From befab215b41840dc44a3fc27af044f8df12b5194 Mon Sep 17 00:00:00 2001 From: Vanilagy <1696106+Vanilagy@users.noreply.github.com> Date: Wed, 18 Jun 2025 12:25:10 +0200 Subject: [PATCH] Add "Quick start" to docs, small API and docs adjustments --- docs/guide/installation.md | 2 +- docs/guide/introduction.md | 10 + docs/guide/media-sources.md | 4 - docs/guide/output-formats.md | 18 +- docs/guide/quick-start.md | 503 +++++++++++++++++++++++++++++- docs/guide/writing-media-files.md | 4 +- src/matroska/matroska-muxer.ts | 16 +- src/output-format.ts | 12 +- 8 files changed, 533 insertions(+), 36 deletions(-) diff --git a/docs/guide/installation.md b/docs/guide/installation.md index 4ba90b2..2f9c0bc 100644 --- a/docs/guide/installation.md +++ b/docs/guide/installation.md @@ -23,7 +23,7 @@ import { ... } from 'mediabunny'; // ESM const { ... } = require('mediabunny'); // or CommonJS ``` -ESM is prefered because it gives you tree shaking. +ESM is preferred because it gives you tree shaking. You can also just include the library using a script tag in your HTML: ```html diff --git a/docs/guide/introduction.md b/docs/guide/introduction.md index 428aeb3..ed1d32f 100644 --- a/docs/guide/introduction.md +++ b/docs/guide/introduction.md @@ -18,6 +18,7 @@ Here's a long list of stuff this library does: - Input and output streaming, arbitrary file size support - File location independence (memory, disk, network, ...) - Utilities for compression, resizing, rotation, resampling, trimming +- Transmuxing and transcoding - Microsecond-accurate reading and writing precision - Efficient seeking through time - Pipelined design for efficient hardware usage and automatic backpressure @@ -41,6 +42,15 @@ Mediabunny is a general-purpose toolkit and can be used in infinitely many ways. Check out the [Examples](/examples) page for demo implementations of many of these ideas! +## Getting started + +To get going with Mediabunny, here are some starting points: +- Check out [Quick start](./quick-start) for a collection of useful code snippets +- Start with [Reading media files](./reading-media-files) if you want to do read operations. +- Start with [Writing media files](./writing-media-files) if you want to do write operations. +- Start with [Converting media files](./converting-media-files) if you care about file conversions. +- Dive into [Packets & samples](./packets-and-samples) for a deeper understanding of the concepts underlying this library. + ## Motivation Mediabunny is the evolution of my previous libraries, [mp4-muxer](https://github.com/Vanilagy/mp4-muxer) and [webm-muxer](https://github.com/Vanilagy/webm-muxer), which were both created due to the advent of the WebCodecs API. While they fulfilled their job just fine, I saw a few painpoints: diff --git a/docs/guide/media-sources.md b/docs/guide/media-sources.md index c26b1cc..0314e38 100644 --- a/docs/guide/media-sources.md +++ b/docs/guide/media-sources.md @@ -1,9 +1,5 @@ # Media sources -::: info -Media sources are not to be confused with [MediaSource](https://developer.mozilla.org/en-US/docs/Web/API/MediaSource) of the Media Source Extensions API. -::: - ## Introduction _Media sources_ provide APIs for adding media data to an output file. Different media sources provide different levels of abstraction and cater to different use cases. diff --git a/docs/guide/output-formats.md b/docs/guide/output-formats.md index e175eca..ef7f83a 100644 --- a/docs/guide/output-formats.md +++ b/docs/guide/output-formats.md @@ -41,6 +41,12 @@ type TrackCountLimits = { }; ``` +### Append-only writing + +Some output format configurations write in an *append-only* fashion. This means they only ever add new data to the end, and never have to seek back to overwrite a previously-written section of the file. Or, put formally: the byte offset of any write is exactly equal to the number of bytes written before it. + +Append-only formats, in combination with [`StreamTarget`](./writing-media-files#streamtarget), have some useful properties. They enable use with [Media Source Extensions](https://developer.mozilla.org/en-US/docs/Web/API/Media_Source_Extensions_API) and allow for trivial streaming across the network, such as for file uploads. + ## MP4 This output format creates MP4 files. @@ -72,12 +78,12 @@ type IsobmffOutputFormatOptions = { - `'in-memory'`\ Produces a file with Fast Start by keeping all media chunks in memory until the file is finalized. This produces a high-quality and compact output at the cost of a more expensive finalization step and higher memory requirements. ::: info - This option ensures append-only writing. + This option ensures [append-only writing](#append-only-writing), although all the writing happens in bulk, at the end. ::: - `'fragmented'`\ Produces a _fragmented MP4 (fMP4)_ file, evenly placing sample metadata throughout the file by grouping it into "fragments" (short sections of media), while placing general metadata at the beginning of the file. Fragmented files are ideal in streaming contexts, as each fragment can be played individually without requiring knowledge of the other fragments. Furthermore, they remain lightweight to create no matter how large the file becomes, as they don't require media to be kept in memory for very long. However, fragmented files are not as widely and wholly supported as regular MP4 files, and some players don't provide seeking functionality for them. ::: info - This option ensures append-only writing. + This option ensures [append-only writing](#append-only-writing). ::: ::: warning This option requires [packet buffering](./writing-media-files#packet-buffering). @@ -124,7 +130,7 @@ const output = new Output({ The following options are available: ```ts type MkvOutputFormatOptions = { - streamable?: boolean; + appendOnly?: boolean; minimumClusterDuration?: number; onEbmlHeader?: (data: Uint8Array, position: number) => void; @@ -132,10 +138,10 @@ type MkvOutputFormatOptions = { onCluster?: (data: Uint8Array, position: number, timestamp: number) => unknown; }; ``` -- `streamable`\ +- `appendOnly`\ Configures the output to write data in an append-only fashion. This is useful for live-streaming the output as it's being created. Note that when enabled, certain features like file duration or seeking will be disabled or impacted, so don't use this option when you want to write out a media file for later use. ::: info - This option ensures append-only writing. + This option ensures [append-only writing](#append-only-writing). ::: - `minimumClusterDuration`\ Sets the minimum duration in seconds a cluster must have to be finalized and written to the file. Defaults to 1 second. @@ -173,7 +179,7 @@ const output = new Output({ ``` ::: info -This format ensures append-only writing. +This format ensures [append-only writing](#append-only-writing). ::: The following options are available: diff --git a/docs/guide/quick-start.md b/docs/guide/quick-start.md index 3f6d8d4..d8ac87c 100644 --- a/docs/guide/quick-start.md +++ b/docs/guide/quick-start.md @@ -1,31 +1,516 @@ # Quick start -This page is a collection of short code snippets to showcase the most common operations you may use this library for. +This page is a collection of short code snippets that showcase the most common operations you may use this library for. -## Reading file metadata +## Read file metadata ```ts import { Input, ALL_FORMATS, BlobSource } from 'mediabunny'; const input = new Input({ - formats: ALL_FORMATS, + formats: ALL_FORMATS, // Supporting all file formats source: new BlobSource(file), // Assuming a File instance }); const duration = await input.computeDuration(); // in seconds +const allTracks = await input.getTracks(); // List of all tracks +// Extract video metadata const videoTrack = await input.getPrimaryVideoTrack(); if (videoTrack) { - const width = videoTrack.displayWidth; - const height = videoTrack.displayHeight; - const rotation = videoTrack.rotation; // in degrees clockwise + videoTrack.displayWidth; // in pixels + videoTrack.displayHeight; // in pixels + videoTrack.rotation; // in degrees clockwise + + // Compute FPS (can be expensive) + const packetStats = await videoTrack.computePacketStats(); + const averageFrameRate = packetStats.averagePacketRate; } +// Extract audio metadata const audioTrack = await input.getPrimaryAudioTrack(); if (audioTrack) { - const numberOfChannels = audioTrack.numberOfChannels; - const sampleRate = audioTrack.sampleRate; // in Hz + audioTrack.numberOfChannels; + audioTrack.sampleRate; // in Hz } ``` -## Reading media data \ No newline at end of file +::: info +- Check out the Metadata extraction example for this code in action. +- You can read from more than just `File` instances - check out [Input sources](./reading-media-files#input-sources) for more. +::: + + +## Read media data + +```ts +import { + Input, + ALL_FORMATS, + BlobSource, + VideoSampleSink, + AudioSampleSink, +} from 'mediabunny'; + +const input = new Input({ + formats: ALL_FORMATS, + source: new BlobSource(file), +}); + +// Read video frames +const videoTrack = await input.getPrimaryVideoTrack(); +if (videoTrack) { + const decodable = await videoTrack.canDecode(); + if (decodable) { + const sink = new VideoSampleSink(videoTrack); + + // Get the video frame at timestamp 5s + const videoSample = await sink.getSample(5); + videoSample.timestamp; // in seconds + videoSample.duration; // in seconds + + // Draw the frame to a canvas + videoSample.draw(ctx, 0, 0); + + // Loop over all frames in the first 30s of video + for await (const sample of sink.samples(0, 30)) { + // ... + } + } +} + +// Read audio chunks +const audioTrack = await input.getPrimaryAudioTrack(); +if (audioTrack) { + const decodable = await audioTrack.canDecode(); + if (decodable) { + const sink = new AudioSampleSink(audioTrack); + + // Get audio chunk at timestamp 5s; a short chunk of audio + const audioSample = await sink.getSample(5); + audioSample.timestamp; // in seconds + audioSample.duration; // in seconds + audioSample.numberOfFrames; + + // Convert to AudioBuffer for use with the Web Audio API + const audioBuffer = audioSample.toAudioBuffer(); + + // Loop over all samples in the first 30s of audio + for await (const sample of sink.samples(0, 30)) { + // ... + } + } +} +``` + +::: info +- Check out the Media player example for a demo built on this use case. +- See [Media sinks](./media-sinks) for all the ways to extract media data from tracks. +::: + +## Extract video thumbnails + +```ts +import { + Input, + ALL_FORMATS, + BlobSource, + CanvasSink, +} from 'mediabunny'; + +const input = new Input({ + formats: ALL_FORMATS, + source: new BlobSource(file), +}); + +const videoTrack = await input.getPrimaryVideoTrack(); +if (videoTrack) { + const decodable = await videoTrack.canDecode(); + if (decodable) { + const sink = new CanvasSink(videoTrack, { + width: 320, // Automatically resize the thumbnails + }); + + // Get the thumbnail at timestamp 10s + const result = await sink.getCanvas(10); + result.canvas; // HTMLCanvasElement | OffscreenCanvas + result.timestamp; // in seconds + result.duration; // in seconds + + // Generate five equally-spaced thumbnails through the video + const startTimestamp = await videoTrack.getFirstTimestamp(); + const endTimestamp = await videoTrack.computeDuration(); + const timestamps = [0, 0.2, 0.4, 0.6, 0.8].map( + (t) => startTimestamp + t * (endTimestamp - startTimestamp) + ); + + // Loop over these timestamps + for await (const result of sink.canvasesAtTimestamps(timestamps)) { + // ... + } + } +} +``` + +::: info +- Check out the Thumbnail generation example for this code in action. +- You can further configure [`CanvasSink`](./media-sinks#canvassink). +::: + +## Extract encoded packets + +```ts +import { + Input, + ALL_FORMATS, + BlobSource, + EncodedPacketSink, +} from 'mediabunny'; + +const input = new Input({ + formats: ALL_FORMATS, + source: new BlobSource(file), +}); + +const videoTrack = await input.getPrimaryVideoTrack(); +if (videoTrack) { + const sink = new EncodedPacketSink(videoTrack); + + // Get packet for timestamp 10s + const packet = await sink.getPacket(10); + packet.data; // Uint8Array + packet.type; // 'key' | 'delta' + packet.timestamp; // in seconds + packet.duration; // in seconds + + // Get the closest key packet to timestamp 10s + const keyPacket = await sink.getKeyPacket(10); + + // Get the following packet + const nextPacket = await sink.getNextPacket(keyPacket); + + // Set up a manual decoder + const decoderConfig = await videoTrack.getDecoderConfig(); + const videoDecoder = new VideoDecoder({ + output: console.log, + error: console.error, + }); + videoDecoder.configure(decoderConfig); + + // Loop over all packets in decode order + for await (const packet of sink.packets()) { + videoDecoder.decode(packet.toEncodedVideoChunk()); + } + + await videoDecoder.flush(); +} +``` + +::: info +Check out [`EncodedPacketSink`](./media-sinks#encodedpacketsink) for the full documentation. +::: + +## Write new media files + +```ts +import { + Output, + BufferTarget, + Mp4OutputFormat, + CanvasSource, + AudioBufferSource, + QUALITY_HIGH, +} from 'mediabunny'; + +// An Output represents a new media file +const output = new Output({ + format: new Mp4OutputFormat(), // The format of the file + target: new BufferTarget(), // Where to write the file (here, to memory) +}); + +// Example: add a video track driven by a canvas +const videoSource = new CanvasSource(canvas, { + codec: 'avc', + bitrate: QUALITY_HIGH, +}); +output.addVideoTrack(videoSource); + +// Example: add an audio track driven by AudioBuffers +const audioSource = new AudioBufferSource({ + codec: 'aac', + bitrate: QUALITY_HIGH, +}); +output.addAudioTrack(audioSource); + +await output.start(); + +// Add some video frames +for (let frame = 0; ...) { + await videoSource.add(frame / 30, 1 / 30); +} + +// Add some audio data +await audioSource.add(audioBuffer1); +await audioSource.add(audioBuffer2); + +await output.finalize(); + +const buffer = output.target.buffer; // Uint8Array containing the final MP4 file +``` + +::: info +- Check out the Procedural generation example for a demo of in-browser video generation. +- You can create files of many different formats; check out [Output formats](./output-formats) for the full list. +- Media data can be added from different sources, see [Media sources](./media-sources). +::: + +## Write directly to disk + +```ts +import { + Output, + StreamTarget, +} from 'mediabunny'; + +// File System API +const handle = await window.showSaveFilePicker(); +const writableStream = await handle.createWritable(); + +const output = new Output({ + // `chunked: true` to batch disk operations + target: new StreamTarget(writableStream, { chunked: true }), + // ... +}); + +// ... + +await output.finalize(); + +// The file has been fully written to disk +``` + +## Stream over the network + +```ts +import { + Output, + StreamTarget, + StreamTargetChunk, + Mp4OutputFormat, +} from 'mediabunny'; + +const { writable, readable } = new TransformStream({ + transform: (chunk, controller) => controller.enqueue(chunk.data), +}); + +const output = new Output({ + target: new StreamTarget(writable), + // We must use an append-only format here, such as fragmented MP4 + format: new Mp4OutputFormat({ fastStart: 'fragmented' }), +}); + +const uploadComplete = fetch('https://example.com/upload', { + method: 'POST', + body: readable, + headers: { + 'Content-Type': output.format.mimeType, + }, +}); + +await output.start(); + +// ... + +await output.finalize(); +await uploadComplete; +``` + +::: info +- This code automatically handles the backpressure applied by a slow network. +- Read more on [append-only formats](./output-formats#append-only-writing), a requirement for this pattern. +::: + +## Record live media + +```ts +import { + Output, + BufferTarget, + WebMOutputFormat, + MediaStreamVideoTrackSource, + MediaStreamAudioTrackSource, + QUALITY_MEDIUM +} from 'mediabunny'; + +const userMedia = await navigator.mediaDevices.getUserMedia({ + video: true, + audio: true, +}); +const videoTrack = userMedia.getVideoTracks()[0]; +const audioTrack = userMedia.getAudioTracks()[0]; + +const output = new Output({ + format: new WebMOutputFormat(), + target: new BufferTarget(), +}); + +if (videoTrack) { + const source = new MediaStreamVideoTrackSource(videoTrack, { + codec: 'vp9', + bitrate: QUALITY_MEDIUM, + }); + output.addVideoTrack(source); +} + +if (audioTrack) { + const source = new MediaStreamAudioTrackSource(audioTrack, { + codec: 'opus', + bitrate: QUALITY_MEDIUM, + }); + output.addAudioTrack(source); +} + +await output.start(); + +// Wait... + +await output.finalize(); +``` + +::: info +- Check out the Live recording demo for this code in action. +- This is basically [`MediaRecorder`](https://developer.mozilla.org/en-US/docs/Web/API/MediaRecorder), but less sucky. +::: + +## Check encoding support + +```ts +import { + MovOutputFormat, + getFirstEncodableVideoCodec, + getFirstEncodableAudioCodec, + getEncodableVideoCodecs, + getEncodableAudioCodecs, +} from 'mediabunny'; + +const outputFormat = new MovOutputFormat(); + +// Find the best supported codec for the given container format +const bestVideoCodec = await getFirstEncodableVideoCodec( + outputFormat.getSupportedVideoCodecs(), + // Optionally, constrained by these parameters: + { width: 1920, height: 1080 }, +); +const bestAudioCodec = await getFirstEncodableAudioCodec( + outputFormat.getSupportedAudioCodecs(), +); + +// Find all supported codecs +const supportedVideoCodecs = await getEncodableVideoCodecs(); +const supportedAudioCodecs = await getEncodableAudioCodecs(); +``` + +## Convert files + +```ts +import { + Input, + Output, + Conversion, + ALL_FORMATS, + BlobSource, + Mp4OutputFormat, +} from 'mediabunny'; + +// Check the above snippets for more examples of Input and Output +const input = new Input({ + formats: ALL_FORMATS, + source: new BlobSource(file), +}); +const output = new Output({ + format: new Mp4OutputFormat(), + target: new BufferTarget(), +}); + +const conversion = await Conversion.init({ input, output }); +conversion.discardedTracks; // List of tracks that won't make it into the output + +conversion.onProgress = (progress) => { + progress; // Number between 0 and 1, inclusive +}; + +await conversion.execute(); +// Conversion is complete + +const buffer = output.target.buffer; // Uint8Array containing the final MP4 file +``` + +::: info +- This code will automatically transmux (copy media data) when possible, and transcode (re-encode media data) when necessary. +- Refer to [Converting media files](./converting-media-files) for the full documentation. +::: + +## Extract audio + +```ts +import { + Input, + Output, + Conversion, + WavOutputFormat, +} from 'mediabunny'; + +const input = new Input(...); +const output = new Output({ + // Write to a .wav file, keeping only the audio track + format: new WavOutputFormat(), + // ... +}); + +const conversion = await Conversion.init({ input, output }); +await conversion.execute(); +// Conversion is complete +``` + +::: info +- You can extract to other audio-only formats, such as .mp3, .ogg, or even .m4a. See [Output formats](./output-formats). +::: + +## Compress media + +```ts +import { + Input, + Output, + Conversion, + QUALITY_LOW, +} from 'mediabunny'; + +const input = new Input(...); +const output = new Output(...); + +const conversion = await Conversion.init({ + input, + output, + video: { + width: 480, + bitrate: QUALITY_LOW, + }, + audio: { + numberOfChannels: 1, + bitrate: QUALITY_LOW, + }, + trim: { + // Let's keep only the first 60 seconds + start: 0, + end: 60, + }, +}); + +await conversion.execute(); +// Conversion is complete +``` + +::: info +- Check out the File compression example for this code in action. +::: \ No newline at end of file diff --git a/docs/guide/writing-media-files.md b/docs/guide/writing-media-files.md index 2d85f33..f34db98 100644 --- a/docs/guide/writing-media-files.md +++ b/docs/guide/writing-media-files.md @@ -246,7 +246,7 @@ Each chunk written to the `WritableStream` represents a contiguous chunk of byte ::: warning Note that some byte regions in the output file may be written to multiple times. It is therefore **incorrect** to construct the final file by simply concatenating all `Uint8Array`s together - you **must** write each chunk of data at the specified byte offset position _in the order_ in which the chunks arrived. If you don't do this, your output file will likely be invalid or corrupted. -Some [output formats](./output-formats) have "monotonic" writing modes in which the byte offset of a written chunk will always be equal to the total number of bytes in all previously written chunks. In other words, when writing is monotonic, simply concatening all `Uint8Array`s yields the correct result. Some APIs (like `appendBuffer` of Media Source Extensions) require this, so make sure to configure your output format accordingly for those cases. +Some [output formats](./output-formats) have *append-only* writing modes in which the byte offset of a written chunk will always be equal to the total number of bytes in all previously written chunks. In other words, when writing is append-only, simply concatening all `Uint8Array`s yields the correct result. Some APIs (like `appendBuffer` of Media Source Extensions) require this, so make sure to configure your output format accordingly for those cases. ::: #### Chunked mode @@ -256,7 +256,7 @@ By default, data will be emitted by the `StreamTarget` as soon as it is availabl ```ts new StreamTarget(writable, { chunked: true, - chunkSize: 2**20, // Optional; defaults to 16 MiB + chunkSize: 2 ** 20, // Optional; defaults to 16 MiB }), ``` diff --git a/src/matroska/matroska-muxer.ts b/src/matroska/matroska-muxer.ts index 9d0b03f..be253cc 100644 --- a/src/matroska/matroska-muxer.ts +++ b/src/matroska/matroska-muxer.ts @@ -150,7 +150,7 @@ export class MatroskaMuxer extends Muxer { this.ebmlWriter = new EBMLWriter(this.writer); - if (this.format._options.streamable) { + if (this.format._options.appendOnly) { this.writer.ensureMonotonicity = true; } } @@ -160,7 +160,7 @@ export class MatroskaMuxer extends Muxer { this.writeEBMLHeader(); - if (!this.format._options.streamable) { + if (!this.format._options.appendOnly) { this.createSeekHead(); } @@ -228,7 +228,7 @@ export class MatroskaMuxer extends Muxer { { id: EBMLId.TimestampScale, data: 1e6 }, { id: EBMLId.MuxingApp, data: APP_NAME }, { id: EBMLId.WritingApp, data: APP_NAME }, - !this.format._options.streamable ? segmentDuration : null, + !this.format._options.appendOnly ? segmentDuration : null, ] }; this.segmentInfo = segmentInfo; } @@ -370,9 +370,9 @@ export class MatroskaMuxer extends Muxer { private createSegment() { const segment: EBML = { id: EBMLId.Segment, - size: this.format._options.streamable ? -1 : SEGMENT_SIZE_BYTES, + size: this.format._options.appendOnly ? -1 : SEGMENT_SIZE_BYTES, data: [ - !this.format._options.streamable ? this.seekHead as EBML : null, + !this.format._options.appendOnly ? this.seekHead as EBML : null, this.segmentInfo, this.tracksElement, ], @@ -862,7 +862,7 @@ export class MatroskaMuxer extends Muxer { this.currentCluster = { id: EBMLId.Cluster, - size: this.format._options.streamable ? -1 : CLUSTER_SIZE_BYTES, + size: this.format._options.appendOnly ? -1 : CLUSTER_SIZE_BYTES, data: [ { id: EBMLId.Timestamp, data: msTimestamp }, ], @@ -877,7 +877,7 @@ export class MatroskaMuxer extends Muxer { private finalizeCurrentCluster() { assert(this.currentCluster); - if (!this.format._options.streamable) { + if (!this.format._options.appendOnly) { const clusterSize = this.writer.getPos() - this.ebmlWriter.dataOffsets.get(this.currentCluster)!; const endPos = this.writer.getPos(); @@ -959,7 +959,7 @@ export class MatroskaMuxer extends Muxer { assert(this.cues); this.ebmlWriter.writeEBML(this.cues); - if (!this.format._options.streamable) { + if (!this.format._options.appendOnly) { const endPos = this.writer.getPos(); // Write the Segment size diff --git a/src/output-format.ts b/src/output-format.ts index eabd49c..c75da3b 100644 --- a/src/output-format.ts +++ b/src/output-format.ts @@ -303,11 +303,11 @@ export class MovOutputFormat extends IsobmffOutputFormat { */ export type MkvOutputFormatOptions = { /** - * Configures the output to only write data monotonically, useful for live-streaming the file as it's being muxed. - * When enabled, some features such as storing duration and seeking will be disabled or impacted, so don't use this - * option when you want to write out a file for later use. + * Configures the output to only append new data at the end, useful for live-streaming the file as it's being + * created. When enabled, some features such as storing duration and seeking will be disabled or impacted, so don't + * use this option when you want to write out a clean file for later use. */ - streamable?: boolean; + appendOnly?: boolean; /** * This field controls the minimum duration of each Matroska cluster, in seconds. New clusters will only be created @@ -354,8 +354,8 @@ export class MkvOutputFormat extends OutputFormat { if (!options || typeof options !== 'object') { throw new TypeError('options must be an object.'); } - if (options.streamable !== undefined && typeof options.streamable !== 'boolean') { - throw new TypeError('options.streamable, when provided, must be a boolean.'); + if (options.appendOnly !== undefined && typeof options.appendOnly !== 'boolean') { + throw new TypeError('options.appendOnly, when provided, must be a boolean.'); } if ( options.minimumClusterDuration !== undefined