From 5e224510be67ef09eef50f1e5eceb9f826571b7e Mon Sep 17 00:00:00 2001 From: Vanilagy <1696106+Vanilagy@users.noreply.github.com> Date: Mon, 31 Mar 2025 21:21:10 +0200 Subject: [PATCH] Add documentation on writing files --- docs/.vitepress/config.mts | 16 +- docs/guide/media-sources.md | 396 +++++++++++++++++++++ docs/guide/output-formats.md | 225 ++++++++++++ docs/guide/supported-formats-and-codecs.md | 6 +- docs/guide/writing-overview.md | 300 ++++++++++++++++ docs/guide/writing.md | 0 docs/index.md | 3 +- src/index.ts | 4 +- src/isobmff/isobmff-muxer.ts | 2 + src/output-format.ts | 19 +- src/wave/wave-muxer.ts | 6 +- 11 files changed, 954 insertions(+), 23 deletions(-) create mode 100644 docs/guide/media-sources.md create mode 100644 docs/guide/output-formats.md create mode 100644 docs/guide/writing-overview.md delete mode 100644 docs/guide/writing.md diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index c9b4072..60c8847 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -10,7 +10,7 @@ export default defineConfig({ // https://vitepress.dev/reference/default-theme-config nav: [ { text: 'Home', link: '/' }, - { text: 'Guide', link: '/guide/introduction' }, + { text: 'Guide', link: '/guide/introduction', activeMatch: '/guide' }, ], sidebar: [ @@ -31,7 +31,7 @@ export default defineConfig({ { text: 'Writing media files', items: [ - { text: 'Writing basics', link: '/guide/writing' }, + { text: 'Writing overview', link: '/guide/writing-overview' }, { text: 'Media sources', link: '/guide/media-sources' }, { text: 'Output formats', link: '/guide/output-formats' }, ], @@ -39,8 +39,8 @@ export default defineConfig({ { text: 'Miscellaneous', items: [ - { text: 'Supported formats & codecs', link: 'guide/supported-formats-and-codecs' }, - { text: 'Custom coders', link: 'guide/custom-coders' }, + { text: 'Supported formats & codecs', link: '/guide/supported-formats-and-codecs' }, + { text: 'Custom coders', link: '/guide/custom-coders' }, ], }, ], @@ -48,6 +48,14 @@ export default defineConfig({ socialLinks: [ { icon: 'github', link: 'https://github.com/vuejs/vitepress' }, ], + + search: { + provider: 'local', + }, + + outline: { + level: [2, 3], + }, }, markdown: { config(md) { diff --git a/docs/guide/media-sources.md b/docs/guide/media-sources.md new file mode 100644 index 0000000..c34287b --- /dev/null +++ b/docs/guide/media-sources.md @@ -0,0 +1,396 @@ +# Media sources + +::: info +Media sources are not to be confused with [MediaSource](https://developer.mozilla.org/en-US/docs/Web/API/MediaSource) of the Media Source Extensions API. +::: + +## Introduction + +_Media sources_ provide APIs for adding media data to an output file. Different media sources provide different levels of abstraction and cater to different use cases. + +For information on how to use media sources to create output tracks, check the [writing overview](./writing-overview). + +Most media sources follow this code pattern to add media data: +```ts +await mediaSource.add(...); +``` + +### Closing sources + +When you're done using the source, meaning no additional media data will be added, it's best to close the source as soon as possible: +```ts +void mediaSource.close(); +``` +Closing sources manually is _technically_ not required and will happen automatically when finalizing the `Output`. However, if your `Output` has multiple tracks and not all of them finish supplying their data at the same time (for example, adding all audio first and then all video), closing sources early will improve performance and lower memory usage. This is because the `Output` can better "plan ahead", knowing it doesn't have to wait for certain tracks anymore (see [Packet buffering](./writing-overview#packet-buffering)). Therefore, it is good practice to always manually close all media sources as soon as you are done using them. + +### Backpressure + +Media sources are the means by which backpressure is propagated from the output pipeline into your application logic. The `Output` may want to apply backpressure if the encoders or the [StreamTarget](./writing-overview/#streamtarget)'s writable can't keep up. + +Backpressure is communicated by media sources via promises. All media sources with an `add` method return a promise: +```ts +mediaSource.add(...); // => Promise +``` +This promise resolves when the source is ready to receive more data. In most cases, the promise will resolve instantly, but if some part of the output pipeline is overworked, it will remain pending until the output is ready to continue. Therefore, by awaiting this promise, you automatically propagate backpressure into your application logic: +```ts +// Wrong: // [!code error] +while (notDone) { // [!code error] + mediaSource.add(...); // [!code error] +} // [!code error] + +// Correct: +while (notDone) { + await mediaSource.add(...); +} +``` + +### Video encoding config + +All video sources that handle encoding internally require you to specify a `VideoEncodingConfiguration`, specifying the codec configuration to use: +```ts +type VideoEncodingConfig = { + codec: VideoCodec; + bitrate: number | Quality; + latencyMode?: 'quality' | 'realtime'; + keyFrameInterval?: number; + + onEncodedPacket?: ( + packet: EncodedPacket, + meta: EncodedVideoChunkMetadata | undefined + ) => unknown; + onEncodingError?: ( + error: Error + ) => unknown; +}; +``` +- `codec`: The [video codec](./supported-formats-and-codecs/#video-codecs) used for encoding. +- `bitrate`: The target number of bits per second. Alternatively, this can be a [subjective quality](#subjective-qualities). +- `latencyMode`: The latency mode as specified by the WebCodecs API. Media stream-driven video sources will automatically use the `realtime` setting. +- `keyFrameInterval`: The maximum interval in seconds between two adjacent key frames. Defaults to 5 seconds. More frequent key frames improve seeking behavior but increase file size. When using multiple video tracks, this value should be set to the same value for all tracks. +- `onEncodedPacket`: Called for each successfully encoded packet. Useful for determining encoding progress. +- `onEncodingError`: Called when an error occurs during encoding. + +### Audio encoding config + +All audio sources that handle encoding internally require you to specify an `AudioEncodingConfiguration`, specifying the codec configuration to use: +```ts +type AudioEncodingConfig = { + codec: AudioCodec; + bitrate?: number | Quality; + + onEncodedPacket?: ( + packet: EncodedPacket, + meta: EncodedAudioChunkMetadata | undefined + ) => unknown; + onEncodingError?: ( + error: Error + ) => unknown; +}; +``` +- `codec`: The [audio codec](./supported-formats-and-codecs/#audio-codecs) used for encoding. Can be omitted for uncompressed PCM codecs. +- `bitrate`: The target number of bits per second. Alternatively, this can be a [subjective quality](#subjective-qualities). +- `onEncodedPacket`: Called for each successfully encoded packet. Useful for determining encoding progress. +- `onEncodingError`: Called when an error occurs during encoding. + +### Subjective qualities + +Mediakit provides five subjective quality options as an alternative to manually providing a bitrate. From a subjective quality, a bitrate will be calculated internally based on the codec and track information (width, height, sample rate, ...). + +```ts +import { + QUALITY_VERY_LOW, + QUALITY_LOW, + QUALITY_MEDIUM, + QUALITY_HIGH, + QUALITY_VERY_HIGH, +} from 'mediakit'; +``` + +## Video sources + +Video sources feed data to video tracks on an `Output`. They all extend the abstract `VideoSource` class. + +### `VideoSampleSource` + +This source takes [video samples](TODO), encodes them, and passes the encoded data to the output. + +```ts +import { VideoSampleSource } from 'mediakit'; + +const sampleSource = new VideoSampleSource({ + codec: 'avc', + bitrate: 1e6, +}); + +await sampleSource.add(videoSample); + +// You may optionally force samples to be encoded as key frames: +await sampleSource.add(videoSample, { keyFrame: true }); +``` + +### `CanvasSource` + +This source simplifies a common pattern: A single canvas is repeatedly updated in a render loop and each frame is added to the output file. + +```ts +import { CanvasSource, QUALITY_MEDIUM } from 'mediakit'; + +const canvasSource = new CanvasSource(canvasElement, { + codec: 'av1', + bitrate: QUALITY_MEDIUM, +}); + +await canvasSource.add(0.0, 0.1); // Timestamp, duration (in seconds) +await canvasSource.add(0.1, 0.1); +await canvasSource.add(0.2, 0.1); + +// You may optionally force frames to be encoded as key frames: +await canvasSource.add(0.3, 0.1, { keyFrame: true }); +``` + +### `MediaStreamVideoTrackSource` + +This is a source for use with the [Media Capture and Streams API](https://developer.mozilla.org/en-US/docs/Web/API/Media_Capture_and_Streams_API). Use this source if you want to pipe a real-time video source (such as a webcam or screen recording) to an output file. + +```ts +import { MediaStreamVideoTrackSource } from 'mediakit'; + +// Get the user's screen +const stream = await navigator.mediaDevices.getDisplayMedia({ video: true }); +const videoTrack = stream.getVideoTracks()[0]; + +const videoTrackSource = new MediaStreamVideoTrackSource(videoTrack, { + codec: 'vp9', + bitrate: 1e7, +}); +``` + +This source requires no additional method calls; data will automatically be captured and piped to the output file as soon as `start()` is called on the `Output`. Make sure to `stop()` on `videoTrack` after finalizing the `Output` if you don't need the user's media anymore. + +### `EncodedVideoPacketSource` + +The most barebones of all video sources, this source can be used to directly pipe [encoded packets](TODO) of video data to the output. This source requires that you take care of the encoding process yourself, which enables you to use the WebCodecs API manually or to plug in your own encoding stack. Alternatively, you may retrieve the encoded packets directly by reading them from another media file, allowing you to skip decoding and reencoding video data. + +```ts +import { EncodedVideoPacketSource } from 'mediakit'; + +// You must specify the codec name: +const packetSource = new EncodedVideoPacketSource('vp9'); + +await packetSource.add(packet1); +await packetSource.add(packet2); +``` + +> [!IMPORTANT] +> You must add the packets in decode order. + +You will need to provide additional metadata alongside your first call to `add` to give the `Output` more information about the shape and form of the video data. This metadata must be in the form of the WebCodecs API's `EncodedVideoChunkMetadata`. It might look like this: +```ts +await packetSource.add(firstPacket, { + decoderConfig: { + codec: 'vp09.00.31.08', + codedWidth: 1280, + codedHeight: 720, + colorSpace: { + primaries: 'bt709', + transfer: 'iec61966-2-1', + matrix: 'smpte170m', + fullRange: false, + }, + description: undefined, + }, +}); +``` + +`codec`, `codedWidth`, and `codedHeight` are required for all codecs, whereas `description` is required for some codecs. Additional fields, such as `colorSpace`, are optional. The [WebCodecs Codec Registry](https://www.w3.org/TR/webcodecs-codec-registry/) specifies the formats of `codec` and `description` for each video codec, which you must adhere to. + +#### B-frames + +Some video codecs use *B-frames*, which are frames that require both the previous and the next frame to be decoded. For example, you may have something like this: +```md +Frame 1: 0.0s, I-frame (key frame) +Frame 2: 0.1s, B-frame +Frame 3: 0.2s, P-frame +``` +The decode order for these frames will be: +```md +Frame 1 -> Frame 3 -> Frame 2 +``` +Some file formats have an explicit notion of both a "decode timestamp" and a "presentation timestamp" to model B-frames or out-of-order decoding. However, Mediakit packets only specify their *presentation timestamp*. Decode order is determined by the order in which you add the packets, so in our example, you must add the packets like this: +```ts +await packetSource.add(packetForFrame1); // 0.0s +await packetSource.add(packetForFrame3); // 0.2s +await packetSource.add(packetForFrame2); // 0.1s +``` + +You are allowed to provide wildly out-of-order presentation timestamp sequences, but there is a hard constraint: + +> [!IMPORTANT] +> A packet you add must not have a smaller timestamp than the largest timestamp you added before adding the last key frame. + +This is quite a mouthful, so this example will hopefully clarify it: +```md +# Legal: +Packet 1: 0.0s, key frame +Packet 2: 0.3s, delta frame +Packet 3: 0.2s, delta frame +Packet 4: 0.1s, delta frame +Packet 5: 0.4s, key frame +Packet 6: 0.5s, delta frame + +# Also legal: +Packet 1: 0.0s, key frame +Packet 2: 0.3s, delta frame +Packet 3: 0.2s, delta frame +Packet 4: 0.1s, delta frame +Packet 5: 0.4s, key frame +Packet 6: 0.35s, delta frame +Packet 7: 0.3s, delta frame +Packet 8: 0.5s, delta frame + +# Illegal: +Packet 1: 0.0s, key frame +Packet 2: 0.3s, delta frame +Packet 3: 0.2s, delta frame +Packet 4: 0.1s, delta frame +Packet 5: 0.4s, key frame +Packet 6: 0.25s, delta frame +``` + +## Audio sources + +Audio sources feed data to audio tracks on an `Output`. They all extend the abstract `AudioSource` class. + +### `AudioSampleSource` + +This source takes [audio samples](TODO), encodes them, and passes the encoded data to the output. + +```ts +import { AudioSampleSource } from 'mediakit'; + +const sampleSource = new AudioSampleSource({ + codec: 'aac', + bitrate: 128e3, +}); + +await sampleSource.add(audioSample); +``` + +### `AudioBufferSource` + +This source directly accepts instances of `AudioBuffer` as data, simplifying usage with the Web Audio API. The first AudioBuffer will be played at timestamp 0, and any subsequent AudioBuffer will be appended after all previous AudioBuffers. + +```ts +import { AudioBufferSource, QUALITY_MEDIUM } from 'mediakit'; + +const bufferSource = new AudioBufferSource({ + codec: 'opus', + bitrate: QUALITY_MEDIUM, +}); + +await bufferSource.add(audioBuffer1); +await bufferSource.add(audioBuffer2); +await bufferSource.add(audioBuffer3); +``` + +### `MediaStreamAudioTrackSource` + +This is a source for use with the [Media Capture and Streams API](https://developer.mozilla.org/en-US/docs/Web/API/Media_Capture_and_Streams_API). Use this source if you want to pipe a real-time audio source (such as a microphone or audio from the user's computer) to an output file. + +```ts +import { MediaStreamAudioTrackSource } from 'mediakit'; + +// Get the user's microphone +const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); +const audioTrack = stream.getAudioTracks()[0]; + +const audioTrackSource = new MediaStreamAudioTrackSource(audioTrack, { + codec: 'opus', + bitrate: 128e3, +}); +``` + +This source requires no additional method calls; data will automatically be captured and piped to the output file as soon as `start()` is called on the `Output`. Make sure to `stop()` on `audioTrack` after finalizing the `Output` if you don't need the user's media anymore. + +### `EncodedAudioPacketSource` + +The most barebones of all audio sources, this source can be used to directly pipe [encoded packets](TODO) of audio data to the output. This source requires that you take care of the encoding process yourself, which enables you to use the WebCodecs API manually or to plug in your own encoding stack. Alternatively, you may retrieve the encoded packets directly by reading them from another media file, allowing you to skip decoding and reencoding audio data. + +```ts +import { EncodedAudioPacketSource } from 'mediakit'; + +// You must specify the codec name: +const packetSource = new EncodedAudioPacketSource('aac'); + +await packetSource.add(packet); +``` + +You will need to provide additional metadata alongside your first call to `add` to give the `Output` more information about the shape and form of the audio data. This metadata must be in the form of the WebCodecs API's `EncodedAudioChunkMetadata`. It might look like this: +```ts +await packetSource.add(firstPacket, { + decoderConfig: { + codec: 'mp4a.40.2', + numberOfChannels: 2, + sampleRate: 48000, + description: new Uint8Array([17, 144]), + }, +}); +``` + +`codec`, `numberOfChannels`, and `sampleRate` are required for all codecs, whereas `description` is required for some codecs. The [WebCodecs Codec Registry](https://www.w3.org/TR/webcodecs-codec-registry/) specifies the formats of `codec` and `description` for each audio codec, which you must adhere to. + +## Subtitle sources + +Subtitle sources feed data to subtitle tracks on an `Output`. They all extend the abstract `SubtitleSource` class. + +### `TextSubtitleSource` + +This source feeds subtitle cues to the output from a text file in which the subtitles are defined. + +```ts +import { TextSubtitleSource } from 'mediakit'; + +const textSource = new TextSubtitleSource('webvtt'); + +const text = +`WEBVTT + +00:00:00.000 --> 00:00:02.000 +This is your last chance. + +00:00:02.500 --> 00:00:04.000 +After this, there is no turning back. + +00:00:04.500 --> 00:00:06.000 +If you take the blue pill, the story ends. + +00:00:06.500 --> 00:00:08.000 +You wake up in your bed and believe whatever you want to believe. + +00:00:08.500 --> 00:00:10.000 +If you take the red pill, you stay in Wonderland + +00:00:10.500 --> 00:00:12.000 +and I show you how deep the rabbit hole goes. +`; + +await textSource.add(text); +``` + +If you add the entire subtitle file at once, make sure to [close the source](#closing-sources) immediately after: +```ts +void textSource.close(); +``` + +You can also add cues individually in small chunks: +```ts +import { TextSubtitleSource } from 'mediakit'; + +const textSource = new TextSubtitleSource('webvtt'); + +await textSource.add('WEBVTT\n\n'); +await textSource.add('00:00:00.000 --> 00:00:02.000\nHello there!\n\n'); +await textSource.add('00:00:02.500 --> 00:00:04.000\nChunky chunks.\n\n'); +``` + +The chunks have certain constraints: A cue must be fully contained within a chunk and cannot be split across multiple smaller chunks (although a chunk can contain multiple cues). Also, the WebVTT preamble must be added first and all at once. \ No newline at end of file diff --git a/docs/guide/output-formats.md b/docs/guide/output-formats.md new file mode 100644 index 0000000..a81c37f --- /dev/null +++ b/docs/guide/output-formats.md @@ -0,0 +1,225 @@ +# Output formats + +## Introduction + +An _output format_ specifies the container format of the data written by an `Output`. Mediakit supports many commonly used container formats, each having format-specific options. + +Many formats also offer *data callbacks*, which are special callbacks that fire for specific data regions in the output file. + +### Format properties + +All output formats have a common set of properties you can query. + +```ts +// Get the format's file extension: +format.fileExtension; // => '.mp4' + +// Check which codecs can be contained by the format: +format.getSupportedCodecs(); // => MediaCodec[] +format.getSupportedVideoCodecs(); // => VideoCodec[] +format.getSupportedAudioCodecs(); // => AudioCodec[] +format.getSupportedSubtitleCodecs(); // => SubtitleCodec[] + +// Check if the format supports video tracks with rotation metadata: +format.supportsVideoRotationMetadata; // => boolean +``` + +Refer to the [compatibility table](./supported-formats-and-codecs.md#compatibility-table) to see which codecs can be used with which output format. + +Formats also differ in the amount and types of tracks they can contain. You can retrieve this information using: +```ts +format.getSupportedTrackCounts(); // => TrackCountLimits + +type TrackCountLimits = { + video: { min: number, max: number }, + audio: { min: number, max: number }, + subtitle: { min: number, max: number }, + total: { min: number, max: number }, +}; +``` + +## MP4 + +This output format creates MP4 files. +```ts +import { Output, Mp4OutputFormat } from 'mediakit'; + +const output = new Output({ + format: new Mp4OutputFormat(options), + // ... +}); +``` + +The following options are available: +```ts +type IsobmffOutputFormatOptions = { + fastStart?: false | 'in-memory' | 'fragmented'; + minimumFragmentDuration?: number; + + onFtyp?: (data: Uint8Array, position: number) => unknown; + onMoov?: (data: Uint8Array, position: number) => unknown; + onMdat?: (data: Uint8Array, position: number) => unknown; + onMoof?: (data: Uint8Array, position: number, timestamp: number) => unknown; +}; +``` +- `fastStart`\ + Controls the placement of metadata in the file. Placing metadata at the start of the file is known as "Fast Start" and provides certain benefits: The file becomes easier to stream over the web without range requests, and sites like YouTube can start processing the video while it's uploading. However, placing metadata at the start of the file can require more processing and memory in the writing step. This library provides full control over the placement of metadata by setting `fastStart` to one of these options: + - `false`\ + Disables Fast Start, placing the metadata at the end of the file. Fastest and uses the least memory. + - `'in-memory'`\ + Produces a file with Fast Start by keeping all media chunks in memory until the file is finalized. This produces a high-quality and compact output at the cost of a more expensive finalization step and higher memory requirements. + ::: info + This option ensures append-only writing. + ::: + - `'fragmented'`\ + Produces a _fragmented MP4 (fMP4)_ file, evenly placing sample metadata throughout the file by grouping it into "fragments" (short sections of media), while placing general metadata at the beginning of the file. Fragmented files are ideal in streaming contexts, as each fragment can be played individually without requiring knowledge of the other fragments. Furthermore, they remain lightweight to create no matter how large the file becomes, as they don't require media to be kept in memory for very long. However, fragmented files are not as widely and wholly supported as regular MP4 files, and some players don't provide seeking functionality for them. + ::: info + This option ensures append-only writing. + ::: + ::: warning + This option requires [packet buffering](./writing-overview#packet-buffering). + ::: + - `undefined`\ + The default option; it behaves like `'in-memory'` when using [`BufferTarget`](./writing-overview#buffertarget) and like `false` otherwise. +- `minimumFragmentDuration`\ + Only relevant when `fastStart` is `'fragmented'`. Sets the minimum duration in seconds a fragment must have to be finalized and written to the file. Defaults to 1 second. +- `onFtyp`\ + Will be called once the ftyp (File Type) box of the output file has been written. +- `onMoov`\ + Will be called once the moov (Movie) box of the output file has been written. +- `onMdat`\ + Will be called for each finalized mdat (Media Data) box of the output file. Usage of this callback is not recommended when not using `fastStart: 'fragmented'`, as there will be one monolithic mdat box which might require large amounts of memory. +- `onMoof`\ + Will be called for each finalized moof (Movie Fragment) box of the output file. The fragment's start timestamp in seconds is also passed. + +## QuickTime File Format (.mov) + +This output format creates QuickTime files (.mov). +```ts +import { Output, MovOutputFormat } from 'mediakit'; + +const output = new Output({ + format: new MovOutputFormat(options), + // ... +}); +``` + +The available options are the same `IsobmffOutputFormatOptions` used by [MP4](#mp4). + +## WebM + +This output format creates WebM files. +```ts +import { Output, WebmOutputFormat } from 'mediakit'; + +const output = new Output({ + format: new WebmOutputFormat(options), + // ... +}); +``` + +The following options are available: +```ts +type MkvOutputFormatOptions = { + streamable?: boolean; + minimumClusterDuration?: number; + + onEbmlHeader?: (data: Uint8Array, position: number) => void; + onSegmentHeader?: (data: Uint8Array, position: number) => unknown; + onCluster?: (data: Uint8Array, position: number, timestamp: number) => unknown; +}; +``` +- `streamable`\ + Configures the output to write data in an append-only fashion. This is useful for live-streaming the output as it's being created. Note that when enabled, certain features like file duration or seeking will be disabled or impacted, so don't use this option when you want to write out a media file for later use. + ::: info + This option ensures append-only writing. + ::: +- `minimumClusterDuration`\ + Sets the minimum duration in seconds a cluster must have to be finalized and written to the file. Defaults to 1 second. +- `onEbmlHeader`\ + Will be called once the EBML header of the output file has been written. +- `onSegmentHeader`\ + Will be called once the header part of the Matroska Segment element has been written. The header data includes the Segment element and everything inside it, up to (but excluding) the first Matroska Cluster. +- `onCluster`\ + Will be called for each finalized Matroska Cluster of the output file. The cluster's start timestamp in seconds is also passed. + +## Matroska (.mkv) + +This output format creates Matroska files (.mkv). +```ts +import { Output, MkvOutputFormat } from 'mediakit'; + +const output = new Output({ + format: new MkvOutputFormat(options), + // ... +}); +``` + +The available options are the same `MkvOutputFormatOptions` used by [WebM](#webm). + +## Ogg + +This output format creates Ogg files. +```ts +import { Output, OggOutputFormat } from 'mediakit'; + +const output = new Output({ + format: new OggOutputFormat(options), + // ... +}); +``` + +::: info +This format ensures append-only writing. +::: + +The following options are available: +```ts +type OggOutputFormatOptions = { + onPage?: (data: Uint8Array, position: number, source: MediaSource) => unknown; +}; +``` +- `onPage`\ + Will be called for each finalized Ogg page of the output file. The [media source](./media-sources) backing the page's track (logical bitstream) is also passed. + +## MP3 + +This output format creates MP3 files. +```ts +import { Output, Mp3OutputFormat } from 'mediakit'; + +const output = new Output({ + format: new Mp3OutputFormat(options), + // ... +}); +``` + +The following options are available: +```ts +type Mp3OutputFormatOptions = { + onXingFrame?: (data: Uint8Array, position: number) => unknown; +}; +``` +- `onXingFrame`\ + Will be called once the Xing metadata frame is finalized, which happens at the end of the writing process. + +## WAVE + +This output format creates WAVE (.wav) files. +```ts +import { Output, WavOutputFormat } from 'mediakit'; + +const output = new Output({ + format: new WavOutputFormat(options), + // ... +}); +``` + +The following options are available: +```ts +type WavOutputFormatOptions = { + onHeader?: (data: Uint8Array, position: number) => unknown; +}; +``` +- `onHeader`\ + Will be called once the file header is written. The header consists of the RIFF header, the format chunk, and the start of the data chunk (with a placeholder size of 0). \ No newline at end of file diff --git a/docs/guide/supported-formats-and-codecs.md b/docs/guide/supported-formats-and-codecs.md index 4413e1c..70f71c1 100644 --- a/docs/guide/supported-formats-and-codecs.md +++ b/docs/guide/supported-formats-and-codecs.md @@ -1,6 +1,4 @@ # Supported formats & codecs - -Mediakit supports many container formats and media codecs. ## Container formats @@ -16,9 +14,9 @@ Mediakit supports many commonly used media container formats, all of which are s ## Codecs -Mediakit supports a wide range of video, audio and subtitle codecs. More specifically, it supports all codecs specified by the WebCodecs API and a few additional PCM codecs out of the box. +Mediakit supports a wide range of video, audio, and subtitle codecs. More specifically, it supports all codecs specified by the WebCodecs API and a few additional PCM codecs out of the box. -The availability of the codecs provided by the WebCodecs API depends on the browser and cannot be guaranteed by this library. Mediakit provides [special utility functions](#querying-codec-encodability) to check which codecs are able to encoded. You can also specify [custom coders](./custom-coders) to provide your own encoder/decoder implementation if the browser doesn't support the codec natively. +The availability of the codecs provided by the WebCodecs API depends on the browser and cannot be guaranteed by this library. Mediakit provides [special utility functions](#querying-codec-encodability) to check which codecs are able to be encoded. You can also specify [custom coders](./custom-coders) to provide your own encoder/decoder implementation if the browser doesn't support the codec natively. ### Video codecs diff --git a/docs/guide/writing-overview.md b/docs/guide/writing-overview.md new file mode 100644 index 0000000..f40fe22 --- /dev/null +++ b/docs/guide/writing-overview.md @@ -0,0 +1,300 @@ +# Writing overview + +Mediakit enables you to create media files with very fine levels of control. You can add an arbitrary number of video, audio and subtitle tracks to a media file, and precisely control the timing of media data. This library supports [many output file formats](./output-formats). Using [output targets](#output-targets), you can decide if you want to build up the entire file in memory or stream it out in chunks as it's being created—allowing you to create very large files. + +Mediakit provides many ways to supply media data for output tracks, nicely integrating with the WebCodecs API, but also allowing you to use your own encoding stack if you wish. These [media sources](./media-sources) come in multiple levels of abstraction, enabling easy use for common use cases while still giving you fine-grained control if you need it. + +## Creating a new output + +Media file creation in Mediakit revolves around a central class, `Output`. One instance of `Output` represents one media file we want to create. + +Start by creating a new instance of `Output` using the desired configuration of the file you want to create: +```ts +import { Output, Mp4OutputFormat, BufferTarget } from 'mediakit'; + +// In this example, we'll be creating an MP4 file in memory: +const output = new Output({ + format: new Mp4OutputFormat(), + target: new BufferTarget(), +}); +``` + +See [Output formats](./output-formats) for a full list of available output formats.\ +See [Output targets](#output-targets) for a full list of available output targets. + +You can always access `format` and `target` on the output: +```ts +output.format; // => Mp4OutputFormat +output.target; // => BufferTarget +``` + +## Adding tracks + +There are a couple of methods on an `Output` that you can use to add tracks to it: + +```ts +output.addVideoTrack(videoSource); +output.addAudioTrack(audioSource); +output.addSubtitleTrack(subtitleSource); +``` + +For each track you want to add, you'll need to create a unique [media source](./media-source) for it. You'll be able to add media data to the output via these media sources. A media source can only ever be used for one output track. + +Optionally, you can specify additional track metadata when adding tracks: +```ts +// This specifies that the video track should be rotated by 90 degrees clockwise +// before being displayed by video players, and that a frame rate of 30 FPS is +// expected. +output.addVideoTrack(videoSource, { + rotation: 90, // Clockwise rotation in degrees + frameRate: 30, +}); + +// This adds two audio tracks; one in English and one in German. +output.addAudioTrack(audioSourceEng, { + language: 'eng', // ISO 639-2/T language code +}); +output.addAudioTrack(audioSourceGer, { + language: 'ger', +}); + +// This adds multiple subtitle tracks, all for different languages. +output.addSubtitleTrack(subtitleSourceEng, { language: 'eng' }); +output.addSubtitleTrack(subtitleSourceGer, { language: 'ger' }); +output.addSubtitleTrack(subtitleSourceSpa, { language: 'spa' }); +output.addSubtitleTrack(subtitleSourceFre, { language: 'fre' }); +output.addSubtitleTrack(subtitleSourceIta, { language: 'ita' }); +``` + +As an example, let's add two tracks to our output: +- A video track driven by the contents of a `` element, encoded using AVC +- An audio track driven by the user's microphone input, encoded using AAC + +```ts +import { CanvasSource, MediaStreamAudioTrackSource } from 'mediakit'; + +// Assuming `canvasElement` exists +const videoSource = new CanvasSource(canvasElement, { + codec: 'avc', + bitrate: 1e6, // 1 Mbps +}); + +const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); +const audioStreamTrack = stream.getAudioTracks()[0]; +const audioSource = new MediaStreamAudioTrackSource(audioStreamTrack, { + codec: 'aac', + bitrate: 128e3, // 128 kbps +}); + +output.addVideoTrack(videoSource, { frameRate: 30 }); +output.addAudioTrack(audioSource); +``` + +::: warning +Adding tracks to an `Output` will throw if the track is not compatible with the output format. Be sure to respect the [properties](./output-formats#format-properties) of the output format when adding tracks. +::: + +## Starting the output + +After all tracks have been added to the `Output`, you need to *start* it. Starting an output spins up the writing process, allowing you to now start sending media data to the output file. It also prevents you from adding any new tracks to it. + +```ts +await output.start(); // Resolves once the output is ready to receive media data +``` + +## Adding media data + +After starting an `Output`, you can use the media sources you used to add tracks to pipe media data to the output file. The API for this is different for each [media source](./media-source), but it typically looks something like this: +```ts +mediaSource.add(...); +``` + +In our example, as soon as we called `start`, the user's microphone input will be piped to the output file. However, we still need to add the data from our canvas. We might do something like this: +```ts +let framesAdded = 0; +const intervalId = setInterval(() => { + const timestampInSeconds = framesAdded / 30; + const durationInSeconds = 1 / 30; + + // Captures the canvas state at the time of calling `add`: + videoSource.add(timestampInSeconds, durationInSeconds); + framesAdded++; +}, 1000 / 30); +``` + +And then we'll let this run for as long as we want to capture media data. + +## Finalizing the output + +Once all media data has been added, the `Output` needs to be *finalized*. Finalization finishes all remaining encoding +work and writes the remaining data to create the final, playable media file. +```ts +await output.finalize(); // Resolves once the output is finalized +``` + +::: warning +After calling `finalize`, adding more media data to the output results in an error. +::: + +In our example, we'll need to do this: +```ts +clearInterval(intervalId); // Stops the canvas loop +audioStreamTrack.stop(); // Stops capturing the user's microphone + +await output.finalize(); + +const file = output.target.buffer; // => Uint8Array +``` + +## Canceling an output + +Sometimes, you may want to cancel the ongoing creation of an output file. For this, use the `cancel` method: +```ts +await output.cancel(); // Resolves once the output is canceled +``` + +This automatically frees up all resources used by the output process, such as closing all encoders or releasing the +writer. + +::: warning +After calling `cancel`, adding more media data to the output results in an error. +::: + +In our example, we would do this: +```ts +clearInterval(intervalId); // Stops the canvas loop +audioStreamTrack.stop(); // Stops capturing the user's microphone + +await output.cancel(); + +// The output is canceled +``` + +## Checking output state + +You can always check the current state the output is in using its `state` property: +```ts +output.state; // => 'pending' | 'started' | 'canceled' | 'finalizing' | 'finalized' +``` + +- `'pending'` - The output hasn't been started or canceled yet; new tracks can be added. +- `'started'` - The output has been started and is ready to receive media data; tracks can no longer be added. +- `'finalizing'` - `finalize` has been called but hasn't resolved yet; no more media data can be added. +- `'finalized'` - The output has been finalized and is done writing the file. +- `'canceled'` - The output has been canceled. + +## Output targets + +The _output target_ determines where the data created by the `Output` will be written. This library offers two targets: + +### `BufferTarget` + +This target writes all data to a single, contiguous, in-memory `ArrayBuffer`. This buffer will automatically grow as the file becomes larger. Usage is straightforward: +```ts +import { Output, BufferTarget } from 'mediakit'; + +const output = new Output({ + target: new BufferTarget(), + // ... +}); + +// ... + +output.target.buffer; // => null +await output.finalize(); +output.target.buffer; // => Uint8Array +``` + +This target is a great choice for small-ish files (< 100 MB), but since all data will be kept in memory, using it for large files is suboptimal. If the output gets very large, the page might crash due to memory exhaustion. For these cases, using `StreamTarget` is recommended. + +### `StreamTarget` + +This target passes you the data written by the `Output` in small chunks, requiring you to pipe that data elsewhere to manually assemble the final file. Example use cases include writing the file directly to disk, or uploading it to a server over the network. + +`StreamTarget` makes use of the Streams API, meaning you'll need to pass it an instance of `WritableStream`: +```ts +import { Output, StreamTarget, StreamTargetChunk } from 'mediakit'; + +const writable = new WritableStream({ + write(chunk: StreamTargetChunk) { + chunk.data; // => Uint8Array + chunk.position; // => number + + // Do something with the data... + } +}); + +const output = new Output({ + target: new StreamTarget(writable), + // ... +}); +``` + +Each chunk written to the `WritableStream` represents a contiguous chunk of bytes of the output file, `data`, that is expected to be written at the given byte offset, `position`. The `WritableStream` will automatically be closed when `finalize` or `cancel` are called on the `Output`. + +::: warning +Note that some byte regions in the output file may be written to multiple times. It is therefore **incorrect** to construct the final file by simply concatenating all `Uint8Array`s together - you **must** write each chunk of data at the specified byte offset position _in the order_ in which the chunks arrived. If you don't do this, your output file will likely be invalid or corrupted. + +Some [output formats](./output-formats) have "monotonic" writing modes in which the byte offset of a written chunk will always be equal to the total number of bytes in all previously written chunks. In other words, when writing is monotonic, simply concatening all `Uint8Array`s yields the correct result. Some APIs (like `appendBuffer` of Media Source Extensions) require this, so make sure to configure your output format accordingly for those cases. +::: + +#### Chunked mode + +By default, data will be emitted by the `StreamTarget` as soon as it is available. In some formats, these may lead to hundreds of write events per second. If you want to reduce the frequency of writes, `StreamTarget` offers an alternative "chunked mode" in which data will first be accumulated into large chunks of a given size in memory, and then only be emitted once a chunk is completely full. + +```ts +new StreamTarget(writable, { + chunked: true, + chunkSize: 2**20, // Optional; defaults to 16 MiB +}), +``` + +#### Applying backpressure + +Sometimes, the `Output` may produce new data faster than you are able to write it. In this case, you want to communicate to the `Output` that it should "chill out" and slow down to match the pace that the `WritableStream` is able to handle. When using `StreamTarget`, the `Output` will automatically respect the backpressure applied by the `WritableStream`. For this, it is useful to understand the [Stream API concepts](https://developer.mozilla.org/en-US/docs/Web/API/Streams_API/Concepts) of how to apply backpressure. + +For example, the writable may apply backpressure by returning a promise in `write`: +```ts +const writable = new WritableStream({ + write(chunk: StreamTargetChunk) { + // Pretend writing out data takes 10 milliseconds: + return new Promise(resolve => setTimeout(resolve, 10)); + } +}); +``` + +::: info +In order for the writable's backpressure to ripple through the entire pipeline, you must make sure to correctly respect the [backpressure applied by media sources](./media-sources#backpressure). +::: + +#### Usage with the File System API + +`StreamTargetChunk` is designed such that it is compatible with the File System API's `FileSystemWritableFileStream`. This means, if you want to write data directly to disk, you can simply do something like this: + +```ts +const handle = await window.showSaveFilePicker(); +const writableStream = await handle.createWritable(); + +const output = new Output({ + target: new StreamTarget(writableStream), + // ... +}); + +// ... + +await output.finalize(); // Will automatically close the writable stream +``` + +## Packet buffering + +Some [output formats](./output-formats) require *packet buffering* for multi-track outputs. Packet buffering occurs because the `Output` must wait for data from all tracks for a given timestamp to continue writing data. For example, should you first encode all your video frames and then encode the audio afterward, the `Output` will have to hold all of the video frames in memory until the audio packets start coming in. This might lead to memory exhaustion should your video be very long. When there is only one media track, this issue does not arise. + +Check the [Output formats](./output-formats) page to see which format configurations require packet buffering. + +--- + +If your output format configuration requires packet buffering, make sure to add media data in a somewhat interleaved way to keep memory usage low. For example, if you're creating a 5-minute file, add your data in chunks—10 seconds of video, then 10 seconds of audio, then repeat—instead of first adding all 300 seconds of video followed by all 300 seconds of audio. + +::: info +If this kind of chunking isn't possible for your use case, try adding the media with the overall smaller data footprint first: First add the 300 seconds of audio, then add the 300 seconds of video. +::: diff --git a/docs/guide/writing.md b/docs/guide/writing.md deleted file mode 100644 index e69de29..0000000 diff --git a/docs/index.md b/docs/index.md index 6cd5008..85e049c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -34,4 +34,5 @@ things the guide needs to cover: - all media sinks - utility functions - supported containers / codecs -- samples & packets \ No newline at end of file +- samples & packets +- conversion \ No newline at end of file diff --git a/src/index.ts b/src/index.ts index af28b88..bce8770 100644 --- a/src/index.ts +++ b/src/index.ts @@ -20,8 +20,8 @@ export { WebMOutputFormatOptions, Mp3OutputFormat, Mp3OutputFormatOptions, - WaveOutputFormat, - WaveOutputFormatOptions, + WavOutputFormat, + WavOutputFormatOptions, OggOutputFormat, OggOutputFormatOptions, TrackCountLimits, diff --git a/src/isobmff/isobmff-muxer.ts b/src/isobmff/isobmff-muxer.ts index b7068b9..140c161 100644 --- a/src/isobmff/isobmff-muxer.ts +++ b/src/isobmff/isobmff-muxer.ts @@ -335,6 +335,8 @@ export class IsobmffMuxer extends Muxer { async addEncodedAudioPacket(track: OutputAudioTrack, packet: EncodedPacket, meta?: EncodedAudioChunkMetadata) { const release = await this.mutex.acquire(); + console.log(meta); + try { const trackData = this.getAudioTrackData(track, meta); diff --git a/src/output-format.ts b/src/output-format.ts index ffc3398..f4676d8 100644 --- a/src/output-format.ts +++ b/src/output-format.ts @@ -100,12 +100,13 @@ export type IsobmffOutputFormatOptions = { * finalized. This produces a high-quality and compact output at the cost of a more expensive finalization step and * higher memory requirements. Data will be written monotonically (in order) when this option is set. * - * Use `'fragmented'` to place metadata at the start of the file by creating a fragmented file. In a + * Use `'fragmented'` to place metadata at the start of the file by creating a fragmented file (fMP4). In a * fragmented file, chunks of media and their metadata are written to the file in "fragments", eliminating the need - * to put all metadata in one place. Fragmented files are useful for streaming, as they allow for better random - * access. Furthermore, they remain lightweight to create even for very large files, as they don't require all media - * to be kept in memory. However, fragmented files are not as widely and wholly supported as regular MP4/MOV files. - * Data will be written monotonically (in order) when this option is set. + * to put all metadata in one place. Fragmented files are useful for streaming contexts, as each fragment can be + * played individually without requiring knowledge of the other fragments. Furthermore, they remain lightweight to + * create even for very large files, as they don't require all media to be kept in memory. However, fragmented files + * are not as widely and wholly supported as regular MP4/MOV files. Data will be written monotonically (in order) + * when this option is set. * * When this field is not defined, either `false` or `'in-memory'` will be used, automatically determined based on * the type of output target used. @@ -503,7 +504,7 @@ export class Mp3OutputFormat extends OutputFormat { * WAVE-specific output options. * @public */ -export type WaveOutputFormatOptions = { +export type WavOutputFormatOptions = { /** * Will be called once the file header is written. The header consists of the RIFF header, the format chunk, and the * start of the data chunk (with a placeholder size of 0). @@ -515,11 +516,11 @@ export type WaveOutputFormatOptions = { * WAVE file format, based on RIFF. * @public */ -export class WaveOutputFormat extends OutputFormat { +export class WavOutputFormat extends OutputFormat { /** @internal */ - _options: WaveOutputFormatOptions; + _options: WavOutputFormatOptions; - constructor(options: WaveOutputFormatOptions = {}) { + constructor(options: WavOutputFormatOptions = {}) { if (!options || typeof options !== 'object') { throw new TypeError('options must be an object.'); } diff --git a/src/wave/wave-muxer.ts b/src/wave/wave-muxer.ts index a4eca79..d800d73 100644 --- a/src/wave/wave-muxer.ts +++ b/src/wave/wave-muxer.ts @@ -5,16 +5,16 @@ import { WaveFormat } from './wave-demuxer'; import { RiffWriter } from './riff-writer'; import { Writer } from '../writer'; import { EncodedPacket } from '../packet'; -import { WaveOutputFormat } from '../output-format'; +import { WavOutputFormat } from '../output-format'; export class WaveMuxer extends Muxer { - private format: WaveOutputFormat; + private format: WavOutputFormat; private writer: Writer; private riffWriter: RiffWriter; private headerWritten = false; private dataSize = 0; - constructor(output: Output, format: WaveOutputFormat) { + constructor(output: Output, format: WavOutputFormat) { super(output); this.format = format;