+
+By default, Mediabunny requires a browser environment for full access to decoders, encoders, and video processing features. `@mediabunny/server` uses [NodeAV](https://github.com/seydx/node-av) to polyfill this functionality for server-side environments such as Node, Bun, or Deno, enabling the usage of all Mediabunny features on the server. The result is a server-side media processing API that integrates naturally with TypeScript as opposed to the awkwardness and inefficiencies of calling out to the FFmpeg CLI.
+
+Features added by this package include:
+- Video decoders and encoders for AVC (H.264), HEVC (H.265), VP8, VP9, and AV1. Supports both length-prefixed and Annex B AVC/HEVC as well as transparent video via VP9.
+- Audio decoders and encoders for AAC, MP3, Vorbis, Opus, FLAC, AC-3 and E-AC-3. Supports AAC in both AAC and ADTS formats.
+- Video frame transformation support (resize, rotate, crop)
+- Automatic hardware acceleration on all platforms (macOS, Linux, Windows)
+- Built-in multithreading
+- Zero-copy decode and encode paths
+
+> This package, like the rest of Mediabunny, is enabled by its [sponsors](https://mediabunny.dev/#sponsors) and their donations. If you've derived value from this package, please consider [leaving a donation](https://github.com/sponsors/Vanilagy)! 💘
+
+> This package was made possible in large part due to seydx's amazing work on [NodeAV](https://github.com/seydx/node-av). The library is truly modern, a joy to work with, and incredibly powerful. Give them a star!
+
+## Installation
+
+This library peer-depends on Mediabunny. Install both using npm:
+```bash
+npm install mediabunny @mediabunny/server
+```
+
+## Usage
+
+```ts
+import { registerMediabunnyServer } from '@mediabunny/server';
+registerMediabunnyServer();
+```
+
+That's it - you now have access to the full Mediabunny feature set on the server.
+
+## Upload media compression example
+
+Here, we set up a simple media compression server in Node.js. The client's request body is streamed to Mediabunny, the media gets processed, and the output is streamed directly to the disk. Memory usage is O(1) due to pipelining, and an overly fast uploader is automatically slowed down due to stream backpressure.
+
+```ts
+import { ALL_FORMATS, Conversion, FilePathTarget, Input, Mp4OutputFormat, Output, QUALITY_MEDIUM, ReadableStreamSource } from "mediabunny";
+import { registerMediabunnyServer } from "@mediabunny/server";
+import { Readable } from "node:stream";
+import http from "node:http";
+
+registerMediabunnyServer();
+
+const server = http.createServer(async (req, res) => {
+ // Read the request body as a stream
+ const stream = Readable.toWeb(req) as ReadableStream;
+ const input = new Input({
+ source: new ReadableStreamSource(stream),
+ formats: ALL_FORMATS,
+ });
+
+ // Stream the output directly to the disk, could also stream to S3 etc.
+ const output = new Output({
+ format: new Mp4OutputFormat(),
+ target: new FilePathTarget(`./converted-${crypto.randomUUID()}.mp4`),
+ });
+
+ try {
+ const conversion = await Conversion.init({
+ input,
+ output,
+ video: async track => ({
+ codec: 'avc',
+ height: Math.min(720, await track.getDisplayHeight()),
+ bitrate: QUALITY_MEDIUM,
+ }),
+ });
+ await conversion.execute();
+
+ res.statusCode = 204;
+ res.end();
+ } catch (error) {
+ res.statusCode = 500;
+ res.end();
+
+ console.error("Error processing media:", error);
+ }
+});
+
+server.listen(3000);
+```
+
+For all the other ways to use Mediabunny, refer to its [guide](https://mediabunny.dev/guide/introduction).
+
+## Performance
+
+`@mediabunny/server` is extremely performant as it is a thin wrapper around [NodeAV](https://github.com/seydx/node-av), which itself is a thin wrapper around the FFmpeg C API. All decoders and encoders automatically run on separate threads, keeping the main thread unblocked. Hardware acceleration is automatically detected and utilized on all operating systems whenever available (unless explicitly disabled using `hardwareAcceleration: 'prefer-software'`). Video frame and audio sample data is never copied from FFmpeg unless explicitly requested via `VideoSample.copyTo()` and `AudioSample.copyTo()`, and zero-copy GPU decode -> encode paths are used automatically whenever possible.
+
+## Advanced usage
+
+### Usage with NodeAV
+
+`@mediabunny/server` provides `AvFrameVideoSampleResource` and `AvFrameAudioSampleResource` as a means to create `VideoSample` and `AudioSample` instances that are directly backed by data residing in NodeAV's [`Frame`](https://seydx.github.io/node-av/api/lib/classes/Frame.html) (and therefore FFmpeg's `AVFrame`) without ever having to copy data to or from JavaScript. Reading NodeAV's documentation can help you make full use of this integration.
+
+To convert between Mediabunny and NodeAV (FFmpeg) worlds, you can do this:
+```ts
+import { VideoSample, AudioSample } from 'mediabunny';
+import { AvFrameVideoSampleResource, AvFrameAudioSampleResource, toAvFrame } from '@mediabunny/server';
+
+// Frame -> VideoSample
+new VideoSample(new AvFrameVideoSampleResource(frame), { timestamp });
+
+// Frame -> AudioSample
+new AudioSample(new AvFrameAudioSampleResource(frame));
+// (uses the timestamp in the frame)
+
+// VideoSample -> Frame
+await toAvFrame(videoSample, frame);
+
+// AudioSample -> Frame
+await toAvFrame(audioSample, frame);
+```
+
+#### Electron example
+
+For example, when using Electron, we may want to capture the app's contents without moving video data from the GPU to the CPU:
+```ts
+import { VideoSample } from 'mediabunny';
+import { AvFrameVideoSampleResource } from '@mediabunny/server';
+import { HardwareContext, SharedTexture, AV_HWDEVICE_TYPE_VIDEOTOOLBOX } from 'node-av';
+
+// Create hardware context (platform-specific)
+const hw = HardwareContext.create(AV_HWDEVICE_TYPE_VIDEOTOOLBOX);
+using sharedTexture = SharedTexture.create(hw);
+
+// In Electron paint event with offscreen rendering
+offscreen.webContents.on('paint', (event) => {
+ const texture = event.texture;
+ if (!texture?.textureInfo) {
+ return;
+ }
+
+ // Import as hardware frame (zero-copy)
+ const frame = sharedTexture.importTexture(texture.textureInfo, { pts: 0n });
+ const sample = new VideoSample(new AvFrameVideoSampleResource(frame), {
+ timestamp: 0,
+ duration: 0,
+ });
+
+ texture.release();
+});
+```
+
+#### Microphone recording example
+
+Here, we're using NodeAV's Device API to access the user's microphone:
+```ts
+import { AudioSample } from 'mediabunny';
+import { AvFrameAudioSampleResource } from '@mediabunny/server';
+import { DeviceAPI, Decoder } from 'node-av';
+
+await using mic = await DeviceAPI.openMicrophone();
+const audioStream = mic.audio()!;
+using decoder = await Decoder.create(audioStream);
+
+let firstTimestamp: number | null = null;
+for await (const frame of decoder.frames(mic.packets(audioStream.index))) {
+ if (!frame) {
+ break;
+ }
+
+ const sample = new AudioSample(new AvFrameAudioSampleResource(frame));
+
+ if (firstTimestamp === null) {
+ firstTimestamp = sample.timestamp;
+ }
+
+ // Offset timestamps so they start at 0
+ sample.setTimestamp(sample.timestamp - firstTimestamp);
+
+ // Do something with the sample now, like passing it to an AudioSampleSource
+ // ...
+}
+```
+
+### Video and audio processing
+
+Browser environments ship with many API goodies such as the Canvas 2D API which are a naturally great fit for doing video frame processing, and they integrate well with Mediabunny. On the server, these APIs don't exist, so other approaches must be used:
+
+#### VideoSample.transform()
+
+This method allows for simple transformations on `VideoSample` instances and works when `@mediabunny/server` has been registered:
+```ts
+const transformed = await sample.transform({
+ width: 640,
+ height: 360,
+ fit: 'cover',
+});
+```
+
+#### NodeAV filter graphs
+
+FFmpeg's `libavfilter` is an incredibly powerful and generic media processing library, and all of it is directly accessible via NodeAV. It works for both video as well as audio data.
+
+For example, here we combine Mediabunny's Conversion API with a filter graph to grayscale a video:
+```ts
+import { Conversion } from 'mediabunny';
+import { AvFrameVideoSampleResource, toAvFrame } from '@mediabunny/server';
+import { Frame, FilterAPI } from 'node-av';
+
+async function* one(f: Frame) { yield f; }
+
+const conversion = await Conversion.init({
+ // ...
+ video: {
+ process: async (sample) => {
+ // VideoSample -> Frame
+ using inFrame = new Frame();
+ inFrame.alloc();
+ await toAvFrame(sample, inFrame);
+
+ // Frame -> filter -> AvFrameVideoSampleResource
+ using filter = FilterAPI.create('format=gray');
+ for await (const outFrame of filter.frames(one(inFrame))) {
+ return outFrame && new AvFrameVideoSampleResource(outFrame);
+ }
+
+ return null;
+ },
+ },
+ // ...
+});
+await conversion.execute();
+```
+
+#### Canvas API polyfills
+
+Libraries like [Skia Canvas](https://github.com/samizdatco/skia-canvas) provide GPU-enabled polyfills for the Canvas 2D API. Using it with Mediabunny is simply a matter of converting from and to the Canvas API:
+```ts
+const width = videoSample.displayWidth;
+const height = videoSample.displayHeight;
+
+const canvas = new Canvas(width, height);
+const ctx = canvas.getContext('2d');
+
+// Copy data from VideoSample
+const imageData = ctx.createImageData(width, height);
+await videoSample.copyTo(imageData.data, { format: 'RGBA' });
+ctx.putImageData(imageData, 0, 0);
+
+// Issue draw commands
+ctx.fillStyle = 'red';
+ctx.fillRect(20, 20, 100, 60);
+
+// Convert to VideoSample again
+const pixels = ctx.getImageData(0, 0, width, height).data;
+return new VideoSample(pixels, {
+ format: 'RGBA',
+ codedWidth: width,
+ codedHeight: height,
+ timestamp: videoSample.timestamp,
+ duration: videoSample.duration,
+});
+```
+
+## Implementation details
+
+`@mediabunny/server` uses [NodeAV](https://github.com/seydx/node-av) under the hood which provides N-API C bindings to FFmpeg's C API. Using NodeAV, this package implements [custom decoders and encoders](https://mediabunny.dev/guide/supported-formats-and-codecs#custom-coders) by directly using the APIs provided by `libavcodec`.
+
+For encoding, video frames and audio samples are transferred to FFmpeg by converting them to an `AVFrame` and are then passed to the correct encoder. The resulting packets are then normalized into the format expected by WebCodecs and the [Mediabunny Codec Registry](https://mediabunny.dev/codec-registry/overview). For decoding, the above process is inverted: packets and decoder metadata are passed to the correct decoder, and the resulting `AVFrame` instances are wrapped in `VideoSample` or `AudioSample` instances. Video frame transformations (resize, rotate, crop) are implemented using the `libavfilter` API.
+
+Whenever possible, `AVFrame`s are never copied over to JavaScript unless explicitly needed. This enables zero-copy decode -> transformation -> encode paths.
+
+## License
+
+`@mediabunny/server` uses the same MPL-2.0 license as Mediabunny.
\ No newline at end of file
diff --git a/packages/server/logo.svg b/packages/server/logo.svg
new file mode 100644
index 0000000..21bddf7
--- /dev/null
+++ b/packages/server/logo.svg
@@ -0,0 +1,4094 @@
+
+
+
diff --git a/packages/server/src/audio-decoder.ts b/packages/server/src/audio-decoder.ts
index a273214..4d94a42 100644
--- a/packages/server/src/audio-decoder.ts
+++ b/packages/server/src/audio-decoder.ts
@@ -10,7 +10,7 @@ import { AudioCodec, AudioSample, CustomAudioDecoder, EncodedPacket, type MaybeP
import * as NodeAv from 'node-av';
import { CODEC_TO_CODEC_ID, getChannelLayout } from './misc';
import { assert, toUint8Array } from '../../../src/misc';
-import { NodeAvFrameAudioSampleResource } from './audio-sample';
+import { AvFrameAudioSampleResource } from './audio-sample';
export class NodeAvAudioDecoder extends CustomAudioDecoder {
frame!: NodeAv.Frame;
@@ -90,7 +90,7 @@ export class NodeAvAudioDecoder extends CustomAudioDecoder {
}
clone.timeBase = new NodeAv.Rational(1, this.config.sampleRate);
- this.onSample(new AudioSample(new NodeAvFrameAudioSampleResource(clone)));
+ this.onSample(new AudioSample(new AvFrameAudioSampleResource(clone)));
}
async flush(): Promise {
diff --git a/packages/server/src/audio-encoder.ts b/packages/server/src/audio-encoder.ts
index a20030a..4dd4494 100644
--- a/packages/server/src/audio-encoder.ts
+++ b/packages/server/src/audio-encoder.ts
@@ -15,9 +15,9 @@ import {
EncodedPacket,
} from 'mediabunny';
import * as NodeAv from 'node-av';
-import { CODEC_TO_CODEC_ID, fromAudioSampleFormat, getChannelLayout } from './misc';
+import { CODEC_TO_CODEC_ID, getChannelLayout } from './misc';
import { assert, toUint8Array } from '../../../src/misc';
-import { NodeAvFrameAudioSampleResource } from './audio-sample';
+import { copyAudioSampleToAvFrame, AvFrameAudioSampleResource } from './audio-sample';
import {
AdtsHeaderTemplate,
buildAdtsHeaderTemplate,
@@ -133,26 +133,14 @@ export class NodeAvAudioEncoder extends CustomAudioEncoder {
this.firstExpectedTimestamp ??= audioSample.timestamp;
- if (audioSample._data instanceof NodeAvFrameAudioSampleResource) {
+ if (audioSample._data instanceof AvFrameAudioSampleResource) {
this.frame.ref(audioSample._data.frame);
} else {
- // Copy audio data from AudioData to FFmpeg Frame
- const format = fromAudioSampleFormat(audioSample.format);
- this.frame.format = format;
- this.frame.nbSamples = audioSample.numberOfFrames;
- this.frame.sampleRate = audioSample.sampleRate;
- this.frame.channelLayout = getChannelLayout(audioSample.numberOfChannels);
- this.frame.duration = BigInt(Math.round(audioSample.duration * this.config.sampleRate));
-
- this.frame.allocBuffer();
- assert(this.frame.data);
-
- for (let i = 0; i < this.frame.data.length; i++) {
- audioSample.copyTo(this.frame.data[i]!, { planeIndex: i });
- }
+ copyAudioSampleToAvFrame(audioSample, this.frame);
}
this.frame.pts = BigInt(Math.round(audioSample.timestamp * this.config.sampleRate));
+ this.frame.duration = BigInt(Math.round(audioSample.duration * this.config.sampleRate));
this.frame.timeBase = new NodeAv.Rational(1, this.config.sampleRate);
const key = `${this.frame.sampleRate}:${this.frame.channels}:${this.frame.format}`;
diff --git a/packages/server/src/audio-sample.ts b/packages/server/src/audio-sample.ts
index e09d2cb..b9abb65 100644
--- a/packages/server/src/audio-sample.ts
+++ b/packages/server/src/audio-sample.ts
@@ -6,9 +6,9 @@
* file, You can obtain one at https://mozilla.org/MPL/2.0/.
*/
-import { AudioSampleResource } from 'mediabunny';
+import { AudioSample, AudioSampleResource } from 'mediabunny';
import * as NodeAv from 'node-av';
-import { toAudioSampleFormat } from './misc';
+import { fromAudioSampleFormat, getChannelLayout, toAudioSampleFormat } from './misc';
import { assert, toUint8Array } from '../../../src/misc';
/**
@@ -17,10 +17,13 @@ import { assert, toUint8Array } from '../../../src/misc';
* [`AVFrame`](https://ffmpeg.org/doxygen/2.7/structAVFrame.html). You can use this resource to create `AudioSample`
* instances that are directly backed by FFmpeg's `AVFrame` without data having to be copied.
*
+ * When passed, the `Frame` is now owned by resource, meaning it takes care of closing the frame later. If you want to
+ * keep a copy for your own use, clone the frame first.
+ *
* @group \@mediabunny/server
* @public
*/
-export class NodeAvFrameAudioSampleResource extends AudioSampleResource {
+export class AvFrameAudioSampleResource extends AudioSampleResource {
/** @internal */
_frame: NodeAv.Frame | null;
@@ -30,7 +33,7 @@ export class NodeAvFrameAudioSampleResource extends AudioSampleResource {
*/
get frame() {
if (!this._frame) {
- throw new Error('NodeAvFrameAudioSampleResource has been closed.');
+ throw new Error('AvFrameAudioSampleResource has been closed.');
}
return this._frame;
@@ -40,7 +43,7 @@ export class NodeAvFrameAudioSampleResource extends AudioSampleResource {
super();
if (frame.getMediaType() !== NodeAv.AVMEDIA_TYPE_AUDIO) {
- throw new Error('NodeAvFrameAudioSampleResource must be initialized with an audio frame.');
+ throw new Error('AvFrameAudioSampleResource must be initialized with an audio frame.');
}
this._frame = frame;
@@ -49,7 +52,8 @@ export class NodeAvFrameAudioSampleResource extends AudioSampleResource {
getFormat(): AudioSampleFormat {
const result = toAudioSampleFormat(this.frame.format as NodeAv.AVSampleFormat);
if (result === null) {
- throw new TypeError('Unsupported audio sample format: ' + this.frame.format);
+ const name = NodeAv.avGetSampleFmtName(this.frame.format as NodeAv.AVSampleFormat);
+ throw new TypeError(`Unsupported audio sample format: ${name} (${this.frame.format})`);
}
return result;
@@ -81,3 +85,17 @@ export class NodeAvFrameAudioSampleResource extends AudioSampleResource {
return toUint8Array(this.frame.data[planeIndex]!);
}
}
+
+export const copyAudioSampleToAvFrame = (sample: AudioSample, frame: NodeAv.Frame) => {
+ frame.format = fromAudioSampleFormat(sample.format);
+ frame.nbSamples = sample.numberOfFrames;
+ frame.sampleRate = sample.sampleRate;
+ frame.channelLayout = getChannelLayout(sample.numberOfChannels);
+
+ frame.allocBuffer();
+ assert(frame.data);
+
+ for (let i = 0; i < frame.data.length; i++) {
+ sample.copyTo(frame.data[i]!, { planeIndex: i });
+ }
+};
diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts
index c0eada0..a59337c 100644
--- a/packages/server/src/index.ts
+++ b/packages/server/src/index.ts
@@ -6,13 +6,14 @@
* file, You can obtain one at https://mozilla.org/MPL/2.0/.
*/
-import { registerDecoder, registerEncoder, registerVideoSampleTransformer } from 'mediabunny';
+import { AudioSample, registerDecoder, registerEncoder, registerVideoSampleTransformer, VideoSample } from 'mediabunny';
import * as NodeAv from 'node-av';
import { NodeAvVideoDecoder } from './video-decoder';
import { NodeAvVideoEncoder } from './video-encoder';
import { NodeAvAudioDecoder } from './audio-decoder';
import { NodeAvAudioEncoder } from './audio-encoder';
-import { transformVideoSample } from './video-sample';
+import { copyVideoSampleToAvFrame, AvFrameVideoSampleResource, transformVideoSample } from './video-sample';
+import { copyAudioSampleToAvFrame, AvFrameAudioSampleResource } from './audio-sample';
const SERVER_LOADED_SYMBOL = Symbol.for('@mediabunny/server loaded');
if ((globalThis as Record)[SERVER_LOADED_SYMBOL]) {
@@ -59,5 +60,45 @@ export const registerMediabunnyServer = () => {
registerVideoSampleTransformer(transformVideoSample);
};
-export { NodeAvFrameVideoSampleResource } from './video-sample';
-export { NodeAvFrameAudioSampleResource } from './audio-sample';
+export { AvFrameVideoSampleResource } from './video-sample';
+export { AvFrameAudioSampleResource } from './audio-sample';
+
+/**
+ * Copies a `VideoSample` or `AudioSample` into the given NodeAV
+ * [`Frame`](https://seydx.github.io/node-av/api/lib/classes/Frame.html), setting up the frame's format, media type,
+ * timing and data. When the sample is already backed by an `AVFrame` (via `AvFrameVideoSampleResource` or
+ * `AvFrameAudioSampleResource`), the frame is ref'd instead of copied for zero-copy reuse.
+ *
+ * For video samples, the frame's time base is always set to 1/1000000 (microsecond accuracy). For audio samples, the
+ * frame's time base is always set to 1/sampleRate.
+ *
+ * @group \@mediabunny/server
+ * @public
+ */
+export const toAvFrame = async (sample: VideoSample | AudioSample, frame: NodeAv.Frame) => {
+ if (sample instanceof VideoSample) {
+ if (sample._data instanceof AvFrameVideoSampleResource) {
+ frame.ref(sample._data.frame);
+ } else {
+ if (sample.format === null) {
+ throw new Error('Cannot convert foreign VideoSample with unknown (null) format.');
+ }
+
+ await copyVideoSampleToAvFrame(sample, frame, null);
+ }
+
+ frame.pts = BigInt(sample.microsecondTimestamp);
+ frame.duration = BigInt(sample.microsecondDuration);
+ frame.timeBase = new NodeAv.Rational(1, 1e6);
+ } else {
+ if (sample._data instanceof AvFrameAudioSampleResource) {
+ frame.ref(sample._data.frame);
+ } else {
+ copyAudioSampleToAvFrame(sample, frame);
+ }
+
+ frame.timeBase = new NodeAv.Rational(1, sample.sampleRate);
+ frame.pts = BigInt(Math.round(sample.timestamp * sample.sampleRate));
+ frame.duration = BigInt(sample.numberOfFrames);
+ }
+};
diff --git a/packages/server/src/video-decoder.ts b/packages/server/src/video-decoder.ts
index e234a3b..14272a2 100644
--- a/packages/server/src/video-decoder.ts
+++ b/packages/server/src/video-decoder.ts
@@ -10,7 +10,7 @@ import { CustomVideoDecoder, VideoCodec, EncodedPacket, VideoSample, type MaybeP
import * as NodeAv from 'node-av';
import { CODEC_TO_CODEC_ID, getHardwareDecoderCodec, LIBVPX_VP9 } from './misc';
import { assert, binarySearchLessOrEqual, simplifyRational, toUint8Array } from '../../../src/misc';
-import { NodeAvFrameVideoSampleResource } from './video-sample';
+import { AvFrameVideoSampleResource } from './video-sample';
export class NodeAvVideoDecoder extends CustomVideoDecoder {
frame!: NodeAv.Frame;
@@ -187,7 +187,7 @@ export class NodeAvVideoDecoder extends CustomVideoDecoder {
throw new Error('Frame clone allocation failed.');
}
- this.onSample(new VideoSample(new NodeAvFrameVideoSampleResource(clone), {
+ this.onSample(new VideoSample(new AvFrameVideoSampleResource(clone), {
timestamp,
duration,
}));
diff --git a/packages/server/src/video-encoder.ts b/packages/server/src/video-encoder.ts
index d2c4055..1cdfc09 100644
--- a/packages/server/src/video-encoder.ts
+++ b/packages/server/src/video-encoder.ts
@@ -24,7 +24,7 @@ import {
unmapMatrixCoefficients,
unmapTransferCharacteristics,
} from './misc';
-import { copyVideoSampleToAvFrame, NodeAvFrameVideoSampleResource } from './video-sample';
+import { copyVideoSampleToAvFrame, AvFrameVideoSampleResource } from './video-sample';
import {
AvcNalUnitType,
extractAv1CodecInfoFromPacket,
@@ -187,7 +187,7 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
assert(this.codecContext);
}
- if (videoSample._data instanceof NodeAvFrameVideoSampleResource) {
+ if (videoSample._data instanceof AvFrameVideoSampleResource) {
this.frame.ref(videoSample._data.frame);
} else {
if (videoSample.format === null) {
diff --git a/packages/server/src/video-sample.ts b/packages/server/src/video-sample.ts
index 549470e..ee0368e 100644
--- a/packages/server/src/video-sample.ts
+++ b/packages/server/src/video-sample.ts
@@ -48,10 +48,13 @@ const JPEG_RANGE_PIX_FORMATS = new Set([
* When using Electron, you can directly create `Frame` instances without the data having to leave the GPU. For more,
* see [NodeAV's docs](https://seydx.github.io/node-av/api/lib/classes/Frame.html).
*
+ * When passed, the `Frame` is now owned by resource, meaning it takes care of closing the frame later. If you want to
+ * keep a copy for your own use, clone the frame first.
+ *
* @group \@mediabunny/server
* @public
*/
-export class NodeAvFrameVideoSampleResource extends VideoSampleResource {
+export class AvFrameVideoSampleResource extends VideoSampleResource {
/** @internal */
_frame: NodeAv.Frame | null;
@@ -61,7 +64,7 @@ export class NodeAvFrameVideoSampleResource extends VideoSampleResource {
*/
get frame() {
if (!this._frame) {
- throw new Error('NodeAvFrameVideoSampleResource has been closed.');
+ throw new Error('AvFrameVideoSampleResource has been closed.');
}
return this._frame;
@@ -71,7 +74,7 @@ export class NodeAvFrameVideoSampleResource extends VideoSampleResource {
super();
if (frame.getMediaType() !== NodeAv.AVMEDIA_TYPE_VIDEO) {
- throw new Error('NodeAvFrameVideoSampleResource must be initialized with a video frame.');
+ throw new Error('AvFrameVideoSampleResource must be initialized with a video frame.');
}
this._frame = frame;
@@ -169,7 +172,7 @@ export class NodeAvFrameVideoSampleResource extends VideoSampleResource {
dstFrame.sampleAspectRatio = srcFrame.sampleAspectRatio;
- return new VideoSample(new NodeAvFrameVideoSampleResource(dstFrame), init);
+ return new VideoSample(new AvFrameVideoSampleResource(dstFrame), init);
}
}
@@ -213,7 +216,7 @@ export const transformVideoSample = async (
let srcFrame: NodeAv.Frame;
let srcFrameOwned = false;
- if (sample._data instanceof NodeAvFrameVideoSampleResource) {
+ if (sample._data instanceof AvFrameVideoSampleResource) {
srcFrame = sample._data.frame;
} else {
if (sample.format === null) {
@@ -295,7 +298,7 @@ export const transformVideoSample = async (
const getRet = await bufferSink.buffersinkGetFrame(dstFrame);
NodeAv.FFmpegError.throwIfError(getRet, 'buffersinkGetFrame');
- return new VideoSample(new NodeAvFrameVideoSampleResource(dstFrame), {
+ return new VideoSample(new AvFrameVideoSampleResource(dstFrame), {
timestamp: sample.timestamp,
duration: sample.duration,
rotation: 0, // baked in by the filter graph
diff --git a/src/conversion.ts b/src/conversion.ts
index f286ade..f7106fe 100644
--- a/src/conversion.ts
+++ b/src/conversion.ts
@@ -56,6 +56,7 @@ import {
toInterleavedAudioFormat,
validateCropRectangle,
VideoSample,
+ VideoSampleResource,
} from './sample';
import { MetadataTags, validateMetadataTags } from './metadata';
import { NullTarget } from './target';
@@ -225,16 +226,17 @@ export type ConversionVideoOptions = {
* timestamp modifications. Will be called for each input video sample after transformations and frame rate
* corrections.
*
- * Must return a {@link VideoSample} or a `CanvasImageSource`, an array of them, or `null` for dropping the frame.
- * When non-timestamped data is returned, the timestamp and duration from the source sample will be used. Rotation
- * metadata of the returned sample will be ignored.
+ * Must return a {@link VideoSample}, a {@link VideoSampleResource} or a `CanvasImageSource`, an array of them, or
+ * `null` for dropping the frame. When non-timestamped data is returned, the timestamp and duration from the source
+ * sample will be used. Rotation metadata of the returned sample will be ignored.
*
* This function can also be used to manually resize frames. When doing so, you should signal the post-process
* dimensions using the `processedWidth` and `processedHeight` fields, which enables the encoder to better know what
* to expect. If these fields aren't set, Mediabunny will assume you won't perform any resizing.
*/
process?: (sample: VideoSample) => MaybePromise<
- CanvasImageSource | VideoSample | (CanvasImageSource | VideoSample)[] | null
+ CanvasImageSource | VideoSample | VideoSampleResource
+ | (CanvasImageSource | VideoSample | VideoSampleResource)[] | null
>;
/**
* An optional hint specifying the width of video samples returned by the `process` function, for better
@@ -1057,6 +1059,10 @@ export class Conversion {
}
this._executed = true;
+ for (const id of this._outputTrackIds) {
+ this._synchronizer.declareTrack(id);
+ }
+
if (this.onProgress) {
// Compute duration using only the utilized tracks
const uniqueUtilizedTracks = new Set(this.utilizedTracks);
@@ -1432,7 +1438,7 @@ export class Conversion {
// Calling the VideoSample constructor here will automatically handle input validation for us
// (it throws for any non-legal argument).
- return new VideoSample(x, {
+ return new VideoSample(x as CanvasImageSource, {
timestamp: sample.timestamp,
duration: sample.duration,
});
@@ -1819,7 +1825,7 @@ export class ConversionCanceledError extends Error {
}
}
-const MAX_TIMESTAMP_GAP = 5;
+const MAX_TIMESTAMP_GAP = 1; // in seconds
/**
* Utility class for synchronizing multiple track packet consumers with one another. We don't want one consumer to get
@@ -1834,6 +1840,36 @@ class TrackSynchronizer {
resolve: () => void;
}[] = [];
+ declareTrack(trackId: number) {
+ this.maxTimestamps.set(trackId, 0);
+ }
+
+ shouldWait(trackId: number, timestamp: number) {
+ const currentValue = this.maxTimestamps.get(trackId);
+ assert(currentValue !== undefined);
+
+ this.maxTimestamps.set(trackId, Math.max(timestamp, currentValue));
+
+ const newMin = this.computeMinAndMaybeResolve();
+ return timestamp - newMin > MAX_TIMESTAMP_GAP; // Should wait if it is too far ahead of the slowest consumer
+ }
+
+ wait(timestamp: number) {
+ const { promise, resolve } = promiseWithResolvers();
+
+ this.resolvers.push({
+ timestamp,
+ resolve,
+ });
+
+ return promise;
+ }
+
+ closeTrack(trackId: number) {
+ this.maxTimestamps.delete(trackId);
+ this.computeMinAndMaybeResolve();
+ }
+
computeMinAndMaybeResolve() {
let newMin = Infinity;
for (const [, timestamp] of this.maxTimestamps) {
@@ -1853,27 +1889,4 @@ class TrackSynchronizer {
return newMin;
}
-
- shouldWait(trackId: number, timestamp: number) {
- this.maxTimestamps.set(trackId, Math.max(timestamp, this.maxTimestamps.get(trackId) ?? -Infinity));
-
- const newMin = this.computeMinAndMaybeResolve();
- return timestamp - newMin >= MAX_TIMESTAMP_GAP; // Should wait if it is too far ahead of the slowest consumer
- }
-
- wait(timestamp: number) {
- const { promise, resolve } = promiseWithResolvers();
-
- this.resolvers.push({
- timestamp,
- resolve,
- });
-
- return promise;
- }
-
- closeTrack(trackId: number) {
- this.maxTimestamps.delete(trackId);
- this.computeMinAndMaybeResolve();
- }
}
diff --git a/src/encode.ts b/src/encode.ts
index ec0a0cd..3565da1 100644
--- a/src/encode.ts
+++ b/src/encode.ts
@@ -24,7 +24,7 @@ import {
import { customAudioEncoders, customVideoEncoders } from './custom-coder';
import { isFirefox, MaybePromise, Rotation } from './misc';
import { EncodedPacket } from './packet';
-import { AudioSample, CropRectangle, validateCropRectangle, VideoSample } from './sample';
+import { AudioSample, CropRectangle, validateCropRectangle, VideoSample, VideoSampleResource } from './sample';
export const canEncodeVideoMemo = new Map>();
export const canEncodeAudioMemo = new Map>();
@@ -126,11 +126,13 @@ export type VideoTransformOptions = {
* or timestamp modifications. Will be called for each video frame after transformations and frame rate
* corrections.
*
- * Must return a {@link VideoSample} or a `CanvasImageSource`, an array of them, or `null` for dropping the
- * frame. When non-timestamped data is returned, the timestamp and duration from the input sample will be used.
+ * Must return a {@link VideoSample}, a {@link VideoSampleResource} or a `CanvasImageSource`, an array of them, or
+ * `null` for dropping the frame. When non-timestamped data is returned, the timestamp and duration from the input
+ * sample will be used.
*/
process?: (sample: VideoSample) => MaybePromise<
- CanvasImageSource | VideoSample | (CanvasImageSource | VideoSample)[] | null
+ CanvasImageSource | VideoSample | VideoSampleResource
+ | (CanvasImageSource | VideoSample | VideoSampleResource)[] | null
>;
/**
* Forces every video frame through the transformation step even if no transformation properties are defined.
diff --git a/src/media-source.ts b/src/media-source.ts
index 6a77819..795aa58 100644
--- a/src/media-source.ts
+++ b/src/media-source.ts
@@ -427,7 +427,7 @@ class VideoEncoderWrapper {
return new VideoSample(x);
}
- return new VideoSample(x, {
+ return new VideoSample(x as CanvasImageSource, {
timestamp: videoSample.timestamp,
duration: videoSample.duration,
});
diff --git a/src/source.ts b/src/source.ts
index 8b64e9e..be6db57 100644
--- a/src/source.ts
+++ b/src/source.ts
@@ -1240,7 +1240,7 @@ type ReadableStreamSourcePendingSlice = {
* @public
*/
export type ReadableStreamSourceOptions = {
- /** The maximum number of bytes the cache is allowed to hold in memory. Defaults to 16 MiB. */
+ /** The maximum number of bytes the cache is allowed to hold in memory. Defaults to 32 MiB. */
maxCacheSize?: number;
};
@@ -1298,7 +1298,7 @@ export class ReadableStreamSource extends Source {
super();
this._stream = stream;
- this._maxCacheSize = options.maxCacheSize ?? (16 * 2 ** 20 /* 16 MiB */);
+ this._maxCacheSize = options.maxCacheSize ?? (32 * 2 ** 20 /* 32 MiB */);
}
/** @internal */
@@ -1431,6 +1431,8 @@ export class ReadableStreamSource extends Source {
const startIndex = this._currentIndex;
const endIndex = this._currentIndex + value.byteLength;
+ this._dispatchRead(startIndex, endIndex);
+
// Fill the pending slices with the data
for (let i = 0; i < this._pendingSlices.length; i++) {
const pendingSlice = this._pendingSlices[i]!;
diff --git a/test/browser/media-sources.test.ts b/test/browser/media-sources.test.ts
index 316c209..7742163 100644
--- a/test/browser/media-sources.test.ts
+++ b/test/browser/media-sources.test.ts
@@ -243,10 +243,7 @@ test('VideoSampleSource, transform.process manual resize', async () => {
sample.draw(ctx, 0, 0, 60, 40);
sample.close();
- return new VideoSample(canvas, {
- timestamp: sample.timestamp,
- duration: sample.duration,
- });
+ return canvas;
},
},
},
@@ -474,10 +471,7 @@ test('VideoSampleSource, transform.frameRate works with process', async () => {
const ctx = canvas.getContext('2d')!;
sample.draw(ctx, 0, 0, 60, 40);
- return new VideoSample(canvas, {
- timestamp: sample.timestamp,
- duration: sample.duration,
- });
+ return canvas;
},
},
},
diff --git a/test/node/server-extension.test.ts b/test/node/server-extension.test.ts
index a9077a6..6267dff 100644
--- a/test/node/server-extension.test.ts
+++ b/test/node/server-extension.test.ts
@@ -24,8 +24,10 @@ import { Output } from '../../src/output.js';
import { Mp4OutputFormat } from '../../src/output-format.js';
import { BufferTarget } from '../../src/target.js';
import { Conversion } from '../../src/conversion.js';
-import { NodeAvFrameVideoSampleResource } from '../../packages/server/src/video-sample.js';
-import { NodeAvFrameAudioSampleResource } from '../../packages/server/src/audio-sample.js';
+import { AvFrameVideoSampleResource } from '../../packages/server/src/video-sample.js';
+import { AvFrameAudioSampleResource } from '../../packages/server/src/audio-sample.js';
+import { toAvFrame } from '../../packages/server/src/index.js';
+import * as NodeAv from 'node-av';
beforeAll(() => {
registerMediabunnyServer();
@@ -687,7 +689,7 @@ describe('Video', async () => {
using sample = await sink.getSample(0);
assert(sample);
- expect(sample._data).toBeInstanceOf(NodeAvFrameVideoSampleResource);
+ expect(sample._data).toBeInstanceOf(AvFrameVideoSampleResource);
expect(sample.format).toBe('I420');
expect(sample.codedWidth).toBe(1920);
expect(sample.codedHeight).toBe(1080);
@@ -822,6 +824,47 @@ describe('Video', async () => {
]);
});
+ test('toAvFrame with RGBA VideoSample', async () => {
+ const width = 16;
+ const height = 8;
+ const data = new Uint8Array(width * height * 4);
+ for (let i = 0; i < width * height; i++) {
+ data[i * 4] = 0x12;
+ data[i * 4 + 1] = 0x34;
+ data[i * 4 + 2] = 0x56;
+ data[i * 4 + 3] = 0xff;
+ }
+
+ using sample = new VideoSample(data, {
+ format: 'RGBA',
+ codedWidth: width,
+ codedHeight: height,
+ timestamp: 0.5,
+ duration: 1 / 30,
+ });
+
+ using frame = new NodeAv.Frame();
+ frame.alloc();
+
+ await toAvFrame(sample, frame);
+
+ expect(frame.getMediaType()).toBe(NodeAv.AVMEDIA_TYPE_VIDEO);
+ expect(frame.format).toBe(NodeAv.AV_PIX_FMT_RGBA);
+ expect(frame.width).toBe(width);
+ expect(frame.height).toBe(height);
+ expect(Number(frame.pts)).toBe(Math.round(0.5 * 1e6));
+ expect(Number(frame.duration)).toBe(Math.round((1 / 30) * 1e6));
+ expect(frame.timeBase.num).toBe(1);
+ expect(frame.timeBase.den).toBe(1e6);
+
+ assert(frame.data);
+ const plane = toUint8Array(frame.data[0]!);
+ expect(plane[0]).toBe(0x12);
+ expect(plane[1]).toBe(0x34);
+ expect(plane[2]).toBe(0x56);
+ expect(plane[3]).toBe(0xff);
+ });
+
describe('VideoSample transformation', () => {
// 400x400 image: red everywhere, with a 200x200 blue square filling the bottom-left quadrant.
const TEST_IMAGE = (() => {
@@ -1558,7 +1601,7 @@ describe('Audio', async () => {
using sample = await sink.getSample(await audioTrack.getFirstTimestamp());
assert(sample);
- expect(sample._data).toBeInstanceOf(NodeAvFrameAudioSampleResource);
+ expect(sample._data).toBeInstanceOf(AvFrameAudioSampleResource);
expect(sample.format).toBe('f32-planar');
expect(sample.numberOfChannels).toBe(6);
expect(sample.sampleRate).toBe(48000);
@@ -1579,4 +1622,40 @@ describe('Audio', async () => {
sample.copyTo(buf, { planeIndex: 5 });
sample.copyTo(buf, { format: 'f32', planeIndex: 0 });
});
+
+ test('toAvFrame with f32 AudioSample', async () => {
+ const sampleRate = 48000;
+ const numberOfChannels = 2;
+ const numberOfFrames = 1024;
+ const data = createF32SineWave(sampleRate, numberOfChannels, numberOfFrames / sampleRate);
+
+ using sample = new AudioSample({
+ data,
+ format: 'f32',
+ timestamp: 0.25,
+ numberOfChannels,
+ sampleRate,
+ });
+
+ using frame = new NodeAv.Frame();
+ frame.alloc();
+
+ await toAvFrame(sample, frame);
+
+ expect(frame.getMediaType()).toBe(NodeAv.AVMEDIA_TYPE_AUDIO);
+ expect(frame.format).toBe(NodeAv.AV_SAMPLE_FMT_FLT);
+ expect(frame.sampleRate).toBe(sampleRate);
+ expect(frame.channels).toBe(numberOfChannels);
+ expect(frame.nbSamples).toBe(numberOfFrames);
+ expect(Number(frame.pts)).toBe(Math.round(0.25 * sampleRate));
+ expect(Number(frame.duration)).toBe(numberOfFrames);
+ expect(frame.timeBase.num).toBe(1);
+ expect(frame.timeBase.den).toBe(sampleRate);
+
+ assert(frame.data);
+ const u8 = toUint8Array(frame.data[0]!);
+ const plane = new Float32Array(u8.buffer, u8.byteOffset, numberOfFrames * numberOfChannels);
+ expect(plane[0]).toBe(data[0]);
+ expect(plane[1]).toBe(data[1]);
+ });
});
+