Add toAvFrame, NodeAv* -> Av*, add server README, fix track synchronizer in conversion, increase default cache size for ReadableStreamSource

packages/server/src/audio-decoder.ts

@@ -10,7 +10,7 @@ import { AudioCodec, AudioSample, CustomAudioDecoder, EncodedPacket, type MaybeP
 import * as NodeAv from 'node-av';
 import { CODEC_TO_CODEC_ID, getChannelLayout } from './misc';
 import { assert, toUint8Array } from '../../../src/misc';
-import { NodeAvFrameAudioSampleResource } from './audio-sample';
+import { AvFrameAudioSampleResource } from './audio-sample';
 
 export class NodeAvAudioDecoder extends CustomAudioDecoder {
 	frame!: NodeAv.Frame;
@@ -90,7 +90,7 @@ export class NodeAvAudioDecoder extends CustomAudioDecoder {
 		}
 
 		clone.timeBase = new NodeAv.Rational(1, this.config.sampleRate);
-		this.onSample(new AudioSample(new NodeAvFrameAudioSampleResource(clone)));
+		this.onSample(new AudioSample(new AvFrameAudioSampleResource(clone)));
 	}
 
 	async flush(): Promise<void> {

packages/server/src/audio-encoder.ts

@@ -15,9 +15,9 @@ import {
 	EncodedPacket,
 } from 'mediabunny';
 import * as NodeAv from 'node-av';
-import { CODEC_TO_CODEC_ID, fromAudioSampleFormat, getChannelLayout } from './misc';
+import { CODEC_TO_CODEC_ID, getChannelLayout } from './misc';
 import { assert, toUint8Array } from '../../../src/misc';
-import { NodeAvFrameAudioSampleResource } from './audio-sample';
+import { copyAudioSampleToAvFrame, AvFrameAudioSampleResource } from './audio-sample';
 import {
 	AdtsHeaderTemplate,
 	buildAdtsHeaderTemplate,
@@ -133,26 +133,14 @@ export class NodeAvAudioEncoder extends CustomAudioEncoder {
 
 		this.firstExpectedTimestamp ??= audioSample.timestamp;
 
-		if (audioSample._data instanceof NodeAvFrameAudioSampleResource) {
+		if (audioSample._data instanceof AvFrameAudioSampleResource) {
 			this.frame.ref(audioSample._data.frame);
 		} else {
-			// Copy audio data from AudioData to FFmpeg Frame
-			const format = fromAudioSampleFormat(audioSample.format);
-			this.frame.format = format;
-			this.frame.nbSamples = audioSample.numberOfFrames;
-			this.frame.sampleRate = audioSample.sampleRate;
-			this.frame.channelLayout = getChannelLayout(audioSample.numberOfChannels);
-			this.frame.duration = BigInt(Math.round(audioSample.duration * this.config.sampleRate));
-
-			this.frame.allocBuffer();
-			assert(this.frame.data);
-
-			for (let i = 0; i < this.frame.data.length; i++) {
-				audioSample.copyTo(this.frame.data[i]!, { planeIndex: i });
-			}
+			copyAudioSampleToAvFrame(audioSample, this.frame);
 		}
 
 		this.frame.pts = BigInt(Math.round(audioSample.timestamp * this.config.sampleRate));
+		this.frame.duration = BigInt(Math.round(audioSample.duration * this.config.sampleRate));
 		this.frame.timeBase = new NodeAv.Rational(1, this.config.sampleRate);
 
 		const key = `${this.frame.sampleRate}:${this.frame.channels}:${this.frame.format}`;

packages/server/src/audio-sample.ts

@@ -6,9 +6,9 @@
  * file, You can obtain one at https://mozilla.org/MPL/2.0/.
  */
 
-import { AudioSampleResource } from 'mediabunny';
+import { AudioSample, AudioSampleResource } from 'mediabunny';
 import * as NodeAv from 'node-av';
-import { toAudioSampleFormat } from './misc';
+import { fromAudioSampleFormat, getChannelLayout, toAudioSampleFormat } from './misc';
 import { assert, toUint8Array } from '../../../src/misc';
 
 /**
@@ -17,10 +17,13 @@ import { assert, toUint8Array } from '../../../src/misc';
  * [`AVFrame`](https://ffmpeg.org/doxygen/2.7/structAVFrame.html). You can use this resource to create `AudioSample`
  * instances that are directly backed by FFmpeg's `AVFrame` without data having to be copied.
  *
+ * When passed, the `Frame` is now owned by resource, meaning it takes care of closing the frame later. If you want to
+ * keep a copy for your own use, clone the frame first.
+ *
  * @group \@mediabunny/server
  * @public
  */
-export class NodeAvFrameAudioSampleResource extends AudioSampleResource {
+export class AvFrameAudioSampleResource extends AudioSampleResource {
 	/** @internal */
 	_frame: NodeAv.Frame | null;
 
@@ -30,7 +33,7 @@ export class NodeAvFrameAudioSampleResource extends AudioSampleResource {
 	 */
 	get frame() {
 		if (!this._frame) {
-			throw new Error('NodeAvFrameAudioSampleResource has been closed.');
+			throw new Error('AvFrameAudioSampleResource has been closed.');
 		}
 
 		return this._frame;
@@ -40,7 +43,7 @@ export class NodeAvFrameAudioSampleResource extends AudioSampleResource {
 		super();
 
 		if (frame.getMediaType() !== NodeAv.AVMEDIA_TYPE_AUDIO) {
-			throw new Error('NodeAvFrameAudioSampleResource must be initialized with an audio frame.');
+			throw new Error('AvFrameAudioSampleResource must be initialized with an audio frame.');
 		}
 
 		this._frame = frame;
@@ -49,7 +52,8 @@ export class NodeAvFrameAudioSampleResource extends AudioSampleResource {
 	getFormat(): AudioSampleFormat {
 		const result = toAudioSampleFormat(this.frame.format as NodeAv.AVSampleFormat);
 		if (result === null) {
-			throw new TypeError('Unsupported audio sample format: ' + this.frame.format);
+			const name = NodeAv.avGetSampleFmtName(this.frame.format as NodeAv.AVSampleFormat);
+			throw new TypeError(`Unsupported audio sample format: ${name} (${this.frame.format})`);
 		}
 
 		return result;
@@ -81,3 +85,17 @@ export class NodeAvFrameAudioSampleResource extends AudioSampleResource {
 		return toUint8Array(this.frame.data[planeIndex]!);
 	}
 }
+
+export const copyAudioSampleToAvFrame = (sample: AudioSample, frame: NodeAv.Frame) => {
+	frame.format = fromAudioSampleFormat(sample.format);
+	frame.nbSamples = sample.numberOfFrames;
+	frame.sampleRate = sample.sampleRate;
+	frame.channelLayout = getChannelLayout(sample.numberOfChannels);
+
+	frame.allocBuffer();
+	assert(frame.data);
+
+	for (let i = 0; i < frame.data.length; i++) {
+		sample.copyTo(frame.data[i]!, { planeIndex: i });
+	}
+};

packages/server/src/index.ts

@@ -6,13 +6,14 @@
  * file, You can obtain one at https://mozilla.org/MPL/2.0/.
  */
 
-import { registerDecoder, registerEncoder, registerVideoSampleTransformer } from 'mediabunny';
+import { AudioSample, registerDecoder, registerEncoder, registerVideoSampleTransformer, VideoSample } from 'mediabunny';
 import * as NodeAv from 'node-av';
 import { NodeAvVideoDecoder } from './video-decoder';
 import { NodeAvVideoEncoder } from './video-encoder';
 import { NodeAvAudioDecoder } from './audio-decoder';
 import { NodeAvAudioEncoder } from './audio-encoder';
-import { transformVideoSample } from './video-sample';
+import { copyVideoSampleToAvFrame, AvFrameVideoSampleResource, transformVideoSample } from './video-sample';
+import { copyAudioSampleToAvFrame, AvFrameAudioSampleResource } from './audio-sample';
 
 const SERVER_LOADED_SYMBOL = Symbol.for('@mediabunny/server loaded');
 if ((globalThis as Record<symbol, unknown>)[SERVER_LOADED_SYMBOL]) {
@@ -59,5 +60,45 @@ export const registerMediabunnyServer = () => {
 	registerVideoSampleTransformer(transformVideoSample);
 };
 
-export { NodeAvFrameVideoSampleResource } from './video-sample';
-export { NodeAvFrameAudioSampleResource } from './audio-sample';
+export { AvFrameVideoSampleResource } from './video-sample';
+export { AvFrameAudioSampleResource } from './audio-sample';
+
+/**
+ * Copies a `VideoSample` or `AudioSample` into the given NodeAV
+ * [`Frame`](https://seydx.github.io/node-av/api/lib/classes/Frame.html), setting up the frame's format, media type,
+ * timing and data. When the sample is already backed by an `AVFrame` (via `AvFrameVideoSampleResource` or
+ * `AvFrameAudioSampleResource`), the frame is ref'd instead of copied for zero-copy reuse.
+ *
+ * For video samples, the frame's time base is always set to 1/1000000 (microsecond accuracy). For audio samples, the
+ * frame's time base is always set to 1/sampleRate.
+ *
+ * @group \@mediabunny/server
+ * @public
+ */
+export const toAvFrame = async (sample: VideoSample | AudioSample, frame: NodeAv.Frame) => {
+	if (sample instanceof VideoSample) {
+		if (sample._data instanceof AvFrameVideoSampleResource) {
+			frame.ref(sample._data.frame);
+		} else {
+			if (sample.format === null) {
+				throw new Error('Cannot convert foreign VideoSample with unknown (null) format.');
+			}
+
+			await copyVideoSampleToAvFrame(sample, frame, null);
+		}
+
+		frame.pts = BigInt(sample.microsecondTimestamp);
+		frame.duration = BigInt(sample.microsecondDuration);
+		frame.timeBase = new NodeAv.Rational(1, 1e6);
+	} else {
+		if (sample._data instanceof AvFrameAudioSampleResource) {
+			frame.ref(sample._data.frame);
+		} else {
+			copyAudioSampleToAvFrame(sample, frame);
+		}
+
+		frame.timeBase = new NodeAv.Rational(1, sample.sampleRate);
+		frame.pts = BigInt(Math.round(sample.timestamp * sample.sampleRate));
+		frame.duration = BigInt(sample.numberOfFrames);
+	}
+};

packages/server/src/video-decoder.ts

@@ -10,7 +10,7 @@ import { CustomVideoDecoder, VideoCodec, EncodedPacket, VideoSample, type MaybeP
 import * as NodeAv from 'node-av';
 import { CODEC_TO_CODEC_ID, getHardwareDecoderCodec, LIBVPX_VP9 } from './misc';
 import { assert, binarySearchLessOrEqual, simplifyRational, toUint8Array } from '../../../src/misc';
-import { NodeAvFrameVideoSampleResource } from './video-sample';
+import { AvFrameVideoSampleResource } from './video-sample';
 
 export class NodeAvVideoDecoder extends CustomVideoDecoder {
 	frame!: NodeAv.Frame;
@@ -187,7 +187,7 @@ export class NodeAvVideoDecoder extends CustomVideoDecoder {
 			throw new Error('Frame clone allocation failed.');
 		}
 
-		this.onSample(new VideoSample(new NodeAvFrameVideoSampleResource(clone), {
+		this.onSample(new VideoSample(new AvFrameVideoSampleResource(clone), {
 			timestamp,
 			duration,
 		}));

packages/server/src/video-encoder.ts

@@ -24,7 +24,7 @@ import {
 	unmapMatrixCoefficients,
 	unmapTransferCharacteristics,
 } from './misc';
-import { copyVideoSampleToAvFrame, NodeAvFrameVideoSampleResource } from './video-sample';
+import { copyVideoSampleToAvFrame, AvFrameVideoSampleResource } from './video-sample';
 import {
 	AvcNalUnitType,
 	extractAv1CodecInfoFromPacket,
@@ -187,7 +187,7 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
 			assert(this.codecContext);
 		}
 
-		if (videoSample._data instanceof NodeAvFrameVideoSampleResource) {
+		if (videoSample._data instanceof AvFrameVideoSampleResource) {
 			this.frame.ref(videoSample._data.frame);
 		} else {
 			if (videoSample.format === null) {

packages/server/src/video-sample.ts

@@ -48,10 +48,13 @@ const JPEG_RANGE_PIX_FORMATS = new Set([
  * When using Electron, you can directly create `Frame` instances without the data having to leave the GPU. For more,
  * see [NodeAV's docs](https://seydx.github.io/node-av/api/lib/classes/Frame.html).
  *
+ * When passed, the `Frame` is now owned by resource, meaning it takes care of closing the frame later. If you want to
+ * keep a copy for your own use, clone the frame first.
+ *
  * @group \@mediabunny/server
  * @public
  */
-export class NodeAvFrameVideoSampleResource extends VideoSampleResource {
+export class AvFrameVideoSampleResource extends VideoSampleResource {
 	/** @internal */
 	_frame: NodeAv.Frame | null;
 
@@ -61,7 +64,7 @@ export class NodeAvFrameVideoSampleResource extends VideoSampleResource {
 	 */
 	get frame() {
 		if (!this._frame) {
-			throw new Error('NodeAvFrameVideoSampleResource has been closed.');
+			throw new Error('AvFrameVideoSampleResource has been closed.');
 		}
 
 		return this._frame;
@@ -71,7 +74,7 @@ export class NodeAvFrameVideoSampleResource extends VideoSampleResource {
 		super();
 
 		if (frame.getMediaType() !== NodeAv.AVMEDIA_TYPE_VIDEO) {
-			throw new Error('NodeAvFrameVideoSampleResource must be initialized with a video frame.');
+			throw new Error('AvFrameVideoSampleResource must be initialized with a video frame.');
 		}
 
 		this._frame = frame;
@@ -169,7 +172,7 @@ export class NodeAvFrameVideoSampleResource extends VideoSampleResource {
 
 		dstFrame.sampleAspectRatio = srcFrame.sampleAspectRatio;
 
-		return new VideoSample(new NodeAvFrameVideoSampleResource(dstFrame), init);
+		return new VideoSample(new AvFrameVideoSampleResource(dstFrame), init);
 	}
 }
 
@@ -213,7 +216,7 @@ export const transformVideoSample = async (
 	let srcFrame: NodeAv.Frame;
 	let srcFrameOwned = false;
 
-	if (sample._data instanceof NodeAvFrameVideoSampleResource) {
+	if (sample._data instanceof AvFrameVideoSampleResource) {
 		srcFrame = sample._data.frame;
 	} else {
 		if (sample.format === null) {
@@ -295,7 +298,7 @@ export const transformVideoSample = async (
 		const getRet = await bufferSink.buffersinkGetFrame(dstFrame);
 		NodeAv.FFmpegError.throwIfError(getRet, 'buffersinkGetFrame');
 
-		return new VideoSample(new NodeAvFrameVideoSampleResource(dstFrame), {
+		return new VideoSample(new AvFrameVideoSampleResource(dstFrame), {
 			timestamp: sample.timestamp,
 			duration: sample.duration,
 			rotation: 0, // baked in by the filter graph