Add VideoSample.transform(), add custom transformer functions, ditch CanvasSink path in Conversion API, add libavfilter-based VideoSample transformation

packages/server/src/index.ts

@@ -1,9 +1,10 @@
-import { registerDecoder, registerEncoder } from 'mediabunny';
+import { registerDecoder, registerEncoder, registerVideoSampleTransformer } from 'mediabunny';
 import * as NodeAv from 'node-av';
 import { NodeAvVideoDecoder } from './video-decoder';
 import { NodeAvVideoEncoder } from './video-encoder';
 import { NodeAvAudioDecoder } from './audio-decoder';
 import { NodeAvAudioEncoder } from './audio-encoder';
+import { transformSample } from './video-sample';
 
 const SERVER_LOADED_SYMBOL = Symbol.for('@mediabunny/server loaded');
 if ((globalThis as Record<symbol, unknown>)[SERVER_LOADED_SYMBOL]) {
@@ -32,4 +33,6 @@ export const registerMediabunnyServer = () => {
 	// Audio
 	registerDecoder(NodeAvAudioDecoder);
 	registerEncoder(NodeAvAudioEncoder);
+
+	registerVideoSampleTransformer(transformSample);
 };

packages/server/src/video-encoder.ts

@@ -2,16 +2,12 @@ import { CustomVideoEncoder, MaybePromise, QUALITY_MEDIUM, VideoCodec, VideoSamp
 import * as NodeAv from 'node-av';
 import {
 	CODEC_TO_CODEC_ID,
-	fromPixelFormat,
 	getHardwareEncoderCodec,
-	mapColorPrimaries,
-	mapMatrixCoefficients,
-	mapTransferCharacteristics,
 	unmapColorPrimaries,
 	unmapMatrixCoefficients,
 	unmapTransferCharacteristics,
 } from './misc';
-import { NodeAvFrameVideoSampleResource } from './video-sample';
+import { copyVideoSampleToAvFrame, NodeAvFrameVideoSampleResource } from './video-sample';
 import {
 	AvcNalUnitType,
 	extractAv1CodecInfoFromPacket,
@@ -172,33 +168,8 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
 				throw new Error('Cannot encode foreign VideoSample with unknown (null) format.');
 			}
 
-			// Copy frame data from VideoSample to FFmpeg Frame
-			this.frame.format = fromPixelFormat(videoSample.format);
-			this.frame.width = videoSample.codedWidth;
-			this.frame.height = videoSample.codedHeight;
-			this.frame.pts = BigInt(videoSample.microsecondTimestamp);
-			this.frame.duration = BigInt(videoSample.microsecondDuration);
-			this.frame.colorPrimaries = mapColorPrimaries(videoSample.colorSpace.primaries ?? 'unknown')
-				?? NodeAv.AVCOL_PRI_UNSPECIFIED;
-			this.frame.colorSpace = mapMatrixCoefficients(videoSample.colorSpace.matrix ?? 'unknown')
-				?? NodeAv.AVCOL_SPC_UNSPECIFIED;
-			this.frame.colorTrc = mapTransferCharacteristics(videoSample.colorSpace.transfer ?? 'unknown')
-				?? NodeAv.AVCOL_TRC_UNSPECIFIED;
-			this.frame.colorRange = videoSample.colorSpace.fullRange === false
-				? NodeAv.AVCOL_RANGE_MPEG
-				: videoSample.colorSpace.fullRange === true
-					? NodeAv.AVCOL_RANGE_JPEG
-					: NodeAv.AVCOL_RANGE_UNSPECIFIED;
-
+			this.lastBuffer = await copyVideoSampleToAvFrame(videoSample, this.frame, this.lastBuffer);
 			this.frame.keyFrame = options?.keyFrame ? 1 : 0;
-
-			const size = videoSample.allocationSize();
-			if (!this.lastBuffer || this.lastBuffer.byteLength !== size) {
-				this.lastBuffer = Buffer.from({ length: size });
-			}
-
-			await videoSample.copyTo(this.lastBuffer);
-			this.frame.fromBuffer(this.lastBuffer);
 		}
 
 		let frameToEncode = this.frame;

packages/server/src/video-sample.ts

@@ -4,13 +4,16 @@ import { VideoSampleColorSpace } from 'mediabunny';
 import { VideoSampleResource } from 'mediabunny';
 import * as NodeAv from 'node-av';
 import { MaybePromise, toUint8Array } from '../../../src/misc';
-import { VideoDataPlane, VideoSample } from '../../../src/sample';
+import { VideoSampleTransformationDescription, VideoDataPlane, VideoSample } from '../../../src/sample';
 import {
 	toPixelFormat,
 	unmapColorPrimaries,
 	unmapTransferCharacteristics,
 	unmapMatrixCoefficients,
 	fromPixelFormat,
+	mapColorPrimaries,
+	mapMatrixCoefficients,
+	mapTransferCharacteristics,
 } from './misc';
 import { SetRequired } from 'mediabunny';
 import { VideoSampleInit } from 'mediabunny';
@@ -89,7 +92,6 @@ export class NodeAvFrameVideoSampleResource extends VideoSampleResource {
 
 	async toRgbSample(
 		init: SetRequired<VideoSampleInit, 'timestamp'>,
-		format: 'RGBA' | 'RGBX' | 'BGRA' | 'BGRX',
 		// Will respect it when somebody complains
 		// eslint-disable-next-line @typescript-eslint/no-unused-vars
 		colorSpace: PredefinedColorSpace,
@@ -99,7 +101,7 @@ export class NodeAvFrameVideoSampleResource extends VideoSampleResource {
 
 		const scaler = new NodeAv.SoftwareScaleContext();
 		const srcFmt = this.frame.format as NodeAv.AVPixelFormat;
-		const dstFmt = fromPixelFormat(format);
+		const dstFmt = fromPixelFormat('RGBA');
 
 		scaler.getContext(
 			width, height, srcFmt,
@@ -122,9 +124,146 @@ export class NodeAvFrameVideoSampleResource extends VideoSampleResource {
 			scaler.freeContext();
 		}
 
-		dstFrame.format = dstFmt; // FFmpeg messes up RGBA and RGBX
 		dstFrame.sampleAspectRatio = srcFrame.sampleAspectRatio;
 
 		return new VideoSample(new NodeAvFrameVideoSampleResource(dstFrame), init);
 	}
 }
+
+export const copyVideoSampleToAvFrame = async (sample: VideoSample, frame: NodeAv.Frame, lastBuffer: Buffer | null) => {
+	assert(sample.format !== null);
+
+	frame.format = fromPixelFormat(sample.format);
+	frame.width = sample.codedWidth;
+	frame.height = sample.codedHeight;
+	frame.sampleAspectRatio = new NodeAv.Rational(
+		sample.pixelAspectRatio.num,
+		sample.pixelAspectRatio.den,
+	);
+	frame.pts = BigInt(sample.microsecondTimestamp);
+	frame.duration = BigInt(sample.microsecondDuration);
+	frame.colorPrimaries = mapColorPrimaries(sample.colorSpace.primaries ?? 'unknown')
+		?? NodeAv.AVCOL_PRI_UNSPECIFIED;
+	frame.colorSpace = mapMatrixCoefficients(sample.colorSpace.matrix ?? 'unknown')
+		?? NodeAv.AVCOL_SPC_UNSPECIFIED;
+	frame.colorTrc = mapTransferCharacteristics(sample.colorSpace.transfer ?? 'unknown')
+		?? NodeAv.AVCOL_TRC_UNSPECIFIED;
+	frame.colorRange = sample.colorSpace.fullRange === false
+		? NodeAv.AVCOL_RANGE_MPEG
+		: sample.colorSpace.fullRange === true
+			? NodeAv.AVCOL_RANGE_JPEG
+			: NodeAv.AVCOL_RANGE_UNSPECIFIED;
+
+	const size = sample.allocationSize();
+	if (!lastBuffer || lastBuffer.byteLength !== size) {
+		lastBuffer = Buffer.from({ length: size });
+	}
+
+	await sample.copyTo(lastBuffer);
+	frame.fromBuffer(lastBuffer);
+
+	return lastBuffer;
+};
+
+export const transformSample = async (
+	sample: VideoSample,
+	description: VideoSampleTransformationDescription,
+): Promise<VideoSample | null> => {
+	let srcFrame: NodeAv.Frame;
+	let srcFrameOwned = false;
+
+	if (sample._data instanceof NodeAvFrameVideoSampleResource) {
+		srcFrame = sample._data.frame;
+	} else {
+		if (sample.format === null) {
+			return null;
+		}
+
+		srcFrame = new NodeAv.Frame();
+		srcFrame.alloc();
+		srcFrameOwned = true;
+
+		await copyVideoSampleToAvFrame(sample, srcFrame, null);
+	}
+
+	// Build the filter chain. Order: square-pixel normalize -> rotate -> crop -> resize-with-fit.
+	const chain: string[] = [];
+
+	if (sample.squarePixelWidth !== sample.codedWidth || sample.squarePixelHeight !== sample.codedHeight) {
+		chain.push(`scale=${sample.squarePixelWidth}:${sample.squarePixelHeight}`);
+		chain.push('setsar=1');
+	}
+
+	if (description.rotation === 90) {
+		chain.push('transpose=1');
+	} else if (description.rotation === 180) {
+		chain.push('transpose=1,transpose=1');
+	} else if (description.rotation === 270) {
+		chain.push('transpose=2');
+	}
+
+	chain.push(`crop=${Math.round(description.crop.width)}:${Math.round(description.crop.height)}`
+		+ `:${Math.round(description.crop.left)}:${Math.round(description.crop.top)}`);
+
+	if (description.fit === 'fill') {
+		chain.push(`scale=${description.width}:${description.height}`);
+	} else if (description.fit === 'contain') {
+		chain.push(`scale=${description.width}:${description.height}:force_original_aspect_ratio=decrease`);
+		chain.push(`pad=${description.width}:${description.height}:(ow-iw)/2:(oh-ih)/2:color=black@0`);
+	} else if (description.fit === 'cover') {
+		chain.push(`scale=${description.width}:${description.height}:force_original_aspect_ratio=increase`);
+		chain.push(`crop=${description.width}:${description.height}`);
+	}
+
+	chain.push('setsar=1');
+
+	const graph = new NodeAv.FilterGraph();
+	graph.alloc();
+
+	try {
+		const srcArgs = `video_size=${srcFrame.width}x${srcFrame.height}`
+			+ `:pix_fmt=${srcFrame.format}`
+			+ `:time_base=1/1000000`
+			+ `:pixel_aspect=${sample.pixelAspectRatio.num}/${sample.pixelAspectRatio.den}`;
+
+		const bufferSrc = graph.createFilter(NodeAv.Filter.getByName('buffer')!, 'src', srcArgs);
+		const bufferSink = graph.createFilter(NodeAv.Filter.getByName('buffersink')!, 'sink');
+		assert(bufferSrc && bufferSink);
+
+		// The naming here looks inverted but matches FFmpeg's parse semantics: from the parsed chain's
+		// perspective, its inputs are fed by the graph's existing outputs (the buffer src), and its outputs
+		// feed the graph's existing inputs (the buffer sink).
+		const outputs = NodeAv.FilterInOut.createList([{ name: 'in', filterCtx: bufferSrc, padIdx: 0 }]);
+		const inputs = NodeAv.FilterInOut.createList([{ name: 'out', filterCtx: bufferSink, padIdx: 0 }]);
+
+		const parseRet = graph.parsePtr(`[in]${chain.join(',')}[out]`, inputs, outputs);
+		NodeAv.FFmpegError.throwIfError(parseRet, 'FilterGraph.parsePtr');
+
+		const configRet = await graph.config();
+		NodeAv.FFmpegError.throwIfError(configRet, 'FilterGraph.config');
+
+		const addRet = await bufferSrc.buffersrcAddFrame(srcFrame);
+		NodeAv.FFmpegError.throwIfError(addRet, 'buffersrcAddFrame');
+
+		// Flush - we only ever push a single frame through this graph.
+		await bufferSrc.buffersrcAddFrame(null);
+
+		const dstFrame = new NodeAv.Frame();
+		dstFrame.alloc();
+
+		const getRet = await bufferSink.buffersinkGetFrame(dstFrame);
+		NodeAv.FFmpegError.throwIfError(getRet, 'buffersinkGetFrame');
+
+		return new VideoSample(new NodeAvFrameVideoSampleResource(dstFrame), {
+			timestamp: sample.timestamp,
+			duration: sample.duration,
+			rotation: 0, // baked in by the filter graph
+		});
+	} finally {
+		graph.free();
+
+		if (srcFrameOwned) {
+			srcFrame.free();
+		}
+	}
+};