Add VideoSample.transform(), add custom transformer functions, ditch CanvasSink path in Conversion API, add libavfilter-based VideoSample transformation

packages/server/src/index.ts

@@ -1,9 +1,10 @@
-import { registerDecoder, registerEncoder } from 'mediabunny';
+import { registerDecoder, registerEncoder, registerVideoSampleTransformer } from 'mediabunny';
 import * as NodeAv from 'node-av';
 import { NodeAvVideoDecoder } from './video-decoder';
 import { NodeAvVideoEncoder } from './video-encoder';
 import { NodeAvAudioDecoder } from './audio-decoder';
 import { NodeAvAudioEncoder } from './audio-encoder';
+import { transformSample } from './video-sample';
 
 const SERVER_LOADED_SYMBOL = Symbol.for('@mediabunny/server loaded');
 if ((globalThis as Record<symbol, unknown>)[SERVER_LOADED_SYMBOL]) {
@@ -32,4 +33,6 @@ export const registerMediabunnyServer = () => {
 	// Audio
 	registerDecoder(NodeAvAudioDecoder);
 	registerEncoder(NodeAvAudioEncoder);
+
+	registerVideoSampleTransformer(transformSample);
 };

packages/server/src/video-encoder.ts

@@ -2,16 +2,12 @@ import { CustomVideoEncoder, MaybePromise, QUALITY_MEDIUM, VideoCodec, VideoSamp
 import * as NodeAv from 'node-av';
 import {
 	CODEC_TO_CODEC_ID,
-	fromPixelFormat,
 	getHardwareEncoderCodec,
-	mapColorPrimaries,
-	mapMatrixCoefficients,
-	mapTransferCharacteristics,
 	unmapColorPrimaries,
 	unmapMatrixCoefficients,
 	unmapTransferCharacteristics,
 } from './misc';
-import { NodeAvFrameVideoSampleResource } from './video-sample';
+import { copyVideoSampleToAvFrame, NodeAvFrameVideoSampleResource } from './video-sample';
 import {
 	AvcNalUnitType,
 	extractAv1CodecInfoFromPacket,
@@ -172,33 +168,8 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
 				throw new Error('Cannot encode foreign VideoSample with unknown (null) format.');
 			}
 
-			// Copy frame data from VideoSample to FFmpeg Frame
-			this.frame.format = fromPixelFormat(videoSample.format);
-			this.frame.width = videoSample.codedWidth;
-			this.frame.height = videoSample.codedHeight;
-			this.frame.pts = BigInt(videoSample.microsecondTimestamp);
-			this.frame.duration = BigInt(videoSample.microsecondDuration);
-			this.frame.colorPrimaries = mapColorPrimaries(videoSample.colorSpace.primaries ?? 'unknown')
-				?? NodeAv.AVCOL_PRI_UNSPECIFIED;
-			this.frame.colorSpace = mapMatrixCoefficients(videoSample.colorSpace.matrix ?? 'unknown')
-				?? NodeAv.AVCOL_SPC_UNSPECIFIED;
-			this.frame.colorTrc = mapTransferCharacteristics(videoSample.colorSpace.transfer ?? 'unknown')
-				?? NodeAv.AVCOL_TRC_UNSPECIFIED;
-			this.frame.colorRange = videoSample.colorSpace.fullRange === false
-				? NodeAv.AVCOL_RANGE_MPEG
-				: videoSample.colorSpace.fullRange === true
-					? NodeAv.AVCOL_RANGE_JPEG
-					: NodeAv.AVCOL_RANGE_UNSPECIFIED;
-
+			this.lastBuffer = await copyVideoSampleToAvFrame(videoSample, this.frame, this.lastBuffer);
 			this.frame.keyFrame = options?.keyFrame ? 1 : 0;
-
-			const size = videoSample.allocationSize();
-			if (!this.lastBuffer || this.lastBuffer.byteLength !== size) {
-				this.lastBuffer = Buffer.from({ length: size });
-			}
-
-			await videoSample.copyTo(this.lastBuffer);
-			this.frame.fromBuffer(this.lastBuffer);
 		}
 
 		let frameToEncode = this.frame;

packages/server/src/video-sample.ts

@@ -4,13 +4,16 @@ import { VideoSampleColorSpace } from 'mediabunny';
 import { VideoSampleResource } from 'mediabunny';
 import * as NodeAv from 'node-av';
 import { MaybePromise, toUint8Array } from '../../../src/misc';
-import { VideoDataPlane, VideoSample } from '../../../src/sample';
+import { VideoSampleTransformationDescription, VideoDataPlane, VideoSample } from '../../../src/sample';
 import {
 	toPixelFormat,
 	unmapColorPrimaries,
 	unmapTransferCharacteristics,
 	unmapMatrixCoefficients,
 	fromPixelFormat,
+	mapColorPrimaries,
+	mapMatrixCoefficients,
+	mapTransferCharacteristics,
 } from './misc';
 import { SetRequired } from 'mediabunny';
 import { VideoSampleInit } from 'mediabunny';
@@ -89,7 +92,6 @@ export class NodeAvFrameVideoSampleResource extends VideoSampleResource {
 
 	async toRgbSample(
 		init: SetRequired<VideoSampleInit, 'timestamp'>,
-		format: 'RGBA' | 'RGBX' | 'BGRA' | 'BGRX',
 		// Will respect it when somebody complains
 		// eslint-disable-next-line @typescript-eslint/no-unused-vars
 		colorSpace: PredefinedColorSpace,
@@ -99,7 +101,7 @@ export class NodeAvFrameVideoSampleResource extends VideoSampleResource {
 
 		const scaler = new NodeAv.SoftwareScaleContext();
 		const srcFmt = this.frame.format as NodeAv.AVPixelFormat;
-		const dstFmt = fromPixelFormat(format);
+		const dstFmt = fromPixelFormat('RGBA');
 
 		scaler.getContext(
 			width, height, srcFmt,
@@ -122,9 +124,146 @@ export class NodeAvFrameVideoSampleResource extends VideoSampleResource {
 			scaler.freeContext();
 		}
 
-		dstFrame.format = dstFmt; // FFmpeg messes up RGBA and RGBX
 		dstFrame.sampleAspectRatio = srcFrame.sampleAspectRatio;
 
 		return new VideoSample(new NodeAvFrameVideoSampleResource(dstFrame), init);
 	}
 }
+
+export const copyVideoSampleToAvFrame = async (sample: VideoSample, frame: NodeAv.Frame, lastBuffer: Buffer | null) => {
+	assert(sample.format !== null);
+
+	frame.format = fromPixelFormat(sample.format);
+	frame.width = sample.codedWidth;
+	frame.height = sample.codedHeight;
+	frame.sampleAspectRatio = new NodeAv.Rational(
+		sample.pixelAspectRatio.num,
+		sample.pixelAspectRatio.den,
+	);
+	frame.pts = BigInt(sample.microsecondTimestamp);
+	frame.duration = BigInt(sample.microsecondDuration);
+	frame.colorPrimaries = mapColorPrimaries(sample.colorSpace.primaries ?? 'unknown')
+		?? NodeAv.AVCOL_PRI_UNSPECIFIED;
+	frame.colorSpace = mapMatrixCoefficients(sample.colorSpace.matrix ?? 'unknown')
+		?? NodeAv.AVCOL_SPC_UNSPECIFIED;
+	frame.colorTrc = mapTransferCharacteristics(sample.colorSpace.transfer ?? 'unknown')
+		?? NodeAv.AVCOL_TRC_UNSPECIFIED;
+	frame.colorRange = sample.colorSpace.fullRange === false
+		? NodeAv.AVCOL_RANGE_MPEG
+		: sample.colorSpace.fullRange === true
+			? NodeAv.AVCOL_RANGE_JPEG
+			: NodeAv.AVCOL_RANGE_UNSPECIFIED;
+
+	const size = sample.allocationSize();
+	if (!lastBuffer || lastBuffer.byteLength !== size) {
+		lastBuffer = Buffer.from({ length: size });
+	}
+
+	await sample.copyTo(lastBuffer);
+	frame.fromBuffer(lastBuffer);
+
+	return lastBuffer;
+};
+
+export const transformSample = async (
+	sample: VideoSample,
+	description: VideoSampleTransformationDescription,
+): Promise<VideoSample | null> => {
+	let srcFrame: NodeAv.Frame;
+	let srcFrameOwned = false;
+
+	if (sample._data instanceof NodeAvFrameVideoSampleResource) {
+		srcFrame = sample._data.frame;
+	} else {
+		if (sample.format === null) {
+			return null;
+		}
+
+		srcFrame = new NodeAv.Frame();
+		srcFrame.alloc();
+		srcFrameOwned = true;
+
+		await copyVideoSampleToAvFrame(sample, srcFrame, null);
+	}
+
+	// Build the filter chain. Order: square-pixel normalize -> rotate -> crop -> resize-with-fit.
+	const chain: string[] = [];
+
+	if (sample.squarePixelWidth !== sample.codedWidth || sample.squarePixelHeight !== sample.codedHeight) {
+		chain.push(`scale=${sample.squarePixelWidth}:${sample.squarePixelHeight}`);
+		chain.push('setsar=1');
+	}
+
+	if (description.rotation === 90) {
+		chain.push('transpose=1');
+	} else if (description.rotation === 180) {
+		chain.push('transpose=1,transpose=1');
+	} else if (description.rotation === 270) {
+		chain.push('transpose=2');
+	}
+
+	chain.push(`crop=${Math.round(description.crop.width)}:${Math.round(description.crop.height)}`
+		+ `:${Math.round(description.crop.left)}:${Math.round(description.crop.top)}`);
+
+	if (description.fit === 'fill') {
+		chain.push(`scale=${description.width}:${description.height}`);
+	} else if (description.fit === 'contain') {
+		chain.push(`scale=${description.width}:${description.height}:force_original_aspect_ratio=decrease`);
+		chain.push(`pad=${description.width}:${description.height}:(ow-iw)/2:(oh-ih)/2:color=black@0`);
+	} else if (description.fit === 'cover') {
+		chain.push(`scale=${description.width}:${description.height}:force_original_aspect_ratio=increase`);
+		chain.push(`crop=${description.width}:${description.height}`);
+	}
+
+	chain.push('setsar=1');
+
+	const graph = new NodeAv.FilterGraph();
+	graph.alloc();
+
+	try {
+		const srcArgs = `video_size=${srcFrame.width}x${srcFrame.height}`
+			+ `:pix_fmt=${srcFrame.format}`
+			+ `:time_base=1/1000000`
+			+ `:pixel_aspect=${sample.pixelAspectRatio.num}/${sample.pixelAspectRatio.den}`;
+
+		const bufferSrc = graph.createFilter(NodeAv.Filter.getByName('buffer')!, 'src', srcArgs);
+		const bufferSink = graph.createFilter(NodeAv.Filter.getByName('buffersink')!, 'sink');
+		assert(bufferSrc && bufferSink);
+
+		// The naming here looks inverted but matches FFmpeg's parse semantics: from the parsed chain's
+		// perspective, its inputs are fed by the graph's existing outputs (the buffer src), and its outputs
+		// feed the graph's existing inputs (the buffer sink).
+		const outputs = NodeAv.FilterInOut.createList([{ name: 'in', filterCtx: bufferSrc, padIdx: 0 }]);
+		const inputs = NodeAv.FilterInOut.createList([{ name: 'out', filterCtx: bufferSink, padIdx: 0 }]);
+
+		const parseRet = graph.parsePtr(`[in]${chain.join(',')}[out]`, inputs, outputs);
+		NodeAv.FFmpegError.throwIfError(parseRet, 'FilterGraph.parsePtr');
+
+		const configRet = await graph.config();
+		NodeAv.FFmpegError.throwIfError(configRet, 'FilterGraph.config');
+
+		const addRet = await bufferSrc.buffersrcAddFrame(srcFrame);
+		NodeAv.FFmpegError.throwIfError(addRet, 'buffersrcAddFrame');
+
+		// Flush - we only ever push a single frame through this graph.
+		await bufferSrc.buffersrcAddFrame(null);
+
+		const dstFrame = new NodeAv.Frame();
+		dstFrame.alloc();
+
+		const getRet = await bufferSink.buffersinkGetFrame(dstFrame);
+		NodeAv.FFmpegError.throwIfError(getRet, 'buffersinkGetFrame');
+
+		return new VideoSample(new NodeAvFrameVideoSampleResource(dstFrame), {
+			timestamp: sample.timestamp,
+			duration: sample.duration,
+			rotation: 0, // baked in by the filter graph
+		});
+	} finally {
+		graph.free();
+
+		if (srcFrameOwned) {
+			srcFrame.free();
+		}
+	}
+};

src/conversion.ts

@@ -24,7 +24,6 @@ import { Input } from './input';
 import { InputAudioTrack, InputTrack, InputVideoTrack } from './input-track';
 import {
 	AudioSampleSink,
-	CanvasSink,
 	EncodedPacketSink,
 	VideoSampleSink,
 } from './media-sink';
@@ -41,7 +40,6 @@ import {
 	assertNever,
 	ceilToMultipleOfTwo,
 	clamp,
-	floorToDivisor,
 	isIso639Dash2LanguageCode,
 	MaybePromise,
 	normalizeRotation,
@@ -1142,7 +1140,8 @@ export class Conversion {
 
 		let videoSource: VideoSource;
 
-		const totalRotation = normalizeRotation(await track.getRotation() + (trackOptions.rotate ?? 0));
+		const innateRotation = await track.getRotation();
+		const totalRotation = normalizeRotation(innateRotation + (trackOptions.rotate ?? 0));
 		let outputTrackRotation = totalRotation;
 		const canUseRotationMetadata = this.output.format.supportsVideoRotationMetadata
 			&& (trackOptions.allowRotationMetadata ?? true);
@@ -1284,10 +1283,9 @@ export class Conversion {
 				sizeChangeBehavior: trackOptions.fit ?? 'passThrough',
 				alpha,
 				hardwareAcceleration: trackOptions.hardwareAcceleration,
+				transform: {},
 			};
-
-			const source = new VideoSampleSource(encodingConfig);
-			videoSource = source;
+			assert(encodingConfig.transform);
 
 			let needsRerender = width !== originalWidth
 				|| height !== originalHeight
@@ -1334,180 +1332,47 @@ export class Conversion {
 				}
 			}
 
+			if (trackOptions.frameRate) {
+				encodingConfig.transform.frameRate = trackOptions.frameRate;
+			}
+
 			if (needsRerender) {
 				outputTrackRotation = 0; // Since the rotation is baked into the output
 
-				this._trackPromises.push((async () => {
-					await this._started;
-
-					const sink = new CanvasSink(track, {
-						width,
-						height,
-						fit: trackOptions.fit ?? 'fill',
-						rotation: totalRotation, // Bake the rotation into the output
-						crop: trackOptions.crop,
-						poolSize: 1,
-						alpha: alpha === 'keep',
-					});
-					const iterator = sink.canvases(this._startTimestamp, this._endTimestamp);
-					const frameRate = trackOptions.frameRate;
-
-					let lastCanvas: HTMLCanvasElement | OffscreenCanvas | null = null;
-					let lastCanvasTimestamp: number | null = null;
-					let lastCanvasEndTimestamp: number | null = null;
-
-					/** Repeats the last sample to pad out the time until the specified timestamp. */
-					const padFrames = async (until: number) => {
-						assert(lastCanvas);
-						assert(frameRate !== undefined);
-
-						const frameDifference = Math.round((until - lastCanvasTimestamp!) * frameRate);
-
-						for (let i = 1; i < frameDifference; i++) {
-							const sample = new VideoSample(lastCanvas, {
-								timestamp: lastCanvasTimestamp! + i / frameRate,
-								duration: 1 / frameRate,
-							});
-							await this._registerVideoSample(trackOptions, outputTrackId, source, sample);
-							sample.close();
-						}
-					};
-
-					for await (const { canvas, timestamp, duration } of iterator) {
-						if (this._canceled) {
-							return;
+				encodingConfig.transform.width = width;
+				encodingConfig.transform.height = height;
+				encodingConfig.transform.fit = trackOptions.fit ?? 'fill';
+				encodingConfig.transform.rotate = normalizeRotation(totalRotation - innateRotation);
+				encodingConfig.transform.crop = crop;
+				encodingConfig.transform.alpha = alpha;
 			}
 
-						let adjustedSampleTimestamp = Math.max(timestamp - this._startTimestamp, 0);
-						lastCanvasEndTimestamp = adjustedSampleTimestamp + duration;
+			const source = new VideoSampleSource(encodingConfig);
+			videoSource = source;
 
-						if (frameRate !== undefined) {
-							// Logic for skipping/repeating frames when a frame rate is set
-							const alignedTimestamp = floorToDivisor(adjustedSampleTimestamp, frameRate);
-
-							if (lastCanvas !== null) {
-								if (alignedTimestamp <= lastCanvasTimestamp!) {
-									lastCanvas = canvas;
-									lastCanvasTimestamp = alignedTimestamp;
-
-									// Skip this sample, since we already added one for this frame
-									continue;
-								} else {
-									// Check if we may need to repeat the previous frame
-									await padFrames(alignedTimestamp);
-								}
-							}
-
-							adjustedSampleTimestamp = alignedTimestamp;
-						}
-
-						const sample = new VideoSample(canvas, {
-							timestamp: adjustedSampleTimestamp,
-							duration: frameRate !== undefined ? 1 / frameRate : duration,
-						});
-						await this._registerVideoSample(trackOptions, outputTrackId, source, sample);
-						sample.close();
-
-						if (frameRate !== undefined) {
-							lastCanvas = canvas;
-							lastCanvasTimestamp = adjustedSampleTimestamp;
-						}
-					}
-
-					if (lastCanvas) {
-						assert(lastCanvasEndTimestamp !== null);
-						assert(frameRate !== undefined);
-
-						// If necessary, pad until the end timestamp of the last sample
-						await padFrames(floorToDivisor(lastCanvasEndTimestamp, frameRate));
-					}
-
-					source.close();
-					this._synchronizer.closeTrack(outputTrackId);
-				})());
-			} else {
 			this._trackPromises.push((async () => {
 				await this._started;
 
 				const sink = new VideoSampleSink(track);
-					const frameRate = trackOptions.frameRate;
-
-					let lastSample: VideoSample | null = null;
-					let lastSampleTimestamp: number | null = null;
-					let lastSampleEndTimestamp: number | null = null;
-
-					/** Repeats the last sample to pad out the time until the specified timestamp. */
-					const padFrames = async (until: number) => {
-						assert(lastSample);
-						assert(frameRate !== undefined);
-
-						const frameDifference = Math.round((until - lastSampleTimestamp!) * frameRate);
-
-						for (let i = 1; i < frameDifference; i++) {
-							lastSample.setTimestamp(lastSampleTimestamp! + i / frameRate);
-							lastSample.setDuration(1 / frameRate);
-							await this._registerVideoSample(trackOptions, outputTrackId, source, lastSample);
-						}
-
-						lastSample.close();
-					};
 
 				for await (const sample of sink.samples(this._startTimestamp, this._endTimestamp)) {
 					if (this._canceled) {
 						sample.close();
-							lastSample?.close();
 						return;
 					}
 
-						let adjustedSampleTimestamp = Math.max(sample.timestamp - this._startTimestamp, 0);
-						lastSampleEndTimestamp = adjustedSampleTimestamp + sample.duration;
-
-						if (frameRate !== undefined) {
-							// Logic for skipping/repeating frames when a frame rate is set
-							const alignedTimestamp = floorToDivisor(adjustedSampleTimestamp, frameRate);
-
-							if (lastSample !== null) {
-								if (alignedTimestamp <= lastSampleTimestamp!) {
-									lastSample.close();
-									lastSample = sample;
-									lastSampleTimestamp = alignedTimestamp;
-
-									// Skip this sample, since we already added one for this frame
-									continue;
-								} else {
-									// Check if we may need to repeat the previous frame
-									await padFrames(alignedTimestamp);
-								}
-							}
-
-							adjustedSampleTimestamp = alignedTimestamp;
-							sample.setDuration(1 / frameRate);
-						}
-
+					const adjustedSampleTimestamp = Math.max(sample.timestamp - this._startTimestamp, 0);
 					sample.setTimestamp(adjustedSampleTimestamp);
+
 					await this._registerVideoSample(trackOptions, outputTrackId, source, sample);
 
-						if (frameRate !== undefined) {
-							lastSample = sample;
-							lastSampleTimestamp = adjustedSampleTimestamp;
-						} else {
 					sample.close();
 				}
-					}
-
-					if (lastSample) {
-						assert(lastSampleEndTimestamp !== null);
-						assert(frameRate !== undefined);
-
-						// If necessary, pad until the end timestamp of the last sample
-						await padFrames(floorToDivisor(lastSampleEndTimestamp, frameRate));
-					}
 
 				source.close();
 				this._synchronizer.closeTrack(outputTrackId);
 			})());
 		}
-		}
 
 		let ownGroup: OutputTrackGroup | null = null;
 		if (!trackOptions.group) {

src/encode.ts

@@ -113,6 +113,10 @@ export type VideoTransformOptions = {
 	 * clamped to the dimensions of the frame. Cropping is performed after rotation but before resizing.
 	 */
 	crop?: CropRectangle;
+	/**
+	 * Whether to discard or keep the transparency information of the video samples. The default is `'keep'`.
+	 */
+	alpha?: 'keep' | 'discard';
 	/**
 	 * The frame rate in hertz to normalize the video frame stream to.
 	 */
@@ -193,10 +197,12 @@ export const validateVideoEncodingConfig = (config: VideoEncodingConfig) => {
 		if (
 			config.transform.fit !== undefined
 			&& ['fill', 'contain', 'cover'].includes(config.sizeChangeBehavior!)
+			&& config.transform.fit !== config.sizeChangeBehavior
 		) {
 			throw new TypeError(
-				'config.transform.fit cannot be used when config.sizeChangeBehavior is \'fill\', \'contain\' or'
-				+ ' \'cover\', as sizeChangeBehavior already determines the fitting algorithm.',
+				'config.transform.fit, when provided, cannot differ from config.sizeChangeBehavior when'
+				+ ' config.sizeChangeBehavior is \'fill\', \'contain\' or \'cover\', as sizeChangeBehavior already'
+				+ ' determines the fitting algorithm.',
 			);
 		}
 		if (config.transform.rotate !== undefined && ![0, 90, 180, 270].includes(config.transform.rotate)) {

src/index.ts

@@ -249,8 +249,12 @@ export {
 	type VideoSamplePixelFormat,
 	VideoSampleColorSpace,
 	VideoSampleResource,
+	type VideoSampleTransformOptions,
+	type VideoSampleTransformationDescription,
 	type CropRectangle,
 	VIDEO_SAMPLE_PIXEL_FORMATS,
+	type VideoDataPlane,
+	registerVideoSampleTransformer,
 } from './sample';
 export {
 	AudioBufferSink,

src/media-source.ts

@@ -25,13 +25,10 @@ import {
 	assertNever,
 	binarySearchLessOrEqual,
 	CallSerializer,
-	ceilToMultipleOfTwo,
 	clamp,
 	clearIntervalUnthrottled,
 	floorToDivisor,
-	isFirefox,
 	last,
-	normalizeRotation,
 	promiseWithResolvers,
 	roundToDivisor,
 	setInt24,
@@ -53,7 +50,6 @@ import { EncodedPacket, EncodedPacketSideData } from './packet';
 import {
 	AudioSample,
 	audioSampleToInterleavedFormat,
-	clampCropRectangle,
 	toInterleavedAudioFormat,
 	VideoSample,
 } from './sample';
@@ -233,7 +229,6 @@ class VideoEncoderWrapper {
 	private muxer: Muxer | null = null;
 	private lastMultipleOfKeyFrameInterval = -1;
 
-	private resizeCanvas: HTMLCanvasElement | OffscreenCanvas | null = null;
 	// Tracks the input dimensions of the first frame
 	private codedWidth: number | null = null;
 	private codedHeight: number | null = null;
@@ -275,15 +270,7 @@ class VideoEncoderWrapper {
 
 	private lastMuxerPromise: Promise<void> = Promise.resolve();
 
-	constructor(private source: VideoSource, private encodingConfig: VideoEncodingConfig) {
-		const sizeChangeBehavior = encodingConfig.sizeChangeBehavior ?? 'deny';
-		if (['fill', 'contain', 'cover'].includes(sizeChangeBehavior) && encodingConfig.transform?.fit !== undefined) {
-			throw new TypeError(
-				`Cannot set 'fit' when 'sizeChangeBehavior' is '${sizeChangeBehavior}'. `
-				+ `The size change behavior determines the fit in this case.`,
-			);
-		}
-	}
+	constructor(private source: VideoSource, private encodingConfig: VideoEncodingConfig) {}
 
 	async add(videoSample: VideoSample, shouldClose: boolean, encodeOptions?: VideoEncoderEncodeOptions) {
 		const originalSample = videoSample;
@@ -323,23 +310,8 @@ class VideoEncoderWrapper {
 			const needsTransform = hasTransformConfig || (isSizeChange && sizeChangeBehavior !== 'passThrough');
 
 			if (needsTransform) {
-				const rotation = normalizeRotation(videoSample.rotation + (config.transform?.rotate ?? 0));
-				const [rotatedWidth, rotatedHeight] = rotation % 180 === 0
-					? [videoSample.codedWidth, videoSample.codedHeight]
-					: [videoSample.codedHeight, videoSample.codedWidth];
-
-				// Clamp crop rectangle to the rotated video dimensions
-				let finalCrop = config.transform?.crop;
-				if (finalCrop) {
-					finalCrop = clampCropRectangle(finalCrop, rotatedWidth, rotatedHeight);
-				}
-
-				const cropWidth = finalCrop ? finalCrop.width : rotatedWidth;
-				const cropHeight = finalCrop ? finalCrop.height : rotatedHeight;
-				const originalAspectRatio = cropWidth / cropHeight;
-
-				let targetWidth: number;
-				let targetHeight: number;
+				let targetWidth = config.transform?.width;
+				let targetHeight = config.transform?.height;
 				let appliedFit: 'fill' | 'contain' | 'cover' = config.transform?.fit ?? 'fill';
 
 				// If the size changed and behavior is fill/contain/cover, lock to the original output dimensions
@@ -351,81 +323,29 @@ class VideoEncoderWrapper {
 					targetWidth = this.outputWidth!;
 					targetHeight = this.outputHeight!;
 					appliedFit = sizeChangeBehavior;
-				} else {
-					// Otherwise, dynamically calculate the target dimensions based on config and aspect ratio
-					if (config.transform?.width !== undefined && config.transform?.height === undefined) {
-						targetWidth = config.transform.width;
-						targetHeight = ceilToMultipleOfTwo(Math.round(targetWidth / originalAspectRatio));
-					} else if (config.transform?.width === undefined && config.transform?.height !== undefined) {
-						targetHeight = config.transform.height;
-						targetWidth = ceilToMultipleOfTwo(Math.round(targetHeight * originalAspectRatio));
-					} else if (config.transform?.width !== undefined && config.transform?.height !== undefined) {
-						targetWidth = config.transform?.width;
-						targetHeight = config.transform?.height;
-					} else {
-						targetWidth = cropWidth;
-						targetHeight = cropHeight;
-					}
 				}
 
+				const transformed = await videoSample.transform({
+					width: targetWidth,
+					height: targetHeight,
+					roundDimensionsTo: 2,
+					crop: config.transform?.crop,
+					rotate: config.transform?.rotate,
+					fit: appliedFit,
+					alpha: config.alpha,
+				});
+
 				// Save the output dimensions of the first frame
 				if (this.outputWidth === null || this.outputHeight === null) {
-					this.outputWidth = targetWidth;
-					this.outputHeight = targetHeight;
+					this.outputWidth = transformed.displayWidth;
+					this.outputHeight = transformed.displayHeight;
 				}
 
-				let canvasIsNew = false;
-
-				if (!this.resizeCanvas) {
-					if (typeof document !== 'undefined') {
-						// Prefer an HTMLCanvasElement
-						this.resizeCanvas = document.createElement('canvas');
-						this.resizeCanvas.width = targetWidth;
-						this.resizeCanvas.height = targetHeight;
-					} else {
-						this.resizeCanvas = new OffscreenCanvas(targetWidth, targetHeight);
-					}
-					canvasIsNew = true;
-				} else if (this.resizeCanvas.width !== targetWidth || this.resizeCanvas.height !== targetHeight) {
-					// Dynamically resize the canvas if the target dimensions have changed
-					this.resizeCanvas.width = targetWidth;
-					this.resizeCanvas.height = targetHeight;
-				}
-
-				const context = this.resizeCanvas.getContext('2d', {
-					// Firefox has VideoFrame glitches with opaque canvases
-					alpha: this.encodingConfig.alpha === 'keep' || isFirefox(),
-				}) as CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;
-				assert(context);
-
-				if (typeof context.resetTransform === 'function') {
-					context.resetTransform();
-				}
-
-				if (!canvasIsNew) {
-					if (isFirefox()) {
-						context.fillStyle = 'black';
-						context.fillRect(0, 0, targetWidth, targetHeight);
-					} else {
-						context.clearRect(0, 0, targetWidth, targetHeight);
-					}
-				}
-
-				videoSample.drawWithFit(context, {
-					fit: appliedFit,
-					rotation: rotation,
-					crop: finalCrop,
-				});
-
 				if (shouldClose) {
 					videoSample.close();
 				}
 
-				videoSample = new VideoSample(this.resizeCanvas, {
-					timestamp: videoSample.timestamp,
-					duration: videoSample.duration,
-					rotation: 0, // Rotation is now baked into the canvas
-				});
+				videoSample = transformed;
 				shouldClose = true;
 			} else {
 				// If no canvas is needed, we still need to record the output dimensions for the first frame

src/sample.ts

@@ -26,6 +26,9 @@ import {
 	simplifyRational,
 	Rectangle,
 	validateRectangle,
+	normalizeRotation,
+	roundToMultiple,
+	arrayArgmin,
 	MaybePromise,
 } from './misc';
 
@@ -103,8 +106,10 @@ export abstract class VideoSampleResource {
 	/** Returns the height of the frame in pixels. */
 	abstract getCodedHeight(): number;
 
+	/** Returns the width of the frame in square pixels, respecting pixel aspect ratio. */
 	abstract getSquarePixelWidth(): number;
 
+	/** Returns the height of the frame in square pixels, respecting pixel aspect ratio. */
 	abstract getSquarePixelHeight(): number;
 
 	/** Returns the color space of the frame. */
@@ -116,17 +121,32 @@ export abstract class VideoSampleResource {
 	 */
 	abstract close(): void;
 
+	/**
+	 * Returns the data planes that hold the video data for this sample. The returned planes and data must be in the
+	 * format returned by `getFormat()`.
+	 */
 	abstract getDataPlanes(): MaybePromise<VideoDataPlane[]>;
 
+	/**
+	 * Returns a new RGB {@link VideoSample} that contains the same content as this sample. The provided `init` object
+	 * must be used to set the metadata of this new video sample. When converting from a non-RGB format to RGB, the
+	 * conversion must respect `colorSpace`.
+	 */
 	abstract toRgbSample(
 		init: SetRequired<VideoSampleInit, 'timestamp'>,
-		format: 'RGBA' | 'RGBX' | 'BGRA' | 'BGRX',
 		colorSpace: PredefinedColorSpace,
 	): MaybePromise<VideoSample>;
 }
 
+/**
+ * Describes a single data plane of a video frame.
+ * @group Samples
+ * @public
+ */
 export type VideoDataPlane = {
+	/** The data of the plane. */
 	data: Uint8Array;
+	/** The stride of the plane, in bytes. This is the distance in bytes between the start of each row of pixels. */
 	stride: number;
 };
 
@@ -239,9 +259,9 @@ export class VideoSample implements Disposable {
 	readonly format!: VideoSamplePixelFormat | null;
 	/** The visible region of the frame in the coded pixel grid. */
 	readonly visibleRect!: Rectangle;
-	/** The width of the frame in square pixels, before rotation is applied. */
+	/** The width of the frame in square pixels (respecting pixel aspect ratio), before rotation is applied. */
 	readonly squarePixelWidth!: number;
-	/** The height of the frame in square pixels, before rotation is applied. */
+	/** The height of the frame in square pixels (respecting pixel aspect ratio), before rotation is applied. */
 	readonly squarePixelHeight!: number;
 	/** The rotation of the frame in degrees, clockwise. */
 	readonly rotation!: Rotation;
@@ -754,7 +774,6 @@ export class VideoSample implements Disposable {
 						duration: this.duration,
 						rotation: this.rotation,
 					},
-					options.format as 'RGBA' | 'RGBX' | 'BGRA' | 'BGRX',
 					options.colorSpace ?? 'srgb',
 				);
 				if (!(rgbSample instanceof VideoSample)) {
@@ -1333,6 +1352,179 @@ export class VideoSample implements Disposable {
 		}
 	}
 
+	/**
+	 * Transform this video sample to a new video sample given the options. Can be used to resize, rotate, and crop
+	 * the sample.
+	 *
+	 * In non-browser environments, this method will not work by default. To make it work, register a custom
+	 * transformer function via {@link registerVideoSampleTransformer}.
+	 */
+	async transform(options: VideoSampleTransformOptions) {
+		if (!options || typeof options !== 'object') {
+			throw new TypeError('options must be an object.');
+		}
+		if (options.width !== undefined && (!Number.isInteger(options.width) || options.width <= 0)) {
+			throw new TypeError('options.width, when provided, must be a positive integer.');
+		}
+		if (options.height !== undefined && (!Number.isInteger(options.height) || options.height <= 0)) {
+			throw new TypeError('options.height, when provided, must be a positive integer.');
+		}
+		if (
+			options.roundDimensionsTo !== undefined
+			&& (!Number.isInteger(options.roundDimensionsTo) || options.roundDimensionsTo <= 0)
+		) {
+			throw new TypeError('options.roundDimensionsTo, when provided, must be a positive integer.');
+		}
+		if (options.fit !== undefined && !['fill', 'contain', 'cover'].includes(options.fit)) {
+			throw new TypeError('options.fit, when provided, must be one of "fill", "contain", or "cover".');
+		}
+		if (
+			options.width !== undefined
+			&& options.height !== undefined
+			&& options.fit === undefined
+		) {
+			throw new TypeError(
+				'When both options.width and options.height are provided, options.fit must also be provided.',
+			);
+		}
+		if (options.rotate !== undefined && ![0, 90, 180, 270].includes(options.rotate)) {
+			throw new TypeError('options.rotate, when provided, must be 0, 90, 180 or 270.');
+		}
+		if (options.crop !== undefined) {
+			validateCropRectangle(options.crop, 'options.');
+		}
+		if (options.alpha !== undefined && !['keep', 'discard'].includes(options.alpha)) {
+			throw new TypeError('options.alpha, when provided, must be \'keep\' or \'discard\'.');
+		}
+
+		const rotation = normalizeRotation(this.rotation + (options.rotate ?? 0));
+		const [rotatedWidth, rotatedHeight] = rotation % 180 === 0
+			? [this.squarePixelWidth, this.squarePixelHeight]
+			: [this.squarePixelHeight, this.squarePixelWidth];
+
+		// Clamp crop rectangle to the rotated video dimensions
+		let finalCrop = options.crop;
+		if (finalCrop) {
+			finalCrop = clampCropRectangle(finalCrop, rotatedWidth, rotatedHeight);
+		}
+
+		const cropWidth = finalCrop ? finalCrop.width : rotatedWidth;
+		const cropHeight = finalCrop ? finalCrop.height : rotatedHeight;
+		const originalAspectRatio = cropWidth / cropHeight;
+
+		let targetWidth: number;
+		let targetHeight: number;
+
+		if (options.width !== undefined && options.height === undefined) {
+			targetWidth = options.width;
+			targetHeight = targetWidth / originalAspectRatio;
+		} else if (options.width === undefined && options.height !== undefined) {
+			targetHeight = options.height;
+			targetWidth = targetHeight * originalAspectRatio;
+		} else if (options.width !== undefined && options.height !== undefined) {
+			targetWidth = options.width;
+			targetHeight = options.height;
+		} else {
+			targetWidth = cropWidth;
+			targetHeight = cropHeight;
+		}
+
+		targetWidth = roundToMultiple(targetWidth, options.roundDimensionsTo ?? 1);
+		targetHeight = roundToMultiple(targetHeight, options.roundDimensionsTo ?? 1);
+
+		const description: VideoSampleTransformationDescription = {
+			width: targetWidth,
+			height: targetHeight,
+			fit: options.fit ?? 'fill',
+			rotation,
+			crop: finalCrop ?? {
+				left: 0,
+				top: 0,
+				width: rotatedWidth,
+				height: rotatedHeight,
+			},
+			alpha: options.alpha ?? 'keep',
+		};
+
+		// Description's finalized; let's see if a registered transformer wants to handle it
+		for (const transformer of registeredVideoSampleTransformers) {
+			let result = transformer(this, description);
+			if (result instanceof Promise) result = await result;
+
+			if (result !== null) {
+				return result;
+			}
+		}
+
+		// We need to handle it ourselves, and we use canvases to do it
+
+		let canvas: HTMLCanvasElement | OffscreenCanvas | null = null;
+		let canvasIsNew = false;
+
+		for (const entry of transformationCanvasCache) {
+			if (entry.canvas.width === description.width && entry.canvas.height === description.height) {
+				canvas = entry.canvas;
+				entry.age = transformationCanvasCacheNextAge++;
+
+				break;
+			}
+		}
+
+		if (canvas === null) {
+			if (typeof OffscreenCanvas !== 'undefined') {
+				canvas = new OffscreenCanvas(description.width, description.height);
+			} else {
+				if (typeof window === 'undefined' || typeof document === 'undefined') {
+					throw new Error(
+						'Cannot transform VideoSamples in this environment. Either run in an environment with'
+						+ ' OffscreenCanvas or HTMLCanvasElement, or supply a custom VideoSample transformer using'
+						+ ' registerVideoSampleTransformer().',
+					);
+				}
+
+				canvas = document.createElement('canvas');
+				canvas.width = description.width;
+				canvas.height = description.height;
+			}
+
+			canvasIsNew = true;
+
+			if (transformationCanvasCache.length >= TRANSFORMATION_CANVAS_CACHE_MAX_SIZE) {
+				transformationCanvasCache.splice(arrayArgmin(transformationCanvasCache, x => x.age), 1);
+			}
+
+			transformationCanvasCache.push({
+				canvas,
+				age: transformationCanvasCacheNextAge++,
+			});
+		}
+
+		const context = canvas.getContext('2d', {
+			alpha: true,
+		}) as CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;
+		assert(context);
+
+		if (description.alpha === 'discard') {
+			context.fillStyle = 'black';
+			context.fillRect(0, 0, description.width, description.height);
+		} else if (!canvasIsNew) {
+			// Cached canvases carry stale pixels from a prior draw
+			context.clearRect(0, 0, description.width, description.height);
+		}
+
+		this.drawWithFit(context, {
+			fit: description.fit,
+			rotation: description.rotation,
+			crop: description.crop,
+		});
+
+		return new VideoSample(canvas, {
+			timestamp: this.timestamp,
+			duration: this.duration,
+			rotation: 0, // Any previous rotation is now baked in
+		});
+	}
+
 	/** Sets the rotation metadata of this video sample. */
 	setRotation(newRotation: Rotation) {
 		if (![0, 90, 180, 270].includes(newRotation)) {
@@ -1369,6 +1561,123 @@ export class VideoSample implements Disposable {
 	}
 }
 
+/**
+ * Options for transforming a {@link VideoSample}. The order of operations are:
+ *
+ * 1. Pixel aspect ratio normalization (always applied)
+ * 2. Rotation
+ * 3. Crop
+ * 4. Resize using fit
+ * @group Samples
+ * @public
+ */
+export type VideoSampleTransformOptions = {
+	/**
+	 * The width in pixels to resize the frames to. If height is not set, it will be deduced
+	 * automatically based on aspect ratio.
+	 */
+	width?: number;
+	/**
+	 * The height in pixels to resize the frames to. If width is not set, it will be deduced
+	 * automatically based on aspect ratio.
+	 */
+	height?: number;
+	/**
+	 * A positive integer. When provided, both the width and height will be rounded to the nearest multiple of
+	 * this number.
+	 */
+	roundDimensionsTo?: number;
+	/**
+	 * The fitting algorithm in case both width and height are set.
+	 *
+	 * - `'fill'` will stretch the image to fill the entire box, potentially altering aspect ratio.
+	 * - `'contain'` will contain the entire image within the box while preserving aspect ratio. This may lead to
+	 * letterboxing.
+	 * - `'cover'` will scale the image until the entire box is filled, while preserving aspect ratio.
+	 */
+	fit?: 'fill' | 'contain' | 'cover';
+	/**
+	 * The clockwise rotation by which to rotate the frames. Rotation is applied before resizing.
+	 */
+	rotate?: Rotation;
+	/**
+	 * Specifies the rectangular region of the frames to crop to. The crop region will automatically be
+	 * clamped to the dimensions of the frame. Cropping is performed after rotation but before resizing.
+	 */
+	crop?: CropRectangle;
+	/**
+	 * Whether to discard or keep the transparency information of the video sample. The default is `'keep'`.
+	 */
+	alpha?: 'keep' | 'discard';
+};
+
+/**
+ * A fully-resolved description of a video sample transformation, with all defaults and constraints baked in.
+ *
+ * The order of operations must be:
+ * 1. Pixel aspect ratio normalization (always applied)
+ * 2. Rotation
+ * 3. Crop
+ * 4. Resize using fit
+ * @group Samples
+ * @public
+ */
+export type VideoSampleTransformationDescription = {
+	/** The width in pixels to resize the frames to. */
+	width: number;
+	/** The height in pixels to resize the frames to. */
+	height: number;
+	/**
+	 * The fitting algorithm.
+	 *
+	 * - `'fill'` will stretch the image to fill the entire box, potentially altering aspect ratio.
+	 * - `'contain'` will contain the entire image within the box while preserving aspect ratio. This may lead to
+	 * letterboxing.
+	 * - `'cover'` will scale the image until the entire box is filled, while preserving aspect ratio.
+	 */
+	fit: 'fill' | 'contain' | 'cover';
+	/** The clockwise rotation by which to rotate the frames. Rotation is applied before resizing. */
+	rotation: Rotation;
+	/**
+	 * The rectangular region of the frames to crop to, clamped to the dimensions of the frame. Cropping is
+	 * performed after rotation but before resizing.
+	 */
+	crop: CropRectangle;
+	/** Whether to discard or keep the transparency information of the video sample. */
+	alpha: 'keep' | 'discard';
+};
+
+const registeredVideoSampleTransformers: ((
+	sample: VideoSample,
+	description: VideoSampleTransformationDescription,
+) => MaybePromise<VideoSample | null>)[] = [];
+
+/**
+ * Registers a callback to handle the transformation of {@link VideoSample} instances. The callback can either return
+ * the transformed sample, or `null` to indicate that it doesn't want to handle the given transformation task.
+ * @group Samples
+ * @public
+ */
+export const registerVideoSampleTransformer = (
+	transformer: (
+		sample: VideoSample,
+		description: VideoSampleTransformationDescription,
+	) => MaybePromise<VideoSample | null>,
+) => {
+	if (registeredVideoSampleTransformers.includes(transformer)) {
+		return; // Already in there
+	}
+
+	registeredVideoSampleTransformers.push(transformer);
+};
+
+const TRANSFORMATION_CANVAS_CACHE_MAX_SIZE = 3;
+const transformationCanvasCache: {
+	canvas: HTMLCanvasElement | OffscreenCanvas;
+	age: number;
+}[] = [];
+let transformationCanvasCacheNextAge = 0;
+
 /**
  * Describes the color space of a {@link VideoSample}. Corresponds to the WebCodecs API's VideoColorSpace.
  * @group Samples
@@ -1927,6 +2236,10 @@ export abstract class AudioSampleResource {
 	 */
 	abstract close(): void;
 
+	/**
+	 * Returns the audio sample data for the plane given by `planeIndex`. The audio data must be in the format returned
+	 * by `getFormat()`. For interleaved formats, there is only one plane.
+	 */
 	abstract getDataPlane(planeIndex: number): Uint8Array;
 }
 

test/browser/custom-sample-resources.test.ts

@@ -61,8 +61,6 @@ class ImageVideoSampleResource extends VideoSampleResource {
 	toRgbSample(
 		init: SetRequired<VideoSampleInit, 'timestamp'>,
 		// eslint-disable-next-line @typescript-eslint/no-unused-vars
-		format: 'RGBA' | 'RGBX' | 'BGRA' | 'BGRX',
-		// eslint-disable-next-line @typescript-eslint/no-unused-vars
 		colorSpace: PredefinedColorSpace,
 	): MaybePromise<VideoSample> {
 		return new VideoSample(this, init);

test/node/server-extension.test.ts

@@ -736,6 +736,181 @@ describe('Video', async () => {
 			{ offset: 0, stride: 400 * 4 },
 		]);
 	});
+
+	describe('VideoSample transformation', () => {
+		// 400x400 image: red everywhere, with a 200x200 blue square filling the bottom-left quadrant.
+		const TEST_IMAGE = (() => {
+			const buf = new Uint8Array(400 * 400 * 4);
+			for (let i = 0; i < 400 * 400; i++) {
+				buf[i * 4] = 255;
+				buf[i * 4 + 1] = 0;
+				buf[i * 4 + 2] = 0;
+				buf[i * 4 + 3] = 255;
+			}
+			for (let y = 200; y < 400; y++) {
+				for (let x = 0; x < 200; x++) {
+					const i = y * 400 + x;
+					buf[i * 4] = 0;
+					buf[i * 4 + 1] = 0;
+					buf[i * 4 + 2] = 255;
+					buf[i * 4 + 3] = 255;
+				}
+			}
+			return buf;
+		})();
+
+		const makeSample = () => new VideoSample(TEST_IMAGE, {
+			format: 'RGBA',
+			codedWidth: 400,
+			codedHeight: 400,
+			timestamp: 0,
+		});
+
+		const readRgba = async (sample: VideoSample) => {
+			const buffer = new Uint8Array(sample.codedWidth * sample.codedHeight * 4);
+			await sample.copyTo(buffer, { format: 'RGBA' });
+			return buffer;
+		};
+
+		const getColorAt = (rgba: Uint8Array, width: number, x: number, y: number) => {
+			const i = (y * width + x) * 4;
+			return [rgba[i]!, rgba[i + 1]!, rgba[i + 2]!];
+		};
+
+		const expectColor = (actual: number[], expected: number[], tolerance = 4) => {
+			for (let i = 0; i < 3; i++) {
+				expect(
+					Math.abs(actual[i]! - expected[i]!),
+					`channel ${i}: got ${actual[i]}, expected ${expected[i]} (+/-${tolerance})`,
+				).toBeLessThanOrEqual(tolerance);
+			}
+		};
+
+		test('resize to 200x200', async () => {
+			using sample = makeSample();
+			using result = await sample.transform({ width: 200, height: 200, fit: 'fill' });
+
+			expect(result.codedWidth).toBe(200);
+			expect(result.codedHeight).toBe(200);
+
+			const rgba = await readRgba(result);
+			expectColor(getColorAt(rgba, 200, 50, 50), [255, 0, 0]); // top-left
+			expectColor(getColorAt(rgba, 200, 150, 50), [255, 0, 0]); // top-right
+			expectColor(getColorAt(rgba, 200, 50, 150), [0, 0, 255]); // bottom-left (blue)
+			expectColor(getColorAt(rgba, 200, 150, 150), [255, 0, 0]); // bottom-right
+		});
+
+		test('rotate 90 deg clockwise', async () => {
+			using sample = makeSample();
+			using result = await sample.transform({ rotate: 90 });
+
+			expect(result.codedWidth).toBe(400);
+			expect(result.codedHeight).toBe(400);
+
+			const rgba = await readRgba(result);
+			expectColor(getColorAt(rgba, 400, 50, 50), [0, 0, 255]); // top-left (blue)
+			expectColor(getColorAt(rgba, 400, 350, 50), [255, 0, 0]);
+			expectColor(getColorAt(rgba, 400, 50, 350), [255, 0, 0]);
+			expectColor(getColorAt(rgba, 400, 350, 350), [255, 0, 0]);
+		});
+
+		test('crop top-left, no rotation', async () => {
+			using sample = makeSample();
+			using result = await sample.transform({ crop: { left: 0, top: 0, width: 200, height: 200 } });
+
+			expect(result.codedWidth).toBe(200);
+			expect(result.codedHeight).toBe(200);
+
+			const rgba = await readRgba(result);
+			expectColor(getColorAt(rgba, 200, 50, 50), [255, 0, 0]);
+			expectColor(getColorAt(rgba, 200, 150, 50), [255, 0, 0]);
+			expectColor(getColorAt(rgba, 200, 50, 150), [255, 0, 0]);
+			expectColor(getColorAt(rgba, 200, 150, 150), [255, 0, 0]);
+		});
+
+		test('rotate 90 deg then crop top-left, crop applies after rotation', async () => {
+			using sample = makeSample();
+			using result = await sample.transform({
+				rotate: 90,
+				crop: { left: 0, top: 0, width: 200, height: 200 },
+			});
+
+			expect(result.codedWidth).toBe(200);
+			expect(result.codedHeight).toBe(200);
+
+			const rgba = await readRgba(result);
+			expectColor(getColorAt(rgba, 200, 50, 50), [0, 0, 255]);
+			expectColor(getColorAt(rgba, 200, 150, 50), [0, 0, 255]);
+			expectColor(getColorAt(rgba, 200, 50, 150), [0, 0, 255]);
+			expectColor(getColorAt(rgba, 200, 150, 150), [0, 0, 255]);
+		});
+
+		test('resize to 400x200 with fill, vertically squished', async () => {
+			using sample = makeSample();
+			using result = await sample.transform({ width: 400, height: 200, fit: 'fill' });
+
+			expect(result.codedWidth).toBe(400);
+			expect(result.codedHeight).toBe(200);
+
+			const rgba = await readRgba(result);
+			expectColor(getColorAt(rgba, 400, 50, 50), [255, 0, 0]); // top-left
+			expectColor(getColorAt(rgba, 400, 300, 50), [255, 0, 0]); // top-right
+			expectColor(getColorAt(rgba, 400, 50, 175), [0, 0, 255]); // bottom-left (blue)
+			expectColor(getColorAt(rgba, 400, 300, 175), [255, 0, 0]); // bottom-right
+		});
+
+		test('resize to 400x200 with contain, letterboxed', async () => {
+			using sample = makeSample();
+			using result = await sample.transform({ width: 400, height: 200, fit: 'contain' });
+
+			expect(result.codedWidth).toBe(400);
+			expect(result.codedHeight).toBe(200);
+
+			const rgba = await readRgba(result);
+			expectColor(getColorAt(rgba, 400, 50, 100), [0, 0, 0]); // left letterbox
+			expectColor(getColorAt(rgba, 400, 350, 100), [0, 0, 0]); // right letterbox
+			expectColor(getColorAt(rgba, 400, 150, 50), [255, 0, 0]); // top-left of image (red)
+			expectColor(getColorAt(rgba, 400, 150, 150), [0, 0, 255]); // bottom-left of image (blue)
+			expectColor(getColorAt(rgba, 400, 250, 150), [255, 0, 0]); // bottom-right of image (red)
+		});
+
+		test('resize to 400x200 with cover, vertical center crop', async () => {
+			using sample = makeSample();
+			using result = await sample.transform({ width: 400, height: 200, fit: 'cover' });
+
+			expect(result.codedWidth).toBe(400);
+			expect(result.codedHeight).toBe(200);
+
+			const rgba = await readRgba(result);
+			expectColor(getColorAt(rgba, 400, 50, 50), [255, 0, 0]); // top-left (red)
+			expectColor(getColorAt(rgba, 400, 350, 50), [255, 0, 0]); // top-right (red)
+			expectColor(getColorAt(rgba, 400, 50, 150), [0, 0, 255]); // bottom-left (blue)
+			expectColor(getColorAt(rgba, 400, 350, 150), [255, 0, 0]); // bottom-right (red)
+		});
+
+		test('VideoSample transformation via Conversion API', async () => {
+			using input = new Input({
+				source: new FilePathSource('./test/public/video.mp4'),
+				formats: ALL_FORMATS,
+			});
+
+			const output = new Output({
+				format: new Mp4OutputFormat(),
+				target: new BufferTarget(),
+			});
+
+			const conversion = await Conversion.init({
+				input,
+				output,
+				video: {
+					width: 300,
+					height: 300,
+					fit: 'contain',
+				},
+			});
+			await conversion.execute();
+		});
+	});
 });
 
 describe('Audio', async () => {