Change VideoSampleResource interface, validate return values, add more proper VideoSample.copyTo() logic, fix decoder/encoder color space issues

packages/server/src/misc.ts

@@ -1,12 +1,14 @@
-import { VideoSamplePixelFormat, VideoCodec } from 'mediabunny';
+import { VideoSamplePixelFormat, MediaCodec } from 'mediabunny';
 import * as NodeAv from 'node-av';
 
-export const CODEC_TO_CODEC_ID: Record<VideoCodec, NodeAv.AVCodecID> = {
+export const CODEC_TO_CODEC_ID: Partial<Record<MediaCodec, NodeAv.AVCodecID>> = {
 	avc: NodeAv.AV_CODEC_ID_H264,
 	hevc: NodeAv.AV_CODEC_ID_HEVC,
 	vp8: NodeAv.AV_CODEC_ID_VP8,
 	vp9: NodeAv.AV_CODEC_ID_VP9,
 	av1: NodeAv.AV_CODEC_ID_AV1,
+
+	aac: NodeAv.AV_CODEC_ID_AAC,
 };
 
 let cachedHardwareContext: NodeAv.HardwareContext | null | undefined = undefined;
@@ -180,3 +182,44 @@ export const fromPixelFormat = (pixelFormat: VideoSamplePixelFormat) => {
 		default: return NodeAv.AV_PIX_FMT_NONE;
 	}
 };
+
+export const toAudioSampleFormat = (ffmpegSampleFormat: NodeAv.AVSampleFormat): AudioSampleFormat | null => {
+	switch (ffmpegSampleFormat) {
+		case NodeAv.AV_SAMPLE_FMT_U8: return 'u8';
+		case NodeAv.AV_SAMPLE_FMT_S16: return 's16';
+		case NodeAv.AV_SAMPLE_FMT_S32: return 's32';
+		case NodeAv.AV_SAMPLE_FMT_FLT: return 'f32';
+		case NodeAv.AV_SAMPLE_FMT_U8P: return 'u8-planar';
+		case NodeAv.AV_SAMPLE_FMT_S16P: return 's16-planar';
+		case NodeAv.AV_SAMPLE_FMT_S32P: return 's32-planar';
+		case NodeAv.AV_SAMPLE_FMT_FLTP: return 'f32-planar';
+
+		default: return null;
+	}
+};
+
+export const fromAudioSampleFormat = (sampleFormat: AudioSampleFormat): NodeAv.AVSampleFormat => {
+	switch (sampleFormat) {
+		case 'u8': return NodeAv.AV_SAMPLE_FMT_U8;
+		case 's16': return NodeAv.AV_SAMPLE_FMT_S16;
+		case 's32': return NodeAv.AV_SAMPLE_FMT_S32;
+		case 'f32': return NodeAv.AV_SAMPLE_FMT_FLT;
+		case 'u8-planar': return NodeAv.AV_SAMPLE_FMT_U8P;
+		case 's16-planar': return NodeAv.AV_SAMPLE_FMT_S16P;
+		case 's32-planar': return NodeAv.AV_SAMPLE_FMT_S32P;
+		case 'f32-planar': return NodeAv.AV_SAMPLE_FMT_FLTP;
+
+		default: return NodeAv.AV_SAMPLE_FMT_NONE;
+	}
+};
+
+export const getChannelLayout = (numChannels: number): NodeAv.ChannelLayout => {
+	switch (numChannels) {
+		case 1: return NodeAv.AV_CHANNEL_LAYOUT_MONO;
+		case 2: return NodeAv.AV_CHANNEL_LAYOUT_STEREO;
+		case 4: return NodeAv.AV_CHANNEL_LAYOUT_QUAD;
+		case 6: return NodeAv.AV_CHANNEL_LAYOUT_5POINT1;
+		case 8: return NodeAv.AV_CHANNEL_LAYOUT_7POINT1;
+		default: return { nbChannels: numChannels, order: NodeAv.AV_CHANNEL_ORDER_UNSPEC, mask: 0n };
+	}
+};

packages/server/src/video-decoder.ts

@@ -1,13 +1,7 @@
 import { CustomVideoDecoder, VideoCodec, EncodedPacket, VideoSample, MaybePromise, Rational } from 'mediabunny';
 import * as NodeAv from 'node-av';
-import {
+import { CODEC_TO_CODEC_ID, getHardwareDecoderCodec } from './misc';
-	CODEC_TO_CODEC_ID,
+import { assert, binarySearchLessOrEqual, simplifyRational, toUint8Array } from '../../../src/misc';
-	getHardwareDecoderCodec,
-	mapColorPrimaries,
-	mapMatrixCoefficients,
-	mapTransferCharacteristics,
-} from './misc';
-import { binarySearchLessOrEqual, simplifyRational, toUint8Array } from '../../../src/misc';
 import { NodeAvFrameVideoSampleResource } from './video-sample';
 
 export class NodeAvVideoDecoder extends CustomVideoDecoder {
@@ -37,6 +31,7 @@ export class NodeAvVideoDecoder extends CustomVideoDecoder {
 		this.packet.alloc();
 
 		const codecId = CODEC_TO_CODEC_ID[this.codec];
+		assert(codecId !== undefined);
 
 		let codec: NodeAv.Codec | null;
 		if (this.config.hardwareAcceleration !== 'prefer-hardware' || this.codec === 'av1') {
@@ -67,30 +62,6 @@ export class NodeAvVideoDecoder extends CustomVideoDecoder {
 			: null;
 		codecContext.sampleAspectRatio = new NodeAv.Rational(this.pixelAspectRatio.num, this.pixelAspectRatio.den);
 
-		if (this.config.colorSpace?.primaries) {
-			const mapped = mapColorPrimaries(this.config.colorSpace.primaries);
-			if (mapped !== null) {
-				codecContext.colorPrimaries = mapped;
-			}
-		}
-		if (this.config.colorSpace?.transfer) {
-			const mapped = mapTransferCharacteristics(this.config.colorSpace.transfer);
-			if (mapped !== null) {
-				codecContext.colorTrc = mapped;
-			}
-		}
-		if (this.config.colorSpace?.matrix) {
-			const mapped = mapMatrixCoefficients(this.config.colorSpace.matrix);
-			if (mapped !== null) {
-				codecContext.colorSpace = mapped;
-			}
-		}
-		if (this.config.colorSpace?.fullRange != null) {
-			codecContext.colorRange = this.config.colorSpace.fullRange
-				? NodeAv.AVCOL_RANGE_JPEG
-				: NodeAv.AVCOL_RANGE_MPEG;
-		}
-
 		const ret = await codecContext.open2();
 		NodeAv.FFmpegError.throwIfError(ret, 'Open codec context');
 
@@ -178,7 +149,12 @@ export class NodeAvVideoDecoder extends CustomVideoDecoder {
 			}
 		}
 
-		this.onSample(new VideoSample(new NodeAvFrameVideoSampleResource(this.frame), {
+		const clone = this.frame.clone();
+		if (!clone) {
+			throw new Error('Frame clone allocation failed.');
+		}
+
+		this.onSample(new VideoSample(new NodeAvFrameVideoSampleResource(clone), {
 			timestamp,
 			duration,
 		}));

packages/server/src/video-encoder.ts

@@ -35,7 +35,6 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
 	avCodec!: NodeAv.Codec;
 	codecContext: NodeAv.CodecContext | null = null;
 	lastBuffer: Buffer | null = null;
-	colorSpaceSet = false;
 	packetEmitted = false;
 
 	scaler: NodeAv.SoftwareScaleContext | null = null;
@@ -65,6 +64,7 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
 		this.packet.alloc();
 
 		const codecId = CODEC_TO_CODEC_ID[this.codec];
+		assert(codecId !== undefined);
 
 		let codec: NodeAv.Codec | null = null;
 		if (this.config.hardwareAcceleration === 'prefer-software') {
@@ -203,15 +203,6 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
 			this.frame.fromBuffer(this.lastBuffer);
 		}
 
-		if (!this.colorSpaceSet) {
-			this.codecContext.colorPrimaries = this.frame.colorPrimaries;
-			this.codecContext.colorTrc = this.frame.colorTrc;
-			this.codecContext.colorSpace = this.frame.colorSpace;
-			this.codecContext.colorRange = this.frame.colorRange;
-
-			this.colorSpaceSet = true;
-		}
-
 		let frameToEncode = this.frame;
 
 		const requiresScaler
@@ -550,7 +541,6 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
 		}
 
 		this.packetEmitted = false;
-		this.colorSpaceSet = false;
 	}
 
 	close(): MaybePromise<void> {

packages/server/src/video-sample.ts

@@ -1,20 +1,27 @@
 import assert from 'assert';
-import { Rectangle, VideoSamplePixelFormat } from 'mediabunny';
+import { VideoSamplePixelFormat } from 'mediabunny';
 import { VideoSampleColorSpace } from 'mediabunny';
 import { VideoSampleResource } from 'mediabunny';
 import * as NodeAv from 'node-av';
-import { toUint8Array } from '../../../src/misc';
+import { MaybePromise, toUint8Array } from '../../../src/misc';
-import { getPlaneConfigs } from '../../../src/sample';
+import { VideoDataPlane, VideoSample } from '../../../src/sample';
 import {
 	toPixelFormat,
 	unmapColorPrimaries,
 	unmapTransferCharacteristics,
 	unmapMatrixCoefficients,
 	fromPixelFormat,
-	mapColorPrimaries,
-	mapTransferCharacteristics,
-	mapMatrixCoefficients,
 } from './misc';
+import { SetRequired } from 'mediabunny';
+import { VideoSampleInit } from 'mediabunny';
+
+const JPEG_RANGE_PIX_FORMATS = new Set([
+	NodeAv.AV_PIX_FMT_YUVJ411P,
+	NodeAv.AV_PIX_FMT_YUVJ420P,
+	NodeAv.AV_PIX_FMT_YUVJ422P,
+	NodeAv.AV_PIX_FMT_YUVJ440P,
+	NodeAv.AV_PIX_FMT_YUVJ444P,
+]);
 
 export class NodeAvFrameVideoSampleResource extends VideoSampleResource {
 	frame: NodeAv.Frame;
@@ -22,12 +29,7 @@ export class NodeAvFrameVideoSampleResource extends VideoSampleResource {
 	constructor(frame: NodeAv.Frame) {
 		super();
 
-		const clone = frame.clone();
+		this.frame = frame;
-		if (!clone) {
-			throw new Error('Allocation failure during frame clone.');
-		}
-
-		this.frame = clone;
 	}
 
 	getFormat(): VideoSamplePixelFormat | null {
@@ -63,7 +65,12 @@ export class NodeAvFrameVideoSampleResource extends VideoSampleResource {
 			primaries: unmapColorPrimaries(this.frame.colorPrimaries) as VideoColorPrimaries | null,
 			transfer: unmapTransferCharacteristics(this.frame.colorTrc) as VideoTransferCharacteristics | null,
 			matrix: unmapMatrixCoefficients(this.frame.colorSpace) as VideoMatrixCoefficients | null,
-			fullRange: this.frame.colorRange === NodeAv.AVCOL_RANGE_JPEG,
+			fullRange: this.frame.colorRange === NodeAv.AVCOL_RANGE_JPEG
+				|| JPEG_RANGE_PIX_FORMATS.has(this.frame.format as NodeAv.AVPixelFormat)
+				? true
+				: this.frame.colorRange === NodeAv.AVCOL_RANGE_MPEG
+					? false
+					: null,
 		});
 	}
 
@@ -71,450 +78,53 @@ export class NodeAvFrameVideoSampleResource extends VideoSampleResource {
 		this.frame.free();
 	}
 
-	allocationSize(options: VideoFrameCopyToOptions): number {
+	getDataPlanes(): MaybePromise<VideoDataPlane[]> {
-		// 3. Let combinedLayout be the result of running the Parse VideoFrameCopyToOptions algorithm with options.
+		assert(this.frame.data);
-		// 4. If combinedLayout is an exception, throw combinedLayout.
-		const combinedLayout = ParseVideoFrameCopyToOptions(this, options);
 
-		// 5. Return combinedLayout's allocationSize.
+		return this.frame.data.map((data, i) => ({
-		return combinedLayout.allocationSize;
+			data: toUint8Array(data),
+			stride: this.frame.linesize[i]!,
+		}));
 	}
 
-	async copyTo(
+	async toRgbSample(
-		destination: AllowSharedBufferSource,
+		init: SetRequired<VideoSampleInit, 'timestamp'>,
-		options: VideoFrameCopyToOptions,
+		format: 'RGBA' | 'RGBX' | 'BGRA' | 'BGRX',
-	): Promise<PlaneLayout[]> {
+		// Will respect it when somebody complains
-		const format = this.getFormat();
+		// eslint-disable-next-line @typescript-eslint/no-unused-vars
-		assert(format !== null);
+		colorSpace: PredefinedColorSpace,
+	): Promise<VideoSample> {
+		const width = this.frame.width;
+		const height = this.frame.height;
 
-		// 3. Let combinedLayout be the result of running the Parse VideoFrameCopyToOptions algorithm with options.
+		const scaler = new NodeAv.SoftwareScaleContext();
-		const combinedLayout = ParseVideoFrameCopyToOptions(this, options);
+		const srcFmt = this.frame.format as NodeAv.AVPixelFormat;
+		const dstFmt = fromPixelFormat(format);
 
-		// 4. If destination.byteLength is less than combinedLayout’s allocationSize, return a promise rejected with
+		scaler.getContext(
-		// // eslint-disable-next-line @stylistic/max-len
+			width, height, srcFmt,
-		const destBytes = toUint8Array(destination);
+			width, height, dstFmt,
-		if (destBytes.byteLength < combinedLayout.allocationSize) {
+			NodeAv.SWS_BILINEAR,
-			throw new TypeError(
+		);
-				`Destination buffer too small. Required: ${combinedLayout.allocationSize},`
+
-				+ ` Available: ${destBytes.byteLength}`,
+		const dstFrame = new NodeAv.Frame();
-			);
+		dstFrame.width = width;
+		dstFrame.height = height;
+		dstFrame.format = dstFmt;
+		dstFrame.alloc();
+		dstFrame.allocBuffer();
+
+		const srcFrame = this.frame;
+
+		try {
+			await scaler.scaleFrame(dstFrame, srcFrame);
+		} finally {
+			scaler.freeContext();
 		}
 
-		// 5. If options.format is equal to one of RGBA, RGBX, BGRA, BGRX then:
+		dstFrame.format = dstFmt; // FFmpeg messes up RGBA and RGBX
-		if (options.format && ['RGBA', 'RGBX', 'BGRA', 'BGRX'].includes(options.format) && options.format !== format) {
+		dstFrame.sampleAspectRatio = srcFrame.sampleAspectRatio;
-			// Let newOptions be the result of running the Clone Configuration algorithm with options.
-			// Assign undefined to newOptions.format.
-			const newOptions = { ...options };
-			delete newOptions.format;
 
-			// Let rgbFrame be the result of running the Convert to RGB frame algorithm with this, options.format,
+		return new VideoSample(new NodeAvFrameVideoSampleResource(dstFrame), init);
-			// and options.colorSpace.
-			const rgbFrame = await ConvertToRGBFrame(this, options.format, options.colorSpace);
-
-			// Return the result of calling copyTo() on rgbFrame with destination and newOptions.
-			try {
-				const result = await rgbFrame.copyTo(destination, newOptions);
-				rgbFrame.close();
-				return result;
-			} catch (e) {
-				rgbFrame.close();
-				throw e;
-			}
-		}
-
-		// 6. Let p be a new Promise. (Implicit)
-		// 7. Let copyStepsQueue be the result of starting a new parallel queue. (Implicit)
-		// 8. Let planeLayouts be a new list.
-		const planeLayouts: PlaneLayout[] = [];
-
-		// Enqueue the following steps to copyStepsQueue: (fuck the queuing part)
-
-		// Let resource be the media resource referenced by [[resource reference]].
-		// (this.data)
-
-		// Let numPlanes be the number of planes as defined by [[format]].
-		const planes = getPlaneConfigs(format);
-		const numPlanes = planes.length;
-
-		// Let planeIndex be 0.
-		// While planeIndex is less than combinedLayout’s numPlanes:
-		for (let planeIndex = 0; planeIndex < numPlanes; planeIndex++) {
-			const computedLayout = combinedLayout.computedLayouts[planeIndex]!;
-
-			// Let sourceStride be the stride of the plane in resource as identified by planeIndex.
-			let sourceStride = 0;
-			let sourceStartOffset = 0;
-
-			sourceStride = this.frame.linesize[planeIndex]!;
-			const planeBuffer = this.frame.data![planeIndex]!;
-			if (!planeBuffer) {
-				throw new Error(`Missing data for plane ${planeIndex}`);
-			}
-			const sourceData = new Uint8Array(planeBuffer.buffer, planeBuffer.byteOffset, planeBuffer.byteLength);
-			sourceStartOffset = 0;
-
-			// Let sourceOffset be the product of multiplying computedLayout’s sourceTop by sourceStride
-			let sourceOffset = computedLayout.sourceTop * sourceStride;
-
-			// Add computedLayout’s sourceLeftBytes to sourceOffset.
-			sourceOffset += computedLayout.sourceLeftBytes;
-
-			// Adjust for the base offset of the plane in the source buffer
-			sourceOffset += sourceStartOffset;
-
-			// Let destinationOffset be computedLayout’s destinationOffset.
-			let destinationOffset = computedLayout.destinationOffset;
-
-			// Let rowBytes be computedLayout’s sourceWidthBytes.
-			const rowBytes = computedLayout.sourceWidthBytes;
-
-			// Let layout be a new PlaneLayout, with offset set to destinationOffset and stride set to rowBytes.
-			// This is a spec error actually (https://github.com/w3c/webcodecs/issues/918)
-			const layout: PlaneLayout = {
-				offset: destinationOffset,
-				stride: computedLayout.destinationStride,
-			};
-
-			// Let row be 0.
-			// While row is less than computedLayout’s sourceHeight:
-			for (let row = 0; row < computedLayout.sourceHeight; row++) {
-				// Copy rowBytes bytes from resource starting at sourceOffset to destination starting
-				// at destinationOffset.
-				if (sourceOffset + rowBytes > sourceData.byteLength) {
-					throw new Error(`Source buffer OOB read`);
-				}
-				if (destinationOffset + rowBytes > destBytes.byteLength) {
-					throw new Error(`Destination buffer OOB write`);
-				}
-
-				const srcSub = sourceData.subarray(sourceOffset, sourceOffset + rowBytes);
-				destBytes.set(srcSub, destinationOffset);
-
-				// Increment sourceOffset by sourceStride.
-				sourceOffset += sourceStride;
-
-				// Increment destinationOffset by computedLayout’s destinationStride.
-				destinationOffset += computedLayout.destinationStride;
-			}
-
-			// Append layout to planeLayouts.
-			planeLayouts.push(layout);
-		}
-
-		// Queue a task to resolve p with planeLayouts.
-		return planeLayouts;
 	}
 }
-
-type CombinedBufferLayout = {
-	allocationSize: number;
-	computedLayouts: ComputedPlaneLayout[];
-};
-
-type ComputedPlaneLayout = {
-	destinationOffset: number;
-	destinationStride: number;
-	sourceTop: number;
-	sourceHeight: number;
-	sourceLeftBytes: number;
-	sourceWidthBytes: number;
-};
-
-// Taken from the WebCodecs spec
-const ParseVideoFrameCopyToOptions = (
-	frame: VideoSampleResource,
-	options: VideoFrameCopyToOptions,
-): CombinedBufferLayout => {
-	// 1. Let defaultRect be the result of performing the getter steps for visibleRect.
-	const defaultRect: Rectangle = {
-		left: 0,
-		top: 0,
-		width: frame.getCodedWidth(),
-		height: frame.getCodedHeight(),
-	};
-
-	// 2. Let overrideRect be undefined.
-	// 3. If options.rect exists, assign the value of options.rect to overrideRect.
-	const overrideRect = options.rect;
-
-	// 4. Let parsedRect be the result of running the Parse Visible Rect algorithm...
-	const parsedRect = ParseVisibleRect(
-		defaultRect,
-		overrideRect,
-		frame.getCodedWidth(),
-		frame.getCodedHeight(),
-		frame.getFormat(),
-	);
-
-	// 5. If parsedRect is an exception, return parsedRect. (Handled by throw)
-
-	// 6. Let optLayout be undefined.
-	// 7. If options.layout exists, assign its value to optLayout.
-	const optLayout = options.layout;
-
-	// 8. Let format be undefined.
-	let format: VideoSamplePixelFormat | undefined;
-
-	// 9. If options.format does not exist, assign [[format]] to format.
-	if (!options.format || options.format === frame.getFormat()) {
-		format = frame.getFormat()!;
-	} else if (['RGBA', 'RGBX', 'BGRA', 'BGRX'].includes(options.format)) {
-		// 10. Otherwise, if options.format is equal to one of RGBA, RGBX, BGRA, BGRX, then assign options.format
-		//  to format...
-		format = options.format;
-	} else {
-		throw new Error('NotSupportedError: Invalid destination format');
-	}
-
-	// 11. Let combinedLayout be the result of running the Compute Layout and Allocation Size algorithm...
-	return ComputeLayoutAndAllocationSize(parsedRect, format, optLayout);
-};
-
-// Taken from the WebCodecs spec
-const ParseVisibleRect = (
-	defaultRect: DOMRectInit,
-	overrideRect: DOMRectInit | undefined,
-	codedWidth: number,
-	codedHeight: number,
-	format: VideoSamplePixelFormat | null,
-): DOMRectInit => {
-	// 1. Let sourceRect be defaultRect
-	const sourceRect = { ...defaultRect };
-
-	// 2. If overrideRect is not undefined:
-	if (overrideRect !== undefined) {
-		// If either of overrideRect.width or height is 0, return a TypeError.
-		if (overrideRect.width === 0 || overrideRect.height === 0) {
-			throw new TypeError('visibleRect dimensions cannot be zero');
-		}
-		// If the sum of overrideRect.x and overrideRect.width is greater than codedWidth, return a TypeError.
-		if ((overrideRect.x || 0) + (overrideRect.width || 0) > codedWidth) {
-			throw new TypeError('visibleRect exceeds codedWidth');
-		}
-		// If the sum of overrideRect.y and overrideRect.height is greater than codedHeight, return a TypeError.
-		if ((overrideRect.y || 0) + (overrideRect.height || 0) > codedHeight) {
-			throw new TypeError('visibleRect exceeds codedHeight');
-		}
-		// Assign overrideRect to sourceRect.
-		sourceRect.x = overrideRect.x || 0;
-		sourceRect.y = overrideRect.y || 0;
-		sourceRect.width = overrideRect.width || 0;
-		sourceRect.height = overrideRect.height || 0;
-	}
-
-	// 3. Let validAlignment be the result of running the Verify Rect Offset Alignment algorithm.
-	const validAlignment = VerifyRectOffsetAlignment(format, sourceRect);
-
-	// 4. If validAlignment is false, throw a TypeError.
-	if (!validAlignment) {
-		throw new TypeError('visibleRect alignment is invalid for the format');
-	}
-
-	// 5. Return sourceRect.
-	return sourceRect;
-};
-
-// Taken from the WebCodecs spec
-const VerifyRectOffsetAlignment = (format: VideoSamplePixelFormat | null, rect: DOMRectInit): boolean => {
-	// 1. If format is null, return true.
-	if (format === null) return true;
-
-	const planes = getPlaneConfigs(format);
-
-	// 2. Let planeIndex be 0.
-	// 3. Let numPlanes be the number of planes as defined by format.
-	// 4. While planeIndex is less than numPlanes:
-	for (let planeIndex = 0; planeIndex < planes.length; planeIndex++) {
-		const plane = planes[planeIndex]!;
-		const sampleWidth = plane.widthDivisor;
-		const sampleHeight = plane.heightDivisor;
-
-		// If rect.x is not a multiple of sampleWidth, return false.
-		if ((rect.x || 0) % sampleWidth !== 0) return false;
-		// If rect.y is not a multiple of sampleHeight, return false.
-		if ((rect.y || 0) % sampleHeight !== 0) return false;
-	}
-
-	return true;
-};
-
-// Taken from the WebCodecs spec
-const ComputeLayoutAndAllocationSize = (
-	parsedRect: DOMRectInit,
-	format: VideoSamplePixelFormat,
-	layout?: PlaneLayout[],
-): CombinedBufferLayout => {
-	const planes = getPlaneConfigs(format);
-
-	// 1. Let numPlanes be the number of planes as defined by format.
-	const numPlanes = planes.length;
-
-	// 2. If layout is not undefined and its length does not equal numPlanes, throw a TypeError.
-	if (layout !== undefined && layout.length !== numPlanes) {
-		throw new TypeError(`Layout must have ${numPlanes} planes`);
-	}
-
-	// 3. Let minAllocationSize be 0.
-	let minAllocationSize = 0;
-
-	// 4. Let computedLayouts be a new list.
-	const computedLayouts: ComputedPlaneLayout[] = [];
-
-	// 5. Let endOffsets be a new list.
-	const endOffsets: number[] = [];
-
-	// 6. Let planeIndex be 0.
-	// 7. While planeIndex < numPlanes:
-	for (let planeIndex = 0; planeIndex < numPlanes; planeIndex++) {
-		const plane = planes[planeIndex]!;
-		const sampleBytes = plane.sampleBytes;
-		const sampleWidth = plane.widthDivisor;
-		const sampleHeight = plane.heightDivisor;
-
-		// Let computedLayout be a new computed plane layout.
-		const computedLayout: ComputedPlaneLayout = {
-			destinationOffset: 0,
-			destinationStride: 0,
-			sourceTop: 0,
-			sourceHeight: 0,
-			sourceLeftBytes: 0,
-			sourceWidthBytes: 0,
-		};
-
-		// Set computedLayout’s sourceTop...
-		computedLayout.sourceTop = Math.ceil(Math.trunc(parsedRect.y || 0) / sampleHeight);
-		// Set computedLayout’s sourceHeight...
-		computedLayout.sourceHeight = Math.ceil(Math.trunc(parsedRect.height || 0) / sampleHeight);
-		// Set computedLayout’s sourceLeftBytes...
-		computedLayout.sourceLeftBytes = Math.floor(Math.trunc(parsedRect.x || 0) / sampleWidth) * sampleBytes;
-		// Set computedLayout’s sourceWidthBytes...
-		computedLayout.sourceWidthBytes = Math.floor(Math.trunc(parsedRect.width || 0) / sampleWidth) * sampleBytes;
-
-		// If layout is not undefined:
-		if (layout !== undefined) {
-			const planeLayout = layout[planeIndex]!;
-			// If planeLayout.stride is less than computedLayout’s sourceWidthBytes, return a TypeError.
-			if (planeLayout.stride < computedLayout.sourceWidthBytes) {
-				throw new TypeError(`Stride for plane ${planeIndex} is too small`);
-			}
-			// Assign planeLayout.offset to computedLayout’s destinationOffset.
-			computedLayout.destinationOffset = planeLayout.offset;
-			// Assign planeLayout.stride to computedLayout’s destinationStride.
-			computedLayout.destinationStride = planeLayout.stride;
-		} else {
-			// Otherwise:
-			// Assign minAllocationSize to computedLayout’s destinationOffset.
-			computedLayout.destinationOffset = minAllocationSize;
-			// Assign computedLayout’s sourceWidthBytes to computedLayout’s destinationStride.
-			computedLayout.destinationStride = computedLayout.sourceWidthBytes;
-		}
-
-		// Let planeSize be the product of multiplying computedLayout’s destinationStride and sourceHeight.
-		const planeSize = computedLayout.destinationStride * computedLayout.sourceHeight;
-
-		// Let planeEnd be the sum of planeSize and computedLayout’s destinationOffset.
-		const planeEnd = planeSize + computedLayout.destinationOffset;
-
-		// If planeSize or planeEnd is greater than maximum range of unsigned long, return a TypeError.
-		if (planeEnd > 4294967295) throw new TypeError('Allocation size exceeds limit');
-
-		// Append planeEnd to endOffsets.
-		endOffsets.push(planeEnd);
-
-		// Assign the maximum of minAllocationSize and planeEnd to minAllocationSize.
-		minAllocationSize = Math.max(minAllocationSize, planeEnd);
-
-		// Check for overlap
-		for (let earlierPlaneIndex = 0; earlierPlaneIndex < planeIndex; earlierPlaneIndex++) {
-			const earlierLayout = computedLayouts[earlierPlaneIndex]!;
-			// If plane A ends before plane B starts, they do not overlap.
-			if (
-				endOffsets[planeIndex]! <= earlierLayout.destinationOffset
-				|| endOffsets[earlierPlaneIndex]! <= computedLayout.destinationOffset
-			) {
-				continue;
-			}
-			throw new TypeError('Planes overlap');
-		}
-
-		computedLayouts.push(computedLayout);
-	}
-
-	// 12. Return combinedLayout.
-	return {
-		allocationSize: minAllocationSize,
-		computedLayouts: computedLayouts,
-	};
-};
-
-// Taken from the WebCodecs spec
-const ConvertToRGBFrame = async (
-	frame: NodeAvFrameVideoSampleResource,
-	format: VideoPixelFormat,
-	colorSpace?: PredefinedColorSpace,
-): Promise<NodeAvFrameVideoSampleResource> => {
-	// 1. Let convertedFrame be a new VideoFrame...
-	// (We construct it at the end, but we prepare the resource here)
-
-	const width = frame.getCodedWidth();
-	const height = frame.getCodedHeight();
-
-	const scaler = new NodeAv.SoftwareScaleContext();
-	const srcFmt = fromPixelFormat(frame.getFormat()!);
-	const dstFmt = fromPixelFormat(format);
-
-	// Configure scaling: Source -> Destination (Visible Size)
-	scaler.getContext(
-		width, height, srcFmt,
-		width, height, dstFmt,
-		NodeAv.SWS_BILINEAR,
-	);
-
-	// Allocate destination frame
-	const dstFrame = new NodeAv.Frame();
-	dstFrame.width = width;
-	dstFrame.height = height;
-	dstFrame.format = dstFmt;
-	dstFrame.alloc();
-	dstFrame.allocBuffer();
-
-	// Apply destination color space settings to dstFrame
-	// If colorSpace is not provided, srgb is used
-	const targetColorSpace = colorSpace || 'srgb';
-	if (targetColorSpace === 'srgb') {
-		dstFrame.colorPrimaries = NodeAv.AVCOL_PRI_BT709;
-		dstFrame.colorTrc = NodeAv.AVCOL_TRC_IEC61966_2_1;
-		dstFrame.colorSpace = NodeAv.AVCOL_SPC_RGB;
-		dstFrame.colorRange = NodeAv.AVCOL_RANGE_JPEG;
-	} else if (targetColorSpace === 'display-p3') {
-		dstFrame.colorPrimaries = NodeAv.AVCOL_PRI_SMPTE432;
-		dstFrame.colorTrc = NodeAv.AVCOL_TRC_IEC61966_2_1;
-		dstFrame.colorSpace = NodeAv.AVCOL_SPC_RGB;
-		dstFrame.colorRange = NodeAv.AVCOL_RANGE_JPEG;
-	}
-
-	const srcFrame = frame.frame;
-
-	// Apply source color space settings to srcFrame (if known)
-	// This ensures the scaler knows the input color space for correct conversion
-	const frameColorSpace = frame.getColorSpace();
-	if (colorSpace) {
-		srcFrame.colorPrimaries = mapColorPrimaries(frameColorSpace.primaries ?? 'unknown')
-			?? NodeAv.AVCOL_PRI_UNSPECIFIED;
-		srcFrame.colorTrc = mapTransferCharacteristics(frameColorSpace.transfer ?? 'unknown')
-			?? NodeAv.AVCOL_TRC_UNSPECIFIED;
-		srcFrame.colorSpace = mapMatrixCoefficients(frameColorSpace.matrix ?? 'unknown')
-			?? NodeAv.AVCOL_SPC_UNSPECIFIED;
-		srcFrame.colorRange = frameColorSpace.fullRange
-			? NodeAv.AVCOL_RANGE_JPEG
-			: NodeAv.AVCOL_RANGE_MPEG;
-	}
-
-	try {
-		await scaler.scaleFrame(dstFrame, srcFrame);
-	} finally {
-		scaler.freeContext();
-	}
-
-	return new NodeAvFrameVideoSampleResource(dstFrame);
-};

src/sample.ts

@@ -116,16 +116,20 @@ export abstract class VideoSampleResource {
 	 */
 	abstract close(): void;
 
-	/** Returns the number of bytes required to hold this video sample's pixel data. */
+	abstract getDataPlanes(): MaybePromise<VideoDataPlane[]>;
-	abstract allocationSize(options: VideoFrameCopyToOptions): number;
 
-	/** Copies this video sample's pixel data to an ArrayBuffer or ArrayBufferView. */
+	abstract toRgbSample(
-	abstract copyTo(
+		init: SetRequired<VideoSampleInit, 'timestamp'>,
-		destination: AllowSharedBufferSource,
+		format: 'RGBA' | 'RGBX' | 'BGRA' | 'BGRX',
-		options: VideoFrameCopyToOptions
+		colorSpace: PredefinedColorSpace,
-	): MaybePromise<PlaneLayout[]>;
+	): MaybePromise<VideoSample>;
 }
 
+export type VideoDataPlane = {
+	data: Uint8Array;
+	stride: number;
+};
+
 /**
  * The list of {@link VideoSample} pixel formats.
  * @group Samples
@@ -567,9 +571,33 @@ export class VideoSample implements Disposable {
 			data._referenceCount++;
 
 			this.format = data.getFormat();
-			this.visibleRect = { left: 0, top: 0, width: data.getCodedWidth(), height: data.getCodedHeight() };
+			if (this.format !== null && !VIDEO_SAMPLE_PIXEL_FORMATS.includes(this.format)) {
+				throw new TypeError('getFormat() must return a VideoSamplePixelFormat or null.');
+			}
+
+			this.visibleRect = {
+				left: 0,
+				top: 0,
+				width: data.getCodedWidth(),
+				height: data.getCodedHeight(),
+			};
+			if (!Number.isInteger(this.visibleRect.width) || this.visibleRect.width <= 0) {
+				throw new TypeError('getCodedWidth() must return a positive integer.');
+			}
+			if (!Number.isInteger(this.visibleRect.height) || this.visibleRect.height <= 0) {
+				throw new TypeError('getCodedHeight() must return a positive integer.');
+			}
+
 			this.squarePixelWidth = data.getSquarePixelWidth();
+			if (!Number.isInteger(this.squarePixelWidth) || this.squarePixelWidth <= 0) {
+				throw new TypeError('getSquarePixelWidth() must return a positive integer.');
+			}
+
 			this.squarePixelHeight = data.getSquarePixelHeight();
+			if (!Number.isInteger(this.squarePixelHeight) || this.squarePixelHeight <= 0) {
+				throw new TypeError('getSquarePixelHeight() must return a positive integer.');
+			}
+
 			this.rotation = init.rotation ?? 0;
 			this.timestamp = init.timestamp!;
 			this.duration = init.duration ?? 0;
@@ -682,42 +710,20 @@ export class VideoSample implements Disposable {
 		if (this._closed) {
 			throw new Error('VideoSample is closed.');
 		}
-		if (this.format === null) {
+
+		if ((options.format ?? this.format) == null) {
 			// https://github.com/Vanilagy/mediabunny/issues/267
 			// https://github.com/w3c/webcodecs/issues/920
-			throw new Error('Cannot get allocation size when format is null. Sorry!');
+			throw new Error('Cannot get allocation size when format is null.');
-		}
-
-		assert(this._data !== null);
-
-		if (this._data instanceof VideoSampleResource) {
-			return this._data.allocationSize(options);
-		}
-
-		if (!isVideoFrame(this._data)) {
-			if (
-				options.colorSpace
-				|| (options.format && options.format !== this.format)
-				|| options.layout
-				|| options.rect
-			) {
-				// Temporarily convert to VideoFrame to get it done
-				// TODO: Compute this directly without needing to go through VideoFrame
-				const videoFrame = this.toVideoFrame();
-				const size = videoFrame.allocationSize(options);
-				videoFrame.close();
-
-				return size;
-			}
 		}
 
 		if (isVideoFrame(this._data)) {
+			// Call the native method purely for performance
 			return this._data.allocationSize(options);
-		} else if (this._data instanceof Uint8Array) {
-			return this._data.byteLength;
-		} else {
-			return this.codedWidth * this.codedHeight * 4; // RGBX
 		}
+
+		const combinedLayout = ParseVideoFrameCopyToOptions(this, options);
+		return combinedLayout.allocationSize;
 	}
 
 	/**
@@ -733,56 +739,200 @@ export class VideoSample implements Disposable {
 		if (this._closed) {
 			throw new Error('VideoSample is closed.');
 		}
-		if (this.format === null) {
+		if ((options.format ?? this.format) == null) {
 			throw new Error('Cannot copy video sample data when format is null. Sorry!');
 		}
 
 		assert(this._data !== null);
 
-		if (this._data instanceof VideoSampleResource) {
+		if (isVideoFrame(this._data)) {
 			return this._data.copyTo(destination, options);
 		}
 
-		if (!isVideoFrame(this._data)) {
+		if (
-			if (
+			options.format
-				options.colorSpace
+			&& this.format !== options.format
-				|| (options.format && options.format !== this.format)
+			&& ['RGBA', 'RGBX', 'BGRA', 'BGRX'].includes(options.format)
-				|| options.layout
+		) {
-				|| options.rect
+			// RGB conversion for custom VideoSampleResource
-			) {
+			if (this._data instanceof VideoSampleResource) {
-				// Temporarily convert to VideoFrame to get it done
+				using rgbSample = await this._data.toRgbSample(
-				// TODO: Do this directly without needing to go through VideoFrame
+					{
-				const videoFrame = this.toVideoFrame();
+						timestamp: this.timestamp,
-				const layout = await videoFrame.copyTo(destination, options);
+						duration: this.duration,
-				videoFrame.close();
+						rotation: this.rotation,
+					},
+					options.format as 'RGBA' | 'RGBX' | 'BGRA' | 'BGRX',
+					options.colorSpace ?? 'srgb',
+				);
+				if (!(rgbSample instanceof VideoSample)) {
+					throw new TypeError('toRgbSample() must return a VideoSample.');
+				}
+				if (rgbSample.format !== options.format) {
+					throw new Error(
+						`Sample returned by toRgbSample was expected to have format '${options.format}', got`
+						+ ` '${rgbSample.format}' instead.`,
+					);
+				}
 
-				return layout;
+				return await rgbSample.copyTo(destination, options); // 'await' is intentional here cuz of using
+			} else if (!['RGBA', 'RGBX', 'BGRA', 'BGRX'].includes(this.format!)) {
+				if (typeof VideoFrame === 'undefined') {
+					throw new Error(
+						'For this sample, converting from a non-RGB to an RGB format requires VideoFrame to'
+						+ ' be defined.',
+					);
+				}
+
+				const tempFrame = this.toVideoFrame();
+				const result = await tempFrame.copyTo(destination, options);
+				tempFrame.close();
+
+				return result;
 			}
 		}
 
-		if (isVideoFrame(this._data)) {
+		const combinedLayout = ParseVideoFrameCopyToOptions(this, options);
-			return this._data.copyTo(destination, options);
+		assert(this.format);
+
+		// 4. If destination.byteLength is less than combinedLayout’s allocationSize, return a promise rejected with
+		const destBytes = toUint8Array(destination);
+		if (destBytes.byteLength < combinedLayout.allocationSize) {
+			throw new TypeError(
+				`Destination buffer too small. Required: ${combinedLayout.allocationSize},`
+				+ ` Available: ${destBytes.byteLength}`,
+			);
+		}
+
+		const planeConfigs = getPlaneConfigs(this.format);
+		let dataPlanes: VideoDataPlane[];
+
+		if (this._data instanceof VideoSampleResource) {
+			let result = this._data.getDataPlanes();
+			if (result instanceof Promise) result = await result;
+
+			if (
+				!Array.isArray(result)
+				|| result.some(x => !(x.data instanceof Uint8Array) || !Number.isInteger(x.stride) || x.stride < 0)
+			) {
+				throw new TypeError(
+					'getDataPlanes() must return an array of objects with a Uint8Array "data" property and a'
+					+ ' non-negative integer "stride" property.',
+				);
+			}
+
+			dataPlanes = result;
 		} else if (this._data instanceof Uint8Array) {
 			assert(this._layout);
+			assert(this._layout.length === planeConfigs.length);
 
-			const dest = toUint8Array(destination);
+			dataPlanes = this._layout.map((planeLayout, i) => {
-			dest.set(this._data);
+				const height = Math.ceil(this.codedHeight / planeConfigs[i]!.heightDivisor);
 
-			return this._layout;
+				return {
+					data: (this._data as Uint8Array).subarray(
+						planeLayout.offset,
+						planeLayout.offset + planeLayout.stride * height,
+					),
+					stride: planeLayout.stride,
+				};
+			});
 		} else {
 			const canvas = this._data;
 			const context = canvas.getContext('2d');
 			assert(context);
 
 			const imageData = context.getImageData(0, 0, this.codedWidth, this.codedHeight);
-			const dest = toUint8Array(destination);
-			dest.set(imageData.data);
 
-			return [{
+			dataPlanes = [{
-				offset: 0,
+				data: toUint8Array(imageData.data),
 				stride: 4 * this.codedWidth,
 			}];
 		}
+
+		// 6. Let p be a new Promise. (Implicit)
+		// 7. Let copyStepsQueue be the result of starting a new parallel queue. (Implicit)
+		// 8. Let planeLayouts be a new list.
+		const planeLayouts: PlaneLayout[] = [];
+
+		// Enqueue the following steps to copyStepsQueue: (fuck the queuing part)
+
+		// Let resource be the media resource referenced by [[resource reference]].
+		// (this.data)
+
+		// Let numPlanes be the number of planes as defined by [[format]].
+		const numPlanes = planeConfigs.length;
+
+		// Let planeIndex be 0.
+		// While planeIndex is less than combinedLayout’s numPlanes:
+		for (let planeIndex = 0; planeIndex < numPlanes; planeIndex++) {
+			const computedLayout = combinedLayout.computedLayouts[planeIndex]!;
+
+			// Let sourceStride be the stride of the plane in resource as identified by planeIndex.
+			const sourceStride = dataPlanes[planeIndex]!.stride;
+			const sourceData = dataPlanes[planeIndex]!.data;
+
+			// Let sourceOffset be the product of multiplying computedLayout’s sourceTop by sourceStride
+			let sourceOffset = computedLayout.sourceTop * sourceStride;
+
+			// Add computedLayout’s sourceLeftBytes to sourceOffset.
+			sourceOffset += computedLayout.sourceLeftBytes;
+
+			// Let destinationOffset be computedLayout’s destinationOffset.
+			let destinationOffset = computedLayout.destinationOffset;
+
+			// Let rowBytes be computedLayout’s sourceWidthBytes.
+			const rowBytes = computedLayout.sourceWidthBytes;
+
+			// Let layout be a new PlaneLayout, with offset set to destinationOffset and stride set to rowBytes.
+			// This is a spec error actually (https://github.com/w3c/webcodecs/issues/918)
+			const layout: PlaneLayout = {
+				offset: destinationOffset,
+				stride: computedLayout.destinationStride,
+			};
+
+			// Let row be 0.
+			// While row is less than computedLayout’s sourceHeight:
+			for (let row = 0; row < computedLayout.sourceHeight; row++) {
+				// Copy rowBytes bytes from resource starting at sourceOffset to destination starting
+				// at destinationOffset.
+				if (sourceOffset + rowBytes > sourceData.byteLength) {
+					throw new Error(`Source buffer OOB read.`);
+				}
+				if (destinationOffset + rowBytes > destBytes.byteLength) {
+					throw new Error(`Destination buffer OOB write.`);
+				}
+
+				const srcSub = sourceData.subarray(sourceOffset, sourceOffset + rowBytes);
+				destBytes.set(srcSub, destinationOffset);
+
+				// Increment sourceOffset by sourceStride.
+				sourceOffset += sourceStride;
+
+				// Increment destinationOffset by computedLayout’s destinationStride.
+				destinationOffset += computedLayout.destinationStride;
+			}
+
+			// Append layout to planeLayouts.
+			planeLayouts.push(layout);
+		}
+
+		// RGB conversion for ArrayBuffer and canvas-backed samples
+		if (options.format !== undefined && !(this._data instanceof VideoSampleResource)) {
+			const needsRgbConversion = this.format.startsWith('RGB') !== options.format.startsWith('RGB');
+			if (needsRgbConversion) {
+				// Loop over the destination bytes, swapping R and B
+				for (let i = 0; i < combinedLayout.allocationSize; i += 4) {
+					const r = destBytes[i]!;
+					const b = destBytes[i + 2]!;
+					destBytes[i] = b;
+					destBytes[i + 2] = r;
+				}
+			}
+		}
+
+		// Queue a task to resolve p with planeLayouts.
+		return planeLayouts;
 	}
 
 	/**
@@ -797,35 +947,10 @@ export class VideoSample implements Disposable {
 		assert(this._data !== null);
 
 		if (this._data instanceof VideoSampleResource) {
-			const format = this._data.getFormat() ?? 'RGBA';
+			throw new Error(
-			const allocationSize = this._data.allocationSize({ format: format as VideoPixelFormat });
+				'Creating a VideoFrame from a VideoSampleResource is currently not supported. To work around this,'
-
+				+ ' copy the data into a buffer and then create a VideoFrame from it.',
-			let data: ArrayBuffer;
+			);
-			if (this._data._lastAllocationBuffer?.byteLength === allocationSize) {
-				data = this._data._lastAllocationBuffer;
-			} else {
-				data = this._data._lastAllocationBuffer = new ArrayBuffer(allocationSize);
-			}
-			const layoutResult = this._data.copyTo(data, { format: format as VideoPixelFormat });
-
-			if (layoutResult instanceof Promise) {
-				throw new Error(
-					'Cannot create a VideoFrame from a VideoSampleResource if copyTo returns a Promise. To work'
-					+ ' around this, copy the data into a buffer and then create a VideoFrame from it.',
-				);
-			}
-
-			const layout = layoutResult;
-
-			return new VideoFrame(data, {
-				format: format as VideoPixelFormat,
-				codedWidth: this._data.getCodedWidth(),
-				codedHeight: this._data.getCodedHeight(),
-				colorSpace: this._data.getColorSpace(),
-				layout,
-				timestamp: this.microsecondTimestamp,
-				duration: this.microsecondDuration,
-			});
 		} else if (isVideoFrame(this._data)) {
 			return new VideoFrame(this._data, {
 				timestamp: this.microsecondTimestamp,
@@ -1474,6 +1599,250 @@ export const getPlaneConfigs = (format: VideoSamplePixelFormat): PlaneConfig[] =
 	}
 };
 
+type CombinedBufferLayout = {
+	allocationSize: number;
+	computedLayouts: ComputedPlaneLayout[];
+};
+
+type ComputedPlaneLayout = {
+	destinationOffset: number;
+	destinationStride: number;
+	sourceTop: number;
+	sourceHeight: number;
+	sourceLeftBytes: number;
+	sourceWidthBytes: number;
+};
+
+/** Taken from the WebCodecs spec. */
+const ParseVideoFrameCopyToOptions = (
+	sample: VideoSample,
+	options: VideoFrameCopyToOptions,
+): CombinedBufferLayout => {
+	// 1. Let defaultRect be the result of performing the getter steps for visibleRect.
+	const defaultRect: Rectangle = {
+		left: 0,
+		top: 0,
+		width: sample.codedWidth,
+		height: sample.codedHeight,
+	};
+
+	// 2. Let overrideRect be undefined.
+	// 3. If options.rect exists, assign the value of options.rect to overrideRect.
+	const overrideRect = options.rect;
+
+	// 4. Let parsedRect be the result of running the Parse Visible Rect algorithm...
+	const parsedRect = ParseVisibleRect(
+		defaultRect,
+		overrideRect,
+		sample.codedWidth,
+		sample.codedHeight,
+		sample.format,
+	);
+
+	// 5. If parsedRect is an exception, return parsedRect. (Handled by throw)
+
+	// 6. Let optLayout be undefined.
+	// 7. If options.layout exists, assign its value to optLayout.
+	const optLayout = options.layout;
+
+	// 8. Let format be undefined.
+	let format: VideoSamplePixelFormat | undefined;
+
+	// 9. If options.format does not exist, assign [[format]] to format.
+	if (!options.format || options.format === sample.format) {
+		format = sample.format!;
+	} else if (['RGBA', 'RGBX', 'BGRA', 'BGRX'].includes(options.format)) {
+		// 10. Otherwise, if options.format is equal to one of RGBA, RGBX, BGRA, BGRX, then assign options.format
+		//  to format...
+		format = options.format;
+	} else {
+		throw new Error('NotSupportedError: Invalid destination format.');
+	}
+
+	// 11. Let combinedLayout be the result of running the Compute Layout and Allocation Size algorithm...
+	return ComputeLayoutAndAllocationSize(parsedRect, format, optLayout);
+};
+
+/** Taken from the WebCodecs spec. */
+const ParseVisibleRect = (
+	defaultRect: DOMRectInit,
+	overrideRect: DOMRectInit | undefined,
+	codedWidth: number,
+	codedHeight: number,
+	format: VideoSamplePixelFormat | null,
+): DOMRectInit => {
+	// 1. Let sourceRect be defaultRect
+	const sourceRect = { ...defaultRect };
+
+	// 2. If overrideRect is not undefined:
+	if (overrideRect !== undefined) {
+		// If either of overrideRect.width or height is 0, return a TypeError.
+		if (overrideRect.width === 0 || overrideRect.height === 0) {
+			throw new TypeError('visibleRect dimensions cannot be zero.');
+		}
+		// If the sum of overrideRect.x and overrideRect.width is greater than codedWidth, return a TypeError.
+		if ((overrideRect.x || 0) + (overrideRect.width || 0) > codedWidth) {
+			throw new TypeError('visibleRect exceeds codedWidth.');
+		}
+		// If the sum of overrideRect.y and overrideRect.height is greater than codedHeight, return a TypeError.
+		if ((overrideRect.y || 0) + (overrideRect.height || 0) > codedHeight) {
+			throw new TypeError('visibleRect exceeds codedHeight.');
+		}
+		// Assign overrideRect to sourceRect.
+		sourceRect.x = overrideRect.x || 0;
+		sourceRect.y = overrideRect.y || 0;
+		sourceRect.width = overrideRect.width || 0;
+		sourceRect.height = overrideRect.height || 0;
+	}
+
+	// 3. Let validAlignment be the result of running the Verify Rect Offset Alignment algorithm.
+	const validAlignment = VerifyRectOffsetAlignment(format, sourceRect);
+
+	// 4. If validAlignment is false, throw a TypeError.
+	if (!validAlignment) {
+		throw new TypeError('visibleRect alignment is invalid for the format.');
+	}
+
+	// 5. Return sourceRect.
+	return sourceRect;
+};
+
+/** Taken from the WebCodecs spec. */
+const VerifyRectOffsetAlignment = (format: VideoSamplePixelFormat | null, rect: DOMRectInit): boolean => {
+	// 1. If format is null, return true.
+	if (format === null) return true;
+
+	const planes = getPlaneConfigs(format);
+
+	// 2. Let planeIndex be 0.
+	// 3. Let numPlanes be the number of planes as defined by format.
+	// 4. While planeIndex is less than numPlanes:
+	for (let planeIndex = 0; planeIndex < planes.length; planeIndex++) {
+		const plane = planes[planeIndex]!;
+		const sampleWidth = plane.widthDivisor;
+		const sampleHeight = plane.heightDivisor;
+
+		// If rect.x is not a multiple of sampleWidth, return false.
+		if ((rect.x || 0) % sampleWidth !== 0) return false;
+		// If rect.y is not a multiple of sampleHeight, return false.
+		if ((rect.y || 0) % sampleHeight !== 0) return false;
+	}
+
+	return true;
+};
+
+/** Taken from the WebCodecs spec. */
+const ComputeLayoutAndAllocationSize = (
+	parsedRect: DOMRectInit,
+	format: VideoSamplePixelFormat,
+	layout?: PlaneLayout[],
+): CombinedBufferLayout => {
+	const planes = getPlaneConfigs(format);
+
+	// 1. Let numPlanes be the number of planes as defined by format.
+	const numPlanes = planes.length;
+
+	// 2. If layout is not undefined and its length does not equal numPlanes, throw a TypeError.
+	if (layout !== undefined && layout.length !== numPlanes) {
+		throw new TypeError(`Layout must have ${numPlanes} planes.`);
+	}
+
+	// 3. Let minAllocationSize be 0.
+	let minAllocationSize = 0;
+
+	// 4. Let computedLayouts be a new list.
+	const computedLayouts: ComputedPlaneLayout[] = [];
+
+	// 5. Let endOffsets be a new list.
+	const endOffsets: number[] = [];
+
+	// 6. Let planeIndex be 0.
+	// 7. While planeIndex < numPlanes:
+	for (let planeIndex = 0; planeIndex < numPlanes; planeIndex++) {
+		const plane = planes[planeIndex]!;
+		const sampleBytes = plane.sampleBytes;
+		const sampleWidth = plane.widthDivisor;
+		const sampleHeight = plane.heightDivisor;
+
+		// Let computedLayout be a new computed plane layout.
+		const computedLayout: ComputedPlaneLayout = {
+			destinationOffset: 0,
+			destinationStride: 0,
+			sourceTop: 0,
+			sourceHeight: 0,
+			sourceLeftBytes: 0,
+			sourceWidthBytes: 0,
+		};
+
+		// Set computedLayout’s sourceTop...
+		computedLayout.sourceTop = Math.ceil(Math.trunc(parsedRect.y || 0) / sampleHeight);
+		// Set computedLayout’s sourceHeight...
+		computedLayout.sourceHeight = Math.ceil(Math.trunc(parsedRect.height || 0) / sampleHeight);
+		// Set computedLayout’s sourceLeftBytes...
+		computedLayout.sourceLeftBytes = Math.floor(Math.trunc(parsedRect.x || 0) / sampleWidth) * sampleBytes;
+		// Set computedLayout’s sourceWidthBytes...
+		computedLayout.sourceWidthBytes = Math.floor(Math.trunc(parsedRect.width || 0) / sampleWidth) * sampleBytes;
+
+		// If layout is not undefined:
+		if (layout !== undefined) {
+			const planeLayout = layout[planeIndex]!;
+			// If planeLayout.stride is less than computedLayout’s sourceWidthBytes, return a TypeError.
+			if (planeLayout.stride < computedLayout.sourceWidthBytes) {
+				throw new TypeError(`Stride for plane ${planeIndex} is too small.`);
+			}
+			// Assign planeLayout.offset to computedLayout’s destinationOffset.
+			computedLayout.destinationOffset = planeLayout.offset;
+			// Assign planeLayout.stride to computedLayout’s destinationStride.
+			computedLayout.destinationStride = planeLayout.stride;
+		} else {
+			// Otherwise:
+			// Assign minAllocationSize to computedLayout’s destinationOffset.
+			computedLayout.destinationOffset = minAllocationSize;
+			// Assign computedLayout’s sourceWidthBytes to computedLayout’s destinationStride.
+			computedLayout.destinationStride = computedLayout.sourceWidthBytes;
+		}
+
+		// Let planeSize be the product of multiplying computedLayout’s destinationStride and sourceHeight.
+		const planeSize = computedLayout.destinationStride * computedLayout.sourceHeight;
+
+		// Let planeEnd be the sum of planeSize and computedLayout’s destinationOffset.
+		const planeEnd = planeSize + computedLayout.destinationOffset;
+
+		// If planeSize or planeEnd is greater than maximum range of unsigned long, return a TypeError.
+		if (planeEnd > 4294967295) {
+			throw new TypeError('Allocation size exceeds limit.');
+		}
+
+		// Append planeEnd to endOffsets.
+		endOffsets.push(planeEnd);
+
+		// Assign the maximum of minAllocationSize and planeEnd to minAllocationSize.
+		minAllocationSize = Math.max(minAllocationSize, planeEnd);
+
+		// Check for overlap
+		for (let earlierPlaneIndex = 0; earlierPlaneIndex < planeIndex; earlierPlaneIndex++) {
+			const earlierLayout = computedLayouts[earlierPlaneIndex]!;
+			// If plane A ends before plane B starts, they do not overlap.
+			if (
+				endOffsets[planeIndex]! <= earlierLayout.destinationOffset
+				|| endOffsets[earlierPlaneIndex]! <= computedLayout.destinationOffset
+			) {
+				continue;
+			}
+
+			throw new TypeError('Planes overlap.');
+		}
+
+		computedLayouts.push(computedLayout);
+	}
+
+	// 12. Return combinedLayout.
+	return {
+		allocationSize: minAllocationSize,
+		computedLayouts: computedLayouts,
+	};
+};
+
 const AUDIO_SAMPLE_FORMATS = new Set<AudioSampleFormat>(
 	['f32', 'f32-planar', 's16', 's16-planar', 's32', 's32-planar', 'u8', 'u8-planar'],
 );

test/browser/video-samples.test.ts

@@ -154,7 +154,7 @@ test('copyTo and plane layouts', async () => {
 
 		expect(layout).toEqual([{
 			offset: 0,
-			stride: 1300 * 4,
+			stride: 1280 * 4,
 		}]);
 
 		using clone = sample.clone();
@@ -162,7 +162,7 @@ test('copyTo and plane layouts', async () => {
 
 		expect(clonedLayout).toEqual([{
 			offset: 0,
-			stride: 1300 * 4,
+			stride: 1280 * 4,
 		}]);
 	}
 
@@ -186,6 +186,184 @@ test('copyTo and plane layouts', async () => {
 			offset: 1280 * 720 + (1280 / 2) * (720 / 2),
 			stride: 1280 / 2,
 		}]);
+
+		const rgbLayout = await sample.copyTo(buffer, { format: 'RGBA' });
+
+		expect(rgbLayout).toEqual([{
+			offset: 0,
+			stride: 4 * 1280,
+		}]);
+	}
+});
+
+test('RGB conversion for ArrayBuffer-backed data', async () => {
+	// 2x2 RGBA image with distinct, easily-verifiable pixels
+	const src = new Uint8Array([
+		10, 20, 30, 255, 40, 50, 60, 255,
+		70, 80, 90, 255, 100, 110, 120, 255,
+	]);
+
+	{
+		// RGBA -> BGRA: R and B should swap
+		using sample = new VideoSample(src.slice(), {
+			timestamp: 0,
+			codedWidth: 2,
+			codedHeight: 2,
+			format: 'RGBA',
+		});
+
+		const dest = new Uint8Array(2 * 2 * 4);
+		const layout = await sample.copyTo(dest, { format: 'BGRA' });
+
+		expect(layout).toEqual([{ offset: 0, stride: 2 * 4 }]);
+		expect(Array.from(dest)).toEqual([
+			30, 20, 10, 255, 60, 50, 40, 255,
+			90, 80, 70, 255, 120, 110, 100, 255,
+		]);
+	}
+
+	{
+		// RGBA -> RGBA: no swap, bytes copied through unchanged
+		using sample = new VideoSample(src.slice(), {
+			timestamp: 0,
+			codedWidth: 2,
+			codedHeight: 2,
+			format: 'RGBA',
+		});
+
+		const dest = new Uint8Array(2 * 2 * 4);
+		await sample.copyTo(dest, { format: 'RGBA' });
+
+		expect(Array.from(dest)).toEqual(Array.from(src));
+	}
+
+	{
+		// RGBA -> BGRX: R and B should swap
+		using sample = new VideoSample(src.slice(), {
+			timestamp: 0,
+			codedWidth: 2,
+			codedHeight: 2,
+			format: 'RGBA',
+		});
+
+		const dest = new Uint8Array(2 * 2 * 4);
+		await sample.copyTo(dest, { format: 'BGRX' });
+
+		expect(Array.from(dest)).toEqual([
+			30, 20, 10, 255, 60, 50, 40, 255,
+			90, 80, 70, 255, 120, 110, 100, 255,
+		]);
+	}
+
+	{
+		// BGRA -> RGBA: R and B should swap
+		using sample = new VideoSample(src.slice(), {
+			timestamp: 0,
+			codedWidth: 2,
+			codedHeight: 2,
+			format: 'BGRA',
+		});
+
+		const dest = new Uint8Array(2 * 2 * 4);
+		await sample.copyTo(dest, { format: 'RGBA' });
+
+		expect(Array.from(dest)).toEqual([
+			30, 20, 10, 255, 60, 50, 40, 255,
+			90, 80, 70, 255, 120, 110, 100, 255,
+		]);
+	}
+
+	{
+		// BGRA -> BGRA: no swap
+		using sample = new VideoSample(src.slice(), {
+			timestamp: 0,
+			codedWidth: 2,
+			codedHeight: 2,
+			format: 'BGRA',
+		});
+
+		const dest = new Uint8Array(2 * 2 * 4);
+		await sample.copyTo(dest, { format: 'BGRA' });
+
+		expect(Array.from(dest)).toEqual(Array.from(src));
+	}
+});
+
+test('crop (rect option) for ArrayBuffer-backed data', async () => {
+	// 4x4 RGBA image where each pixel's R/G channels encode (x, y)
+	const width = 4;
+	const height = 4;
+	const src = new Uint8Array(width * height * 4);
+	for (let y = 0; y < height; y++) {
+		for (let x = 0; x < width; x++) {
+			const i = (y * width + x) * 4;
+			src[i] = x;
+			src[i + 1] = y;
+			src[i + 2] = 0;
+			src[i + 3] = 255;
+		}
+	}
+
+	{
+		// Crop a 2x2 region at (1, 1)
+		using sample = new VideoSample(src.slice(), {
+			timestamp: 0,
+			codedWidth: width,
+			codedHeight: height,
+			format: 'RGBA',
+		});
+
+		const size = sample.allocationSize({ rect: { x: 1, y: 1, width: 2, height: 2 } });
+		expect(size).toBe(2 * 2 * 4);
+
+		const dest = new Uint8Array(size);
+		const layout = await sample.copyTo(dest, { rect: { x: 1, y: 1, width: 2, height: 2 } });
+
+		expect(layout).toEqual([{ offset: 0, stride: 2 * 4 }]);
+		expect(Array.from(dest)).toEqual([
+			1, 1, 0, 255, 2, 1, 0, 255,
+			1, 2, 0, 255, 2, 2, 0, 255,
+		]);
+	}
+
+	{
+		// Crop an offset rect at the top-right corner
+		using sample = new VideoSample(src.slice(), {
+			timestamp: 0,
+			codedWidth: width,
+			codedHeight: height,
+			format: 'RGBA',
+		});
+
+		const dest = new Uint8Array(2 * 1 * 4);
+		const layout = await sample.copyTo(dest, { rect: { x: 2, y: 0, width: 2, height: 1 } });
+
+		expect(layout).toEqual([{ offset: 0, stride: 2 * 4 }]);
+		expect(Array.from(dest)).toEqual([
+			2, 0, 0, 255, 3, 0, 0, 255,
+		]);
+	}
+
+	{
+		// Crop combined with a custom destination stride (padding between rows)
+		using sample = new VideoSample(src.slice(), {
+			timestamp: 0,
+			codedWidth: width,
+			codedHeight: height,
+			format: 'RGBA',
+		});
+
+		const stride = 3 * 4; // wider than the crop, leaving trailing padding
+		const dest = new Uint8Array(stride * 2);
+		const layout = await sample.copyTo(dest, {
+			rect: { x: 0, y: 2, width: 2, height: 2 },
+			layout: [{ offset: 0, stride }],
+		});
+
+		expect(layout).toEqual([{ offset: 0, stride }]);
+		// Row 0 of crop (y=2): pixels (0,2) and (1,2), then 4 bytes of untouched padding
+		expect(Array.from(dest.subarray(0, 8))).toEqual([0, 2, 0, 255, 1, 2, 0, 255]);
+		expect(Array.from(dest.subarray(stride, stride + 8))).toEqual([0, 3, 0, 255, 1, 3, 0, 255]);
 	}
 });
 
@@ -201,12 +379,7 @@ test('null format', async () => {
 	expect(() => sample.allocationSize()).toThrow('when format is null');
 	await expect(async () => sample.copyTo(new ArrayBuffer())).rejects.toThrow('when format is null');
 
-	// Even this throws :(
+	// BUT: See https://github.com/Vanilagy/mediabunny/issues/267
-	// See https://github.com/Vanilagy/mediabunny/issues/267
-	expect(() => sample.allocationSize({ format: 'RGBA' })).toThrow('when format is null');
-
-	// Uncomment this if the RGBA conversion works again:
-	/*
 	const size = sample.allocationSize({ format: 'RGBA' });
 	expect(size).toBe(1280 * 720 * 4);
 	const buffer = new ArrayBuffer(size);
@@ -223,5 +396,4 @@ test('null format', async () => {
 	await sample.copyTo(buffer, { format: 'RGBX' });
 	await sample.copyTo(buffer, { format: 'BGRA' });
 	await sample.copyTo(buffer, { format: 'BGRX' });
-	*/
 });

test/node/server-extension.test.ts

@@ -7,7 +7,8 @@ import { assert, toUint8Array } from '../../src/misc.js';
 import { EncodedPacketSink, VideoSampleSink } from '../../src/media-sink.js';
 import { NodeAvVideoDecoder } from '../../packages/server/src/video-decoder.js';
 import { NodeAvVideoEncoder } from '../../packages/server/src/video-encoder.js';
-import { VideoSample } from '../../src/sample.js';
+import { NodeAvAudioDecoder } from '../../packages/server/src/audio-decoder.js';
+import { AudioSample, VideoSample } from '../../src/sample.js';
 import { buildVideoCodecString, VideoCodec } from '../../src/codec.js';
 import { EncodedPacket } from '../../src/packet.js';
 import {
@@ -122,12 +123,6 @@ describe('Video', async () => {
 				expect(meta.decoderConfig!.codec.startsWith('avc1.')).toBe(true);
 				expect(meta.decoderConfig!.codedWidth).toBe(1280);
 				expect(meta.decoderConfig!.codedHeight).toBe(720);
-				expect(meta.decoderConfig!.colorSpace).toEqual({
-					primaries: 'bt709',
-					transfer: 'iec61966-2-1',
-					matrix: 'rgb',
-					fullRange: true,
-				});
 				expect(meta.decoderConfig!.description).toBeDefined();
 			}
 
@@ -208,12 +203,6 @@ describe('Video', async () => {
 			expect(sample.codedHeight).toBe(720);
 			expect(sample.timestamp).toBe(i / 30);
 			expect(sample.duration).toBe(1 / 30);
-			expect(sample.colorSpace).toEqual({
-				primaries: 'bt709',
-				transfer: 'iec61966-2-1',
-				matrix: 'rgb',
-				fullRange: true,
-			});
 
 			const buf = new Uint8Array(sample.allocationSize({ format: 'RGBX' }));
 			await sample.copyTo(buf, { format: 'RGBX' });
@@ -248,12 +237,6 @@ describe('Video', async () => {
 			expect(sample.codedHeight).toBe(720);
 			expect(sample.timestamp).toBe(i / 30);
 			expect(sample.duration).toBe(1 / 30);
-			expect(sample.colorSpace).toEqual({
-				primaries: 'bt709',
-				transfer: 'iec61966-2-1',
-				matrix: 'rgb',
-				fullRange: true,
-			});
 
 			const buf = new Uint8Array(sample.allocationSize({ format: 'RGBX' }));
 			await sample.copyTo(buf, { format: 'RGBX' });
@@ -292,14 +275,6 @@ describe('Video', async () => {
 			expect(sample.timestamp).toBe(i / 30);
 			expect(sample.duration).toBe(1 / 30);
 
-			// Undefined, for some reason:
-			expect(sample.colorSpace).toEqual({
-				primaries: null,
-				transfer: null,
-				matrix: null,
-				fullRange: false,
-			});
-
 			const buf = new Uint8Array(sample.allocationSize({ format: 'RGBX' }));
 			await sample.copyTo(buf, { format: 'RGBX' });
 
@@ -336,14 +311,6 @@ describe('Video', async () => {
 			expect(sample.timestamp).toBe(i / 30);
 			expect(sample.duration).toBe(1 / 30);
 
-			// Undefined, for some reason:
-			expect(sample.colorSpace).toEqual({
-				primaries: null,
-				transfer: null,
-				matrix: null,
-				fullRange: false,
-			});
-
 			const buf = new Uint8Array(sample.allocationSize({ format: 'RGBX' }));
 			await sample.copyTo(buf, { format: 'RGBX' });
 
@@ -368,12 +335,6 @@ describe('Video', async () => {
 			expect(sample.codedHeight).toBe(720);
 			expect(sample.timestamp).toBe(i / 30);
 			expect(sample.duration).toBe(1 / 30);
-			expect(sample.colorSpace).toEqual({
-				primaries: 'bt709',
-				transfer: 'iec61966-2-1',
-				matrix: 'bt470bg',
-				fullRange: false,
-			});
 
 			const buf = new Uint8Array(sample.allocationSize({ format: 'RGBX' }));
 			await sample.copyTo(buf, { format: 'RGBX' });
@@ -399,12 +360,6 @@ describe('Video', async () => {
 			expect(sample.codedHeight).toBe(720);
 			expect(sample.timestamp).toBe(i / 30);
 			expect(sample.duration).toBe(1 / 30);
-			expect(sample.colorSpace).toEqual({
-				primaries: 'bt709',
-				transfer: 'iec61966-2-1',
-				matrix: null,
-				fullRange: false,
-			});
 
 			const buf = new Uint8Array(sample.allocationSize({ format: 'RGBX' }));
 			await sample.copyTo(buf, { format: 'RGBX' });
@@ -430,12 +385,6 @@ describe('Video', async () => {
 			expect(sample.codedHeight).toBe(720);
 			expect(sample.timestamp).toBe(i / 30);
 			expect(sample.duration).toBe(1 / 30);
-			expect(sample.colorSpace).toEqual({
-				primaries: 'bt709',
-				transfer: 'iec61966-2-1',
-				matrix: 'rgb',
-				fullRange: false,
-			});
 
 			const buf = new Uint8Array(sample.allocationSize({ format: 'RGBX' }));
 			await sample.copyTo(buf, { format: 'RGBX' });
@@ -668,12 +617,6 @@ describe('Video', async () => {
 		expect(sample.rotation).toBe(0);
 		expect(sample.timestamp).toBe(0);
 		expect(sample.duration).toBe(1 / 25);
-		expect(sample.colorSpace).toEqual({
-			primaries: null,
-			transfer: null,
-			matrix: 'bt470bg',
-			fullRange: true,
-		});
 
 		// Default expected YUV size
 		expect(sample.allocationSize()).toBe(1920 * 1080 * 1.5);
@@ -792,3 +735,77 @@ describe('Video', async () => {
 		]);
 	});
 });
+
+/*
+describe('Audio', async () => {
+	test('Decoder lifecycle', async () => {
+		using input = new Input({
+			source: new FilePathSource('./test/public/trim-buck-bunny-ffmpeg.ts'),
+			formats: ALL_FORMATS,
+		});
+
+		const audioTrack = await input.getPrimaryAudioTrack();
+		assert(audioTrack);
+
+		const decoder = new NodeAvAudioDecoder();
+		// @ts-expect-error Readonly
+		decoder.codec = await audioTrack.getCodec();
+		// @ts-expect-error Readonly
+		decoder.config = await audioTrack.getDecoderConfig();
+
+		let sampleCount = 0;
+		const packetTimestamps: number[] = [];
+
+		// @ts-expect-error Readonly
+		decoder.onSample = (sample: AudioSample) => {
+			expect(sample.timestamp).toBe(packetTimestamps[sampleCount]);
+
+			if (sampleCount > 0) {
+				expect(sample.duration).toBeCloseTo(
+					packetTimestamps[sampleCount]! - packetTimestamps[sampleCount - 1]!,
+				);
+			}
+
+			sampleCount++;
+			sample.close();
+		};
+
+		await decoder.init();
+
+		const sink = new EncodedPacketSink(audioTrack);
+		let packetCount = 0;
+		for await (const packet of sink.packets()) {
+			packetTimestamps.push(packet.timestamp);
+			await decoder.decode(packet);
+
+			if (++packetCount === 10) {
+				break;
+			}
+		}
+
+		await decoder.flush();
+
+		expect(sampleCount).toBe(10);
+
+		// And, go again
+		sampleCount = 0;
+		packetTimestamps.length = 0;
+
+		packetCount = 0;
+		for await (const packet of sink.packets((await sink.getKeyPacket(5))!)) {
+			packetTimestamps.push(packet.timestamp);
+			await decoder.decode(packet);
+
+			if (++packetCount === 10) {
+				break;
+			}
+		}
+
+		await decoder.flush();
+
+		expect(sampleCount).toBe(10);
+
+		await decoder.close();
+	});
+});
+*/