Add options to VideoSample allocationSize and copyTo methods, add explicit error message when format is null (closes #256)

docs/guide/packets-and-samples.md

@@ -378,13 +378,11 @@ const bytesNeeded = videoSample.allocationSize(); // => number
 Then, use `copyTo` to copy the pixel data into the destination buffer:
 ```ts
 const bytes = new Uint8Array(bytesNeeded);
-videoSample.copyTo(bytes);
+const planeLayout = await videoSample.copyTo(bytes);
 ```
 
 ::: info
-The data will always be in the pixel format specified in the `format` field.
-
-To convert the data into a different pixel format, or to extract only a section of the frame, please use the [`allocationSize`](https://developer.mozilla.org/en-US/docs/Web/API/VideoFrame/allocationSize) and [`copyTo`](https://developer.mozilla.org/en-US/docs/Web/API/VideoFrame/copyTo) methods on `VideoFrame` instead. Get a `VideoFrame` by running `videoSample.toVideoFrame()`.
+You can pass additional options to `allocationSize` and `copyTo` to extract data in a different pixel format.
 :::
 
 ---

src/index.ts

@@ -168,8 +168,10 @@ export {
 	AudioSampleCopyToOptions,
 	VideoSample,
 	VideoSampleInit,
+	VideoSamplePixelFormat,
 	VideoSampleColorSpace,
 	CropRectangle,
+	VIDEO_SAMPLE_PIXEL_FORMATS,
 } from './sample';
 export {
 	AudioBufferSink,

src/sample.ts

@@ -17,6 +17,7 @@ import {
 	SetRequired,
 	isFirefox,
 	polyfillSymbolDispose,
+	assertNever,
 } from './misc';
 
 polyfillSymbolDispose();
@@ -69,6 +70,57 @@ if (typeof FinalizationRegistry !== 'undefined') {
 	});
 }
 
+/**
+ * The list of {@link VideoSample} pixel formats.
+ * @group Samples
+ * @public
+ */
+export const VIDEO_SAMPLE_PIXEL_FORMATS = [
+	// 4:2:0 Y, U, V
+	'I420',
+	'I420P10',
+	'I420P12',
+	// 4:2:0 Y, U, V, A
+	'I420A',
+	'I420AP10',
+	'I420AP12',
+	// 4:2:2 Y, U, V
+	'I422',
+	'I422P10',
+	'I422P12',
+	// 4:2:2 Y, U, V, A
+	'I422A',
+	'I422AP10',
+	'I422AP12',
+	// 4:4:4 Y, U, V
+	'I444',
+	'I444P10',
+	'I444P12',
+	// 4:4:4 Y, U, V, A
+	'I444A',
+	'I444AP10',
+	'I444AP12',
+	// 4:2:0 Y, UV
+	'NV12',
+	// 4:4:4 RGBA
+	'RGBA',
+	// 4:4:4 RGBX (opaque)
+	'RGBX',
+	// 4:4:4 BGRA
+	'BGRA',
+	// 4:4:4 BGRX (opaque)
+	'BGRX',
+] as const;
+const VIDEO_SAMPLE_PIXEL_FORMATS_SET = new Set(VIDEO_SAMPLE_PIXEL_FORMATS);
+
+/**
+ * The internal pixel format with which a {@link VideoSample} is stored.
+ * [See pixel formats](https://www.w3.org/TR/webcodecs/#pixel-format) for more.
+ * @group Samples
+ * @public
+ */
+export type VideoSamplePixelFormat = typeof VIDEO_SAMPLE_PIXEL_FORMATS[number];
+
 /**
  * Metadata used for VideoSample initialization.
  * @group Samples
@@ -77,9 +129,9 @@ if (typeof FinalizationRegistry !== 'undefined') {
 export type VideoSampleInit = {
 	/**
 	 * The internal pixel format in which the frame is stored.
-	 * [See pixel formats](https://developer.mozilla.org/en-US/docs/Web/API/VideoFrame/format)
+	 * [See pixel formats](https://www.w3.org/TR/webcodecs/#pixel-format)
 	 */
-	format?: VideoPixelFormat;
+	format?: VideoSamplePixelFormat;
 	/** The width of the frame in pixels. */
 	codedWidth?: number;
 	/** The height of the frame in pixels. */
@@ -92,6 +144,8 @@ export type VideoSampleInit = {
 	duration?: number;
 	/** The color space of the frame. */
 	colorSpace?: VideoColorSpaceInit;
+	/** The byte layout of the planes of the frame. */
+	layout?: PlaneLayout[];
 };
 
 /**
@@ -103,14 +157,20 @@ export type VideoSampleInit = {
 export class VideoSample implements Disposable {
 	/** @internal */
 	_data!: VideoFrame | OffscreenCanvas | Uint8Array | null;
+	/**
+	 * Used for the ArrayBuffer-backed case.
+	 * @internal
+	 */
+	_layout!: PlaneLayout[] | null;
 	/** @internal */
 	_closed: boolean = false;
 
 	/**
-	 * The internal pixel format in which the frame is stored.
-	 * [See pixel formats](https://developer.mozilla.org/en-US/docs/Web/API/VideoFrame/format)
+	 * The internal pixel format in which the frame is stored. Will be `null` if it's using an arbitrary internal
+	 * format not representable by `VideoPixelFormat`.
+	 * [See pixel formats](https://www.w3.org/TR/webcodecs/#pixel-format)
 	 */
-	readonly format!: VideoPixelFormat | null;
+	readonly format!: VideoSamplePixelFormat | null;
 	/** The width of the frame in pixels. */
 	readonly codedWidth!: number;
 	/** The height of the frame in pixels. */
@@ -189,8 +249,8 @@ export class VideoSample implements Disposable {
 			if (!init || typeof init !== 'object') {
 				throw new TypeError('init must be an object.');
 			}
-			if (!('format' in init) || typeof init.format !== 'string') {
-				throw new TypeError('init.format must be a string.');
+			if (init.format === undefined || !VIDEO_SAMPLE_PIXEL_FORMATS_SET.has(init.format)) {
+				throw new TypeError('init.format must be one of: ' + VIDEO_SAMPLE_PIXEL_FORMATS.join(', '));
 			}
 			if (!Number.isInteger(init.codedWidth) || init.codedWidth! <= 0) {
 				throw new TypeError('init.codedWidth must be a positive integer.');
@@ -209,6 +269,7 @@ export class VideoSample implements Disposable {
 			}
 
 			this._data = toUint8Array(data).slice(); // Copy it
+			this._layout = init.layout ?? createDefaultPlaneLayout(init.format, init.codedWidth!, init.codedHeight!);
 
 			this.format = init.format;
 			this.codedWidth = init.codedWidth!;
@@ -229,6 +290,7 @@ export class VideoSample implements Disposable {
 			}
 
 			this._data = data;
+			this._layout = null;
 
 			this.format = data.format;
 			// Copying the display dimensions here, assuming no innate VideoFrame rotation
@@ -301,6 +363,7 @@ export class VideoSample implements Disposable {
 			// Draw it to a canvas
 			context.drawImage(data, 0, 0);
 			this._data = canvas;
+			this._layout = null;
 
 			this.format = 'RGBX';
 			this.codedWidth = width;
@@ -336,8 +399,11 @@ export class VideoSample implements Disposable {
 				rotation: this.rotation,
 			});
 		} else if (this._data instanceof Uint8Array) {
-			return new VideoSample(this._data.slice(), {
+			assert(this._layout);
+
+			return new VideoSample(this._data, {
 				format: this.format!,
+				layout: this._layout,
 				codedWidth: this.codedWidth,
 				codedHeight: this.codedHeight,
 				timestamp: this.timestamp,
@@ -378,16 +444,43 @@ export class VideoSample implements Disposable {
 		this._closed = true;
 	}
 
-	/** Returns the number of bytes required to hold this video sample's pixel data. */
-	allocationSize() {
+	/**
+	 * Returns the number of bytes required to hold this video sample's pixel data. Throws if `format` is `null`;
+	 * specify an explicit RGB format in the options in this case.
+	 */
+	allocationSize(options: VideoFrameCopyToOptions = {}): number {
+		validateVideoFrameCopyToOptions(options);
+
 		if (this._closed) {
 			throw new Error('VideoSample is closed.');
 		}
+		if ((options.format ?? this.format) === null) {
+			throw new Error(
+				'Cannot get allocation size when format is null. Please manually provide an RGB pixel format in the'
+				+ ' options instead.',
+			);
+		}
 
 		assert(this._data !== null);
 
+		if (!isVideoFrame(this._data)) {
+			if (
+				options.colorSpace
+				|| (options.format && options.format !== this.format)
+				|| options.layout
+				|| options.rect
+			) {
+				// Temporarily convert to VideoFrame to get it done
+				const videoFrame = this.toVideoFrame();
+				const size = videoFrame.allocationSize(options);
+				videoFrame.close();
+
+				return size;
+			}
+		}
+
 		if (isVideoFrame(this._data)) {
-			return this._data.allocationSize();
+			return this._data.allocationSize(options);
 		} else if (this._data instanceof Uint8Array) {
 			return this._data.byteLength;
 		} else {
@@ -395,23 +488,54 @@ export class VideoSample implements Disposable {
 		}
 	}
 
-	/** Copies this video sample's pixel data to an ArrayBuffer or ArrayBufferView. */
-	async copyTo(destination: AllowSharedBufferSource) {
+	/**
+	 * Copies this video sample's pixel data to an ArrayBuffer or ArrayBufferView. Throws if `format` is `null`;
+	 * specify an explicit RGB format in the options in this case.
+	 * @returns The byte layout of the planes of the copied data.
+	 */
+	async copyTo(destination: AllowSharedBufferSource, options: VideoFrameCopyToOptions = {}): Promise<PlaneLayout[]> {
 		if (!isAllowSharedBufferSource(destination)) {
 			throw new TypeError('destination must be an ArrayBuffer or an ArrayBuffer view.');
 		}
+		validateVideoFrameCopyToOptions(options);
 
 		if (this._closed) {
 			throw new Error('VideoSample is closed.');
 		}
+		if ((options.format ?? this.format) === null) {
+			throw new Error(
+				'Cannot copy video sample data when format is null. Please manually provide an RGB pixel format in the'
+				+ ' options instead.',
+			);
+		}
 
 		assert(this._data !== null);
 
+		if (!isVideoFrame(this._data)) {
+			if (
+				options.colorSpace
+				|| (options.format && options.format !== this.format)
+				|| options.layout
+				|| options.rect
+			) {
+				// Temporarily convert to VideoFrame to get it done
+				const videoFrame = this.toVideoFrame();
+				const layout = await videoFrame.copyTo(destination, options);
+				videoFrame.close();
+
+				return layout;
+			}
+		}
+
 		if (isVideoFrame(this._data)) {
-			await this._data.copyTo(destination);
+			return this._data.copyTo(destination, options);
 		} else if (this._data instanceof Uint8Array) {
+			assert(this._layout);
+
 			const dest = toUint8Array(destination);
 			dest.set(this._data);
+
+			return this._layout;
 		} else {
 			const canvas = this._data;
 			const context = canvas.getContext('2d');
@@ -420,6 +544,11 @@ export class VideoSample implements Disposable {
 			const imageData = context.getImageData(0, 0, this.codedWidth, this.codedHeight);
 			const dest = toUint8Array(destination);
 			dest.set(imageData.data);
+
+			return [{
+				offset: 0,
+				stride: 4 * this.codedWidth,
+			}];
 		}
 	}
 
@@ -441,7 +570,7 @@ export class VideoSample implements Disposable {
 			});
 		} else if (this._data instanceof Uint8Array) {
 			return new VideoFrame(this._data, {
-				format: this.format!,
+				format: this.format! as VideoPixelFormat,
 				codedWidth: this.codedWidth,
 				codedHeight: this.codedHeight,
 				timestamp: this.microsecondTimestamp,
@@ -887,7 +1016,168 @@ export const validateCropRectangle = (crop: CropRectangle, prefix: string) => {
 	}
 };
 
-const AUDIO_SAMPLE_FORMATS = new Set(
+const validateVideoFrameCopyToOptions = (options: VideoFrameCopyToOptions) => {
+	if (!options || typeof options !== 'object') {
+		throw new TypeError('options must be an object.');
+	}
+	if (options.colorSpace !== undefined && !['display-p3', 'srgb'].includes(options.colorSpace)) {
+		throw new TypeError('options.colorSpace, when provided, must be \'display-p3\' or \'srgb\'.');
+	}
+	if (options.format !== undefined && typeof options.format !== 'string') {
+		throw new TypeError('options.format, when provided, must be a string.');
+	}
+	if (options.layout !== undefined) {
+		if (!Array.isArray(options.layout)) {
+			throw new TypeError('options.layout, when provided, must be an array.');
+		}
+
+		for (const plane of options.layout) {
+			if (!plane || typeof plane !== 'object') {
+				throw new TypeError('Each entry in options.layout must be an object.');
+			}
+			if (!Number.isInteger(plane.offset) || plane.offset < 0) {
+				throw new TypeError('plane.offset must be a non-negative integer.');
+			}
+			if (!Number.isInteger(plane.stride) || plane.stride < 0) {
+				throw new TypeError('plane.stride must be a non-negative integer.');
+			}
+		}
+	}
+	if (options.rect !== undefined) {
+		if (!options.rect || typeof options.rect !== 'object') {
+			throw new TypeError('options.rect, when provided, must be an object.');
+		}
+		if (options.rect.x !== undefined && (!Number.isInteger(options.rect.x) || options.rect.x < 0)) {
+			throw new TypeError('options.rect.x, when provided, must be a non-negative integer.');
+		}
+		if (options.rect.y !== undefined && (!Number.isInteger(options.rect.y) || options.rect.y < 0)) {
+			throw new TypeError('options.rect.y, when provided, must be a non-negative integer.');
+		}
+		if (options.rect.width !== undefined && (!Number.isInteger(options.rect.width) || options.rect.width < 0)) {
+			throw new TypeError('options.rect.width, when provided, must be a non-negative integer.');
+		}
+		if (options.rect.height !== undefined && (!Number.isInteger(options.rect.height) || options.rect.height < 0)) {
+			throw new TypeError('options.rect.height, when provided, must be a non-negative integer.');
+		}
+	}
+};
+
+/** Implements logic from WebCodecs § 9.4.6 "Compute Layout and Allocation Size" */
+const createDefaultPlaneLayout = (
+	format: VideoSamplePixelFormat,
+	codedWidth: number,
+	codedHeight: number,
+): PlaneLayout[] => {
+	const planes = getPlaneConfigs(format);
+	const layouts: PlaneLayout[] = [];
+	let currentOffset = 0;
+
+	for (const plane of planes) {
+		// Per § 9.8, dimensions are usually "rounded up to the nearest integer".
+		const planeWidth = Math.ceil(codedWidth / plane.widthDivisor);
+		const planeHeight = Math.ceil(codedHeight / plane.heightDivisor);
+
+		const stride = planeWidth * plane.sampleBytes;
+
+		// Tight packing
+		const planeSize = stride * planeHeight;
+
+		layouts.push({
+			offset: currentOffset,
+			stride: stride,
+		});
+
+		currentOffset += planeSize;
+	}
+
+	return layouts;
+};
+
+type PlaneConfig = {
+	sampleBytes: number;
+	widthDivisor: number; // Horizontal sub-sampling factor
+	heightDivisor: number; // Vertical sub-sampling factor
+};
+
+/** Helper to retrieve plane configurations based on WebCodecs § 9.8 Pixel Format definitions. */
+const getPlaneConfigs = (format: VideoSamplePixelFormat): PlaneConfig[] => {
+	// Helper for standard YUV planes
+	const yuv = (
+		yBytes: number,
+		uvBytes: number,
+		subX: number,
+		subY: number,
+		hasAlpha: boolean,
+	): PlaneConfig[] => {
+		const configs: PlaneConfig[] = [
+			{ sampleBytes: yBytes, widthDivisor: 1, heightDivisor: 1 },
+			{ sampleBytes: uvBytes, widthDivisor: subX, heightDivisor: subY },
+			{ sampleBytes: uvBytes, widthDivisor: subX, heightDivisor: subY },
+		];
+
+		if (hasAlpha) {
+			// Match luma dimensions
+			configs.push({ sampleBytes: yBytes, widthDivisor: 1, heightDivisor: 1 });
+		}
+
+		return configs;
+	};
+
+	switch (format) {
+		case 'I420':
+			return yuv(1, 1, 2, 2, false);
+		case 'I420P10':
+		case 'I420P12':
+			return yuv(2, 2, 2, 2, false);
+		case 'I420A':
+			return yuv(1, 1, 2, 2, true);
+		case 'I420AP10':
+		case 'I420AP12':
+			return yuv(2, 2, 2, 2, true);
+
+		case 'I422':
+			return yuv(1, 1, 2, 1, false);
+		case 'I422P10':
+		case 'I422P12':
+			return yuv(2, 2, 2, 1, false);
+		case 'I422A':
+			return yuv(1, 1, 2, 1, true);
+		case 'I422AP10':
+		case 'I422AP12':
+			return yuv(2, 2, 2, 1, true);
+
+		case 'I444':
+			return yuv(1, 1, 1, 1, false);
+		case 'I444P10':
+		case 'I444P12':
+			return yuv(2, 2, 1, 1, false);
+		case 'I444A':
+			return yuv(1, 1, 1, 1, true);
+		case 'I444AP10':
+		case 'I444AP12':
+			return yuv(2, 2, 1, 1, true);
+
+		case 'NV12':
+			return [
+				{ sampleBytes: 1, widthDivisor: 1, heightDivisor: 1 },
+				{ sampleBytes: 2, widthDivisor: 2, heightDivisor: 2 }, // Interleaved U and V
+			];
+
+		case 'RGBA':
+		case 'RGBX':
+		case 'BGRA':
+		case 'BGRX':
+			return [
+				{ sampleBytes: 4, widthDivisor: 1, heightDivisor: 1 },
+			];
+
+		default:
+			assertNever(format);
+			assert(false);
+	}
+};
+
+const AUDIO_SAMPLE_FORMATS = new Set<AudioSampleFormat>(
 	['f32', 'f32-planar', 's16', 's16-planar', 's32', 's32-planar', 'u8', 'u8-planar'],
 );
 

test/browser/video-samples.test.ts

@@ -0,0 +1,220 @@
+import { expect, test } from 'vitest';
+import { VideoSample } from '../../src/sample.js';
+
+test('allocationSize', async () => {
+	{
+		const canvas = new OffscreenCanvas(1280, 720);
+		canvas.getContext('2d');
+
+		using sample = new VideoSample(canvas, { timestamp: 0 });
+
+		const size1 = sample.allocationSize();
+		expect(size1).toBe(1280 * 720 * 4);
+
+		const size2 = sample.allocationSize({
+			layout: [{
+				offset: 0,
+				stride: 1300 * 4,
+			}],
+		});
+		expect(size2).toBe(1300 * 720 * 4);
+	}
+
+	{
+		const data = new Uint8Array(1280 * 720 * 4);
+		using sample = new VideoSample(data, {
+			timestamp: 0,
+			codedWidth: 1280,
+			codedHeight: 720,
+			format: 'RGBA',
+		});
+
+		const size1 = sample.allocationSize();
+		expect(size1).toBe(1280 * 720 * 4);
+
+		const size2 = sample.allocationSize({
+			layout: [{
+				offset: 0,
+				stride: 1300 * 4,
+			}],
+		});
+		expect(size2).toBe(1300 * 720 * 4);
+	}
+
+	{
+		const data = new Uint8Array(1280 * 720 * 1.5);
+		using sample = new VideoSample(data, {
+			timestamp: 0,
+			codedWidth: 1280,
+			codedHeight: 720,
+			format: 'I420',
+		});
+
+		const size1 = sample.allocationSize();
+		expect(size1).toBe(1280 * 720 * 1.5);
+
+		const size2 = sample.allocationSize({ format: 'RGBA' });
+		expect(size2).toBe(1280 * 720 * 4);
+
+		const size3 = sample.allocationSize({
+			format: 'RGBA',
+			layout: [{
+				offset: 0,
+				stride: 1300 * 4,
+			}],
+		});
+		expect(size3).toBe(1300 * 720 * 4);
+	}
+});
+
+test('copyTo and plane layouts', async () => {
+	const buffer = new ArrayBuffer(1e7);
+
+	{
+		const canvas = new OffscreenCanvas(1280, 720);
+		canvas.getContext('2d');
+
+		using sample = new VideoSample(canvas, { timestamp: 0 });
+		const layout = await sample.copyTo(buffer);
+
+		expect(layout).toEqual([{
+			offset: 0,
+			stride: 1280 * 4,
+		}]);
+	}
+
+	{
+		const canvas = new OffscreenCanvas(1280, 720);
+		canvas.getContext('2d');
+
+		using sample = new VideoSample(canvas, { timestamp: 0 });
+		const layout = await sample.copyTo(buffer, {
+			layout: [{
+				offset: 0,
+				stride: 1300 * 4,
+			}],
+		});
+
+		expect(layout).toEqual([{
+			offset: 0,
+			stride: 1300 * 4,
+		}]);
+	}
+
+	{
+		const data = new Uint8Array(1280 * 720 * 4);
+		using sample = new VideoSample(data, {
+			timestamp: 0,
+			codedWidth: 1280,
+			codedHeight: 720,
+			format: 'RGBA',
+		});
+		const layout = await sample.copyTo(buffer);
+
+		expect(layout).toEqual([{
+			offset: 0,
+			stride: 1280 * 4,
+		}]);
+	}
+
+	{
+		const data = new Uint8Array(1280 * 720 * 4);
+		using sample = new VideoSample(data, {
+			timestamp: 0,
+			codedWidth: 1280,
+			codedHeight: 720,
+			format: 'RGBA',
+		});
+		const layout = await sample.copyTo(buffer, {
+			layout: [{
+				offset: 0,
+				stride: 1300 * 4,
+			}],
+		});
+
+		expect(layout).toEqual([{
+			offset: 0,
+			stride: 1300 * 4,
+		}]);
+	}
+
+	{
+		const data = new Uint8Array(1300 * 720 * 4);
+		using sample = new VideoSample(data, {
+			timestamp: 0,
+			codedWidth: 1280,
+			codedHeight: 720,
+			format: 'RGBA',
+			layout: [{
+				offset: 0,
+				stride: 1300 * 4,
+			}],
+		});
+		const layout = await sample.copyTo(buffer);
+
+		expect(layout).toEqual([{
+			offset: 0,
+			stride: 1300 * 4,
+		}]);
+
+		using clone = sample.clone();
+		const clonedLayout = await clone.copyTo(buffer);
+
+		expect(clonedLayout).toEqual([{
+			offset: 0,
+			stride: 1300 * 4,
+		}]);
+	}
+
+	{
+		const data = new Uint8Array(1280 * 720 * 1.5);
+		using sample = new VideoSample(data, {
+			timestamp: 0,
+			codedWidth: 1280,
+			codedHeight: 720,
+			format: 'I420',
+		});
+		const layout = await sample.copyTo(buffer);
+
+		expect(layout).toEqual([{
+			offset: 0,
+			stride: 1280,
+		}, {
+			offset: 1280 * 720,
+			stride: 1280 / 2,
+		}, {
+			offset: 1280 * 720 + (1280 / 2) * (720 / 2),
+			stride: 1280 / 2,
+		}]);
+	}
+});
+
+test('null format', async () => {
+	const canvas = new OffscreenCanvas(1280, 720);
+	canvas.getContext('2d');
+
+	const frame = new VideoFrame(canvas, { timestamp: 0 });
+	using sample = new VideoSample(frame);
+	// @ts-expect-error Sybau
+	sample.format = null;
+
+	expect(() => sample.allocationSize()).toThrow('when format is null');
+	await expect(async () => sample.copyTo(new ArrayBuffer())).rejects.toThrow('when format is null');
+
+	const size = sample.allocationSize({ format: 'RGBA' });
+	expect(size).toBe(1280 * 720 * 4);
+	const buffer = new ArrayBuffer(size);
+	const layout = await sample.copyTo(buffer, { format: 'RGBA' });
+
+	expect(layout).toEqual([{
+		offset: 0,
+		stride: 1280 * 4,
+	}]);
+
+	sample.allocationSize({ format: 'RGBX' });
+	sample.allocationSize({ format: 'BGRA' });
+	sample.allocationSize({ format: 'BGRX' });
+	await sample.copyTo(buffer, { format: 'RGBX' });
+	await sample.copyTo(buffer, { format: 'BGRA' });
+	await sample.copyTo(buffer, { format: 'BGRX' });
+});