diff --git a/docs/guide/packets-and-samples.md b/docs/guide/packets-and-samples.md index 18baa04..aa5c295 100644 --- a/docs/guide/packets-and-samples.md +++ b/docs/guide/packets-and-samples.md @@ -378,13 +378,11 @@ const bytesNeeded = videoSample.allocationSize(); // => number Then, use `copyTo` to copy the pixel data into the destination buffer: ```ts const bytes = new Uint8Array(bytesNeeded); -videoSample.copyTo(bytes); +const planeLayout = await videoSample.copyTo(bytes); ``` ::: info -The data will always be in the pixel format specified in the `format` field. - -To convert the data into a different pixel format, or to extract only a section of the frame, please use the [`allocationSize`](https://developer.mozilla.org/en-US/docs/Web/API/VideoFrame/allocationSize) and [`copyTo`](https://developer.mozilla.org/en-US/docs/Web/API/VideoFrame/copyTo) methods on `VideoFrame` instead. Get a `VideoFrame` by running `videoSample.toVideoFrame()`. +You can pass additional options to `allocationSize` and `copyTo` to extract data in a different pixel format. ::: --- diff --git a/src/index.ts b/src/index.ts index 5434d1a..6cd723f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -168,8 +168,10 @@ export { AudioSampleCopyToOptions, VideoSample, VideoSampleInit, + VideoSamplePixelFormat, VideoSampleColorSpace, CropRectangle, + VIDEO_SAMPLE_PIXEL_FORMATS, } from './sample'; export { AudioBufferSink, diff --git a/src/sample.ts b/src/sample.ts index 02e454c..37ec910 100644 --- a/src/sample.ts +++ b/src/sample.ts @@ -17,6 +17,7 @@ import { SetRequired, isFirefox, polyfillSymbolDispose, + assertNever, } from './misc'; polyfillSymbolDispose(); @@ -69,6 +70,57 @@ if (typeof FinalizationRegistry !== 'undefined') { }); } +/** + * The list of {@link VideoSample} pixel formats. + * @group Samples + * @public + */ +export const VIDEO_SAMPLE_PIXEL_FORMATS = [ + // 4:2:0 Y, U, V + 'I420', + 'I420P10', + 'I420P12', + // 4:2:0 Y, U, V, A + 'I420A', + 'I420AP10', + 'I420AP12', + // 4:2:2 Y, U, V + 'I422', + 'I422P10', + 'I422P12', + // 4:2:2 Y, U, V, A + 'I422A', + 'I422AP10', + 'I422AP12', + // 4:4:4 Y, U, V + 'I444', + 'I444P10', + 'I444P12', + // 4:4:4 Y, U, V, A + 'I444A', + 'I444AP10', + 'I444AP12', + // 4:2:0 Y, UV + 'NV12', + // 4:4:4 RGBA + 'RGBA', + // 4:4:4 RGBX (opaque) + 'RGBX', + // 4:4:4 BGRA + 'BGRA', + // 4:4:4 BGRX (opaque) + 'BGRX', +] as const; +const VIDEO_SAMPLE_PIXEL_FORMATS_SET = new Set(VIDEO_SAMPLE_PIXEL_FORMATS); + +/** + * The internal pixel format with which a {@link VideoSample} is stored. + * [See pixel formats](https://www.w3.org/TR/webcodecs/#pixel-format) for more. + * @group Samples + * @public + */ +export type VideoSamplePixelFormat = typeof VIDEO_SAMPLE_PIXEL_FORMATS[number]; + /** * Metadata used for VideoSample initialization. * @group Samples @@ -77,9 +129,9 @@ if (typeof FinalizationRegistry !== 'undefined') { export type VideoSampleInit = { /** * The internal pixel format in which the frame is stored. - * [See pixel formats](https://developer.mozilla.org/en-US/docs/Web/API/VideoFrame/format) + * [See pixel formats](https://www.w3.org/TR/webcodecs/#pixel-format) */ - format?: VideoPixelFormat; + format?: VideoSamplePixelFormat; /** The width of the frame in pixels. */ codedWidth?: number; /** The height of the frame in pixels. */ @@ -92,6 +144,8 @@ export type VideoSampleInit = { duration?: number; /** The color space of the frame. */ colorSpace?: VideoColorSpaceInit; + /** The byte layout of the planes of the frame. */ + layout?: PlaneLayout[]; }; /** @@ -103,14 +157,20 @@ export type VideoSampleInit = { export class VideoSample implements Disposable { /** @internal */ _data!: VideoFrame | OffscreenCanvas | Uint8Array | null; + /** + * Used for the ArrayBuffer-backed case. + * @internal + */ + _layout!: PlaneLayout[] | null; /** @internal */ _closed: boolean = false; /** - * The internal pixel format in which the frame is stored. - * [See pixel formats](https://developer.mozilla.org/en-US/docs/Web/API/VideoFrame/format) + * The internal pixel format in which the frame is stored. Will be `null` if it's using an arbitrary internal + * format not representable by `VideoPixelFormat`. + * [See pixel formats](https://www.w3.org/TR/webcodecs/#pixel-format) */ - readonly format!: VideoPixelFormat | null; + readonly format!: VideoSamplePixelFormat | null; /** The width of the frame in pixels. */ readonly codedWidth!: number; /** The height of the frame in pixels. */ @@ -189,8 +249,8 @@ export class VideoSample implements Disposable { if (!init || typeof init !== 'object') { throw new TypeError('init must be an object.'); } - if (!('format' in init) || typeof init.format !== 'string') { - throw new TypeError('init.format must be a string.'); + if (init.format === undefined || !VIDEO_SAMPLE_PIXEL_FORMATS_SET.has(init.format)) { + throw new TypeError('init.format must be one of: ' + VIDEO_SAMPLE_PIXEL_FORMATS.join(', ')); } if (!Number.isInteger(init.codedWidth) || init.codedWidth! <= 0) { throw new TypeError('init.codedWidth must be a positive integer.'); @@ -209,6 +269,7 @@ export class VideoSample implements Disposable { } this._data = toUint8Array(data).slice(); // Copy it + this._layout = init.layout ?? createDefaultPlaneLayout(init.format, init.codedWidth!, init.codedHeight!); this.format = init.format; this.codedWidth = init.codedWidth!; @@ -229,6 +290,7 @@ export class VideoSample implements Disposable { } this._data = data; + this._layout = null; this.format = data.format; // Copying the display dimensions here, assuming no innate VideoFrame rotation @@ -301,6 +363,7 @@ export class VideoSample implements Disposable { // Draw it to a canvas context.drawImage(data, 0, 0); this._data = canvas; + this._layout = null; this.format = 'RGBX'; this.codedWidth = width; @@ -336,8 +399,11 @@ export class VideoSample implements Disposable { rotation: this.rotation, }); } else if (this._data instanceof Uint8Array) { - return new VideoSample(this._data.slice(), { + assert(this._layout); + + return new VideoSample(this._data, { format: this.format!, + layout: this._layout, codedWidth: this.codedWidth, codedHeight: this.codedHeight, timestamp: this.timestamp, @@ -378,16 +444,43 @@ export class VideoSample implements Disposable { this._closed = true; } - /** Returns the number of bytes required to hold this video sample's pixel data. */ - allocationSize() { + /** + * Returns the number of bytes required to hold this video sample's pixel data. Throws if `format` is `null`; + * specify an explicit RGB format in the options in this case. + */ + allocationSize(options: VideoFrameCopyToOptions = {}): number { + validateVideoFrameCopyToOptions(options); + if (this._closed) { throw new Error('VideoSample is closed.'); } + if ((options.format ?? this.format) === null) { + throw new Error( + 'Cannot get allocation size when format is null. Please manually provide an RGB pixel format in the' + + ' options instead.', + ); + } assert(this._data !== null); + if (!isVideoFrame(this._data)) { + if ( + options.colorSpace + || (options.format && options.format !== this.format) + || options.layout + || options.rect + ) { + // Temporarily convert to VideoFrame to get it done + const videoFrame = this.toVideoFrame(); + const size = videoFrame.allocationSize(options); + videoFrame.close(); + + return size; + } + } + if (isVideoFrame(this._data)) { - return this._data.allocationSize(); + return this._data.allocationSize(options); } else if (this._data instanceof Uint8Array) { return this._data.byteLength; } else { @@ -395,23 +488,54 @@ export class VideoSample implements Disposable { } } - /** Copies this video sample's pixel data to an ArrayBuffer or ArrayBufferView. */ - async copyTo(destination: AllowSharedBufferSource) { + /** + * Copies this video sample's pixel data to an ArrayBuffer or ArrayBufferView. Throws if `format` is `null`; + * specify an explicit RGB format in the options in this case. + * @returns The byte layout of the planes of the copied data. + */ + async copyTo(destination: AllowSharedBufferSource, options: VideoFrameCopyToOptions = {}): Promise { if (!isAllowSharedBufferSource(destination)) { throw new TypeError('destination must be an ArrayBuffer or an ArrayBuffer view.'); } + validateVideoFrameCopyToOptions(options); if (this._closed) { throw new Error('VideoSample is closed.'); } + if ((options.format ?? this.format) === null) { + throw new Error( + 'Cannot copy video sample data when format is null. Please manually provide an RGB pixel format in the' + + ' options instead.', + ); + } assert(this._data !== null); + if (!isVideoFrame(this._data)) { + if ( + options.colorSpace + || (options.format && options.format !== this.format) + || options.layout + || options.rect + ) { + // Temporarily convert to VideoFrame to get it done + const videoFrame = this.toVideoFrame(); + const layout = await videoFrame.copyTo(destination, options); + videoFrame.close(); + + return layout; + } + } + if (isVideoFrame(this._data)) { - await this._data.copyTo(destination); + return this._data.copyTo(destination, options); } else if (this._data instanceof Uint8Array) { + assert(this._layout); + const dest = toUint8Array(destination); dest.set(this._data); + + return this._layout; } else { const canvas = this._data; const context = canvas.getContext('2d'); @@ -420,6 +544,11 @@ export class VideoSample implements Disposable { const imageData = context.getImageData(0, 0, this.codedWidth, this.codedHeight); const dest = toUint8Array(destination); dest.set(imageData.data); + + return [{ + offset: 0, + stride: 4 * this.codedWidth, + }]; } } @@ -441,7 +570,7 @@ export class VideoSample implements Disposable { }); } else if (this._data instanceof Uint8Array) { return new VideoFrame(this._data, { - format: this.format!, + format: this.format! as VideoPixelFormat, codedWidth: this.codedWidth, codedHeight: this.codedHeight, timestamp: this.microsecondTimestamp, @@ -887,7 +1016,168 @@ export const validateCropRectangle = (crop: CropRectangle, prefix: string) => { } }; -const AUDIO_SAMPLE_FORMATS = new Set( +const validateVideoFrameCopyToOptions = (options: VideoFrameCopyToOptions) => { + if (!options || typeof options !== 'object') { + throw new TypeError('options must be an object.'); + } + if (options.colorSpace !== undefined && !['display-p3', 'srgb'].includes(options.colorSpace)) { + throw new TypeError('options.colorSpace, when provided, must be \'display-p3\' or \'srgb\'.'); + } + if (options.format !== undefined && typeof options.format !== 'string') { + throw new TypeError('options.format, when provided, must be a string.'); + } + if (options.layout !== undefined) { + if (!Array.isArray(options.layout)) { + throw new TypeError('options.layout, when provided, must be an array.'); + } + + for (const plane of options.layout) { + if (!plane || typeof plane !== 'object') { + throw new TypeError('Each entry in options.layout must be an object.'); + } + if (!Number.isInteger(plane.offset) || plane.offset < 0) { + throw new TypeError('plane.offset must be a non-negative integer.'); + } + if (!Number.isInteger(plane.stride) || plane.stride < 0) { + throw new TypeError('plane.stride must be a non-negative integer.'); + } + } + } + if (options.rect !== undefined) { + if (!options.rect || typeof options.rect !== 'object') { + throw new TypeError('options.rect, when provided, must be an object.'); + } + if (options.rect.x !== undefined && (!Number.isInteger(options.rect.x) || options.rect.x < 0)) { + throw new TypeError('options.rect.x, when provided, must be a non-negative integer.'); + } + if (options.rect.y !== undefined && (!Number.isInteger(options.rect.y) || options.rect.y < 0)) { + throw new TypeError('options.rect.y, when provided, must be a non-negative integer.'); + } + if (options.rect.width !== undefined && (!Number.isInteger(options.rect.width) || options.rect.width < 0)) { + throw new TypeError('options.rect.width, when provided, must be a non-negative integer.'); + } + if (options.rect.height !== undefined && (!Number.isInteger(options.rect.height) || options.rect.height < 0)) { + throw new TypeError('options.rect.height, when provided, must be a non-negative integer.'); + } + } +}; + +/** Implements logic from WebCodecs § 9.4.6 "Compute Layout and Allocation Size" */ +const createDefaultPlaneLayout = ( + format: VideoSamplePixelFormat, + codedWidth: number, + codedHeight: number, +): PlaneLayout[] => { + const planes = getPlaneConfigs(format); + const layouts: PlaneLayout[] = []; + let currentOffset = 0; + + for (const plane of planes) { + // Per § 9.8, dimensions are usually "rounded up to the nearest integer". + const planeWidth = Math.ceil(codedWidth / plane.widthDivisor); + const planeHeight = Math.ceil(codedHeight / plane.heightDivisor); + + const stride = planeWidth * plane.sampleBytes; + + // Tight packing + const planeSize = stride * planeHeight; + + layouts.push({ + offset: currentOffset, + stride: stride, + }); + + currentOffset += planeSize; + } + + return layouts; +}; + +type PlaneConfig = { + sampleBytes: number; + widthDivisor: number; // Horizontal sub-sampling factor + heightDivisor: number; // Vertical sub-sampling factor +}; + +/** Helper to retrieve plane configurations based on WebCodecs § 9.8 Pixel Format definitions. */ +const getPlaneConfigs = (format: VideoSamplePixelFormat): PlaneConfig[] => { + // Helper for standard YUV planes + const yuv = ( + yBytes: number, + uvBytes: number, + subX: number, + subY: number, + hasAlpha: boolean, + ): PlaneConfig[] => { + const configs: PlaneConfig[] = [ + { sampleBytes: yBytes, widthDivisor: 1, heightDivisor: 1 }, + { sampleBytes: uvBytes, widthDivisor: subX, heightDivisor: subY }, + { sampleBytes: uvBytes, widthDivisor: subX, heightDivisor: subY }, + ]; + + if (hasAlpha) { + // Match luma dimensions + configs.push({ sampleBytes: yBytes, widthDivisor: 1, heightDivisor: 1 }); + } + + return configs; + }; + + switch (format) { + case 'I420': + return yuv(1, 1, 2, 2, false); + case 'I420P10': + case 'I420P12': + return yuv(2, 2, 2, 2, false); + case 'I420A': + return yuv(1, 1, 2, 2, true); + case 'I420AP10': + case 'I420AP12': + return yuv(2, 2, 2, 2, true); + + case 'I422': + return yuv(1, 1, 2, 1, false); + case 'I422P10': + case 'I422P12': + return yuv(2, 2, 2, 1, false); + case 'I422A': + return yuv(1, 1, 2, 1, true); + case 'I422AP10': + case 'I422AP12': + return yuv(2, 2, 2, 1, true); + + case 'I444': + return yuv(1, 1, 1, 1, false); + case 'I444P10': + case 'I444P12': + return yuv(2, 2, 1, 1, false); + case 'I444A': + return yuv(1, 1, 1, 1, true); + case 'I444AP10': + case 'I444AP12': + return yuv(2, 2, 1, 1, true); + + case 'NV12': + return [ + { sampleBytes: 1, widthDivisor: 1, heightDivisor: 1 }, + { sampleBytes: 2, widthDivisor: 2, heightDivisor: 2 }, // Interleaved U and V + ]; + + case 'RGBA': + case 'RGBX': + case 'BGRA': + case 'BGRX': + return [ + { sampleBytes: 4, widthDivisor: 1, heightDivisor: 1 }, + ]; + + default: + assertNever(format); + assert(false); + } +}; + +const AUDIO_SAMPLE_FORMATS = new Set( ['f32', 'f32-planar', 's16', 's16-planar', 's32', 's32-planar', 'u8', 'u8-planar'], ); diff --git a/test/browser/video-samples.test.ts b/test/browser/video-samples.test.ts new file mode 100644 index 0000000..b0c12e6 --- /dev/null +++ b/test/browser/video-samples.test.ts @@ -0,0 +1,220 @@ +import { expect, test } from 'vitest'; +import { VideoSample } from '../../src/sample.js'; + +test('allocationSize', async () => { + { + const canvas = new OffscreenCanvas(1280, 720); + canvas.getContext('2d'); + + using sample = new VideoSample(canvas, { timestamp: 0 }); + + const size1 = sample.allocationSize(); + expect(size1).toBe(1280 * 720 * 4); + + const size2 = sample.allocationSize({ + layout: [{ + offset: 0, + stride: 1300 * 4, + }], + }); + expect(size2).toBe(1300 * 720 * 4); + } + + { + const data = new Uint8Array(1280 * 720 * 4); + using sample = new VideoSample(data, { + timestamp: 0, + codedWidth: 1280, + codedHeight: 720, + format: 'RGBA', + }); + + const size1 = sample.allocationSize(); + expect(size1).toBe(1280 * 720 * 4); + + const size2 = sample.allocationSize({ + layout: [{ + offset: 0, + stride: 1300 * 4, + }], + }); + expect(size2).toBe(1300 * 720 * 4); + } + + { + const data = new Uint8Array(1280 * 720 * 1.5); + using sample = new VideoSample(data, { + timestamp: 0, + codedWidth: 1280, + codedHeight: 720, + format: 'I420', + }); + + const size1 = sample.allocationSize(); + expect(size1).toBe(1280 * 720 * 1.5); + + const size2 = sample.allocationSize({ format: 'RGBA' }); + expect(size2).toBe(1280 * 720 * 4); + + const size3 = sample.allocationSize({ + format: 'RGBA', + layout: [{ + offset: 0, + stride: 1300 * 4, + }], + }); + expect(size3).toBe(1300 * 720 * 4); + } +}); + +test('copyTo and plane layouts', async () => { + const buffer = new ArrayBuffer(1e7); + + { + const canvas = new OffscreenCanvas(1280, 720); + canvas.getContext('2d'); + + using sample = new VideoSample(canvas, { timestamp: 0 }); + const layout = await sample.copyTo(buffer); + + expect(layout).toEqual([{ + offset: 0, + stride: 1280 * 4, + }]); + } + + { + const canvas = new OffscreenCanvas(1280, 720); + canvas.getContext('2d'); + + using sample = new VideoSample(canvas, { timestamp: 0 }); + const layout = await sample.copyTo(buffer, { + layout: [{ + offset: 0, + stride: 1300 * 4, + }], + }); + + expect(layout).toEqual([{ + offset: 0, + stride: 1300 * 4, + }]); + } + + { + const data = new Uint8Array(1280 * 720 * 4); + using sample = new VideoSample(data, { + timestamp: 0, + codedWidth: 1280, + codedHeight: 720, + format: 'RGBA', + }); + const layout = await sample.copyTo(buffer); + + expect(layout).toEqual([{ + offset: 0, + stride: 1280 * 4, + }]); + } + + { + const data = new Uint8Array(1280 * 720 * 4); + using sample = new VideoSample(data, { + timestamp: 0, + codedWidth: 1280, + codedHeight: 720, + format: 'RGBA', + }); + const layout = await sample.copyTo(buffer, { + layout: [{ + offset: 0, + stride: 1300 * 4, + }], + }); + + expect(layout).toEqual([{ + offset: 0, + stride: 1300 * 4, + }]); + } + + { + const data = new Uint8Array(1300 * 720 * 4); + using sample = new VideoSample(data, { + timestamp: 0, + codedWidth: 1280, + codedHeight: 720, + format: 'RGBA', + layout: [{ + offset: 0, + stride: 1300 * 4, + }], + }); + const layout = await sample.copyTo(buffer); + + expect(layout).toEqual([{ + offset: 0, + stride: 1300 * 4, + }]); + + using clone = sample.clone(); + const clonedLayout = await clone.copyTo(buffer); + + expect(clonedLayout).toEqual([{ + offset: 0, + stride: 1300 * 4, + }]); + } + + { + const data = new Uint8Array(1280 * 720 * 1.5); + using sample = new VideoSample(data, { + timestamp: 0, + codedWidth: 1280, + codedHeight: 720, + format: 'I420', + }); + const layout = await sample.copyTo(buffer); + + expect(layout).toEqual([{ + offset: 0, + stride: 1280, + }, { + offset: 1280 * 720, + stride: 1280 / 2, + }, { + offset: 1280 * 720 + (1280 / 2) * (720 / 2), + stride: 1280 / 2, + }]); + } +}); + +test('null format', async () => { + const canvas = new OffscreenCanvas(1280, 720); + canvas.getContext('2d'); + + const frame = new VideoFrame(canvas, { timestamp: 0 }); + using sample = new VideoSample(frame); + // @ts-expect-error Sybau + sample.format = null; + + expect(() => sample.allocationSize()).toThrow('when format is null'); + await expect(async () => sample.copyTo(new ArrayBuffer())).rejects.toThrow('when format is null'); + + const size = sample.allocationSize({ format: 'RGBA' }); + expect(size).toBe(1280 * 720 * 4); + const buffer = new ArrayBuffer(size); + const layout = await sample.copyTo(buffer, { format: 'RGBA' }); + + expect(layout).toEqual([{ + offset: 0, + stride: 1280 * 4, + }]); + + sample.allocationSize({ format: 'RGBX' }); + sample.allocationSize({ format: 'BGRA' }); + sample.allocationSize({ format: 'BGRX' }); + await sample.copyTo(buffer, { format: 'RGBX' }); + await sample.copyTo(buffer, { format: 'BGRA' }); + await sample.copyTo(buffer, { format: 'BGRX' }); +});