Add VideoSampleResource and AudioSampleResource

2026-10-03 05:43:50 +02:00 · 2025-12-17 11:14:12 +01:00
parent 228721d68f
commit 81d3d1e6a4
3 changed files with 443 additions and 53 deletions
@@ -23,10 +23,10 @@ polyfillSymbolDispose();

 type FinalizationRegistryValue = {
 	type: 'video';
-	data: VideoFrame | OffscreenCanvas | Uint8Array;
+	data: VideoFrame | OffscreenCanvas | Uint8Array | VideoSampleResource;
 } | {
 	type: 'audio';
-	data: AudioData | Uint8Array;
+	data: AudioData | Uint8Array | AudioSampleResource;
 };

 // Let's manually handle logging the garbage collection errors that are typically logged by the browser. This way, they
@@ -69,6 +69,46 @@ if (typeof FinalizationRegistry !== 'undefined') {
 	});
 }

+/**
+ * Abstract base class for custom video sample resources. Implement this class to provide custom backing
+ * for VideoSample instances.
+ * @group Samples
+ * @public
+ */
+export abstract class VideoSampleResource {
+	/** @internal */
+	_referenceCount: number = 0;
+	/** @internal */
+	_lastAllocationBuffer: ArrayBuffer | null = null;
+
+	/**
+	 * Returns the internal pixel format in which the frame is stored.
+	 * [See pixel formats](https://developer.mozilla.org/en-US/docs/Web/API/VideoFrame/format)
+	 */
+	abstract getFormat(): VideoPixelFormat | null;
+
+	/** Returns the width of the frame in pixels. */
+	abstract getCodedWidth(): number;
+
+	/** Returns the height of the frame in pixels. */
+	abstract getCodedHeight(): number;
+
+	/** Returns the color space of the frame. */
+	abstract getColorSpace(): VideoSampleColorSpace;
+
+	/**
+	 * Closes this resource, releasing held resources. Called automatically when the last {@link VideoSample} using this
+	 * resource is closed.
+	 */
+	abstract close(): void;
+
+	/** Returns the number of bytes required to hold this video sample's pixel data. */
+	abstract allocationSize(options: VideoFrameCopyToOptions): number;
+
+	/** Copies this video sample's pixel data to an ArrayBuffer or ArrayBufferView. */
+	abstract copyTo(destination: AllowSharedBufferSource, options: VideoFrameCopyToOptions): PlaneLayout[];
+}
+
 /**
 * Metadata used for VideoSample initialization.
 * @group Samples
@@ -102,7 +142,7 @@ export type VideoSampleInit = {
 */
 export class VideoSample implements Disposable {
 	/** @internal */
-	_data!: VideoFrame | OffscreenCanvas | Uint8Array | null;
+	_data!: VideoFrame | OffscreenCanvas | Uint8Array | VideoSampleResource | null;
 	/** @internal */
 	_closed: boolean = false;

@@ -177,8 +217,12 @@ export class VideoSample implements Disposable {
 		data: AllowSharedBufferSource,
 		init: SetRequired<VideoSampleInit, 'format' | 'codedWidth' | 'codedHeight' | 'timestamp'>
 	);
+	/**
+	 * Creates a new {@link VideoSample} backed by a custom {@link VideoSampleResource}.
+	 */
+	constructor(resource: VideoSampleResource, init: SetRequired<VideoSampleInit, 'timestamp'>);
 	constructor(
-		data: VideoFrame | CanvasImageSource | AllowSharedBufferSource,
+		data: VideoFrame | CanvasImageSource | AllowSharedBufferSource | VideoSampleResource,
 		init?: VideoSampleInit,
 	) {
 		if (
@@ -314,8 +358,34 @@ export class VideoSample implements Disposable {
 				transfer: 'iec61966-2-1',
 				fullRange: true,
 			});
+		} else if (data instanceof VideoSampleResource) {
+			if (!init || typeof init !== 'object') {
+				throw new TypeError('init must be an object.');
+			}
+			if (init.rotation !== undefined && ![0, 90, 180, 270].includes(init.rotation)) {
+				throw new TypeError('init.rotation, when provided, must be 0, 90, 180, or 270.');
+			}
+			if (!Number.isFinite(init.timestamp)) {
+				throw new TypeError('init.timestamp must be a number.');
+			}
+			if (init.duration !== undefined && (!Number.isFinite(init.duration) || init.duration < 0)) {
+				throw new TypeError('init.duration, when provided, must be a non-negative number.');
+			}
+
+			this._data = data;
+			data._referenceCount++;
+
+			this.format = data.getFormat();
+			this.codedWidth = data.getCodedWidth();
+			this.codedHeight = data.getCodedHeight();
+			this.rotation = init.rotation ?? 0;
+			this.timestamp = init.timestamp!;
+			this.duration = init.duration ?? 0;
+			this.colorSpace = data.getColorSpace();
 		} else {
-			throw new TypeError('Invalid data type: Must be a BufferSource or CanvasImageSource.');
+			throw new TypeError(
+				'Invalid data type: Must be a BufferSource, CanvasImageSource, or VideoSampleResource.',
+			);
 		}

 		finalizationRegistry?.register(this, { type: 'video', data: this._data }, this);
@@ -329,7 +399,13 @@ export class VideoSample implements Disposable {

 		assert(this._data !== null);

-		if (isVideoFrame(this._data)) {
+		if (this._data instanceof VideoSampleResource) {
+			return new VideoSample(this._data, {
+				timestamp: this.timestamp,
+				duration: this.duration,
+				rotation: this.rotation,
+			});
+		} else if (isVideoFrame(this._data)) {
 			return new VideoSample(this._data.clone(), {
 				timestamp: this.timestamp,
 				duration: this.duration,
@@ -369,7 +445,12 @@ export class VideoSample implements Disposable {

 		finalizationRegistry?.unregister(this);

-		if (isVideoFrame(this._data)) {
+		if (this._data instanceof VideoSampleResource) {
+			this._data._referenceCount--;
+			if (this._data._referenceCount === 0) {
+				this._data.close();
+			}
+		} else if (isVideoFrame(this._data)) {
 			this._data.close();
 		} else {
 			this._data = null; // GC that shit
@@ -379,15 +460,17 @@ export class VideoSample implements Disposable {
 	}

 	/** Returns the number of bytes required to hold this video sample's pixel data. */
-	allocationSize() {
+	allocationSize(options: VideoFrameCopyToOptions = {}) {
 		if (this._closed) {
 			throw new Error('VideoSample is closed.');
 		}

 		assert(this._data !== null);

-		if (isVideoFrame(this._data)) {
-			return this._data.allocationSize();
+		if (this._data instanceof VideoSampleResource) {
+			return this._data.allocationSize(options);
+		} else if (isVideoFrame(this._data)) {
+			return this._data.allocationSize(options);
 		} else if (this._data instanceof Uint8Array) {
 			return this._data.byteLength;
 		} else {
@@ -396,7 +479,7 @@ export class VideoSample implements Disposable {
 	}

 	/** Copies this video sample's pixel data to an ArrayBuffer or ArrayBufferView. */
-	async copyTo(destination: AllowSharedBufferSource) {
+	async copyTo(destination: AllowSharedBufferSource, options: VideoFrameCopyToOptions = {}) {
 		if (!isAllowSharedBufferSource(destination)) {
 			throw new TypeError('destination must be an ArrayBuffer or an ArrayBuffer view.');
 		}
@@ -407,8 +490,10 @@ export class VideoSample implements Disposable {

 		assert(this._data !== null);

-		if (isVideoFrame(this._data)) {
-			await this._data.copyTo(destination);
+		if (this._data instanceof VideoSampleResource) {
+			this._data.copyTo(destination, options);
+		} else if (isVideoFrame(this._data)) {
+			await this._data.copyTo(destination, options);
 		} else if (this._data instanceof Uint8Array) {
 			const dest = toUint8Array(destination);
 			dest.set(this._data);
@@ -427,14 +512,35 @@ export class VideoSample implements Disposable {
 	 * Converts this video sample to a VideoFrame for use with the WebCodecs API. The VideoFrame returned by this
 	 * method *must* be closed separately from this video sample.
 	 */
-	toVideoFrame() {
+	toVideoFrame(): VideoFrame {
 		if (this._closed) {
 			throw new Error('VideoSample is closed.');
 		}

 		assert(this._data !== null);

-		if (isVideoFrame(this._data)) {
+		if (this._data instanceof VideoSampleResource) {
+			const format = this._data.getFormat() ?? 'RGBA';
+			const allocationSize = this._data.allocationSize({ format });
+
+			let data: ArrayBuffer;
+			if (this._data._lastAllocationBuffer?.byteLength === allocationSize) {
+				data = this._data._lastAllocationBuffer;
+			} else {
+				data = this._data._lastAllocationBuffer = new ArrayBuffer(allocationSize);
+			}
+			const layout = this._data.copyTo(data, { format });
+
+			return new VideoFrame(data, {
+				format,
+				codedWidth: this._data.getCodedWidth(),
+				codedHeight: this._data.getCodedHeight(),
+				colorSpace: this._data.getColorSpace(),
+				layout,
+				timestamp: this.microsecondTimestamp,
+				duration: this.microsecondDuration,
+			});
+		} else if (isVideoFrame(this._data)) {
 			return new VideoFrame(this._data, {
 				timestamp: this.microsecondTimestamp,
 				duration: this.microsecondDuration || undefined, // Drag 0 duration to undefined, glitches some codecs
@@ -748,7 +854,7 @@ export class VideoSample implements Disposable {
 	 * Converts this video sample to a
 	 * [`CanvasImageSource`](https://udn.realityripple.com/docs/Web/API/CanvasImageSource) for drawing to a canvas.
 	 *
-	 * You must use the value returned by this method immediately, as any VideoFrame created internally will
+	 * You must use the value returned by this method immediately, as any VideoFrame created internally may
 	 * automatically be closed in the next microtask.
 	 */
 	toCanvasImageSource() {
@@ -758,7 +864,7 @@ export class VideoSample implements Disposable {

 		assert(this._data !== null);

-		if (this._data instanceof Uint8Array) {
+		if (this._data instanceof VideoSampleResource || this._data instanceof Uint8Array) {
 			// Requires VideoFrame to be defined
 			const videoFrame = this.toVideoFrame();
 			queueMicrotask(() => videoFrame.close()); // Let's automatically close the frame in the next microtask
@@ -891,6 +997,51 @@ const AUDIO_SAMPLE_FORMATS = new Set(
 	['f32', 'f32-planar', 's16', 's16-planar', 's32', 's32-planar', 'u8', 'u8-planar'],
 );

+/**
+ * Abstract base class for custom audio sample resources. Implement this class to provide custom backing
+ * for AudioSample instances.
+ * @group Samples
+ * @public
+ */
+export abstract class AudioSampleResource {
+	/** @internal */
+	_referenceCount: number = 0;
+
+	/**
+	 * Returns the audio sample format.
+	 * [See sample formats](https://developer.mozilla.org/en-US/docs/Web/API/AudioData/format)
+	 */
+	abstract getFormat(): AudioSampleFormat;
+
+	/** Returns the audio sample rate in hertz. */
+	abstract getSampleRate(): number;
+
+	/** Returns the number of audio frames in the sample, per channel. */
+	abstract getNumberOfFrames(): number;
+
+	/** Returns the number of audio channels. */
+	abstract getNumberOfChannels(): number;
+
+	/** Returns the presentation timestamp of the sample in seconds. */
+	abstract getTimestamp(): number;
+
+	/**
+	 * Closes this resource, releasing held resources. Called automatically when the last {@link AudioSample} using this
+	 * resource is closed.
+	 */
+	abstract close(): void;
+
+	/**
+	 * Returns the number of bytes required to hold the underlying audio data as specified by the given options.
+	 */
+	abstract allocationSize(options: AudioSampleCopyToOptions): number;
+
+	/**
+	 * Copies the underlying audio data to an ArrayBuffer or ArrayBufferView as specified by the given options.
+	 */
+	abstract copyTo(destination: AllowSharedBufferSource, options: AudioSampleCopyToOptions): void;
+}
+
 /**
 * Metadata used for AudioSample initialization.
 * @group Samples
@@ -943,7 +1094,7 @@ export type AudioSampleCopyToOptions = {
 */
 export class AudioSample implements Disposable {
 	/** @internal */
-	_data: AudioData | Uint8Array;
+	_data: AudioData | Uint8Array | AudioSampleResource;
 	/** @internal */
 	_closed: boolean = false;

@@ -983,7 +1134,8 @@ export class AudioSample implements Disposable {
 	 * [`AudioData`](https://developer.mozilla.org/en-US/docs/Web/API/AudioData) or from raw bytes specified in
 	 * {@link AudioSampleInit}.
 	 */
-	constructor(init: AudioData | AudioSampleInit) {
+	constructor(init: AudioData | AudioSampleInit | AudioSampleResource);
+	constructor(init: AudioData | AudioSampleInit | AudioSampleResource) {
 		if (isAudioData(init)) {
 			if (init.format === null) {
 				throw new TypeError('AudioData with null format is not supported.');
@@ -997,6 +1149,16 @@ export class AudioSample implements Disposable {
 			this.numberOfChannels = init.numberOfChannels;
 			this.timestamp = init.timestamp / 1e6;
 			this.duration = init.numberOfFrames / init.sampleRate;
+		} else if (init instanceof AudioSampleResource) {
+			this._data = init;
+			init._referenceCount++;
+
+			this.format = init.getFormat();
+			this.sampleRate = init.getSampleRate();
+			this.numberOfFrames = init.getNumberOfFrames();
+			this.numberOfChannels = init.getNumberOfChannels();
+			this.timestamp = init.getTimestamp();
+			this.duration = this.numberOfFrames / this.sampleRate;
 		} else {
 			if (!init || typeof init !== 'object') {
 				throw new TypeError('Invalid AudioDataInit: must be an object.');
@@ -1072,6 +1234,10 @@ export class AudioSample implements Disposable {
 			throw new Error('AudioSample is closed.');
 		}

+		if (this._data instanceof AudioSampleResource) {
+			return this._data.allocationSize(options);
+		}
+
 		const destFormat = options.format ?? this.format;

 		const frameOffset = options.frameOffset ?? 0;
@@ -1124,6 +1290,10 @@ export class AudioSample implements Disposable {
 			throw new Error('AudioSample is closed.');
 		}

+		if (this._data instanceof AudioSampleResource) {
+			return this._data.copyTo(destination, options);
+		}
+
 		const { planeIndex, format, frameCount: optFrameCount, frameOffset: optFrameOffset } = options;

 		const destFormat = format ?? this.format;
@@ -1253,9 +1423,14 @@ export class AudioSample implements Disposable {
 			throw new Error('AudioSample is closed.');
 		}

-		if (isAudioData(this._data)) {
+		if (this._data instanceof AudioSampleResource) {
+			const sample = new AudioSample(this._data);
+			sample.setTimestamp(this.timestamp); // Make sure the timestamp is correct
+
+			return sample;
+		} else if (isAudioData(this._data)) {
 			const sample = new AudioSample(this._data.clone());
-			sample.setTimestamp(this.timestamp); // Make sure the timestamp is precise (beyond microsecond accuracy)
+			sample.setTimestamp(this.timestamp);

 			return sample;
 		} else {
@@ -1281,7 +1456,12 @@ export class AudioSample implements Disposable {

 		finalizationRegistry?.unregister(this);

-		if (isAudioData(this._data)) {
+		if (this._data instanceof AudioSampleResource) {
+			this._data._referenceCount--;
+			if (this._data._referenceCount === 0) {
+				this._data.close();
+			}
+		} else if (isAudioData(this._data)) {
 			this._data.close();
 		} else {
 			this._data = new Uint8Array(0);
@@ -1299,12 +1479,32 @@ export class AudioSample implements Disposable {
 			throw new Error('AudioSample is closed.');
 		}

-		if (isAudioData(this._data)) {
+		if (this._data instanceof AudioSampleResource) {
+			return this._createAudioDataFromData();
+		} else if (isAudioData(this._data)) {
 			if (this._data.timestamp === this.microsecondTimestamp) {
 				// Timestamp matches, let's just return the data (but cloned)
 				return this._data.clone();
 			} else {
 				// It's impossible to simply change an AudioData's timestamp, so we'll need to create a new one
+				return this._createAudioDataFromData();
+			}
+		} else {
+			return new AudioData({
+				format: this.format,
+				sampleRate: this.sampleRate,
+				numberOfFrames: this.numberOfFrames,
+				numberOfChannels: this.numberOfChannels,
+				timestamp: this.microsecondTimestamp,
+				data: this._data.buffer instanceof ArrayBuffer
+					? this._data.buffer
+					: this._data.slice(), // In the case of SharedArrayBuffer, convert to ArrayBuffer
+			});
+		}
+	}
+
+	/** @internal */
+	_createAudioDataFromData() {
 		if (formatIsPlanar(this.format)) {
 			const size = this.allocationSize({ planeIndex: 0, format: this.format });
 			const data = new ArrayBuffer(size * this.numberOfChannels);
@@ -1336,19 +1536,6 @@ export class AudioSample implements Disposable {
 			});
 		}
 	}
-		} else {
-			return new AudioData({
-				format: this.format,
-				sampleRate: this.sampleRate,
-				numberOfFrames: this.numberOfFrames,
-				numberOfChannels: this.numberOfChannels,
-				timestamp: this.microsecondTimestamp,
-				data: this._data.buffer instanceof ArrayBuffer
-					? this._data.buffer
-					: this._data.slice(), // In the case of SharedArrayBuffer, convert to ArrayBuffer
-			});
-		}
-	}

 	/** Convert this audio sample to an AudioBuffer for use with the Web Audio API. */
 	toAudioBuffer() {
@@ -0,0 +1,203 @@
+import { expect, test } from 'vitest';
+import {
+	VideoSampleResource,
+	VideoSampleColorSpace,
+	VideoSample,
+	AudioSampleResource,
+	AudioSample,
+} from '../../src/sample.js';
+import { toUint8Array } from '../../src/misc.js';
+
+class ImageVideoSampleResource extends VideoSampleResource {
+	constructor(public image: HTMLImageElement | null) {
+		super();
+	}
+
+	getFormat(): VideoPixelFormat {
+		return 'RGBA';
+	}
+
+	getCodedWidth() {
+		return this.image!.width;
+	}
+
+	getCodedHeight() {
+		return this.image!.height;
+	}
+
+	getColorSpace() {
+		return new VideoSampleColorSpace({
+			matrix: 'rgb',
+			primaries: 'bt709',
+			transfer: 'iec61966-2-1',
+			fullRange: true,
+		});
+	}
+
+	allocationSize() {
+		return this.image!.width * this.image!.height * 4;
+	}
+
+	copyTo(destination: AllowSharedBufferSource): PlaneLayout[] {
+		const canvas = new OffscreenCanvas(this.image!.width, this.image!.height);
+		const ctx = canvas.getContext('2d')!;
+
+		ctx.drawImage(this.image!, 0, 0);
+
+		const imageData = ctx.getImageData(0, 0, this.image!.width, this.image!.height);
+		toUint8Array(destination).set(imageData.data);
+
+		return [{
+			offset: 0,
+			stride: this.image!.width * 4,
+		}];
+	}
+
+	close() {
+		this.image = null;
+	}
+}
+
+test('Custom VideoSample resource usage', async () => {
+	const image = new Image();
+	image.src = '/red-image.png';
+	await image.decode();
+
+	const resource = new ImageVideoSampleResource(image);
+	const sample = new VideoSample(resource, { timestamp: 0 });
+	expect(sample.format).toBe('RGBA');
+	expect(sample.codedWidth).toBe(2048);
+	expect(sample.codedHeight).toBe(2048);
+	expect(sample.colorSpace).toEqual({
+		matrix: 'rgb',
+		primaries: 'bt709',
+		transfer: 'iec61966-2-1',
+		fullRange: true,
+	});
+
+	const videoFrame = sample.toVideoFrame();
+	const canvas = new OffscreenCanvas(1024, 1024);
+	const context = canvas.getContext('2d')!;
+	context.drawImage(videoFrame, 0, 0);
+	const imageData = context.getImageData(0, 0, 1024, 1024);
+	expect([...imageData.data.slice(0, 4)]).toEqual([255, 0, 0, 255]);
+
+	videoFrame.close();
+
+	const clone = sample.clone();
+	sample.close();
+	expect(resource.image).not.toBe(null);
+
+	clone.close();
+	expect(resource.image).toBe(null);
+});
+
+test('Custom VideoSample resource, invalid construction', async () => {
+	const image = new Image();
+	image.src = '/red-image.png';
+	await image.decode();
+
+	const resource = new ImageVideoSampleResource(image);
+	// @ts-expect-error Wrong
+	expect(() => new VideoSample(resource)).toThrow(TypeError);
+	// @ts-expect-error Wrong
+	expect(() => new VideoSample(resource, { duration: 0 })).toThrow(TypeError);
+});
+
+class AudioBufferAudioSampleResource extends AudioSampleResource {
+	constructor(
+		public audioBuffer: AudioBuffer | null,
+		public timestamp: number,
+	) {
+		super();
+	}
+
+	getFormat(): AudioSampleFormat {
+		return 'f32-planar';
+	}
+
+	getSampleRate() {
+		return this.audioBuffer!.sampleRate;
+	}
+
+	getNumberOfFrames() {
+		return this.audioBuffer!.length;
+	}
+
+	getNumberOfChannels() {
+		return this.audioBuffer!.numberOfChannels;
+	}
+
+	getTimestamp() {
+		return this.timestamp;
+	}
+
+	allocationSize() {
+		const frameCount = this.audioBuffer!.length;
+		return frameCount * Float32Array.BYTES_PER_ELEMENT;
+	}
+
+	copyTo(destination: AllowSharedBufferSource, options: AudioDataCopyToOptions): void {
+		const frameCount = this.audioBuffer!.length;
+		const planeIndex = options.planeIndex;
+
+		const channel = this.audioBuffer!.getChannelData(planeIndex);
+
+		let destView: Float32Array;
+		if (destination instanceof ArrayBuffer) {
+			destView = new Float32Array(destination);
+		} else if (destination instanceof SharedArrayBuffer) {
+			destView = new Float32Array(destination);
+		} else {
+			destView = new Float32Array(destination.buffer, destination.byteOffset, destination.byteLength / 4);
+		}
+
+		const frameOffset = options.frameOffset ?? 0;
+		const copyFrameCount = options.frameCount !== undefined ? options.frameCount : (frameCount - frameOffset);
+
+		for (let i = 0; i < copyFrameCount; i++) {
+			destView[i] = channel[i + frameOffset]!;
+		}
+	}
+
+	close() {
+		this.audioBuffer = null;
+	}
+}
+
+test('Custom AudioSample resource usage', async () => {
+	const audioContext = new AudioContext();
+	const audioBuffer = audioContext.createBuffer(2, 48000, 48000);
+
+	const leftChannel = audioBuffer.getChannelData(0);
+	const rightChannel = audioBuffer.getChannelData(1);
+	for (let i = 0; i < 48000; i++) {
+		const t = i / 48000;
+		leftChannel[i] = Math.sin(2 * Math.PI * 440 * t);
+		rightChannel[i] = Math.cos(2 * Math.PI * 440 * t);
+	}
+
+	const resource = new AudioBufferAudioSampleResource(audioBuffer, 0);
+	const sample = new AudioSample(resource);
+
+	expect(sample.format).toBe('f32-planar');
+	expect(sample.sampleRate).toBe(48000);
+	expect(sample.numberOfFrames).toBe(48000);
+	expect(sample.numberOfChannels).toBe(2);
+	expect(sample.timestamp).toBe(0);
+	expect(sample.duration).toBeCloseTo(1);
+
+	const buffer = new ArrayBuffer(48000 * 4);
+	sample.copyTo(buffer, { planeIndex: 0 });
+
+	const view = new Float32Array(buffer);
+	expect(view[0]).toBeCloseTo(0, 4);
+	expect(view[13]).toBeCloseTo(Math.sin(2 * Math.PI * 440 * (13 / 48000)));
+
+	const clone = sample.clone();
+	sample.close();
+	expect(resource.audioBuffer).not.toBe(null);
+
+	clone.close();
+	expect(resource.audioBuffer).toBe(null);
+});