diff --git a/src/sample.ts b/src/sample.ts index 02e454c..f0f9dda 100644 --- a/src/sample.ts +++ b/src/sample.ts @@ -23,10 +23,10 @@ polyfillSymbolDispose(); type FinalizationRegistryValue = { type: 'video'; - data: VideoFrame | OffscreenCanvas | Uint8Array; + data: VideoFrame | OffscreenCanvas | Uint8Array | VideoSampleResource; } | { type: 'audio'; - data: AudioData | Uint8Array; + data: AudioData | Uint8Array | AudioSampleResource; }; // Let's manually handle logging the garbage collection errors that are typically logged by the browser. This way, they @@ -69,6 +69,46 @@ if (typeof FinalizationRegistry !== 'undefined') { }); } +/** + * Abstract base class for custom video sample resources. Implement this class to provide custom backing + * for VideoSample instances. + * @group Samples + * @public + */ +export abstract class VideoSampleResource { + /** @internal */ + _referenceCount: number = 0; + /** @internal */ + _lastAllocationBuffer: ArrayBuffer | null = null; + + /** + * Returns the internal pixel format in which the frame is stored. + * [See pixel formats](https://developer.mozilla.org/en-US/docs/Web/API/VideoFrame/format) + */ + abstract getFormat(): VideoPixelFormat | null; + + /** Returns the width of the frame in pixels. */ + abstract getCodedWidth(): number; + + /** Returns the height of the frame in pixels. */ + abstract getCodedHeight(): number; + + /** Returns the color space of the frame. */ + abstract getColorSpace(): VideoSampleColorSpace; + + /** + * Closes this resource, releasing held resources. Called automatically when the last {@link VideoSample} using this + * resource is closed. + */ + abstract close(): void; + + /** Returns the number of bytes required to hold this video sample's pixel data. */ + abstract allocationSize(options: VideoFrameCopyToOptions): number; + + /** Copies this video sample's pixel data to an ArrayBuffer or ArrayBufferView. */ + abstract copyTo(destination: AllowSharedBufferSource, options: VideoFrameCopyToOptions): PlaneLayout[]; +} + /** * Metadata used for VideoSample initialization. * @group Samples @@ -102,7 +142,7 @@ export type VideoSampleInit = { */ export class VideoSample implements Disposable { /** @internal */ - _data!: VideoFrame | OffscreenCanvas | Uint8Array | null; + _data!: VideoFrame | OffscreenCanvas | Uint8Array | VideoSampleResource | null; /** @internal */ _closed: boolean = false; @@ -177,8 +217,12 @@ export class VideoSample implements Disposable { data: AllowSharedBufferSource, init: SetRequired ); + /** + * Creates a new {@link VideoSample} backed by a custom {@link VideoSampleResource}. + */ + constructor(resource: VideoSampleResource, init: SetRequired); constructor( - data: VideoFrame | CanvasImageSource | AllowSharedBufferSource, + data: VideoFrame | CanvasImageSource | AllowSharedBufferSource | VideoSampleResource, init?: VideoSampleInit, ) { if ( @@ -314,8 +358,34 @@ export class VideoSample implements Disposable { transfer: 'iec61966-2-1', fullRange: true, }); + } else if (data instanceof VideoSampleResource) { + if (!init || typeof init !== 'object') { + throw new TypeError('init must be an object.'); + } + if (init.rotation !== undefined && ![0, 90, 180, 270].includes(init.rotation)) { + throw new TypeError('init.rotation, when provided, must be 0, 90, 180, or 270.'); + } + if (!Number.isFinite(init.timestamp)) { + throw new TypeError('init.timestamp must be a number.'); + } + if (init.duration !== undefined && (!Number.isFinite(init.duration) || init.duration < 0)) { + throw new TypeError('init.duration, when provided, must be a non-negative number.'); + } + + this._data = data; + data._referenceCount++; + + this.format = data.getFormat(); + this.codedWidth = data.getCodedWidth(); + this.codedHeight = data.getCodedHeight(); + this.rotation = init.rotation ?? 0; + this.timestamp = init.timestamp!; + this.duration = init.duration ?? 0; + this.colorSpace = data.getColorSpace(); } else { - throw new TypeError('Invalid data type: Must be a BufferSource or CanvasImageSource.'); + throw new TypeError( + 'Invalid data type: Must be a BufferSource, CanvasImageSource, or VideoSampleResource.', + ); } finalizationRegistry?.register(this, { type: 'video', data: this._data }, this); @@ -329,7 +399,13 @@ export class VideoSample implements Disposable { assert(this._data !== null); - if (isVideoFrame(this._data)) { + if (this._data instanceof VideoSampleResource) { + return new VideoSample(this._data, { + timestamp: this.timestamp, + duration: this.duration, + rotation: this.rotation, + }); + } else if (isVideoFrame(this._data)) { return new VideoSample(this._data.clone(), { timestamp: this.timestamp, duration: this.duration, @@ -369,7 +445,12 @@ export class VideoSample implements Disposable { finalizationRegistry?.unregister(this); - if (isVideoFrame(this._data)) { + if (this._data instanceof VideoSampleResource) { + this._data._referenceCount--; + if (this._data._referenceCount === 0) { + this._data.close(); + } + } else if (isVideoFrame(this._data)) { this._data.close(); } else { this._data = null; // GC that shit @@ -379,15 +460,17 @@ export class VideoSample implements Disposable { } /** Returns the number of bytes required to hold this video sample's pixel data. */ - allocationSize() { + allocationSize(options: VideoFrameCopyToOptions = {}) { if (this._closed) { throw new Error('VideoSample is closed.'); } assert(this._data !== null); - if (isVideoFrame(this._data)) { - return this._data.allocationSize(); + if (this._data instanceof VideoSampleResource) { + return this._data.allocationSize(options); + } else if (isVideoFrame(this._data)) { + return this._data.allocationSize(options); } else if (this._data instanceof Uint8Array) { return this._data.byteLength; } else { @@ -396,7 +479,7 @@ export class VideoSample implements Disposable { } /** Copies this video sample's pixel data to an ArrayBuffer or ArrayBufferView. */ - async copyTo(destination: AllowSharedBufferSource) { + async copyTo(destination: AllowSharedBufferSource, options: VideoFrameCopyToOptions = {}) { if (!isAllowSharedBufferSource(destination)) { throw new TypeError('destination must be an ArrayBuffer or an ArrayBuffer view.'); } @@ -407,8 +490,10 @@ export class VideoSample implements Disposable { assert(this._data !== null); - if (isVideoFrame(this._data)) { - await this._data.copyTo(destination); + if (this._data instanceof VideoSampleResource) { + this._data.copyTo(destination, options); + } else if (isVideoFrame(this._data)) { + await this._data.copyTo(destination, options); } else if (this._data instanceof Uint8Array) { const dest = toUint8Array(destination); dest.set(this._data); @@ -427,14 +512,35 @@ export class VideoSample implements Disposable { * Converts this video sample to a VideoFrame for use with the WebCodecs API. The VideoFrame returned by this * method *must* be closed separately from this video sample. */ - toVideoFrame() { + toVideoFrame(): VideoFrame { if (this._closed) { throw new Error('VideoSample is closed.'); } assert(this._data !== null); - if (isVideoFrame(this._data)) { + if (this._data instanceof VideoSampleResource) { + const format = this._data.getFormat() ?? 'RGBA'; + const allocationSize = this._data.allocationSize({ format }); + + let data: ArrayBuffer; + if (this._data._lastAllocationBuffer?.byteLength === allocationSize) { + data = this._data._lastAllocationBuffer; + } else { + data = this._data._lastAllocationBuffer = new ArrayBuffer(allocationSize); + } + const layout = this._data.copyTo(data, { format }); + + return new VideoFrame(data, { + format, + codedWidth: this._data.getCodedWidth(), + codedHeight: this._data.getCodedHeight(), + colorSpace: this._data.getColorSpace(), + layout, + timestamp: this.microsecondTimestamp, + duration: this.microsecondDuration, + }); + } else if (isVideoFrame(this._data)) { return new VideoFrame(this._data, { timestamp: this.microsecondTimestamp, duration: this.microsecondDuration || undefined, // Drag 0 duration to undefined, glitches some codecs @@ -748,7 +854,7 @@ export class VideoSample implements Disposable { * Converts this video sample to a * [`CanvasImageSource`](https://udn.realityripple.com/docs/Web/API/CanvasImageSource) for drawing to a canvas. * - * You must use the value returned by this method immediately, as any VideoFrame created internally will + * You must use the value returned by this method immediately, as any VideoFrame created internally may * automatically be closed in the next microtask. */ toCanvasImageSource() { @@ -758,7 +864,7 @@ export class VideoSample implements Disposable { assert(this._data !== null); - if (this._data instanceof Uint8Array) { + if (this._data instanceof VideoSampleResource || this._data instanceof Uint8Array) { // Requires VideoFrame to be defined const videoFrame = this.toVideoFrame(); queueMicrotask(() => videoFrame.close()); // Let's automatically close the frame in the next microtask @@ -891,6 +997,51 @@ const AUDIO_SAMPLE_FORMATS = new Set( ['f32', 'f32-planar', 's16', 's16-planar', 's32', 's32-planar', 'u8', 'u8-planar'], ); +/** + * Abstract base class for custom audio sample resources. Implement this class to provide custom backing + * for AudioSample instances. + * @group Samples + * @public + */ +export abstract class AudioSampleResource { + /** @internal */ + _referenceCount: number = 0; + + /** + * Returns the audio sample format. + * [See sample formats](https://developer.mozilla.org/en-US/docs/Web/API/AudioData/format) + */ + abstract getFormat(): AudioSampleFormat; + + /** Returns the audio sample rate in hertz. */ + abstract getSampleRate(): number; + + /** Returns the number of audio frames in the sample, per channel. */ + abstract getNumberOfFrames(): number; + + /** Returns the number of audio channels. */ + abstract getNumberOfChannels(): number; + + /** Returns the presentation timestamp of the sample in seconds. */ + abstract getTimestamp(): number; + + /** + * Closes this resource, releasing held resources. Called automatically when the last {@link AudioSample} using this + * resource is closed. + */ + abstract close(): void; + + /** + * Returns the number of bytes required to hold the underlying audio data as specified by the given options. + */ + abstract allocationSize(options: AudioSampleCopyToOptions): number; + + /** + * Copies the underlying audio data to an ArrayBuffer or ArrayBufferView as specified by the given options. + */ + abstract copyTo(destination: AllowSharedBufferSource, options: AudioSampleCopyToOptions): void; +} + /** * Metadata used for AudioSample initialization. * @group Samples @@ -943,7 +1094,7 @@ export type AudioSampleCopyToOptions = { */ export class AudioSample implements Disposable { /** @internal */ - _data: AudioData | Uint8Array; + _data: AudioData | Uint8Array | AudioSampleResource; /** @internal */ _closed: boolean = false; @@ -983,7 +1134,8 @@ export class AudioSample implements Disposable { * [`AudioData`](https://developer.mozilla.org/en-US/docs/Web/API/AudioData) or from raw bytes specified in * {@link AudioSampleInit}. */ - constructor(init: AudioData | AudioSampleInit) { + constructor(init: AudioData | AudioSampleInit | AudioSampleResource); + constructor(init: AudioData | AudioSampleInit | AudioSampleResource) { if (isAudioData(init)) { if (init.format === null) { throw new TypeError('AudioData with null format is not supported.'); @@ -997,6 +1149,16 @@ export class AudioSample implements Disposable { this.numberOfChannels = init.numberOfChannels; this.timestamp = init.timestamp / 1e6; this.duration = init.numberOfFrames / init.sampleRate; + } else if (init instanceof AudioSampleResource) { + this._data = init; + init._referenceCount++; + + this.format = init.getFormat(); + this.sampleRate = init.getSampleRate(); + this.numberOfFrames = init.getNumberOfFrames(); + this.numberOfChannels = init.getNumberOfChannels(); + this.timestamp = init.getTimestamp(); + this.duration = this.numberOfFrames / this.sampleRate; } else { if (!init || typeof init !== 'object') { throw new TypeError('Invalid AudioDataInit: must be an object.'); @@ -1072,6 +1234,10 @@ export class AudioSample implements Disposable { throw new Error('AudioSample is closed.'); } + if (this._data instanceof AudioSampleResource) { + return this._data.allocationSize(options); + } + const destFormat = options.format ?? this.format; const frameOffset = options.frameOffset ?? 0; @@ -1124,6 +1290,10 @@ export class AudioSample implements Disposable { throw new Error('AudioSample is closed.'); } + if (this._data instanceof AudioSampleResource) { + return this._data.copyTo(destination, options); + } + const { planeIndex, format, frameCount: optFrameCount, frameOffset: optFrameOffset } = options; const destFormat = format ?? this.format; @@ -1253,9 +1423,14 @@ export class AudioSample implements Disposable { throw new Error('AudioSample is closed.'); } - if (isAudioData(this._data)) { + if (this._data instanceof AudioSampleResource) { + const sample = new AudioSample(this._data); + sample.setTimestamp(this.timestamp); // Make sure the timestamp is correct + + return sample; + } else if (isAudioData(this._data)) { const sample = new AudioSample(this._data.clone()); - sample.setTimestamp(this.timestamp); // Make sure the timestamp is precise (beyond microsecond accuracy) + sample.setTimestamp(this.timestamp); return sample; } else { @@ -1281,7 +1456,12 @@ export class AudioSample implements Disposable { finalizationRegistry?.unregister(this); - if (isAudioData(this._data)) { + if (this._data instanceof AudioSampleResource) { + this._data._referenceCount--; + if (this._data._referenceCount === 0) { + this._data.close(); + } + } else if (isAudioData(this._data)) { this._data.close(); } else { this._data = new Uint8Array(0); @@ -1299,42 +1479,15 @@ export class AudioSample implements Disposable { throw new Error('AudioSample is closed.'); } - if (isAudioData(this._data)) { + if (this._data instanceof AudioSampleResource) { + return this._createAudioDataFromData(); + } else if (isAudioData(this._data)) { if (this._data.timestamp === this.microsecondTimestamp) { // Timestamp matches, let's just return the data (but cloned) return this._data.clone(); } else { // It's impossible to simply change an AudioData's timestamp, so we'll need to create a new one - if (formatIsPlanar(this.format)) { - const size = this.allocationSize({ planeIndex: 0, format: this.format }); - const data = new ArrayBuffer(size * this.numberOfChannels); - - // We gotta read out each plane individually - for (let i = 0; i < this.numberOfChannels; i++) { - this.copyTo(new Uint8Array(data, i * size, size), { planeIndex: i, format: this.format }); - } - - return new AudioData({ - format: this.format, - sampleRate: this.sampleRate, - numberOfFrames: this.numberOfFrames, - numberOfChannels: this.numberOfChannels, - timestamp: this.microsecondTimestamp, - data, - }); - } else { - const data = new ArrayBuffer(this.allocationSize({ planeIndex: 0, format: this.format })); - this.copyTo(data, { planeIndex: 0, format: this.format }); - - return new AudioData({ - format: this.format, - sampleRate: this.sampleRate, - numberOfFrames: this.numberOfFrames, - numberOfChannels: this.numberOfChannels, - timestamp: this.microsecondTimestamp, - data, - }); - } + return this._createAudioDataFromData(); } } else { return new AudioData({ @@ -1350,6 +1503,40 @@ export class AudioSample implements Disposable { } } + /** @internal */ + _createAudioDataFromData() { + if (formatIsPlanar(this.format)) { + const size = this.allocationSize({ planeIndex: 0, format: this.format }); + const data = new ArrayBuffer(size * this.numberOfChannels); + + // We gotta read out each plane individually + for (let i = 0; i < this.numberOfChannels; i++) { + this.copyTo(new Uint8Array(data, i * size, size), { planeIndex: i, format: this.format }); + } + + return new AudioData({ + format: this.format, + sampleRate: this.sampleRate, + numberOfFrames: this.numberOfFrames, + numberOfChannels: this.numberOfChannels, + timestamp: this.microsecondTimestamp, + data, + }); + } else { + const data = new ArrayBuffer(this.allocationSize({ planeIndex: 0, format: this.format })); + this.copyTo(data, { planeIndex: 0, format: this.format }); + + return new AudioData({ + format: this.format, + sampleRate: this.sampleRate, + numberOfFrames: this.numberOfFrames, + numberOfChannels: this.numberOfChannels, + timestamp: this.microsecondTimestamp, + data, + }); + } + } + /** Convert this audio sample to an AudioBuffer for use with the Web Audio API. */ toAudioBuffer() { if (this._closed) { diff --git a/test/browser/custom-sample-resource.test.ts b/test/browser/custom-sample-resource.test.ts new file mode 100644 index 0000000..46f6f39 --- /dev/null +++ b/test/browser/custom-sample-resource.test.ts @@ -0,0 +1,203 @@ +import { expect, test } from 'vitest'; +import { + VideoSampleResource, + VideoSampleColorSpace, + VideoSample, + AudioSampleResource, + AudioSample, +} from '../../src/sample.js'; +import { toUint8Array } from '../../src/misc.js'; + +class ImageVideoSampleResource extends VideoSampleResource { + constructor(public image: HTMLImageElement | null) { + super(); + } + + getFormat(): VideoPixelFormat { + return 'RGBA'; + } + + getCodedWidth() { + return this.image!.width; + } + + getCodedHeight() { + return this.image!.height; + } + + getColorSpace() { + return new VideoSampleColorSpace({ + matrix: 'rgb', + primaries: 'bt709', + transfer: 'iec61966-2-1', + fullRange: true, + }); + } + + allocationSize() { + return this.image!.width * this.image!.height * 4; + } + + copyTo(destination: AllowSharedBufferSource): PlaneLayout[] { + const canvas = new OffscreenCanvas(this.image!.width, this.image!.height); + const ctx = canvas.getContext('2d')!; + + ctx.drawImage(this.image!, 0, 0); + + const imageData = ctx.getImageData(0, 0, this.image!.width, this.image!.height); + toUint8Array(destination).set(imageData.data); + + return [{ + offset: 0, + stride: this.image!.width * 4, + }]; + } + + close() { + this.image = null; + } +} + +test('Custom VideoSample resource usage', async () => { + const image = new Image(); + image.src = '/red-image.png'; + await image.decode(); + + const resource = new ImageVideoSampleResource(image); + const sample = new VideoSample(resource, { timestamp: 0 }); + expect(sample.format).toBe('RGBA'); + expect(sample.codedWidth).toBe(2048); + expect(sample.codedHeight).toBe(2048); + expect(sample.colorSpace).toEqual({ + matrix: 'rgb', + primaries: 'bt709', + transfer: 'iec61966-2-1', + fullRange: true, + }); + + const videoFrame = sample.toVideoFrame(); + const canvas = new OffscreenCanvas(1024, 1024); + const context = canvas.getContext('2d')!; + context.drawImage(videoFrame, 0, 0); + const imageData = context.getImageData(0, 0, 1024, 1024); + expect([...imageData.data.slice(0, 4)]).toEqual([255, 0, 0, 255]); + + videoFrame.close(); + + const clone = sample.clone(); + sample.close(); + expect(resource.image).not.toBe(null); + + clone.close(); + expect(resource.image).toBe(null); +}); + +test('Custom VideoSample resource, invalid construction', async () => { + const image = new Image(); + image.src = '/red-image.png'; + await image.decode(); + + const resource = new ImageVideoSampleResource(image); + // @ts-expect-error Wrong + expect(() => new VideoSample(resource)).toThrow(TypeError); + // @ts-expect-error Wrong + expect(() => new VideoSample(resource, { duration: 0 })).toThrow(TypeError); +}); + +class AudioBufferAudioSampleResource extends AudioSampleResource { + constructor( + public audioBuffer: AudioBuffer | null, + public timestamp: number, + ) { + super(); + } + + getFormat(): AudioSampleFormat { + return 'f32-planar'; + } + + getSampleRate() { + return this.audioBuffer!.sampleRate; + } + + getNumberOfFrames() { + return this.audioBuffer!.length; + } + + getNumberOfChannels() { + return this.audioBuffer!.numberOfChannels; + } + + getTimestamp() { + return this.timestamp; + } + + allocationSize() { + const frameCount = this.audioBuffer!.length; + return frameCount * Float32Array.BYTES_PER_ELEMENT; + } + + copyTo(destination: AllowSharedBufferSource, options: AudioDataCopyToOptions): void { + const frameCount = this.audioBuffer!.length; + const planeIndex = options.planeIndex; + + const channel = this.audioBuffer!.getChannelData(planeIndex); + + let destView: Float32Array; + if (destination instanceof ArrayBuffer) { + destView = new Float32Array(destination); + } else if (destination instanceof SharedArrayBuffer) { + destView = new Float32Array(destination); + } else { + destView = new Float32Array(destination.buffer, destination.byteOffset, destination.byteLength / 4); + } + + const frameOffset = options.frameOffset ?? 0; + const copyFrameCount = options.frameCount !== undefined ? options.frameCount : (frameCount - frameOffset); + + for (let i = 0; i < copyFrameCount; i++) { + destView[i] = channel[i + frameOffset]!; + } + } + + close() { + this.audioBuffer = null; + } +} + +test('Custom AudioSample resource usage', async () => { + const audioContext = new AudioContext(); + const audioBuffer = audioContext.createBuffer(2, 48000, 48000); + + const leftChannel = audioBuffer.getChannelData(0); + const rightChannel = audioBuffer.getChannelData(1); + for (let i = 0; i < 48000; i++) { + const t = i / 48000; + leftChannel[i] = Math.sin(2 * Math.PI * 440 * t); + rightChannel[i] = Math.cos(2 * Math.PI * 440 * t); + } + + const resource = new AudioBufferAudioSampleResource(audioBuffer, 0); + const sample = new AudioSample(resource); + + expect(sample.format).toBe('f32-planar'); + expect(sample.sampleRate).toBe(48000); + expect(sample.numberOfFrames).toBe(48000); + expect(sample.numberOfChannels).toBe(2); + expect(sample.timestamp).toBe(0); + expect(sample.duration).toBeCloseTo(1); + + const buffer = new ArrayBuffer(48000 * 4); + sample.copyTo(buffer, { planeIndex: 0 }); + + const view = new Float32Array(buffer); + expect(view[0]).toBeCloseTo(0, 4); + expect(view[13]).toBeCloseTo(Math.sin(2 * Math.PI * 440 * (13 / 48000))); + + const clone = sample.clone(); + sample.close(); + expect(resource.audioBuffer).not.toBe(null); + + clone.close(); + expect(resource.audioBuffer).toBe(null); +}); diff --git a/test/public/red-image.png b/test/public/red-image.png new file mode 100644 index 0000000..3c1b03c Binary files /dev/null and b/test/public/red-image.png differ