Add VideoSample.encodeOptions, optimized ArrayBuffer-backed VideoSample cloning

docs/guide/packets-and-samples.md

@@ -330,9 +330,12 @@ videoSample.microsecondDuration; // => Duration in microseconds
 videoSample.colorSpace; // => VideoColorSpace
 
 videoSample.visibleRect; // Rectangle
+
+// Encode options used when this sample is passed to an encoder
+videoSample.encodeOptions; // => VideoEncoderEncodeOptions (defaults to {})
 ```
 
-While all of these properties are read-only, you can use the `setTimestamp`, `setDuration` and `setRotation` methods to modify some of the metadata of the video sample.
+While all of these properties are read-only, you can use the `setTimestamp`, `setDuration`, `setRotation` and `setEncodeOptions` methods to modify some of the metadata of the video sample.
 
 ::: warning
 Timestamps can be [negative](#negative-timestamps).

src/index.ts

@@ -145,6 +145,7 @@ export {
 export {
 	type AnyIterable,
 	ConcurrentRunner,
+	type DeepReadonly,
 	EventEmitter,
 	type EventListenerOptions,
 	type FilePath,

src/media-source.ts

@@ -460,12 +460,14 @@ class VideoEncoderWrapper {
 				const keyFrameInterval = this.encodingConfig.keyFrameInterval ?? 2;
 				const multipleOfKeyFrameInterval = Math.floor(sampleToEncode.timestamp / keyFrameInterval);
 
+				const mergedEncodeOptions = { ...sampleToEncode.encodeOptions, ...encodeOptions };
+
 				// Ensure a key frame every keyFrameInterval seconds. It is important that all video tracks
 				// follow the same "key frame" rhythm, because aligned key frames are required to start new
 				// fragments in ISOBMFF or clusters in Matroska (or at least desirable).
 				const finalEncodeOptions = {
-					...encodeOptions,
+					...mergedEncodeOptions,
-					keyFrame: encodeOptions?.keyFrame
+					keyFrame: mergedEncodeOptions.keyFrame
 						|| keyFrameInterval === 0
 						|| multipleOfKeyFrameInterval !== this.lastMultipleOfKeyFrameInterval,
 				};

src/misc.ts

@@ -482,6 +482,15 @@ export const SECOND_TO_MICROSECOND_FACTOR = 1e6 * (1 + Number.EPSILON);
  */
 export type SetRequired<T, K extends keyof T> = T & Required<Pick<T, K>>;
 
+/**
+ * Recursively makes all properties of T readonly.
+ * @group Miscellaneous
+ * @public
+ */
+export type DeepReadonly<T> = T extends object ? {
+	readonly [K in keyof T]: DeepReadonly<T[K]>;
+} : T;
+
 /**
  * Sets all keys K of T to be optional.
  * @group Miscellaneous

src/sample.ts

@@ -30,6 +30,7 @@ import {
 	roundToMultiple,
 	arrayArgmin,
 	MaybePromise,
+	DeepReadonly,
 } from './misc';
 
 polyfillSymbolDispose();
@@ -232,6 +233,11 @@ export type VideoSampleInit = {
 	displayWidth?: number | undefined;
 	/** Height of the frame in pixels after applying aspect ratio adjustments and rotation. */
 	displayHeight?: number | undefined;
+	/** The encode options to use when this sample is passed to an encoder. */
+	encodeOptions?: DeepReadonly<VideoEncoderEncodeOptions>;
+
+	/** @internal */
+	_doNotCopy?: boolean;
 };
 
 /**
@@ -279,6 +285,8 @@ export class VideoSample implements Disposable {
 	readonly duration!: number;
 	/** The color space of the frame. */
 	readonly colorSpace!: VideoSampleColorSpace;
+	/** The encode options to use when this sample is passed to an encoder. */
+	readonly encodeOptions!: DeepReadonly<VideoEncoderEncodeOptions>;
 
 	/** The width of the frame in pixels. */
 	get codedWidth() {
@@ -414,7 +422,9 @@ export class VideoSample implements Disposable {
 				);
 			}
 
-			this._data = toUint8Array(data).slice(); // Copy it
+			this._data = init._doNotCopy
+				? toUint8Array(data)
+				: toUint8Array(data).slice(); // Copy it
 			this._layout = init.layout ?? createDefaultPlaneLayout(init.format, init.codedWidth!, init.codedHeight!);
 
 			this.format = init.format;
@@ -633,6 +643,8 @@ export class VideoSample implements Disposable {
 			);
 		}
 
+		this.encodeOptions = init?.encodeOptions ?? {};
+
 		this.pixelAspectRatio = simplifyRational({
 			num: this.squarePixelWidth * this.codedHeight,
 			den: this.squarePixelHeight * this.codedWidth,
@@ -653,12 +665,14 @@ export class VideoSample implements Disposable {
 				timestamp: this.timestamp,
 				duration: this.duration,
 				rotation: this.rotation,
+				encodeOptions: this.encodeOptions,
 			});
 		} else if (isVideoFrame(this._data)) {
 			return new VideoSample(this._data.clone(), {
 				timestamp: this.timestamp,
 				duration: this.duration,
 				rotation: this.rotation,
+				encodeOptions: this.encodeOptions,
 			});
 		} else if (this._data instanceof Uint8Array) {
 			assert(this._layout);
@@ -675,6 +689,10 @@ export class VideoSample implements Disposable {
 				visibleRect: this.visibleRect,
 				displayWidth: this.displayWidth,
 				displayHeight: this.displayHeight,
+				encodeOptions: this.encodeOptions,
+
+				// It's already been copied, if we copy it again we make the clone unnecessarily expensive
+				_doNotCopy: true,
 			});
 		} else {
 			return new VideoSample(this._data, {
@@ -688,6 +706,7 @@ export class VideoSample implements Disposable {
 				visibleRect: this.visibleRect,
 				displayWidth: this.displayWidth,
 				displayHeight: this.displayHeight,
+				encodeOptions: this.encodeOptions,
 			});
 		}
 	}
@@ -1566,6 +1585,16 @@ export class VideoSample implements Disposable {
 		(this.duration as number) = newDuration;
 	}
 
+	/** Sets the encode options used when this sample is passed to an encoder. */
+	setEncodeOptions(newEncodeOptions: VideoEncoderEncodeOptions) {
+		if (!newEncodeOptions || typeof newEncodeOptions !== 'object') {
+			throw new TypeError('newEncodeOptions must be an object.');
+		}
+
+		// eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion
+		(this.encodeOptions as DeepReadonly<VideoEncoderEncodeOptions>) = newEncodeOptions;
+	}
+
 	/** Calls `.close()`. */
 	[Symbol.dispose]() {
 		this.close();

test/browser/video-samples.test.ts

@@ -1,5 +1,11 @@
 import { expect, test } from 'vitest';
 import { VideoSample } from '../../src/sample.js';
+import { VideoSampleSource } from '../../src/media-source.js';
+import { Output } from '../../src/output.js';
+import { Mp4OutputFormat } from '../../src/output-format.js';
+import { BufferTarget } from '../../src/target.js';
+import { QUALITY_MEDIUM } from '../../src/encode.js';
+import { PacketType } from '../../src/packet.js';
 
 test('allocationSize', async () => {
 	{
@@ -443,3 +449,83 @@ test('null format', async () => {
 	await sample.copyTo(buffer, { format: 'BGRA' });
 	await sample.copyTo(buffer, { format: 'BGRX' });
 });
+
+test('encodeOptions', () => {
+	const canvas = new OffscreenCanvas(100, 100);
+	canvas.getContext('2d');
+
+	// The default is an empty object
+	using a = new VideoSample(canvas, { timestamp: 0 });
+	expect(a.encodeOptions).toEqual({});
+
+	// It can be overridden via the setter
+	a.setEncodeOptions({ keyFrame: true });
+	expect(a.encodeOptions).toEqual({ keyFrame: true });
+
+	// Cloning carries the encode options over
+	using aClone = a.clone();
+	expect(aClone.encodeOptions).toEqual({ keyFrame: true });
+
+	// It can be set via the constructor
+	using b = new VideoSample(canvas, { timestamp: 0, encodeOptions: { keyFrame: true } });
+	expect(b.encodeOptions).toEqual({ keyFrame: true });
+});
+
+test('encodeOptions, key frame forcing', async () => {
+	const samples = Array.from({ length: 5 }, (_, i) => makeRedSample(i / 30, { keyFrame: true }));
+
+	await encodeAndAssertPacketTypes(
+		samples,
+		['key', 'key', 'key', 'key', 'key'],
+	);
+});
+
+test('encodeOptions add() override', async () => {
+	const samples = Array.from({ length: 5 }, (_, i) => makeRedSample(i / 30, { keyFrame: true }));
+
+	await encodeAndAssertPacketTypes(
+		samples,
+		['key', 'delta', 'delta', 'delta', 'delta'],
+		samples.map(() => ({ keyFrame: false })),
+	);
+});
+
+const makeRedSample = (timestamp: number, encodeOptions?: VideoEncoderEncodeOptions) => {
+	const canvas = new OffscreenCanvas(100, 100);
+	const ctx = canvas.getContext('2d')!;
+	ctx.fillStyle = 'red';
+	ctx.fillRect(0, 0, 100, 100);
+
+	return new VideoSample(canvas, {
+		timestamp,
+		duration: 1 / 30,
+		encodeOptions,
+	});
+};
+
+const encodeAndAssertPacketTypes = async (
+	samples: VideoSample[],
+	expectedPacketTypes: PacketType[],
+	addOptions?: (VideoEncoderEncodeOptions | undefined)[],
+) => {
+	const output = new Output({
+		format: new Mp4OutputFormat(),
+		target: new BufferTarget(),
+	});
+
+	let i = 0;
+	const videoSource = new VideoSampleSource({
+		codec: 'vp8',
+		bitrate: QUALITY_MEDIUM,
+		onEncodedPacket: packet => expect(packet.type).toBe(expectedPacketTypes[i++]),
+	});
+	output.addVideoTrack(videoSource);
+	await output.start();
+
+	for (let i = 0; i < samples.length; i++) {
+		await videoSource.add(samples[i]!, addOptions?.[i]);
+		samples[i]!.close();
+	}
+
+	await output.finalize();
+};