From 0cdf0063e29efb695f93d646691ed79dee5bcbe0 Mon Sep 17 00:00:00 2001 From: Vanilagy <1696106+Vanilagy@users.noreply.github.com> Date: Tue, 2 Jun 2026 18:01:22 +0200 Subject: [PATCH] Add VideoSample.encodeOptions, optimized ArrayBuffer-backed VideoSample cloning --- docs/guide/packets-and-samples.md | 5 +- src/index.ts | 1 + src/media-source.ts | 6 ++- src/misc.ts | 9 ++++ src/sample.ts | 31 ++++++++++- test/browser/video-samples.test.ts | 86 ++++++++++++++++++++++++++++++ 6 files changed, 134 insertions(+), 4 deletions(-) diff --git a/docs/guide/packets-and-samples.md b/docs/guide/packets-and-samples.md index 77c09b7..c6d1e77 100644 --- a/docs/guide/packets-and-samples.md +++ b/docs/guide/packets-and-samples.md @@ -330,9 +330,12 @@ videoSample.microsecondDuration; // => Duration in microseconds videoSample.colorSpace; // => VideoColorSpace videoSample.visibleRect; // Rectangle + +// Encode options used when this sample is passed to an encoder +videoSample.encodeOptions; // => VideoEncoderEncodeOptions (defaults to {}) ``` -While all of these properties are read-only, you can use the `setTimestamp`, `setDuration` and `setRotation` methods to modify some of the metadata of the video sample. +While all of these properties are read-only, you can use the `setTimestamp`, `setDuration`, `setRotation` and `setEncodeOptions` methods to modify some of the metadata of the video sample. ::: warning Timestamps can be [negative](#negative-timestamps). diff --git a/src/index.ts b/src/index.ts index f87c1bd..241229d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -145,6 +145,7 @@ export { export { type AnyIterable, ConcurrentRunner, + type DeepReadonly, EventEmitter, type EventListenerOptions, type FilePath, diff --git a/src/media-source.ts b/src/media-source.ts index fd40e0a..34c4f42 100644 --- a/src/media-source.ts +++ b/src/media-source.ts @@ -460,12 +460,14 @@ class VideoEncoderWrapper { const keyFrameInterval = this.encodingConfig.keyFrameInterval ?? 2; const multipleOfKeyFrameInterval = Math.floor(sampleToEncode.timestamp / keyFrameInterval); + const mergedEncodeOptions = { ...sampleToEncode.encodeOptions, ...encodeOptions }; + // Ensure a key frame every keyFrameInterval seconds. It is important that all video tracks // follow the same "key frame" rhythm, because aligned key frames are required to start new // fragments in ISOBMFF or clusters in Matroska (or at least desirable). const finalEncodeOptions = { - ...encodeOptions, - keyFrame: encodeOptions?.keyFrame + ...mergedEncodeOptions, + keyFrame: mergedEncodeOptions.keyFrame || keyFrameInterval === 0 || multipleOfKeyFrameInterval !== this.lastMultipleOfKeyFrameInterval, }; diff --git a/src/misc.ts b/src/misc.ts index cfc3008..ee938d3 100644 --- a/src/misc.ts +++ b/src/misc.ts @@ -482,6 +482,15 @@ export const SECOND_TO_MICROSECOND_FACTOR = 1e6 * (1 + Number.EPSILON); */ export type SetRequired = T & Required>; +/** + * Recursively makes all properties of T readonly. + * @group Miscellaneous + * @public + */ +export type DeepReadonly = T extends object ? { + readonly [K in keyof T]: DeepReadonly; +} : T; + /** * Sets all keys K of T to be optional. * @group Miscellaneous diff --git a/src/sample.ts b/src/sample.ts index 258345b..bd97ba8 100644 --- a/src/sample.ts +++ b/src/sample.ts @@ -30,6 +30,7 @@ import { roundToMultiple, arrayArgmin, MaybePromise, + DeepReadonly, } from './misc'; polyfillSymbolDispose(); @@ -232,6 +233,11 @@ export type VideoSampleInit = { displayWidth?: number | undefined; /** Height of the frame in pixels after applying aspect ratio adjustments and rotation. */ displayHeight?: number | undefined; + /** The encode options to use when this sample is passed to an encoder. */ + encodeOptions?: DeepReadonly; + + /** @internal */ + _doNotCopy?: boolean; }; /** @@ -279,6 +285,8 @@ export class VideoSample implements Disposable { readonly duration!: number; /** The color space of the frame. */ readonly colorSpace!: VideoSampleColorSpace; + /** The encode options to use when this sample is passed to an encoder. */ + readonly encodeOptions!: DeepReadonly; /** The width of the frame in pixels. */ get codedWidth() { @@ -414,7 +422,9 @@ export class VideoSample implements Disposable { ); } - this._data = toUint8Array(data).slice(); // Copy it + this._data = init._doNotCopy + ? toUint8Array(data) + : toUint8Array(data).slice(); // Copy it this._layout = init.layout ?? createDefaultPlaneLayout(init.format, init.codedWidth!, init.codedHeight!); this.format = init.format; @@ -633,6 +643,8 @@ export class VideoSample implements Disposable { ); } + this.encodeOptions = init?.encodeOptions ?? {}; + this.pixelAspectRatio = simplifyRational({ num: this.squarePixelWidth * this.codedHeight, den: this.squarePixelHeight * this.codedWidth, @@ -653,12 +665,14 @@ export class VideoSample implements Disposable { timestamp: this.timestamp, duration: this.duration, rotation: this.rotation, + encodeOptions: this.encodeOptions, }); } else if (isVideoFrame(this._data)) { return new VideoSample(this._data.clone(), { timestamp: this.timestamp, duration: this.duration, rotation: this.rotation, + encodeOptions: this.encodeOptions, }); } else if (this._data instanceof Uint8Array) { assert(this._layout); @@ -675,6 +689,10 @@ export class VideoSample implements Disposable { visibleRect: this.visibleRect, displayWidth: this.displayWidth, displayHeight: this.displayHeight, + encodeOptions: this.encodeOptions, + + // It's already been copied, if we copy it again we make the clone unnecessarily expensive + _doNotCopy: true, }); } else { return new VideoSample(this._data, { @@ -688,6 +706,7 @@ export class VideoSample implements Disposable { visibleRect: this.visibleRect, displayWidth: this.displayWidth, displayHeight: this.displayHeight, + encodeOptions: this.encodeOptions, }); } } @@ -1566,6 +1585,16 @@ export class VideoSample implements Disposable { (this.duration as number) = newDuration; } + /** Sets the encode options used when this sample is passed to an encoder. */ + setEncodeOptions(newEncodeOptions: VideoEncoderEncodeOptions) { + if (!newEncodeOptions || typeof newEncodeOptions !== 'object') { + throw new TypeError('newEncodeOptions must be an object.'); + } + + // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion + (this.encodeOptions as DeepReadonly) = newEncodeOptions; + } + /** Calls `.close()`. */ [Symbol.dispose]() { this.close(); diff --git a/test/browser/video-samples.test.ts b/test/browser/video-samples.test.ts index 2057799..9f6c424 100644 --- a/test/browser/video-samples.test.ts +++ b/test/browser/video-samples.test.ts @@ -1,5 +1,11 @@ import { expect, test } from 'vitest'; import { VideoSample } from '../../src/sample.js'; +import { VideoSampleSource } from '../../src/media-source.js'; +import { Output } from '../../src/output.js'; +import { Mp4OutputFormat } from '../../src/output-format.js'; +import { BufferTarget } from '../../src/target.js'; +import { QUALITY_MEDIUM } from '../../src/encode.js'; +import { PacketType } from '../../src/packet.js'; test('allocationSize', async () => { { @@ -443,3 +449,83 @@ test('null format', async () => { await sample.copyTo(buffer, { format: 'BGRA' }); await sample.copyTo(buffer, { format: 'BGRX' }); }); + +test('encodeOptions', () => { + const canvas = new OffscreenCanvas(100, 100); + canvas.getContext('2d'); + + // The default is an empty object + using a = new VideoSample(canvas, { timestamp: 0 }); + expect(a.encodeOptions).toEqual({}); + + // It can be overridden via the setter + a.setEncodeOptions({ keyFrame: true }); + expect(a.encodeOptions).toEqual({ keyFrame: true }); + + // Cloning carries the encode options over + using aClone = a.clone(); + expect(aClone.encodeOptions).toEqual({ keyFrame: true }); + + // It can be set via the constructor + using b = new VideoSample(canvas, { timestamp: 0, encodeOptions: { keyFrame: true } }); + expect(b.encodeOptions).toEqual({ keyFrame: true }); +}); + +test('encodeOptions, key frame forcing', async () => { + const samples = Array.from({ length: 5 }, (_, i) => makeRedSample(i / 30, { keyFrame: true })); + + await encodeAndAssertPacketTypes( + samples, + ['key', 'key', 'key', 'key', 'key'], + ); +}); + +test('encodeOptions add() override', async () => { + const samples = Array.from({ length: 5 }, (_, i) => makeRedSample(i / 30, { keyFrame: true })); + + await encodeAndAssertPacketTypes( + samples, + ['key', 'delta', 'delta', 'delta', 'delta'], + samples.map(() => ({ keyFrame: false })), + ); +}); + +const makeRedSample = (timestamp: number, encodeOptions?: VideoEncoderEncodeOptions) => { + const canvas = new OffscreenCanvas(100, 100); + const ctx = canvas.getContext('2d')!; + ctx.fillStyle = 'red'; + ctx.fillRect(0, 0, 100, 100); + + return new VideoSample(canvas, { + timestamp, + duration: 1 / 30, + encodeOptions, + }); +}; + +const encodeAndAssertPacketTypes = async ( + samples: VideoSample[], + expectedPacketTypes: PacketType[], + addOptions?: (VideoEncoderEncodeOptions | undefined)[], +) => { + const output = new Output({ + format: new Mp4OutputFormat(), + target: new BufferTarget(), + }); + + let i = 0; + const videoSource = new VideoSampleSource({ + codec: 'vp8', + bitrate: QUALITY_MEDIUM, + onEncodedPacket: packet => expect(packet.type).toBe(expectedPacketTypes[i++]), + }); + output.addVideoTrack(videoSource); + await output.start(); + + for (let i = 0; i < samples.length; i++) { + await videoSource.add(samples[i]!, addOptions?.[i]); + samples[i]!.close(); + } + + await output.finalize(); +};