From 4affad934ea4bafa73f8a53a50d2036e6b761a04 Mon Sep 17 00:00:00 2001 From: Tom Yang <82199152+tomyang11@users.noreply.github.com> Date: Tue, 16 Jun 2026 04:42:24 -0700 Subject: [PATCH] Add per-sink decoder preferences (hardwareAcceleration, optimizeForLatency) (#406) * Add per-sink decoder preferences to VideoSampleSink and CanvasSink Adds an optional VideoSinkDecoderOptions ({ hardwareAcceleration, optimizeForLatency }) parameter to VideoSampleSink, exposed on CanvasSink via options.decoderOptions, applied to the decoder config before the VideoDecoderWrapper is constructed. Motivation: applications that run many sinks concurrently (multi-track video editors) need to manage hardware decode sessions deliberately - the number of concurrent hardware sessions is OS-limited, undocumented, and exceeding it fails silently on some platforms (macOS VideoToolbox accepts configure() and decode() and simply never outputs). Such an application places overflow sinks on 'prefer-software' explicitly. optimizeForLatency is exposed alongside it since it is the other WebCodecs decoder-config preference an application may want per sink. The override composes with the existing interlaced-AVC Chromium workaround, which runs later and can only strengthen the preference toward software. Validation mirrors decode.ts's validateVideoDecodingConfig. Co-Authored-By: Claude Opus 4.8 * Modify docs --------- Co-authored-by: Claude Opus 4.8 Co-authored-by: Vanilagy <1696106+Vanilagy@users.noreply.github.com> --- docs/guide/media-sinks.md | 11 ++++++++ src/index.ts | 1 + src/media-sink.ts | 57 ++++++++++++++++++++++++++++++++++++--- 3 files changed, 66 insertions(+), 3 deletions(-) diff --git a/docs/guide/media-sinks.md b/docs/guide/media-sinks.md index aed2ce2..6d32b3b 100644 --- a/docs/guide/media-sinks.md +++ b/docs/guide/media-sinks.md @@ -225,6 +225,11 @@ Create the sink like so: import { VideoSampleSink } from 'mediabunny'; const sink = new VideoSampleSink(videoTrack); + +// Optionally, configure the decoder: +const sink = new VideoSampleSink(videoTrack, { + hardwarePreference: 'prefer-software', +}); ``` #### Single retrieval @@ -363,6 +368,8 @@ type CanvasSinkOptions = { rotation?: 0 | 90 | 180 | 270; crop?: { left: number; top: number; width: number; height: number }; poolSize?: number; + alpha?: boolean; + decoderOptions?: VideoSinkDecoderOptions; }; ``` - `width`\ @@ -380,6 +387,10 @@ type CanvasSinkOptions = { Specifies the rectangular region of the input video to crop to. The crop region will automatically be clamped to the dimensions of the input video track. Cropping is performed after rotation but before resizing. The crop region is in the _display pixel space_ of the underlying video data. - `poolSize`\ See [Canvas pool](#canvas-pool). +- `alpha`\ + Whether the output canvases should have transparency instead of a black background. Defaults to `false`. Set this to `true` when using this sink to read transparent videos. +- `decoderOptions`\ + Additional preferences for the underlying video decoder. Some examples: ```ts diff --git a/src/index.ts b/src/index.ts index 241229d..3f7e95a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -270,6 +270,7 @@ export { EncodedPacketSink, type PacketRetrievalOptions, VideoSampleSink, + type VideoSinkDecoderOptions, type WrappedAudioBuffer, type WrappedCanvas, } from './media-sink'; diff --git a/src/media-sink.ts b/src/media-sink.ts index c245b16..36f6174 100644 --- a/src/media-sink.ts +++ b/src/media-sink.ts @@ -1749,6 +1749,42 @@ const colorAlphaMergerWorkerCode = () => { }; }; +/** + * Describes additional decoder preferences for video sinks. + * @group Media sinks + * @public + */ +export type VideoSinkDecoderOptions = { + /** + * A hint that configures the hardware acceleration method of the decoder. This is best left on `'no-preference'`, + * the default. + */ + hardwareAcceleration?: 'no-preference' | 'prefer-hardware' | 'prefer-software'; + /** + * Hint that the selected decoder should be configured to minimize the number of packets that have to be decoded + * before video frames are output. + */ + optimizeForLatency?: boolean; +}; + +const validateVideoSinkDecoderOptions = (decoderOptions: VideoSinkDecoderOptions) => { + if (!decoderOptions || typeof decoderOptions !== 'object') { + throw new TypeError('decoderOptions must be an object.'); + } + if ( + decoderOptions.hardwareAcceleration !== undefined + && !['no-preference', 'prefer-hardware', 'prefer-software'].includes(decoderOptions.hardwareAcceleration) + ) { + throw new TypeError( + 'decoderOptions.hardwareAcceleration, when provided, must be \'no-preference\', \'prefer-hardware\' or' + + ' \'prefer-software\'.', + ); + } + if (decoderOptions.optimizeForLatency !== undefined && typeof decoderOptions.optimizeForLatency !== 'boolean') { + throw new TypeError('decoderOptions.optimizeForLatency, when provided, must be a boolean.'); + } +}; + /** * A sink that retrieves decoded video samples (video frames) from a video track. * @group Media sinks @@ -1757,16 +1793,20 @@ const colorAlphaMergerWorkerCode = () => { export class VideoSampleSink extends BaseMediaSampleSink { /** @internal */ _track: InputVideoTrack; + /** @internal */ + _decoderOptions: VideoSinkDecoderOptions; /** Creates a new {@link VideoSampleSink} for the given {@link InputVideoTrack}. */ - constructor(videoTrack: InputVideoTrack) { + constructor(videoTrack: InputVideoTrack, decoderOptions: VideoSinkDecoderOptions = {}) { if (!(videoTrack instanceof InputVideoTrack)) { throw new TypeError('videoTrack must be an InputVideoTrack.'); } + validateVideoSinkDecoderOptions(decoderOptions); super(); this._track = videoTrack; + this._decoderOptions = decoderOptions; } /** @internal */ @@ -1783,10 +1823,16 @@ export class VideoSampleSink extends BaseMediaSampleSink { const codec = await this._track.getCodec(); const rotation = await this._track.getRotation(); - const decoderConfig = await this._track.getDecoderConfig(); + let decoderConfig = await this._track.getDecoderConfig(); const timeResolution = await this._track.getTimeResolution(); assert(codec && decoderConfig); + decoderConfig = { + ...decoderConfig, + hardwareAcceleration: this._decoderOptions.hardwareAcceleration, + optimizeForLatency: this._decoderOptions.optimizeForLatency, + }; + return new VideoDecoderWrapper(onSample, onError, codec, decoderConfig, rotation, timeResolution); } @@ -1903,6 +1949,8 @@ export type CanvasSinkOptions = { * canvas is created each time. */ poolSize?: number; + /** Additional preferences for the underlying video decoder. */ + decoderOptions?: VideoSinkDecoderOptions; }; /** @@ -1982,12 +2030,15 @@ export class CanvasSink { ) { throw new TypeError('poolSize must be a non-negative integer.'); } + if (options.decoderOptions !== undefined) { + validateVideoSinkDecoderOptions(options.decoderOptions); + } this._videoTrack = videoTrack; this._alpha = options.alpha ?? false; this._options = options; this._fit = options.fit ?? 'fill'; - this._videoSampleSink = new VideoSampleSink(videoTrack); + this._videoSampleSink = new VideoSampleSink(videoTrack, options.decoderOptions); this._canvasPool = Array.from({ length: options.poolSize ?? 0 }, () => null); }