Add per-sink decoder preferences (hardwareAcceleration, optimizeForLatency) (#406)

* Add per-sink decoder preferences to VideoSampleSink and CanvasSink

Adds an optional VideoSinkDecoderOptions ({ hardwareAcceleration,
optimizeForLatency }) parameter to VideoSampleSink, exposed on CanvasSink
via options.decoderOptions, applied to the decoder config before the
VideoDecoderWrapper is constructed.

Motivation: applications that run many sinks concurrently (multi-track
video editors) need to manage hardware decode sessions deliberately -
the number of concurrent hardware sessions is OS-limited, undocumented,
and exceeding it fails silently on some platforms (macOS VideoToolbox
accepts configure() and decode() and simply never outputs). Such an
application places overflow sinks on 'prefer-software' explicitly.
optimizeForLatency is exposed alongside it since it is the other
WebCodecs decoder-config preference an application may want per sink.

The override composes with the existing interlaced-AVC Chromium
workaround, which runs later and can only strengthen the preference
toward software.

Validation mirrors decode.ts's validateVideoDecodingConfig.

Co-Authored-By: Claude Opus 4.8 <[email protected]>

* Modify docs

---------

Co-authored-by: Claude Opus 4.8 <[email protected]>
Co-authored-by: Vanilagy <[email protected]>

docs/guide/media-sinks.md

@@ -225,6 +225,11 @@ Create the sink like so:
 import { VideoSampleSink } from 'mediabunny';
 
 const sink = new VideoSampleSink(videoTrack);
+
+// Optionally, configure the decoder:
+const sink = new VideoSampleSink(videoTrack, {
+	hardwarePreference: 'prefer-software',
+});
 ```
 
 #### Single retrieval
@@ -363,6 +368,8 @@ type CanvasSinkOptions = {
 	rotation?: 0 | 90 | 180 | 270;
 	crop?: { left: number; top: number; width: number; height: number };
 	poolSize?: number;
+	alpha?: boolean;
+	decoderOptions?: VideoSinkDecoderOptions;
 };
 ```
 - `width`\
@@ -380,6 +387,10 @@ type CanvasSinkOptions = {
 	Specifies the rectangular region of the input video to crop to. The crop region will automatically be clamped to the dimensions of the input video track. Cropping is performed after rotation but before resizing. The crop region is in the _display pixel space_ of the underlying video data.
 - `poolSize`\
 	See [Canvas pool](#canvas-pool).
+- `alpha`\
+	Whether the output canvases should have transparency instead of a black background. Defaults to `false`. Set this to `true` when using this sink to read transparent videos.
+- `decoderOptions`\
+	Additional preferences for the underlying video decoder.
 
 Some examples:
 ```ts

src/index.ts

@@ -270,6 +270,7 @@ export {
 	EncodedPacketSink,
 	type PacketRetrievalOptions,
 	VideoSampleSink,
+	type VideoSinkDecoderOptions,
 	type WrappedAudioBuffer,
 	type WrappedCanvas,
 } from './media-sink';

src/media-sink.ts

@@ -1749,6 +1749,42 @@ const colorAlphaMergerWorkerCode = () => {
 	};
 };
 
+/**
+ * Describes additional decoder preferences for video sinks.
+ * @group Media sinks
+ * @public
+ */
+export type VideoSinkDecoderOptions = {
+	/**
+	 * A hint that configures the hardware acceleration method of the decoder. This is best left on `'no-preference'`,
+	 * the default.
+	 */
+	hardwareAcceleration?: 'no-preference' | 'prefer-hardware' | 'prefer-software';
+	/**
+	 * Hint that the selected decoder should be configured to minimize the number of packets that have to be decoded
+	 * before video frames are output.
+	 */
+	optimizeForLatency?: boolean;
+};
+
+const validateVideoSinkDecoderOptions = (decoderOptions: VideoSinkDecoderOptions) => {
+	if (!decoderOptions || typeof decoderOptions !== 'object') {
+		throw new TypeError('decoderOptions must be an object.');
+	}
+	if (
+		decoderOptions.hardwareAcceleration !== undefined
+		&& !['no-preference', 'prefer-hardware', 'prefer-software'].includes(decoderOptions.hardwareAcceleration)
+	) {
+		throw new TypeError(
+			'decoderOptions.hardwareAcceleration, when provided, must be \'no-preference\', \'prefer-hardware\' or'
+			+ ' \'prefer-software\'.',
+		);
+	}
+	if (decoderOptions.optimizeForLatency !== undefined && typeof decoderOptions.optimizeForLatency !== 'boolean') {
+		throw new TypeError('decoderOptions.optimizeForLatency, when provided, must be a boolean.');
+	}
+};
+
 /**
  * A sink that retrieves decoded video samples (video frames) from a video track.
  * @group Media sinks
@@ -1757,16 +1793,20 @@ const colorAlphaMergerWorkerCode = () => {
 export class VideoSampleSink extends BaseMediaSampleSink<VideoSample> {
 	/** @internal */
 	_track: InputVideoTrack;
+	/** @internal */
+	_decoderOptions: VideoSinkDecoderOptions;
 
 	/** Creates a new {@link VideoSampleSink} for the given {@link InputVideoTrack}. */
-	constructor(videoTrack: InputVideoTrack) {
+	constructor(videoTrack: InputVideoTrack, decoderOptions: VideoSinkDecoderOptions = {}) {
 		if (!(videoTrack instanceof InputVideoTrack)) {
 			throw new TypeError('videoTrack must be an InputVideoTrack.');
 		}
+		validateVideoSinkDecoderOptions(decoderOptions);
 
 		super();
 
 		this._track = videoTrack;
+		this._decoderOptions = decoderOptions;
 	}
 
 	/** @internal */
@@ -1783,10 +1823,16 @@ export class VideoSampleSink extends BaseMediaSampleSink<VideoSample> {
 
 		const codec = await this._track.getCodec();
 		const rotation = await this._track.getRotation();
-		const decoderConfig = await this._track.getDecoderConfig();
+		let decoderConfig = await this._track.getDecoderConfig();
 		const timeResolution = await this._track.getTimeResolution();
 		assert(codec && decoderConfig);
 
+		decoderConfig = {
+			...decoderConfig,
+			hardwareAcceleration: this._decoderOptions.hardwareAcceleration,
+			optimizeForLatency: this._decoderOptions.optimizeForLatency,
+		};
+
 		return new VideoDecoderWrapper(onSample, onError, codec, decoderConfig, rotation, timeResolution);
 	}
 
@@ -1903,6 +1949,8 @@ export type CanvasSinkOptions = {
 	 * canvas is created each time.
 	 */
 	poolSize?: number;
+	/** Additional preferences for the underlying video decoder. */
+	decoderOptions?: VideoSinkDecoderOptions;
 };
 
 /**
@@ -1982,12 +2030,15 @@ export class CanvasSink {
 		) {
 			throw new TypeError('poolSize must be a non-negative integer.');
 		}
+		if (options.decoderOptions !== undefined) {
+			validateVideoSinkDecoderOptions(options.decoderOptions);
+		}
 
 		this._videoTrack = videoTrack;
 		this._alpha = options.alpha ?? false;
 		this._options = options;
 		this._fit = options.fit ?? 'fill';
-		this._videoSampleSink = new VideoSampleSink(videoTrack);
+		this._videoSampleSink = new VideoSampleSink(videoTrack, options.decoderOptions);
 		this._canvasPool = Array.from({ length: options.poolSize ?? 0 }, () => null);
 	}