diff --git a/docs/guide/media-sinks.md b/docs/guide/media-sinks.md index aed2ce2..6d32b3b 100644 --- a/docs/guide/media-sinks.md +++ b/docs/guide/media-sinks.md @@ -225,6 +225,11 @@ Create the sink like so: import { VideoSampleSink } from 'mediabunny'; const sink = new VideoSampleSink(videoTrack); + +// Optionally, configure the decoder: +const sink = new VideoSampleSink(videoTrack, { + hardwarePreference: 'prefer-software', +}); ``` #### Single retrieval @@ -363,6 +368,8 @@ type CanvasSinkOptions = { rotation?: 0 | 90 | 180 | 270; crop?: { left: number; top: number; width: number; height: number }; poolSize?: number; + alpha?: boolean; + decoderOptions?: VideoSinkDecoderOptions; }; ``` - `width`\ @@ -380,6 +387,10 @@ type CanvasSinkOptions = { Specifies the rectangular region of the input video to crop to. The crop region will automatically be clamped to the dimensions of the input video track. Cropping is performed after rotation but before resizing. The crop region is in the _display pixel space_ of the underlying video data. - `poolSize`\ See [Canvas pool](#canvas-pool). +- `alpha`\ + Whether the output canvases should have transparency instead of a black background. Defaults to `false`. Set this to `true` when using this sink to read transparent videos. +- `decoderOptions`\ + Additional preferences for the underlying video decoder. Some examples: ```ts diff --git a/src/index.ts b/src/index.ts index 241229d..3f7e95a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -270,6 +270,7 @@ export { EncodedPacketSink, type PacketRetrievalOptions, VideoSampleSink, + type VideoSinkDecoderOptions, type WrappedAudioBuffer, type WrappedCanvas, } from './media-sink'; diff --git a/src/media-sink.ts b/src/media-sink.ts index c245b16..36f6174 100644 --- a/src/media-sink.ts +++ b/src/media-sink.ts @@ -1749,6 +1749,42 @@ const colorAlphaMergerWorkerCode = () => { }; }; +/** + * Describes additional decoder preferences for video sinks. + * @group Media sinks + * @public + */ +export type VideoSinkDecoderOptions = { + /** + * A hint that configures the hardware acceleration method of the decoder. This is best left on `'no-preference'`, + * the default. + */ + hardwareAcceleration?: 'no-preference' | 'prefer-hardware' | 'prefer-software'; + /** + * Hint that the selected decoder should be configured to minimize the number of packets that have to be decoded + * before video frames are output. + */ + optimizeForLatency?: boolean; +}; + +const validateVideoSinkDecoderOptions = (decoderOptions: VideoSinkDecoderOptions) => { + if (!decoderOptions || typeof decoderOptions !== 'object') { + throw new TypeError('decoderOptions must be an object.'); + } + if ( + decoderOptions.hardwareAcceleration !== undefined + && !['no-preference', 'prefer-hardware', 'prefer-software'].includes(decoderOptions.hardwareAcceleration) + ) { + throw new TypeError( + 'decoderOptions.hardwareAcceleration, when provided, must be \'no-preference\', \'prefer-hardware\' or' + + ' \'prefer-software\'.', + ); + } + if (decoderOptions.optimizeForLatency !== undefined && typeof decoderOptions.optimizeForLatency !== 'boolean') { + throw new TypeError('decoderOptions.optimizeForLatency, when provided, must be a boolean.'); + } +}; + /** * A sink that retrieves decoded video samples (video frames) from a video track. * @group Media sinks @@ -1757,16 +1793,20 @@ const colorAlphaMergerWorkerCode = () => { export class VideoSampleSink extends BaseMediaSampleSink { /** @internal */ _track: InputVideoTrack; + /** @internal */ + _decoderOptions: VideoSinkDecoderOptions; /** Creates a new {@link VideoSampleSink} for the given {@link InputVideoTrack}. */ - constructor(videoTrack: InputVideoTrack) { + constructor(videoTrack: InputVideoTrack, decoderOptions: VideoSinkDecoderOptions = {}) { if (!(videoTrack instanceof InputVideoTrack)) { throw new TypeError('videoTrack must be an InputVideoTrack.'); } + validateVideoSinkDecoderOptions(decoderOptions); super(); this._track = videoTrack; + this._decoderOptions = decoderOptions; } /** @internal */ @@ -1783,10 +1823,16 @@ export class VideoSampleSink extends BaseMediaSampleSink { const codec = await this._track.getCodec(); const rotation = await this._track.getRotation(); - const decoderConfig = await this._track.getDecoderConfig(); + let decoderConfig = await this._track.getDecoderConfig(); const timeResolution = await this._track.getTimeResolution(); assert(codec && decoderConfig); + decoderConfig = { + ...decoderConfig, + hardwareAcceleration: this._decoderOptions.hardwareAcceleration, + optimizeForLatency: this._decoderOptions.optimizeForLatency, + }; + return new VideoDecoderWrapper(onSample, onError, codec, decoderConfig, rotation, timeResolution); } @@ -1903,6 +1949,8 @@ export type CanvasSinkOptions = { * canvas is created each time. */ poolSize?: number; + /** Additional preferences for the underlying video decoder. */ + decoderOptions?: VideoSinkDecoderOptions; }; /** @@ -1982,12 +2030,15 @@ export class CanvasSink { ) { throw new TypeError('poolSize must be a non-negative integer.'); } + if (options.decoderOptions !== undefined) { + validateVideoSinkDecoderOptions(options.decoderOptions); + } this._videoTrack = videoTrack; this._alpha = options.alpha ?? false; this._options = options; this._fit = options.fit ?? 'fill'; - this._videoSampleSink = new VideoSampleSink(videoTrack); + this._videoSampleSink = new VideoSampleSink(videoTrack, options.decoderOptions); this._canvasPool = Array.from({ length: options.poolSize ?? 0 }, () => null); }