From 63b81901850b50ef378f069b301628ec7c2bd466 Mon Sep 17 00:00:00 2001 From: Vanilagy <1696106+Vanilagy@users.noreply.github.com> Date: Sun, 24 Aug 2025 16:37:03 +0200 Subject: [PATCH] Add logic for encoding video frames that change size over time (#63) --- dev/convert.html | 2 + docs/guide/converting-media-files.md | 2 + docs/guide/media-sources.md | 2 + docs/guide/packets-and-samples.md | 11 +++++ examples/media-player/media-player.ts | 5 ++- src/conversion.ts | 3 +- src/encode.ts | 14 ++++++ src/media-sink.ts | 42 +++++------------ src/media-source.ts | 65 ++++++++++++++++++++++----- src/sample.ts | 58 ++++++++++++++++++++++++ 10 files changed, 160 insertions(+), 44 deletions(-) diff --git a/dev/convert.html b/dev/convert.html index 094ed90..ba67311 100644 --- a/dev/convert.html +++ b/dev/convert.html @@ -74,6 +74,8 @@ }, */ video: () => ({ + codec: 'avc', + //fit: 'contain', //frameRate: 27.123, //width: 320, //forceTranscode: true, diff --git a/docs/guide/converting-media-files.md b/docs/guide/converting-media-files.md index 14505df..6e4b8ce 100644 --- a/docs/guide/converting-media-files.md +++ b/docs/guide/converting-media-files.md @@ -141,6 +141,8 @@ If `width` or `height` is used in conjunction with `rotation`, they control the If you want to apply max/min constraints to a video's dimensions, check out [track-specific options](#track-specific-options). +In the rare case that the input video changes size over time, the `fit` field can be used to control the size change behavior (see [`VideoEncodingConfig`](./media-sources#video-encoding-config)). When unset, the behavior is `'passThrough'`. + ### Adjusting frame rate The `frameRate` property can be used to set the frame rate of the output video in Hz. If not specified, the original input frame rate will be used (which may be variable). diff --git a/docs/guide/media-sources.md b/docs/guide/media-sources.md index 57b8cbe..96dd142 100644 --- a/docs/guide/media-sources.md +++ b/docs/guide/media-sources.md @@ -54,6 +54,7 @@ type VideoEncodingConfig = { hardwareAcceleration?: 'no-preference' | 'prefer-hardware' | 'prefer-software'; scalabilityMode?: string; contentHint?: string; + sizeChangeBehavior?: 'deny' | 'passThrough' | 'fill' | 'contain' | 'cover'; onEncodedPacket?: ( packet: EncodedPacket, @@ -73,6 +74,7 @@ type VideoEncodingConfig = { - `hardwareAcceleration`: A hint that configures the hardware acceleration method of this codec. This is best left on `'no-preference'`. - `scalabilityMode`: An encoding scalability mode identifier as defined by [WebRTC-SVC](https://w3c.github.io/webrtc-svc/#scalabilitymodes*). - `contentHint`: An encoding video content hint as defined by [mst-content-hint](https://w3c.github.io/mst-content-hint/#video-content-hints). +- `sizeChangeBehavior`: Video frames may change size overtime. This field controls the behavior in case this happens. Defaults to `'deny'`. - `onEncodedPacket`: Called for each successfully encoded packet. Useful for determining encoding progress. - `onEncoderConfig`: Called when the internal encoder config, as used by the WebCodecs API, is created. You can use this to introspect the full codec string. diff --git a/docs/guide/packets-and-samples.md b/docs/guide/packets-and-samples.md index 5780ebd..b9d8a82 100644 --- a/docs/guide/packets-and-samples.md +++ b/docs/guide/packets-and-samples.md @@ -343,6 +343,17 @@ draw( ``` These methods behave like [drawImage](https://developer.mozilla.org/en-US/docs/Web/API/CanvasRenderingContext2D/drawImage) and paint the video frame at the given position with the given dimensions. This method will automatically draw the frame with the correct rotation based on its `rotation` property. +The `drawWithFit` method can be used to draw the video sample to fill an entire canvas with a specified fitting algorithm: +```ts +drawWithFit( + context: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D, + options: { + fit: 'fill' | 'contain' | 'cover'; + rotation?: Rotation; // Overrides the sample's rotation + }, +): void; +``` + If you want to draw the raw underlying image to a canvas directly (without respecting the rotation metadata), then you can use the following method: ```ts videoSample.toCanvasImageSource(); // => VideoFrame | OffscreenCanvas; diff --git a/examples/media-player/media-player.ts b/examples/media-player/media-player.ts index c684116..3c84c89 100644 --- a/examples/media-player/media-player.ts +++ b/examples/media-player/media-player.ts @@ -144,7 +144,10 @@ const initMediaPlayer = async (file: File) => { // For video, let's use a CanvasSink as it handles rotation and closing video samples for us. // Pool size of 2: We'll only ever have the current and the next frame around, so we only need two canvases. - videoSink = videoTrack && new CanvasSink(videoTrack, { poolSize: 2 }); + videoSink = videoTrack && new CanvasSink(videoTrack, { + poolSize: 2, + fit: 'contain', // In case the video changes dimensions over time + }); // For audio, we'll use an AudioBufferSink to directly retrieve AudioBuffers compatible with the Web Audio API audioSink = audioTrack && new AudioBufferSink(audioTrack); diff --git a/src/conversion.ts b/src/conversion.ts index 733b578..b392717 100644 --- a/src/conversion.ts +++ b/src/conversion.ts @@ -59,7 +59,7 @@ export type ConversionVideoOptions = { */ height?: number; /** - * The fitting algorithm in case both width and height are set. + * The fitting algorithm in case both width and height are set, or if the input video changes its size over time. * * - 'fill' will stretch the image to fill the entire box, potentially altering aspect ratio. * - 'contain' will contain the entire image within the box while preserving aspect ratio. This may lead to @@ -623,6 +623,7 @@ export class Conversion { const encodingConfig: VideoEncodingConfig = { codec: encodableCodec, bitrate, + sizeChangeBehavior: trackOptions.fit ?? 'passThrough', onEncodedPacket: sample => this._reportProgress(track.id, sample.timestamp + sample.duration), }; diff --git a/src/encode.ts b/src/encode.ts index e3ea32e..8166c30 100644 --- a/src/encode.ts +++ b/src/encode.ts @@ -43,6 +43,19 @@ export type VideoEncodingConfig = { * all the same key frame interval. */ keyFrameInterval?: number; + /** + * Video frames may change size overtime. This field controls the behavior in case this happens. + * + * - 'deny' (default) will throw an error, requiring all frames to have the exact same dimensions. + * - 'passThrough' will allow the change and directly pass the frame to the encoder. + * - 'fill' will stretch the image to fill the entire original box, potentially altering aspect ratio. + * - 'contain' will contain the entire image within the originalbox while preserving aspect ratio. This may lead to + * letterboxing. + * - 'cover' will scale the image until the entire original box is filled, while preserving aspect ratio. + * + * The "original box" refers to the dimensions of the first encoded frame. + */ + sizeChangeBehavior?: 'deny' | 'passThrough' | 'fill' | 'contain' | 'cover'; /** Called for each successfully encoded packet. Both the packet and the encoding metadata are passed. */ onEncodedPacket?: (packet: EncodedPacket, meta: EncodedVideoChunkMetadata | undefined) => unknown; @@ -66,6 +79,7 @@ export const validateVideoEncodingConfig = (config: VideoEncodingConfig) => { ) { throw new TypeError('config.keyFrameInterval, when provided, must be a non-negative number.'); } + // todo here if (config.onEncodedPacket !== undefined && typeof config.onEncodedPacket !== 'function') { throw new TypeError('config.onEncodedChunk, when provided, must be a function.'); } diff --git a/src/media-sink.ts b/src/media-sink.ts index 6c4de94..6506f4f 100644 --- a/src/media-sink.ts +++ b/src/media-sink.ts @@ -1118,6 +1118,8 @@ export class CanvasSink { /** @internal */ _videoSampleToWrappedCanvas(sample: VideoSample): WrappedCanvas { let canvas = this._canvasPool[this._nextCanvasIndex]; + let canvasIsNew = false; + if (!canvas) { if (typeof document !== 'undefined') { // Prefer an HTMLCanvasElement @@ -1131,6 +1133,8 @@ export class CanvasSink { if (this._canvasPool.length > 0) { this._canvasPool[this._nextCanvasIndex] = canvas; } + + canvasIsNew = true; } if (this._canvasPool.length > 0) { @@ -1143,40 +1147,14 @@ export class CanvasSink { context.resetTransform(); - // These variables specify where the final sample will be drawn on the canvas - let dx: number; - let dy: number; - let newWidth: number; - let newHeight: number; - - if (this._fit === 'fill') { - dx = 0; - dy = 0; - newWidth = this._width; - newHeight = this._height; - } else { - const [sampleWidth, sampleHeight] = this._rotation % 180 === 0 - ? [sample.codedWidth, sample.codedHeight] - : [sample.codedHeight, sample.codedWidth]; - - const scale = this._fit === 'contain' - ? Math.min(this._width / sampleWidth, this._height / sampleHeight) - : Math.max(this._width / sampleWidth, this._height / sampleHeight); - newWidth = sampleWidth * scale; - newHeight = sampleHeight * scale; - dx = (this._width - newWidth) / 2; - dy = (this._height - newHeight) / 2; + if (!canvasIsNew) { + context.clearRect(0, 0, this._width, this._height); } - const aspectRatioChange = this._rotation % 180 === 0 ? 1 : newWidth / newHeight; - context.translate(this._width / 2, this._height / 2); - context.rotate(this._rotation * Math.PI / 180); - // This aspect ratio compensation is done so that we can draw the sample with the intended dimensions and - // don't need to think about how those dimensions change after the rotation - context.scale(1 / aspectRatioChange, aspectRatioChange); - context.translate(-this._width / 2, -this._height / 2); - - context.drawImage(sample.toCanvasImageSource(), dx, dy, newWidth, newHeight); + sample.drawWithFit(context, { + fit: this._fit, + rotation: this._rotation, + }); const result = { canvas, diff --git a/src/media-source.ts b/src/media-source.ts index 14b1d93..d8730e1 100644 --- a/src/media-source.ts +++ b/src/media-source.ts @@ -191,8 +191,9 @@ class VideoEncoderWrapper { private encoder: VideoEncoder | null = null; private muxer: Muxer | null = null; private lastMultipleOfKeyFrameInterval = -1; - private lastWidth: number | null = null; - private lastHeight: number | null = null; + private codedWidth: number | null = null; + private codedHeight: number | null = null; + private resizeCanvas: HTMLCanvasElement | OffscreenCanvas | null = null; private customEncoder: CustomVideoEncoder | null = null; private customEncoderCallSerializer = new CallSerializer(); @@ -213,16 +214,60 @@ class VideoEncoderWrapper { this.source._ensureValidAdd(); // Ensure video sample size remains constant - if (this.lastWidth !== null && this.lastHeight !== null) { - if (videoSample.codedWidth !== this.lastWidth || videoSample.codedHeight !== this.lastHeight) { - throw new Error( - `Video sample size must remain constant. Expected ${this.lastWidth}x${this.lastHeight},` - + ` got ${videoSample.codedWidth}x${videoSample.codedHeight}.`, - ); + if (this.codedWidth !== null && this.codedHeight !== null) { + if (videoSample.codedWidth !== this.codedWidth || videoSample.codedHeight !== this.codedHeight) { + const sizeChangeBehavior = this.encodingConfig.sizeChangeBehavior ?? 'deny'; + + if (sizeChangeBehavior === 'passThrough') { + // Do nada + } else if (sizeChangeBehavior === 'deny') { + throw new Error( + `Video sample size must remain constant. Expected ${this.codedWidth}x${this.codedHeight},` + + ` got ${videoSample.codedWidth}x${videoSample.codedHeight}. To allow the sample size to` + + ` change over time, set \`sizeChangeBehavior\` to a value other than 'strict' in the` + + ` encoding options.`, + ); + } else { + let canvasIsNew = false; + + if (!this.resizeCanvas) { + if (typeof document !== 'undefined') { + // Prefer an HTMLCanvasElement + this.resizeCanvas = document.createElement('canvas'); + this.resizeCanvas.width = this.codedWidth; + this.resizeCanvas.height = this.codedHeight; + } else { + this.resizeCanvas = new OffscreenCanvas(this.codedWidth, this.codedHeight); + } + + canvasIsNew = true; + } + + const context = this.resizeCanvas.getContext('2d', { alpha: false }) as + CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D; + assert(context); + + if (!canvasIsNew) { + context.clearRect(0, 0, this.codedWidth, this.codedHeight); + } + + videoSample.drawWithFit(context, { fit: sizeChangeBehavior }); + + if (shouldClose) { + videoSample.close(); + } + + videoSample = new VideoSample(this.resizeCanvas, { + timestamp: videoSample.timestamp, + duration: videoSample.duration, + rotation: videoSample.rotation, + }); + shouldClose = true; + } } } else { - this.lastWidth = videoSample.codedWidth; - this.lastHeight = videoSample.codedHeight; + this.codedWidth = videoSample.codedWidth; + this.codedHeight = videoSample.codedHeight; } if (!this.encoderInitialized) { diff --git a/src/sample.ts b/src/sample.ts index 44ec41b..2ccf78d 100644 --- a/src/sample.ts +++ b/src/sample.ts @@ -534,6 +534,64 @@ export class VideoSample { context.restore(); } + /** + * Draws the sample in the middle of the canvas corresponding to the context with the specified fit behavior. + */ + drawWithFit(context: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D, options: { + /** + * Controls the fitting algorithm. + * + * - 'fill' will stretch the image to fill the entire box, potentially altering aspect ratio. + * - 'contain' will contain the entire image within the box while preserving aspect ratio. This may lead to + * letterboxing. + * - 'cover' will scale the image until the entire box is filled, while preserving aspect ratio. + */ + fit: 'fill' | 'contain' | 'cover'; + /** A way to override rotation. Defaults to the rotation of the sample. */ + rotation?: Rotation; + }) { + const canvasWidth = context.canvas.width; + const canvasHeight = context.canvas.height; + const rotation = options.rotation ?? this.rotation; + + // These variables specify where the final sample will be drawn on the canvas + let dx: number; + let dy: number; + let newWidth: number; + let newHeight: number; + + if (options.fit === 'fill') { + dx = 0; + dy = 0; + newWidth = canvasWidth; + newHeight = canvasHeight; + } else { + const [sampleWidth, sampleHeight] = rotation % 180 === 0 + ? [this.codedWidth, this.codedHeight] + : [this.codedHeight, this.codedWidth]; + + const scale = options.fit === 'contain' + ? Math.min(canvasWidth / sampleWidth, canvasHeight / sampleHeight) + : Math.max(canvasWidth / sampleWidth, canvasHeight / sampleHeight); + newWidth = sampleWidth * scale; + newHeight = sampleHeight * scale; + dx = (canvasWidth - newWidth) / 2; + dy = (canvasHeight - newHeight) / 2; + } + + const aspectRatioChange = rotation % 180 === 0 ? 1 : newWidth / newHeight; + context.translate(canvasWidth / 2, canvasHeight / 2); + context.rotate(rotation * Math.PI / 180); + // This aspect ratio compensation is done so that we can draw the sample with the intended dimensions and + // don't need to think about how those dimensions change after the rotation + context.scale(1 / aspectRatioChange, aspectRatioChange); + context.translate(-canvasWidth / 2, -canvasHeight / 2); + + // Important that we don't use .draw() here since that would take rotation into account, but we wanna handle it + // ourselves here + context.drawImage(this.toCanvasImageSource(), dx, dy, newWidth, newHeight); + } + /** * Converts this video sample to a CanvasImageSource for drawing to a canvas. *