Add logic for encoding video frames that change size over time (#63)

dev/convert.html

@@ -74,6 +74,8 @@
 			},
 			*/
 			video: () => ({
+				codec: 'avc',
+				//fit: 'contain',
 				//frameRate: 27.123,
 				//width: 320,
 				//forceTranscode: true,

docs/guide/converting-media-files.md

@@ -141,6 +141,8 @@ If `width` or `height` is used in conjunction with `rotation`, they control the
 
 If you want to apply max/min constraints to a video's dimensions, check out [track-specific options](#track-specific-options).
 
+In the rare case that the input video changes size over time, the `fit` field can be used to control the size change behavior (see [`VideoEncodingConfig`](./media-sources#video-encoding-config)). When unset, the behavior is `'passThrough'`.
+
 ### Adjusting frame rate
 
 The `frameRate` property can be used to set the frame rate of the output video in Hz. If not specified, the original input frame rate will be used (which may be variable).

docs/guide/media-sources.md

@@ -54,6 +54,7 @@ type VideoEncodingConfig = {
 	hardwareAcceleration?: 'no-preference' | 'prefer-hardware' | 'prefer-software';
 	scalabilityMode?: string;
 	contentHint?: string;
+	sizeChangeBehavior?: 'deny' | 'passThrough' | 'fill' | 'contain' | 'cover';
 
 	onEncodedPacket?: (
 		packet: EncodedPacket,
@@ -73,6 +74,7 @@ type VideoEncodingConfig = {
 - `hardwareAcceleration`: A hint that configures the hardware acceleration method of this codec. This is best left on `'no-preference'`.
 - `scalabilityMode`: An encoding scalability mode identifier as defined by [WebRTC-SVC](https://w3c.github.io/webrtc-svc/#scalabilitymodes*).
 - `contentHint`: An encoding video content hint as defined by [mst-content-hint](https://w3c.github.io/mst-content-hint/#video-content-hints).
+- `sizeChangeBehavior`: Video frames may change size overtime. This field controls the behavior in case this happens. Defaults to `'deny'`. 
 - `onEncodedPacket`: Called for each successfully encoded packet. Useful for determining encoding progress.
 - `onEncoderConfig`: Called when the internal encoder config, as used by the WebCodecs API, is created. You can use this to introspect the full codec string.
 

docs/guide/packets-and-samples.md

@@ -343,6 +343,17 @@ draw(
 ```
 These methods behave like [drawImage](https://developer.mozilla.org/en-US/docs/Web/API/CanvasRenderingContext2D/drawImage) and paint the video frame at the given position with the given dimensions. This method will automatically draw the frame with the correct rotation based on its `rotation` property.
 
+The `drawWithFit` method can be used to draw the video sample to fill an entire canvas with a specified fitting algorithm:
+```ts
+drawWithFit(
+	context: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D,
+	options: {
+		fit: 'fill' | 'contain' | 'cover';
+		rotation?: Rotation; // Overrides the sample's rotation
+	},
+): void;
+```
+
 If you want to draw the raw underlying image to a canvas directly (without respecting the rotation metadata), then you can use the following method:
 ```ts
 videoSample.toCanvasImageSource(); // => VideoFrame | OffscreenCanvas;

examples/media-player/media-player.ts

@@ -144,7 +144,10 @@ const initMediaPlayer = async (file: File) => {
 
 		// For video, let's use a CanvasSink as it handles rotation and closing video samples for us.
 		// Pool size of 2: We'll only ever have the current and the next frame around, so we only need two canvases.
-		videoSink = videoTrack && new CanvasSink(videoTrack, { poolSize: 2 });
+		videoSink = videoTrack && new CanvasSink(videoTrack, {
+			poolSize: 2,
+			fit: 'contain', // In case the video changes dimensions over time
+		});
 		// For audio, we'll use an AudioBufferSink to directly retrieve AudioBuffers compatible with the Web Audio API
 		audioSink = audioTrack && new AudioBufferSink(audioTrack);
 

src/conversion.ts

@@ -59,7 +59,7 @@ export type ConversionVideoOptions = {
 	 */
 	height?: number;
 	/**
-	 * The fitting algorithm in case both width and height are set.
+	 * The fitting algorithm in case both width and height are set, or if the input video changes its size over time.
 	 *
 	 * - 'fill' will stretch the image to fill the entire box, potentially altering aspect ratio.
 	 * - 'contain' will contain the entire image within the box while preserving aspect ratio. This may lead to
@@ -623,6 +623,7 @@ export class Conversion {
 			const encodingConfig: VideoEncodingConfig = {
 				codec: encodableCodec,
 				bitrate,
+				sizeChangeBehavior: trackOptions.fit ?? 'passThrough',
 				onEncodedPacket: sample => this._reportProgress(track.id, sample.timestamp + sample.duration),
 			};
 

src/encode.ts

@@ -43,6 +43,19 @@ export type VideoEncodingConfig = {
 	 * all the same key frame interval.
 	 */
 	keyFrameInterval?: number;
+	/**
+	 * Video frames may change size overtime. This field controls the behavior in case this happens.
+	 *
+	 * - 'deny' (default) will throw an error, requiring all frames to have the exact same dimensions.
+	 * - 'passThrough' will allow the change and directly pass the frame to the encoder.
+	 * - 'fill' will stretch the image to fill the entire original box, potentially altering aspect ratio.
+	 * - 'contain' will contain the entire image within the  originalbox while preserving aspect ratio. This may lead to
+	 * letterboxing.
+	 * - 'cover' will scale the image until the entire original box is filled, while preserving aspect ratio.
+	 *
+	 * The "original box" refers to the dimensions of the first encoded frame.
+	 */
+	sizeChangeBehavior?: 'deny' | 'passThrough' | 'fill' | 'contain' | 'cover';
 
 	/** Called for each successfully encoded packet. Both the packet and the encoding metadata are passed. */
 	onEncodedPacket?: (packet: EncodedPacket, meta: EncodedVideoChunkMetadata | undefined) => unknown;
@@ -66,6 +79,7 @@ export const validateVideoEncodingConfig = (config: VideoEncodingConfig) => {
 	) {
 		throw new TypeError('config.keyFrameInterval, when provided, must be a non-negative number.');
 	}
+	// todo here
 	if (config.onEncodedPacket !== undefined && typeof config.onEncodedPacket !== 'function') {
 		throw new TypeError('config.onEncodedChunk, when provided, must be a function.');
 	}

src/media-sink.ts

@@ -1118,6 +1118,8 @@ export class CanvasSink {
 	/** @internal */
 	_videoSampleToWrappedCanvas(sample: VideoSample): WrappedCanvas {
 		let canvas = this._canvasPool[this._nextCanvasIndex];
+		let canvasIsNew = false;
+
 		if (!canvas) {
 			if (typeof document !== 'undefined') {
 				// Prefer an HTMLCanvasElement
@@ -1131,6 +1133,8 @@ export class CanvasSink {
 			if (this._canvasPool.length > 0) {
 				this._canvasPool[this._nextCanvasIndex] = canvas;
 			}
+
+			canvasIsNew = true;
 		}
 
 		if (this._canvasPool.length > 0) {
@@ -1143,40 +1147,14 @@ export class CanvasSink {
 
 		context.resetTransform();
 
-		// These variables specify where the final sample will be drawn on the canvas
-		let dx: number;
-		let dy: number;
-		let newWidth: number;
-		let newHeight: number;
-
-		if (this._fit === 'fill') {
-			dx = 0;
-			dy = 0;
-			newWidth = this._width;
-			newHeight = this._height;
-		} else {
-			const [sampleWidth, sampleHeight] = this._rotation % 180 === 0
-				? [sample.codedWidth, sample.codedHeight]
-				: [sample.codedHeight, sample.codedWidth];
-
-			const scale = this._fit === 'contain'
-				? Math.min(this._width / sampleWidth, this._height / sampleHeight)
-				: Math.max(this._width / sampleWidth, this._height / sampleHeight);
-			newWidth = sampleWidth * scale;
-			newHeight = sampleHeight * scale;
-			dx = (this._width - newWidth) / 2;
-			dy = (this._height - newHeight) / 2;
+		if (!canvasIsNew) {
+			context.clearRect(0, 0, this._width, this._height);
 		}
 
-		const aspectRatioChange = this._rotation % 180 === 0 ? 1 : newWidth / newHeight;
-		context.translate(this._width / 2, this._height / 2);
-		context.rotate(this._rotation * Math.PI / 180);
-		// This aspect ratio compensation is done so that we can draw the sample with the intended dimensions and
-		// don't need to think about how those dimensions change after the rotation
-		context.scale(1 / aspectRatioChange, aspectRatioChange);
-		context.translate(-this._width / 2, -this._height / 2);
-
-		context.drawImage(sample.toCanvasImageSource(), dx, dy, newWidth, newHeight);
+		sample.drawWithFit(context, {
+			fit: this._fit,
+			rotation: this._rotation,
+		});
 
 		const result = {
 			canvas,

src/media-source.ts

@@ -191,8 +191,9 @@ class VideoEncoderWrapper {
 	private encoder: VideoEncoder | null = null;
 	private muxer: Muxer | null = null;
 	private lastMultipleOfKeyFrameInterval = -1;
-	private lastWidth: number | null = null;
-	private lastHeight: number | null = null;
+	private codedWidth: number | null = null;
+	private codedHeight: number | null = null;
+	private resizeCanvas: HTMLCanvasElement | OffscreenCanvas | null = null;
 
 	private customEncoder: CustomVideoEncoder | null = null;
 	private customEncoderCallSerializer = new CallSerializer();
@@ -213,16 +214,60 @@ class VideoEncoderWrapper {
 			this.source._ensureValidAdd();
 
 			// Ensure video sample size remains constant
-			if (this.lastWidth !== null && this.lastHeight !== null) {
-				if (videoSample.codedWidth !== this.lastWidth || videoSample.codedHeight !== this.lastHeight) {
-					throw new Error(
-						`Video sample size must remain constant. Expected ${this.lastWidth}x${this.lastHeight},`
-						+ ` got ${videoSample.codedWidth}x${videoSample.codedHeight}.`,
-					);
+			if (this.codedWidth !== null && this.codedHeight !== null) {
+				if (videoSample.codedWidth !== this.codedWidth || videoSample.codedHeight !== this.codedHeight) {
+					const sizeChangeBehavior = this.encodingConfig.sizeChangeBehavior ?? 'deny';
+
+					if (sizeChangeBehavior === 'passThrough') {
+						// Do nada
+					} else if (sizeChangeBehavior === 'deny') {
+						throw new Error(
+							`Video sample size must remain constant. Expected ${this.codedWidth}x${this.codedHeight},`
+							+ ` got ${videoSample.codedWidth}x${videoSample.codedHeight}. To allow the sample size to`
+							+ ` change over time, set \`sizeChangeBehavior\` to a value other than 'strict' in the`
+							+ ` encoding options.`,
+						);
+					} else {
+						let canvasIsNew = false;
+
+						if (!this.resizeCanvas) {
+							if (typeof document !== 'undefined') {
+								// Prefer an HTMLCanvasElement
+								this.resizeCanvas = document.createElement('canvas');
+								this.resizeCanvas.width = this.codedWidth;
+								this.resizeCanvas.height = this.codedHeight;
+							} else {
+								this.resizeCanvas = new OffscreenCanvas(this.codedWidth, this.codedHeight);
+							}
+
+							canvasIsNew = true;
+						}
+
+						const context = this.resizeCanvas.getContext('2d', { alpha: false }) as
+							CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;
+						assert(context);
+
+						if (!canvasIsNew) {
+							context.clearRect(0, 0, this.codedWidth, this.codedHeight);
+						}
+
+						videoSample.drawWithFit(context, { fit: sizeChangeBehavior });
+
+						if (shouldClose) {
+							videoSample.close();
+						}
+
+						videoSample = new VideoSample(this.resizeCanvas, {
+							timestamp: videoSample.timestamp,
+							duration: videoSample.duration,
+							rotation: videoSample.rotation,
+						});
+						shouldClose = true;
+					}
 				}
 			} else {
-				this.lastWidth = videoSample.codedWidth;
-				this.lastHeight = videoSample.codedHeight;
+				this.codedWidth = videoSample.codedWidth;
+				this.codedHeight = videoSample.codedHeight;
 			}
 
 			if (!this.encoderInitialized) {

src/sample.ts

@@ -534,6 +534,64 @@ export class VideoSample {
 		context.restore();
 	}
 
+	/**
+	 * Draws the sample in the middle of the canvas corresponding to the context with the specified fit behavior.
+	 */
+	drawWithFit(context: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D, options: {
+		/**
+		 * Controls the fitting algorithm.
+		 *
+		 * - 'fill' will stretch the image to fill the entire box, potentially altering aspect ratio.
+		 * - 'contain' will contain the entire image within the box while preserving aspect ratio. This may lead to
+		 * letterboxing.
+		 * - 'cover' will scale the image until the entire box is filled, while preserving aspect ratio.
+		 */
+		fit: 'fill' | 'contain' | 'cover';
+		/** A way to override rotation. Defaults to the rotation of the sample. */
+		rotation?: Rotation;
+	}) {
+		const canvasWidth = context.canvas.width;
+		const canvasHeight = context.canvas.height;
+		const rotation = options.rotation ?? this.rotation;
+
+		// These variables specify where the final sample will be drawn on the canvas
+		let dx: number;
+		let dy: number;
+		let newWidth: number;
+		let newHeight: number;
+
+		if (options.fit === 'fill') {
+			dx = 0;
+			dy = 0;
+			newWidth = canvasWidth;
+			newHeight = canvasHeight;
+		} else {
+			const [sampleWidth, sampleHeight] = rotation % 180 === 0
+				? [this.codedWidth, this.codedHeight]
+				: [this.codedHeight, this.codedWidth];
+
+			const scale = options.fit === 'contain'
+				? Math.min(canvasWidth / sampleWidth, canvasHeight / sampleHeight)
+				: Math.max(canvasWidth / sampleWidth, canvasHeight / sampleHeight);
+			newWidth = sampleWidth * scale;
+			newHeight = sampleHeight * scale;
+			dx = (canvasWidth - newWidth) / 2;
+			dy = (canvasHeight - newHeight) / 2;
+		}
+
+		const aspectRatioChange = rotation % 180 === 0 ? 1 : newWidth / newHeight;
+		context.translate(canvasWidth / 2, canvasHeight / 2);
+		context.rotate(rotation * Math.PI / 180);
+		// This aspect ratio compensation is done so that we can draw the sample with the intended dimensions and
+		// don't need to think about how those dimensions change after the rotation
+		context.scale(1 / aspectRatioChange, aspectRatioChange);
+		context.translate(-canvasWidth / 2, -canvasHeight / 2);
+
+		// Important that we don't use .draw() here since that would take rotation into account, but we wanna handle it
+		// ourselves here
+		context.drawImage(this.toCanvasImageSource(), dx, dy, newWidth, newHeight);
+	}
+
 	/**
 	 * Converts this video sample to a CanvasImageSource for drawing to a canvas.
 	 *