Add logic for encoding video frames that change size over time (#63)

src/conversion.ts

@@ -59,7 +59,7 @@ export type ConversionVideoOptions = {
 	 */
 	height?: number;
 	/**
-	 * The fitting algorithm in case both width and height are set.
+	 * The fitting algorithm in case both width and height are set, or if the input video changes its size over time.
 	 *
 	 * - 'fill' will stretch the image to fill the entire box, potentially altering aspect ratio.
 	 * - 'contain' will contain the entire image within the box while preserving aspect ratio. This may lead to
@@ -623,6 +623,7 @@ export class Conversion {
 			const encodingConfig: VideoEncodingConfig = {
 				codec: encodableCodec,
 				bitrate,
+				sizeChangeBehavior: trackOptions.fit ?? 'passThrough',
 				onEncodedPacket: sample => this._reportProgress(track.id, sample.timestamp + sample.duration),
 			};
 

src/encode.ts

@@ -43,6 +43,19 @@ export type VideoEncodingConfig = {
 	 * all the same key frame interval.
 	 */
 	keyFrameInterval?: number;
+	/**
+	 * Video frames may change size overtime. This field controls the behavior in case this happens.
+	 *
+	 * - 'deny' (default) will throw an error, requiring all frames to have the exact same dimensions.
+	 * - 'passThrough' will allow the change and directly pass the frame to the encoder.
+	 * - 'fill' will stretch the image to fill the entire original box, potentially altering aspect ratio.
+	 * - 'contain' will contain the entire image within the  originalbox while preserving aspect ratio. This may lead to
+	 * letterboxing.
+	 * - 'cover' will scale the image until the entire original box is filled, while preserving aspect ratio.
+	 *
+	 * The "original box" refers to the dimensions of the first encoded frame.
+	 */
+	sizeChangeBehavior?: 'deny' | 'passThrough' | 'fill' | 'contain' | 'cover';
 
 	/** Called for each successfully encoded packet. Both the packet and the encoding metadata are passed. */
 	onEncodedPacket?: (packet: EncodedPacket, meta: EncodedVideoChunkMetadata | undefined) => unknown;
@@ -66,6 +79,7 @@ export const validateVideoEncodingConfig = (config: VideoEncodingConfig) => {
 	) {
 		throw new TypeError('config.keyFrameInterval, when provided, must be a non-negative number.');
 	}
+	// todo here
 	if (config.onEncodedPacket !== undefined && typeof config.onEncodedPacket !== 'function') {
 		throw new TypeError('config.onEncodedChunk, when provided, must be a function.');
 	}

src/media-sink.ts

@@ -1118,6 +1118,8 @@ export class CanvasSink {
 	/** @internal */
 	_videoSampleToWrappedCanvas(sample: VideoSample): WrappedCanvas {
 		let canvas = this._canvasPool[this._nextCanvasIndex];
+		let canvasIsNew = false;
+
 		if (!canvas) {
 			if (typeof document !== 'undefined') {
 				// Prefer an HTMLCanvasElement
@@ -1131,6 +1133,8 @@ export class CanvasSink {
 			if (this._canvasPool.length > 0) {
 				this._canvasPool[this._nextCanvasIndex] = canvas;
 			}
+
+			canvasIsNew = true;
 		}
 
 		if (this._canvasPool.length > 0) {
@@ -1143,40 +1147,14 @@ export class CanvasSink {
 
 		context.resetTransform();
 
-		// These variables specify where the final sample will be drawn on the canvas
-		let dx: number;
-		let dy: number;
-		let newWidth: number;
-		let newHeight: number;
-
-		if (this._fit === 'fill') {
-			dx = 0;
-			dy = 0;
-			newWidth = this._width;
-			newHeight = this._height;
-		} else {
-			const [sampleWidth, sampleHeight] = this._rotation % 180 === 0
-				? [sample.codedWidth, sample.codedHeight]
-				: [sample.codedHeight, sample.codedWidth];
-
-			const scale = this._fit === 'contain'
-				? Math.min(this._width / sampleWidth, this._height / sampleHeight)
-				: Math.max(this._width / sampleWidth, this._height / sampleHeight);
-			newWidth = sampleWidth * scale;
-			newHeight = sampleHeight * scale;
-			dx = (this._width - newWidth) / 2;
-			dy = (this._height - newHeight) / 2;
+		if (!canvasIsNew) {
+			context.clearRect(0, 0, this._width, this._height);
 		}
 
-		const aspectRatioChange = this._rotation % 180 === 0 ? 1 : newWidth / newHeight;
-		context.translate(this._width / 2, this._height / 2);
-		context.rotate(this._rotation * Math.PI / 180);
-		// This aspect ratio compensation is done so that we can draw the sample with the intended dimensions and
-		// don't need to think about how those dimensions change after the rotation
-		context.scale(1 / aspectRatioChange, aspectRatioChange);
-		context.translate(-this._width / 2, -this._height / 2);
-
-		context.drawImage(sample.toCanvasImageSource(), dx, dy, newWidth, newHeight);
+		sample.drawWithFit(context, {
+			fit: this._fit,
+			rotation: this._rotation,
+		});
 
 		const result = {
 			canvas,

src/media-source.ts

@@ -191,8 +191,9 @@ class VideoEncoderWrapper {
 	private encoder: VideoEncoder | null = null;
 	private muxer: Muxer | null = null;
 	private lastMultipleOfKeyFrameInterval = -1;
-	private lastWidth: number | null = null;
-	private lastHeight: number | null = null;
+	private codedWidth: number | null = null;
+	private codedHeight: number | null = null;
+	private resizeCanvas: HTMLCanvasElement | OffscreenCanvas | null = null;
 
 	private customEncoder: CustomVideoEncoder | null = null;
 	private customEncoderCallSerializer = new CallSerializer();
@@ -213,16 +214,60 @@ class VideoEncoderWrapper {
 			this.source._ensureValidAdd();
 
 			// Ensure video sample size remains constant
-			if (this.lastWidth !== null && this.lastHeight !== null) {
-				if (videoSample.codedWidth !== this.lastWidth || videoSample.codedHeight !== this.lastHeight) {
-					throw new Error(
-						`Video sample size must remain constant. Expected ${this.lastWidth}x${this.lastHeight},`
-						+ ` got ${videoSample.codedWidth}x${videoSample.codedHeight}.`,
-					);
+			if (this.codedWidth !== null && this.codedHeight !== null) {
+				if (videoSample.codedWidth !== this.codedWidth || videoSample.codedHeight !== this.codedHeight) {
+					const sizeChangeBehavior = this.encodingConfig.sizeChangeBehavior ?? 'deny';
+
+					if (sizeChangeBehavior === 'passThrough') {
+						// Do nada
+					} else if (sizeChangeBehavior === 'deny') {
+						throw new Error(
+							`Video sample size must remain constant. Expected ${this.codedWidth}x${this.codedHeight},`
+							+ ` got ${videoSample.codedWidth}x${videoSample.codedHeight}. To allow the sample size to`
+							+ ` change over time, set \`sizeChangeBehavior\` to a value other than 'strict' in the`
+							+ ` encoding options.`,
+						);
+					} else {
+						let canvasIsNew = false;
+
+						if (!this.resizeCanvas) {
+							if (typeof document !== 'undefined') {
+								// Prefer an HTMLCanvasElement
+								this.resizeCanvas = document.createElement('canvas');
+								this.resizeCanvas.width = this.codedWidth;
+								this.resizeCanvas.height = this.codedHeight;
+							} else {
+								this.resizeCanvas = new OffscreenCanvas(this.codedWidth, this.codedHeight);
+							}
+
+							canvasIsNew = true;
+						}
+
+						const context = this.resizeCanvas.getContext('2d', { alpha: false }) as
+							CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;
+						assert(context);
+
+						if (!canvasIsNew) {
+							context.clearRect(0, 0, this.codedWidth, this.codedHeight);
+						}
+
+						videoSample.drawWithFit(context, { fit: sizeChangeBehavior });
+
+						if (shouldClose) {
+							videoSample.close();
+						}
+
+						videoSample = new VideoSample(this.resizeCanvas, {
+							timestamp: videoSample.timestamp,
+							duration: videoSample.duration,
+							rotation: videoSample.rotation,
+						});
+						shouldClose = true;
+					}
 				}
 			} else {
-				this.lastWidth = videoSample.codedWidth;
-				this.lastHeight = videoSample.codedHeight;
+				this.codedWidth = videoSample.codedWidth;
+				this.codedHeight = videoSample.codedHeight;
 			}
 
 			if (!this.encoderInitialized) {

src/sample.ts

@@ -534,6 +534,64 @@ export class VideoSample {
 		context.restore();
 	}
 
+	/**
+	 * Draws the sample in the middle of the canvas corresponding to the context with the specified fit behavior.
+	 */
+	drawWithFit(context: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D, options: {
+		/**
+		 * Controls the fitting algorithm.
+		 *
+		 * - 'fill' will stretch the image to fill the entire box, potentially altering aspect ratio.
+		 * - 'contain' will contain the entire image within the box while preserving aspect ratio. This may lead to
+		 * letterboxing.
+		 * - 'cover' will scale the image until the entire box is filled, while preserving aspect ratio.
+		 */
+		fit: 'fill' | 'contain' | 'cover';
+		/** A way to override rotation. Defaults to the rotation of the sample. */
+		rotation?: Rotation;
+	}) {
+		const canvasWidth = context.canvas.width;
+		const canvasHeight = context.canvas.height;
+		const rotation = options.rotation ?? this.rotation;
+
+		// These variables specify where the final sample will be drawn on the canvas
+		let dx: number;
+		let dy: number;
+		let newWidth: number;
+		let newHeight: number;
+
+		if (options.fit === 'fill') {
+			dx = 0;
+			dy = 0;
+			newWidth = canvasWidth;
+			newHeight = canvasHeight;
+		} else {
+			const [sampleWidth, sampleHeight] = rotation % 180 === 0
+				? [this.codedWidth, this.codedHeight]
+				: [this.codedHeight, this.codedWidth];
+
+			const scale = options.fit === 'contain'
+				? Math.min(canvasWidth / sampleWidth, canvasHeight / sampleHeight)
+				: Math.max(canvasWidth / sampleWidth, canvasHeight / sampleHeight);
+			newWidth = sampleWidth * scale;
+			newHeight = sampleHeight * scale;
+			dx = (canvasWidth - newWidth) / 2;
+			dy = (canvasHeight - newHeight) / 2;
+		}
+
+		const aspectRatioChange = rotation % 180 === 0 ? 1 : newWidth / newHeight;
+		context.translate(canvasWidth / 2, canvasHeight / 2);
+		context.rotate(rotation * Math.PI / 180);
+		// This aspect ratio compensation is done so that we can draw the sample with the intended dimensions and
+		// don't need to think about how those dimensions change after the rotation
+		context.scale(1 / aspectRatioChange, aspectRatioChange);
+		context.translate(-canvasWidth / 2, -canvasHeight / 2);
+
+		// Important that we don't use .draw() here since that would take rotation into account, but we wanna handle it
+		// ourselves here
+		context.drawImage(this.toCanvasImageSource(), dx, dy, newWidth, newHeight);
+	}
+
 	/**
 	 * Converts this video sample to a CanvasImageSource for drawing to a canvas.
 	 *