Fixed missing mipmapping in the VideoSample transform case, now guaranteed via a manual mipmap algo

dev/convert.html

@@ -129,6 +129,7 @@
 			},
 			*/
 			video: {
+				height: 320,
 				//codec: 'avc',
 				//bitrate: new Mediabunny.Quality({
 				//	quality: Infinity,
@@ -157,7 +158,7 @@
 				}
 			},
 			trim: {
-				end: 0.5,
+				end: 5,
 				//start: -2,
 				//end: 10,
 				//start: 300.14984567374756 - 100,
@@ -197,7 +198,7 @@
 		document.body.append(video);
 		video.play();
 
-		download(new Blob([target.buffer]), 'converted' + outputFormat.fileExtension);
+		//download(new Blob([target.buffer]), 'converted' + outputFormat.fileExtension);
 
 		function download(blob, filename) {
 			const url = URL.createObjectURL(blob);

src/media-sink.ts

@@ -1989,21 +1989,12 @@ export class CanvasSink {
 		}) as CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;
 		assert(context);
 
-		context.imageSmoothingQuality = 'high';
+		sample._drawWithFitAndMipmapping(canvas, context, {
-
-		if (!canvasIsNew) {
-			if (!this._alpha && isFirefox()) {
-				context.fillStyle = 'black';
-				context.fillRect(0, 0, width, height);
-			} else {
-				context.clearRect(0, 0, width, height);
-			}
-		}
-
-		sample.drawWithFit(context, {
 			fit: this._fit,
 			rotation: this._rotation,
 			crop: this._crop,
+			targetIsFresh: canvasIsNew,
+			fillBlack: !this._alpha && isFirefox(),
 		});
 
 		const result = {

src/sample.ts

@@ -534,6 +534,9 @@ export class VideoSample implements Disposable {
 			if (init.duration !== undefined && (!Number.isFinite(init.duration) || init.duration < 0)) {
 				throw new TypeError('init.duration, when provided, must be a non-negative number.');
 			}
+			if (init.visibleRect !== undefined) {
+				validateRectangle(init.visibleRect, 'init.visibleRect');
+			}
 
 			if (typeof VideoFrame !== 'undefined') {
 				return new VideoSample(
@@ -541,6 +544,13 @@ export class VideoSample implements Disposable {
 						timestamp: Math.trunc(init.timestamp! * SECOND_TO_MICROSECOND_FACTOR),
 						// Drag 0 to undefined
 						duration: Math.trunc((init.duration ?? 0) * SECOND_TO_MICROSECOND_FACTOR) || undefined,
+						// WebCodecs wants DOMRectInit
+						visibleRect: init.visibleRect && {
+							x: init.visibleRect.left,
+							y: init.visibleRect.top,
+							width: init.visibleRect.width,
+							height: init.visibleRect.height,
+						},
 					}),
 					init,
 				);
@@ -565,7 +575,9 @@ export class VideoSample implements Disposable {
 				throw new TypeError('Could not determine dimensions.');
 			}
 
-			const canvas = new OffscreenCanvas(width, height);
+			const visibleRect = init.visibleRect ?? { left: 0, top: 0, width, height };
+
+			const canvas = new OffscreenCanvas(visibleRect.width, visibleRect.height);
 			const context = canvas.getContext('2d', {
 				alpha: isFirefox(), // Firefox has VideoFrame glitches with opaque canvases
 				willReadFrequently: true,
@@ -577,15 +589,15 @@ export class VideoSample implements Disposable {
 				);
 			}
 
-			// Draw it to a canvas
+			// Draw it to a canvas, cropped to the visible rect
-			context.drawImage(data, 0, 0);
+			context.drawImage(data, -visibleRect.left, -visibleRect.top);
 			this._data = canvas;
 			this._layout = null;
 
 			this.format = 'RGBX';
-			this.visibleRect = { left: 0, top: 0, width, height };
+			this.visibleRect = { left: 0, top: 0, width: visibleRect.width, height: visibleRect.height };
-			this.squarePixelWidth = width;
+			this.squarePixelWidth = visibleRect.width;
-			this.squarePixelHeight = height;
+			this.squarePixelHeight = visibleRect.height;
 			this.rotation = init.rotation ?? 0;
 			this.timestamp = init.timestamp!;
 			this.duration = init.duration ?? 0;
@@ -1364,6 +1376,94 @@ export class VideoSample implements Disposable {
 		return { sx, sy, sWidth, sHeight };
 	}
 
+	/**
+	 * Draws the sample onto the target canvas with fit behavior, manually mipmapping on strong downscales for quality.
+	 * @internal
+	 */
+	_drawWithFitAndMipmapping(
+		targetCanvas: HTMLCanvasElement | OffscreenCanvas,
+		targetContext: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D,
+		options: {
+			fit: 'fill' | 'contain' | 'cover';
+			rotation: Rotation;
+			crop: CropRectangle | undefined;
+			/** Freshly-created target canvases carry no stale pixels and don't need to be cleared. */
+			targetIsFresh: boolean;
+			/** Paints on a black background instead of transparency (for alpha discard or opaque emulation). */
+			fillBlack: boolean;
+		},
+	) {
+		const targetWidth = targetCanvas.width;
+		const targetHeight = targetCanvas.height;
+
+		const [rotatedWidth, rotatedHeight] = options.rotation % 180 === 0
+			? [this.squarePixelWidth, this.squarePixelHeight]
+			: [this.squarePixelHeight, this.squarePixelWidth];
+		const sourceWidth = options.crop ? options.crop.width : rotatedWidth;
+		const sourceHeight = options.crop ? options.crop.height : rotatedHeight;
+
+		// Once we downscale by more than 2x, bilinear filtering starts skipping pixels and the result aliases. So in
+		// that case, we do manual mipmapping: draw at a larger size first, then iteratively halve the image until we
+		// arrive at the target size. Sucks that we have to do this honestly, but imageSmoothingQuality is very flaky -
+		// some resources are just not mipmapped in the browser and thus always fall back to bilinear downsampling.
+		let mipLevels = 0;
+		if (2 * targetWidth < sourceWidth && 2 * targetHeight < sourceHeight) {
+			mipLevels = Math.floor(Math.log2(
+				Math.min(sourceWidth / targetWidth, sourceHeight / targetHeight),
+			));
+		}
+
+		const drawWidth = targetWidth * 2 ** mipLevels;
+		const drawHeight = targetHeight * 2 ** mipLevels;
+
+		const { canvas, context, isNew } = mipLevels > 0
+			? getTransformationCanvas(drawWidth, drawHeight)
+			: { canvas: targetCanvas, context: targetContext, isNew: options.targetIsFresh };
+
+		context.imageSmoothingQuality = 'high';
+
+		if (options.fillBlack) {
+			context.fillStyle = 'black';
+			context.fillRect(0, 0, drawWidth, drawHeight);
+		} else if (!isNew) {
+			// Reused canvases carry stale pixels from a prior draw
+			context.clearRect(0, 0, drawWidth, drawHeight);
+		}
+
+		this.drawWithFit(context, {
+			fit: options.fit,
+			rotation: options.rotation,
+			crop: options.crop,
+		});
+
+		// Walk down the mip chain by repeatedly drawing the canvas onto itself at half size. The 'copy' composite
+		// operation makes sure pixels get replaced instead of blended with what's already there.
+		context.globalCompositeOperation = 'copy';
+		for (let i = mipLevels; i > 1; i--) {
+			const levelWidth = targetWidth * 2 ** i;
+			const levelHeight = targetHeight * 2 ** i;
+
+			context.drawImage(canvas, 0, 0, levelWidth, levelHeight, 0, 0, levelWidth / 2, levelHeight / 2);
+		}
+		context.globalCompositeOperation = 'source-over';
+
+		if (mipLevels > 0) {
+			// We'd love to skip this step and hand over the oversized mip canvas cropped with visibleRect, but sadly,
+			// all browsers ignore a frame's visibleRect when encoding it and consume the full coded frame instead
+			// (violating the WebCodecs spec, I think? See https://issues.chromium.org/issues/543284189). So, the result
+			// must live on a canvas of exactly the target size, which is why the last halving step draws onto the
+			// target canvas directly. 'copy' also conveniently disposes of any stale pixels there.
+			targetContext.imageSmoothingQuality = 'high';
+			targetContext.globalCompositeOperation = 'copy';
+			targetContext.drawImage(
+				canvas,
+				0, 0, 2 * targetWidth, 2 * targetHeight,
+				0, 0, targetWidth, targetHeight,
+			);
+			targetContext.globalCompositeOperation = 'source-over';
+		}
+	}
+
 	/**
 	 * Converts this video sample to a
 	 * [`CanvasImageSource`](https://udn.realityripple.com/docs/Web/API/CanvasImageSource) for drawing to a canvas.
@@ -1495,71 +1595,14 @@ export class VideoSample implements Disposable {
 
 		// We need to handle it ourselves, and we use canvases to do it
 
-		let canvas: HTMLCanvasElement | OffscreenCanvas | null = null;
+		const { canvas, context, isNew } = getTransformationCanvas(description.width, description.height);
-		let canvasIsNew = false;
 
-		for (const entry of transformationCanvasCache) {
+		this._drawWithFitAndMipmapping(canvas, context, {
-			if (entry.canvas.width === description.width && entry.canvas.height === description.height) {
-				canvas = entry.canvas;
-				entry.age = transformationCanvasCacheNextAge++;
-
-				break;
-			}
-		}
-
-		if (canvas === null) {
-			if (typeof OffscreenCanvas !== 'undefined') {
-				canvas = new OffscreenCanvas(description.width, description.height);
-			} else {
-				if (typeof window === 'undefined' || typeof document === 'undefined') {
-					throw new Error(
-						'Cannot transform VideoSamples in this environment. Either run in an environment with'
-						+ ' OffscreenCanvas or HTMLCanvasElement, or supply a custom VideoSample transformer using'
-						+ ' registerVideoSampleTransformer().',
-					);
-				}
-
-				canvas = document.createElement('canvas');
-				canvas.width = description.width;
-				canvas.height = description.height;
-			}
-
-			canvasIsNew = true;
-
-			if (transformationCanvasCache.length >= TRANSFORMATION_CANVAS_CACHE_MAX_SIZE) {
-				transformationCanvasCache.splice(arrayArgmin(transformationCanvasCache, x => x.age), 1);
-			}
-
-			transformationCanvasCache.push({
-				canvas,
-				age: transformationCanvasCacheNextAge++,
-			});
-		}
-
-		const context = canvas.getContext('2d', {
-			alpha: true,
-		}) as CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;
-		if (!context) {
-			throw new Error(
-				'The \'2d\' canvas context is required to transform VideoSamples. Register a custom transformer using'
-				+ ' registerVideoSampleTransformer to work around this limitation.',
-			);
-		}
-
-		context.imageSmoothingQuality = 'high';
-
-		if (description.alpha === 'discard') {
-			context.fillStyle = 'black';
-			context.fillRect(0, 0, description.width, description.height);
-		} else if (!canvasIsNew) {
-			// Cached canvases carry stale pixels from a prior draw
-			context.clearRect(0, 0, description.width, description.height);
-		}
-
-		this.drawWithFit(context, {
 			fit: description.fit,
 			rotation: description.rotation,
 			crop: description.crop,
+			targetIsFresh: isNew,
+			fillBlack: description.alpha === 'discard',
 		});
 
 		return new VideoSample(canvas, {
@@ -1728,10 +1771,60 @@ export const registerVideoSampleTransformer = (
 const TRANSFORMATION_CANVAS_CACHE_MAX_SIZE = 3;
 const transformationCanvasCache: {
 	canvas: HTMLCanvasElement | OffscreenCanvas;
+	context: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;
 	age: number;
 }[] = [];
 let transformationCanvasCacheNextAge = 0;
 
+const getTransformationCanvas = (width: number, height: number) => {
+	for (const entry of transformationCanvasCache) {
+		if (entry.canvas.width === width && entry.canvas.height === height) {
+			entry.age = transformationCanvasCacheNextAge++;
+			return { canvas: entry.canvas, context: entry.context, isNew: false };
+		}
+	}
+
+	let canvas: HTMLCanvasElement | OffscreenCanvas;
+	if (typeof OffscreenCanvas !== 'undefined') {
+		canvas = new OffscreenCanvas(width, height);
+	} else {
+		if (typeof window === 'undefined' || typeof document === 'undefined') {
+			throw new Error(
+				'Cannot transform VideoSamples in this environment. Either run in an environment with'
+				+ ' OffscreenCanvas or HTMLCanvasElement, or supply a custom VideoSample transformer using'
+				+ ' registerVideoSampleTransformer().',
+			);
+		}
+
+		canvas = document.createElement('canvas');
+		canvas.width = width;
+		canvas.height = height;
+	}
+
+	const context = canvas.getContext('2d', {
+		alpha: true,
+		willReadFrequently: false,
+	}) as CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;
+	if (!context) {
+		throw new Error(
+			'The \'2d\' canvas context is required to transform VideoSamples. Register a custom transformer using'
+			+ ' registerVideoSampleTransformer to work around this limitation.',
+		);
+	}
+
+	if (transformationCanvasCache.length >= TRANSFORMATION_CANVAS_CACHE_MAX_SIZE) {
+		transformationCanvasCache.splice(arrayArgmin(transformationCanvasCache, x => x.age), 1);
+	}
+
+	transformationCanvasCache.push({
+		canvas,
+		context,
+		age: transformationCanvasCacheNextAge++,
+	});
+
+	return { canvas, context, isNew: true };
+};
+
 /**
  * Describes the color space of a {@link VideoSample}. Corresponds to the WebCodecs API's VideoColorSpace.
  * @group Samples

test/browser/conversion.test.ts

@@ -759,3 +759,27 @@ test('Pre-signaled pause signal', async () => {
 	await conversion.execute();
 	expect(conversion.state).toBe('done');
 });
+
+test('Resizing at various scale factors', async () => {
+	// The source is 1080p. 720p downscales by less than 2x, 240p downscales by more than 2x (which kicks in the manual
+	// mipmapping path), and 1440p upscales.
+	for (const height of [720, 240, 1440]) {
+		using input = new Input({
+			source: new UrlSource('/video.mp4'),
+			formats: ALL_FORMATS,
+		});
+
+		const output = new Output({ format: new Mp4OutputFormat(), target: new BufferTarget() });
+		const conversion = await Conversion.init({
+			input,
+			output,
+			video: { height },
+			trim: { end: 1 },
+		});
+		await conversion.execute();
+
+		using result = new Input({ source: new BufferSource(output.target.buffer!), formats: ALL_FORMATS });
+		const videoTrack = await result.getPrimaryVideoTrack();
+		expect(await videoTrack!.getDisplayHeight()).toBe(height);
+	}
+});