Merge pull request #212 from JonnyBurger/apply-rotation-to-videoframe

Preserve rotation when calling `.toVideoFrame()`

dev/demux.html

@@ -7,6 +7,37 @@
 	fileInput.type = 'file';
 	document.body.append(fileInput);
 
+	const canvas = document.createElement('canvas');
+	const ctx = canvas.getContext('2d');
+	canvas.style.border = '1px solid black';
+
+	ctx.fillStyle = 'blue';
+	ctx.fillRect(0, 0, canvas.width, canvas.height);
+	ctx.fillStyle = 'red';
+	ctx.fillRect(0, 0, 200, 30);
+
+	document.body.append(canvas);
+
+	console.log("argh", new VideoFrame(canvas, {timestamp: 0, rotation: 90}))
+
+	const sample = new Mediabunny.VideoSample(new VideoFrame(canvas, {timestamp: 0, rotation: 90}));
+	console.log(sample);
+
+	sample.setRotation(0);
+
+	/*
+	//sample.setRotation(90);
+	console.log(sample);
+	
+	const frame = sample.toVideoFrame();
+	console.log(frame)
+	*/
+
+	ctx.clearRect(0, 0, canvas.width, canvas.height);
+	sample.draw(ctx, 0, 0);
+	//sample.drawWithFit(ctx, { fit: 'contain' });
+	//ctx.drawImage(frame, 0, 0);
+
 	fileInput.addEventListener('change', async () => {
 		const file = fileInput.files[0];
 		const input = new Mediabunny.Input({

src/sample.ts

@@ -17,10 +17,26 @@ import {
 	SetRequired,
 	isFirefox,
 	polyfillSymbolDispose,
+	normalizeRotation,
 } from './misc';
 
 polyfillSymbolDispose();
 
+// Patch the type
+declare global {
+	interface VideoFrame {
+		rotation?: number;
+	}
+
+	interface VideoFrameInit {
+		rotation?: number;
+	}
+
+	interface VideoFrameBufferInit {
+		rotation?: number;
+	}
+}
+
 /**
  * Metadata used for VideoSample initialization.
  * @group Samples
@@ -179,12 +195,21 @@ export class VideoSample implements Disposable {
 			this._data = data;
 
 			this.format = data.format;
-			// Copying the display dimensions here, assuming no innate VideoFrame rotation
+
+			// Copying the display dimensions here because the way codedWidth and codedHeight are defined is a bit wrong
+			// (they're defined as the pre-rotation dimensions, not the coded dimensions which is a different thing)
 			this.codedWidth = data.displayWidth;
 			this.codedHeight = data.displayHeight;
+
+			const frameRotation = normalizeRotation(data.rotation ?? 0);
+			if (frameRotation % 180 !== 0) {
+				this.codedWidth = data.displayHeight;
+				this.codedHeight = data.displayWidth;
+			}
+
 			// The VideoFrame's rotation is ignored here. It's still a new field, and I'm not sure of any application
 			// where the browser makes use of it. If a case gets found, I'll add it.
-			this.rotation = init?.rotation ?? 0;
+			this.rotation = init?.rotation ?? frameRotation;
 			this.timestamp = init?.timestamp ?? data.timestamp / 1e6;
 			this.duration = init?.duration ?? (data.duration ?? 0) / 1e6;
 			this.colorSpace = data.colorSpace;
@@ -379,24 +404,33 @@ export class VideoSample implements Disposable {
 		assert(this._data !== null);
 
 		if (isVideoFrame(this._data)) {
-			return new VideoFrame(this._data, {
+			const init: VideoFrameInit = {
 				timestamp: this.microsecondTimestamp,
 				duration: this.microsecondDuration || undefined, // Drag 0 duration to undefined, glitches some codecs
-			});
+				// The rotation field here is additive (adds to the existing rotation), but we want it to act like an
+				// override instead, so we need to do some subtraction
+				rotation: this.rotation - (this._data.rotation ?? 0),
+			};
+			return new VideoFrame(this._data, init);
 		} else if (this._data instanceof Uint8Array) {
-			return new VideoFrame(this._data, {
+			const init: VideoFrameBufferInit = {
 				format: this.format!,
 				codedWidth: this.codedWidth,
 				codedHeight: this.codedHeight,
 				timestamp: this.microsecondTimestamp,
 				duration: this.microsecondDuration || undefined,
 				colorSpace: this.colorSpace,
-			});
+				rotation: this.rotation,
+			};
+
+			return new VideoFrame(this._data, init);
 		} else {
-			return new VideoFrame(this._data, {
+			const init: VideoFrameInit = {
 				timestamp: this.microsecondTimestamp,
 				duration: this.microsecondDuration || undefined,
-			});
+				rotation: this.rotation,
+			};
+			return new VideoFrame(this._data, init);
 		}
 	}
 
@@ -523,19 +557,24 @@ export class VideoSample implements Disposable {
 			throw new Error('VideoSample is closed.');
 		}
 
-		({ sx, sy, sWidth, sHeight } = this._rotateSourceRegion(sx, sy, sWidth, sHeight, this.rotation));
-
 		const source = this.toCanvasImageSource();
 
+		// Relative to the innate rotation of the source
+		const relativeRotation = typeof VideoFrame !== 'undefined' && source instanceof VideoFrame
+			? normalizeRotation(this.rotation - (source.rotation ?? 0))
+			: this.rotation;
+
+		({ sx, sy, sWidth, sHeight } = this._rotateSourceRegion(sx, sy, sWidth, sHeight, relativeRotation));
+
 		context.save();
 
 		const centerX = dx + dWidth / 2;
 		const centerY = dy + dHeight / 2;
 
 		context.translate(centerX, centerY);
-		context.rotate(this.rotation * Math.PI / 180);
+		context.rotate(relativeRotation * Math.PI / 180);
 
-		const aspectRatioChange = this.rotation % 180 === 0 ? 1 : dWidth / dHeight;
+		const aspectRatioChange = relativeRotation % 180 === 0 ? 1 : dWidth / dHeight;
 
 		// Scale to compensate for aspect ratio changes when rotated
 		context.scale(1 / aspectRatioChange, aspectRatioChange);
@@ -602,6 +641,13 @@ export class VideoSample implements Disposable {
 		const canvasHeight = context.canvas.height;
 		const rotation = options.rotation ?? this.rotation;
 
+		const source = this.toCanvasImageSource();
+
+		// Relative to the innate rotation of the source
+		const relativeRotation = typeof VideoFrame !== 'undefined' && source instanceof VideoFrame
+			? normalizeRotation(rotation - (source.rotation ?? 0))
+			: rotation;
+
 		const [rotatedWidth, rotatedHeight] = rotation % 180 === 0
 			? [this.codedWidth, this.codedHeight]
 			: [this.codedHeight, this.codedWidth];
@@ -621,7 +667,7 @@ export class VideoSample implements Disposable {
 			options.crop?.top ?? 0,
 			options.crop?.width ?? rotatedWidth,
 			options.crop?.height ?? rotatedHeight,
-			rotation,
+			relativeRotation,
 		);
 
 		if (options.fit === 'fill') {
@@ -645,9 +691,9 @@ export class VideoSample implements Disposable {
 
 		context.save();
 
-		const aspectRatioChange = rotation % 180 === 0 ? 1 : newWidth / newHeight;
+		const aspectRatioChange = relativeRotation % 180 === 0 ? 1 : newWidth / newHeight;
 		context.translate(canvasWidth / 2, canvasHeight / 2);
-		context.rotate(rotation * Math.PI / 180);
+		context.rotate(relativeRotation * Math.PI / 180);
 		// This aspect ratio compensation is done so that we can draw the sample with the intended dimensions and
 		// don't need to think about how those dimensions change after the rotation
 		context.scale(1 / aspectRatioChange, aspectRatioChange);
@@ -655,30 +701,39 @@ export class VideoSample implements Disposable {
 
 		// Important that we don't use .draw() here since that would take rotation into account, but we wanna handle it
 		// ourselves here
-		context.drawImage(this.toCanvasImageSource(), sx, sy, sWidth, sHeight, dx, dy, newWidth, newHeight);
+		context.drawImage(source, sx, sy, sWidth, sHeight, dx, dy, newWidth, newHeight);
 
 		context.restore();
 	}
 
 	/** @internal */
-	_rotateSourceRegion(sx: number, sy: number, sWidth: number, sHeight: number, rotation: number) {
+	_rotateSourceRegion(sx: number, sy: number, sWidth: number, sHeight: number, rotation: Rotation) {
+		let sourceWidth = this.codedWidth;
+		let sourceHeight = this.codedHeight;
+
+		if (typeof VideoFrame !== 'undefined' && this._data instanceof VideoFrame) {
+			// Kinda dirty but has to be done
+			sourceWidth = this._data.displayWidth;
+			sourceHeight = this._data.displayHeight;
+		}
+
 		// The provided sx,sy,sWidth,sHeight refer to the final rotated image, but that's not actually how the image is
 		// stored. Therefore, we must map these back onto the original, pre-rotation image.
 		if (rotation === 90) {
 			[sx, sy, sWidth, sHeight] = [
 				sy,
-				this.codedHeight - sx - sWidth,
+				sourceHeight - sx - sWidth,
 				sHeight,
 				sWidth,
 			];
 		} else if (rotation === 180) {
 			[sx, sy] = [
-				this.codedWidth - sx - sWidth,
+				sourceWidth - sx - sWidth,
-				this.codedHeight - sy - sHeight,
+				sourceHeight - sy - sHeight,
 			];
 		} else if (rotation === 270) {
 			[sx, sy, sWidth, sHeight] = [
-				this.codedWidth - sy - sHeight,
+				sourceWidth - sy - sHeight,
 				sx,
 				sHeight,
 				sWidth,

test/browser/video-sample.test.ts

@@ -0,0 +1,118 @@
+import { expect, test } from 'vitest';
+import { VideoSample } from '../../src/sample.js';
+
+const createCanvas = () => {
+	const canvas = document.createElement('canvas');
+	canvas.width = 300;
+	canvas.height = 150;
+
+	const ctx = canvas.getContext('2d')!;
+
+	ctx.fillStyle = '#0000ff';
+	ctx.fillRect(0, 0, canvas.width, canvas.height);
+
+	ctx.fillStyle = '#ff0000';
+	ctx.fillRect(0, 0, 100, 50);
+
+	return canvas;
+};
+
+type Color = {
+	r: number;
+	g: number;
+	b: number;
+};
+
+const colorDistance = (c1: Color, c2: Color) => {
+	return Math.hypot(c1.r - c2.r, c1.g - c2.g, c1.b - c2.b);
+};
+
+const sampleCanvasColor = (ctx: CanvasRenderingContext2D, x: number, y: number): Color => {
+	const pixel = ctx.getImageData(x, y, 1, 1).data;
+
+	return {
+		r: pixel[0]!,
+		g: pixel[1]!,
+		b: pixel[2]!,
+	};
+};
+
+test('Can create VideoSample from VideoFrame and modify rotation', () => {
+	const frame = new VideoFrame(createCanvas(), { timestamp: 0 });
+	const sample = new VideoSample(frame);
+
+	expect(frame.rotation).toBe(0);
+	expect(sample.rotation).toBe(0);
+
+	const canvas = document.createElement('canvas');
+	canvas.width = 300;
+	canvas.height = 150;
+	const ctx = canvas.getContext('2d')!;
+
+	sample.draw(ctx, 0, 0);
+
+	expect(colorDistance(sampleCanvasColor(ctx, 5, 5), { r: 255, g: 0, b: 0 })).toBeLessThan(10);
+	expect(colorDistance(sampleCanvasColor(ctx, 5, 55), { r: 0, g: 0, b: 255 })).toBeLessThan(10);
+	expect(colorDistance(sampleCanvasColor(ctx, 110, 5), { r: 0, g: 0, b: 255 })).toBeLessThan(10);
+
+	sample.setRotation(90);
+	expect(sample.rotation).toBe(90);
+
+	ctx.clearRect(0, 0, canvas.width, canvas.height);
+	sample.draw(ctx, 0, 0);
+
+	expect(colorDistance(sampleCanvasColor(ctx, 145, 5), { r: 255, g: 0, b: 0 })).toBeLessThan(10);
+	expect(colorDistance(sampleCanvasColor(ctx, 95, 5), { r: 0, g: 0, b: 255 })).toBeLessThan(10);
+	expect(colorDistance(sampleCanvasColor(ctx, 145, 110), { r: 0, g: 0, b: 255 })).toBeLessThan(10);
+
+	const extracted = sample.toVideoFrame();
+	expect(extracted.rotation).toBe(90); // It was changed
+
+	ctx.clearRect(0, 0, canvas.width, canvas.height);
+	sample.drawWithFit(ctx, { fit: 'fill' });
+
+	expect(colorDistance(sampleCanvasColor(ctx, 295, 5), { r: 255, g: 0, b: 0 })).toBeLessThan(10);
+	expect(colorDistance(sampleCanvasColor(ctx, 295, 55), { r: 0, g: 0, b: 255 })).toBeLessThan(10);
+});
+
+test('Can create VideoSample from rotated VideoFrame', () => {
+	const frame = new VideoFrame(createCanvas(), { timestamp: 0, rotation: 90 });
+	const sample = new VideoSample(frame);
+
+	expect(frame.rotation).toBe(90);
+	expect(sample.rotation).toBe(90);
+
+	const canvas = document.createElement('canvas');
+	canvas.width = 300;
+	canvas.height = 150;
+	const ctx = canvas.getContext('2d')!;
+
+	sample.draw(ctx, 0, 0);
+
+	expect(colorDistance(sampleCanvasColor(ctx, 145, 5), { r: 255, g: 0, b: 0 })).toBeLessThan(10);
+	expect(colorDistance(sampleCanvasColor(ctx, 95, 5), { r: 0, g: 0, b: 255 })).toBeLessThan(10);
+	expect(colorDistance(sampleCanvasColor(ctx, 145, 110), { r: 0, g: 0, b: 255 })).toBeLessThan(10);
+
+	ctx.clearRect(0, 0, canvas.width, canvas.height);
+	sample.drawWithFit(ctx, { fit: 'fill' });
+
+	expect(colorDistance(sampleCanvasColor(ctx, 295, 5), { r: 255, g: 0, b: 0 })).toBeLessThan(10);
+	expect(colorDistance(sampleCanvasColor(ctx, 295, 55), { r: 0, g: 0, b: 255 })).toBeLessThan(10);
+
+	sample.setRotation(0);
+	expect(sample.rotation).toBe(0);
+
+	ctx.clearRect(0, 0, canvas.width, canvas.height);
+	sample.draw(ctx, 0, 0);
+
+	expect(colorDistance(sampleCanvasColor(ctx, 5, 5), { r: 255, g: 0, b: 0 })).toBeLessThan(10);
+	expect(colorDistance(sampleCanvasColor(ctx, 5, 55), { r: 0, g: 0, b: 255 })).toBeLessThan(10);
+	expect(colorDistance(sampleCanvasColor(ctx, 110, 5), { r: 0, g: 0, b: 255 })).toBeLessThan(10);
+
+	ctx.clearRect(0, 0, canvas.width, canvas.height);
+	sample.drawWithFit(ctx, { fit: 'fill' });
+
+	expect(colorDistance(sampleCanvasColor(ctx, 5, 5), { r: 255, g: 0, b: 0 })).toBeLessThan(10);
+	expect(colorDistance(sampleCanvasColor(ctx, 5, 55), { r: 0, g: 0, b: 255 })).toBeLessThan(10);
+	expect(colorDistance(sampleCanvasColor(ctx, 110, 5), { r: 0, g: 0, b: 255 })).toBeLessThan(10);
+});