From ea9b0501fda3a15003853b9b8fbaa59033a5a4b0 Mon Sep 17 00:00:00 2001 From: JonnyBurger Date: Thu, 6 Nov 2025 10:03:42 +0100 Subject: [PATCH 1/3] Preserve rotation when calling `.toVideoFrame()` --- src/sample.ts | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/sample.ts b/src/sample.ts index 805aebe..b66b636 100644 --- a/src/sample.ts +++ b/src/sample.ts @@ -379,24 +379,34 @@ export class VideoSample implements Disposable { assert(this._data !== null); if (isVideoFrame(this._data)) { - return new VideoFrame(this._data, { + const init: VideoFrameInit = { timestamp: this.microsecondTimestamp, duration: this.microsecondDuration || undefined, // Drag 0 duration to undefined, glitches some codecs - }); + // @ts-expect-error - WebCodecs types do not yet support rotation + rotation: this.rotation, + }; + return new VideoFrame(this._data, init); } else if (this._data instanceof Uint8Array) { - return new VideoFrame(this._data, { + const init: VideoFrameBufferInit = { format: this.format!, codedWidth: this.codedWidth, codedHeight: this.codedHeight, timestamp: this.microsecondTimestamp, duration: this.microsecondDuration || undefined, colorSpace: this.colorSpace, - }); + // @ts-expect-error - WebCodecs types do not yet support rotation + rotation: this.rotation, + }; + + return new VideoFrame(this._data, init); } else { - return new VideoFrame(this._data, { + const init: VideoFrameInit = { timestamp: this.microsecondTimestamp, duration: this.microsecondDuration || undefined, - }); + // @ts-expect-error - WebCodecs types do not yet support rotation + rotation: this.rotation, + }; + return new VideoFrame(this._data, init); } } From f0d0e51537bede0d5f019517a4210558799bb336 Mon Sep 17 00:00:00 2001 From: JonnyBurger Date: Thu, 6 Nov 2025 13:51:17 +0100 Subject: [PATCH 2/3] Inherit rotation from VideoFrame when constructing VideoSample from it --- src/sample.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/sample.ts b/src/sample.ts index b66b636..34724e6 100644 --- a/src/sample.ts +++ b/src/sample.ts @@ -161,7 +161,10 @@ export class VideoSample implements Disposable { this.format = init.format; this.codedWidth = init.codedWidth!; this.codedHeight = init.codedHeight!; - this.rotation = init.rotation ?? 0; + this.rotation = init.rotation + ?? (typeof VideoFrame !== 'undefined' && data instanceof VideoFrame && 'rotation' in data + ? data.rotation as Rotation + : 0); this.timestamp = init.timestamp!; this.duration = init.duration ?? 0; this.colorSpace = new VideoColorSpace(init.colorSpace); From 4f7a980fcc95579217b398e244cf9eb4ec679f8f Mon Sep 17 00:00:00 2001 From: Vanilagy <1696106+Vanilagy@users.noreply.github.com> Date: Tue, 11 Nov 2025 16:17:41 +0100 Subject: [PATCH 3/3] Fix VideoSample drawing when the underlying VideoFrame is rotated, fix incorrect sample initialization from a rotated VideoFrame, and fix a few other things --- dev/demux.html | 31 ++++++++ src/sample.ts | 88 ++++++++++++++++------ test/browser/video-sample.test.ts | 118 ++++++++++++++++++++++++++++++ 3 files changed, 214 insertions(+), 23 deletions(-) create mode 100644 test/browser/video-sample.test.ts diff --git a/dev/demux.html b/dev/demux.html index 2bca413..65b062a 100644 --- a/dev/demux.html +++ b/dev/demux.html @@ -7,6 +7,37 @@ fileInput.type = 'file'; document.body.append(fileInput); + const canvas = document.createElement('canvas'); + const ctx = canvas.getContext('2d'); + canvas.style.border = '1px solid black'; + + ctx.fillStyle = 'blue'; + ctx.fillRect(0, 0, canvas.width, canvas.height); + ctx.fillStyle = 'red'; + ctx.fillRect(0, 0, 200, 30); + + document.body.append(canvas); + + console.log("argh", new VideoFrame(canvas, {timestamp: 0, rotation: 90})) + + const sample = new Mediabunny.VideoSample(new VideoFrame(canvas, {timestamp: 0, rotation: 90})); + console.log(sample); + + sample.setRotation(0); + + /* + //sample.setRotation(90); + console.log(sample); + + const frame = sample.toVideoFrame(); + console.log(frame) + */ + + ctx.clearRect(0, 0, canvas.width, canvas.height); + sample.draw(ctx, 0, 0); + //sample.drawWithFit(ctx, { fit: 'contain' }); + //ctx.drawImage(frame, 0, 0); + fileInput.addEventListener('change', async () => { const file = fileInput.files[0]; const input = new Mediabunny.Input({ diff --git a/src/sample.ts b/src/sample.ts index 34724e6..2a9bb43 100644 --- a/src/sample.ts +++ b/src/sample.ts @@ -17,10 +17,26 @@ import { SetRequired, isFirefox, polyfillSymbolDispose, + normalizeRotation, } from './misc'; polyfillSymbolDispose(); +// Patch the type +declare global { + interface VideoFrame { + rotation?: number; + } + + interface VideoFrameInit { + rotation?: number; + } + + interface VideoFrameBufferInit { + rotation?: number; + } +} + /** * Metadata used for VideoSample initialization. * @group Samples @@ -161,10 +177,7 @@ export class VideoSample implements Disposable { this.format = init.format; this.codedWidth = init.codedWidth!; this.codedHeight = init.codedHeight!; - this.rotation = init.rotation - ?? (typeof VideoFrame !== 'undefined' && data instanceof VideoFrame && 'rotation' in data - ? data.rotation as Rotation - : 0); + this.rotation = init.rotation ?? 0; this.timestamp = init.timestamp!; this.duration = init.duration ?? 0; this.colorSpace = new VideoColorSpace(init.colorSpace); @@ -182,12 +195,21 @@ export class VideoSample implements Disposable { this._data = data; this.format = data.format; - // Copying the display dimensions here, assuming no innate VideoFrame rotation + + // Copying the display dimensions here because the way codedWidth and codedHeight are defined is a bit wrong + // (they're defined as the pre-rotation dimensions, not the coded dimensions which is a different thing) this.codedWidth = data.displayWidth; this.codedHeight = data.displayHeight; + + const frameRotation = normalizeRotation(data.rotation ?? 0); + if (frameRotation % 180 !== 0) { + this.codedWidth = data.displayHeight; + this.codedHeight = data.displayWidth; + } + // The VideoFrame's rotation is ignored here. It's still a new field, and I'm not sure of any application // where the browser makes use of it. If a case gets found, I'll add it. - this.rotation = init?.rotation ?? 0; + this.rotation = init?.rotation ?? frameRotation; this.timestamp = init?.timestamp ?? data.timestamp / 1e6; this.duration = init?.duration ?? (data.duration ?? 0) / 1e6; this.colorSpace = data.colorSpace; @@ -385,8 +407,9 @@ export class VideoSample implements Disposable { const init: VideoFrameInit = { timestamp: this.microsecondTimestamp, duration: this.microsecondDuration || undefined, // Drag 0 duration to undefined, glitches some codecs - // @ts-expect-error - WebCodecs types do not yet support rotation - rotation: this.rotation, + // The rotation field here is additive (adds to the existing rotation), but we want it to act like an + // override instead, so we need to do some subtraction + rotation: this.rotation - (this._data.rotation ?? 0), }; return new VideoFrame(this._data, init); } else if (this._data instanceof Uint8Array) { @@ -397,7 +420,6 @@ export class VideoSample implements Disposable { timestamp: this.microsecondTimestamp, duration: this.microsecondDuration || undefined, colorSpace: this.colorSpace, - // @ts-expect-error - WebCodecs types do not yet support rotation rotation: this.rotation, }; @@ -406,7 +428,6 @@ export class VideoSample implements Disposable { const init: VideoFrameInit = { timestamp: this.microsecondTimestamp, duration: this.microsecondDuration || undefined, - // @ts-expect-error - WebCodecs types do not yet support rotation rotation: this.rotation, }; return new VideoFrame(this._data, init); @@ -536,19 +557,24 @@ export class VideoSample implements Disposable { throw new Error('VideoSample is closed.'); } - ({ sx, sy, sWidth, sHeight } = this._rotateSourceRegion(sx, sy, sWidth, sHeight, this.rotation)); - const source = this.toCanvasImageSource(); + // Relative to the innate rotation of the source + const relativeRotation = typeof VideoFrame !== 'undefined' && source instanceof VideoFrame + ? normalizeRotation(this.rotation - (source.rotation ?? 0)) + : this.rotation; + + ({ sx, sy, sWidth, sHeight } = this._rotateSourceRegion(sx, sy, sWidth, sHeight, relativeRotation)); + context.save(); const centerX = dx + dWidth / 2; const centerY = dy + dHeight / 2; context.translate(centerX, centerY); - context.rotate(this.rotation * Math.PI / 180); + context.rotate(relativeRotation * Math.PI / 180); - const aspectRatioChange = this.rotation % 180 === 0 ? 1 : dWidth / dHeight; + const aspectRatioChange = relativeRotation % 180 === 0 ? 1 : dWidth / dHeight; // Scale to compensate for aspect ratio changes when rotated context.scale(1 / aspectRatioChange, aspectRatioChange); @@ -616,6 +642,13 @@ export class VideoSample implements Disposable { const canvasHeight = context.canvas.height; const rotation = options.rotation ?? this.rotation; + const source = this.toCanvasImageSource(); + + // Relative to the innate rotation of the source + const relativeRotation = typeof VideoFrame !== 'undefined' && source instanceof VideoFrame + ? normalizeRotation(rotation - (source.rotation ?? 0)) + : rotation; + const [rotatedWidth, rotatedHeight] = rotation % 180 === 0 ? [this.codedWidth, this.codedHeight] : [this.codedHeight, this.codedWidth]; @@ -635,7 +668,7 @@ export class VideoSample implements Disposable { options.crop?.top ?? 0, options.crop?.width ?? rotatedWidth, options.crop?.height ?? rotatedHeight, - rotation, + relativeRotation, ); if (options.fit === 'fill') { @@ -657,9 +690,9 @@ export class VideoSample implements Disposable { dy = (canvasHeight - newHeight) / 2; } - const aspectRatioChange = rotation % 180 === 0 ? 1 : newWidth / newHeight; + const aspectRatioChange = relativeRotation % 180 === 0 ? 1 : newWidth / newHeight; context.translate(canvasWidth / 2, canvasHeight / 2); - context.rotate(rotation * Math.PI / 180); + context.rotate(relativeRotation * Math.PI / 180); // This aspect ratio compensation is done so that we can draw the sample with the intended dimensions and // don't need to think about how those dimensions change after the rotation context.scale(1 / aspectRatioChange, aspectRatioChange); @@ -667,28 +700,37 @@ export class VideoSample implements Disposable { // Important that we don't use .draw() here since that would take rotation into account, but we wanna handle it // ourselves here - context.drawImage(this.toCanvasImageSource(), sx, sy, sWidth, sHeight, dx, dy, newWidth, newHeight); + context.drawImage(source, sx, sy, sWidth, sHeight, dx, dy, newWidth, newHeight); } /** @internal */ - _rotateSourceRegion(sx: number, sy: number, sWidth: number, sHeight: number, rotation: number) { + _rotateSourceRegion(sx: number, sy: number, sWidth: number, sHeight: number, rotation: Rotation) { + let sourceWidth = this.codedWidth; + let sourceHeight = this.codedHeight; + + if (typeof VideoFrame !== 'undefined' && this._data instanceof VideoFrame) { + // Kinda dirty but has to be done + sourceWidth = this._data.displayWidth; + sourceHeight = this._data.displayHeight; + } + // The provided sx,sy,sWidth,sHeight refer to the final rotated image, but that's not actually how the image is // stored. Therefore, we must map these back onto the original, pre-rotation image. if (rotation === 90) { [sx, sy, sWidth, sHeight] = [ sy, - this.codedHeight - sx - sWidth, + sourceHeight - sx - sWidth, sHeight, sWidth, ]; } else if (rotation === 180) { [sx, sy] = [ - this.codedWidth - sx - sWidth, - this.codedHeight - sy - sHeight, + sourceWidth - sx - sWidth, + sourceHeight - sy - sHeight, ]; } else if (rotation === 270) { [sx, sy, sWidth, sHeight] = [ - this.codedWidth - sy - sHeight, + sourceWidth - sy - sHeight, sx, sHeight, sWidth, diff --git a/test/browser/video-sample.test.ts b/test/browser/video-sample.test.ts new file mode 100644 index 0000000..79da806 --- /dev/null +++ b/test/browser/video-sample.test.ts @@ -0,0 +1,118 @@ +import { expect, test } from 'vitest'; +import { VideoSample } from '../../src/sample.js'; + +const createCanvas = () => { + const canvas = document.createElement('canvas'); + canvas.width = 300; + canvas.height = 150; + + const ctx = canvas.getContext('2d')!; + + ctx.fillStyle = '#0000ff'; + ctx.fillRect(0, 0, canvas.width, canvas.height); + + ctx.fillStyle = '#ff0000'; + ctx.fillRect(0, 0, 100, 50); + + return canvas; +}; + +type Color = { + r: number; + g: number; + b: number; +}; + +const colorDistance = (c1: Color, c2: Color) => { + return Math.hypot(c1.r - c2.r, c1.g - c2.g, c1.b - c2.b); +}; + +const sampleCanvasColor = (ctx: CanvasRenderingContext2D, x: number, y: number): Color => { + const pixel = ctx.getImageData(x, y, 1, 1).data; + + return { + r: pixel[0]!, + g: pixel[1]!, + b: pixel[2]!, + }; +}; + +test('Can create VideoSample from VideoFrame and modify rotation', () => { + const frame = new VideoFrame(createCanvas(), { timestamp: 0 }); + const sample = new VideoSample(frame); + + expect(frame.rotation).toBe(0); + expect(sample.rotation).toBe(0); + + const canvas = document.createElement('canvas'); + canvas.width = 300; + canvas.height = 150; + const ctx = canvas.getContext('2d')!; + + sample.draw(ctx, 0, 0); + + expect(colorDistance(sampleCanvasColor(ctx, 5, 5), { r: 255, g: 0, b: 0 })).toBeLessThan(10); + expect(colorDistance(sampleCanvasColor(ctx, 5, 55), { r: 0, g: 0, b: 255 })).toBeLessThan(10); + expect(colorDistance(sampleCanvasColor(ctx, 110, 5), { r: 0, g: 0, b: 255 })).toBeLessThan(10); + + sample.setRotation(90); + expect(sample.rotation).toBe(90); + + ctx.clearRect(0, 0, canvas.width, canvas.height); + sample.draw(ctx, 0, 0); + + expect(colorDistance(sampleCanvasColor(ctx, 145, 5), { r: 255, g: 0, b: 0 })).toBeLessThan(10); + expect(colorDistance(sampleCanvasColor(ctx, 95, 5), { r: 0, g: 0, b: 255 })).toBeLessThan(10); + expect(colorDistance(sampleCanvasColor(ctx, 145, 110), { r: 0, g: 0, b: 255 })).toBeLessThan(10); + + const extracted = sample.toVideoFrame(); + expect(extracted.rotation).toBe(90); // It was changed + + ctx.clearRect(0, 0, canvas.width, canvas.height); + sample.drawWithFit(ctx, { fit: 'fill' }); + + expect(colorDistance(sampleCanvasColor(ctx, 295, 5), { r: 255, g: 0, b: 0 })).toBeLessThan(10); + expect(colorDistance(sampleCanvasColor(ctx, 295, 55), { r: 0, g: 0, b: 255 })).toBeLessThan(10); +}); + +test('Can create VideoSample from rotated VideoFrame', () => { + const frame = new VideoFrame(createCanvas(), { timestamp: 0, rotation: 90 }); + const sample = new VideoSample(frame); + + expect(frame.rotation).toBe(90); + expect(sample.rotation).toBe(90); + + const canvas = document.createElement('canvas'); + canvas.width = 300; + canvas.height = 150; + const ctx = canvas.getContext('2d')!; + + sample.draw(ctx, 0, 0); + + expect(colorDistance(sampleCanvasColor(ctx, 145, 5), { r: 255, g: 0, b: 0 })).toBeLessThan(10); + expect(colorDistance(sampleCanvasColor(ctx, 95, 5), { r: 0, g: 0, b: 255 })).toBeLessThan(10); + expect(colorDistance(sampleCanvasColor(ctx, 145, 110), { r: 0, g: 0, b: 255 })).toBeLessThan(10); + + ctx.clearRect(0, 0, canvas.width, canvas.height); + sample.drawWithFit(ctx, { fit: 'fill' }); + + expect(colorDistance(sampleCanvasColor(ctx, 295, 5), { r: 255, g: 0, b: 0 })).toBeLessThan(10); + expect(colorDistance(sampleCanvasColor(ctx, 295, 55), { r: 0, g: 0, b: 255 })).toBeLessThan(10); + + sample.setRotation(0); + expect(sample.rotation).toBe(0); + + ctx.clearRect(0, 0, canvas.width, canvas.height); + sample.draw(ctx, 0, 0); + + expect(colorDistance(sampleCanvasColor(ctx, 5, 5), { r: 255, g: 0, b: 0 })).toBeLessThan(10); + expect(colorDistance(sampleCanvasColor(ctx, 5, 55), { r: 0, g: 0, b: 255 })).toBeLessThan(10); + expect(colorDistance(sampleCanvasColor(ctx, 110, 5), { r: 0, g: 0, b: 255 })).toBeLessThan(10); + + ctx.clearRect(0, 0, canvas.width, canvas.height); + sample.drawWithFit(ctx, { fit: 'fill' }); + + expect(colorDistance(sampleCanvasColor(ctx, 5, 5), { r: 255, g: 0, b: 0 })).toBeLessThan(10); + expect(colorDistance(sampleCanvasColor(ctx, 5, 55), { r: 0, g: 0, b: 255 })).toBeLessThan(10); + expect(colorDistance(sampleCanvasColor(ctx, 110, 5), { r: 0, g: 0, b: 255 })).toBeLessThan(10); +});