diff --git a/dev/demux.html b/dev/demux.html
index 5309287..19688b4 100644
--- a/dev/demux.html
+++ b/dev/demux.html
@@ -7,6 +7,37 @@
fileInput.type = 'file';
document.body.append(fileInput);
+ const canvas = document.createElement('canvas');
+ const ctx = canvas.getContext('2d');
+ canvas.style.border = '1px solid black';
+
+ ctx.fillStyle = 'blue';
+ ctx.fillRect(0, 0, canvas.width, canvas.height);
+ ctx.fillStyle = 'red';
+ ctx.fillRect(0, 0, 200, 30);
+
+ document.body.append(canvas);
+
+ console.log("argh", new VideoFrame(canvas, {timestamp: 0, rotation: 90}))
+
+ const sample = new Mediabunny.VideoSample(new VideoFrame(canvas, {timestamp: 0, rotation: 90}));
+ console.log(sample);
+
+ sample.setRotation(0);
+
+ /*
+ //sample.setRotation(90);
+ console.log(sample);
+
+ const frame = sample.toVideoFrame();
+ console.log(frame)
+ */
+
+ ctx.clearRect(0, 0, canvas.width, canvas.height);
+ sample.draw(ctx, 0, 0);
+ //sample.drawWithFit(ctx, { fit: 'contain' });
+ //ctx.drawImage(frame, 0, 0);
+
fileInput.addEventListener('change', async () => {
const file = fileInput.files[0];
const input = new Mediabunny.Input({
diff --git a/src/sample.ts b/src/sample.ts
index 4a179d7..1e786dc 100644
--- a/src/sample.ts
+++ b/src/sample.ts
@@ -17,10 +17,26 @@ import {
SetRequired,
isFirefox,
polyfillSymbolDispose,
+ normalizeRotation,
} from './misc';
polyfillSymbolDispose();
+// Patch the type
+declare global {
+ interface VideoFrame {
+ rotation?: number;
+ }
+
+ interface VideoFrameInit {
+ rotation?: number;
+ }
+
+ interface VideoFrameBufferInit {
+ rotation?: number;
+ }
+}
+
type FinalizationRegistryValue = {
type: 'video';
data: VideoFrame | OffscreenCanvas | Uint8Array;
@@ -242,12 +258,21 @@ export class VideoSample implements Disposable {
this._data = data;
this.format = data.format;
- // Copying the display dimensions here, assuming no innate VideoFrame rotation
+
+ // Copying the display dimensions here because the way codedWidth and codedHeight are defined is a bit wrong
+ // (they're defined as the pre-rotation dimensions, not the coded dimensions which is a different thing)
this.codedWidth = data.displayWidth;
this.codedHeight = data.displayHeight;
+
+ const frameRotation = normalizeRotation(data.rotation ?? 0);
+ if (frameRotation % 180 !== 0) {
+ this.codedWidth = data.displayHeight;
+ this.codedHeight = data.displayWidth;
+ }
+
// The VideoFrame's rotation is ignored here. It's still a new field, and I'm not sure of any application
// where the browser makes use of it. If a case gets found, I'll add it.
- this.rotation = init?.rotation ?? 0;
+ this.rotation = init?.rotation ?? frameRotation;
this.timestamp = init?.timestamp ?? data.timestamp / 1e6;
this.duration = init?.duration ?? (data.duration ?? 0) / 1e6;
this.colorSpace = data.colorSpace;
@@ -456,24 +481,33 @@ export class VideoSample implements Disposable {
assert(this._data !== null);
if (isVideoFrame(this._data)) {
- return new VideoFrame(this._data, {
+ const init: VideoFrameInit = {
timestamp: this.microsecondTimestamp,
duration: this.microsecondDuration || undefined, // Drag 0 duration to undefined, glitches some codecs
- });
+ // The rotation field here is additive (adds to the existing rotation), but we want it to act like an
+ // override instead, so we need to do some subtraction
+ rotation: this.rotation - (this._data.rotation ?? 0),
+ };
+ return new VideoFrame(this._data, init);
} else if (this._data instanceof Uint8Array) {
- return new VideoFrame(this._data, {
+ const init: VideoFrameBufferInit = {
format: this.format!,
codedWidth: this.codedWidth,
codedHeight: this.codedHeight,
timestamp: this.microsecondTimestamp,
duration: this.microsecondDuration || undefined,
colorSpace: this.colorSpace,
- });
+ rotation: this.rotation,
+ };
+
+ return new VideoFrame(this._data, init);
} else {
- return new VideoFrame(this._data, {
+ const init: VideoFrameInit = {
timestamp: this.microsecondTimestamp,
duration: this.microsecondDuration || undefined,
- });
+ rotation: this.rotation,
+ };
+ return new VideoFrame(this._data, init);
}
}
@@ -600,19 +634,24 @@ export class VideoSample implements Disposable {
throw new Error('VideoSample is closed.');
}
- ({ sx, sy, sWidth, sHeight } = this._rotateSourceRegion(sx, sy, sWidth, sHeight, this.rotation));
-
const source = this.toCanvasImageSource();
+ // Relative to the innate rotation of the source
+ const relativeRotation = typeof VideoFrame !== 'undefined' && source instanceof VideoFrame
+ ? normalizeRotation(this.rotation - (source.rotation ?? 0))
+ : this.rotation;
+
+ ({ sx, sy, sWidth, sHeight } = this._rotateSourceRegion(sx, sy, sWidth, sHeight, relativeRotation));
+
context.save();
const centerX = dx + dWidth / 2;
const centerY = dy + dHeight / 2;
context.translate(centerX, centerY);
- context.rotate(this.rotation * Math.PI / 180);
+ context.rotate(relativeRotation * Math.PI / 180);
- const aspectRatioChange = this.rotation % 180 === 0 ? 1 : dWidth / dHeight;
+ const aspectRatioChange = relativeRotation % 180 === 0 ? 1 : dWidth / dHeight;
// Scale to compensate for aspect ratio changes when rotated
context.scale(1 / aspectRatioChange, aspectRatioChange);
@@ -679,6 +718,13 @@ export class VideoSample implements Disposable {
const canvasHeight = context.canvas.height;
const rotation = options.rotation ?? this.rotation;
+ const source = this.toCanvasImageSource();
+
+ // Relative to the innate rotation of the source
+ const relativeRotation = typeof VideoFrame !== 'undefined' && source instanceof VideoFrame
+ ? normalizeRotation(rotation - (source.rotation ?? 0))
+ : rotation;
+
const [rotatedWidth, rotatedHeight] = rotation % 180 === 0
? [this.codedWidth, this.codedHeight]
: [this.codedHeight, this.codedWidth];
@@ -698,7 +744,7 @@ export class VideoSample implements Disposable {
options.crop?.top ?? 0,
options.crop?.width ?? rotatedWidth,
options.crop?.height ?? rotatedHeight,
- rotation,
+ relativeRotation,
);
if (options.fit === 'fill') {
@@ -722,9 +768,9 @@ export class VideoSample implements Disposable {
context.save();
- const aspectRatioChange = rotation % 180 === 0 ? 1 : newWidth / newHeight;
+ const aspectRatioChange = relativeRotation % 180 === 0 ? 1 : newWidth / newHeight;
context.translate(canvasWidth / 2, canvasHeight / 2);
- context.rotate(rotation * Math.PI / 180);
+ context.rotate(relativeRotation * Math.PI / 180);
// This aspect ratio compensation is done so that we can draw the sample with the intended dimensions and
// don't need to think about how those dimensions change after the rotation
context.scale(1 / aspectRatioChange, aspectRatioChange);
@@ -732,30 +778,39 @@ export class VideoSample implements Disposable {
// Important that we don't use .draw() here since that would take rotation into account, but we wanna handle it
// ourselves here
- context.drawImage(this.toCanvasImageSource(), sx, sy, sWidth, sHeight, dx, dy, newWidth, newHeight);
+ context.drawImage(source, sx, sy, sWidth, sHeight, dx, dy, newWidth, newHeight);
context.restore();
}
/** @internal */
- _rotateSourceRegion(sx: number, sy: number, sWidth: number, sHeight: number, rotation: number) {
+ _rotateSourceRegion(sx: number, sy: number, sWidth: number, sHeight: number, rotation: Rotation) {
+ let sourceWidth = this.codedWidth;
+ let sourceHeight = this.codedHeight;
+
+ if (typeof VideoFrame !== 'undefined' && this._data instanceof VideoFrame) {
+ // Kinda dirty but has to be done
+ sourceWidth = this._data.displayWidth;
+ sourceHeight = this._data.displayHeight;
+ }
+
// The provided sx,sy,sWidth,sHeight refer to the final rotated image, but that's not actually how the image is
// stored. Therefore, we must map these back onto the original, pre-rotation image.
if (rotation === 90) {
[sx, sy, sWidth, sHeight] = [
sy,
- this.codedHeight - sx - sWidth,
+ sourceHeight - sx - sWidth,
sHeight,
sWidth,
];
} else if (rotation === 180) {
[sx, sy] = [
- this.codedWidth - sx - sWidth,
- this.codedHeight - sy - sHeight,
+ sourceWidth - sx - sWidth,
+ sourceHeight - sy - sHeight,
];
} else if (rotation === 270) {
[sx, sy, sWidth, sHeight] = [
- this.codedWidth - sy - sHeight,
+ sourceWidth - sy - sHeight,
sx,
sHeight,
sWidth,
diff --git a/test/browser/video-sample.test.ts b/test/browser/video-sample.test.ts
new file mode 100644
index 0000000..79da806
--- /dev/null
+++ b/test/browser/video-sample.test.ts
@@ -0,0 +1,118 @@
+import { expect, test } from 'vitest';
+import { VideoSample } from '../../src/sample.js';
+
+const createCanvas = () => {
+ const canvas = document.createElement('canvas');
+ canvas.width = 300;
+ canvas.height = 150;
+
+ const ctx = canvas.getContext('2d')!;
+
+ ctx.fillStyle = '#0000ff';
+ ctx.fillRect(0, 0, canvas.width, canvas.height);
+
+ ctx.fillStyle = '#ff0000';
+ ctx.fillRect(0, 0, 100, 50);
+
+ return canvas;
+};
+
+type Color = {
+ r: number;
+ g: number;
+ b: number;
+};
+
+const colorDistance = (c1: Color, c2: Color) => {
+ return Math.hypot(c1.r - c2.r, c1.g - c2.g, c1.b - c2.b);
+};
+
+const sampleCanvasColor = (ctx: CanvasRenderingContext2D, x: number, y: number): Color => {
+ const pixel = ctx.getImageData(x, y, 1, 1).data;
+
+ return {
+ r: pixel[0]!,
+ g: pixel[1]!,
+ b: pixel[2]!,
+ };
+};
+
+test('Can create VideoSample from VideoFrame and modify rotation', () => {
+ const frame = new VideoFrame(createCanvas(), { timestamp: 0 });
+ const sample = new VideoSample(frame);
+
+ expect(frame.rotation).toBe(0);
+ expect(sample.rotation).toBe(0);
+
+ const canvas = document.createElement('canvas');
+ canvas.width = 300;
+ canvas.height = 150;
+ const ctx = canvas.getContext('2d')!;
+
+ sample.draw(ctx, 0, 0);
+
+ expect(colorDistance(sampleCanvasColor(ctx, 5, 5), { r: 255, g: 0, b: 0 })).toBeLessThan(10);
+ expect(colorDistance(sampleCanvasColor(ctx, 5, 55), { r: 0, g: 0, b: 255 })).toBeLessThan(10);
+ expect(colorDistance(sampleCanvasColor(ctx, 110, 5), { r: 0, g: 0, b: 255 })).toBeLessThan(10);
+
+ sample.setRotation(90);
+ expect(sample.rotation).toBe(90);
+
+ ctx.clearRect(0, 0, canvas.width, canvas.height);
+ sample.draw(ctx, 0, 0);
+
+ expect(colorDistance(sampleCanvasColor(ctx, 145, 5), { r: 255, g: 0, b: 0 })).toBeLessThan(10);
+ expect(colorDistance(sampleCanvasColor(ctx, 95, 5), { r: 0, g: 0, b: 255 })).toBeLessThan(10);
+ expect(colorDistance(sampleCanvasColor(ctx, 145, 110), { r: 0, g: 0, b: 255 })).toBeLessThan(10);
+
+ const extracted = sample.toVideoFrame();
+ expect(extracted.rotation).toBe(90); // It was changed
+
+ ctx.clearRect(0, 0, canvas.width, canvas.height);
+ sample.drawWithFit(ctx, { fit: 'fill' });
+
+ expect(colorDistance(sampleCanvasColor(ctx, 295, 5), { r: 255, g: 0, b: 0 })).toBeLessThan(10);
+ expect(colorDistance(sampleCanvasColor(ctx, 295, 55), { r: 0, g: 0, b: 255 })).toBeLessThan(10);
+});
+
+test('Can create VideoSample from rotated VideoFrame', () => {
+ const frame = new VideoFrame(createCanvas(), { timestamp: 0, rotation: 90 });
+ const sample = new VideoSample(frame);
+
+ expect(frame.rotation).toBe(90);
+ expect(sample.rotation).toBe(90);
+
+ const canvas = document.createElement('canvas');
+ canvas.width = 300;
+ canvas.height = 150;
+ const ctx = canvas.getContext('2d')!;
+
+ sample.draw(ctx, 0, 0);
+
+ expect(colorDistance(sampleCanvasColor(ctx, 145, 5), { r: 255, g: 0, b: 0 })).toBeLessThan(10);
+ expect(colorDistance(sampleCanvasColor(ctx, 95, 5), { r: 0, g: 0, b: 255 })).toBeLessThan(10);
+ expect(colorDistance(sampleCanvasColor(ctx, 145, 110), { r: 0, g: 0, b: 255 })).toBeLessThan(10);
+
+ ctx.clearRect(0, 0, canvas.width, canvas.height);
+ sample.drawWithFit(ctx, { fit: 'fill' });
+
+ expect(colorDistance(sampleCanvasColor(ctx, 295, 5), { r: 255, g: 0, b: 0 })).toBeLessThan(10);
+ expect(colorDistance(sampleCanvasColor(ctx, 295, 55), { r: 0, g: 0, b: 255 })).toBeLessThan(10);
+
+ sample.setRotation(0);
+ expect(sample.rotation).toBe(0);
+
+ ctx.clearRect(0, 0, canvas.width, canvas.height);
+ sample.draw(ctx, 0, 0);
+
+ expect(colorDistance(sampleCanvasColor(ctx, 5, 5), { r: 255, g: 0, b: 0 })).toBeLessThan(10);
+ expect(colorDistance(sampleCanvasColor(ctx, 5, 55), { r: 0, g: 0, b: 255 })).toBeLessThan(10);
+ expect(colorDistance(sampleCanvasColor(ctx, 110, 5), { r: 0, g: 0, b: 255 })).toBeLessThan(10);
+
+ ctx.clearRect(0, 0, canvas.width, canvas.height);
+ sample.drawWithFit(ctx, { fit: 'fill' });
+
+ expect(colorDistance(sampleCanvasColor(ctx, 5, 5), { r: 255, g: 0, b: 0 })).toBeLessThan(10);
+ expect(colorDistance(sampleCanvasColor(ctx, 5, 55), { r: 0, g: 0, b: 255 })).toBeLessThan(10);
+ expect(colorDistance(sampleCanvasColor(ctx, 110, 5), { r: 0, g: 0, b: 255 })).toBeLessThan(10);
+});