arcodange/mediabunny
1
0
Fork 0
mirror of https://github.com/arcodange-org/mediabunny.git synced 2026-09-30 12:23:52 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add support for reading/writing non-square pixel video frames (fixes #309)

Browse Source
This commit is contained in:
Vanilagy
2026-02-25 17:16:33 +01:00
parent a7dc2c198b
commit bb4749cd8d
25 changed files with 744 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/sample.ts
+157 -22
View File
@@ -9,16 +9,23 @@
import {
assert,
clamp,
COLOR_PRIMARIES_MAP,
isAllowSharedBufferSource,
MATRIX_COEFFICIENTS_MAP,
Rotation,
SECOND_TO_MICROSECOND_FACTOR,
toDataView,
toUint8Array,
SetRequired,
TRANSFER_CHARACTERISTICS_MAP,
isFirefox,
polyfillSymbolDispose,
assertNever,
isWebKit,
Rational,
simplifyRational,
Rectangle,
validateRectangle,
} from './misc';
polyfillSymbolDispose();
@@ -147,6 +154,12 @@ export type VideoSampleInit = {
colorSpace?: VideoColorSpaceInit;
/** The byte layout of the planes of the frame. */
layout?: PlaneLayout[];
/** Visible region in the coded frame. When omitted, the rect defaults to `(0, 0, codedWidth, codedHeight)`. */
visibleRect?: Rectangle | undefined;
/** Width of the frame in pixels after applying aspect ratio adjustments and rotation. */
displayWidth?: number | undefined;
/** Height of the frame in pixels after applying aspect ratio adjustments and rotation. */
displayHeight?: number | undefined;
};
/**
@@ -172,12 +185,19 @@ export class VideoSample implements Disposable {
* [See pixel formats](https://www.w3.org/TR/webcodecs/#pixel-format)
*/
readonly format!: VideoSamplePixelFormat | null;
/** The width of the frame in pixels. */
readonly codedWidth!: number;
/** The height of the frame in pixels. */
readonly codedHeight!: number;
/** The visible region of the frame in the coded pixel grid. */
readonly visibleRect!: Rectangle;
/** The width of the frame in square pixels, before rotation is applied. */
readonly squarePixelWidth!: number;
/** The height of the frame in square pixels, before rotation is applied. */
readonly squarePixelHeight!: number;
/** The rotation of the frame in degrees, clockwise. */
readonly rotation!: Rotation;
/**
* The pixel aspect ratio of the frame, as a rational number in its reduced form. Most videos use
* square pixels (1:1).
*/
readonly pixelAspectRatio!: Rational;
/**
* The presentation timestamp of the frame in seconds. May be negative. Frames with negative end timestamps should
* not be presented.
@@ -188,14 +208,26 @@ export class VideoSample implements Disposable {
/** The color space of the frame. */
readonly colorSpace!: VideoSampleColorSpace;
/** The width of the frame in pixels after rotation. */
get displayWidth() {
return this.rotation % 180 === 0 ? this.codedWidth : this.codedHeight;
/** The width of the frame in pixels. */
get codedWidth() {
// This is wrong, but the fix is a v2 thing
return this.visibleRect.width;
}
/** The height of the frame in pixels after rotation. */
/** The height of the frame in pixels. */
get codedHeight() {
// Same here
return this.visibleRect.height;
}
/** The display width of the frame in pixels, after aspect ratio adjustment and rotation. */
get displayWidth() {
return this.rotation % 180 === 0 ? this.squarePixelWidth : this.squarePixelHeight;
}
/** The display height of the frame in pixels, after aspect ratio adjustment and rotation. */
get displayHeight() {
return this.rotation % 180 === 0 ? this.codedHeight : this.codedWidth;
return this.rotation % 180 === 0 ? this.squarePixelHeight : this.squarePixelWidth;
}
/** The presentation timestamp of the frame in microseconds. */
@@ -268,17 +300,66 @@ export class VideoSample implements Disposable {
if (init.duration !== undefined && (!Number.isFinite(init.duration) || init.duration < 0)) {
throw new TypeError('init.duration, when provided, must be a non-negative number.');
}
if (init.layout !== undefined) {
if (!Array.isArray(init.layout)) {
throw new TypeError('init.layout, when provided, must be an array.');
}
for (const plane of init.layout) {
if (!plane || typeof plane !== 'object' || Array.isArray(plane)) {
throw new TypeError('Each entry in init.layout must be an object.');
}
if (!Number.isInteger(plane.offset) || plane.offset < 0) {
throw new TypeError('plane.offset must be a non-negative integer.');
}
if (!Number.isInteger(plane.stride) || plane.stride < 0) {
throw new TypeError('plane.stride must be a non-negative integer.');
}
}
}
if (init.visibleRect !== undefined) {
validateRectangle(init.visibleRect, 'init.visibleRect');
}
if (
init.displayWidth !== undefined
&& (!Number.isInteger(init.displayWidth) || init.displayWidth <= 0)
) {
throw new TypeError('init.displayWidth, when provided, must be a positive integer.');
}
if (
init.displayHeight !== undefined
&& (!Number.isInteger(init.displayHeight) || init.displayHeight <= 0)
) {
throw new TypeError('init.displayHeight, when provided, must be a positive integer.');
}
if ((init.displayWidth !== undefined) !== (init.displayHeight !== undefined)) {
throw new TypeError(
'init.displayWidth and init.displayHeight must be either both provided or both omitted.',
);
}
this._data = toUint8Array(data).slice(); // Copy it
this._layout = init.layout ?? createDefaultPlaneLayout(init.format, init.codedWidth!, init.codedHeight!);
this.format = init.format;
this.codedWidth = init.codedWidth!;
this.codedHeight = init.codedHeight!;
this.rotation = init.rotation ?? 0;
this.timestamp = init.timestamp!;
this.duration = init.duration ?? 0;
this.colorSpace = new VideoSampleColorSpace(init.colorSpace);
this.visibleRect = {
left: init.visibleRect?.left ?? 0,
top: init.visibleRect?.top ?? 0,
width: init.visibleRect?.width ?? init.codedWidth!,
height: init.visibleRect?.height ?? init.codedHeight!,
};
if (init.displayWidth !== undefined) {
this.squarePixelWidth = this.rotation % 180 === 0 ? init.displayWidth : init.displayHeight!;
this.squarePixelHeight = this.rotation % 180 === 0 ? init.displayHeight! : init.displayWidth;
} else {
this.squarePixelWidth = this.codedWidth;
this.squarePixelHeight = this.codedHeight;
}
} else if (typeof VideoFrame !== 'undefined' && data instanceof VideoFrame) {
if (init?.rotation !== undefined && ![0, 90, 180, 270].includes(init.rotation)) {
throw new TypeError('init.rotation, when provided, must be 0, 90, 180, or 270.');
@@ -289,17 +370,28 @@ export class VideoSample implements Disposable {
if (init?.duration !== undefined && (!Number.isFinite(init.duration) || init.duration < 0)) {
throw new TypeError('init.duration, when provided, must be a non-negative number.');
}
if (init?.visibleRect !== undefined) {
validateRectangle(init.visibleRect, 'init.visibleRect');
}
this._data = data;
this._layout = null;
this.format = data.format;
// Copying the display dimensions here, assuming no innate VideoFrame rotation
this.codedWidth = data.displayWidth;
this.codedHeight = data.displayHeight;
this.visibleRect = {
left: data.visibleRect?.x ?? 0,
top: data.visibleRect?.y ?? 0,
width: data.visibleRect?.width ?? data.codedWidth,
height: data.visibleRect?.height ?? data.codedHeight,
};
// The VideoFrame's rotation is ignored here. It's still a new field, and I'm not sure of any application
// where the browser makes use of it. If a case gets found, I'll add it.
this.rotation = init?.rotation ?? 0;
// Assuming no innate VideoFrame rotation here
this.squarePixelWidth = data.displayWidth;
this.squarePixelHeight = data.displayHeight;
this.timestamp = init?.timestamp ?? data.timestamp / 1e6;
this.duration = init?.duration ?? (data.duration ?? 0) / 1e6;
this.colorSpace = new VideoSampleColorSpace(data.colorSpace);
@@ -367,8 +459,9 @@ export class VideoSample implements Disposable {
this._layout = null;
this.format = 'RGBX';
this.codedWidth = width;
this.codedHeight = height;
this.visibleRect = { left: 0, top: 0, width, height };
this.squarePixelWidth = width;
this.squarePixelHeight = height;
this.rotation = init.rotation ?? 0;
this.timestamp = init.timestamp!;
this.duration = init.duration ?? 0;
@@ -382,6 +475,10 @@ export class VideoSample implements Disposable {
throw new TypeError('Invalid data type: Must be a BufferSource or CanvasImageSource.');
}
this.pixelAspectRatio = simplifyRational({
num: this.squarePixelWidth * this.codedHeight,
den: this.squarePixelHeight * this.codedWidth,
});
finalizationRegistry?.register(this, { type: 'video', data: this._data }, this);
}
@@ -411,6 +508,9 @@ export class VideoSample implements Disposable {
duration: this.duration,
colorSpace: this.colorSpace,
rotation: this.rotation,
visibleRect: this.visibleRect,
displayWidth: this.displayWidth,
displayHeight: this.displayHeight,
});
} else {
return new VideoSample(this._data, {
@@ -421,6 +521,9 @@ export class VideoSample implements Disposable {
duration: this.duration,
colorSpace: this.colorSpace,
rotation: this.rotation,
visibleRect: this.visibleRect,
displayWidth: this.displayWidth,
displayHeight: this.displayHeight,
});
}
}
@@ -755,6 +858,7 @@ export class VideoSample implements Disposable {
/**
* Specifies the rectangular region of the video sample to crop to. The crop region will automatically be
* clamped to the dimensions of the video sample. Cropping is performed after rotation but before resizing.
* The crop region is in the _display pixel space_ of the underlying video data.
*/
crop?: CropRectangle;
}) {
@@ -785,8 +889,8 @@ export class VideoSample implements Disposable {
const rotation = options.rotation ?? this.rotation;
const [rotatedWidth, rotatedHeight] = rotation % 180 === 0
? [this.codedWidth, this.codedHeight]
: [this.codedHeight, this.codedWidth];
? [this.squarePixelWidth, this.squarePixelHeight]
: [this.squarePixelHeight, this.squarePixelWidth];
if (options.crop) {
clampCropRectangle(options.crop, rotatedWidth, rotatedHeight);
@@ -849,18 +953,18 @@ export class VideoSample implements Disposable {
if (rotation === 90) {
[sx, sy, sWidth, sHeight] = [
sy,
this.codedHeight - sx - sWidth,
this.squarePixelHeight - sx - sWidth,
sHeight,
sWidth,
];
} else if (rotation === 180) {
[sx, sy] = [
this.codedWidth - sx - sWidth,
this.codedHeight - sy - sHeight,
this.squarePixelWidth - sx - sWidth,
this.squarePixelHeight - sy - sHeight,
];
} else if (rotation === 270) {
[sx, sy, sWidth, sHeight] = [
this.codedWidth - sy - sHeight,
this.squarePixelWidth - sy - sHeight,
sx,
sHeight,
sWidth,
@@ -948,6 +1052,37 @@ export class VideoSampleColorSpace {
/** Creates a new VideoSampleColorSpace. */
constructor(init?: VideoColorSpaceInit) {
if (init !== undefined) {
if (!init || typeof init !== 'object') {
throw new TypeError('init.colorSpace, when provided, must be an object.');
}
const primariesValues = Object.keys(COLOR_PRIMARIES_MAP);
if (init.primaries != null && !primariesValues.includes(init.primaries)) {
throw new TypeError(
`init.colorSpace.primaries, when provided, must be one of ${primariesValues.join(', ')}.`,
);
}
const transferValues = Object.keys(TRANSFER_CHARACTERISTICS_MAP);
if (init.transfer != null && !transferValues.includes(init.transfer)) {
throw new TypeError(
`init.colorSpace.transfer, when provided, must be one of ${transferValues.join(', ')}.`,
);
}
const matrixValues = Object.keys(MATRIX_COEFFICIENTS_MAP);
if (init.matrix != null && !matrixValues.includes(init.matrix)) {
throw new TypeError(
`init.colorSpace.matrix, when provided, must be one of ${matrixValues.join(', ')}.`,
);
}
if (init.fullRange != null && typeof init.fullRange !== 'boolean') {
throw new TypeError('init.colorSpace.fullRange, when provided, must be a boolean.');
}
}
this.primaries = init?.primaries ?? null;
this.transfer = init?.transfer ?? null;
this.matrix = init?.matrix ?? null;