arcodange/mediabunny
1
0
Fork 0
mirror of https://github.com/arcodange-org/mediabunny.git synced 2026-09-27 10:53:50 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add VideoSample.transform(), add custom transformer functions, ditch CanvasSink path in Conversion API, add libavfilter-based VideoSample transformation

Browse Source
This commit is contained in:
Vanilagy
2026-05-11 15:57:51 +02:00
parent 3ae73a7c5e
commit a89d715524
10 changed files with 709 additions and 315 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/conversion.ts
+40 -175
View File
@@ -24,7 +24,6 @@ import { Input } from './input';
import { InputAudioTrack, InputTrack, InputVideoTrack } from './input-track';
import {
AudioSampleSink,
CanvasSink,
EncodedPacketSink,
VideoSampleSink,
} from './media-sink';
@@ -41,7 +40,6 @@ import {
assertNever,
ceilToMultipleOfTwo,
clamp,
floorToDivisor,
isIso639Dash2LanguageCode,
MaybePromise,
normalizeRotation,
@@ -1142,7 +1140,8 @@ export class Conversion {
let videoSource: VideoSource;
const totalRotation = normalizeRotation(await track.getRotation() + (trackOptions.rotate ?? 0));
const innateRotation = await track.getRotation();
const totalRotation = normalizeRotation(innateRotation + (trackOptions.rotate ?? 0));
let outputTrackRotation = totalRotation;
const canUseRotationMetadata = this.output.format.supportsVideoRotationMetadata
&& (trackOptions.allowRotationMetadata ?? true);
@@ -1284,10 +1283,9 @@ export class Conversion {
sizeChangeBehavior: trackOptions.fit ?? 'passThrough',
alpha,
hardwareAcceleration: trackOptions.hardwareAcceleration,
transform: {},
};
const source = new VideoSampleSource(encodingConfig);
videoSource = source;
assert(encodingConfig.transform);
let needsRerender = width !== originalWidth
|| height !== originalHeight
@@ -1334,179 +1332,46 @@ export class Conversion {
}
}
if (trackOptions.frameRate) {
encodingConfig.transform.frameRate = trackOptions.frameRate;
}
if (needsRerender) {
outputTrackRotation = 0; // Since the rotation is baked into the output
this._trackPromises.push((async () => {
await this._started;
const sink = new CanvasSink(track, {
width,
height,
fit: trackOptions.fit ?? 'fill',
rotation: totalRotation, // Bake the rotation into the output
crop: trackOptions.crop,
poolSize: 1,
alpha: alpha === 'keep',
});
const iterator = sink.canvases(this._startTimestamp, this._endTimestamp);
const frameRate = trackOptions.frameRate;
let lastCanvas: HTMLCanvasElement | OffscreenCanvas | null = null;
let lastCanvasTimestamp: number | null = null;
let lastCanvasEndTimestamp: number | null = null;
/** Repeats the last sample to pad out the time until the specified timestamp. */
const padFrames = async (until: number) => {
assert(lastCanvas);
assert(frameRate !== undefined);
const frameDifference = Math.round((until - lastCanvasTimestamp!) * frameRate);
for (let i = 1; i < frameDifference; i++) {
const sample = new VideoSample(lastCanvas, {
timestamp: lastCanvasTimestamp! + i / frameRate,
duration: 1 / frameRate,
});
await this._registerVideoSample(trackOptions, outputTrackId, source, sample);
sample.close();
}
};
for await (const { canvas, timestamp, duration } of iterator) {
if (this._canceled) {
return;
}
let adjustedSampleTimestamp = Math.max(timestamp - this._startTimestamp, 0);
lastCanvasEndTimestamp = adjustedSampleTimestamp + duration;
if (frameRate !== undefined) {
// Logic for skipping/repeating frames when a frame rate is set
const alignedTimestamp = floorToDivisor(adjustedSampleTimestamp, frameRate);
if (lastCanvas !== null) {
if (alignedTimestamp <= lastCanvasTimestamp!) {
lastCanvas = canvas;
lastCanvasTimestamp = alignedTimestamp;
// Skip this sample, since we already added one for this frame
continue;
} else {
// Check if we may need to repeat the previous frame
await padFrames(alignedTimestamp);
}
}
adjustedSampleTimestamp = alignedTimestamp;
}
const sample = new VideoSample(canvas, {
timestamp: adjustedSampleTimestamp,
duration: frameRate !== undefined ? 1 / frameRate : duration,
});
await this._registerVideoSample(trackOptions, outputTrackId, source, sample);
sample.close();
if (frameRate !== undefined) {
lastCanvas = canvas;
lastCanvasTimestamp = adjustedSampleTimestamp;
}
}
if (lastCanvas) {
assert(lastCanvasEndTimestamp !== null);
assert(frameRate !== undefined);
// If necessary, pad until the end timestamp of the last sample
await padFrames(floorToDivisor(lastCanvasEndTimestamp, frameRate));
}
source.close();
this._synchronizer.closeTrack(outputTrackId);
})());
} else {
this._trackPromises.push((async () => {
await this._started;
const sink = new VideoSampleSink(track);
const frameRate = trackOptions.frameRate;
let lastSample: VideoSample | null = null;
let lastSampleTimestamp: number | null = null;
let lastSampleEndTimestamp: number | null = null;
/** Repeats the last sample to pad out the time until the specified timestamp. */
const padFrames = async (until: number) => {
assert(lastSample);
assert(frameRate !== undefined);
const frameDifference = Math.round((until - lastSampleTimestamp!) * frameRate);
for (let i = 1; i < frameDifference; i++) {
lastSample.setTimestamp(lastSampleTimestamp! + i / frameRate);
lastSample.setDuration(1 / frameRate);
await this._registerVideoSample(trackOptions, outputTrackId, source, lastSample);
}
lastSample.close();
};
for await (const sample of sink.samples(this._startTimestamp, this._endTimestamp)) {
if (this._canceled) {
sample.close();
lastSample?.close();
return;
}
let adjustedSampleTimestamp = Math.max(sample.timestamp - this._startTimestamp, 0);
lastSampleEndTimestamp = adjustedSampleTimestamp + sample.duration;
if (frameRate !== undefined) {
// Logic for skipping/repeating frames when a frame rate is set
const alignedTimestamp = floorToDivisor(adjustedSampleTimestamp, frameRate);
if (lastSample !== null) {
if (alignedTimestamp <= lastSampleTimestamp!) {
lastSample.close();
lastSample = sample;
lastSampleTimestamp = alignedTimestamp;
// Skip this sample, since we already added one for this frame
continue;
} else {
// Check if we may need to repeat the previous frame
await padFrames(alignedTimestamp);
}
}
adjustedSampleTimestamp = alignedTimestamp;
sample.setDuration(1 / frameRate);
}
sample.setTimestamp(adjustedSampleTimestamp);
await this._registerVideoSample(trackOptions, outputTrackId, source, sample);
if (frameRate !== undefined) {
lastSample = sample;
lastSampleTimestamp = adjustedSampleTimestamp;
} else {
sample.close();
}
}
if (lastSample) {
assert(lastSampleEndTimestamp !== null);
assert(frameRate !== undefined);
// If necessary, pad until the end timestamp of the last sample
await padFrames(floorToDivisor(lastSampleEndTimestamp, frameRate));
}
source.close();
this._synchronizer.closeTrack(outputTrackId);
})());
encodingConfig.transform.width = width;
encodingConfig.transform.height = height;
encodingConfig.transform.fit = trackOptions.fit ?? 'fill';
encodingConfig.transform.rotate = normalizeRotation(totalRotation - innateRotation);
encodingConfig.transform.crop = crop;
encodingConfig.transform.alpha = alpha;
}
const source = new VideoSampleSource(encodingConfig);
videoSource = source;
this._trackPromises.push((async () => {
await this._started;
const sink = new VideoSampleSink(track);
for await (const sample of sink.samples(this._startTimestamp, this._endTimestamp)) {
if (this._canceled) {
sample.close();
return;
}
const adjustedSampleTimestamp = Math.max(sample.timestamp - this._startTimestamp, 0);
sample.setTimestamp(adjustedSampleTimestamp);
await this._registerVideoSample(trackOptions, outputTrackId, source, sample);
sample.close();
}
source.close();
this._synchronizer.closeTrack(outputTrackId);
})());
}
let ownGroup: OutputTrackGroup | null = null;
src/encode.ts
+8 -2
View File
@@ -113,6 +113,10 @@ export type VideoTransformOptions = {
* clamped to the dimensions of the frame. Cropping is performed after rotation but before resizing.
*/
crop?: CropRectangle;
/**
* Whether to discard or keep the transparency information of the video samples. The default is `'keep'`.
*/
alpha?: 'keep' | 'discard';
/**
* The frame rate in hertz to normalize the video frame stream to.
*/
@@ -193,10 +197,12 @@ export const validateVideoEncodingConfig = (config: VideoEncodingConfig) => {
if (
config.transform.fit !== undefined
&& ['fill', 'contain', 'cover'].includes(config.sizeChangeBehavior!)
&& config.transform.fit !== config.sizeChangeBehavior
) {
throw new TypeError(
'config.transform.fit cannot be used when config.sizeChangeBehavior is \'fill\', \'contain\' or'
+ ' \'cover\', as sizeChangeBehavior already determines the fitting algorithm.',
'config.transform.fit, when provided, cannot differ from config.sizeChangeBehavior when'
+ ' config.sizeChangeBehavior is \'fill\', \'contain\' or \'cover\', as sizeChangeBehavior already'
+ ' determines the fitting algorithm.',
);
}
if (config.transform.rotate !== undefined && ![0, 90, 180, 270].includes(config.transform.rotate)) {
src/index.ts
+4
View File
@@ -249,8 +249,12 @@ export {
type VideoSamplePixelFormat,
VideoSampleColorSpace,
VideoSampleResource,
type VideoSampleTransformOptions,
type VideoSampleTransformationDescription,
type CropRectangle,
VIDEO_SAMPLE_PIXEL_FORMATS,
type VideoDataPlane,
registerVideoSampleTransformer,
} from './sample';
export {
AudioBufferSink,
src/media-source.ts
+16 -96
View File
@@ -25,13 +25,10 @@ import {
assertNever,
binarySearchLessOrEqual,
CallSerializer,
ceilToMultipleOfTwo,
clamp,
clearIntervalUnthrottled,
floorToDivisor,
isFirefox,
last,
normalizeRotation,
promiseWithResolvers,
roundToDivisor,
setInt24,
@@ -53,7 +50,6 @@ import { EncodedPacket, EncodedPacketSideData } from './packet';
import {
AudioSample,
audioSampleToInterleavedFormat,
clampCropRectangle,
toInterleavedAudioFormat,
VideoSample,
} from './sample';
@@ -233,7 +229,6 @@ class VideoEncoderWrapper {
private muxer: Muxer | null = null;
private lastMultipleOfKeyFrameInterval = -1;
private resizeCanvas: HTMLCanvasElement | OffscreenCanvas | null = null;
// Tracks the input dimensions of the first frame
private codedWidth: number | null = null;
private codedHeight: number | null = null;
@@ -275,15 +270,7 @@ class VideoEncoderWrapper {
private lastMuxerPromise: Promise<void> = Promise.resolve();
constructor(private source: VideoSource, private encodingConfig: VideoEncodingConfig) {
const sizeChangeBehavior = encodingConfig.sizeChangeBehavior ?? 'deny';
if (['fill', 'contain', 'cover'].includes(sizeChangeBehavior) && encodingConfig.transform?.fit !== undefined) {
throw new TypeError(
`Cannot set 'fit' when 'sizeChangeBehavior' is '${sizeChangeBehavior}'. `
+ `The size change behavior determines the fit in this case.`,
);
}
}
constructor(private source: VideoSource, private encodingConfig: VideoEncodingConfig) {}
async add(videoSample: VideoSample, shouldClose: boolean, encodeOptions?: VideoEncoderEncodeOptions) {
const originalSample = videoSample;
@@ -323,23 +310,8 @@ class VideoEncoderWrapper {
const needsTransform = hasTransformConfig || (isSizeChange && sizeChangeBehavior !== 'passThrough');
if (needsTransform) {
const rotation = normalizeRotation(videoSample.rotation + (config.transform?.rotate ?? 0));
const [rotatedWidth, rotatedHeight] = rotation % 180 === 0
? [videoSample.codedWidth, videoSample.codedHeight]
: [videoSample.codedHeight, videoSample.codedWidth];
// Clamp crop rectangle to the rotated video dimensions
let finalCrop = config.transform?.crop;
if (finalCrop) {
finalCrop = clampCropRectangle(finalCrop, rotatedWidth, rotatedHeight);
}
const cropWidth = finalCrop ? finalCrop.width : rotatedWidth;
const cropHeight = finalCrop ? finalCrop.height : rotatedHeight;
const originalAspectRatio = cropWidth / cropHeight;
let targetWidth: number;
let targetHeight: number;
let targetWidth = config.transform?.width;
let targetHeight = config.transform?.height;
let appliedFit: 'fill' | 'contain' | 'cover' = config.transform?.fit ?? 'fill';
// If the size changed and behavior is fill/contain/cover, lock to the original output dimensions
@@ -351,81 +323,29 @@ class VideoEncoderWrapper {
targetWidth = this.outputWidth!;
targetHeight = this.outputHeight!;
appliedFit = sizeChangeBehavior;
} else {
// Otherwise, dynamically calculate the target dimensions based on config and aspect ratio
if (config.transform?.width !== undefined && config.transform?.height === undefined) {
targetWidth = config.transform.width;
targetHeight = ceilToMultipleOfTwo(Math.round(targetWidth / originalAspectRatio));
} else if (config.transform?.width === undefined && config.transform?.height !== undefined) {
targetHeight = config.transform.height;
targetWidth = ceilToMultipleOfTwo(Math.round(targetHeight * originalAspectRatio));
} else if (config.transform?.width !== undefined && config.transform?.height !== undefined) {
targetWidth = config.transform?.width;
targetHeight = config.transform?.height;
} else {
targetWidth = cropWidth;
targetHeight = cropHeight;
}
}
const transformed = await videoSample.transform({
width: targetWidth,
height: targetHeight,
roundDimensionsTo: 2,
crop: config.transform?.crop,
rotate: config.transform?.rotate,
fit: appliedFit,
alpha: config.alpha,
});
// Save the output dimensions of the first frame
if (this.outputWidth === null || this.outputHeight === null) {
this.outputWidth = targetWidth;
this.outputHeight = targetHeight;
this.outputWidth = transformed.displayWidth;
this.outputHeight = transformed.displayHeight;
}
let canvasIsNew = false;
if (!this.resizeCanvas) {
if (typeof document !== 'undefined') {
// Prefer an HTMLCanvasElement
this.resizeCanvas = document.createElement('canvas');
this.resizeCanvas.width = targetWidth;
this.resizeCanvas.height = targetHeight;
} else {
this.resizeCanvas = new OffscreenCanvas(targetWidth, targetHeight);
}
canvasIsNew = true;
} else if (this.resizeCanvas.width !== targetWidth || this.resizeCanvas.height !== targetHeight) {
// Dynamically resize the canvas if the target dimensions have changed
this.resizeCanvas.width = targetWidth;
this.resizeCanvas.height = targetHeight;
}
const context = this.resizeCanvas.getContext('2d', {
// Firefox has VideoFrame glitches with opaque canvases
alpha: this.encodingConfig.alpha === 'keep' || isFirefox(),
}) as CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;
assert(context);
if (typeof context.resetTransform === 'function') {
context.resetTransform();
}
if (!canvasIsNew) {
if (isFirefox()) {
context.fillStyle = 'black';
context.fillRect(0, 0, targetWidth, targetHeight);
} else {
context.clearRect(0, 0, targetWidth, targetHeight);
}
}
videoSample.drawWithFit(context, {
fit: appliedFit,
rotation: rotation,
crop: finalCrop,
});
if (shouldClose) {
videoSample.close();
}
videoSample = new VideoSample(this.resizeCanvas, {
timestamp: videoSample.timestamp,
duration: videoSample.duration,
rotation: 0, // Rotation is now baked into the canvas
});
videoSample = transformed;
shouldClose = true;
} else {
// If no canvas is needed, we still need to record the output dimensions for the first frame
src/sample.ts
+317 -4
View File
@@ -26,6 +26,9 @@ import {
simplifyRational,
Rectangle,
validateRectangle,
normalizeRotation,
roundToMultiple,
arrayArgmin,
MaybePromise,
} from './misc';
@@ -103,8 +106,10 @@ export abstract class VideoSampleResource {
/** Returns the height of the frame in pixels. */
abstract getCodedHeight(): number;
/** Returns the width of the frame in square pixels, respecting pixel aspect ratio. */
abstract getSquarePixelWidth(): number;
/** Returns the height of the frame in square pixels, respecting pixel aspect ratio. */
abstract getSquarePixelHeight(): number;
/** Returns the color space of the frame. */
@@ -116,17 +121,32 @@ export abstract class VideoSampleResource {
*/
abstract close(): void;
/**
* Returns the data planes that hold the video data for this sample. The returned planes and data must be in the
* format returned by `getFormat()`.
*/
abstract getDataPlanes(): MaybePromise<VideoDataPlane[]>;
/**
* Returns a new RGB {@link VideoSample} that contains the same content as this sample. The provided `init` object
* must be used to set the metadata of this new video sample. When converting from a non-RGB format to RGB, the
* conversion must respect `colorSpace`.
*/
abstract toRgbSample(
init: SetRequired<VideoSampleInit, 'timestamp'>,
format: 'RGBA' | 'RGBX' | 'BGRA' | 'BGRX',
colorSpace: PredefinedColorSpace,
): MaybePromise<VideoSample>;
}
/**
* Describes a single data plane of a video frame.
* @group Samples
* @public
*/
export type VideoDataPlane = {
/** The data of the plane. */
data: Uint8Array;
/** The stride of the plane, in bytes. This is the distance in bytes between the start of each row of pixels. */
stride: number;
};
@@ -239,9 +259,9 @@ export class VideoSample implements Disposable {
readonly format!: VideoSamplePixelFormat | null;
/** The visible region of the frame in the coded pixel grid. */
readonly visibleRect!: Rectangle;
/** The width of the frame in square pixels, before rotation is applied. */
/** The width of the frame in square pixels (respecting pixel aspect ratio), before rotation is applied. */
readonly squarePixelWidth!: number;
/** The height of the frame in square pixels, before rotation is applied. */
/** The height of the frame in square pixels (respecting pixel aspect ratio), before rotation is applied. */
readonly squarePixelHeight!: number;
/** The rotation of the frame in degrees, clockwise. */
readonly rotation!: Rotation;
@@ -754,7 +774,6 @@ export class VideoSample implements Disposable {
duration: this.duration,
rotation: this.rotation,
},
options.format as 'RGBA' | 'RGBX' | 'BGRA' | 'BGRX',
options.colorSpace ?? 'srgb',
);
if (!(rgbSample instanceof VideoSample)) {
@@ -1333,6 +1352,179 @@ export class VideoSample implements Disposable {
}
}
/**
* Transform this video sample to a new video sample given the options. Can be used to resize, rotate, and crop
* the sample.
*
* In non-browser environments, this method will not work by default. To make it work, register a custom
* transformer function via {@link registerVideoSampleTransformer}.
*/
async transform(options: VideoSampleTransformOptions) {
if (!options || typeof options !== 'object') {
throw new TypeError('options must be an object.');
}
if (options.width !== undefined && (!Number.isInteger(options.width) || options.width <= 0)) {
throw new TypeError('options.width, when provided, must be a positive integer.');
}
if (options.height !== undefined && (!Number.isInteger(options.height) || options.height <= 0)) {
throw new TypeError('options.height, when provided, must be a positive integer.');
}
if (
options.roundDimensionsTo !== undefined
&& (!Number.isInteger(options.roundDimensionsTo) || options.roundDimensionsTo <= 0)
) {
throw new TypeError('options.roundDimensionsTo, when provided, must be a positive integer.');
}
if (options.fit !== undefined && !['fill', 'contain', 'cover'].includes(options.fit)) {
throw new TypeError('options.fit, when provided, must be one of "fill", "contain", or "cover".');
}
if (
options.width !== undefined
&& options.height !== undefined
&& options.fit === undefined
) {
throw new TypeError(
'When both options.width and options.height are provided, options.fit must also be provided.',
);
}
if (options.rotate !== undefined && ![0, 90, 180, 270].includes(options.rotate)) {
throw new TypeError('options.rotate, when provided, must be 0, 90, 180 or 270.');
}
if (options.crop !== undefined) {
validateCropRectangle(options.crop, 'options.');
}
if (options.alpha !== undefined && !['keep', 'discard'].includes(options.alpha)) {
throw new TypeError('options.alpha, when provided, must be \'keep\' or \'discard\'.');
}
const rotation = normalizeRotation(this.rotation + (options.rotate ?? 0));
const [rotatedWidth, rotatedHeight] = rotation % 180 === 0
? [this.squarePixelWidth, this.squarePixelHeight]
: [this.squarePixelHeight, this.squarePixelWidth];
// Clamp crop rectangle to the rotated video dimensions
let finalCrop = options.crop;
if (finalCrop) {
finalCrop = clampCropRectangle(finalCrop, rotatedWidth, rotatedHeight);
}
const cropWidth = finalCrop ? finalCrop.width : rotatedWidth;
const cropHeight = finalCrop ? finalCrop.height : rotatedHeight;
const originalAspectRatio = cropWidth / cropHeight;
let targetWidth: number;
let targetHeight: number;
if (options.width !== undefined && options.height === undefined) {
targetWidth = options.width;
targetHeight = targetWidth / originalAspectRatio;
} else if (options.width === undefined && options.height !== undefined) {
targetHeight = options.height;
targetWidth = targetHeight * originalAspectRatio;
} else if (options.width !== undefined && options.height !== undefined) {
targetWidth = options.width;
targetHeight = options.height;
} else {
targetWidth = cropWidth;
targetHeight = cropHeight;
}
targetWidth = roundToMultiple(targetWidth, options.roundDimensionsTo ?? 1);
targetHeight = roundToMultiple(targetHeight, options.roundDimensionsTo ?? 1);
const description: VideoSampleTransformationDescription = {
width: targetWidth,
height: targetHeight,
fit: options.fit ?? 'fill',
rotation,
crop: finalCrop ?? {
left: 0,
top: 0,
width: rotatedWidth,
height: rotatedHeight,
},
alpha: options.alpha ?? 'keep',
};
// Description's finalized; let's see if a registered transformer wants to handle it
for (const transformer of registeredVideoSampleTransformers) {
let result = transformer(this, description);
if (result instanceof Promise) result = await result;
if (result !== null) {
return result;
}
}
// We need to handle it ourselves, and we use canvases to do it
let canvas: HTMLCanvasElement | OffscreenCanvas | null = null;
let canvasIsNew = false;
for (const entry of transformationCanvasCache) {
if (entry.canvas.width === description.width && entry.canvas.height === description.height) {
canvas = entry.canvas;
entry.age = transformationCanvasCacheNextAge++;
break;
}
}
if (canvas === null) {
if (typeof OffscreenCanvas !== 'undefined') {
canvas = new OffscreenCanvas(description.width, description.height);
} else {
if (typeof window === 'undefined' || typeof document === 'undefined') {
throw new Error(
'Cannot transform VideoSamples in this environment. Either run in an environment with'
+ ' OffscreenCanvas or HTMLCanvasElement, or supply a custom VideoSample transformer using'
+ ' registerVideoSampleTransformer().',
);
}
canvas = document.createElement('canvas');
canvas.width = description.width;
canvas.height = description.height;
}
canvasIsNew = true;
if (transformationCanvasCache.length >= TRANSFORMATION_CANVAS_CACHE_MAX_SIZE) {
transformationCanvasCache.splice(arrayArgmin(transformationCanvasCache, x => x.age), 1);
}
transformationCanvasCache.push({
canvas,
age: transformationCanvasCacheNextAge++,
});
}
const context = canvas.getContext('2d', {
alpha: true,
}) as CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;
assert(context);
if (description.alpha === 'discard') {
context.fillStyle = 'black';
context.fillRect(0, 0, description.width, description.height);
} else if (!canvasIsNew) {
// Cached canvases carry stale pixels from a prior draw
context.clearRect(0, 0, description.width, description.height);
}
this.drawWithFit(context, {
fit: description.fit,
rotation: description.rotation,
crop: description.crop,
});
return new VideoSample(canvas, {
timestamp: this.timestamp,
duration: this.duration,
rotation: 0, // Any previous rotation is now baked in
});
}
/** Sets the rotation metadata of this video sample. */
setRotation(newRotation: Rotation) {
if (![0, 90, 180, 270].includes(newRotation)) {
@@ -1369,6 +1561,123 @@ export class VideoSample implements Disposable {
}
}
/**
* Options for transforming a {@link VideoSample}. The order of operations are:
*
* 1. Pixel aspect ratio normalization (always applied)
* 2. Rotation
* 3. Crop
* 4. Resize using fit
* @group Samples
* @public
*/
export type VideoSampleTransformOptions = {
/**
* The width in pixels to resize the frames to. If height is not set, it will be deduced
* automatically based on aspect ratio.
*/
width?: number;
/**
* The height in pixels to resize the frames to. If width is not set, it will be deduced
* automatically based on aspect ratio.
*/
height?: number;
/**
* A positive integer. When provided, both the width and height will be rounded to the nearest multiple of
* this number.
*/
roundDimensionsTo?: number;
/**
* The fitting algorithm in case both width and height are set.
*
* - `'fill'` will stretch the image to fill the entire box, potentially altering aspect ratio.
* - `'contain'` will contain the entire image within the box while preserving aspect ratio. This may lead to
* letterboxing.
* - `'cover'` will scale the image until the entire box is filled, while preserving aspect ratio.
*/
fit?: 'fill' | 'contain' | 'cover';
/**
* The clockwise rotation by which to rotate the frames. Rotation is applied before resizing.
*/
rotate?: Rotation;
/**
* Specifies the rectangular region of the frames to crop to. The crop region will automatically be
* clamped to the dimensions of the frame. Cropping is performed after rotation but before resizing.
*/
crop?: CropRectangle;
/**
* Whether to discard or keep the transparency information of the video sample. The default is `'keep'`.
*/
alpha?: 'keep' | 'discard';
};
/**
* A fully-resolved description of a video sample transformation, with all defaults and constraints baked in.
*
* The order of operations must be:
* 1. Pixel aspect ratio normalization (always applied)
* 2. Rotation
* 3. Crop
* 4. Resize using fit
* @group Samples
* @public
*/
export type VideoSampleTransformationDescription = {
/** The width in pixels to resize the frames to. */
width: number;
/** The height in pixels to resize the frames to. */
height: number;
/**
* The fitting algorithm.
*
* - `'fill'` will stretch the image to fill the entire box, potentially altering aspect ratio.
* - `'contain'` will contain the entire image within the box while preserving aspect ratio. This may lead to
* letterboxing.
* - `'cover'` will scale the image until the entire box is filled, while preserving aspect ratio.
*/
fit: 'fill' | 'contain' | 'cover';
/** The clockwise rotation by which to rotate the frames. Rotation is applied before resizing. */
rotation: Rotation;
/**
* The rectangular region of the frames to crop to, clamped to the dimensions of the frame. Cropping is
* performed after rotation but before resizing.
*/
crop: CropRectangle;
/** Whether to discard or keep the transparency information of the video sample. */
alpha: 'keep' | 'discard';
};
const registeredVideoSampleTransformers: ((
sample: VideoSample,
description: VideoSampleTransformationDescription,
) => MaybePromise<VideoSample | null>)[] = [];
/**
* Registers a callback to handle the transformation of {@link VideoSample} instances. The callback can either return
* the transformed sample, or `null` to indicate that it doesn't want to handle the given transformation task.
* @group Samples
* @public
*/
export const registerVideoSampleTransformer = (
transformer: (
sample: VideoSample,
description: VideoSampleTransformationDescription,
) => MaybePromise<VideoSample | null>,
) => {
if (registeredVideoSampleTransformers.includes(transformer)) {
return; // Already in there
}
registeredVideoSampleTransformers.push(transformer);
};
const TRANSFORMATION_CANVAS_CACHE_MAX_SIZE = 3;
const transformationCanvasCache: {
canvas: HTMLCanvasElement | OffscreenCanvas;
age: number;
}[] = [];
let transformationCanvasCacheNextAge = 0;
/**
* Describes the color space of a {@link VideoSample}. Corresponds to the WebCodecs API's VideoColorSpace.
* @group Samples
@@ -1927,6 +2236,10 @@ export abstract class AudioSampleResource {
*/
abstract close(): void;
/**
* Returns the audio sample data for the plane given by `planeIndex`. The audio data must be in the format returned
* by `getFormat()`. For interleaved formats, there is only one plane.
*/
abstract getDataPlane(planeIndex: number): Uint8Array;
}