arcodange/mediabunny
1
0
Fork 0
mirror of https://github.com/arcodange-org/mediabunny.git synced 2026-09-27 10:53:50 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add video frame preprocessing to VideoEncodingConfig & VideoEncoderWrapper

Browse Source
This commit is contained in:
Vanilagy
2026-04-01 13:30:34 +02:00
parent 3400b3ac12
commit 024660773b
8 changed files with 1021 additions and 201 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/procedural-generation/procedural-generation.ts
+7 -4
View File
@@ -99,7 +99,6 @@ const generateVideo = async () => {
output = new Output({
rootPath: 'master.m3u8',
target: async ({ path }) => {
/*
const fileHandle = await dirHandle.getFileHandle(path, { create: true });
const writable = await fileHandle.createWritable();
@@ -107,8 +106,8 @@ const generateVideo = async () => {
target.on('finalized', () => console.log('Finalizado', path));
return target;
*/
/*
const target = new BufferTarget();
target.on('finalized', async () => {
if (path.includes('m3u8')) {
@@ -123,11 +122,12 @@ const generateVideo = async () => {
}
return target;
*/
},
format: new HlsOutputFormat({
segmentFormat: new CmafOutputFormat(),
// singleFilePerPlaylist: true,
live: true,
// live: true,
getPlaylistPath: info => `sussex-${info.n}.m3u8`,
}),
});
@@ -146,6 +146,9 @@ const generateVideo = async () => {
codec: videoCodec,
bitrate: QUALITY_HIGH,
keyFrameInterval: 2,
transform: {
frameRate: 5,
},
});
output.addVideoTrack(canvasSource, { frameRate });
@@ -206,7 +209,7 @@ const generateVideo = async () => {
// automatically slow down the rendering loop when the encoder can't keep up.
await canvasSource.add(currentTime, 1 / frameRate);
await new Promise(resolve => setTimeout(resolve, 1000 / frameRate));
// await new Promise(resolve => setTimeout(resolve, 1000 / frameRate));
}
// Signal to the output that no more video frames are coming (not necessary, but recommended)
src/conversion.ts
+8 -8
View File
@@ -38,7 +38,9 @@ import {
} from './media-source';
import {
assert,
ceilToMultipleOfTwo,
clamp,
floorToDivisor,
isIso639Dash2LanguageCode,
MaybePromise,
normalizeRotation,
@@ -915,9 +917,9 @@ export class Conversion {
? [track.squarePixelWidth, track.squarePixelHeight]
: [track.squarePixelHeight, track.squarePixelWidth];
const crop = trackOptions.crop;
let crop = trackOptions.crop;
if (crop) {
clampCropRectangle(crop, rotatedWidth, rotatedHeight);
crop = clampCropRectangle(crop, rotatedWidth, rotatedHeight);
}
const [originalWidth, originalHeight] = crop
@@ -929,8 +931,6 @@ export class Conversion {
const aspectRatio = width / height;
// A lot of video encoders require that the dimensions be multiples of 2
const ceilToMultipleOfTwo = (value: number) => Math.ceil(value / 2) * 2;
if (trackOptions.width !== undefined && trackOptions.height === undefined) {
width = ceilToMultipleOfTwo(trackOptions.width);
height = ceilToMultipleOfTwo(Math.round(width / aspectRatio));
@@ -1144,7 +1144,7 @@ export class Conversion {
if (frameRate !== undefined) {
// Logic for skipping/repeating frames when a frame rate is set
const alignedTimestamp = Math.floor(adjustedSampleTimestamp * frameRate) / frameRate;
const alignedTimestamp = floorToDivisor(adjustedSampleTimestamp, frameRate);
if (lastCanvas !== null) {
if (alignedTimestamp <= lastCanvasTimestamp!) {
@@ -1180,7 +1180,7 @@ export class Conversion {
assert(frameRate !== undefined);
// If necessary, pad until the end timestamp of the last sample
await padFrames(Math.floor(lastCanvasEndTimestamp * frameRate) / frameRate);
await padFrames(floorToDivisor(lastCanvasEndTimestamp, frameRate));
}
source.close();
@@ -1225,7 +1225,7 @@ export class Conversion {
if (frameRate !== undefined) {
// Logic for skipping/repeating frames when a frame rate is set
const alignedTimestamp = Math.floor(adjustedSampleTimestamp * frameRate) / frameRate;
const alignedTimestamp = floorToDivisor(adjustedSampleTimestamp, frameRate);
if (lastSample !== null) {
if (alignedTimestamp <= lastSampleTimestamp!) {
@@ -1261,7 +1261,7 @@ export class Conversion {
assert(frameRate !== undefined);
// If necessary, pad until the end timestamp of the last sample
await padFrames(Math.floor(lastSampleEndTimestamp * frameRate) / frameRate);
await padFrames(floorToDivisor(lastSampleEndTimestamp, frameRate));
}
source.close();
src/encode.ts
+110 -2
View File
@@ -22,8 +22,9 @@ import {
VideoCodec,
} from './codec';
import { customAudioEncoders, customVideoEncoders } from './custom-coder';
import { isFirefox } from './misc';
import { isFirefox, MaybePromise, Rotation } from './misc';
import { EncodedPacket } from './packet';
import { CropRectangle, validateCropRectangle, VideoSample } from './sample';
const canEncodeVideoMemo = new Map<string, Promise<boolean>>();
const canEncodeAudioMemo = new Map<string, Promise<boolean>>();
@@ -61,6 +62,58 @@ export type VideoEncodingConfig = {
*/
sizeChangeBehavior?: 'deny' | 'passThrough' | 'fill' | 'contain' | 'cover';
/**
* Optional transformations to apply to the video frames before they are passed to the encoder.
*/
transform?: {
/**
* The width in pixels to resize the frames to. If height is not set, it will be deduced
* automatically based on aspect ratio.
*/
width?: number;
/**
* The height in pixels to resize the frames to. If width is not set, it will be deduced
* automatically based on aspect ratio.
*/
height?: number;
/**
* The fitting algorithm in case both width and height are set.
*
* - `'fill'` will stretch the image to fill the entire box, potentially altering aspect ratio.
* - `'contain'` will contain the entire image within the box while preserving aspect ratio. This may lead to
* letterboxing.
* - `'cover'` will scale the image until the entire box is filled, while preserving aspect ratio.
*
* To avoid ambiguity, this field must not be set when `sizeChangeBehavior` is `'fill'`, `'contain'` or
* `'deny'`, since `sizeChangeBehavior` already determines the fitting algorithm.
*/
fit?: 'fill' | 'contain' | 'cover';
/**
* The clockwise rotation by which to rotate the frames. Rotation is applied before resizing.
*/
rotate?: Rotation;
/**
* Specifies the rectangular region of the frames to crop to. The crop region will automatically be
* clamped to the dimensions of the frame. Cropping is performed after rotation but before resizing.
*/
crop?: CropRectangle;
/**
* The frame rate in hertz to normalize the video frame stream to.
*/
frameRate?: number;
/**
* Allows for custom user-defined processing of video frames, e.g. for applying overlays, color transformations,
* or timestamp modifications. Will be called for each video frame after transformations and frame rate
* corrections.
*
* Must return a {@link VideoSample} or a `CanvasImageSource`, an array of them, or `null` for dropping the
* frame. When non-timestamped data is returned, the timestamp and duration from the input sample will be used.
*/
process?: (sample: VideoSample) => MaybePromise<
CanvasImageSource | VideoSample | (CanvasImageSource | VideoSample)[] | null
>;
};
/** Called for each successfully encoded packet. Both the packet and the encoding metadata are passed. */
onEncodedPacket?: (packet: EncodedPacket, meta: EncodedVideoChunkMetadata | undefined) => unknown;
/**
@@ -95,6 +148,61 @@ export const validateVideoEncodingConfig = (config: VideoEncodingConfig) => {
+ ' or \'cover\'.',
);
}
if (config.transform !== undefined) {
if (typeof config.transform !== 'object' || !config.transform) {
throw new TypeError('config.transform, when provided, must be an object.');
}
if (
config.transform.width !== undefined
&& (!Number.isInteger(config.transform.width) || config.transform.width <= 0)
) {
throw new TypeError('config.transform.width, when provided, must be a positive integer.');
}
if (
config.transform.height !== undefined
&& (!Number.isInteger(config.transform.height) || config.transform.height <= 0)
) {
throw new TypeError('config.transform.height, when provided, must be a positive integer.');
}
if (config.transform.fit !== undefined && !['fill', 'contain', 'cover'].includes(config.transform.fit)) {
throw new TypeError('config.transform.fit, when provided, must be one of "fill", "contain", or "cover".');
}
if (
config.transform.width !== undefined
&& config.transform.height !== undefined
&& config.transform.fit === undefined
&& !['fill', 'contain', 'cover'].includes(config.sizeChangeBehavior!)
) {
throw new TypeError(
'When both config.transform.width and config.transform.height are provided, config.transform.fit'
+ ' must also be provided.',
);
}
if (
config.transform.fit !== undefined
&& ['fill', 'contain', 'cover'].includes(config.sizeChangeBehavior!)
) {
throw new TypeError(
'config.transform.fit cannot be used when config.sizeChangeBehavior is \'fill\', \'contain\' or'
+ ' \'cover\', as sizeChangeBehavior already determines the fitting algorithm.',
);
}
if (config.transform.rotate !== undefined && ![0, 90, 180, 270].includes(config.transform.rotate)) {
throw new TypeError('config.transform.rotate, when provided, must be 0, 90, 180 or 270.');
}
if (config.transform.crop !== undefined) {
validateCropRectangle(config.transform.crop, 'config.');
}
if (config.transform.process !== undefined && typeof config.transform.process !== 'function') {
throw new TypeError('config.transform.process, when provided, must be a function.');
}
if (
config.transform.frameRate !== undefined
&& (!Number.isFinite(config.transform.frameRate) || config.transform.frameRate <= 0)
) {
throw new TypeError('config.transform.frameRate, when provided, must be a finite positive number.');
}
}
if (config.onEncodedPacket !== undefined && typeof config.onEncodedPacket !== 'function') {
throw new TypeError('config.onEncodedChunk, when provided, must be a function.');
}
@@ -106,7 +214,7 @@ export const validateVideoEncodingConfig = (config: VideoEncodingConfig) => {
};
/**
* Additional options that control audio encoding.
* Additional options that control video encoding.
* @group Encoding
* @public
*/
src/media-sink.ts
+2 -2
View File
@@ -1636,9 +1636,9 @@ export class CanvasSink {
? [videoTrack.squarePixelWidth, videoTrack.squarePixelHeight]
: [videoTrack.squarePixelHeight, videoTrack.squarePixelWidth];
const crop = options.crop;
let crop = options.crop;
if (crop) {
clampCropRectangle(crop, rotatedWidth, rotatedHeight);
crop = clampCropRectangle(crop, rotatedWidth, rotatedHeight);
}
let [width, height] = crop
src/media-source.ts
+271 -52
View File
@@ -25,10 +25,13 @@ import {
assertNever,
binarySearchLessOrEqual,
CallSerializer,
ceilToMultipleOfTwo,
clamp,
clearIntervalUnthrottled,
floorToDivisor,
isFirefox,
last,
normalizeRotation,
promiseWithResolvers,
roundToDivisor,
setInt24,
@@ -47,7 +50,7 @@ import {
customAudioEncoders,
} from './custom-coder';
import { EncodedPacket, EncodedPacketSideData } from './packet';
import { AudioSample, VideoSample } from './sample';
import { AudioSample, clampCropRectangle, VideoSample } from './sample';
import {
AudioEncodingConfig,
buildAudioEncoderConfig,
@@ -222,9 +225,19 @@ class VideoEncoderWrapper {
private encoder: VideoEncoder | null = null;
private muxer: Muxer | null = null;
private lastMultipleOfKeyFrameInterval = -1;
private resizeCanvas: HTMLCanvasElement | OffscreenCanvas | null = null;
// Tracks the input dimensions of the first frame
private codedWidth: number | null = null;
private codedHeight: number | null = null;
private resizeCanvas: HTMLCanvasElement | OffscreenCanvas | null = null;
// Tracks the output dimensions of the first frame (used to lock dimensions for fill/contain/cover)
private outputWidth: number | null = null;
private outputHeight: number | null = null;
// Frame rate normalization state
private frameRateLastSample: VideoSample | null = null;
private frameRateLastTimestamp: number | null = null;
private frameRateLastEndTimestamp: number | null = null;
// VideoEncoder converts everything to microseconds, so we need to do some bookkeeping to restore the original
// timing information
@@ -253,58 +266,146 @@ class VideoEncoderWrapper {
*/
private error: Error | null = null;
constructor(private source: VideoSource, private encodingConfig: VideoEncodingConfig) {}
constructor(private source: VideoSource, private encodingConfig: VideoEncodingConfig) {
const sizeChangeBehavior = encodingConfig.sizeChangeBehavior ?? 'deny';
if (['fill', 'contain', 'cover'].includes(sizeChangeBehavior) && encodingConfig.transform?.fit !== undefined) {
throw new TypeError(
`Cannot set 'fit' when 'sizeChangeBehavior' is '${sizeChangeBehavior}'. `
+ `The size change behavior determines the fit in this case.`,
);
}
}
async add(videoSample: VideoSample, shouldClose: boolean, encodeOptions?: VideoEncoderEncodeOptions) {
const originalSample = videoSample;
try {
this.checkForEncoderError();
this.source._ensureValidAdd();
// Ensure video sample size remains constant
const config = this.encodingConfig;
const sizeChangeBehavior = config.sizeChangeBehavior ?? 'deny';
let isSizeChange = false;
// Ensure video sample size remains constant or handle the change
if (this.codedWidth !== null && this.codedHeight !== null) {
if (videoSample.codedWidth !== this.codedWidth || videoSample.codedHeight !== this.codedHeight) {
const sizeChangeBehavior = this.encodingConfig.sizeChangeBehavior ?? 'deny';
if (sizeChangeBehavior === 'passThrough') {
// Do nada
} else if (sizeChangeBehavior === 'deny') {
isSizeChange = true;
if (sizeChangeBehavior === 'deny') {
throw new Error(
`Video sample size must remain constant. Expected ${this.codedWidth}x${this.codedHeight},`
+ ` got ${videoSample.codedWidth}x${videoSample.codedHeight}. To allow the sample size to`
+ ` change over time, set \`sizeChangeBehavior\` to a value other than 'strict' in the`
+ ` change over time, set \`sizeChangeBehavior\` to a value other than 'deny' in the`
+ ` encoding options.`,
);
}
}
} else {
this.codedWidth = videoSample.codedWidth;
this.codedHeight = videoSample.codedHeight;
}
// Determine if we need to apply transformations via canvas
const hasTransformConfig = config.transform?.width !== undefined
|| config.transform?.height !== undefined
|| config.transform?.rotate !== undefined
|| config.transform?.crop !== undefined;
const needsTransform = hasTransformConfig || (isSizeChange && sizeChangeBehavior !== 'passThrough');
if (needsTransform) {
const rotation = normalizeRotation(videoSample.rotation + (config.transform?.rotate ?? 0));
const [rotatedWidth, rotatedHeight] = rotation % 180 === 0
? [videoSample.codedWidth, videoSample.codedHeight]
: [videoSample.codedHeight, videoSample.codedWidth];
// Clamp crop rectangle to the rotated video dimensions
let finalCrop = config.transform?.crop;
if (finalCrop) {
finalCrop = clampCropRectangle(finalCrop, rotatedWidth, rotatedHeight);
}
const cropWidth = finalCrop ? finalCrop.width : rotatedWidth;
const cropHeight = finalCrop ? finalCrop.height : rotatedHeight;
const originalAspectRatio = cropWidth / cropHeight;
let targetWidth: number;
let targetHeight: number;
let appliedFit: 'fill' | 'contain' | 'cover' = config.transform?.fit ?? 'fill';
// If the size changed and behavior is fill/contain/cover, lock to the original output dimensions
if (isSizeChange && sizeChangeBehavior !== 'passThrough') {
assert(this.outputWidth);
assert(this.outputHeight);
assert(sizeChangeBehavior !== 'deny');
targetWidth = this.outputWidth!;
targetHeight = this.outputHeight!;
appliedFit = sizeChangeBehavior;
} else {
// Otherwise, dynamically calculate the target dimensions based on config and aspect ratio
if (config.transform?.width !== undefined && config.transform?.height === undefined) {
targetWidth = config.transform.width;
targetHeight = ceilToMultipleOfTwo(Math.round(targetWidth / originalAspectRatio));
} else if (config.transform?.width === undefined && config.transform?.height !== undefined) {
targetHeight = config.transform.height;
targetWidth = ceilToMultipleOfTwo(Math.round(targetHeight * originalAspectRatio));
} else if (config.transform?.width !== undefined && config.transform?.height !== undefined) {
targetWidth = config.transform?.width;
targetHeight = config.transform?.height;
} else {
targetWidth = cropWidth;
targetHeight = cropHeight;
}
}
// Save the output dimensions of the first frame
if (this.outputWidth === null || this.outputHeight === null) {
this.outputWidth = targetWidth;
this.outputHeight = targetHeight;
}
let canvasIsNew = false;
if (!this.resizeCanvas) {
if (typeof document !== 'undefined') {
// Prefer an HTMLCanvasElement
this.resizeCanvas = document.createElement('canvas');
this.resizeCanvas.width = this.codedWidth;
this.resizeCanvas.height = this.codedHeight;
this.resizeCanvas.width = targetWidth;
this.resizeCanvas.height = targetHeight;
} else {
this.resizeCanvas = new OffscreenCanvas(this.codedWidth, this.codedHeight);
this.resizeCanvas = new OffscreenCanvas(targetWidth, targetHeight);
}
canvasIsNew = true;
} else if (this.resizeCanvas.width !== targetWidth || this.resizeCanvas.height !== targetHeight) {
// Dynamically resize the canvas if the target dimensions have changed
this.resizeCanvas.width = targetWidth;
this.resizeCanvas.height = targetHeight;
}
const context = this.resizeCanvas.getContext('2d', {
alpha: isFirefox(), // Firefox has VideoFrame glitches with opaque canvases
// Firefox has VideoFrame glitches with opaque canvases
alpha: this.encodingConfig.alpha === 'keep' || isFirefox(),
}) as CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;
assert(context);
if (typeof context.resetTransform === 'function') {
context.resetTransform();
}
if (!canvasIsNew) {
if (isFirefox()) {
context.fillStyle = 'black';
context.fillRect(0, 0, this.codedWidth, this.codedHeight);
context.fillRect(0, 0, targetWidth, targetHeight);
} else {
context.clearRect(0, 0, this.codedWidth, this.codedHeight);
context.clearRect(0, 0, targetWidth, targetHeight);
}
}
videoSample.drawWithFit(context, { fit: sizeChangeBehavior });
videoSample.drawWithFit(context, {
fit: appliedFit,
rotation: rotation,
crop: finalCrop,
});
if (shouldClose) {
videoSample.close();
@@ -313,26 +414,115 @@ class VideoEncoderWrapper {
videoSample = new VideoSample(this.resizeCanvas, {
timestamp: videoSample.timestamp,
duration: videoSample.duration,
rotation: videoSample.rotation,
rotation: 0, // Rotation is now baked into the canvas
});
shouldClose = true;
}
}
} else {
this.codedWidth = videoSample.codedWidth;
this.codedHeight = videoSample.codedHeight;
// If no canvas is needed, we still need to record the output dimensions for the first frame
if (this.outputWidth === null || this.outputHeight === null) {
this.outputWidth = videoSample.codedWidth;
this.outputHeight = videoSample.codedHeight;
}
}
const frameRate = config.transform?.frameRate;
if (frameRate !== undefined) {
// Apply frame rate normalization
const originalEndTimestamp = videoSample.timestamp + videoSample.duration;
const alignedTimestamp = floorToDivisor(videoSample.timestamp, frameRate);
if (this.frameRateLastSample !== null) {
if (alignedTimestamp <= this.frameRateLastTimestamp!) {
// Same frame rate slot, replace stored sample with the newer one
this.frameRateLastSample.close();
this.frameRateLastSample = videoSample.clone();
this.frameRateLastEndTimestamp = originalEndTimestamp;
return;
} else {
// Pad the gap by repeating the previous frame
await this.padFrameRate(alignedTimestamp, encodeOptions);
}
}
// Clone if the sample is still the user's, to avoid mutating externally-owned data
if (videoSample === originalSample) {
videoSample = videoSample.clone();
shouldClose = true;
}
videoSample.setTimestamp(alignedTimestamp);
videoSample.setDuration(1 / frameRate);
this.frameRateLastSample?.close();
this.frameRateLastSample = videoSample.clone();
this.frameRateLastTimestamp = alignedTimestamp;
this.frameRateLastEndTimestamp = originalEndTimestamp;
}
await this.processAndEncode(videoSample, encodeOptions);
} finally {
if (shouldClose) {
videoSample.close();
}
}
}
/**
* Runs the process function (if any) and encodes the resulting samples.
*/
private async processAndEncode(
videoSample: VideoSample,
encodeOptions?: VideoEncoderEncodeOptions,
) {
const config = this.encodingConfig;
let samplesToEncode: VideoSample[];
// Apply the user-defined process function, if any
if (config.transform?.process) {
let processed = config.transform.process(videoSample);
if (processed instanceof Promise) {
processed = await processed;
}
if (processed === null) {
return;
}
if (!Array.isArray(processed)) {
processed = [processed];
}
samplesToEncode = processed.map((x) => {
if (x instanceof VideoSample) {
return x;
}
if (typeof VideoFrame !== 'undefined' && x instanceof VideoFrame) {
return new VideoSample(x);
}
return new VideoSample(x, {
timestamp: videoSample.timestamp,
duration: videoSample.duration,
});
});
} else {
samplesToEncode = [videoSample];
}
try {
for (const sampleToEncode of samplesToEncode) {
if (!this.encoderInitialized) {
if (!this.ensureEncoderPromise) {
this.ensureEncoder(videoSample);
this.ensureEncoder(sampleToEncode);
}
// No, this "if" statement is not useless. Sometimes, the above call to `ensureEncoder` might have
// synchronously completed and the encoder is already initialized. In this case, we don't need to await
// the promise anymore. This also fixes nasty async race condition bugs when multiple code paths are
// calling this method: It's important that the call that initialized the encoder go through this
// code first.
// No, this "if" statement is not useless. Sometimes, the above call to
// `ensureEncoder` might have synchronously completed and the encoder is
// already initialized. In this case, we don't need to await the promise
// anymore. This also fixes nasty async race condition bugs when multiple
// code paths are calling this method: It's important that the call that
// initialized the encoder go through this code first.
if (!this.encoderInitialized) {
await this.ensureEncoderPromise;
}
@@ -340,11 +530,11 @@ class VideoEncoderWrapper {
assert(this.encoderInitialized);
const keyFrameInterval = this.encodingConfig.keyFrameInterval ?? 5;
const multipleOfKeyFrameInterval = Math.floor(videoSample.timestamp / keyFrameInterval);
const multipleOfKeyFrameInterval = Math.floor(sampleToEncode.timestamp / keyFrameInterval);
// Ensure a key frame every keyFrameInterval seconds. It is important that all video tracks follow the same
// "key frame" rhythm, because aligned key frames are required to start new fragments in ISOBMFF or clusters
// in Matroska (or at least desirable).
// Ensure a key frame every keyFrameInterval seconds. It is important that all video tracks
// follow the same "key frame" rhythm, because aligned key frames are required to start new
// fragments in ISOBMFF or clusters in Matroska (or at least desirable).
const finalEncodeOptions = {
...encodeOptions,
keyFrame: encodeOptions?.keyFrame
@@ -357,7 +547,7 @@ class VideoEncoderWrapper {
this.customEncoderQueueSize++;
// We clone the sample so it cannot be closed on us from the outside before it reaches the encoder
const clonedSample = videoSample.clone();
const clonedSample = sampleToEncode.clone();
const promise = this.customEncoderCallSerializer
.call(() => this.customEncoder!.encode(clonedSample, finalEncodeOptions))
@@ -365,7 +555,6 @@ class VideoEncoderWrapper {
.catch((error: Error) => this.error ??= error)
.finally(() => {
clonedSample.close();
// `videoSample` gets closed in the finally block at the end of the method
});
if (this.customEncoderQueueSize >= 4) {
@@ -374,7 +563,7 @@ class VideoEncoderWrapper {
} else {
assert(this.encoder);
const videoFrame = videoSample.toVideoFrame();
const videoFrame = sampleToEncode.toVideoFrame();
const preciseTimingIndex = binarySearchLessOrEqual(
this.preciseTimings,
@@ -385,19 +574,19 @@ class VideoEncoderWrapper {
? this.preciseTimings[preciseTimingIndex]
: null;
if (existingEntry && existingEntry.microsecondTimestamp === videoFrame.timestamp) {
if (existingEntry.timestamp !== videoSample.timestamp) {
if (existingEntry.timestamp !== sampleToEncode.timestamp) {
// Mapping isn't unique, can't use the timestamp
existingEntry.timestampIsValid = false;
}
if (existingEntry.duration !== videoSample.duration) {
if (existingEntry.duration !== sampleToEncode.duration) {
// Mapping isn't unique, can't use the duration
existingEntry.durationIsValid = false;
}
} else {
this.preciseTimings.splice(preciseTimingIndex + 1, 0, {
microsecondTimestamp: videoFrame.timestamp,
timestamp: videoSample.timestamp,
duration: videoSample.duration,
timestamp: sampleToEncode.timestamp,
duration: sampleToEncode.duration,
timestampIsValid: true,
durationIsValid: true,
});
@@ -449,33 +638,49 @@ class VideoEncoderWrapper {
}
}
if (shouldClose) {
videoSample.close();
}
// We need to do this after sending the frame to the encoder as the frame otherwise might be closed
if (this.encoder.encodeQueueSize >= 4) {
await new Promise(resolve => this.encoder!.addEventListener('dequeue', resolve, { once: true }));
await new Promise(resolve =>
this.encoder!.addEventListener('dequeue', resolve, { once: true }),
);
}
}
await this.muxer!.mutex.currentPromise; // Allow the writer to apply backpressure
} finally {
if (shouldClose) {
// Make sure it's always closed, even if there was an error
videoSample.close();
}
} finally {
for (const sample of samplesToEncode) {
if (sample !== videoSample) {
sample.close();
}
}
}
}
/** Repeats the last frame rate sample to fill the gap up to the given timestamp. */
private async padFrameRate(until: number, encodeOptions?: VideoEncoderEncodeOptions) {
const frameRate = this.encodingConfig.transform!.frameRate!;
assert(this.frameRateLastSample);
const frameDifference = Math.round((until - this.frameRateLastTimestamp!) * frameRate);
for (let i = 1; i < frameDifference; i++) {
const sample = this.frameRateLastSample.clone();
sample.setTimestamp(this.frameRateLastTimestamp! + i / frameRate);
sample.setDuration(1 / frameRate);
await this.processAndEncode(sample, encodeOptions);
sample.close();
}
}
private ensureEncoder(videoSample: VideoSample) {
this.ensureEncoderPromise = (async () => {
const encoderConfig = buildVideoEncoderConfig({
...this.encodingConfig,
width: videoSample.codedWidth,
height: videoSample.codedHeight,
squarePixelWidth: videoSample.squarePixelWidth,
squarePixelHeight: videoSample.squarePixelHeight,
...this.encodingConfig,
framerate: this.source._connectedTrack?.metadata.frameRate,
});
this.encodingConfig.onEncoderConfig?.(encoderConfig);
@@ -691,7 +896,19 @@ class VideoEncoderWrapper {
}
async flushAndClose(forceClose: boolean) {
if (!forceClose) this.checkForEncoderError();
if (!forceClose) {
this.checkForEncoderError();
}
// Final frame rate padding: fill remaining frames up to the last sample's original end timestamp
if (!forceClose && this.frameRateLastSample) {
const frameRate = this.encodingConfig.transform!.frameRate!;
const alignedEnd = floorToDivisor(this.frameRateLastEndTimestamp!, frameRate);
await this.padFrameRate(alignedEnd);
}
this.frameRateLastSample?.close();
this.frameRateLastSample = null;
if (this.customEncoder) {
if (!forceClose) {
@@ -718,7 +935,9 @@ class VideoEncoderWrapper {
this.splitter?.close();
}
if (!forceClose) this.checkForEncoderError();
if (!forceClose) {
this.checkForEncoderError();
}
}
getQueueSize() {
src/misc.ts
+6
View File
@@ -443,6 +443,10 @@ export const floorToMultiple = (value: number, multiple: number) => {
return Math.floor(value / multiple) * multiple;
};
export const floorToDivisor = (value: number, multiple: number) => {
return Math.floor(value * multiple) / multiple;
};
export const ilog = (x: number) => {
let ret = 0;
while (x) {
@@ -1198,3 +1202,5 @@ export class EventEmitter<TEvents extends Record<string, unknown>> {
}
}
}
export const ceilToMultipleOfTwo = (value: number) => Math.ceil(value / 2) * 2;
src/sample.ts
+9 -7
View File
@@ -1120,14 +1120,16 @@ export type CropRectangle = {
height: number;
};
export const clampCropRectangle = (crop: CropRectangle, outerWidth: number, outerHeight: number) => {
crop.left = Math.min(crop.left, outerWidth);
crop.top = Math.min(crop.top, outerHeight);
crop.width = Math.min(crop.width, outerWidth - crop.left);
crop.height = Math.min(crop.height, outerHeight - crop.top);
export const clampCropRectangle = (crop: CropRectangle, outerWidth: number, outerHeight: number): CropRectangle => {
const left = Math.min(crop.left, outerWidth);
const top = Math.min(crop.top, outerHeight);
const width = Math.min(crop.width, outerWidth - left);
const height = Math.min(crop.height, outerHeight - top);
assert(crop.width >= 0);
assert(crop.height >= 0);
assert(width >= 0);
assert(height >= 0);
return { left, top, width, height };
};
export const validateCropRectangle = (crop: CropRectangle, prefix: string) => {
test/browser/media-sources.test.ts
+484 -2
View File
@@ -1,10 +1,16 @@
import { test } from 'vitest';
import { expect, test } from 'vitest';
import { Output } from '../../src/output.js';
import { WebMOutputFormat } from '../../src/output-format.js';
import { Mp4OutputFormat, WebMOutputFormat } from '../../src/output-format.js';
import { BufferTarget } from '../../src/target.js';
import { VideoSampleSource } from '../../src/media-source.js';
import { VideoSample } from '../../src/sample.js';
import { QUALITY_MEDIUM } from '../../src/encode.js';
import { Input } from '../../src/input.js';
import { ALL_FORMATS } from '../../src/input-format.js';
import { BufferSource } from '../../src/source.js';
import { VideoSampleSink } from '../../src/media-sink.js';
import { assert } from '../../src/misc.js';
import { InputVideoTrack } from '../../src/input-track.js';
test('VideoSampleSource.close() should be idempotent after finalize()', async () => {
const output = new Output({
@@ -33,3 +39,479 @@ test('VideoSampleSource.close() should be idempotent after finalize()', async ()
videoSource.close(); // This previously threw
});
test('Changing input dimensions throws with deny (default)', async () => {
const output = new Output({
format: new Mp4OutputFormat(),
target: new BufferTarget(),
});
const videoSource = new VideoSampleSource({
codec: 'vp8',
bitrate: QUALITY_MEDIUM,
});
output.addVideoTrack(videoSource);
await output.start();
const sample1 = new VideoSample(makeCanvas(100, 100), { timestamp: 0, duration: 1 / 30 });
await videoSource.add(sample1);
sample1.close();
const sample2 = new VideoSample(makeCanvas(200, 150), { timestamp: 1 / 30, duration: 1 / 30 });
await expect(videoSource.add(sample2)).rejects.toThrow(/Video sample size must remain constant/);
sample2.close();
videoSource.close();
});
test('Changing input dimensions with passThrough preserves per-frame dimensions', async () => {
const buffer = await encodeFrames(
{ codec: 'vp8', bitrate: QUALITY_MEDIUM, sizeChangeBehavior: 'passThrough' },
[{ width: 100, height: 100 }, { width: 200, height: 150 }],
);
const samples = await readBackSamples(buffer);
expect(samples).toHaveLength(2);
expect(samples[0]).toMatchObject({ codedWidth: 100, codedHeight: 100 });
expect(
// Either are valid; WebCodecs encoders don't like changing dimensions sometimes
(samples[1]!.codedWidth === 100 && samples[1]!.codedHeight === 100)
|| (samples[1]!.codedWidth === 200 && samples[1]!.codedHeight === 150),
).toBe(true);
});
test('Changing input dimensions with fill/contain/cover locks output to first frame dimensions', async () => {
for (const behavior of ['fill', 'contain', 'cover'] as const) {
const buffer = await encodeFrames(
{ codec: 'vp8', bitrate: QUALITY_MEDIUM, sizeChangeBehavior: behavior },
[{ width: 100, height: 100 }, { width: 200, height: 150 }],
);
const { input, track } = await readBackTrack(buffer);
expect(track.codedWidth).toBe(100);
expect(track.codedHeight).toBe(100);
input.dispose();
}
});
test('Same-sized frames with width and height set', async () => {
const buffer = await encodeFrames(
{ codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { width: 50, height: 80, fit: 'fill' } },
[{ width: 100, height: 100 }, { width: 100, height: 100 }],
);
const { input, track } = await readBackTrack(buffer);
expect(track.codedWidth).toBe(50);
expect(track.codedHeight).toBe(80);
input.dispose();
});
test('Same-sized frames with rotation set to 90', async () => {
const buffer = await encodeFrames(
{ codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { rotate: 90 } },
[{ width: 200, height: 100 }, { width: 200, height: 100 }],
);
const { input, track } = await readBackTrack(buffer);
expect(track.codedWidth).toBe(100);
expect(track.codedHeight).toBe(200);
input.dispose();
});
test('Same-sized frames with rotation, width and height', async () => {
const buffer = await encodeFrames(
{ codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { rotate: 90, width: 50, height: 80, fit: 'contain' } },
[{ width: 200, height: 100 }, { width: 200, height: 100 }],
);
const { input, track } = await readBackTrack(buffer);
expect(track.codedWidth).toBe(50);
expect(track.codedHeight).toBe(80);
input.dispose();
});
test('Changing dimensions with passThrough and rotation 90', async () => {
const buffer = await encodeFrames(
{ codec: 'vp8', bitrate: QUALITY_MEDIUM, sizeChangeBehavior: 'passThrough', transform: { rotate: 90 } },
[{ width: 200, height: 100 }, { width: 300, height: 150 }],
);
const samples = await readBackSamples(buffer);
expect(samples).toHaveLength(2);
expect(samples[0]).toMatchObject({ codedWidth: 100, codedHeight: 200 });
expect(
// Either are valid; WebCodecs encoders don't like changing dimensions sometimes
(samples[1]!.codedWidth === 150 && samples[1]!.codedHeight === 300)
|| (samples[1]!.codedWidth === 100 && samples[1]!.codedHeight === 200),
).toBe(true);
});
test('Changing dimensions with passThrough, width and height set', async () => {
const buffer = await encodeFrames(
{
codec: 'vp8',
bitrate: QUALITY_MEDIUM,
sizeChangeBehavior: 'passThrough',
transform: {
width: 50,
height: 80,
fit: 'fill',
},
},
[{ width: 100, height: 100 }, { width: 200, height: 150 }],
);
const samples = await readBackSamples(buffer);
expect(samples).toHaveLength(2);
// Both frames should be resized to the fixed width/height
expect(samples[0]).toMatchObject({ codedWidth: 50, codedHeight: 80 });
expect(samples[1]).toMatchObject({ codedWidth: 50, codedHeight: 80 });
});
test('transform.process identity function', async () => {
const buffer = await encodeFrames(
{ codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { process: sample => sample } },
[{ width: 100, height: 100 }, { width: 100, height: 100 }, { width: 100, height: 100 }],
);
const samples = await readBackSamples(buffer);
expect(samples).toHaveLength(3);
for (const sample of samples) {
expect(sample.codedWidth).toBe(100);
expect(sample.codedHeight).toBe(100);
}
});
test('transform.process manual resize', async () => {
const buffer = await encodeFrames(
{
codec: 'vp8',
bitrate: QUALITY_MEDIUM,
transform: {
process: (sample) => {
const canvas = new OffscreenCanvas(60, 40);
const ctx = canvas.getContext('2d')!;
sample.draw(ctx, 0, 0, 60, 40);
sample.close();
return new VideoSample(canvas, {
timestamp: sample.timestamp,
duration: sample.duration,
});
},
},
},
[{ width: 100, height: 100 }, { width: 100, height: 100 }],
);
const { input, track } = await readBackTrack(buffer);
expect(track.codedWidth).toBe(60);
expect(track.codedHeight).toBe(40);
input.dispose();
});
test('transform.process receives pre-transformed frames', async () => {
const receivedDimensions: { width: number; height: number }[] = [];
const buffer = await encodeFrames(
{
codec: 'vp8',
bitrate: QUALITY_MEDIUM,
transform: {
width: 50,
height: 80,
fit: 'fill',
process: (sample) => {
receivedDimensions.push({
width: sample.codedWidth,
height: sample.codedHeight,
});
return sample;
},
},
},
[{ width: 200, height: 200 }, { width: 200, height: 200 }],
);
expect(receivedDimensions).toHaveLength(2);
for (const dim of receivedDimensions) {
expect(dim.width).toBe(50);
expect(dim.height).toBe(80);
}
const { input, track } = await readBackTrack(buffer);
expect(track.codedWidth).toBe(50);
expect(track.codedHeight).toBe(80);
input.dispose();
});
test('transform.process drops all frames after the first', async () => {
let frameIndex = 0;
const buffer = await encodeFrames(
{
codec: 'vp8',
bitrate: QUALITY_MEDIUM,
transform: {
process: (sample) => {
if (frameIndex++ > 0) {
return null;
}
return sample;
},
},
},
[{ width: 100, height: 100 }, { width: 100, height: 100 }, { width: 100, height: 100 }],
);
const samples = await readBackSamples(buffer);
expect(samples).toHaveLength(1);
});
test('transform.process expands every frame into two', async () => {
const buffer = await encodeFrames(
{
codec: 'vp8',
bitrate: QUALITY_MEDIUM,
transform: {
process: (sample) => {
const t = sample.timestamp;
const d = sample.duration;
const clone = sample.clone();
clone.setTimestamp(2 * t);
clone.setDuration(d);
const clone2 = sample.clone();
clone2.setTimestamp(2 * t + d);
clone2.setDuration(d);
return [clone, clone2];
},
},
},
[{ width: 100, height: 100 }, { width: 100, height: 100 }, { width: 100, height: 100 }],
);
const samples = await readBackSamples(buffer);
expect(samples).toHaveLength(6);
const d = 1 / 30;
for (let i = 0; i < 6; i++) {
expect(samples[i]!.timestamp).toBe(i * d);
expect(samples[i]!.duration).toBe(d);
}
});
test('transform.frameRate normalizes variable-rate input to fixed rate', async () => {
const buffer = await encodeFrames(
{ codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { frameRate: 10 } },
[
{ width: 100, height: 100, timestamp: 0, duration: 0.15 },
{ width: 100, height: 100, timestamp: 0.15, duration: 0.1 },
{ width: 100, height: 100, timestamp: 0.25, duration: 0.05 },
],
);
const samples = await readBackSamples(buffer);
expect(samples).toHaveLength(3);
for (let i = 0; i < 3; i++) {
expect(samples[i]!.timestamp).toBeCloseTo(i * 0.1);
expect(samples[i]!.duration).toBeCloseTo(0.1);
}
});
test('transform.frameRate pads gaps by repeating last frame', async () => {
const buffer = await encodeFrames(
{ codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { frameRate: 10 } },
[
{ width: 100, height: 100, timestamp: 0, duration: 0.1 },
{ width: 100, height: 100, timestamp: 0.3, duration: 0.1 },
],
);
const samples = await readBackSamples(buffer);
expect(samples).toHaveLength(4);
for (let i = 0; i < 4; i++) {
expect(samples[i]!.timestamp).toBeCloseTo(i * 0.1);
expect(samples[i]!.duration).toBeCloseTo(0.1);
}
});
test('transform.frameRate deduplicates frames in the same slot', async () => {
const buffer = await encodeFrames(
{ codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { frameRate: 10 } },
[
{ width: 100, height: 100, timestamp: 0, duration: 0.03 },
{ width: 100, height: 100, timestamp: 0.03, duration: 0.03 },
{ width: 100, height: 100, timestamp: 0.06, duration: 0.04 },
{ width: 100, height: 100, timestamp: 0.1, duration: 0.1 },
],
);
const samples = await readBackSamples(buffer);
expect(samples).toHaveLength(2);
expect(samples[0]!.timestamp).toBe(0);
expect(samples[1]!.timestamp).toBeCloseTo(0.1);
});
test('transform.frameRate final padding fills remaining duration', async () => {
const buffer = await encodeFrames(
{ codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { frameRate: 10 } },
[
{ width: 100, height: 100, timestamp: 0, duration: 0.5 },
],
);
const samples = await readBackSamples(buffer);
expect(samples).toHaveLength(5);
for (let i = 0; i < 5; i++) {
expect(samples[i]!.timestamp).toBeCloseTo(i * 0.1);
}
});
test('transform.frameRate skipping and padding combined', async () => {
const buffer = await encodeFrames(
{ codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { frameRate: 10 } },
[
{ width: 100, height: 100, timestamp: 0, duration: 0.02 },
{ width: 100, height: 100, timestamp: 0.02, duration: 0.02 },
{ width: 100, height: 100, timestamp: 0.04, duration: 0.02 },
{ width: 100, height: 100, timestamp: 0.3, duration: 0.05 },
{ width: 100, height: 100, timestamp: 0.35, duration: 0.05 },
],
);
const samples = await readBackSamples(buffer);
expect(samples).toHaveLength(4);
for (let i = 0; i < 4; i++) {
expect(samples[i]!.timestamp).toBeCloseTo(i * 0.1);
expect(samples[i]!.duration).toBeCloseTo(0.1);
}
});
test('transform.frameRate works with transform', async () => {
const buffer = await encodeFrames(
{
codec: 'vp8',
bitrate: QUALITY_MEDIUM,
transform: { width: 50, height: 50, fit: 'fill', frameRate: 10 },
},
[
{ width: 100, height: 100, timestamp: 0, duration: 0.3 },
],
);
const samples = await readBackSamples(buffer);
expect(samples).toHaveLength(3);
for (let i = 0; i < 3; i++) {
expect(samples[i]!.timestamp).toBeCloseTo(i * 0.1);
expect(samples[i]!.duration).toBe(0.1);
expect(samples[i]!.codedWidth).toBe(50);
expect(samples[i]!.codedHeight).toBe(50);
}
});
test('transform.frameRate works with process', async () => {
const processedTimestamps: number[] = [];
const buffer = await encodeFrames(
{
codec: 'vp8',
bitrate: QUALITY_MEDIUM,
transform: {
frameRate: 10,
process: (sample) => {
processedTimestamps.push(sample.timestamp);
const canvas = new OffscreenCanvas(60, 40);
const ctx = canvas.getContext('2d')!;
sample.draw(ctx, 0, 0, 60, 40);
return new VideoSample(canvas, {
timestamp: sample.timestamp,
duration: sample.duration,
});
},
},
},
[
{ width: 100, height: 100, timestamp: 0, duration: 0.3 },
],
);
expect(processedTimestamps).toHaveLength(3);
for (let i = 0; i < 3; i++) {
expect(processedTimestamps[i]).toBeCloseTo(i * 0.1);
}
const { input, track } = await readBackTrack(buffer);
expect(track.codedWidth).toBe(60);
expect(track.codedHeight).toBe(40);
input.dispose();
});
const makeCanvas = (width: number, height: number) => {
const canvas = new OffscreenCanvas(width, height);
const ctx = canvas.getContext('2d')!;
ctx.fillStyle = 'red';
ctx.fillRect(0, 0, width, height);
return canvas;
};
const encodeFrames = async (
encodingConfig: ConstructorParameters<typeof VideoSampleSource>[0],
frames: { width: number; height: number; timestamp?: number; duration?: number }[],
) => {
const output = new Output({
format: new Mp4OutputFormat(),
target: new BufferTarget(),
});
const videoSource = new VideoSampleSource(encodingConfig);
output.addVideoTrack(videoSource);
await output.start();
for (let i = 0; i < frames.length; i++) {
const f = frames[i]!;
const canvas = makeCanvas(f.width, f.height);
const sample = new VideoSample(canvas, {
timestamp: f.timestamp ?? i / 30,
duration: f.duration ?? 1 / 30,
});
await videoSource.add(sample);
sample.close();
}
await output.finalize();
return output.target.buffer!;
};
const readBackTrack = async (buffer: ArrayBuffer) => {
const input = new Input({
source: new BufferSource(buffer),
formats: ALL_FORMATS,
});
const track = await input.getPrimaryVideoTrack() as InputVideoTrack;
assert(track);
return { input, track };
};
const readBackSamples = async (buffer: ArrayBuffer) => {
const { input, track } = await readBackTrack(buffer);
const sink = new VideoSampleSink(track);
const samples: { codedWidth: number; codedHeight: number; timestamp: number; duration: number }[] = [];
for await (using sample of sink.samples()) {
samples.push({
codedWidth: sample.codedWidth,
codedHeight: sample.codedHeight,
timestamp: sample.timestamp,
duration: sample.duration,
});
}
input.dispose();
return samples;
};