From 024660773bb9d33ec06f63445a75e3b0c23788cb Mon Sep 17 00:00:00 2001 From: Vanilagy <1696106+Vanilagy@users.noreply.github.com> Date: Wed, 1 Apr 2026 13:30:34 +0200 Subject: [PATCH] Add video frame preprocessing to VideoEncodingConfig & VideoEncoderWrapper --- .../procedural-generation.ts | 11 +- src/conversion.ts | 16 +- src/encode.ts | 112 +++- src/media-sink.ts | 4 +- src/media-source.ts | 571 ++++++++++++------ src/misc.ts | 6 + src/sample.ts | 16 +- test/browser/media-sources.test.ts | 486 ++++++++++++++- 8 files changed, 1021 insertions(+), 201 deletions(-) diff --git a/examples/procedural-generation/procedural-generation.ts b/examples/procedural-generation/procedural-generation.ts index b47d0a8..97a6181 100644 --- a/examples/procedural-generation/procedural-generation.ts +++ b/examples/procedural-generation/procedural-generation.ts @@ -99,7 +99,6 @@ const generateVideo = async () => { output = new Output({ rootPath: 'master.m3u8', target: async ({ path }) => { - /* const fileHandle = await dirHandle.getFileHandle(path, { create: true }); const writable = await fileHandle.createWritable(); @@ -107,8 +106,8 @@ const generateVideo = async () => { target.on('finalized', () => console.log('Finalizado', path)); return target; - */ + /* const target = new BufferTarget(); target.on('finalized', async () => { if (path.includes('m3u8')) { @@ -123,11 +122,12 @@ const generateVideo = async () => { } return target; + */ }, format: new HlsOutputFormat({ segmentFormat: new CmafOutputFormat(), // singleFilePerPlaylist: true, - live: true, + // live: true, getPlaylistPath: info => `sussex-${info.n}.m3u8`, }), }); @@ -146,6 +146,9 @@ const generateVideo = async () => { codec: videoCodec, bitrate: QUALITY_HIGH, keyFrameInterval: 2, + transform: { + frameRate: 5, + }, }); output.addVideoTrack(canvasSource, { frameRate }); @@ -206,7 +209,7 @@ const generateVideo = async () => { // automatically slow down the rendering loop when the encoder can't keep up. await canvasSource.add(currentTime, 1 / frameRate); - await new Promise(resolve => setTimeout(resolve, 1000 / frameRate)); + // await new Promise(resolve => setTimeout(resolve, 1000 / frameRate)); } // Signal to the output that no more video frames are coming (not necessary, but recommended) diff --git a/src/conversion.ts b/src/conversion.ts index b6a76b5..786d54f 100644 --- a/src/conversion.ts +++ b/src/conversion.ts @@ -38,7 +38,9 @@ import { } from './media-source'; import { assert, + ceilToMultipleOfTwo, clamp, + floorToDivisor, isIso639Dash2LanguageCode, MaybePromise, normalizeRotation, @@ -915,9 +917,9 @@ export class Conversion { ? [track.squarePixelWidth, track.squarePixelHeight] : [track.squarePixelHeight, track.squarePixelWidth]; - const crop = trackOptions.crop; + let crop = trackOptions.crop; if (crop) { - clampCropRectangle(crop, rotatedWidth, rotatedHeight); + crop = clampCropRectangle(crop, rotatedWidth, rotatedHeight); } const [originalWidth, originalHeight] = crop @@ -929,8 +931,6 @@ export class Conversion { const aspectRatio = width / height; // A lot of video encoders require that the dimensions be multiples of 2 - const ceilToMultipleOfTwo = (value: number) => Math.ceil(value / 2) * 2; - if (trackOptions.width !== undefined && trackOptions.height === undefined) { width = ceilToMultipleOfTwo(trackOptions.width); height = ceilToMultipleOfTwo(Math.round(width / aspectRatio)); @@ -1144,7 +1144,7 @@ export class Conversion { if (frameRate !== undefined) { // Logic for skipping/repeating frames when a frame rate is set - const alignedTimestamp = Math.floor(adjustedSampleTimestamp * frameRate) / frameRate; + const alignedTimestamp = floorToDivisor(adjustedSampleTimestamp, frameRate); if (lastCanvas !== null) { if (alignedTimestamp <= lastCanvasTimestamp!) { @@ -1180,7 +1180,7 @@ export class Conversion { assert(frameRate !== undefined); // If necessary, pad until the end timestamp of the last sample - await padFrames(Math.floor(lastCanvasEndTimestamp * frameRate) / frameRate); + await padFrames(floorToDivisor(lastCanvasEndTimestamp, frameRate)); } source.close(); @@ -1225,7 +1225,7 @@ export class Conversion { if (frameRate !== undefined) { // Logic for skipping/repeating frames when a frame rate is set - const alignedTimestamp = Math.floor(adjustedSampleTimestamp * frameRate) / frameRate; + const alignedTimestamp = floorToDivisor(adjustedSampleTimestamp, frameRate); if (lastSample !== null) { if (alignedTimestamp <= lastSampleTimestamp!) { @@ -1261,7 +1261,7 @@ export class Conversion { assert(frameRate !== undefined); // If necessary, pad until the end timestamp of the last sample - await padFrames(Math.floor(lastSampleEndTimestamp * frameRate) / frameRate); + await padFrames(floorToDivisor(lastSampleEndTimestamp, frameRate)); } source.close(); diff --git a/src/encode.ts b/src/encode.ts index 537e14c..1a556e3 100644 --- a/src/encode.ts +++ b/src/encode.ts @@ -22,8 +22,9 @@ import { VideoCodec, } from './codec'; import { customAudioEncoders, customVideoEncoders } from './custom-coder'; -import { isFirefox } from './misc'; +import { isFirefox, MaybePromise, Rotation } from './misc'; import { EncodedPacket } from './packet'; +import { CropRectangle, validateCropRectangle, VideoSample } from './sample'; const canEncodeVideoMemo = new Map>(); const canEncodeAudioMemo = new Map>(); @@ -61,6 +62,58 @@ export type VideoEncodingConfig = { */ sizeChangeBehavior?: 'deny' | 'passThrough' | 'fill' | 'contain' | 'cover'; + /** + * Optional transformations to apply to the video frames before they are passed to the encoder. + */ + transform?: { + /** + * The width in pixels to resize the frames to. If height is not set, it will be deduced + * automatically based on aspect ratio. + */ + width?: number; + /** + * The height in pixels to resize the frames to. If width is not set, it will be deduced + * automatically based on aspect ratio. + */ + height?: number; + /** + * The fitting algorithm in case both width and height are set. + * + * - `'fill'` will stretch the image to fill the entire box, potentially altering aspect ratio. + * - `'contain'` will contain the entire image within the box while preserving aspect ratio. This may lead to + * letterboxing. + * - `'cover'` will scale the image until the entire box is filled, while preserving aspect ratio. + * + * To avoid ambiguity, this field must not be set when `sizeChangeBehavior` is `'fill'`, `'contain'` or + * `'deny'`, since `sizeChangeBehavior` already determines the fitting algorithm. + */ + fit?: 'fill' | 'contain' | 'cover'; + /** + * The clockwise rotation by which to rotate the frames. Rotation is applied before resizing. + */ + rotate?: Rotation; + /** + * Specifies the rectangular region of the frames to crop to. The crop region will automatically be + * clamped to the dimensions of the frame. Cropping is performed after rotation but before resizing. + */ + crop?: CropRectangle; + /** + * The frame rate in hertz to normalize the video frame stream to. + */ + frameRate?: number; + /** + * Allows for custom user-defined processing of video frames, e.g. for applying overlays, color transformations, + * or timestamp modifications. Will be called for each video frame after transformations and frame rate + * corrections. + * + * Must return a {@link VideoSample} or a `CanvasImageSource`, an array of them, or `null` for dropping the + * frame. When non-timestamped data is returned, the timestamp and duration from the input sample will be used. + */ + process?: (sample: VideoSample) => MaybePromise< + CanvasImageSource | VideoSample | (CanvasImageSource | VideoSample)[] | null + >; + }; + /** Called for each successfully encoded packet. Both the packet and the encoding metadata are passed. */ onEncodedPacket?: (packet: EncodedPacket, meta: EncodedVideoChunkMetadata | undefined) => unknown; /** @@ -95,6 +148,61 @@ export const validateVideoEncodingConfig = (config: VideoEncodingConfig) => { + ' or \'cover\'.', ); } + if (config.transform !== undefined) { + if (typeof config.transform !== 'object' || !config.transform) { + throw new TypeError('config.transform, when provided, must be an object.'); + } + if ( + config.transform.width !== undefined + && (!Number.isInteger(config.transform.width) || config.transform.width <= 0) + ) { + throw new TypeError('config.transform.width, when provided, must be a positive integer.'); + } + if ( + config.transform.height !== undefined + && (!Number.isInteger(config.transform.height) || config.transform.height <= 0) + ) { + throw new TypeError('config.transform.height, when provided, must be a positive integer.'); + } + if (config.transform.fit !== undefined && !['fill', 'contain', 'cover'].includes(config.transform.fit)) { + throw new TypeError('config.transform.fit, when provided, must be one of "fill", "contain", or "cover".'); + } + if ( + config.transform.width !== undefined + && config.transform.height !== undefined + && config.transform.fit === undefined + && !['fill', 'contain', 'cover'].includes(config.sizeChangeBehavior!) + ) { + throw new TypeError( + 'When both config.transform.width and config.transform.height are provided, config.transform.fit' + + ' must also be provided.', + ); + } + if ( + config.transform.fit !== undefined + && ['fill', 'contain', 'cover'].includes(config.sizeChangeBehavior!) + ) { + throw new TypeError( + 'config.transform.fit cannot be used when config.sizeChangeBehavior is \'fill\', \'contain\' or' + + ' \'cover\', as sizeChangeBehavior already determines the fitting algorithm.', + ); + } + if (config.transform.rotate !== undefined && ![0, 90, 180, 270].includes(config.transform.rotate)) { + throw new TypeError('config.transform.rotate, when provided, must be 0, 90, 180 or 270.'); + } + if (config.transform.crop !== undefined) { + validateCropRectangle(config.transform.crop, 'config.'); + } + if (config.transform.process !== undefined && typeof config.transform.process !== 'function') { + throw new TypeError('config.transform.process, when provided, must be a function.'); + } + if ( + config.transform.frameRate !== undefined + && (!Number.isFinite(config.transform.frameRate) || config.transform.frameRate <= 0) + ) { + throw new TypeError('config.transform.frameRate, when provided, must be a finite positive number.'); + } + } if (config.onEncodedPacket !== undefined && typeof config.onEncodedPacket !== 'function') { throw new TypeError('config.onEncodedChunk, when provided, must be a function.'); } @@ -106,7 +214,7 @@ export const validateVideoEncodingConfig = (config: VideoEncodingConfig) => { }; /** - * Additional options that control audio encoding. + * Additional options that control video encoding. * @group Encoding * @public */ diff --git a/src/media-sink.ts b/src/media-sink.ts index 23c4be2..2f83ff1 100644 --- a/src/media-sink.ts +++ b/src/media-sink.ts @@ -1636,9 +1636,9 @@ export class CanvasSink { ? [videoTrack.squarePixelWidth, videoTrack.squarePixelHeight] : [videoTrack.squarePixelHeight, videoTrack.squarePixelWidth]; - const crop = options.crop; + let crop = options.crop; if (crop) { - clampCropRectangle(crop, rotatedWidth, rotatedHeight); + crop = clampCropRectangle(crop, rotatedWidth, rotatedHeight); } let [width, height] = crop diff --git a/src/media-source.ts b/src/media-source.ts index 3392765..56d938a 100644 --- a/src/media-source.ts +++ b/src/media-source.ts @@ -25,10 +25,13 @@ import { assertNever, binarySearchLessOrEqual, CallSerializer, + ceilToMultipleOfTwo, clamp, clearIntervalUnthrottled, + floorToDivisor, isFirefox, last, + normalizeRotation, promiseWithResolvers, roundToDivisor, setInt24, @@ -47,7 +50,7 @@ import { customAudioEncoders, } from './custom-coder'; import { EncodedPacket, EncodedPacketSideData } from './packet'; -import { AudioSample, VideoSample } from './sample'; +import { AudioSample, clampCropRectangle, VideoSample } from './sample'; import { AudioEncodingConfig, buildAudioEncoderConfig, @@ -222,9 +225,19 @@ class VideoEncoderWrapper { private encoder: VideoEncoder | null = null; private muxer: Muxer | null = null; private lastMultipleOfKeyFrameInterval = -1; + + private resizeCanvas: HTMLCanvasElement | OffscreenCanvas | null = null; + // Tracks the input dimensions of the first frame private codedWidth: number | null = null; private codedHeight: number | null = null; - private resizeCanvas: HTMLCanvasElement | OffscreenCanvas | null = null; + // Tracks the output dimensions of the first frame (used to lock dimensions for fill/contain/cover) + private outputWidth: number | null = null; + private outputHeight: number | null = null; + + // Frame rate normalization state + private frameRateLastSample: VideoSample | null = null; + private frameRateLastTimestamp: number | null = null; + private frameRateLastEndTimestamp: number | null = null; // VideoEncoder converts everything to microseconds, so we need to do some bookkeeping to restore the original // timing information @@ -253,69 +266,38 @@ class VideoEncoderWrapper { */ private error: Error | null = null; - constructor(private source: VideoSource, private encodingConfig: VideoEncodingConfig) {} + constructor(private source: VideoSource, private encodingConfig: VideoEncodingConfig) { + const sizeChangeBehavior = encodingConfig.sizeChangeBehavior ?? 'deny'; + if (['fill', 'contain', 'cover'].includes(sizeChangeBehavior) && encodingConfig.transform?.fit !== undefined) { + throw new TypeError( + `Cannot set 'fit' when 'sizeChangeBehavior' is '${sizeChangeBehavior}'. ` + + `The size change behavior determines the fit in this case.`, + ); + } + } async add(videoSample: VideoSample, shouldClose: boolean, encodeOptions?: VideoEncoderEncodeOptions) { + const originalSample = videoSample; + try { this.checkForEncoderError(); this.source._ensureValidAdd(); - // Ensure video sample size remains constant + const config = this.encodingConfig; + const sizeChangeBehavior = config.sizeChangeBehavior ?? 'deny'; + let isSizeChange = false; + + // Ensure video sample size remains constant or handle the change if (this.codedWidth !== null && this.codedHeight !== null) { if (videoSample.codedWidth !== this.codedWidth || videoSample.codedHeight !== this.codedHeight) { - const sizeChangeBehavior = this.encodingConfig.sizeChangeBehavior ?? 'deny'; - - if (sizeChangeBehavior === 'passThrough') { - // Do nada - } else if (sizeChangeBehavior === 'deny') { + isSizeChange = true; + if (sizeChangeBehavior === 'deny') { throw new Error( `Video sample size must remain constant. Expected ${this.codedWidth}x${this.codedHeight},` + ` got ${videoSample.codedWidth}x${videoSample.codedHeight}. To allow the sample size to` - + ` change over time, set \`sizeChangeBehavior\` to a value other than 'strict' in the` + + ` change over time, set \`sizeChangeBehavior\` to a value other than 'deny' in the` + ` encoding options.`, ); - } else { - let canvasIsNew = false; - - if (!this.resizeCanvas) { - if (typeof document !== 'undefined') { - // Prefer an HTMLCanvasElement - this.resizeCanvas = document.createElement('canvas'); - this.resizeCanvas.width = this.codedWidth; - this.resizeCanvas.height = this.codedHeight; - } else { - this.resizeCanvas = new OffscreenCanvas(this.codedWidth, this.codedHeight); - } - - canvasIsNew = true; - } - - const context = this.resizeCanvas.getContext('2d', { - alpha: isFirefox(), // Firefox has VideoFrame glitches with opaque canvases - }) as CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D; - assert(context); - - if (!canvasIsNew) { - if (isFirefox()) { - context.fillStyle = 'black'; - context.fillRect(0, 0, this.codedWidth, this.codedHeight); - } else { - context.clearRect(0, 0, this.codedWidth, this.codedHeight); - } - } - - videoSample.drawWithFit(context, { fit: sizeChangeBehavior }); - - if (shouldClose) { - videoSample.close(); - } - - videoSample = new VideoSample(this.resizeCanvas, { - timestamp: videoSample.timestamp, - duration: videoSample.duration, - rotation: videoSample.rotation, - }); - shouldClose = true; } } } else { @@ -323,159 +305,382 @@ class VideoEncoderWrapper { this.codedHeight = videoSample.codedHeight; } - if (!this.encoderInitialized) { - if (!this.ensureEncoderPromise) { - this.ensureEncoder(videoSample); + // Determine if we need to apply transformations via canvas + const hasTransformConfig = config.transform?.width !== undefined + || config.transform?.height !== undefined + || config.transform?.rotate !== undefined + || config.transform?.crop !== undefined; + const needsTransform = hasTransformConfig || (isSizeChange && sizeChangeBehavior !== 'passThrough'); + + if (needsTransform) { + const rotation = normalizeRotation(videoSample.rotation + (config.transform?.rotate ?? 0)); + const [rotatedWidth, rotatedHeight] = rotation % 180 === 0 + ? [videoSample.codedWidth, videoSample.codedHeight] + : [videoSample.codedHeight, videoSample.codedWidth]; + + // Clamp crop rectangle to the rotated video dimensions + let finalCrop = config.transform?.crop; + if (finalCrop) { + finalCrop = clampCropRectangle(finalCrop, rotatedWidth, rotatedHeight); } - // No, this "if" statement is not useless. Sometimes, the above call to `ensureEncoder` might have - // synchronously completed and the encoder is already initialized. In this case, we don't need to await - // the promise anymore. This also fixes nasty async race condition bugs when multiple code paths are - // calling this method: It's important that the call that initialized the encoder go through this - // code first. - if (!this.encoderInitialized) { - await this.ensureEncoderPromise; - } - } - assert(this.encoderInitialized); + const cropWidth = finalCrop ? finalCrop.width : rotatedWidth; + const cropHeight = finalCrop ? finalCrop.height : rotatedHeight; + const originalAspectRatio = cropWidth / cropHeight; - const keyFrameInterval = this.encodingConfig.keyFrameInterval ?? 5; - const multipleOfKeyFrameInterval = Math.floor(videoSample.timestamp / keyFrameInterval); + let targetWidth: number; + let targetHeight: number; + let appliedFit: 'fill' | 'contain' | 'cover' = config.transform?.fit ?? 'fill'; - // Ensure a key frame every keyFrameInterval seconds. It is important that all video tracks follow the same - // "key frame" rhythm, because aligned key frames are required to start new fragments in ISOBMFF or clusters - // in Matroska (or at least desirable). - const finalEncodeOptions = { - ...encodeOptions, - keyFrame: encodeOptions?.keyFrame - || keyFrameInterval === 0 - || multipleOfKeyFrameInterval !== this.lastMultipleOfKeyFrameInterval, - }; - this.lastMultipleOfKeyFrameInterval = multipleOfKeyFrameInterval; + // If the size changed and behavior is fill/contain/cover, lock to the original output dimensions + if (isSizeChange && sizeChangeBehavior !== 'passThrough') { + assert(this.outputWidth); + assert(this.outputHeight); + assert(sizeChangeBehavior !== 'deny'); - if (this.customEncoder) { - this.customEncoderQueueSize++; - - // We clone the sample so it cannot be closed on us from the outside before it reaches the encoder - const clonedSample = videoSample.clone(); - - const promise = this.customEncoderCallSerializer - .call(() => this.customEncoder!.encode(clonedSample, finalEncodeOptions)) - .then(() => this.customEncoderQueueSize--) - .catch((error: Error) => this.error ??= error) - .finally(() => { - clonedSample.close(); - // `videoSample` gets closed in the finally block at the end of the method - }); - - if (this.customEncoderQueueSize >= 4) { - await promise; - } - } else { - assert(this.encoder); - - const videoFrame = videoSample.toVideoFrame(); - - const preciseTimingIndex = binarySearchLessOrEqual( - this.preciseTimings, - videoFrame.timestamp, - x => x.microsecondTimestamp, - ); - const existingEntry = preciseTimingIndex !== -1 - ? this.preciseTimings[preciseTimingIndex] - : null; - if (existingEntry && existingEntry.microsecondTimestamp === videoFrame.timestamp) { - if (existingEntry.timestamp !== videoSample.timestamp) { - // Mapping isn't unique, can't use the timestamp - existingEntry.timestampIsValid = false; - } - if (existingEntry.duration !== videoSample.duration) { - // Mapping isn't unique, can't use the duration - existingEntry.durationIsValid = false; - } + targetWidth = this.outputWidth!; + targetHeight = this.outputHeight!; + appliedFit = sizeChangeBehavior; } else { - this.preciseTimings.splice(preciseTimingIndex + 1, 0, { - microsecondTimestamp: videoFrame.timestamp, - timestamp: videoSample.timestamp, - duration: videoSample.duration, - timestampIsValid: true, - durationIsValid: true, - }); - - // Make sure it doesn't grow indefinitely - if (this.preciseTimings.length > 128) { - this.preciseTimings.shift(); - } - } - - if (!this.alphaEncoder) { - // No alpha encoder, simple case - this.encoder.encode(videoFrame, finalEncodeOptions); - videoFrame.close(); - } else { - // We're expected to encode alpha as well - const frameDefinitelyHasNoAlpha = !!videoFrame.format && !videoFrame.format.includes('A'); - - if (frameDefinitelyHasNoAlpha || this.splitterCreationFailed) { - this.alphaFrameQueue.push(null); - this.encoder.encode(videoFrame, finalEncodeOptions); - videoFrame.close(); + // Otherwise, dynamically calculate the target dimensions based on config and aspect ratio + if (config.transform?.width !== undefined && config.transform?.height === undefined) { + targetWidth = config.transform.width; + targetHeight = ceilToMultipleOfTwo(Math.round(targetWidth / originalAspectRatio)); + } else if (config.transform?.width === undefined && config.transform?.height !== undefined) { + targetHeight = config.transform.height; + targetWidth = ceilToMultipleOfTwo(Math.round(targetHeight * originalAspectRatio)); + } else if (config.transform?.width !== undefined && config.transform?.height !== undefined) { + targetWidth = config.transform?.width; + targetHeight = config.transform?.height; } else { - const width = videoFrame.displayWidth; - const height = videoFrame.displayHeight; - - if (!this.splitter) { - try { - this.splitter = new ColorAlphaSplitter(width, height); - } catch (error) { - console.error('Due to an error, only color data will be encoded.', error); - - this.splitterCreationFailed = true; - this.alphaFrameQueue.push(null); - this.encoder.encode(videoFrame, finalEncodeOptions); - videoFrame.close(); - } - } - - if (this.splitter) { - const colorFrame = this.splitter.extractColor(videoFrame); - const alphaFrame = this.splitter.extractAlpha(videoFrame); - - this.alphaFrameQueue.push(alphaFrame); - this.encoder.encode(colorFrame, finalEncodeOptions); - colorFrame.close(); - videoFrame.close(); - } + targetWidth = cropWidth; + targetHeight = cropHeight; } } + // Save the output dimensions of the first frame + if (this.outputWidth === null || this.outputHeight === null) { + this.outputWidth = targetWidth; + this.outputHeight = targetHeight; + } + + let canvasIsNew = false; + + if (!this.resizeCanvas) { + if (typeof document !== 'undefined') { + // Prefer an HTMLCanvasElement + this.resizeCanvas = document.createElement('canvas'); + this.resizeCanvas.width = targetWidth; + this.resizeCanvas.height = targetHeight; + } else { + this.resizeCanvas = new OffscreenCanvas(targetWidth, targetHeight); + } + canvasIsNew = true; + } else if (this.resizeCanvas.width !== targetWidth || this.resizeCanvas.height !== targetHeight) { + // Dynamically resize the canvas if the target dimensions have changed + this.resizeCanvas.width = targetWidth; + this.resizeCanvas.height = targetHeight; + } + + const context = this.resizeCanvas.getContext('2d', { + // Firefox has VideoFrame glitches with opaque canvases + alpha: this.encodingConfig.alpha === 'keep' || isFirefox(), + }) as CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D; + assert(context); + + if (typeof context.resetTransform === 'function') { + context.resetTransform(); + } + + if (!canvasIsNew) { + if (isFirefox()) { + context.fillStyle = 'black'; + context.fillRect(0, 0, targetWidth, targetHeight); + } else { + context.clearRect(0, 0, targetWidth, targetHeight); + } + } + + videoSample.drawWithFit(context, { + fit: appliedFit, + rotation: rotation, + crop: finalCrop, + }); + if (shouldClose) { videoSample.close(); } - // We need to do this after sending the frame to the encoder as the frame otherwise might be closed - if (this.encoder.encodeQueueSize >= 4) { - await new Promise(resolve => this.encoder!.addEventListener('dequeue', resolve, { once: true })); + videoSample = new VideoSample(this.resizeCanvas, { + timestamp: videoSample.timestamp, + duration: videoSample.duration, + rotation: 0, // Rotation is now baked into the canvas + }); + shouldClose = true; + } else { + // If no canvas is needed, we still need to record the output dimensions for the first frame + if (this.outputWidth === null || this.outputHeight === null) { + this.outputWidth = videoSample.codedWidth; + this.outputHeight = videoSample.codedHeight; } } - await this.muxer!.mutex.currentPromise; // Allow the writer to apply backpressure + const frameRate = config.transform?.frameRate; + if (frameRate !== undefined) { + // Apply frame rate normalization + const originalEndTimestamp = videoSample.timestamp + videoSample.duration; + const alignedTimestamp = floorToDivisor(videoSample.timestamp, frameRate); + + if (this.frameRateLastSample !== null) { + if (alignedTimestamp <= this.frameRateLastTimestamp!) { + // Same frame rate slot, replace stored sample with the newer one + this.frameRateLastSample.close(); + this.frameRateLastSample = videoSample.clone(); + this.frameRateLastEndTimestamp = originalEndTimestamp; + return; + } else { + // Pad the gap by repeating the previous frame + await this.padFrameRate(alignedTimestamp, encodeOptions); + } + } + + // Clone if the sample is still the user's, to avoid mutating externally-owned data + if (videoSample === originalSample) { + videoSample = videoSample.clone(); + shouldClose = true; + } + + videoSample.setTimestamp(alignedTimestamp); + videoSample.setDuration(1 / frameRate); + + this.frameRateLastSample?.close(); + this.frameRateLastSample = videoSample.clone(); + this.frameRateLastTimestamp = alignedTimestamp; + this.frameRateLastEndTimestamp = originalEndTimestamp; + } + + await this.processAndEncode(videoSample, encodeOptions); } finally { if (shouldClose) { - // Make sure it's always closed, even if there was an error videoSample.close(); } } } + /** + * Runs the process function (if any) and encodes the resulting samples. + */ + private async processAndEncode( + videoSample: VideoSample, + encodeOptions?: VideoEncoderEncodeOptions, + ) { + const config = this.encodingConfig; + let samplesToEncode: VideoSample[]; + + // Apply the user-defined process function, if any + if (config.transform?.process) { + let processed = config.transform.process(videoSample); + if (processed instanceof Promise) { + processed = await processed; + } + + if (processed === null) { + return; + } + + if (!Array.isArray(processed)) { + processed = [processed]; + } + + samplesToEncode = processed.map((x) => { + if (x instanceof VideoSample) { + return x; + } + + if (typeof VideoFrame !== 'undefined' && x instanceof VideoFrame) { + return new VideoSample(x); + } + + return new VideoSample(x, { + timestamp: videoSample.timestamp, + duration: videoSample.duration, + }); + }); + } else { + samplesToEncode = [videoSample]; + } + + try { + for (const sampleToEncode of samplesToEncode) { + if (!this.encoderInitialized) { + if (!this.ensureEncoderPromise) { + this.ensureEncoder(sampleToEncode); + } + + // No, this "if" statement is not useless. Sometimes, the above call to + // `ensureEncoder` might have synchronously completed and the encoder is + // already initialized. In this case, we don't need to await the promise + // anymore. This also fixes nasty async race condition bugs when multiple + // code paths are calling this method: It's important that the call that + // initialized the encoder go through this code first. + if (!this.encoderInitialized) { + await this.ensureEncoderPromise; + } + } + assert(this.encoderInitialized); + + const keyFrameInterval = this.encodingConfig.keyFrameInterval ?? 5; + const multipleOfKeyFrameInterval = Math.floor(sampleToEncode.timestamp / keyFrameInterval); + + // Ensure a key frame every keyFrameInterval seconds. It is important that all video tracks + // follow the same "key frame" rhythm, because aligned key frames are required to start new + // fragments in ISOBMFF or clusters in Matroska (or at least desirable). + const finalEncodeOptions = { + ...encodeOptions, + keyFrame: encodeOptions?.keyFrame + || keyFrameInterval === 0 + || multipleOfKeyFrameInterval !== this.lastMultipleOfKeyFrameInterval, + }; + this.lastMultipleOfKeyFrameInterval = multipleOfKeyFrameInterval; + + if (this.customEncoder) { + this.customEncoderQueueSize++; + + // We clone the sample so it cannot be closed on us from the outside before it reaches the encoder + const clonedSample = sampleToEncode.clone(); + + const promise = this.customEncoderCallSerializer + .call(() => this.customEncoder!.encode(clonedSample, finalEncodeOptions)) + .then(() => this.customEncoderQueueSize--) + .catch((error: Error) => this.error ??= error) + .finally(() => { + clonedSample.close(); + }); + + if (this.customEncoderQueueSize >= 4) { + await promise; + } + } else { + assert(this.encoder); + + const videoFrame = sampleToEncode.toVideoFrame(); + + const preciseTimingIndex = binarySearchLessOrEqual( + this.preciseTimings, + videoFrame.timestamp, + x => x.microsecondTimestamp, + ); + const existingEntry = preciseTimingIndex !== -1 + ? this.preciseTimings[preciseTimingIndex] + : null; + if (existingEntry && existingEntry.microsecondTimestamp === videoFrame.timestamp) { + if (existingEntry.timestamp !== sampleToEncode.timestamp) { + // Mapping isn't unique, can't use the timestamp + existingEntry.timestampIsValid = false; + } + if (existingEntry.duration !== sampleToEncode.duration) { + // Mapping isn't unique, can't use the duration + existingEntry.durationIsValid = false; + } + } else { + this.preciseTimings.splice(preciseTimingIndex + 1, 0, { + microsecondTimestamp: videoFrame.timestamp, + timestamp: sampleToEncode.timestamp, + duration: sampleToEncode.duration, + timestampIsValid: true, + durationIsValid: true, + }); + + // Make sure it doesn't grow indefinitely + if (this.preciseTimings.length > 128) { + this.preciseTimings.shift(); + } + } + + if (!this.alphaEncoder) { + // No alpha encoder, simple case + this.encoder.encode(videoFrame, finalEncodeOptions); + videoFrame.close(); + } else { + // We're expected to encode alpha as well + const frameDefinitelyHasNoAlpha = !!videoFrame.format && !videoFrame.format.includes('A'); + + if (frameDefinitelyHasNoAlpha || this.splitterCreationFailed) { + this.alphaFrameQueue.push(null); + this.encoder.encode(videoFrame, finalEncodeOptions); + videoFrame.close(); + } else { + const width = videoFrame.displayWidth; + const height = videoFrame.displayHeight; + + if (!this.splitter) { + try { + this.splitter = new ColorAlphaSplitter(width, height); + } catch (error) { + console.error('Due to an error, only color data will be encoded.', error); + + this.splitterCreationFailed = true; + this.alphaFrameQueue.push(null); + this.encoder.encode(videoFrame, finalEncodeOptions); + videoFrame.close(); + } + } + + if (this.splitter) { + const colorFrame = this.splitter.extractColor(videoFrame); + const alphaFrame = this.splitter.extractAlpha(videoFrame); + + this.alphaFrameQueue.push(alphaFrame); + this.encoder.encode(colorFrame, finalEncodeOptions); + colorFrame.close(); + videoFrame.close(); + } + } + } + + // We need to do this after sending the frame to the encoder as the frame otherwise might be closed + if (this.encoder.encodeQueueSize >= 4) { + await new Promise(resolve => + this.encoder!.addEventListener('dequeue', resolve, { once: true }), + ); + } + } + + await this.muxer!.mutex.currentPromise; // Allow the writer to apply backpressure + } + } finally { + for (const sample of samplesToEncode) { + if (sample !== videoSample) { + sample.close(); + } + } + } + } + + /** Repeats the last frame rate sample to fill the gap up to the given timestamp. */ + private async padFrameRate(until: number, encodeOptions?: VideoEncoderEncodeOptions) { + const frameRate = this.encodingConfig.transform!.frameRate!; + assert(this.frameRateLastSample); + + const frameDifference = Math.round((until - this.frameRateLastTimestamp!) * frameRate); + + for (let i = 1; i < frameDifference; i++) { + const sample = this.frameRateLastSample.clone(); + sample.setTimestamp(this.frameRateLastTimestamp! + i / frameRate); + sample.setDuration(1 / frameRate); + await this.processAndEncode(sample, encodeOptions); + sample.close(); + } + } + private ensureEncoder(videoSample: VideoSample) { this.ensureEncoderPromise = (async () => { const encoderConfig = buildVideoEncoderConfig({ + ...this.encodingConfig, width: videoSample.codedWidth, height: videoSample.codedHeight, squarePixelWidth: videoSample.squarePixelWidth, squarePixelHeight: videoSample.squarePixelHeight, - ...this.encodingConfig, framerate: this.source._connectedTrack?.metadata.frameRate, }); this.encodingConfig.onEncoderConfig?.(encoderConfig); @@ -691,7 +896,19 @@ class VideoEncoderWrapper { } async flushAndClose(forceClose: boolean) { - if (!forceClose) this.checkForEncoderError(); + if (!forceClose) { + this.checkForEncoderError(); + } + + // Final frame rate padding: fill remaining frames up to the last sample's original end timestamp + if (!forceClose && this.frameRateLastSample) { + const frameRate = this.encodingConfig.transform!.frameRate!; + const alignedEnd = floorToDivisor(this.frameRateLastEndTimestamp!, frameRate); + await this.padFrameRate(alignedEnd); + } + + this.frameRateLastSample?.close(); + this.frameRateLastSample = null; if (this.customEncoder) { if (!forceClose) { @@ -718,7 +935,9 @@ class VideoEncoderWrapper { this.splitter?.close(); } - if (!forceClose) this.checkForEncoderError(); + if (!forceClose) { + this.checkForEncoderError(); + } } getQueueSize() { diff --git a/src/misc.ts b/src/misc.ts index 128c469..a715043 100644 --- a/src/misc.ts +++ b/src/misc.ts @@ -443,6 +443,10 @@ export const floorToMultiple = (value: number, multiple: number) => { return Math.floor(value / multiple) * multiple; }; +export const floorToDivisor = (value: number, multiple: number) => { + return Math.floor(value * multiple) / multiple; +}; + export const ilog = (x: number) => { let ret = 0; while (x) { @@ -1198,3 +1202,5 @@ export class EventEmitter> { } } } + +export const ceilToMultipleOfTwo = (value: number) => Math.ceil(value / 2) * 2; diff --git a/src/sample.ts b/src/sample.ts index b403a1a..c60474c 100644 --- a/src/sample.ts +++ b/src/sample.ts @@ -1120,14 +1120,16 @@ export type CropRectangle = { height: number; }; -export const clampCropRectangle = (crop: CropRectangle, outerWidth: number, outerHeight: number) => { - crop.left = Math.min(crop.left, outerWidth); - crop.top = Math.min(crop.top, outerHeight); - crop.width = Math.min(crop.width, outerWidth - crop.left); - crop.height = Math.min(crop.height, outerHeight - crop.top); +export const clampCropRectangle = (crop: CropRectangle, outerWidth: number, outerHeight: number): CropRectangle => { + const left = Math.min(crop.left, outerWidth); + const top = Math.min(crop.top, outerHeight); + const width = Math.min(crop.width, outerWidth - left); + const height = Math.min(crop.height, outerHeight - top); - assert(crop.width >= 0); - assert(crop.height >= 0); + assert(width >= 0); + assert(height >= 0); + + return { left, top, width, height }; }; export const validateCropRectangle = (crop: CropRectangle, prefix: string) => { diff --git a/test/browser/media-sources.test.ts b/test/browser/media-sources.test.ts index cd218ba..29e95aa 100644 --- a/test/browser/media-sources.test.ts +++ b/test/browser/media-sources.test.ts @@ -1,10 +1,16 @@ -import { test } from 'vitest'; +import { expect, test } from 'vitest'; import { Output } from '../../src/output.js'; -import { WebMOutputFormat } from '../../src/output-format.js'; +import { Mp4OutputFormat, WebMOutputFormat } from '../../src/output-format.js'; import { BufferTarget } from '../../src/target.js'; import { VideoSampleSource } from '../../src/media-source.js'; import { VideoSample } from '../../src/sample.js'; import { QUALITY_MEDIUM } from '../../src/encode.js'; +import { Input } from '../../src/input.js'; +import { ALL_FORMATS } from '../../src/input-format.js'; +import { BufferSource } from '../../src/source.js'; +import { VideoSampleSink } from '../../src/media-sink.js'; +import { assert } from '../../src/misc.js'; +import { InputVideoTrack } from '../../src/input-track.js'; test('VideoSampleSource.close() should be idempotent after finalize()', async () => { const output = new Output({ @@ -33,3 +39,479 @@ test('VideoSampleSource.close() should be idempotent after finalize()', async () videoSource.close(); // This previously threw }); + +test('Changing input dimensions throws with deny (default)', async () => { + const output = new Output({ + format: new Mp4OutputFormat(), + target: new BufferTarget(), + }); + + const videoSource = new VideoSampleSource({ + codec: 'vp8', + bitrate: QUALITY_MEDIUM, + }); + + output.addVideoTrack(videoSource); + await output.start(); + + const sample1 = new VideoSample(makeCanvas(100, 100), { timestamp: 0, duration: 1 / 30 }); + await videoSource.add(sample1); + sample1.close(); + + const sample2 = new VideoSample(makeCanvas(200, 150), { timestamp: 1 / 30, duration: 1 / 30 }); + await expect(videoSource.add(sample2)).rejects.toThrow(/Video sample size must remain constant/); + sample2.close(); + + videoSource.close(); +}); + +test('Changing input dimensions with passThrough preserves per-frame dimensions', async () => { + const buffer = await encodeFrames( + { codec: 'vp8', bitrate: QUALITY_MEDIUM, sizeChangeBehavior: 'passThrough' }, + [{ width: 100, height: 100 }, { width: 200, height: 150 }], + ); + + const samples = await readBackSamples(buffer); + expect(samples).toHaveLength(2); + expect(samples[0]).toMatchObject({ codedWidth: 100, codedHeight: 100 }); + expect( + // Either are valid; WebCodecs encoders don't like changing dimensions sometimes + (samples[1]!.codedWidth === 100 && samples[1]!.codedHeight === 100) + || (samples[1]!.codedWidth === 200 && samples[1]!.codedHeight === 150), + ).toBe(true); +}); + +test('Changing input dimensions with fill/contain/cover locks output to first frame dimensions', async () => { + for (const behavior of ['fill', 'contain', 'cover'] as const) { + const buffer = await encodeFrames( + { codec: 'vp8', bitrate: QUALITY_MEDIUM, sizeChangeBehavior: behavior }, + [{ width: 100, height: 100 }, { width: 200, height: 150 }], + ); + + const { input, track } = await readBackTrack(buffer); + expect(track.codedWidth).toBe(100); + expect(track.codedHeight).toBe(100); + input.dispose(); + } +}); + +test('Same-sized frames with width and height set', async () => { + const buffer = await encodeFrames( + { codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { width: 50, height: 80, fit: 'fill' } }, + [{ width: 100, height: 100 }, { width: 100, height: 100 }], + ); + + const { input, track } = await readBackTrack(buffer); + expect(track.codedWidth).toBe(50); + expect(track.codedHeight).toBe(80); + input.dispose(); +}); + +test('Same-sized frames with rotation set to 90', async () => { + const buffer = await encodeFrames( + { codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { rotate: 90 } }, + [{ width: 200, height: 100 }, { width: 200, height: 100 }], + ); + + const { input, track } = await readBackTrack(buffer); + expect(track.codedWidth).toBe(100); + expect(track.codedHeight).toBe(200); + input.dispose(); +}); + +test('Same-sized frames with rotation, width and height', async () => { + const buffer = await encodeFrames( + { codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { rotate: 90, width: 50, height: 80, fit: 'contain' } }, + [{ width: 200, height: 100 }, { width: 200, height: 100 }], + ); + + const { input, track } = await readBackTrack(buffer); + expect(track.codedWidth).toBe(50); + expect(track.codedHeight).toBe(80); + input.dispose(); +}); + +test('Changing dimensions with passThrough and rotation 90', async () => { + const buffer = await encodeFrames( + { codec: 'vp8', bitrate: QUALITY_MEDIUM, sizeChangeBehavior: 'passThrough', transform: { rotate: 90 } }, + [{ width: 200, height: 100 }, { width: 300, height: 150 }], + ); + + const samples = await readBackSamples(buffer); + expect(samples).toHaveLength(2); + expect(samples[0]).toMatchObject({ codedWidth: 100, codedHeight: 200 }); + expect( + // Either are valid; WebCodecs encoders don't like changing dimensions sometimes + (samples[1]!.codedWidth === 150 && samples[1]!.codedHeight === 300) + || (samples[1]!.codedWidth === 100 && samples[1]!.codedHeight === 200), + ).toBe(true); +}); + +test('Changing dimensions with passThrough, width and height set', async () => { + const buffer = await encodeFrames( + { + codec: 'vp8', + bitrate: QUALITY_MEDIUM, + sizeChangeBehavior: 'passThrough', + transform: { + width: 50, + height: 80, + fit: 'fill', + }, + }, + [{ width: 100, height: 100 }, { width: 200, height: 150 }], + ); + + const samples = await readBackSamples(buffer); + expect(samples).toHaveLength(2); + // Both frames should be resized to the fixed width/height + expect(samples[0]).toMatchObject({ codedWidth: 50, codedHeight: 80 }); + expect(samples[1]).toMatchObject({ codedWidth: 50, codedHeight: 80 }); +}); + +test('transform.process identity function', async () => { + const buffer = await encodeFrames( + { codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { process: sample => sample } }, + [{ width: 100, height: 100 }, { width: 100, height: 100 }, { width: 100, height: 100 }], + ); + + const samples = await readBackSamples(buffer); + expect(samples).toHaveLength(3); + for (const sample of samples) { + expect(sample.codedWidth).toBe(100); + expect(sample.codedHeight).toBe(100); + } +}); + +test('transform.process manual resize', async () => { + const buffer = await encodeFrames( + { + codec: 'vp8', + bitrate: QUALITY_MEDIUM, + transform: { + process: (sample) => { + const canvas = new OffscreenCanvas(60, 40); + const ctx = canvas.getContext('2d')!; + sample.draw(ctx, 0, 0, 60, 40); + sample.close(); + + return new VideoSample(canvas, { + timestamp: sample.timestamp, + duration: sample.duration, + }); + }, + }, + }, + [{ width: 100, height: 100 }, { width: 100, height: 100 }], + ); + + const { input, track } = await readBackTrack(buffer); + expect(track.codedWidth).toBe(60); + expect(track.codedHeight).toBe(40); + input.dispose(); +}); + +test('transform.process receives pre-transformed frames', async () => { + const receivedDimensions: { width: number; height: number }[] = []; + + const buffer = await encodeFrames( + { + codec: 'vp8', + bitrate: QUALITY_MEDIUM, + transform: { + width: 50, + height: 80, + fit: 'fill', + process: (sample) => { + receivedDimensions.push({ + width: sample.codedWidth, + height: sample.codedHeight, + }); + return sample; + }, + }, + }, + [{ width: 200, height: 200 }, { width: 200, height: 200 }], + ); + + expect(receivedDimensions).toHaveLength(2); + for (const dim of receivedDimensions) { + expect(dim.width).toBe(50); + expect(dim.height).toBe(80); + } + + const { input, track } = await readBackTrack(buffer); + expect(track.codedWidth).toBe(50); + expect(track.codedHeight).toBe(80); + input.dispose(); +}); + +test('transform.process drops all frames after the first', async () => { + let frameIndex = 0; + + const buffer = await encodeFrames( + { + codec: 'vp8', + bitrate: QUALITY_MEDIUM, + transform: { + process: (sample) => { + if (frameIndex++ > 0) { + return null; + } + return sample; + }, + }, + }, + [{ width: 100, height: 100 }, { width: 100, height: 100 }, { width: 100, height: 100 }], + ); + + const samples = await readBackSamples(buffer); + expect(samples).toHaveLength(1); +}); + +test('transform.process expands every frame into two', async () => { + const buffer = await encodeFrames( + { + codec: 'vp8', + bitrate: QUALITY_MEDIUM, + transform: { + process: (sample) => { + const t = sample.timestamp; + const d = sample.duration; + const clone = sample.clone(); + clone.setTimestamp(2 * t); + clone.setDuration(d); + const clone2 = sample.clone(); + clone2.setTimestamp(2 * t + d); + clone2.setDuration(d); + return [clone, clone2]; + }, + }, + }, + [{ width: 100, height: 100 }, { width: 100, height: 100 }, { width: 100, height: 100 }], + ); + + const samples = await readBackSamples(buffer); + expect(samples).toHaveLength(6); + + const d = 1 / 30; + for (let i = 0; i < 6; i++) { + expect(samples[i]!.timestamp).toBe(i * d); + expect(samples[i]!.duration).toBe(d); + } +}); + +test('transform.frameRate normalizes variable-rate input to fixed rate', async () => { + const buffer = await encodeFrames( + { codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { frameRate: 10 } }, + [ + { width: 100, height: 100, timestamp: 0, duration: 0.15 }, + { width: 100, height: 100, timestamp: 0.15, duration: 0.1 }, + { width: 100, height: 100, timestamp: 0.25, duration: 0.05 }, + ], + ); + + const samples = await readBackSamples(buffer); + expect(samples).toHaveLength(3); + for (let i = 0; i < 3; i++) { + expect(samples[i]!.timestamp).toBeCloseTo(i * 0.1); + expect(samples[i]!.duration).toBeCloseTo(0.1); + } +}); + +test('transform.frameRate pads gaps by repeating last frame', async () => { + const buffer = await encodeFrames( + { codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { frameRate: 10 } }, + [ + { width: 100, height: 100, timestamp: 0, duration: 0.1 }, + { width: 100, height: 100, timestamp: 0.3, duration: 0.1 }, + ], + ); + + const samples = await readBackSamples(buffer); + expect(samples).toHaveLength(4); + for (let i = 0; i < 4; i++) { + expect(samples[i]!.timestamp).toBeCloseTo(i * 0.1); + expect(samples[i]!.duration).toBeCloseTo(0.1); + } +}); + +test('transform.frameRate deduplicates frames in the same slot', async () => { + const buffer = await encodeFrames( + { codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { frameRate: 10 } }, + [ + { width: 100, height: 100, timestamp: 0, duration: 0.03 }, + { width: 100, height: 100, timestamp: 0.03, duration: 0.03 }, + { width: 100, height: 100, timestamp: 0.06, duration: 0.04 }, + { width: 100, height: 100, timestamp: 0.1, duration: 0.1 }, + ], + ); + + const samples = await readBackSamples(buffer); + expect(samples).toHaveLength(2); + expect(samples[0]!.timestamp).toBe(0); + expect(samples[1]!.timestamp).toBeCloseTo(0.1); +}); + +test('transform.frameRate final padding fills remaining duration', async () => { + const buffer = await encodeFrames( + { codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { frameRate: 10 } }, + [ + { width: 100, height: 100, timestamp: 0, duration: 0.5 }, + ], + ); + + const samples = await readBackSamples(buffer); + expect(samples).toHaveLength(5); + for (let i = 0; i < 5; i++) { + expect(samples[i]!.timestamp).toBeCloseTo(i * 0.1); + } +}); + +test('transform.frameRate skipping and padding combined', async () => { + const buffer = await encodeFrames( + { codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { frameRate: 10 } }, + [ + { width: 100, height: 100, timestamp: 0, duration: 0.02 }, + { width: 100, height: 100, timestamp: 0.02, duration: 0.02 }, + { width: 100, height: 100, timestamp: 0.04, duration: 0.02 }, + { width: 100, height: 100, timestamp: 0.3, duration: 0.05 }, + { width: 100, height: 100, timestamp: 0.35, duration: 0.05 }, + ], + ); + + const samples = await readBackSamples(buffer); + expect(samples).toHaveLength(4); + for (let i = 0; i < 4; i++) { + expect(samples[i]!.timestamp).toBeCloseTo(i * 0.1); + expect(samples[i]!.duration).toBeCloseTo(0.1); + } +}); + +test('transform.frameRate works with transform', async () => { + const buffer = await encodeFrames( + { + codec: 'vp8', + bitrate: QUALITY_MEDIUM, + transform: { width: 50, height: 50, fit: 'fill', frameRate: 10 }, + }, + [ + { width: 100, height: 100, timestamp: 0, duration: 0.3 }, + ], + ); + + const samples = await readBackSamples(buffer); + expect(samples).toHaveLength(3); + for (let i = 0; i < 3; i++) { + expect(samples[i]!.timestamp).toBeCloseTo(i * 0.1); + expect(samples[i]!.duration).toBe(0.1); + expect(samples[i]!.codedWidth).toBe(50); + expect(samples[i]!.codedHeight).toBe(50); + } +}); + +test('transform.frameRate works with process', async () => { + const processedTimestamps: number[] = []; + + const buffer = await encodeFrames( + { + codec: 'vp8', + bitrate: QUALITY_MEDIUM, + transform: { + frameRate: 10, + process: (sample) => { + processedTimestamps.push(sample.timestamp); + + const canvas = new OffscreenCanvas(60, 40); + const ctx = canvas.getContext('2d')!; + sample.draw(ctx, 0, 0, 60, 40); + + return new VideoSample(canvas, { + timestamp: sample.timestamp, + duration: sample.duration, + }); + }, + }, + }, + [ + { width: 100, height: 100, timestamp: 0, duration: 0.3 }, + ], + ); + + expect(processedTimestamps).toHaveLength(3); + for (let i = 0; i < 3; i++) { + expect(processedTimestamps[i]).toBeCloseTo(i * 0.1); + } + + const { input, track } = await readBackTrack(buffer); + expect(track.codedWidth).toBe(60); + expect(track.codedHeight).toBe(40); + + input.dispose(); +}); + +const makeCanvas = (width: number, height: number) => { + const canvas = new OffscreenCanvas(width, height); + const ctx = canvas.getContext('2d')!; + ctx.fillStyle = 'red'; + ctx.fillRect(0, 0, width, height); + return canvas; +}; + +const encodeFrames = async ( + encodingConfig: ConstructorParameters[0], + frames: { width: number; height: number; timestamp?: number; duration?: number }[], +) => { + const output = new Output({ + format: new Mp4OutputFormat(), + target: new BufferTarget(), + }); + + const videoSource = new VideoSampleSource(encodingConfig); + output.addVideoTrack(videoSource); + await output.start(); + + for (let i = 0; i < frames.length; i++) { + const f = frames[i]!; + const canvas = makeCanvas(f.width, f.height); + const sample = new VideoSample(canvas, { + timestamp: f.timestamp ?? i / 30, + duration: f.duration ?? 1 / 30, + }); + await videoSource.add(sample); + sample.close(); + } + + await output.finalize(); + return output.target.buffer!; +}; + +const readBackTrack = async (buffer: ArrayBuffer) => { + const input = new Input({ + source: new BufferSource(buffer), + formats: ALL_FORMATS, + }); + + const track = await input.getPrimaryVideoTrack() as InputVideoTrack; + assert(track); + + return { input, track }; +}; + +const readBackSamples = async (buffer: ArrayBuffer) => { + const { input, track } = await readBackTrack(buffer); + const sink = new VideoSampleSink(track); + const samples: { codedWidth: number; codedHeight: number; timestamp: number; duration: number }[] = []; + + for await (using sample of sink.samples()) { + samples.push({ + codedWidth: sample.codedWidth, + codedHeight: sample.codedHeight, + timestamp: sample.timestamp, + duration: sample.duration, + }); + } + + input.dispose(); + return samples; +};