diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 5471ac5..ca6bcba 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -1,9 +1,10 @@ -import { registerDecoder, registerEncoder } from 'mediabunny'; +import { registerDecoder, registerEncoder, registerVideoSampleTransformer } from 'mediabunny'; import * as NodeAv from 'node-av'; import { NodeAvVideoDecoder } from './video-decoder'; import { NodeAvVideoEncoder } from './video-encoder'; import { NodeAvAudioDecoder } from './audio-decoder'; import { NodeAvAudioEncoder } from './audio-encoder'; +import { transformSample } from './video-sample'; const SERVER_LOADED_SYMBOL = Symbol.for('@mediabunny/server loaded'); if ((globalThis as Record)[SERVER_LOADED_SYMBOL]) { @@ -32,4 +33,6 @@ export const registerMediabunnyServer = () => { // Audio registerDecoder(NodeAvAudioDecoder); registerEncoder(NodeAvAudioEncoder); + + registerVideoSampleTransformer(transformSample); }; diff --git a/packages/server/src/video-encoder.ts b/packages/server/src/video-encoder.ts index 2e932c7..abb3fcd 100644 --- a/packages/server/src/video-encoder.ts +++ b/packages/server/src/video-encoder.ts @@ -2,16 +2,12 @@ import { CustomVideoEncoder, MaybePromise, QUALITY_MEDIUM, VideoCodec, VideoSamp import * as NodeAv from 'node-av'; import { CODEC_TO_CODEC_ID, - fromPixelFormat, getHardwareEncoderCodec, - mapColorPrimaries, - mapMatrixCoefficients, - mapTransferCharacteristics, unmapColorPrimaries, unmapMatrixCoefficients, unmapTransferCharacteristics, } from './misc'; -import { NodeAvFrameVideoSampleResource } from './video-sample'; +import { copyVideoSampleToAvFrame, NodeAvFrameVideoSampleResource } from './video-sample'; import { AvcNalUnitType, extractAv1CodecInfoFromPacket, @@ -172,33 +168,8 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder { throw new Error('Cannot encode foreign VideoSample with unknown (null) format.'); } - // Copy frame data from VideoSample to FFmpeg Frame - this.frame.format = fromPixelFormat(videoSample.format); - this.frame.width = videoSample.codedWidth; - this.frame.height = videoSample.codedHeight; - this.frame.pts = BigInt(videoSample.microsecondTimestamp); - this.frame.duration = BigInt(videoSample.microsecondDuration); - this.frame.colorPrimaries = mapColorPrimaries(videoSample.colorSpace.primaries ?? 'unknown') - ?? NodeAv.AVCOL_PRI_UNSPECIFIED; - this.frame.colorSpace = mapMatrixCoefficients(videoSample.colorSpace.matrix ?? 'unknown') - ?? NodeAv.AVCOL_SPC_UNSPECIFIED; - this.frame.colorTrc = mapTransferCharacteristics(videoSample.colorSpace.transfer ?? 'unknown') - ?? NodeAv.AVCOL_TRC_UNSPECIFIED; - this.frame.colorRange = videoSample.colorSpace.fullRange === false - ? NodeAv.AVCOL_RANGE_MPEG - : videoSample.colorSpace.fullRange === true - ? NodeAv.AVCOL_RANGE_JPEG - : NodeAv.AVCOL_RANGE_UNSPECIFIED; - + this.lastBuffer = await copyVideoSampleToAvFrame(videoSample, this.frame, this.lastBuffer); this.frame.keyFrame = options?.keyFrame ? 1 : 0; - - const size = videoSample.allocationSize(); - if (!this.lastBuffer || this.lastBuffer.byteLength !== size) { - this.lastBuffer = Buffer.from({ length: size }); - } - - await videoSample.copyTo(this.lastBuffer); - this.frame.fromBuffer(this.lastBuffer); } let frameToEncode = this.frame; diff --git a/packages/server/src/video-sample.ts b/packages/server/src/video-sample.ts index d1d798f..9d10a47 100644 --- a/packages/server/src/video-sample.ts +++ b/packages/server/src/video-sample.ts @@ -4,13 +4,16 @@ import { VideoSampleColorSpace } from 'mediabunny'; import { VideoSampleResource } from 'mediabunny'; import * as NodeAv from 'node-av'; import { MaybePromise, toUint8Array } from '../../../src/misc'; -import { VideoDataPlane, VideoSample } from '../../../src/sample'; +import { VideoSampleTransformationDescription, VideoDataPlane, VideoSample } from '../../../src/sample'; import { toPixelFormat, unmapColorPrimaries, unmapTransferCharacteristics, unmapMatrixCoefficients, fromPixelFormat, + mapColorPrimaries, + mapMatrixCoefficients, + mapTransferCharacteristics, } from './misc'; import { SetRequired } from 'mediabunny'; import { VideoSampleInit } from 'mediabunny'; @@ -89,7 +92,6 @@ export class NodeAvFrameVideoSampleResource extends VideoSampleResource { async toRgbSample( init: SetRequired, - format: 'RGBA' | 'RGBX' | 'BGRA' | 'BGRX', // Will respect it when somebody complains // eslint-disable-next-line @typescript-eslint/no-unused-vars colorSpace: PredefinedColorSpace, @@ -99,7 +101,7 @@ export class NodeAvFrameVideoSampleResource extends VideoSampleResource { const scaler = new NodeAv.SoftwareScaleContext(); const srcFmt = this.frame.format as NodeAv.AVPixelFormat; - const dstFmt = fromPixelFormat(format); + const dstFmt = fromPixelFormat('RGBA'); scaler.getContext( width, height, srcFmt, @@ -122,9 +124,146 @@ export class NodeAvFrameVideoSampleResource extends VideoSampleResource { scaler.freeContext(); } - dstFrame.format = dstFmt; // FFmpeg messes up RGBA and RGBX dstFrame.sampleAspectRatio = srcFrame.sampleAspectRatio; return new VideoSample(new NodeAvFrameVideoSampleResource(dstFrame), init); } } + +export const copyVideoSampleToAvFrame = async (sample: VideoSample, frame: NodeAv.Frame, lastBuffer: Buffer | null) => { + assert(sample.format !== null); + + frame.format = fromPixelFormat(sample.format); + frame.width = sample.codedWidth; + frame.height = sample.codedHeight; + frame.sampleAspectRatio = new NodeAv.Rational( + sample.pixelAspectRatio.num, + sample.pixelAspectRatio.den, + ); + frame.pts = BigInt(sample.microsecondTimestamp); + frame.duration = BigInt(sample.microsecondDuration); + frame.colorPrimaries = mapColorPrimaries(sample.colorSpace.primaries ?? 'unknown') + ?? NodeAv.AVCOL_PRI_UNSPECIFIED; + frame.colorSpace = mapMatrixCoefficients(sample.colorSpace.matrix ?? 'unknown') + ?? NodeAv.AVCOL_SPC_UNSPECIFIED; + frame.colorTrc = mapTransferCharacteristics(sample.colorSpace.transfer ?? 'unknown') + ?? NodeAv.AVCOL_TRC_UNSPECIFIED; + frame.colorRange = sample.colorSpace.fullRange === false + ? NodeAv.AVCOL_RANGE_MPEG + : sample.colorSpace.fullRange === true + ? NodeAv.AVCOL_RANGE_JPEG + : NodeAv.AVCOL_RANGE_UNSPECIFIED; + + const size = sample.allocationSize(); + if (!lastBuffer || lastBuffer.byteLength !== size) { + lastBuffer = Buffer.from({ length: size }); + } + + await sample.copyTo(lastBuffer); + frame.fromBuffer(lastBuffer); + + return lastBuffer; +}; + +export const transformSample = async ( + sample: VideoSample, + description: VideoSampleTransformationDescription, +): Promise => { + let srcFrame: NodeAv.Frame; + let srcFrameOwned = false; + + if (sample._data instanceof NodeAvFrameVideoSampleResource) { + srcFrame = sample._data.frame; + } else { + if (sample.format === null) { + return null; + } + + srcFrame = new NodeAv.Frame(); + srcFrame.alloc(); + srcFrameOwned = true; + + await copyVideoSampleToAvFrame(sample, srcFrame, null); + } + + // Build the filter chain. Order: square-pixel normalize -> rotate -> crop -> resize-with-fit. + const chain: string[] = []; + + if (sample.squarePixelWidth !== sample.codedWidth || sample.squarePixelHeight !== sample.codedHeight) { + chain.push(`scale=${sample.squarePixelWidth}:${sample.squarePixelHeight}`); + chain.push('setsar=1'); + } + + if (description.rotation === 90) { + chain.push('transpose=1'); + } else if (description.rotation === 180) { + chain.push('transpose=1,transpose=1'); + } else if (description.rotation === 270) { + chain.push('transpose=2'); + } + + chain.push(`crop=${Math.round(description.crop.width)}:${Math.round(description.crop.height)}` + + `:${Math.round(description.crop.left)}:${Math.round(description.crop.top)}`); + + if (description.fit === 'fill') { + chain.push(`scale=${description.width}:${description.height}`); + } else if (description.fit === 'contain') { + chain.push(`scale=${description.width}:${description.height}:force_original_aspect_ratio=decrease`); + chain.push(`pad=${description.width}:${description.height}:(ow-iw)/2:(oh-ih)/2:color=black@0`); + } else if (description.fit === 'cover') { + chain.push(`scale=${description.width}:${description.height}:force_original_aspect_ratio=increase`); + chain.push(`crop=${description.width}:${description.height}`); + } + + chain.push('setsar=1'); + + const graph = new NodeAv.FilterGraph(); + graph.alloc(); + + try { + const srcArgs = `video_size=${srcFrame.width}x${srcFrame.height}` + + `:pix_fmt=${srcFrame.format}` + + `:time_base=1/1000000` + + `:pixel_aspect=${sample.pixelAspectRatio.num}/${sample.pixelAspectRatio.den}`; + + const bufferSrc = graph.createFilter(NodeAv.Filter.getByName('buffer')!, 'src', srcArgs); + const bufferSink = graph.createFilter(NodeAv.Filter.getByName('buffersink')!, 'sink'); + assert(bufferSrc && bufferSink); + + // The naming here looks inverted but matches FFmpeg's parse semantics: from the parsed chain's + // perspective, its inputs are fed by the graph's existing outputs (the buffer src), and its outputs + // feed the graph's existing inputs (the buffer sink). + const outputs = NodeAv.FilterInOut.createList([{ name: 'in', filterCtx: bufferSrc, padIdx: 0 }]); + const inputs = NodeAv.FilterInOut.createList([{ name: 'out', filterCtx: bufferSink, padIdx: 0 }]); + + const parseRet = graph.parsePtr(`[in]${chain.join(',')}[out]`, inputs, outputs); + NodeAv.FFmpegError.throwIfError(parseRet, 'FilterGraph.parsePtr'); + + const configRet = await graph.config(); + NodeAv.FFmpegError.throwIfError(configRet, 'FilterGraph.config'); + + const addRet = await bufferSrc.buffersrcAddFrame(srcFrame); + NodeAv.FFmpegError.throwIfError(addRet, 'buffersrcAddFrame'); + + // Flush - we only ever push a single frame through this graph. + await bufferSrc.buffersrcAddFrame(null); + + const dstFrame = new NodeAv.Frame(); + dstFrame.alloc(); + + const getRet = await bufferSink.buffersinkGetFrame(dstFrame); + NodeAv.FFmpegError.throwIfError(getRet, 'buffersinkGetFrame'); + + return new VideoSample(new NodeAvFrameVideoSampleResource(dstFrame), { + timestamp: sample.timestamp, + duration: sample.duration, + rotation: 0, // baked in by the filter graph + }); + } finally { + graph.free(); + + if (srcFrameOwned) { + srcFrame.free(); + } + } +}; diff --git a/src/conversion.ts b/src/conversion.ts index 8c3182b..36948f2 100644 --- a/src/conversion.ts +++ b/src/conversion.ts @@ -24,7 +24,6 @@ import { Input } from './input'; import { InputAudioTrack, InputTrack, InputVideoTrack } from './input-track'; import { AudioSampleSink, - CanvasSink, EncodedPacketSink, VideoSampleSink, } from './media-sink'; @@ -41,7 +40,6 @@ import { assertNever, ceilToMultipleOfTwo, clamp, - floorToDivisor, isIso639Dash2LanguageCode, MaybePromise, normalizeRotation, @@ -1142,7 +1140,8 @@ export class Conversion { let videoSource: VideoSource; - const totalRotation = normalizeRotation(await track.getRotation() + (trackOptions.rotate ?? 0)); + const innateRotation = await track.getRotation(); + const totalRotation = normalizeRotation(innateRotation + (trackOptions.rotate ?? 0)); let outputTrackRotation = totalRotation; const canUseRotationMetadata = this.output.format.supportsVideoRotationMetadata && (trackOptions.allowRotationMetadata ?? true); @@ -1284,10 +1283,9 @@ export class Conversion { sizeChangeBehavior: trackOptions.fit ?? 'passThrough', alpha, hardwareAcceleration: trackOptions.hardwareAcceleration, + transform: {}, }; - - const source = new VideoSampleSource(encodingConfig); - videoSource = source; + assert(encodingConfig.transform); let needsRerender = width !== originalWidth || height !== originalHeight @@ -1334,179 +1332,46 @@ export class Conversion { } } + if (trackOptions.frameRate) { + encodingConfig.transform.frameRate = trackOptions.frameRate; + } + if (needsRerender) { outputTrackRotation = 0; // Since the rotation is baked into the output - this._trackPromises.push((async () => { - await this._started; - - const sink = new CanvasSink(track, { - width, - height, - fit: trackOptions.fit ?? 'fill', - rotation: totalRotation, // Bake the rotation into the output - crop: trackOptions.crop, - poolSize: 1, - alpha: alpha === 'keep', - }); - const iterator = sink.canvases(this._startTimestamp, this._endTimestamp); - const frameRate = trackOptions.frameRate; - - let lastCanvas: HTMLCanvasElement | OffscreenCanvas | null = null; - let lastCanvasTimestamp: number | null = null; - let lastCanvasEndTimestamp: number | null = null; - - /** Repeats the last sample to pad out the time until the specified timestamp. */ - const padFrames = async (until: number) => { - assert(lastCanvas); - assert(frameRate !== undefined); - - const frameDifference = Math.round((until - lastCanvasTimestamp!) * frameRate); - - for (let i = 1; i < frameDifference; i++) { - const sample = new VideoSample(lastCanvas, { - timestamp: lastCanvasTimestamp! + i / frameRate, - duration: 1 / frameRate, - }); - await this._registerVideoSample(trackOptions, outputTrackId, source, sample); - sample.close(); - } - }; - - for await (const { canvas, timestamp, duration } of iterator) { - if (this._canceled) { - return; - } - - let adjustedSampleTimestamp = Math.max(timestamp - this._startTimestamp, 0); - lastCanvasEndTimestamp = adjustedSampleTimestamp + duration; - - if (frameRate !== undefined) { - // Logic for skipping/repeating frames when a frame rate is set - const alignedTimestamp = floorToDivisor(adjustedSampleTimestamp, frameRate); - - if (lastCanvas !== null) { - if (alignedTimestamp <= lastCanvasTimestamp!) { - lastCanvas = canvas; - lastCanvasTimestamp = alignedTimestamp; - - // Skip this sample, since we already added one for this frame - continue; - } else { - // Check if we may need to repeat the previous frame - await padFrames(alignedTimestamp); - } - } - - adjustedSampleTimestamp = alignedTimestamp; - } - - const sample = new VideoSample(canvas, { - timestamp: adjustedSampleTimestamp, - duration: frameRate !== undefined ? 1 / frameRate : duration, - }); - await this._registerVideoSample(trackOptions, outputTrackId, source, sample); - sample.close(); - - if (frameRate !== undefined) { - lastCanvas = canvas; - lastCanvasTimestamp = adjustedSampleTimestamp; - } - } - - if (lastCanvas) { - assert(lastCanvasEndTimestamp !== null); - assert(frameRate !== undefined); - - // If necessary, pad until the end timestamp of the last sample - await padFrames(floorToDivisor(lastCanvasEndTimestamp, frameRate)); - } - - source.close(); - this._synchronizer.closeTrack(outputTrackId); - })()); - } else { - this._trackPromises.push((async () => { - await this._started; - - const sink = new VideoSampleSink(track); - const frameRate = trackOptions.frameRate; - - let lastSample: VideoSample | null = null; - let lastSampleTimestamp: number | null = null; - let lastSampleEndTimestamp: number | null = null; - - /** Repeats the last sample to pad out the time until the specified timestamp. */ - const padFrames = async (until: number) => { - assert(lastSample); - assert(frameRate !== undefined); - - const frameDifference = Math.round((until - lastSampleTimestamp!) * frameRate); - - for (let i = 1; i < frameDifference; i++) { - lastSample.setTimestamp(lastSampleTimestamp! + i / frameRate); - lastSample.setDuration(1 / frameRate); - await this._registerVideoSample(trackOptions, outputTrackId, source, lastSample); - } - - lastSample.close(); - }; - - for await (const sample of sink.samples(this._startTimestamp, this._endTimestamp)) { - if (this._canceled) { - sample.close(); - lastSample?.close(); - return; - } - - let adjustedSampleTimestamp = Math.max(sample.timestamp - this._startTimestamp, 0); - lastSampleEndTimestamp = adjustedSampleTimestamp + sample.duration; - - if (frameRate !== undefined) { - // Logic for skipping/repeating frames when a frame rate is set - const alignedTimestamp = floorToDivisor(adjustedSampleTimestamp, frameRate); - - if (lastSample !== null) { - if (alignedTimestamp <= lastSampleTimestamp!) { - lastSample.close(); - lastSample = sample; - lastSampleTimestamp = alignedTimestamp; - - // Skip this sample, since we already added one for this frame - continue; - } else { - // Check if we may need to repeat the previous frame - await padFrames(alignedTimestamp); - } - } - - adjustedSampleTimestamp = alignedTimestamp; - sample.setDuration(1 / frameRate); - } - - sample.setTimestamp(adjustedSampleTimestamp); - await this._registerVideoSample(trackOptions, outputTrackId, source, sample); - - if (frameRate !== undefined) { - lastSample = sample; - lastSampleTimestamp = adjustedSampleTimestamp; - } else { - sample.close(); - } - } - - if (lastSample) { - assert(lastSampleEndTimestamp !== null); - assert(frameRate !== undefined); - - // If necessary, pad until the end timestamp of the last sample - await padFrames(floorToDivisor(lastSampleEndTimestamp, frameRate)); - } - - source.close(); - this._synchronizer.closeTrack(outputTrackId); - })()); + encodingConfig.transform.width = width; + encodingConfig.transform.height = height; + encodingConfig.transform.fit = trackOptions.fit ?? 'fill'; + encodingConfig.transform.rotate = normalizeRotation(totalRotation - innateRotation); + encodingConfig.transform.crop = crop; + encodingConfig.transform.alpha = alpha; } + + const source = new VideoSampleSource(encodingConfig); + videoSource = source; + + this._trackPromises.push((async () => { + await this._started; + + const sink = new VideoSampleSink(track); + + for await (const sample of sink.samples(this._startTimestamp, this._endTimestamp)) { + if (this._canceled) { + sample.close(); + return; + } + + const adjustedSampleTimestamp = Math.max(sample.timestamp - this._startTimestamp, 0); + sample.setTimestamp(adjustedSampleTimestamp); + + await this._registerVideoSample(trackOptions, outputTrackId, source, sample); + + sample.close(); + } + + source.close(); + this._synchronizer.closeTrack(outputTrackId); + })()); } let ownGroup: OutputTrackGroup | null = null; diff --git a/src/encode.ts b/src/encode.ts index e5be0b8..ec0a0cd 100644 --- a/src/encode.ts +++ b/src/encode.ts @@ -113,6 +113,10 @@ export type VideoTransformOptions = { * clamped to the dimensions of the frame. Cropping is performed after rotation but before resizing. */ crop?: CropRectangle; + /** + * Whether to discard or keep the transparency information of the video samples. The default is `'keep'`. + */ + alpha?: 'keep' | 'discard'; /** * The frame rate in hertz to normalize the video frame stream to. */ @@ -193,10 +197,12 @@ export const validateVideoEncodingConfig = (config: VideoEncodingConfig) => { if ( config.transform.fit !== undefined && ['fill', 'contain', 'cover'].includes(config.sizeChangeBehavior!) + && config.transform.fit !== config.sizeChangeBehavior ) { throw new TypeError( - 'config.transform.fit cannot be used when config.sizeChangeBehavior is \'fill\', \'contain\' or' - + ' \'cover\', as sizeChangeBehavior already determines the fitting algorithm.', + 'config.transform.fit, when provided, cannot differ from config.sizeChangeBehavior when' + + ' config.sizeChangeBehavior is \'fill\', \'contain\' or \'cover\', as sizeChangeBehavior already' + + ' determines the fitting algorithm.', ); } if (config.transform.rotate !== undefined && ![0, 90, 180, 270].includes(config.transform.rotate)) { diff --git a/src/index.ts b/src/index.ts index f3f0146..4e5c9f0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -249,8 +249,12 @@ export { type VideoSamplePixelFormat, VideoSampleColorSpace, VideoSampleResource, + type VideoSampleTransformOptions, + type VideoSampleTransformationDescription, type CropRectangle, VIDEO_SAMPLE_PIXEL_FORMATS, + type VideoDataPlane, + registerVideoSampleTransformer, } from './sample'; export { AudioBufferSink, diff --git a/src/media-source.ts b/src/media-source.ts index 34a054a..1b164cf 100644 --- a/src/media-source.ts +++ b/src/media-source.ts @@ -25,13 +25,10 @@ import { assertNever, binarySearchLessOrEqual, CallSerializer, - ceilToMultipleOfTwo, clamp, clearIntervalUnthrottled, floorToDivisor, - isFirefox, last, - normalizeRotation, promiseWithResolvers, roundToDivisor, setInt24, @@ -53,7 +50,6 @@ import { EncodedPacket, EncodedPacketSideData } from './packet'; import { AudioSample, audioSampleToInterleavedFormat, - clampCropRectangle, toInterleavedAudioFormat, VideoSample, } from './sample'; @@ -233,7 +229,6 @@ class VideoEncoderWrapper { private muxer: Muxer | null = null; private lastMultipleOfKeyFrameInterval = -1; - private resizeCanvas: HTMLCanvasElement | OffscreenCanvas | null = null; // Tracks the input dimensions of the first frame private codedWidth: number | null = null; private codedHeight: number | null = null; @@ -275,15 +270,7 @@ class VideoEncoderWrapper { private lastMuxerPromise: Promise = Promise.resolve(); - constructor(private source: VideoSource, private encodingConfig: VideoEncodingConfig) { - const sizeChangeBehavior = encodingConfig.sizeChangeBehavior ?? 'deny'; - if (['fill', 'contain', 'cover'].includes(sizeChangeBehavior) && encodingConfig.transform?.fit !== undefined) { - throw new TypeError( - `Cannot set 'fit' when 'sizeChangeBehavior' is '${sizeChangeBehavior}'. ` - + `The size change behavior determines the fit in this case.`, - ); - } - } + constructor(private source: VideoSource, private encodingConfig: VideoEncodingConfig) {} async add(videoSample: VideoSample, shouldClose: boolean, encodeOptions?: VideoEncoderEncodeOptions) { const originalSample = videoSample; @@ -323,23 +310,8 @@ class VideoEncoderWrapper { const needsTransform = hasTransformConfig || (isSizeChange && sizeChangeBehavior !== 'passThrough'); if (needsTransform) { - const rotation = normalizeRotation(videoSample.rotation + (config.transform?.rotate ?? 0)); - const [rotatedWidth, rotatedHeight] = rotation % 180 === 0 - ? [videoSample.codedWidth, videoSample.codedHeight] - : [videoSample.codedHeight, videoSample.codedWidth]; - - // Clamp crop rectangle to the rotated video dimensions - let finalCrop = config.transform?.crop; - if (finalCrop) { - finalCrop = clampCropRectangle(finalCrop, rotatedWidth, rotatedHeight); - } - - const cropWidth = finalCrop ? finalCrop.width : rotatedWidth; - const cropHeight = finalCrop ? finalCrop.height : rotatedHeight; - const originalAspectRatio = cropWidth / cropHeight; - - let targetWidth: number; - let targetHeight: number; + let targetWidth = config.transform?.width; + let targetHeight = config.transform?.height; let appliedFit: 'fill' | 'contain' | 'cover' = config.transform?.fit ?? 'fill'; // If the size changed and behavior is fill/contain/cover, lock to the original output dimensions @@ -351,81 +323,29 @@ class VideoEncoderWrapper { targetWidth = this.outputWidth!; targetHeight = this.outputHeight!; appliedFit = sizeChangeBehavior; - } else { - // Otherwise, dynamically calculate the target dimensions based on config and aspect ratio - if (config.transform?.width !== undefined && config.transform?.height === undefined) { - targetWidth = config.transform.width; - targetHeight = ceilToMultipleOfTwo(Math.round(targetWidth / originalAspectRatio)); - } else if (config.transform?.width === undefined && config.transform?.height !== undefined) { - targetHeight = config.transform.height; - targetWidth = ceilToMultipleOfTwo(Math.round(targetHeight * originalAspectRatio)); - } else if (config.transform?.width !== undefined && config.transform?.height !== undefined) { - targetWidth = config.transform?.width; - targetHeight = config.transform?.height; - } else { - targetWidth = cropWidth; - targetHeight = cropHeight; - } } + const transformed = await videoSample.transform({ + width: targetWidth, + height: targetHeight, + roundDimensionsTo: 2, + crop: config.transform?.crop, + rotate: config.transform?.rotate, + fit: appliedFit, + alpha: config.alpha, + }); + // Save the output dimensions of the first frame if (this.outputWidth === null || this.outputHeight === null) { - this.outputWidth = targetWidth; - this.outputHeight = targetHeight; + this.outputWidth = transformed.displayWidth; + this.outputHeight = transformed.displayHeight; } - let canvasIsNew = false; - - if (!this.resizeCanvas) { - if (typeof document !== 'undefined') { - // Prefer an HTMLCanvasElement - this.resizeCanvas = document.createElement('canvas'); - this.resizeCanvas.width = targetWidth; - this.resizeCanvas.height = targetHeight; - } else { - this.resizeCanvas = new OffscreenCanvas(targetWidth, targetHeight); - } - canvasIsNew = true; - } else if (this.resizeCanvas.width !== targetWidth || this.resizeCanvas.height !== targetHeight) { - // Dynamically resize the canvas if the target dimensions have changed - this.resizeCanvas.width = targetWidth; - this.resizeCanvas.height = targetHeight; - } - - const context = this.resizeCanvas.getContext('2d', { - // Firefox has VideoFrame glitches with opaque canvases - alpha: this.encodingConfig.alpha === 'keep' || isFirefox(), - }) as CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D; - assert(context); - - if (typeof context.resetTransform === 'function') { - context.resetTransform(); - } - - if (!canvasIsNew) { - if (isFirefox()) { - context.fillStyle = 'black'; - context.fillRect(0, 0, targetWidth, targetHeight); - } else { - context.clearRect(0, 0, targetWidth, targetHeight); - } - } - - videoSample.drawWithFit(context, { - fit: appliedFit, - rotation: rotation, - crop: finalCrop, - }); - if (shouldClose) { videoSample.close(); } - videoSample = new VideoSample(this.resizeCanvas, { - timestamp: videoSample.timestamp, - duration: videoSample.duration, - rotation: 0, // Rotation is now baked into the canvas - }); + videoSample = transformed; shouldClose = true; } else { // If no canvas is needed, we still need to record the output dimensions for the first frame diff --git a/src/sample.ts b/src/sample.ts index 9824d7e..e61ffdd 100644 --- a/src/sample.ts +++ b/src/sample.ts @@ -26,6 +26,9 @@ import { simplifyRational, Rectangle, validateRectangle, + normalizeRotation, + roundToMultiple, + arrayArgmin, MaybePromise, } from './misc'; @@ -103,8 +106,10 @@ export abstract class VideoSampleResource { /** Returns the height of the frame in pixels. */ abstract getCodedHeight(): number; + /** Returns the width of the frame in square pixels, respecting pixel aspect ratio. */ abstract getSquarePixelWidth(): number; + /** Returns the height of the frame in square pixels, respecting pixel aspect ratio. */ abstract getSquarePixelHeight(): number; /** Returns the color space of the frame. */ @@ -116,17 +121,32 @@ export abstract class VideoSampleResource { */ abstract close(): void; + /** + * Returns the data planes that hold the video data for this sample. The returned planes and data must be in the + * format returned by `getFormat()`. + */ abstract getDataPlanes(): MaybePromise; + /** + * Returns a new RGB {@link VideoSample} that contains the same content as this sample. The provided `init` object + * must be used to set the metadata of this new video sample. When converting from a non-RGB format to RGB, the + * conversion must respect `colorSpace`. + */ abstract toRgbSample( init: SetRequired, - format: 'RGBA' | 'RGBX' | 'BGRA' | 'BGRX', colorSpace: PredefinedColorSpace, ): MaybePromise; } +/** + * Describes a single data plane of a video frame. + * @group Samples + * @public + */ export type VideoDataPlane = { + /** The data of the plane. */ data: Uint8Array; + /** The stride of the plane, in bytes. This is the distance in bytes between the start of each row of pixels. */ stride: number; }; @@ -239,9 +259,9 @@ export class VideoSample implements Disposable { readonly format!: VideoSamplePixelFormat | null; /** The visible region of the frame in the coded pixel grid. */ readonly visibleRect!: Rectangle; - /** The width of the frame in square pixels, before rotation is applied. */ + /** The width of the frame in square pixels (respecting pixel aspect ratio), before rotation is applied. */ readonly squarePixelWidth!: number; - /** The height of the frame in square pixels, before rotation is applied. */ + /** The height of the frame in square pixels (respecting pixel aspect ratio), before rotation is applied. */ readonly squarePixelHeight!: number; /** The rotation of the frame in degrees, clockwise. */ readonly rotation!: Rotation; @@ -754,7 +774,6 @@ export class VideoSample implements Disposable { duration: this.duration, rotation: this.rotation, }, - options.format as 'RGBA' | 'RGBX' | 'BGRA' | 'BGRX', options.colorSpace ?? 'srgb', ); if (!(rgbSample instanceof VideoSample)) { @@ -1333,6 +1352,179 @@ export class VideoSample implements Disposable { } } + /** + * Transform this video sample to a new video sample given the options. Can be used to resize, rotate, and crop + * the sample. + * + * In non-browser environments, this method will not work by default. To make it work, register a custom + * transformer function via {@link registerVideoSampleTransformer}. + */ + async transform(options: VideoSampleTransformOptions) { + if (!options || typeof options !== 'object') { + throw new TypeError('options must be an object.'); + } + if (options.width !== undefined && (!Number.isInteger(options.width) || options.width <= 0)) { + throw new TypeError('options.width, when provided, must be a positive integer.'); + } + if (options.height !== undefined && (!Number.isInteger(options.height) || options.height <= 0)) { + throw new TypeError('options.height, when provided, must be a positive integer.'); + } + if ( + options.roundDimensionsTo !== undefined + && (!Number.isInteger(options.roundDimensionsTo) || options.roundDimensionsTo <= 0) + ) { + throw new TypeError('options.roundDimensionsTo, when provided, must be a positive integer.'); + } + if (options.fit !== undefined && !['fill', 'contain', 'cover'].includes(options.fit)) { + throw new TypeError('options.fit, when provided, must be one of "fill", "contain", or "cover".'); + } + if ( + options.width !== undefined + && options.height !== undefined + && options.fit === undefined + ) { + throw new TypeError( + 'When both options.width and options.height are provided, options.fit must also be provided.', + ); + } + if (options.rotate !== undefined && ![0, 90, 180, 270].includes(options.rotate)) { + throw new TypeError('options.rotate, when provided, must be 0, 90, 180 or 270.'); + } + if (options.crop !== undefined) { + validateCropRectangle(options.crop, 'options.'); + } + if (options.alpha !== undefined && !['keep', 'discard'].includes(options.alpha)) { + throw new TypeError('options.alpha, when provided, must be \'keep\' or \'discard\'.'); + } + + const rotation = normalizeRotation(this.rotation + (options.rotate ?? 0)); + const [rotatedWidth, rotatedHeight] = rotation % 180 === 0 + ? [this.squarePixelWidth, this.squarePixelHeight] + : [this.squarePixelHeight, this.squarePixelWidth]; + + // Clamp crop rectangle to the rotated video dimensions + let finalCrop = options.crop; + if (finalCrop) { + finalCrop = clampCropRectangle(finalCrop, rotatedWidth, rotatedHeight); + } + + const cropWidth = finalCrop ? finalCrop.width : rotatedWidth; + const cropHeight = finalCrop ? finalCrop.height : rotatedHeight; + const originalAspectRatio = cropWidth / cropHeight; + + let targetWidth: number; + let targetHeight: number; + + if (options.width !== undefined && options.height === undefined) { + targetWidth = options.width; + targetHeight = targetWidth / originalAspectRatio; + } else if (options.width === undefined && options.height !== undefined) { + targetHeight = options.height; + targetWidth = targetHeight * originalAspectRatio; + } else if (options.width !== undefined && options.height !== undefined) { + targetWidth = options.width; + targetHeight = options.height; + } else { + targetWidth = cropWidth; + targetHeight = cropHeight; + } + + targetWidth = roundToMultiple(targetWidth, options.roundDimensionsTo ?? 1); + targetHeight = roundToMultiple(targetHeight, options.roundDimensionsTo ?? 1); + + const description: VideoSampleTransformationDescription = { + width: targetWidth, + height: targetHeight, + fit: options.fit ?? 'fill', + rotation, + crop: finalCrop ?? { + left: 0, + top: 0, + width: rotatedWidth, + height: rotatedHeight, + }, + alpha: options.alpha ?? 'keep', + }; + + // Description's finalized; let's see if a registered transformer wants to handle it + for (const transformer of registeredVideoSampleTransformers) { + let result = transformer(this, description); + if (result instanceof Promise) result = await result; + + if (result !== null) { + return result; + } + } + + // We need to handle it ourselves, and we use canvases to do it + + let canvas: HTMLCanvasElement | OffscreenCanvas | null = null; + let canvasIsNew = false; + + for (const entry of transformationCanvasCache) { + if (entry.canvas.width === description.width && entry.canvas.height === description.height) { + canvas = entry.canvas; + entry.age = transformationCanvasCacheNextAge++; + + break; + } + } + + if (canvas === null) { + if (typeof OffscreenCanvas !== 'undefined') { + canvas = new OffscreenCanvas(description.width, description.height); + } else { + if (typeof window === 'undefined' || typeof document === 'undefined') { + throw new Error( + 'Cannot transform VideoSamples in this environment. Either run in an environment with' + + ' OffscreenCanvas or HTMLCanvasElement, or supply a custom VideoSample transformer using' + + ' registerVideoSampleTransformer().', + ); + } + + canvas = document.createElement('canvas'); + canvas.width = description.width; + canvas.height = description.height; + } + + canvasIsNew = true; + + if (transformationCanvasCache.length >= TRANSFORMATION_CANVAS_CACHE_MAX_SIZE) { + transformationCanvasCache.splice(arrayArgmin(transformationCanvasCache, x => x.age), 1); + } + + transformationCanvasCache.push({ + canvas, + age: transformationCanvasCacheNextAge++, + }); + } + + const context = canvas.getContext('2d', { + alpha: true, + }) as CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D; + assert(context); + + if (description.alpha === 'discard') { + context.fillStyle = 'black'; + context.fillRect(0, 0, description.width, description.height); + } else if (!canvasIsNew) { + // Cached canvases carry stale pixels from a prior draw + context.clearRect(0, 0, description.width, description.height); + } + + this.drawWithFit(context, { + fit: description.fit, + rotation: description.rotation, + crop: description.crop, + }); + + return new VideoSample(canvas, { + timestamp: this.timestamp, + duration: this.duration, + rotation: 0, // Any previous rotation is now baked in + }); + } + /** Sets the rotation metadata of this video sample. */ setRotation(newRotation: Rotation) { if (![0, 90, 180, 270].includes(newRotation)) { @@ -1369,6 +1561,123 @@ export class VideoSample implements Disposable { } } +/** + * Options for transforming a {@link VideoSample}. The order of operations are: + * + * 1. Pixel aspect ratio normalization (always applied) + * 2. Rotation + * 3. Crop + * 4. Resize using fit + * @group Samples + * @public + */ +export type VideoSampleTransformOptions = { + /** + * The width in pixels to resize the frames to. If height is not set, it will be deduced + * automatically based on aspect ratio. + */ + width?: number; + /** + * The height in pixels to resize the frames to. If width is not set, it will be deduced + * automatically based on aspect ratio. + */ + height?: number; + /** + * A positive integer. When provided, both the width and height will be rounded to the nearest multiple of + * this number. + */ + roundDimensionsTo?: number; + /** + * The fitting algorithm in case both width and height are set. + * + * - `'fill'` will stretch the image to fill the entire box, potentially altering aspect ratio. + * - `'contain'` will contain the entire image within the box while preserving aspect ratio. This may lead to + * letterboxing. + * - `'cover'` will scale the image until the entire box is filled, while preserving aspect ratio. + */ + fit?: 'fill' | 'contain' | 'cover'; + /** + * The clockwise rotation by which to rotate the frames. Rotation is applied before resizing. + */ + rotate?: Rotation; + /** + * Specifies the rectangular region of the frames to crop to. The crop region will automatically be + * clamped to the dimensions of the frame. Cropping is performed after rotation but before resizing. + */ + crop?: CropRectangle; + /** + * Whether to discard or keep the transparency information of the video sample. The default is `'keep'`. + */ + alpha?: 'keep' | 'discard'; +}; + +/** + * A fully-resolved description of a video sample transformation, with all defaults and constraints baked in. + * + * The order of operations must be: + * 1. Pixel aspect ratio normalization (always applied) + * 2. Rotation + * 3. Crop + * 4. Resize using fit + * @group Samples + * @public + */ +export type VideoSampleTransformationDescription = { + /** The width in pixels to resize the frames to. */ + width: number; + /** The height in pixels to resize the frames to. */ + height: number; + /** + * The fitting algorithm. + * + * - `'fill'` will stretch the image to fill the entire box, potentially altering aspect ratio. + * - `'contain'` will contain the entire image within the box while preserving aspect ratio. This may lead to + * letterboxing. + * - `'cover'` will scale the image until the entire box is filled, while preserving aspect ratio. + */ + fit: 'fill' | 'contain' | 'cover'; + /** The clockwise rotation by which to rotate the frames. Rotation is applied before resizing. */ + rotation: Rotation; + /** + * The rectangular region of the frames to crop to, clamped to the dimensions of the frame. Cropping is + * performed after rotation but before resizing. + */ + crop: CropRectangle; + /** Whether to discard or keep the transparency information of the video sample. */ + alpha: 'keep' | 'discard'; +}; + +const registeredVideoSampleTransformers: (( + sample: VideoSample, + description: VideoSampleTransformationDescription, +) => MaybePromise)[] = []; + +/** + * Registers a callback to handle the transformation of {@link VideoSample} instances. The callback can either return + * the transformed sample, or `null` to indicate that it doesn't want to handle the given transformation task. + * @group Samples + * @public + */ +export const registerVideoSampleTransformer = ( + transformer: ( + sample: VideoSample, + description: VideoSampleTransformationDescription, + ) => MaybePromise, +) => { + if (registeredVideoSampleTransformers.includes(transformer)) { + return; // Already in there + } + + registeredVideoSampleTransformers.push(transformer); +}; + +const TRANSFORMATION_CANVAS_CACHE_MAX_SIZE = 3; +const transformationCanvasCache: { + canvas: HTMLCanvasElement | OffscreenCanvas; + age: number; +}[] = []; +let transformationCanvasCacheNextAge = 0; + /** * Describes the color space of a {@link VideoSample}. Corresponds to the WebCodecs API's VideoColorSpace. * @group Samples @@ -1927,6 +2236,10 @@ export abstract class AudioSampleResource { */ abstract close(): void; + /** + * Returns the audio sample data for the plane given by `planeIndex`. The audio data must be in the format returned + * by `getFormat()`. For interleaved formats, there is only one plane. + */ abstract getDataPlane(planeIndex: number): Uint8Array; } diff --git a/test/browser/custom-sample-resources.test.ts b/test/browser/custom-sample-resources.test.ts index 3d3b09a..b3f28c7 100644 --- a/test/browser/custom-sample-resources.test.ts +++ b/test/browser/custom-sample-resources.test.ts @@ -61,8 +61,6 @@ class ImageVideoSampleResource extends VideoSampleResource { toRgbSample( init: SetRequired, // eslint-disable-next-line @typescript-eslint/no-unused-vars - format: 'RGBA' | 'RGBX' | 'BGRA' | 'BGRX', - // eslint-disable-next-line @typescript-eslint/no-unused-vars colorSpace: PredefinedColorSpace, ): MaybePromise { return new VideoSample(this, init); diff --git a/test/node/server-extension.test.ts b/test/node/server-extension.test.ts index 5df9892..6e1a526 100644 --- a/test/node/server-extension.test.ts +++ b/test/node/server-extension.test.ts @@ -736,6 +736,181 @@ describe('Video', async () => { { offset: 0, stride: 400 * 4 }, ]); }); + + describe('VideoSample transformation', () => { + // 400x400 image: red everywhere, with a 200x200 blue square filling the bottom-left quadrant. + const TEST_IMAGE = (() => { + const buf = new Uint8Array(400 * 400 * 4); + for (let i = 0; i < 400 * 400; i++) { + buf[i * 4] = 255; + buf[i * 4 + 1] = 0; + buf[i * 4 + 2] = 0; + buf[i * 4 + 3] = 255; + } + for (let y = 200; y < 400; y++) { + for (let x = 0; x < 200; x++) { + const i = y * 400 + x; + buf[i * 4] = 0; + buf[i * 4 + 1] = 0; + buf[i * 4 + 2] = 255; + buf[i * 4 + 3] = 255; + } + } + return buf; + })(); + + const makeSample = () => new VideoSample(TEST_IMAGE, { + format: 'RGBA', + codedWidth: 400, + codedHeight: 400, + timestamp: 0, + }); + + const readRgba = async (sample: VideoSample) => { + const buffer = new Uint8Array(sample.codedWidth * sample.codedHeight * 4); + await sample.copyTo(buffer, { format: 'RGBA' }); + return buffer; + }; + + const getColorAt = (rgba: Uint8Array, width: number, x: number, y: number) => { + const i = (y * width + x) * 4; + return [rgba[i]!, rgba[i + 1]!, rgba[i + 2]!]; + }; + + const expectColor = (actual: number[], expected: number[], tolerance = 4) => { + for (let i = 0; i < 3; i++) { + expect( + Math.abs(actual[i]! - expected[i]!), + `channel ${i}: got ${actual[i]}, expected ${expected[i]} (+/-${tolerance})`, + ).toBeLessThanOrEqual(tolerance); + } + }; + + test('resize to 200x200', async () => { + using sample = makeSample(); + using result = await sample.transform({ width: 200, height: 200, fit: 'fill' }); + + expect(result.codedWidth).toBe(200); + expect(result.codedHeight).toBe(200); + + const rgba = await readRgba(result); + expectColor(getColorAt(rgba, 200, 50, 50), [255, 0, 0]); // top-left + expectColor(getColorAt(rgba, 200, 150, 50), [255, 0, 0]); // top-right + expectColor(getColorAt(rgba, 200, 50, 150), [0, 0, 255]); // bottom-left (blue) + expectColor(getColorAt(rgba, 200, 150, 150), [255, 0, 0]); // bottom-right + }); + + test('rotate 90 deg clockwise', async () => { + using sample = makeSample(); + using result = await sample.transform({ rotate: 90 }); + + expect(result.codedWidth).toBe(400); + expect(result.codedHeight).toBe(400); + + const rgba = await readRgba(result); + expectColor(getColorAt(rgba, 400, 50, 50), [0, 0, 255]); // top-left (blue) + expectColor(getColorAt(rgba, 400, 350, 50), [255, 0, 0]); + expectColor(getColorAt(rgba, 400, 50, 350), [255, 0, 0]); + expectColor(getColorAt(rgba, 400, 350, 350), [255, 0, 0]); + }); + + test('crop top-left, no rotation', async () => { + using sample = makeSample(); + using result = await sample.transform({ crop: { left: 0, top: 0, width: 200, height: 200 } }); + + expect(result.codedWidth).toBe(200); + expect(result.codedHeight).toBe(200); + + const rgba = await readRgba(result); + expectColor(getColorAt(rgba, 200, 50, 50), [255, 0, 0]); + expectColor(getColorAt(rgba, 200, 150, 50), [255, 0, 0]); + expectColor(getColorAt(rgba, 200, 50, 150), [255, 0, 0]); + expectColor(getColorAt(rgba, 200, 150, 150), [255, 0, 0]); + }); + + test('rotate 90 deg then crop top-left, crop applies after rotation', async () => { + using sample = makeSample(); + using result = await sample.transform({ + rotate: 90, + crop: { left: 0, top: 0, width: 200, height: 200 }, + }); + + expect(result.codedWidth).toBe(200); + expect(result.codedHeight).toBe(200); + + const rgba = await readRgba(result); + expectColor(getColorAt(rgba, 200, 50, 50), [0, 0, 255]); + expectColor(getColorAt(rgba, 200, 150, 50), [0, 0, 255]); + expectColor(getColorAt(rgba, 200, 50, 150), [0, 0, 255]); + expectColor(getColorAt(rgba, 200, 150, 150), [0, 0, 255]); + }); + + test('resize to 400x200 with fill, vertically squished', async () => { + using sample = makeSample(); + using result = await sample.transform({ width: 400, height: 200, fit: 'fill' }); + + expect(result.codedWidth).toBe(400); + expect(result.codedHeight).toBe(200); + + const rgba = await readRgba(result); + expectColor(getColorAt(rgba, 400, 50, 50), [255, 0, 0]); // top-left + expectColor(getColorAt(rgba, 400, 300, 50), [255, 0, 0]); // top-right + expectColor(getColorAt(rgba, 400, 50, 175), [0, 0, 255]); // bottom-left (blue) + expectColor(getColorAt(rgba, 400, 300, 175), [255, 0, 0]); // bottom-right + }); + + test('resize to 400x200 with contain, letterboxed', async () => { + using sample = makeSample(); + using result = await sample.transform({ width: 400, height: 200, fit: 'contain' }); + + expect(result.codedWidth).toBe(400); + expect(result.codedHeight).toBe(200); + + const rgba = await readRgba(result); + expectColor(getColorAt(rgba, 400, 50, 100), [0, 0, 0]); // left letterbox + expectColor(getColorAt(rgba, 400, 350, 100), [0, 0, 0]); // right letterbox + expectColor(getColorAt(rgba, 400, 150, 50), [255, 0, 0]); // top-left of image (red) + expectColor(getColorAt(rgba, 400, 150, 150), [0, 0, 255]); // bottom-left of image (blue) + expectColor(getColorAt(rgba, 400, 250, 150), [255, 0, 0]); // bottom-right of image (red) + }); + + test('resize to 400x200 with cover, vertical center crop', async () => { + using sample = makeSample(); + using result = await sample.transform({ width: 400, height: 200, fit: 'cover' }); + + expect(result.codedWidth).toBe(400); + expect(result.codedHeight).toBe(200); + + const rgba = await readRgba(result); + expectColor(getColorAt(rgba, 400, 50, 50), [255, 0, 0]); // top-left (red) + expectColor(getColorAt(rgba, 400, 350, 50), [255, 0, 0]); // top-right (red) + expectColor(getColorAt(rgba, 400, 50, 150), [0, 0, 255]); // bottom-left (blue) + expectColor(getColorAt(rgba, 400, 350, 150), [255, 0, 0]); // bottom-right (red) + }); + + test('VideoSample transformation via Conversion API', async () => { + using input = new Input({ + source: new FilePathSource('./test/public/video.mp4'), + formats: ALL_FORMATS, + }); + + const output = new Output({ + format: new Mp4OutputFormat(), + target: new BufferTarget(), + }); + + const conversion = await Conversion.init({ + input, + output, + video: { + width: 300, + height: 300, + fit: 'contain', + }, + }); + await conversion.execute(); + }); + }); }); describe('Audio', async () => {