diff --git a/packages/server/src/misc.ts b/packages/server/src/misc.ts index b458119..d072c28 100644 --- a/packages/server/src/misc.ts +++ b/packages/server/src/misc.ts @@ -229,3 +229,6 @@ export const getChannelLayout = (numChannels: number): NodeAv.ChannelLayout => { default: return { nbChannels: numChannels, order: NodeAv.AV_CHANNEL_ORDER_UNSPEC, mask: 0n }; } }; + +// The value is incorrect in the node-av source code, so we do: +export const LIBVPX_VP9 = 'libvpx-vp9' as (NodeAv.FFEncoderCodec & NodeAv.FFDecoderCodec); diff --git a/packages/server/src/video-decoder.ts b/packages/server/src/video-decoder.ts index b77e6ce..3d479de 100644 --- a/packages/server/src/video-decoder.ts +++ b/packages/server/src/video-decoder.ts @@ -1,13 +1,13 @@ import { CustomVideoDecoder, VideoCodec, EncodedPacket, VideoSample, MaybePromise, Rational } from 'mediabunny'; import * as NodeAv from 'node-av'; -import { CODEC_TO_CODEC_ID, getHardwareDecoderCodec } from './misc'; +import { CODEC_TO_CODEC_ID, getHardwareDecoderCodec, LIBVPX_VP9 } from './misc'; import { assert, binarySearchLessOrEqual, simplifyRational, toUint8Array } from '../../../src/misc'; import { NodeAvFrameVideoSampleResource } from './video-sample'; export class NodeAvVideoDecoder extends CustomVideoDecoder { frame!: NodeAv.Frame; packet!: NodeAv.Packet; - codecContext!: NodeAv.CodecContext; + codecContext: NodeAv.CodecContext | null = null; pixelAspectRatio!: Rational; // Bookkeeping to restore the original timing information @@ -29,12 +29,18 @@ export class NodeAvVideoDecoder extends CustomVideoDecoder { this.frame.alloc(); this.packet = new NodeAv.Packet(); this.packet.alloc(); + } + + async initCodecContext(packet: EncodedPacket) { + assert(this.codecContext === null); const codecId = CODEC_TO_CODEC_ID[this.codec]; assert(codecId !== undefined); let codec: NodeAv.Codec | null; - if (this.config.hardwareAcceleration !== 'prefer-hardware' || this.codec === 'av1') { + if (this.codec === 'vp9' && packet.sideData.alpha) { + codec = NodeAv.Codec.findDecoderByName(LIBVPX_VP9) ?? NodeAv.Codec.findDecoder(codecId); + } else if (this.config.hardwareAcceleration !== 'prefer-hardware' || this.codec === 'av1') { // https://github.com/opencv/opencv/issues/24430 codec = NodeAv.Codec.findDecoder(codecId); } else { @@ -69,6 +75,11 @@ export class NodeAvVideoDecoder extends CustomVideoDecoder { } async decode(packet: EncodedPacket) { + if (this.codecContext === null) { + await this.initCodecContext(packet); + assert(this.codecContext); + } + this.packet.isKeyframe = packet.type === 'key'; this.packet.data = Buffer.from(packet.data); this.packet.timeBase = { num: 1, den: 1e6 }; @@ -76,6 +87,14 @@ export class NodeAvVideoDecoder extends CustomVideoDecoder { this.packet.dts = NodeAv.AV_NOPTS_VALUE; this.packet.duration = BigInt(packet.microsecondDuration); + if (packet.sideData.alpha) { + const matroskaBlockAdditional = Buffer.alloc(8 + packet.sideData.alpha.byteLength); + matroskaBlockAdditional[7] = 1; // BlockAddId + matroskaBlockAdditional.set(packet.sideData.alpha, 8); + + this.packet.addSideData(NodeAv.AV_PKT_DATA_MATROSKA_BLOCKADDITIONAL, matroskaBlockAdditional); + } + const preciseTimingIndex = binarySearchLessOrEqual( this.preciseTimings, packet.microsecondTimestamp, @@ -161,6 +180,10 @@ export class NodeAvVideoDecoder extends CustomVideoDecoder { } async flush() { + if (!this.codecContext) { + return; + } + // Send null packet to signal flush const ret = await this.codecContext.sendPacket(null); NodeAv.FFmpegError.throwIfError(ret, 'Flush decoder'); @@ -180,7 +203,7 @@ export class NodeAvVideoDecoder extends CustomVideoDecoder { } close(): MaybePromise { - this.codecContext.freeContext(); + this.codecContext?.freeContext(); this.frame.free(); this.packet.free(); } diff --git a/packages/server/src/video-encoder.ts b/packages/server/src/video-encoder.ts index abb3fcd..2a5524d 100644 --- a/packages/server/src/video-encoder.ts +++ b/packages/server/src/video-encoder.ts @@ -3,6 +3,7 @@ import * as NodeAv from 'node-av'; import { CODEC_TO_CODEC_ID, getHardwareEncoderCodec, + LIBVPX_VP9, unmapColorPrimaries, unmapMatrixCoefficients, unmapTransferCharacteristics, @@ -24,6 +25,7 @@ import { } from '../../../src/codec-data'; import { extractVideoCodecString } from '../../../src/codec'; import { assert, binarySearchLessOrEqual, simplifyRational, toUint8Array } from '../../../src/misc'; +import { EncodedPacketSideData } from 'mediabunny'; export class NodeAvVideoEncoder extends CustomVideoEncoder { frame!: NodeAv.Frame; @@ -63,7 +65,9 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder { assert(codecId !== undefined); let codec: NodeAv.Codec | null = null; - if (this.config.hardwareAcceleration === 'prefer-software') { + if (this.codec === 'vp9' && this.config.alpha === 'keep') { + codec = NodeAv.Codec.findEncoderByName(LIBVPX_VP9) ?? NodeAv.Codec.findEncoder(codecId); + } else if (this.config.hardwareAcceleration === 'prefer-software') { codec = NodeAv.Codec.findEncoder(codecId); } else { codec = getHardwareEncoderCodec(codecId) ?? NodeAv.Codec.findEncoder(codecId); @@ -85,8 +89,15 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder { codecContext.allocContext3(this.avCodec); let pixelFormat = NodeAv.AV_PIX_FMT_YUV420P; - if (this.avCodec.pixelFormats && !this.avCodec.pixelFormats.includes(NodeAv.AV_PIX_FMT_YUV420P)) { - pixelFormat = this.avCodec.pixelFormats[0]!; + + if (this.avCodec.pixelFormats) { + if (!this.avCodec.pixelFormats.includes(NodeAv.AV_PIX_FMT_YUV420P)) { + pixelFormat = this.avCodec.pixelFormats[0]!; + } + + if (this.config.alpha === 'keep' && this.avCodec.pixelFormats.includes(NodeAv.AV_PIX_FMT_YUVA420P)) { + pixelFormat = NodeAv.AV_PIX_FMT_YUVA420P; + } } const pixelAspectRatio = simplifyRational({ @@ -451,11 +462,20 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder { throw new Error('Unreachable.'); } + const sideData: EncodedPacketSideData = {}; + const matroskaBlockAdditional = this.packet.getSideData(NodeAv.AV_PKT_DATA_MATROSKA_BLOCKADDITIONAL); + if (matroskaBlockAdditional) { + sideData.alpha = toUint8Array(matroskaBlockAdditional).subarray(8); // Skip the BlockAddId + } + const packet = new EncodedPacket( packetData, this.packet.isKeyframe ? 'key' : 'delta', timestamp, duration, + undefined, + undefined, + sideData, ); if (decoderConfigCodecString !== null) { diff --git a/src/media-sink.ts b/src/media-sink.ts index 946ce19..4023fd5 100644 --- a/src/media-sink.ts +++ b/src/media-sink.ts @@ -45,7 +45,14 @@ import { } from './misc'; import { EncodedPacket } from './packet'; import { fromAlaw, fromUlaw } from './pcm'; -import { AudioSample, clampCropRectangle, CropRectangle, validateCropRectangle, VideoSample } from './sample'; +import { + AudioSample, + clampCropRectangle, + CropRectangle, + validateCropRectangle, + VideoSample, + VideoSamplePixelFormat, +} from './sample'; /** * Additional options for controlling packet retrieval. @@ -863,13 +870,13 @@ class VideoDecoderWrapper extends DecoderWrapper { colorQueue: VideoFrame[] = []; alphaQueue: (VideoFrame | null)[] = []; merger: ColorAlphaMerger | null = null; - mergerCreationFailed = false; decodedAlphaChunkCount = 0; alphaDecoderQueueSize = 0; /** Each value is the number of decoded alpha chunks at which a null alpha frame should be added. */ nullAlphaFrameQueue: number[] = []; currentAlphaPacketIndex = 0; alphaRaslSkipped = false; // For HEVC stuff + frameHandlerSerializer = new CallSerializer(); constructor( onSample: (sample: VideoSample) => unknown, @@ -901,15 +908,17 @@ class VideoDecoderWrapper extends DecoderWrapper { void this.customDecoderCallSerializer.call(() => this.customDecoder!.init()); } else { const colorHandler = (frame: VideoFrame) => { - if (this.alphaQueue.length > 0) { - // Even when no alpha data is present (most of the time), there will be nulls in this queue - const alphaFrame = this.alphaQueue.shift(); - assert(alphaFrame !== undefined); + this.frameHandlerSerializer.call(async () => { + if (this.alphaQueue.length > 0) { + // Even when no alpha data is present (most of the time), there will be nulls in this queue + const alphaFrame = this.alphaQueue.shift(); + assert(alphaFrame !== undefined); - this.mergeAlpha(frame, alphaFrame); - } else { - this.colorQueue.push(frame); - } + await this.mergeAlpha(frame, alphaFrame); + } else { + this.colorQueue.push(frame); + } + }).catch((error: Error) => this.onError(error)); }; if (codec === 'avc' && this.decoderConfig.description && isChromium()) { @@ -1013,56 +1022,49 @@ class VideoDecoderWrapper extends DecoderWrapper { } decodeAlphaData(packet: EncodedPacket) { - if (!packet.sideData.alpha || this.mergerCreationFailed) { + if (!packet.sideData.alpha) { // No alpha side data in the packet, most common case this.pushNullAlphaFrame(); return; } if (!this.merger) { - try { - this.merger = new ColorAlphaMerger(); - } catch (error) { - console.error('Due to an error, only color data will be decoded.', error); - - this.mergerCreationFailed = true; - this.decodeAlphaData(packet); // Go again - - return; - } + this.merger = new ColorAlphaMerger(); } // Check if we need to set up the alpha decoder if (!this.alphaDecoder) { const alphaHandler = (frame: VideoFrame) => { - this.alphaDecoderQueueSize--; - - if (this.colorQueue.length > 0) { - const colorFrame = this.colorQueue.shift(); - assert(colorFrame !== undefined); - - this.mergeAlpha(colorFrame, frame); - } else { - this.alphaQueue.push(frame); - } - - // Check if any null frames have been queued for this point - this.decodedAlphaChunkCount++; - while ( - this.nullAlphaFrameQueue.length > 0 - && this.nullAlphaFrameQueue[0] === this.decodedAlphaChunkCount - ) { - this.nullAlphaFrameQueue.shift(); + this.frameHandlerSerializer.call(async () => { + this.alphaDecoderQueueSize--; if (this.colorQueue.length > 0) { const colorFrame = this.colorQueue.shift(); assert(colorFrame !== undefined); - this.mergeAlpha(colorFrame, null); + await this.mergeAlpha(colorFrame, frame); } else { - this.alphaQueue.push(null); + this.alphaQueue.push(frame); } - } + + // Check if any null frames have been queued for this point + this.decodedAlphaChunkCount++; + while ( + this.nullAlphaFrameQueue.length > 0 + && this.nullAlphaFrameQueue[0] === this.decodedAlphaChunkCount + ) { + this.nullAlphaFrameQueue.shift(); + + if (this.colorQueue.length > 0) { + const colorFrame = this.colorQueue.shift(); + assert(colorFrame !== undefined); + + await this.mergeAlpha(colorFrame, null); + } else { + this.alphaQueue.push(null); + } + } + }).catch((error: Error) => this.onError(error)); }; const stack = new Error('Decoding error').stack; @@ -1183,7 +1185,7 @@ class VideoDecoderWrapper extends DecoderWrapper { this.onSample(sample); } - mergeAlpha(color: VideoFrame, alpha: VideoFrame | null) { + async mergeAlpha(color: VideoFrame, alpha: VideoFrame | null) { if (!alpha) { // Nothing needs to be merged const finalSample = new VideoSample(color); @@ -1194,14 +1196,8 @@ class VideoDecoderWrapper extends DecoderWrapper { assert(this.merger); - this.merger.update(color, alpha); - color.close(); - alpha.close(); - - const finalFrame = new VideoFrame(this.merger.canvas, { - timestamp: color.timestamp, - duration: color.duration ?? undefined, - }); + // The merger takes ownership of the frames, so no need to close them ourselves + const finalFrame = await this.merger.update(color, alpha); const finalSample = new VideoSample(finalFrame); this.sampleHandler(finalSample); @@ -1216,6 +1212,7 @@ class VideoDecoderWrapper extends DecoderWrapper { this.decoder.flush(), this.alphaDecoder?.flush(), ]); + await this.frameHandlerSerializer.currentPromise; this.colorQueue.forEach(x => x.close()); this.colorQueue.length = 0; @@ -1265,43 +1262,75 @@ class VideoDecoderWrapper extends DecoderWrapper { } } +let mergerGpuUnavailable = false; + /** Utility class that merges together color and alpha information using simple WebGL 2 shaders. */ -class ColorAlphaMerger { - canvas: OffscreenCanvas | HTMLCanvasElement; - private gl: WebGL2RenderingContext; - private program: WebGLProgram; - private vao: WebGLVertexArrayObject; - private colorTexture: WebGLTexture; - private alphaTexture: WebGLTexture; +export class ColorAlphaMerger { + static forceCpu = false; + + canvas: OffscreenCanvas | HTMLCanvasElement | null = null; + private gl: WebGL2RenderingContext | null = null; + private program: WebGLProgram | null = null; + private vao: WebGLVertexArrayObject | null = null; + private colorTexture: WebGLTexture | null = null; + private alphaTexture: WebGLTexture | null = null; + + private worker: Worker | null = null; + private pendingRequests = new Map>>(); + private nextRequestId = 0; constructor() { - // Canvas will be resized later - if (typeof OffscreenCanvas !== 'undefined') { - // Prefer OffscreenCanvas for Worker environments - this.canvas = new OffscreenCanvas(300, 150); + const canMakeCanvas = typeof OffscreenCanvas !== 'undefined' + // eslint-disable-next-line @typescript-eslint/no-deprecated + || (typeof document !== 'undefined' && typeof document.createElement === 'function'); + + if (!ColorAlphaMerger.forceCpu && canMakeCanvas && !mergerGpuUnavailable) { + // Try the GPU path. If anything goes wrong, we silently fall back to the CPU path. + try { + // Canvas will be resized later + if (typeof OffscreenCanvas !== 'undefined') { + // Prefer OffscreenCanvas for Worker environments + this.canvas = new OffscreenCanvas(300, 150); + } else { + this.canvas = document.createElement('canvas'); + } + + const gl = this.canvas.getContext('webgl2', { + premultipliedAlpha: false, + }) as unknown as WebGL2RenderingContext | null; // Casting because of some TypeScript weirdness + if (!gl) { + throw new Error('Couldn\'t acquire WebGL 2 context.'); + } + + this.gl = gl; + this.program = this.createProgram(); + this.vao = this.createVAO(); + this.colorTexture = this.createTexture(); + this.alphaTexture = this.createTexture(); + + this.gl.useProgram(this.program); + this.gl.uniform1i(this.gl.getUniformLocation(this.program, 'u_colorTexture'), 0); + this.gl.uniform1i(this.gl.getUniformLocation(this.program, 'u_alphaTexture'), 1); + } catch (error) { + this.gl = null; + this.canvas = null; + mergerGpuUnavailable = true; + console.warn('Falling back to CPU for color/alpha merging.', error); + } + } + } + + async update(color: VideoFrame, alpha: VideoFrame): Promise { + if (this.gl) { + return this.updateGpu(color, alpha); } else { - this.canvas = document.createElement('canvas'); + return this.updateCpu(color, alpha); } - - const gl = this.canvas.getContext('webgl2', { - premultipliedAlpha: false, - }) as unknown as WebGL2RenderingContext | null; // Casting because of some TypeScript weirdness - if (!gl) { - throw new Error('Couldn\'t acquire WebGL 2 context.'); - } - - this.gl = gl; - this.program = this.createProgram(); - this.vao = this.createVAO(); - this.colorTexture = this.createTexture(); - this.alphaTexture = this.createTexture(); - - this.gl.useProgram(this.program); - this.gl.uniform1i(this.gl.getUniformLocation(this.program, 'u_colorTexture'), 0); - this.gl.uniform1i(this.gl.getUniformLocation(this.program, 'u_alphaTexture'), 1); } private createProgram(): WebGLProgram { + assert(this.gl); + const vertexShader = this.createShader(this.gl.VERTEX_SHADER, `#version 300 es in vec2 a_position; in vec2 a_texCoord; @@ -1337,6 +1366,8 @@ class ColorAlphaMerger { } private createShader(type: number, source: string): WebGLShader { + assert(this.gl); + const shader = this.gl.createShader(type)!; this.gl.shaderSource(shader, source); this.gl.compileShader(shader); @@ -1344,6 +1375,9 @@ class ColorAlphaMerger { } private createVAO(): WebGLVertexArrayObject { + assert(this.gl); + assert(this.program); + const vao = this.gl.createVertexArray(); this.gl.bindVertexArray(vao); @@ -1371,6 +1405,8 @@ class ColorAlphaMerger { } private createTexture(): WebGLTexture { + assert(this.gl); + const texture = this.gl.createTexture(); this.gl.bindTexture(this.gl.TEXTURE_2D, texture); @@ -1382,7 +1418,10 @@ class ColorAlphaMerger { return texture; } - update(color: VideoFrame, alpha: VideoFrame): void { + private updateGpu(color: VideoFrame, alpha: VideoFrame): VideoFrame { + assert(this.gl); + assert(this.canvas); + if (color.displayWidth !== this.canvas.width || color.displayHeight !== this.canvas.height) { this.canvas.width = color.displayWidth; this.canvas.height = color.displayHeight; @@ -1401,14 +1440,306 @@ class ColorAlphaMerger { this.gl.bindVertexArray(this.vao); this.gl.drawArrays(this.gl.TRIANGLE_STRIP, 0, 4); + + const finalFrame = new VideoFrame(this.canvas, { + timestamp: color.timestamp, + duration: color.duration ?? undefined, + }); + + color.close(); + alpha.close(); + + return finalFrame; + } + + private updateCpu(color: VideoFrame, alpha: VideoFrame): Promise { + if (!this.worker) { + const blob = new Blob( + [`(${colorAlphaMergerWorkerCode.toString()})()`], + { type: 'application/javascript' }, + ); + const url = URL.createObjectURL(blob); + this.worker = new Worker(url); + URL.revokeObjectURL(url); + + this.worker.addEventListener('message', (event: MessageEvent) => { + const data = event.data; + const pending = this.pendingRequests.get(data.id); + if (!pending) { + return; + } + this.pendingRequests.delete(data.id); + + if ('error' in data) { + pending.reject(new Error(data.error)); + } else { + pending.resolve(data.frame); + } + }); + + this.worker.addEventListener('error', (event) => { + const error = new Error(event.message || 'Color/alpha merge worker error.'); + for (const pending of this.pendingRequests.values()) { + pending.reject(error); + } + this.pendingRequests.clear(); + }); + } + + const id = this.nextRequestId++; + const pending = promiseWithResolvers(); + this.pendingRequests.set(id, pending); + + this.worker.postMessage({ id, color, alpha }, { transfer: [color, alpha] }); + + return pending.promise; } close() { - this.gl.getExtension('WEBGL_lose_context')?.loseContext(); - this.gl = null as unknown as WebGL2RenderingContext; + this.gl?.getExtension('WEBGL_lose_context')?.loseContext(); + this.gl = null; + this.canvas = null; + + this.worker?.terminate(); + this.worker = null; + + const error = new Error('Color/alpha merger closed.'); + for (const pending of this.pendingRequests.values()) { + pending.reject(error); + } + this.pendingRequests.clear(); } } +type ColorAlphaMergerWorkerRequest = { + id: number; + color: VideoFrame; + alpha: VideoFrame; +}; + +type ColorAlphaMergerWorkerResponse = + | { id: number; frame: VideoFrame } + | { id: number; error: string }; + +const colorAlphaMergerWorkerCode = () => { + // These buffers are reused across frames as long as the size matches, since consecutive frames usually share + // dimensions + let cpuAlphaBuffer: Uint8Array | null = null; + let cpuColorBuffer: Uint8Array | null = null; + + // Serialize execution internally so concurrent requests don't race on the shared cpu*Buffer state. + let chain: Promise = Promise.resolve(); + self.addEventListener('message', (event: MessageEvent) => { + const { id, color, alpha } = event.data; + chain = chain.then(async () => { + try { + const frame = await merge(color, alpha); + self.postMessage({ id, frame }, { transfer: [frame] }); + } catch (error) { + self.postMessage({ id, error: (error as Error).message }); + } finally { + // We took ownership of the inputs via transfer; close them now that the merge (or its error) is done. + color.close(); + alpha.close(); + } + }); + }); + + const merge = async (color: VideoFrame, alpha: VideoFrame): Promise => { + const format = color.format as VideoSamplePixelFormat | null; + const alphaFormat = alpha.format as VideoSamplePixelFormat | null; + if (!format || !alphaFormat) { + throw new Error('CPU color/alpha merging requires a known VideoFrame format.'); + } + + // The alpha frame must have the same bit depth as the color frame + const colorIs10 = format.includes('P10'); + const colorIs12 = format.includes('P12'); + const alphaIs10 = alphaFormat.includes('P10'); + const alphaIs12 = alphaFormat.includes('P12'); + if (alphaIs10 !== colorIs10 || alphaIs12 !== colorIs12) { + throw new Error( + `CPU color/alpha merging requires the alpha frame to have the same bit depth as the color frame` + + ` (color: '${format}', alpha: '${alphaFormat}').`, + ); + } + + const width = color.codedWidth; + const height = color.codedHeight; + + if (format === 'RGBX' || format === 'RGBA' || format === 'BGRX' || format === 'BGRA') { + return await mergeInterleavedRgba(color, alpha, width, height, format); + } else if ( + format === 'I420' || format === 'I420P10' || format === 'I420P12' + || format === 'I422' || format === 'I422P10' || format === 'I422P12' + || format === 'I444' || format === 'I444P10' || format === 'I444P12' + ) { + return await mergePlanarYuv(color, alpha, width, height, format); + } else if (format === 'NV12') { + return await mergeNv12(color, alpha, width, height); + } + + throw new Error(`CPU color/alpha merging does not support format '${format}'.`); + }; + + const mergeInterleavedRgba = async ( + color: VideoFrame, + alpha: VideoFrame, + width: number, + height: number, + format: 'RGBX' | 'RGBA' | 'BGRX' | 'BGRA', + ): Promise => { + const pixelCount = width * height; + const output = new Uint8Array(pixelCount * 4); + + // Color goes straight into the output buffer via copyTo, no intermediate copy needed + await color.copyTo(output); + + // And now add the alpha data + const alphaY = await readAlpha(alpha, width, height, 1); + for (let i = 0, j = 3; i < pixelCount; i++, j += 4) { + output[j] = alphaY[i]!; + } + + const outputFormat = (format === 'RGBX' || format === 'RGBA') ? 'RGBA' : 'BGRA'; + const init = { + format: outputFormat, + codedWidth: width, + codedHeight: height, + timestamp: color.timestamp, + duration: color.duration ?? undefined, + transfer: [output.buffer], + } as const; + + return new VideoFrame(output, init); + }; + + const mergePlanarYuv = async ( + color: VideoFrame, + alpha: VideoFrame, + width: number, + height: number, + format: + | 'I420' | 'I420P10' | 'I420P12' + | 'I422' | 'I422P10' | 'I422P12' + | 'I444' | 'I444P10' | 'I444P12', + ): Promise => { + const is10 = format.includes('P10'); + const is12 = format.includes('P12'); + const bytesPerSample = (is10 || is12) ? 2 : 1; + + let chromaW: number; + let chromaH: number; + if (format.startsWith('I420')) { + chromaW = Math.ceil(width / 2); + chromaH = Math.ceil(height / 2); + } else if (format.startsWith('I422')) { + chromaW = Math.ceil(width / 2); + chromaH = height; + } else { + chromaW = width; + chromaH = height; + } + + const ySamples = width * height; + const uvSamples = chromaW * chromaH; + const yBytes = ySamples * bytesPerSample; + const uvBytes = uvSamples * bytesPerSample; + const aBytes = ySamples * bytesPerSample; + + const outputBytes = yBytes + 2 * uvBytes + aBytes; + const output = new Uint8Array(outputBytes); + + // Write color planes directly into the output buffer via copyTo, no intermediate copy + await color.copyTo(output); + + const alphaY = await readAlpha(alpha, width, height, bytesPerSample); + const aOffset = yBytes + 2 * uvBytes; + output.set(alphaY, aOffset); + + const outputFormat = (format.slice(0, 4) + 'A' + format.slice(4)) as VideoPixelFormat; + + const init = { + format: outputFormat, + codedWidth: width, + codedHeight: height, + timestamp: color.timestamp, + duration: color.duration ?? undefined, + transfer: [output.buffer], + }; + + return new VideoFrame(output, init); + }; + + const mergeNv12 = async ( + color: VideoFrame, + alpha: VideoFrame, + width: number, + height: number, + ): Promise => { + const ySize = width * height; + const chromaW = Math.ceil(width / 2); + const chromaH = Math.ceil(height / 2); + const uvSize = chromaW * chromaH; + + const sourceSize = color.allocationSize(); + if (!cpuColorBuffer || cpuColorBuffer.byteLength !== sourceSize) { + cpuColorBuffer = new Uint8Array(sourceSize); + } + await color.copyTo(cpuColorBuffer); + + const output = new Uint8Array(ySize + 2 * uvSize + ySize); + + // Y plane copies straight over + output.set(cpuColorBuffer.subarray(0, ySize), 0); + + // Deinterleave the UV plane into separate U and V planes + const uOffset = ySize; + const vOffset = ySize + uvSize; + const uvStart = ySize; + for (let i = 0; i < uvSize; i++) { + output[uOffset + i] = cpuColorBuffer[uvStart + i * 2]!; + output[vOffset + i] = cpuColorBuffer[uvStart + i * 2 + 1]!; + } + + const alphaY = await readAlpha(alpha, width, height, 1); + output.set(alphaY, ySize + 2 * uvSize); + + const init = { + format: 'I420A', + codedWidth: width, + codedHeight: height, + timestamp: color.timestamp, + duration: color.duration ?? undefined, + transfer: [output.buffer], + } as const; + + return new VideoFrame(output, init); + }; + + const readAlpha = async (alpha: VideoFrame, width: number, height: number, bytesPerSample: number) => { + const size = alpha.allocationSize(); + if (!cpuAlphaBuffer || cpuAlphaBuffer.byteLength !== size) { + cpuAlphaBuffer = new Uint8Array(size); + } + await alpha.copyTo(cpuAlphaBuffer); + + const format = alpha.format; + if (format === 'RGBA' || format === 'BGRA' || format === 'RGBX' || format === 'BGRX') { + // Pack alpha data tightly + const rOffset = (format === 'RGBA' || format === 'RGBX') ? 0 : 2; + const pixelCount = width * height; + for (let i = 0; i < pixelCount; i++) { + cpuAlphaBuffer[i] = cpuAlphaBuffer[i * 4 + rOffset]!; + } + return cpuAlphaBuffer.subarray(0, pixelCount); + } else { + // For Y-plane-first formats (I*** and NV12), the leading width*height samples are the Y plane + return cpuAlphaBuffer.subarray(0, width * height * bytesPerSample); + } + }; +}; + /** * A sink that retrieves decoded video samples (video frames) from a video track. * @group Media sinks diff --git a/src/media-source.ts b/src/media-source.ts index 1b164cf..c24fd79 100644 --- a/src/media-source.ts +++ b/src/media-source.ts @@ -52,6 +52,7 @@ import { audioSampleToInterleavedFormat, toInterleavedAudioFormat, VideoSample, + VideoSamplePixelFormat, } from './sample'; import { AudioEncodingConfig, @@ -544,27 +545,15 @@ class VideoEncoderWrapper { const height = videoFrame.displayHeight; if (!this.splitter) { - try { - this.splitter = new ColorAlphaSplitter(width, height); - } catch (error) { - console.error('Due to an error, only color data will be encoded.', error); - - this.splitterCreationFailed = true; - this.alphaFrameQueue.push(null); - this.encoder.encode(videoFrame, finalEncodeOptions); - videoFrame.close(); - } + this.splitter = new ColorAlphaSplitter(width, height); } - if (this.splitter) { - const colorFrame = this.splitter.extractColor(videoFrame); - const alphaFrame = this.splitter.extractAlpha(videoFrame); + // The splitter takes ownership, so no need to close the frames ourselves + const { colorFrame, alphaFrame } = await this.splitter.update(videoFrame); - this.alphaFrameQueue.push(alphaFrame); - this.encoder.encode(colorFrame, finalEncodeOptions); - colorFrame.close(); - videoFrame.close(); - } + this.alphaFrameQueue.push(alphaFrame); + this.encoder.encode(colorFrame, finalEncodeOptions); + colorFrame.close(); } } @@ -889,51 +878,107 @@ class VideoEncoderWrapper { } } -/** Utility class for splitting a composite frame into separate color and alpha components. */ -class ColorAlphaSplitter { - canvas: OffscreenCanvas | HTMLCanvasElement; +let splitterGpuUnavailable = false; - private gl: WebGL2RenderingContext; - private colorProgram: WebGLProgram; - private alphaProgram: WebGLProgram; - private vao: WebGLVertexArrayObject; - private sourceTexture: WebGLTexture; - private lastFrame: VideoFrame | null = null; - private alphaResolutionLocation: WebGLUniformLocation; +/** Utility class for splitting a composite frame into separate color and alpha components. */ +export class ColorAlphaSplitter { + static forceCpu = false; + + canvas: OffscreenCanvas | HTMLCanvasElement | null = null; + + private gl: WebGL2RenderingContext | null = null; + private colorProgram: WebGLProgram | null = null; + private alphaProgram: WebGLProgram | null = null; + private vao: WebGLVertexArrayObject | null = null; + private sourceTexture: WebGLTexture | null = null; + private alphaResolutionLocation: WebGLUniformLocation | null = null; + + private worker: Worker | null = null; + private pendingRequests = new Map< + number, + ReturnType> + >(); + + private nextRequestId = 0; constructor(initialWidth: number, initialHeight: number) { - if (typeof OffscreenCanvas !== 'undefined') { - this.canvas = new OffscreenCanvas(initialWidth, initialHeight); + const canMakeCanvas = typeof OffscreenCanvas !== 'undefined' + // eslint-disable-next-line @typescript-eslint/no-deprecated + || (typeof document !== 'undefined' && typeof document.createElement === 'function'); + + if (!ColorAlphaSplitter.forceCpu && canMakeCanvas && !splitterGpuUnavailable) { + // Try the GPU path. If anything goes wrong, we silently fall back to the CPU path. + try { + if (typeof OffscreenCanvas !== 'undefined') { + this.canvas = new OffscreenCanvas(initialWidth, initialHeight); + } else { + this.canvas = document.createElement('canvas'); + this.canvas.width = initialWidth; + this.canvas.height = initialHeight; + } + + const gl = this.canvas.getContext('webgl2', { + alpha: true, // Needed due to the YUV thing we do for alpha + }) as unknown as WebGL2RenderingContext | null; // Casting because of some TypeScript weirdness + if (!gl) { + throw new Error('Couldn\'t acquire WebGL 2 context.'); + } + + this.gl = gl; + + this.colorProgram = this.createColorProgram(); + this.alphaProgram = this.createAlphaProgram(); + this.vao = this.createVAO(); + this.sourceTexture = this.createTexture(); + + this.alphaResolutionLocation = this.gl.getUniformLocation(this.alphaProgram, 'u_resolution')!; + + this.gl.useProgram(this.colorProgram); + this.gl.uniform1i(this.gl.getUniformLocation(this.colorProgram, 'u_sourceTexture'), 0); + + this.gl.useProgram(this.alphaProgram); + this.gl.uniform1i(this.gl.getUniformLocation(this.alphaProgram, 'u_sourceTexture'), 0); + } catch (error) { + this.gl = null; + this.canvas = null; + splitterGpuUnavailable = true; + console.warn('Falling back to CPU for color/alpha splitting.', error); + } + } + } + + async update(sourceFrame: VideoFrame) { + if (this.gl) { + return this.updateGpu(sourceFrame); } else { - this.canvas = document.createElement('canvas'); - this.canvas.width = initialWidth; - this.canvas.height = initialHeight; + return this.updateCpu(sourceFrame); + } + } + + private updateGpu(sourceFrame: VideoFrame) { + assert(this.gl); + assert(this.canvas); + + if (sourceFrame.displayWidth !== this.canvas.width || sourceFrame.displayHeight !== this.canvas.height) { + this.canvas.width = sourceFrame.displayWidth; + this.canvas.height = sourceFrame.displayHeight; } - const gl = this.canvas.getContext('webgl2', { - alpha: true, // Needed due to the YUV thing we do for alpha - }) as unknown as WebGL2RenderingContext | null; // Casting because of some TypeScript weirdness - if (!gl) { - throw new Error('Couldn\'t acquire WebGL 2 context.'); - } + this.gl.activeTexture(this.gl.TEXTURE0); + this.gl.bindTexture(this.gl.TEXTURE_2D, this.sourceTexture); + this.gl.texImage2D(this.gl.TEXTURE_2D, 0, this.gl.RGBA, this.gl.RGBA, this.gl.UNSIGNED_BYTE, sourceFrame); - this.gl = gl; + const colorFrame = this.runColorProgram(sourceFrame); + const alphaFrame = this.runAlphaProgram(sourceFrame); - this.colorProgram = this.createColorProgram(); - this.alphaProgram = this.createAlphaProgram(); - this.vao = this.createVAO(); - this.sourceTexture = this.createTexture(); + sourceFrame.close(); - this.alphaResolutionLocation = this.gl.getUniformLocation(this.alphaProgram, 'u_resolution')!; - - this.gl.useProgram(this.colorProgram); - this.gl.uniform1i(this.gl.getUniformLocation(this.colorProgram, 'u_sourceTexture'), 0); - - this.gl.useProgram(this.alphaProgram); - this.gl.uniform1i(this.gl.getUniformLocation(this.alphaProgram, 'u_sourceTexture'), 0); + return { colorFrame, alphaFrame }; } private createVertexShader(): WebGLShader { + assert(this.gl); + return this.createShader(this.gl.VERTEX_SHADER, `#version 300 es in vec2 a_position; in vec2 a_texCoord; @@ -947,6 +992,8 @@ class ColorAlphaSplitter { } private createColorProgram(): WebGLProgram { + assert(this.gl); + const vertexShader = this.createVertexShader(); // This shader is simple, simply copy the color information while setting alpha to 1 @@ -972,6 +1019,8 @@ class ColorAlphaSplitter { } private createAlphaProgram(): WebGLProgram { + assert(this.gl); + const vertexShader = this.createVertexShader(); // This shader's more complex. The main reason is that this shader writes data in I420 (yuv420) pixel format @@ -1035,6 +1084,8 @@ class ColorAlphaSplitter { } private createShader(type: number, source: string): WebGLShader { + assert(this.gl); + const shader = this.gl.createShader(type)!; this.gl.shaderSource(shader, source); this.gl.compileShader(shader); @@ -1045,6 +1096,9 @@ class ColorAlphaSplitter { } private createVAO(): WebGLVertexArrayObject { + assert(this.gl); + assert(this.colorProgram); + const vao = this.gl.createVertexArray(); this.gl.bindVertexArray(vao); @@ -1072,6 +1126,8 @@ class ColorAlphaSplitter { } private createTexture(): WebGLTexture { + assert(this.gl); + const texture = this.gl.createTexture(); this.gl.bindTexture(this.gl.TEXTURE_2D, texture); @@ -1083,25 +1139,9 @@ class ColorAlphaSplitter { return texture; } - private updateTexture(sourceFrame: VideoFrame): void { - if (this.lastFrame === sourceFrame) { - return; - } - - if (sourceFrame.displayWidth !== this.canvas.width || sourceFrame.displayHeight !== this.canvas.height) { - this.canvas.width = sourceFrame.displayWidth; - this.canvas.height = sourceFrame.displayHeight; - } - - this.gl.activeTexture(this.gl.TEXTURE0); - this.gl.bindTexture(this.gl.TEXTURE_2D, this.sourceTexture); - this.gl.texImage2D(this.gl.TEXTURE_2D, 0, this.gl.RGBA, this.gl.RGBA, this.gl.UNSIGNED_BYTE, sourceFrame); - - this.lastFrame = sourceFrame; - } - - extractColor(sourceFrame: VideoFrame) { - this.updateTexture(sourceFrame); + private runColorProgram(sourceFrame: VideoFrame) { + assert(this.gl); + assert(this.canvas); this.gl.useProgram(this.colorProgram); this.gl.viewport(0, 0, this.canvas.width, this.canvas.height); @@ -1116,8 +1156,9 @@ class ColorAlphaSplitter { }); } - extractAlpha(sourceFrame: VideoFrame) { - this.updateTexture(sourceFrame); + private runAlphaProgram(sourceFrame: VideoFrame) { + assert(this.gl); + assert(this.canvas); this.gl.useProgram(this.alphaProgram); this.gl.uniform2f(this.alphaResolutionLocation, this.canvas.width, this.canvas.height); @@ -1153,12 +1194,239 @@ class ColorAlphaSplitter { return new VideoFrame(yuv, init); } + private updateCpu(sourceFrame: VideoFrame): Promise<{ colorFrame: VideoFrame; alphaFrame: VideoFrame }> { + if (!this.worker) { + const blob = new Blob( + [`(${colorAlphaSplitterWorkerCode.toString()})()`], + { type: 'application/javascript' }, + ); + const url = URL.createObjectURL(blob); + this.worker = new Worker(url); + URL.revokeObjectURL(url); + + this.worker.addEventListener('message', (event: MessageEvent) => { + const data = event.data; + const pending = this.pendingRequests.get(data.id); + if (!pending) { + return; + } + this.pendingRequests.delete(data.id); + + if ('error' in data) { + pending.reject(new Error(data.error)); + } else { + pending.resolve({ colorFrame: data.colorFrame, alphaFrame: data.alphaFrame }); + } + }); + + this.worker.addEventListener('error', (event) => { + const error = new Error(event.message || 'Color/alpha splitter worker error.'); + for (const pending of this.pendingRequests.values()) { + pending.reject(error); + } + this.pendingRequests.clear(); + }); + } + + const id = this.nextRequestId++; + const pending = promiseWithResolvers<{ colorFrame: VideoFrame; alphaFrame: VideoFrame }>(); + this.pendingRequests.set(id, pending); + this.worker.postMessage({ id, sourceFrame }, { transfer: [sourceFrame] }); + return pending.promise; + } + close() { - this.gl.getExtension('WEBGL_lose_context')?.loseContext(); - this.gl = null as unknown as WebGL2RenderingContext; + this.gl?.getExtension('WEBGL_lose_context')?.loseContext(); + this.gl = null; + this.canvas = null; + + this.worker?.terminate(); + this.worker = null; + const error = new Error('Color/alpha splitter closed.'); + for (const pending of this.pendingRequests.values()) { + pending.reject(error); + } + this.pendingRequests.clear(); } } +type ColorAlphaSplitterWorkerRequest = { + id: number; + sourceFrame: VideoFrame; +}; + +type ColorAlphaSplitterWorkerResponse = + | { id: number; colorFrame: VideoFrame; alphaFrame: VideoFrame } + | { id: number; error: string }; + +const colorAlphaSplitterWorkerCode = () => { + // Reused across frames as long as the size matches, since consecutive frames usually share dimensions. + let cpuSourceBuffer: Uint8Array | null = null; + + // Serialize execution internally so concurrent requests don't race on the shared cpuSourceBuffer. + let chain: Promise = Promise.resolve(); + self.addEventListener('message', (event: MessageEvent) => { + const { id, sourceFrame } = event.data; + chain = chain.then(async () => { + try { + const { colorFrame, alphaFrame } = await split(sourceFrame); + self.postMessage({ id, colorFrame, alphaFrame }, { transfer: [colorFrame, alphaFrame] }); + } catch (error) { + self.postMessage({ id, error: (error as Error).message }); + } finally { + sourceFrame.close(); + } + }); + }); + + const split = async (sourceFrame: VideoFrame) => { + const format = sourceFrame.format as VideoSamplePixelFormat | null; + if (!format) { + throw new Error('CPU color/alpha splitting requires a known VideoFrame format.'); + } + + const width = sourceFrame.codedWidth; + const height = sourceFrame.codedHeight; + const sourceSize = sourceFrame.allocationSize(); + + if (!cpuSourceBuffer || cpuSourceBuffer.byteLength !== sourceSize) { + cpuSourceBuffer = new Uint8Array(sourceSize); + } + await sourceFrame.copyTo(cpuSourceBuffer); + + if (format === 'RGBA' || format === 'BGRA') { + return splitInterleavedRgba(cpuSourceBuffer, width, height, format, sourceFrame); + } else if ( + format === 'I420A' || format === 'I420AP10' || format === 'I420AP12' + || format === 'I422A' || format === 'I422AP10' || format === 'I422AP12' + || format === 'I444A' || format === 'I444AP10' || format === 'I444AP12' + ) { + return splitPlanarYuvA(cpuSourceBuffer, width, height, format, sourceFrame); + } + + throw new Error(`CPU color/alpha splitting does not support format '${format}'.`); + }; + + const splitInterleavedRgba = ( + source: Uint8Array, + width: number, + height: number, + format: 'RGBA' | 'BGRA', + sourceFrame: VideoFrame, + ) => { + const pixelCount = width * height; + const chromaW = Math.ceil(width / 2); + const chromaH = Math.ceil(height / 2); + const alphaSize = pixelCount + chromaW * chromaH * 2; + + // Encode alpha as I420: Y = source A bytes, UV = 128 + const alphaBuffer = new Uint8Array(alphaSize); + for (let i = 0, j = 3; i < pixelCount; i++, j += 4) { + alphaBuffer[i] = source[j]!; + } + alphaBuffer.fill(128, pixelCount); + + // Hand the source buffer straight to VideoFrame as RGBX/BGRX so the A bytes are ignored + const colorFrame = new VideoFrame(source, { + format: format === 'RGBA' ? 'RGBX' : 'BGRX', + codedWidth: width, + codedHeight: height, + timestamp: sourceFrame.timestamp, + duration: sourceFrame.duration ?? undefined, + // No transfer! + }); + const alphaInit = { + format: 'I420' as const, + codedWidth: width, + codedHeight: height, + timestamp: sourceFrame.timestamp, + duration: sourceFrame.duration ?? undefined, + transfer: [alphaBuffer.buffer], + }; + const alphaFrame = new VideoFrame(alphaBuffer, alphaInit); + return { colorFrame, alphaFrame }; + }; + + const splitPlanarYuvA = ( + source: Uint8Array, + width: number, + height: number, + format: + | 'I420A' | 'I420AP10' | 'I420AP12' + | 'I422A' | 'I422AP10' | 'I422AP12' + | 'I444A' | 'I444AP10' | 'I444AP12', + sourceFrame: VideoFrame, + ) => { + const is10 = format.includes('P10'); + const is12 = format.includes('P12'); + const bytesPerSample = (is10 || is12) ? 2 : 1; + + let chromaW: number; + let chromaH: number; + if (format.startsWith('I420')) { + chromaW = Math.ceil(width / 2); + chromaH = Math.ceil(height / 2); + } else if (format.startsWith('I422')) { + chromaW = Math.ceil(width / 2); + chromaH = height; + } else { + chromaW = width; + chromaH = height; + } + + const ySamples = width * height; + const uvSamples = chromaW * chromaH; + const yBytes = ySamples * bytesPerSample; + const uvBytes = uvSamples * bytesPerSample; + const aBytes = ySamples * bytesPerSample; + + const colorBytes = yBytes + uvBytes * 2; + const colorFormat = format.replace('A', '') as VideoPixelFormat; + + const alphaChromaW = Math.ceil(width / 2); + const alphaChromaH = Math.ceil(height / 2); + const alphaUvSamples = alphaChromaW * alphaChromaH; + const alphaUvBytes = alphaUvSamples * bytesPerSample; + const alphaSize = aBytes + 2 * alphaUvBytes; + const alphaBuffer = new Uint8Array(alphaSize); + + const aPlaneStart = colorBytes; + alphaBuffer.set(source.subarray(aPlaneStart, aPlaneStart + aBytes), 0); + + // Fill UV planes with the neutral chroma value + const uvOffset = aBytes; + const neutralChroma = is10 ? 512 : (is12 ? 2048 : 128); + + if (bytesPerSample === 1) { + alphaBuffer.fill(neutralChroma, uvOffset); + } else { + const uvView = new Uint16Array(alphaBuffer.buffer, uvOffset, 2 * alphaUvSamples); + uvView.fill(neutralChroma); + } + + const alphaFormat = (is10 ? 'I420P10' : (is12 ? 'I420P12' : 'I420')) as VideoPixelFormat; + + // Color frame is simply a prefix of the combined bytes + const colorFrame = new VideoFrame(source.subarray(0, colorBytes), { + format: colorFormat, + codedWidth: width, + codedHeight: height, + timestamp: sourceFrame.timestamp, + duration: sourceFrame.duration ?? undefined, + }); + const alphaInit = { + format: alphaFormat, + codedWidth: width, + codedHeight: height, + timestamp: sourceFrame.timestamp, + duration: sourceFrame.duration ?? undefined, + transfer: [alphaBuffer.buffer], + }; + const alphaFrame = new VideoFrame(alphaBuffer, alphaInit); + return { colorFrame, alphaFrame }; + }; +}; + /** * This source can be used to add raw, unencoded video samples (frames) to an output video track. These frames will * automatically be encoded and then piped into the output. diff --git a/test/browser/transparency.test.ts b/test/browser/transparency.test.ts index 22ca520..25979f9 100644 --- a/test/browser/transparency.test.ts +++ b/test/browser/transparency.test.ts @@ -2,16 +2,16 @@ import { expect, test } from 'vitest'; import { Input } from '../../src/input.js'; import { BufferSource, UrlSource } from '../../src/source.js'; import { ALL_FORMATS } from '../../src/input-format.js'; -import { CanvasSink, EncodedPacketSink, VideoSampleSink } from '../../src/media-sink.js'; +import { CanvasSink, ColorAlphaMerger, EncodedPacketSink, VideoSampleSink } from '../../src/media-sink.js'; import { Output } from '../../src/output.js'; import { WebMOutputFormat } from '../../src/output-format.js'; import { BufferTarget } from '../../src/target.js'; -import { CanvasSource, VideoSampleSource } from '../../src/media-source.js'; +import { CanvasSource, ColorAlphaSplitter, VideoSampleSource } from '../../src/media-source.js'; import { canEncodeVideo, QUALITY_HIGH } from '../../src/encode.js'; import { VideoSample } from '../../src/sample.js'; import { Conversion } from '../../src/conversion.js'; -test.skip('Can decode transparent video', async () => { +const decodeTransparentVideoTest = async () => { using input = new Input({ source: new UrlSource('/transparency.webm'), formats: ALL_FORMATS, @@ -33,9 +33,22 @@ test.skip('Can decode transparent video', async () => { const imageData = context.getImageData(0, 0, canvas.width, canvas.height); expect(imageData.data[3]).toBeLessThan(255); // Check that there's actually transparent pixels +}; + +test('Can decode transparent video', async () => { + await decodeTransparentVideoTest(); }); -test.skip('Can decode faulty transparent video and behaves gracefully', async () => { +test('Can decode transparent video, forced CPU path', async () => { + try { + ColorAlphaMerger.forceCpu = true; + await decodeTransparentVideoTest(); + } finally { + ColorAlphaMerger.forceCpu = false; + } +}); + +test('Can decode faulty transparent video and behaves gracefully', async () => { using input = new Input({ source: new UrlSource('/transparency-faulty.webm'), formats: ALL_FORMATS, @@ -55,7 +68,7 @@ test.skip('Can decode faulty transparent video and behaves gracefully', async () expect(secondSample.hasAlpha).toBe(false); }); -test.skip('Can extract transparent frames via CanvasSink', async () => { +test('Can extract transparent frames via CanvasSink', async () => { using input = new Input({ source: new UrlSource('/transparency.webm'), formats: ALL_FORMATS, @@ -81,7 +94,7 @@ test.skip('Can extract transparent frames via CanvasSink', async () => { expect(imageData.data[3]).toBe(255); }); -test.skip('Can encode transparent video', async () => { +const encodeTransparentVideoTest = async () => { const output = new Output({ format: new WebMOutputFormat(), target: new BufferTarget(), @@ -172,10 +185,23 @@ test.skip('Can encode transparent video', async () => { imageData = probeContext.getImageData(0, 0, probeCanvas.width, probeCanvas.height); - expect(imageData.data[3]).toBe(0); // Transparent + expect(imageData.data[4 * (2 * probeCanvas.width + 2) + 3]).toBe(0); // Transparent +}; + +test('Can encode transparent video', async () => { + await encodeTransparentVideoTest(); }); -test.skip('Can encode video with alternating transparency', async () => { +test('Can encode transparent video, forced CPU path', async () => { + try { + ColorAlphaSplitter.forceCpu = true; + await encodeTransparentVideoTest(); + } finally { + ColorAlphaSplitter.forceCpu = false; + } +}); + +test('Can encode video with alternating transparency', async () => { const output = new Output({ format: new WebMOutputFormat(), target: new BufferTarget(), @@ -246,7 +272,7 @@ test.skip('Can encode video with alternating transparency', async () => { } }); -test.skip('Can encode transparent video with odd dimensions', async () => { +test('Can encode transparent video with odd dimensions', async () => { const output = new Output({ format: new WebMOutputFormat(), target: new BufferTarget(), @@ -269,12 +295,12 @@ test.skip('Can encode transparent video with odd dimensions', async () => { await output.finalize(); }); -test.skip('Positive encodability check with alpha', async () => { +test('Positive encodability check with alpha', async () => { const result = await canEncodeVideo('vp9', { alpha: 'keep' }); expect(result).toBe(true); }); -test.skip('Can transmux transparent video, discards alpha by default', async () => { +test('Can transmux transparent video, discards alpha by default', async () => { using input = new Input({ source: new UrlSource('/transparency.webm'), formats: ALL_FORMATS, @@ -303,7 +329,7 @@ test.skip('Can transmux transparent video, discards alpha by default', async () expect(sample.hasAlpha).toBe(false); }); -test.skip('Can transmux transparent video, can keep alpha', async () => { +test('Can transmux transparent video, can keep alpha', async () => { using input = new Input({ source: new UrlSource('/transparency.webm'), formats: ALL_FORMATS, @@ -336,7 +362,7 @@ test.skip('Can transmux transparent video, can keep alpha', async () => { expect(sample.hasAlpha).toBe(true); }); -test.skip('Can reencode transparent video, keeping alpha', async () => { +test('Can reencode transparent video, keeping alpha', async () => { using input = new Input({ source: new UrlSource('/transparency.webm'), formats: ALL_FORMATS, diff --git a/test/node/server-extension.test.ts b/test/node/server-extension.test.ts index 6e1a526..2325aa6 100644 --- a/test/node/server-extension.test.ts +++ b/test/node/server-extension.test.ts @@ -345,11 +345,12 @@ describe('Video', async () => { }); }); - test('VP9 encode and decode', async () => { + test('VP9 encode and decode, opaque', async () => { await encodeDecodeTest('vp9', {}, async (packet, meta, i) => { expect(packet.timestamp).toBe(i / 30); expect(packet.duration).toBe(1 / 30); expect(packet.type).toBe(i ? 'delta' : 'key'); + expect(packet.sideData.alpha).toBeUndefined(); if (i === 0) { expect(meta.decoderConfig).toBeDefined(); @@ -370,6 +371,36 @@ describe('Video', async () => { }); }); + test('VP9 encode and decode, transparent', async () => { + await encodeDecodeTest('vp9', { alpha: 'keep' }, async (packet, meta, i) => { + expect(packet.timestamp).toBe(i / 30); + expect(packet.duration).toBe(1 / 30); + expect(packet.type).toBe(i ? 'delta' : 'key'); + expect(packet.sideData.alpha).toBeDefined(); + expect(packet.sideData.alpha!.byteLength).toBeGreaterThan(0); + + if (i === 0) { + expect(meta.decoderConfig).toBeDefined(); + expect(meta.decoderConfig!.codec.startsWith('vp09.')).toBe(true); + expect(meta.decoderConfig!.description).toBeUndefined(); + } + }, async (sample, i) => { + expect(sample.format).toBe('I420A'); + expect(sample.codedWidth).toBe(1280); + expect(sample.codedHeight).toBe(720); + expect(sample.timestamp).toBe(i / 30); + expect(sample.duration).toBe(1 / 30); + + const buf = new Uint8Array(sample.allocationSize({ format: 'RGBA' })); + await sample.copyTo(buf, { format: 'RGBA' }); + + expect(buf[0]).toBeGreaterThan(250); // Quasi-white + + // Check transparency + expect(Math.abs(buf[3]! - 0x80)).toBeLessThan(3); + }, true); + }); + test('AV1 encode and decode', async () => { await encodeDecodeTest('av1', {}, async (packet, meta, i) => { expect(packet.timestamp).toBe(i / 30); @@ -434,6 +465,7 @@ describe('Video', async () => { extraConfig: Partial, onPacket: (packet: EncodedPacket, meta: EncodedVideoChunkMetadata, i: number) => Promise, onSample: (sample: VideoSample, i: number) => Promise, + transparent = false, ) => { const encoder = new NodeAvVideoEncoder(); // @ts-expect-error Readonly @@ -460,9 +492,15 @@ describe('Video', async () => { await encoder.init(); const data = new Uint8Array(1280 * 720 * 4).fill(0xff); // White + if (transparent) { + for (let i = 0; i < data.byteLength; i += 4) { + data[i + 3] = 0x80; + } + } + for (let i = 0; i < 10; i++) { using sample = new VideoSample(data, { - format: 'RGBX', + format: transparent ? 'RGBA' : 'RGBX', codedWidth: 1280, codedHeight: 720, timestamp: i / 30,