From a4884184e942219bdad1cef1754da55c4f3752ba Mon Sep 17 00:00:00 2001 From: Vanilagy <1696106+Vanilagy@users.noreply.github.com> Date: Tue, 30 Jun 2026 18:23:59 +0200 Subject: [PATCH] Remove GPU path for alpha merger/splitter, add worker pool for merger, allow concurrent merging for better performance --- src/media-sink.ts | 378 +++++++++--------------------- src/media-source.ts | 339 ++------------------------- test/browser/transparency.test.ts | 22 +- 3 files changed, 128 insertions(+), 611 deletions(-) diff --git a/src/media-sink.ts b/src/media-sink.ts index 957b78a..3a2d0b8 100644 --- a/src/media-sink.ts +++ b/src/media-sink.ts @@ -28,6 +28,7 @@ import { assert, assertNever, CallSerializer, + clamp, getInt24, getUint24, insertSorted, @@ -38,6 +39,7 @@ import { last, mapAsyncGenerator, promiseWithResolvers, + removeItem, Rotation, toAsyncIterator, toDataView, @@ -54,7 +56,6 @@ import { VideoSample, VideoSamplePixelFormat, } from './sample'; -import { Logging } from './logging'; /** * Additional options for controlling packet retrieval. @@ -886,7 +887,8 @@ class VideoDecoderWrapper extends DecoderWrapper { nullAlphaFrameQueue: number[] = []; currentAlphaPacketIndex = 0; alphaRaslSkipped = false; // For HEVC stuff - frameHandlerSerializer = new CallSerializer(); + finalSamples: { sample: VideoSample | null }[] = []; + mergeAlphaPromises: Promise[] = []; constructor( onSample: (sample: VideoSample) => unknown, @@ -924,17 +926,15 @@ class VideoDecoderWrapper extends DecoderWrapper { .catch(error => onError(error)); } else { const colorHandler = (frame: VideoFrame) => { - this.frameHandlerSerializer.call(async () => { - if (this.alphaQueue.length > 0) { - // Even when no alpha data is present (most of the time), there will be nulls in this queue - const alphaFrame = this.alphaQueue.shift(); - assert(alphaFrame !== undefined); + if (this.alphaQueue.length > 0) { + // Even when no alpha data is present (most of the time), there will be nulls in this queue + const alphaFrame = this.alphaQueue.shift(); + assert(alphaFrame !== undefined); - await this.mergeAlpha(frame, alphaFrame); - } else { - this.colorQueue.push(frame); - } - }).catch(error => this.onError(error)); + void this.mergeAlpha(frame, alphaFrame); + } else { + this.colorQueue.push(frame); + } }; if (codec === 'avc' && this.decoderConfig.description && isChromium()) { @@ -1060,36 +1060,34 @@ class VideoDecoderWrapper extends DecoderWrapper { // Check if we need to set up the alpha decoder if (!this.alphaDecoder) { const alphaHandler = (frame: VideoFrame) => { - this.frameHandlerSerializer.call(async () => { + if (this.colorQueue.length > 0) { + const colorFrame = this.colorQueue.shift(); + assert(colorFrame !== undefined); + + void this.mergeAlpha(colorFrame, frame); + } else { + this.alphaQueue.push(frame); + } + + // Check if any null frames have been queued for this point + this.decodedAlphaChunkCount++; + while ( + this.nullAlphaFrameQueue.length > 0 + && this.nullAlphaFrameQueue[0] === this.decodedAlphaChunkCount + ) { + this.nullAlphaFrameQueue.shift(); + if (this.colorQueue.length > 0) { const colorFrame = this.colorQueue.shift(); assert(colorFrame !== undefined); - await this.mergeAlpha(colorFrame, frame); + void this.mergeAlpha(colorFrame, null); } else { - this.alphaQueue.push(frame); + this.alphaQueue.push(null); } + } - // Check if any null frames have been queued for this point - this.decodedAlphaChunkCount++; - while ( - this.nullAlphaFrameQueue.length > 0 - && this.nullAlphaFrameQueue[0] === this.decodedAlphaChunkCount - ) { - this.nullAlphaFrameQueue.shift(); - - if (this.colorQueue.length > 0) { - const colorFrame = this.colorQueue.shift(); - assert(colorFrame !== undefined); - - await this.mergeAlpha(colorFrame, null); - } else { - this.alphaQueue.push(null); - } - } - - this.alphaDecoderQueueSize--; - }).catch(error => this.onError(error)); + this.alphaDecoderQueueSize--; }; const stack = new Error('Decoding error').stack; @@ -1211,21 +1209,38 @@ class VideoDecoderWrapper extends DecoderWrapper { } async mergeAlpha(color: VideoFrame, alpha: VideoFrame | null) { - if (!alpha) { - // Nothing needs to be merged - const finalSample = new VideoSample(color); - this.sampleHandler(finalSample); + const resolver = promiseWithResolvers(); + this.mergeAlphaPromises.push(resolver.promise); - return; + // Alpha merging is concurrent but the samples must still be emitted in the same order in which the merging + // began. Therefore, serialize the results in an array. + const result: { sample: VideoSample | null } = { sample: null }; + this.finalSamples.push(result); + + try { + if (!alpha) { + // Nothing needs to be merged + result.sample = new VideoSample(color); + } else { + assert(this.merger); + + // The merger takes ownership of the frames, so no need to close them ourselves + const finalFrame = await this.merger.merge(color, alpha); + result.sample = new VideoSample(finalFrame); + } + + // Emit any leading samples that are ready, preserving input order + while (this.finalSamples.length > 0 && this.finalSamples[0]!.sample !== null) { + const next = this.finalSamples.shift()!; + this.sampleHandler(next.sample!); + } + } catch (error) { + removeItem(this.finalSamples, result); + this.onError(error); + } finally { + removeItem(this.mergeAlphaPromises, resolver.promise); + resolver.resolve(); } - - assert(this.merger); - - // The merger takes ownership of the frames, so no need to close them ourselves - const finalFrame = await this.merger.update(color, alpha); - - const finalSample = new VideoSample(finalFrame); - this.sampleHandler(finalSample); } async flush() { @@ -1237,7 +1252,7 @@ class VideoDecoderWrapper extends DecoderWrapper { this.decoder.flush(), this.alphaDecoder?.flush(), ]); - await this.frameHandlerSerializer.currentPromise; + await Promise.all(this.mergeAlphaPromises); this.colorQueue.forEach(x => x.close()); this.colorQueue.length = 0; @@ -1287,246 +1302,73 @@ class VideoDecoderWrapper extends DecoderWrapper { } } -let mergerGpuUnavailable = false; +let mergerWorkerUrl: string | null = null; -/** Utility class that merges together color and alpha information using simple WebGL 2 shaders. */ +/** Utility class that merges together color and alpha information on the CPU in a pool of workers. */ export class ColorAlphaMerger { - static forceCpu = true; - - canvas: OffscreenCanvas | HTMLCanvasElement | null = null; - private gl: WebGL2RenderingContext | null = null; - private program: WebGLProgram | null = null; - private vao: WebGLVertexArrayObject | null = null; - private colorTexture: WebGLTexture | null = null; - private alphaTexture: WebGLTexture | null = null; - - private worker: Worker | null = null; + private workers: Worker[] = []; + private nextWorkerIndex = 0; private pendingRequests = new Map>>(); private nextRequestId = 0; - constructor() { - const canMakeCanvas = typeof OffscreenCanvas !== 'undefined' - // eslint-disable-next-line @typescript-eslint/no-deprecated - || (typeof document !== 'undefined' && typeof document.createElement === 'function'); - - if (!ColorAlphaMerger.forceCpu && canMakeCanvas && !mergerGpuUnavailable) { - // Try the GPU path. If anything goes wrong, we silently fall back to the CPU path. - try { - // Canvas will be resized later - if (typeof OffscreenCanvas !== 'undefined') { - // Prefer OffscreenCanvas for Worker environments - this.canvas = new OffscreenCanvas(300, 150); - } else { - this.canvas = document.createElement('canvas'); - } - - const gl = this.canvas.getContext('webgl2', { - premultipliedAlpha: false, - }) as unknown as WebGL2RenderingContext | null; // Casting because of some TypeScript weirdness - if (!gl) { - throw new Error('Couldn\'t acquire WebGL 2 context.'); - } - - this.gl = gl; - this.program = this.createProgram(); - this.vao = this.createVAO(); - this.colorTexture = this.createTexture(); - this.alphaTexture = this.createTexture(); - - this.gl.useProgram(this.program); - this.gl.uniform1i(this.gl.getUniformLocation(this.program, 'u_colorTexture'), 0); - this.gl.uniform1i(this.gl.getUniformLocation(this.program, 'u_alphaTexture'), 1); - } catch (error) { - this.gl = null; - this.canvas = null; - mergerGpuUnavailable = true; - Logging._warn('Falling back to CPU for color/alpha merging.', error); + merge(color: VideoFrame, alpha: VideoFrame): Promise { + if (this.workers.length === 0) { + if (!mergerWorkerUrl) { + const blob = new Blob( + [`(${colorAlphaMergerWorkerCode.toString()})()`], + { type: 'application/javascript' }, + ); + mergerWorkerUrl = URL.createObjectURL(blob); } - } - } - async update(color: VideoFrame, alpha: VideoFrame): Promise { - if (this.gl) { - return this.updateGpu(color, alpha); - } else { - return this.updateCpu(color, alpha); - } - } + const poolSize = clamp(navigator.hardwareConcurrency, 1, 4); + for (let i = 0; i < poolSize; i++) { + const worker = new Worker(mergerWorkerUrl); - private createProgram(): WebGLProgram { - assert(this.gl); + worker.addEventListener('message', (event: MessageEvent) => { + const data = event.data; + const pending = this.pendingRequests.get(data.id); + if (!pending) { + return; + } + this.pendingRequests.delete(data.id); - const vertexShader = this.createShader(this.gl.VERTEX_SHADER, `#version 300 es - in vec2 a_position; - in vec2 a_texCoord; - out vec2 v_texCoord; - - void main() { - gl_Position = vec4(a_position, 0.0, 1.0); - v_texCoord = a_texCoord; + if ('error' in data) { + pending.reject(new Error(data.error)); + } else { + pending.resolve(data.frame); + } + }); + + worker.addEventListener('error', (event) => { + const error = new Error(event.message || 'Color/alpha merge worker error.'); + for (const pending of this.pendingRequests.values()) { + pending.reject(error); + } + this.pendingRequests.clear(); + }); + + this.workers.push(worker); } - `); - - const fragmentShader = this.createShader(this.gl.FRAGMENT_SHADER, `#version 300 es - precision highp float; - - uniform sampler2D u_colorTexture; - uniform sampler2D u_alphaTexture; - in vec2 v_texCoord; - out vec4 fragColor; - - void main() { - vec3 color = texture(u_colorTexture, v_texCoord).rgb; - float alpha = texture(u_alphaTexture, v_texCoord).r; - fragColor = vec4(color, alpha); - } - `); - - const program = this.gl.createProgram(); - this.gl.attachShader(program, vertexShader); - this.gl.attachShader(program, fragmentShader); - this.gl.linkProgram(program); - - return program; - } - - private createShader(type: number, source: string): WebGLShader { - assert(this.gl); - - const shader = this.gl.createShader(type)!; - this.gl.shaderSource(shader, source); - this.gl.compileShader(shader); - return shader; - } - - private createVAO(): WebGLVertexArrayObject { - assert(this.gl); - assert(this.program); - - const vao = this.gl.createVertexArray(); - this.gl.bindVertexArray(vao); - - const vertices = new Float32Array([ - -1, -1, 0, 1, - 1, -1, 1, 1, - -1, 1, 0, 0, - 1, 1, 1, 0, - ]); - - const buffer = this.gl.createBuffer(); - this.gl.bindBuffer(this.gl.ARRAY_BUFFER, buffer); - this.gl.bufferData(this.gl.ARRAY_BUFFER, vertices, this.gl.STATIC_DRAW); - - const positionLocation = this.gl.getAttribLocation(this.program, 'a_position'); - const texCoordLocation = this.gl.getAttribLocation(this.program, 'a_texCoord'); - - this.gl.enableVertexAttribArray(positionLocation); - this.gl.vertexAttribPointer(positionLocation, 2, this.gl.FLOAT, false, 16, 0); - - this.gl.enableVertexAttribArray(texCoordLocation); - this.gl.vertexAttribPointer(texCoordLocation, 2, this.gl.FLOAT, false, 16, 8); - - return vao; - } - - private createTexture(): WebGLTexture { - assert(this.gl); - - const texture = this.gl.createTexture(); - - this.gl.bindTexture(this.gl.TEXTURE_2D, texture); - this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_S, this.gl.CLAMP_TO_EDGE); - this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_T, this.gl.CLAMP_TO_EDGE); - this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MIN_FILTER, this.gl.LINEAR); - this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MAG_FILTER, this.gl.LINEAR); - - return texture; - } - - private updateGpu(color: VideoFrame, alpha: VideoFrame): VideoFrame { - assert(this.gl); - assert(this.canvas); - - if (color.displayWidth !== this.canvas.width || color.displayHeight !== this.canvas.height) { - this.canvas.width = color.displayWidth; - this.canvas.height = color.displayHeight; - } - - this.gl.activeTexture(this.gl.TEXTURE0); - this.gl.bindTexture(this.gl.TEXTURE_2D, this.colorTexture); - this.gl.texImage2D(this.gl.TEXTURE_2D, 0, this.gl.RGBA, this.gl.RGBA, this.gl.UNSIGNED_BYTE, color); - - this.gl.activeTexture(this.gl.TEXTURE1); - this.gl.bindTexture(this.gl.TEXTURE_2D, this.alphaTexture); - this.gl.texImage2D(this.gl.TEXTURE_2D, 0, this.gl.RGBA, this.gl.RGBA, this.gl.UNSIGNED_BYTE, alpha); - - this.gl.viewport(0, 0, this.canvas.width, this.canvas.height); - this.gl.clear(this.gl.COLOR_BUFFER_BIT); - - this.gl.bindVertexArray(this.vao); - this.gl.drawArrays(this.gl.TRIANGLE_STRIP, 0, 4); - - const finalFrame = new VideoFrame(this.canvas, { - timestamp: color.timestamp, - duration: color.duration ?? undefined, - }); - - color.close(); - alpha.close(); - - return finalFrame; - } - - private updateCpu(color: VideoFrame, alpha: VideoFrame): Promise { - if (!this.worker) { - const blob = new Blob( - [`(${colorAlphaMergerWorkerCode.toString()})()`], - { type: 'application/javascript' }, - ); - const url = URL.createObjectURL(blob); - this.worker = new Worker(url); - URL.revokeObjectURL(url); - - this.worker.addEventListener('message', (event: MessageEvent) => { - const data = event.data; - const pending = this.pendingRequests.get(data.id); - if (!pending) { - return; - } - this.pendingRequests.delete(data.id); - - if ('error' in data) { - pending.reject(new Error(data.error)); - } else { - pending.resolve(data.frame); - } - }); - - this.worker.addEventListener('error', (event) => { - const error = new Error(event.message || 'Color/alpha merge worker error.'); - for (const pending of this.pendingRequests.values()) { - pending.reject(error); - } - this.pendingRequests.clear(); - }); } const id = this.nextRequestId++; const pending = promiseWithResolvers(); this.pendingRequests.set(id, pending); - this.worker.postMessage({ id, color, alpha }, { transfer: [color, alpha] }); + // Hand the job to the next worker in round-robin fashion + const worker = this.workers[this.nextWorkerIndex]!; + this.nextWorkerIndex = (this.nextWorkerIndex + 1) % this.workers.length; + worker.postMessage({ id, color, alpha }, { transfer: [color, alpha] }); return pending.promise; } close() { - this.gl?.getExtension('WEBGL_lose_context')?.loseContext(); - this.gl = null; - this.canvas = null; - - this.worker?.terminate(); - this.worker = null; + for (const worker of this.workers) { + worker.terminate(); + } + this.workers.length = 0; const error = new Error('Color/alpha merger closed.'); for (const pending of this.pendingRequests.values()) { diff --git a/src/media-source.ts b/src/media-source.ts index d375b0a..223a7f7 100644 --- a/src/media-source.ts +++ b/src/media-source.ts @@ -559,15 +559,12 @@ class VideoEncoderWrapper { this.encoder.encode(videoFrame, finalEncodeOptions); videoFrame.close(); } else { - const width = videoFrame.displayWidth; - const height = videoFrame.displayHeight; - if (!this.splitter) { - this.splitter = new ColorAlphaSplitter(width, height); + this.splitter = new ColorAlphaSplitter(); } // The splitter takes ownership, so no need to close the frames ourselves - const { colorFrame, alphaFrame } = await this.splitter.update(videoFrame); + const { colorFrame, alphaFrame } = await this.splitter.split(videoFrame); this.alphaFrameQueue.push(alphaFrame); this.encoder.encode(colorFrame, finalEncodeOptions); @@ -917,21 +914,10 @@ class VideoEncoderWrapper { } } -let splitterGpuUnavailable = false; +let splitterWorkerUrl: string | null = null; -/** Utility class for splitting a composite frame into separate color and alpha components. */ +/** Utility class for splitting a composite frame into separate color and alpha parts on the CPU in a worker. */ export class ColorAlphaSplitter { - static forceCpu = true; - - canvas: OffscreenCanvas | HTMLCanvasElement | null = null; - - private gl: WebGL2RenderingContext | null = null; - private colorProgram: WebGLProgram | null = null; - private alphaProgram: WebGLProgram | null = null; - private vao: WebGLVertexArrayObject | null = null; - private sourceTexture: WebGLTexture | null = null; - private alphaResolutionLocation: WebGLUniformLocation | null = null; - private worker: Worker | null = null; private pendingRequests = new Map< number, @@ -940,308 +926,17 @@ export class ColorAlphaSplitter { private nextRequestId = 0; - constructor(initialWidth: number, initialHeight: number) { - const canMakeCanvas = typeof OffscreenCanvas !== 'undefined' - // eslint-disable-next-line @typescript-eslint/no-deprecated - || (typeof document !== 'undefined' && typeof document.createElement === 'function'); - - if (!ColorAlphaSplitter.forceCpu && canMakeCanvas && !splitterGpuUnavailable) { - // Try the GPU path. If anything goes wrong, we silently fall back to the CPU path. - try { - if (typeof OffscreenCanvas !== 'undefined') { - this.canvas = new OffscreenCanvas(initialWidth, initialHeight); - } else { - this.canvas = document.createElement('canvas'); - this.canvas.width = initialWidth; - this.canvas.height = initialHeight; - } - - const gl = this.canvas.getContext('webgl2', { - alpha: true, // Needed due to the YUV thing we do for alpha - }) as unknown as WebGL2RenderingContext | null; // Casting because of some TypeScript weirdness - if (!gl) { - throw new Error('Couldn\'t acquire WebGL 2 context.'); - } - - this.gl = gl; - - this.colorProgram = this.createColorProgram(); - this.alphaProgram = this.createAlphaProgram(); - this.vao = this.createVAO(); - this.sourceTexture = this.createTexture(); - - this.alphaResolutionLocation = this.gl.getUniformLocation(this.alphaProgram, 'u_resolution')!; - - this.gl.useProgram(this.colorProgram); - this.gl.uniform1i(this.gl.getUniformLocation(this.colorProgram, 'u_sourceTexture'), 0); - - this.gl.useProgram(this.alphaProgram); - this.gl.uniform1i(this.gl.getUniformLocation(this.alphaProgram, 'u_sourceTexture'), 0); - } catch (error) { - this.gl = null; - this.canvas = null; - splitterGpuUnavailable = true; - Logging._warn('Falling back to CPU for color/alpha splitting.', error); - } - } - } - - async update(sourceFrame: VideoFrame) { - if (this.gl) { - return this.updateGpu(sourceFrame); - } else { - return this.updateCpu(sourceFrame); - } - } - - private updateGpu(sourceFrame: VideoFrame) { - assert(this.gl); - assert(this.canvas); - - if (sourceFrame.displayWidth !== this.canvas.width || sourceFrame.displayHeight !== this.canvas.height) { - this.canvas.width = sourceFrame.displayWidth; - this.canvas.height = sourceFrame.displayHeight; - } - - this.gl.activeTexture(this.gl.TEXTURE0); - this.gl.bindTexture(this.gl.TEXTURE_2D, this.sourceTexture); - this.gl.texImage2D(this.gl.TEXTURE_2D, 0, this.gl.RGBA, this.gl.RGBA, this.gl.UNSIGNED_BYTE, sourceFrame); - - const colorFrame = this.runColorProgram(sourceFrame); - const alphaFrame = this.runAlphaProgram(sourceFrame); - - sourceFrame.close(); - - return { colorFrame, alphaFrame }; - } - - private createVertexShader(): WebGLShader { - assert(this.gl); - - return this.createShader(this.gl.VERTEX_SHADER, `#version 300 es - in vec2 a_position; - in vec2 a_texCoord; - out vec2 v_texCoord; - - void main() { - gl_Position = vec4(a_position, 0.0, 1.0); - v_texCoord = a_texCoord; - } - `); - } - - private createColorProgram(): WebGLProgram { - assert(this.gl); - - const vertexShader = this.createVertexShader(); - - // This shader is simple, simply copy the color information while setting alpha to 1 - const fragmentShader = this.createShader(this.gl.FRAGMENT_SHADER, `#version 300 es - precision highp float; - - uniform sampler2D u_sourceTexture; - in vec2 v_texCoord; - out vec4 fragColor; - - void main() { - vec4 source = texture(u_sourceTexture, v_texCoord); - fragColor = vec4(source.rgb, 1.0); - } - `); - - const program = this.gl.createProgram(); - this.gl.attachShader(program, vertexShader); - this.gl.attachShader(program, fragmentShader); - this.gl.linkProgram(program); - - return program; - } - - private createAlphaProgram(): WebGLProgram { - assert(this.gl); - - const vertexShader = this.createVertexShader(); - - // This shader's more complex. The main reason is that this shader writes data in I420 (yuv420) pixel format - // instead of regular RGBA. In other words, we use the shader to write out I420 data into an RGBA canvas, which - // we then later read out with JavaScript. The reason being that browsers weirdly encode canvases and mess up - // the color spaces, and the only way to have full control over the color space is by outputting YUV data - // directly (avoiding the RGB conversion). Doing this conversion in JS is painfully slow, so let's utlize the - // GPU since we're already calling it anyway. - const fragmentShader = this.createShader(this.gl.FRAGMENT_SHADER, `#version 300 es - precision highp float; - - uniform sampler2D u_sourceTexture; - uniform vec2 u_resolution; // The width and height of the canvas - in vec2 v_texCoord; - out vec4 fragColor; - - // This function determines the value for a single byte in the YUV stream - float getByteValue(float byteOffset) { - float width = u_resolution.x; - float height = u_resolution.y; - - float yPlaneSize = width * height; - - if (byteOffset < yPlaneSize) { - // This byte is in the luma plane. Find the corresponding pixel coordinates to sample from - float y = floor(byteOffset / width); - float x = mod(byteOffset, width); - - // Add 0.5 to sample the center of the texel - vec2 sampleCoord = (vec2(x, y) + 0.5) / u_resolution; - - // The luma value is the alpha from the source texture - return texture(u_sourceTexture, sampleCoord).a; - } else { - // Write a fixed value for chroma and beyond - return 128.0 / 255.0; - } - } - - void main() { - // Each fragment writes 4 bytes (R, G, B, A) - float pixelIndex = floor(gl_FragCoord.y) * u_resolution.x + floor(gl_FragCoord.x); - float baseByteOffset = pixelIndex * 4.0; - - vec4 result; - for (int i = 0; i < 4; i++) { - float currentByteOffset = baseByteOffset + float(i); - result[i] = getByteValue(currentByteOffset); - } - - fragColor = result; - } - `); - - const program = this.gl.createProgram(); - this.gl.attachShader(program, vertexShader); - this.gl.attachShader(program, fragmentShader); - this.gl.linkProgram(program); - - return program; - } - - private createShader(type: number, source: string): WebGLShader { - assert(this.gl); - - const shader = this.gl.createShader(type)!; - this.gl.shaderSource(shader, source); - this.gl.compileShader(shader); - if (!this.gl.getShaderParameter(shader, this.gl.COMPILE_STATUS)) { - Logging._error('Shader compile error:', this.gl.getShaderInfoLog(shader)); - } - return shader; - } - - private createVAO(): WebGLVertexArrayObject { - assert(this.gl); - assert(this.colorProgram); - - const vao = this.gl.createVertexArray(); - this.gl.bindVertexArray(vao); - - const vertices = new Float32Array([ - -1, -1, 0, 1, - 1, -1, 1, 1, - -1, 1, 0, 0, - 1, 1, 1, 0, - ]); - - const buffer = this.gl.createBuffer(); - this.gl.bindBuffer(this.gl.ARRAY_BUFFER, buffer); - this.gl.bufferData(this.gl.ARRAY_BUFFER, vertices, this.gl.STATIC_DRAW); - - const positionLocation = this.gl.getAttribLocation(this.colorProgram, 'a_position'); - const texCoordLocation = this.gl.getAttribLocation(this.colorProgram, 'a_texCoord'); - - this.gl.enableVertexAttribArray(positionLocation); - this.gl.vertexAttribPointer(positionLocation, 2, this.gl.FLOAT, false, 16, 0); - - this.gl.enableVertexAttribArray(texCoordLocation); - this.gl.vertexAttribPointer(texCoordLocation, 2, this.gl.FLOAT, false, 16, 8); - - return vao; - } - - private createTexture(): WebGLTexture { - assert(this.gl); - - const texture = this.gl.createTexture(); - - this.gl.bindTexture(this.gl.TEXTURE_2D, texture); - this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_S, this.gl.CLAMP_TO_EDGE); - this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_T, this.gl.CLAMP_TO_EDGE); - this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MIN_FILTER, this.gl.LINEAR); - this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MAG_FILTER, this.gl.LINEAR); - - return texture; - } - - private runColorProgram(sourceFrame: VideoFrame) { - assert(this.gl); - assert(this.canvas); - - this.gl.useProgram(this.colorProgram); - this.gl.viewport(0, 0, this.canvas.width, this.canvas.height); - this.gl.clear(this.gl.COLOR_BUFFER_BIT); - this.gl.bindVertexArray(this.vao); - this.gl.drawArrays(this.gl.TRIANGLE_STRIP, 0, 4); - - return new VideoFrame(this.canvas, { - timestamp: sourceFrame.timestamp, - duration: sourceFrame.duration ?? undefined, - alpha: 'discard', - }); - } - - private runAlphaProgram(sourceFrame: VideoFrame) { - assert(this.gl); - assert(this.canvas); - - this.gl.useProgram(this.alphaProgram); - this.gl.uniform2f(this.alphaResolutionLocation, this.canvas.width, this.canvas.height); - - this.gl.viewport(0, 0, this.canvas.width, this.canvas.height); - this.gl.clear(this.gl.COLOR_BUFFER_BIT); - this.gl.bindVertexArray(this.vao); - this.gl.drawArrays(this.gl.TRIANGLE_STRIP, 0, 4); - - const { width, height } = this.canvas; - - const chromaSamples = Math.ceil(width / 2) * Math.ceil(height / 2); - const yuvSize = width * height + chromaSamples * 2; - const requiredHeight = Math.ceil(yuvSize / (width * 4)); - - let yuv = new Uint8Array(4 * width * requiredHeight); - this.gl.readPixels(0, 0, width, requiredHeight, this.gl.RGBA, this.gl.UNSIGNED_BYTE, yuv); - yuv = yuv.subarray(0, yuvSize); - - assert(yuv[width * height] === 128); // Where chroma data starts - assert(yuv[yuv.length - 1] === 128); // Assert the YUV data has been fully written - - // Defining this separately because TypeScript doesn't know `transfer` and I can't be bothered to do declaration - // merging right now - const init = { - format: 'I420' as const, - codedWidth: width, - codedHeight: height, - timestamp: sourceFrame.timestamp, - duration: sourceFrame.duration ?? undefined, - transfer: [yuv.buffer], - }; - return new VideoFrame(yuv, init); - } - - private updateCpu(sourceFrame: VideoFrame): Promise<{ colorFrame: VideoFrame; alphaFrame: VideoFrame }> { + split(sourceFrame: VideoFrame): Promise<{ colorFrame: VideoFrame; alphaFrame: VideoFrame }> { if (!this.worker) { - const blob = new Blob( - [`(${colorAlphaSplitterWorkerCode.toString()})()`], - { type: 'application/javascript' }, - ); - const url = URL.createObjectURL(blob); - this.worker = new Worker(url); - URL.revokeObjectURL(url); + if (!splitterWorkerUrl) { + const blob = new Blob( + [`(${colorAlphaSplitterWorkerCode.toString()})()`], + { type: 'application/javascript' }, + ); + splitterWorkerUrl = URL.createObjectURL(blob); + } + + this.worker = new Worker(splitterWorkerUrl); this.worker.addEventListener('message', (event: MessageEvent) => { const data = event.data; @@ -1271,16 +966,14 @@ export class ColorAlphaSplitter { const pending = promiseWithResolvers<{ colorFrame: VideoFrame; alphaFrame: VideoFrame }>(); this.pendingRequests.set(id, pending); this.worker.postMessage({ id, sourceFrame }, { transfer: [sourceFrame] }); + return pending.promise; } close() { - this.gl?.getExtension('WEBGL_lose_context')?.loseContext(); - this.gl = null; - this.canvas = null; - this.worker?.terminate(); this.worker = null; + const error = new Error('Color/alpha splitter closed.'); for (const pending of this.pendingRequests.values()) { pending.reject(error); diff --git a/test/browser/transparency.test.ts b/test/browser/transparency.test.ts index 6ca9905..3c0ebc3 100644 --- a/test/browser/transparency.test.ts +++ b/test/browser/transparency.test.ts @@ -2,11 +2,11 @@ import { expect, test } from 'vitest'; import { Input } from '../../src/input.js'; import { BufferSource, UrlSource } from '../../src/source.js'; import { ALL_FORMATS } from '../../src/input-format.js'; -import { CanvasSink, ColorAlphaMerger, EncodedPacketSink, VideoSampleSink } from '../../src/media-sink.js'; +import { CanvasSink, EncodedPacketSink, VideoSampleSink } from '../../src/media-sink.js'; import { Output } from '../../src/output.js'; import { WebMOutputFormat } from '../../src/output-format.js'; import { BufferTarget } from '../../src/target.js'; -import { CanvasSource, ColorAlphaSplitter, VideoSampleSource } from '../../src/media-source.js'; +import { CanvasSource, VideoSampleSource } from '../../src/media-source.js'; import { canEncodeVideo, QUALITY_HIGH } from '../../src/encode.js'; import { VideoSample } from '../../src/sample.js'; import { Conversion } from '../../src/conversion.js'; @@ -39,15 +39,6 @@ test('Can decode transparent video', async () => { await decodeTransparentVideoTest(); }); -test('Can decode transparent video, forced CPU path', async () => { - try { - ColorAlphaMerger.forceCpu = true; - await decodeTransparentVideoTest(); - } finally { - ColorAlphaMerger.forceCpu = false; - } -}); - test('Can decode faulty transparent video and behaves gracefully', async () => { using input = new Input({ source: new UrlSource('/transparency-faulty.webm'), @@ -192,15 +183,6 @@ test('Can encode transparent video', async () => { await encodeTransparentVideoTest(); }); -test('Can encode transparent video, forced CPU path', async () => { - try { - ColorAlphaSplitter.forceCpu = true; - await encodeTransparentVideoTest(); - } finally { - ColorAlphaSplitter.forceCpu = false; - } -}); - test('Can encode video with alternating transparency', async () => { // This test is already brutal when it comes to finding async race conditions, so run it thrice to be MORE brutal for (let j = 0; j < 3; j++) {