mirror of
https://github.com/arcodange-org/mediabunny.git
synced 2026-10-03 05:43:50 +02:00
Remove GPU path for alpha merger/splitter, add worker pool for merger, allow concurrent merging for better performance
This commit is contained in:
Vanilagy
+60
-218
@@ -28,6 +28,7 @@ import {
|
||||
assert,
|
||||
assertNever,
|
||||
CallSerializer,
|
||||
clamp,
|
||||
getInt24,
|
||||
getUint24,
|
||||
insertSorted,
|
||||
@@ -38,6 +39,7 @@ import {
|
||||
last,
|
||||
mapAsyncGenerator,
|
||||
promiseWithResolvers,
|
||||
removeItem,
|
||||
Rotation,
|
||||
toAsyncIterator,
|
||||
toDataView,
|
||||
@@ -54,7 +56,6 @@ import {
|
||||
VideoSample,
|
||||
VideoSamplePixelFormat,
|
||||
} from './sample';
|
||||
import { Logging } from './logging';
|
||||
|
||||
/**
|
||||
* Additional options for controlling packet retrieval.
|
||||
@@ -886,7 +887,8 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
|
||||
nullAlphaFrameQueue: number[] = [];
|
||||
currentAlphaPacketIndex = 0;
|
||||
alphaRaslSkipped = false; // For HEVC stuff
|
||||
frameHandlerSerializer = new CallSerializer();
|
||||
finalSamples: { sample: VideoSample | null }[] = [];
|
||||
mergeAlphaPromises: Promise<void>[] = [];
|
||||
|
||||
constructor(
|
||||
onSample: (sample: VideoSample) => unknown,
|
||||
@@ -924,17 +926,15 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
|
||||
.catch(error => onError(error));
|
||||
} else {
|
||||
const colorHandler = (frame: VideoFrame) => {
|
||||
this.frameHandlerSerializer.call(async () => {
|
||||
if (this.alphaQueue.length > 0) {
|
||||
// Even when no alpha data is present (most of the time), there will be nulls in this queue
|
||||
const alphaFrame = this.alphaQueue.shift();
|
||||
assert(alphaFrame !== undefined);
|
||||
|
||||
await this.mergeAlpha(frame, alphaFrame);
|
||||
void this.mergeAlpha(frame, alphaFrame);
|
||||
} else {
|
||||
this.colorQueue.push(frame);
|
||||
}
|
||||
}).catch(error => this.onError(error));
|
||||
};
|
||||
|
||||
if (codec === 'avc' && this.decoderConfig.description && isChromium()) {
|
||||
@@ -1060,12 +1060,11 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
|
||||
// Check if we need to set up the alpha decoder
|
||||
if (!this.alphaDecoder) {
|
||||
const alphaHandler = (frame: VideoFrame) => {
|
||||
this.frameHandlerSerializer.call(async () => {
|
||||
if (this.colorQueue.length > 0) {
|
||||
const colorFrame = this.colorQueue.shift();
|
||||
assert(colorFrame !== undefined);
|
||||
|
||||
await this.mergeAlpha(colorFrame, frame);
|
||||
void this.mergeAlpha(colorFrame, frame);
|
||||
} else {
|
||||
this.alphaQueue.push(frame);
|
||||
}
|
||||
@@ -1082,14 +1081,13 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
|
||||
const colorFrame = this.colorQueue.shift();
|
||||
assert(colorFrame !== undefined);
|
||||
|
||||
await this.mergeAlpha(colorFrame, null);
|
||||
void this.mergeAlpha(colorFrame, null);
|
||||
} else {
|
||||
this.alphaQueue.push(null);
|
||||
}
|
||||
}
|
||||
|
||||
this.alphaDecoderQueueSize--;
|
||||
}).catch(error => this.onError(error));
|
||||
};
|
||||
|
||||
const stack = new Error('Decoding error').stack;
|
||||
@@ -1211,21 +1209,38 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
|
||||
}
|
||||
|
||||
async mergeAlpha(color: VideoFrame, alpha: VideoFrame | null) {
|
||||
const resolver = promiseWithResolvers();
|
||||
this.mergeAlphaPromises.push(resolver.promise);
|
||||
|
||||
// Alpha merging is concurrent but the samples must still be emitted in the same order in which the merging
|
||||
// began. Therefore, serialize the results in an array.
|
||||
const result: { sample: VideoSample | null } = { sample: null };
|
||||
this.finalSamples.push(result);
|
||||
|
||||
try {
|
||||
if (!alpha) {
|
||||
// Nothing needs to be merged
|
||||
const finalSample = new VideoSample(color);
|
||||
this.sampleHandler(finalSample);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
result.sample = new VideoSample(color);
|
||||
} else {
|
||||
assert(this.merger);
|
||||
|
||||
// The merger takes ownership of the frames, so no need to close them ourselves
|
||||
const finalFrame = await this.merger.update(color, alpha);
|
||||
const finalFrame = await this.merger.merge(color, alpha);
|
||||
result.sample = new VideoSample(finalFrame);
|
||||
}
|
||||
|
||||
const finalSample = new VideoSample(finalFrame);
|
||||
this.sampleHandler(finalSample);
|
||||
// Emit any leading samples that are ready, preserving input order
|
||||
while (this.finalSamples.length > 0 && this.finalSamples[0]!.sample !== null) {
|
||||
const next = this.finalSamples.shift()!;
|
||||
this.sampleHandler(next.sample!);
|
||||
}
|
||||
} catch (error) {
|
||||
removeItem(this.finalSamples, result);
|
||||
this.onError(error);
|
||||
} finally {
|
||||
removeItem(this.mergeAlphaPromises, resolver.promise);
|
||||
resolver.resolve();
|
||||
}
|
||||
}
|
||||
|
||||
async flush() {
|
||||
@@ -1237,7 +1252,7 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
|
||||
this.decoder.flush(),
|
||||
this.alphaDecoder?.flush(),
|
||||
]);
|
||||
await this.frameHandlerSerializer.currentPromise;
|
||||
await Promise.all(this.mergeAlphaPromises);
|
||||
|
||||
this.colorQueue.forEach(x => x.close());
|
||||
this.colorQueue.length = 0;
|
||||
@@ -1287,207 +1302,30 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
|
||||
}
|
||||
}
|
||||
|
||||
let mergerGpuUnavailable = false;
|
||||
let mergerWorkerUrl: string | null = null;
|
||||
|
||||
/** Utility class that merges together color and alpha information using simple WebGL 2 shaders. */
|
||||
/** Utility class that merges together color and alpha information on the CPU in a pool of workers. */
|
||||
export class ColorAlphaMerger {
|
||||
static forceCpu = true;
|
||||
|
||||
canvas: OffscreenCanvas | HTMLCanvasElement | null = null;
|
||||
private gl: WebGL2RenderingContext | null = null;
|
||||
private program: WebGLProgram | null = null;
|
||||
private vao: WebGLVertexArrayObject | null = null;
|
||||
private colorTexture: WebGLTexture | null = null;
|
||||
private alphaTexture: WebGLTexture | null = null;
|
||||
|
||||
private worker: Worker | null = null;
|
||||
private workers: Worker[] = [];
|
||||
private nextWorkerIndex = 0;
|
||||
private pendingRequests = new Map<number, ReturnType<typeof promiseWithResolvers<VideoFrame>>>();
|
||||
private nextRequestId = 0;
|
||||
|
||||
constructor() {
|
||||
const canMakeCanvas = typeof OffscreenCanvas !== 'undefined'
|
||||
// eslint-disable-next-line @typescript-eslint/no-deprecated
|
||||
|| (typeof document !== 'undefined' && typeof document.createElement === 'function');
|
||||
|
||||
if (!ColorAlphaMerger.forceCpu && canMakeCanvas && !mergerGpuUnavailable) {
|
||||
// Try the GPU path. If anything goes wrong, we silently fall back to the CPU path.
|
||||
try {
|
||||
// Canvas will be resized later
|
||||
if (typeof OffscreenCanvas !== 'undefined') {
|
||||
// Prefer OffscreenCanvas for Worker environments
|
||||
this.canvas = new OffscreenCanvas(300, 150);
|
||||
} else {
|
||||
this.canvas = document.createElement('canvas');
|
||||
}
|
||||
|
||||
const gl = this.canvas.getContext('webgl2', {
|
||||
premultipliedAlpha: false,
|
||||
}) as unknown as WebGL2RenderingContext | null; // Casting because of some TypeScript weirdness
|
||||
if (!gl) {
|
||||
throw new Error('Couldn\'t acquire WebGL 2 context.');
|
||||
}
|
||||
|
||||
this.gl = gl;
|
||||
this.program = this.createProgram();
|
||||
this.vao = this.createVAO();
|
||||
this.colorTexture = this.createTexture();
|
||||
this.alphaTexture = this.createTexture();
|
||||
|
||||
this.gl.useProgram(this.program);
|
||||
this.gl.uniform1i(this.gl.getUniformLocation(this.program, 'u_colorTexture'), 0);
|
||||
this.gl.uniform1i(this.gl.getUniformLocation(this.program, 'u_alphaTexture'), 1);
|
||||
} catch (error) {
|
||||
this.gl = null;
|
||||
this.canvas = null;
|
||||
mergerGpuUnavailable = true;
|
||||
Logging._warn('Falling back to CPU for color/alpha merging.', error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async update(color: VideoFrame, alpha: VideoFrame): Promise<VideoFrame> {
|
||||
if (this.gl) {
|
||||
return this.updateGpu(color, alpha);
|
||||
} else {
|
||||
return this.updateCpu(color, alpha);
|
||||
}
|
||||
}
|
||||
|
||||
private createProgram(): WebGLProgram {
|
||||
assert(this.gl);
|
||||
|
||||
const vertexShader = this.createShader(this.gl.VERTEX_SHADER, `#version 300 es
|
||||
in vec2 a_position;
|
||||
in vec2 a_texCoord;
|
||||
out vec2 v_texCoord;
|
||||
|
||||
void main() {
|
||||
gl_Position = vec4(a_position, 0.0, 1.0);
|
||||
v_texCoord = a_texCoord;
|
||||
}
|
||||
`);
|
||||
|
||||
const fragmentShader = this.createShader(this.gl.FRAGMENT_SHADER, `#version 300 es
|
||||
precision highp float;
|
||||
|
||||
uniform sampler2D u_colorTexture;
|
||||
uniform sampler2D u_alphaTexture;
|
||||
in vec2 v_texCoord;
|
||||
out vec4 fragColor;
|
||||
|
||||
void main() {
|
||||
vec3 color = texture(u_colorTexture, v_texCoord).rgb;
|
||||
float alpha = texture(u_alphaTexture, v_texCoord).r;
|
||||
fragColor = vec4(color, alpha);
|
||||
}
|
||||
`);
|
||||
|
||||
const program = this.gl.createProgram();
|
||||
this.gl.attachShader(program, vertexShader);
|
||||
this.gl.attachShader(program, fragmentShader);
|
||||
this.gl.linkProgram(program);
|
||||
|
||||
return program;
|
||||
}
|
||||
|
||||
private createShader(type: number, source: string): WebGLShader {
|
||||
assert(this.gl);
|
||||
|
||||
const shader = this.gl.createShader(type)!;
|
||||
this.gl.shaderSource(shader, source);
|
||||
this.gl.compileShader(shader);
|
||||
return shader;
|
||||
}
|
||||
|
||||
private createVAO(): WebGLVertexArrayObject {
|
||||
assert(this.gl);
|
||||
assert(this.program);
|
||||
|
||||
const vao = this.gl.createVertexArray();
|
||||
this.gl.bindVertexArray(vao);
|
||||
|
||||
const vertices = new Float32Array([
|
||||
-1, -1, 0, 1,
|
||||
1, -1, 1, 1,
|
||||
-1, 1, 0, 0,
|
||||
1, 1, 1, 0,
|
||||
]);
|
||||
|
||||
const buffer = this.gl.createBuffer();
|
||||
this.gl.bindBuffer(this.gl.ARRAY_BUFFER, buffer);
|
||||
this.gl.bufferData(this.gl.ARRAY_BUFFER, vertices, this.gl.STATIC_DRAW);
|
||||
|
||||
const positionLocation = this.gl.getAttribLocation(this.program, 'a_position');
|
||||
const texCoordLocation = this.gl.getAttribLocation(this.program, 'a_texCoord');
|
||||
|
||||
this.gl.enableVertexAttribArray(positionLocation);
|
||||
this.gl.vertexAttribPointer(positionLocation, 2, this.gl.FLOAT, false, 16, 0);
|
||||
|
||||
this.gl.enableVertexAttribArray(texCoordLocation);
|
||||
this.gl.vertexAttribPointer(texCoordLocation, 2, this.gl.FLOAT, false, 16, 8);
|
||||
|
||||
return vao;
|
||||
}
|
||||
|
||||
private createTexture(): WebGLTexture {
|
||||
assert(this.gl);
|
||||
|
||||
const texture = this.gl.createTexture();
|
||||
|
||||
this.gl.bindTexture(this.gl.TEXTURE_2D, texture);
|
||||
this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_S, this.gl.CLAMP_TO_EDGE);
|
||||
this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_T, this.gl.CLAMP_TO_EDGE);
|
||||
this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MIN_FILTER, this.gl.LINEAR);
|
||||
this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MAG_FILTER, this.gl.LINEAR);
|
||||
|
||||
return texture;
|
||||
}
|
||||
|
||||
private updateGpu(color: VideoFrame, alpha: VideoFrame): VideoFrame {
|
||||
assert(this.gl);
|
||||
assert(this.canvas);
|
||||
|
||||
if (color.displayWidth !== this.canvas.width || color.displayHeight !== this.canvas.height) {
|
||||
this.canvas.width = color.displayWidth;
|
||||
this.canvas.height = color.displayHeight;
|
||||
}
|
||||
|
||||
this.gl.activeTexture(this.gl.TEXTURE0);
|
||||
this.gl.bindTexture(this.gl.TEXTURE_2D, this.colorTexture);
|
||||
this.gl.texImage2D(this.gl.TEXTURE_2D, 0, this.gl.RGBA, this.gl.RGBA, this.gl.UNSIGNED_BYTE, color);
|
||||
|
||||
this.gl.activeTexture(this.gl.TEXTURE1);
|
||||
this.gl.bindTexture(this.gl.TEXTURE_2D, this.alphaTexture);
|
||||
this.gl.texImage2D(this.gl.TEXTURE_2D, 0, this.gl.RGBA, this.gl.RGBA, this.gl.UNSIGNED_BYTE, alpha);
|
||||
|
||||
this.gl.viewport(0, 0, this.canvas.width, this.canvas.height);
|
||||
this.gl.clear(this.gl.COLOR_BUFFER_BIT);
|
||||
|
||||
this.gl.bindVertexArray(this.vao);
|
||||
this.gl.drawArrays(this.gl.TRIANGLE_STRIP, 0, 4);
|
||||
|
||||
const finalFrame = new VideoFrame(this.canvas, {
|
||||
timestamp: color.timestamp,
|
||||
duration: color.duration ?? undefined,
|
||||
});
|
||||
|
||||
color.close();
|
||||
alpha.close();
|
||||
|
||||
return finalFrame;
|
||||
}
|
||||
|
||||
private updateCpu(color: VideoFrame, alpha: VideoFrame): Promise<VideoFrame> {
|
||||
if (!this.worker) {
|
||||
merge(color: VideoFrame, alpha: VideoFrame): Promise<VideoFrame> {
|
||||
if (this.workers.length === 0) {
|
||||
if (!mergerWorkerUrl) {
|
||||
const blob = new Blob(
|
||||
[`(${colorAlphaMergerWorkerCode.toString()})()`],
|
||||
{ type: 'application/javascript' },
|
||||
);
|
||||
const url = URL.createObjectURL(blob);
|
||||
this.worker = new Worker(url);
|
||||
URL.revokeObjectURL(url);
|
||||
mergerWorkerUrl = URL.createObjectURL(blob);
|
||||
}
|
||||
|
||||
this.worker.addEventListener('message', (event: MessageEvent<ColorAlphaMergerWorkerResponse>) => {
|
||||
const poolSize = clamp(navigator.hardwareConcurrency, 1, 4);
|
||||
for (let i = 0; i < poolSize; i++) {
|
||||
const worker = new Worker(mergerWorkerUrl);
|
||||
|
||||
worker.addEventListener('message', (event: MessageEvent<ColorAlphaMergerWorkerResponse>) => {
|
||||
const data = event.data;
|
||||
const pending = this.pendingRequests.get(data.id);
|
||||
if (!pending) {
|
||||
@@ -1502,31 +1340,35 @@ export class ColorAlphaMerger {
|
||||
}
|
||||
});
|
||||
|
||||
this.worker.addEventListener('error', (event) => {
|
||||
worker.addEventListener('error', (event) => {
|
||||
const error = new Error(event.message || 'Color/alpha merge worker error.');
|
||||
for (const pending of this.pendingRequests.values()) {
|
||||
pending.reject(error);
|
||||
}
|
||||
this.pendingRequests.clear();
|
||||
});
|
||||
|
||||
this.workers.push(worker);
|
||||
}
|
||||
}
|
||||
|
||||
const id = this.nextRequestId++;
|
||||
const pending = promiseWithResolvers<VideoFrame>();
|
||||
this.pendingRequests.set(id, pending);
|
||||
|
||||
this.worker.postMessage({ id, color, alpha }, { transfer: [color, alpha] });
|
||||
// Hand the job to the next worker in round-robin fashion
|
||||
const worker = this.workers[this.nextWorkerIndex]!;
|
||||
this.nextWorkerIndex = (this.nextWorkerIndex + 1) % this.workers.length;
|
||||
worker.postMessage({ id, color, alpha }, { transfer: [color, alpha] });
|
||||
|
||||
return pending.promise;
|
||||
}
|
||||
|
||||
close() {
|
||||
this.gl?.getExtension('WEBGL_lose_context')?.loseContext();
|
||||
this.gl = null;
|
||||
this.canvas = null;
|
||||
|
||||
this.worker?.terminate();
|
||||
this.worker = null;
|
||||
for (const worker of this.workers) {
|
||||
worker.terminate();
|
||||
}
|
||||
this.workers.length = 0;
|
||||
|
||||
const error = new Error('Color/alpha merger closed.');
|
||||
for (const pending of this.pendingRequests.values()) {
|
||||
|
||||
+12
-319
@@ -559,15 +559,12 @@ class VideoEncoderWrapper {
|
||||
this.encoder.encode(videoFrame, finalEncodeOptions);
|
||||
videoFrame.close();
|
||||
} else {
|
||||
const width = videoFrame.displayWidth;
|
||||
const height = videoFrame.displayHeight;
|
||||
|
||||
if (!this.splitter) {
|
||||
this.splitter = new ColorAlphaSplitter(width, height);
|
||||
this.splitter = new ColorAlphaSplitter();
|
||||
}
|
||||
|
||||
// The splitter takes ownership, so no need to close the frames ourselves
|
||||
const { colorFrame, alphaFrame } = await this.splitter.update(videoFrame);
|
||||
const { colorFrame, alphaFrame } = await this.splitter.split(videoFrame);
|
||||
|
||||
this.alphaFrameQueue.push(alphaFrame);
|
||||
this.encoder.encode(colorFrame, finalEncodeOptions);
|
||||
@@ -917,21 +914,10 @@ class VideoEncoderWrapper {
|
||||
}
|
||||
}
|
||||
|
||||
let splitterGpuUnavailable = false;
|
||||
let splitterWorkerUrl: string | null = null;
|
||||
|
||||
/** Utility class for splitting a composite frame into separate color and alpha components. */
|
||||
/** Utility class for splitting a composite frame into separate color and alpha parts on the CPU in a worker. */
|
||||
export class ColorAlphaSplitter {
|
||||
static forceCpu = true;
|
||||
|
||||
canvas: OffscreenCanvas | HTMLCanvasElement | null = null;
|
||||
|
||||
private gl: WebGL2RenderingContext | null = null;
|
||||
private colorProgram: WebGLProgram | null = null;
|
||||
private alphaProgram: WebGLProgram | null = null;
|
||||
private vao: WebGLVertexArrayObject | null = null;
|
||||
private sourceTexture: WebGLTexture | null = null;
|
||||
private alphaResolutionLocation: WebGLUniformLocation | null = null;
|
||||
|
||||
private worker: Worker | null = null;
|
||||
private pendingRequests = new Map<
|
||||
number,
|
||||
@@ -940,308 +926,17 @@ export class ColorAlphaSplitter {
|
||||
|
||||
private nextRequestId = 0;
|
||||
|
||||
constructor(initialWidth: number, initialHeight: number) {
|
||||
const canMakeCanvas = typeof OffscreenCanvas !== 'undefined'
|
||||
// eslint-disable-next-line @typescript-eslint/no-deprecated
|
||||
|| (typeof document !== 'undefined' && typeof document.createElement === 'function');
|
||||
|
||||
if (!ColorAlphaSplitter.forceCpu && canMakeCanvas && !splitterGpuUnavailable) {
|
||||
// Try the GPU path. If anything goes wrong, we silently fall back to the CPU path.
|
||||
try {
|
||||
if (typeof OffscreenCanvas !== 'undefined') {
|
||||
this.canvas = new OffscreenCanvas(initialWidth, initialHeight);
|
||||
} else {
|
||||
this.canvas = document.createElement('canvas');
|
||||
this.canvas.width = initialWidth;
|
||||
this.canvas.height = initialHeight;
|
||||
}
|
||||
|
||||
const gl = this.canvas.getContext('webgl2', {
|
||||
alpha: true, // Needed due to the YUV thing we do for alpha
|
||||
}) as unknown as WebGL2RenderingContext | null; // Casting because of some TypeScript weirdness
|
||||
if (!gl) {
|
||||
throw new Error('Couldn\'t acquire WebGL 2 context.');
|
||||
}
|
||||
|
||||
this.gl = gl;
|
||||
|
||||
this.colorProgram = this.createColorProgram();
|
||||
this.alphaProgram = this.createAlphaProgram();
|
||||
this.vao = this.createVAO();
|
||||
this.sourceTexture = this.createTexture();
|
||||
|
||||
this.alphaResolutionLocation = this.gl.getUniformLocation(this.alphaProgram, 'u_resolution')!;
|
||||
|
||||
this.gl.useProgram(this.colorProgram);
|
||||
this.gl.uniform1i(this.gl.getUniformLocation(this.colorProgram, 'u_sourceTexture'), 0);
|
||||
|
||||
this.gl.useProgram(this.alphaProgram);
|
||||
this.gl.uniform1i(this.gl.getUniformLocation(this.alphaProgram, 'u_sourceTexture'), 0);
|
||||
} catch (error) {
|
||||
this.gl = null;
|
||||
this.canvas = null;
|
||||
splitterGpuUnavailable = true;
|
||||
Logging._warn('Falling back to CPU for color/alpha splitting.', error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async update(sourceFrame: VideoFrame) {
|
||||
if (this.gl) {
|
||||
return this.updateGpu(sourceFrame);
|
||||
} else {
|
||||
return this.updateCpu(sourceFrame);
|
||||
}
|
||||
}
|
||||
|
||||
private updateGpu(sourceFrame: VideoFrame) {
|
||||
assert(this.gl);
|
||||
assert(this.canvas);
|
||||
|
||||
if (sourceFrame.displayWidth !== this.canvas.width || sourceFrame.displayHeight !== this.canvas.height) {
|
||||
this.canvas.width = sourceFrame.displayWidth;
|
||||
this.canvas.height = sourceFrame.displayHeight;
|
||||
}
|
||||
|
||||
this.gl.activeTexture(this.gl.TEXTURE0);
|
||||
this.gl.bindTexture(this.gl.TEXTURE_2D, this.sourceTexture);
|
||||
this.gl.texImage2D(this.gl.TEXTURE_2D, 0, this.gl.RGBA, this.gl.RGBA, this.gl.UNSIGNED_BYTE, sourceFrame);
|
||||
|
||||
const colorFrame = this.runColorProgram(sourceFrame);
|
||||
const alphaFrame = this.runAlphaProgram(sourceFrame);
|
||||
|
||||
sourceFrame.close();
|
||||
|
||||
return { colorFrame, alphaFrame };
|
||||
}
|
||||
|
||||
private createVertexShader(): WebGLShader {
|
||||
assert(this.gl);
|
||||
|
||||
return this.createShader(this.gl.VERTEX_SHADER, `#version 300 es
|
||||
in vec2 a_position;
|
||||
in vec2 a_texCoord;
|
||||
out vec2 v_texCoord;
|
||||
|
||||
void main() {
|
||||
gl_Position = vec4(a_position, 0.0, 1.0);
|
||||
v_texCoord = a_texCoord;
|
||||
}
|
||||
`);
|
||||
}
|
||||
|
||||
private createColorProgram(): WebGLProgram {
|
||||
assert(this.gl);
|
||||
|
||||
const vertexShader = this.createVertexShader();
|
||||
|
||||
// This shader is simple, simply copy the color information while setting alpha to 1
|
||||
const fragmentShader = this.createShader(this.gl.FRAGMENT_SHADER, `#version 300 es
|
||||
precision highp float;
|
||||
|
||||
uniform sampler2D u_sourceTexture;
|
||||
in vec2 v_texCoord;
|
||||
out vec4 fragColor;
|
||||
|
||||
void main() {
|
||||
vec4 source = texture(u_sourceTexture, v_texCoord);
|
||||
fragColor = vec4(source.rgb, 1.0);
|
||||
}
|
||||
`);
|
||||
|
||||
const program = this.gl.createProgram();
|
||||
this.gl.attachShader(program, vertexShader);
|
||||
this.gl.attachShader(program, fragmentShader);
|
||||
this.gl.linkProgram(program);
|
||||
|
||||
return program;
|
||||
}
|
||||
|
||||
private createAlphaProgram(): WebGLProgram {
|
||||
assert(this.gl);
|
||||
|
||||
const vertexShader = this.createVertexShader();
|
||||
|
||||
// This shader's more complex. The main reason is that this shader writes data in I420 (yuv420) pixel format
|
||||
// instead of regular RGBA. In other words, we use the shader to write out I420 data into an RGBA canvas, which
|
||||
// we then later read out with JavaScript. The reason being that browsers weirdly encode canvases and mess up
|
||||
// the color spaces, and the only way to have full control over the color space is by outputting YUV data
|
||||
// directly (avoiding the RGB conversion). Doing this conversion in JS is painfully slow, so let's utlize the
|
||||
// GPU since we're already calling it anyway.
|
||||
const fragmentShader = this.createShader(this.gl.FRAGMENT_SHADER, `#version 300 es
|
||||
precision highp float;
|
||||
|
||||
uniform sampler2D u_sourceTexture;
|
||||
uniform vec2 u_resolution; // The width and height of the canvas
|
||||
in vec2 v_texCoord;
|
||||
out vec4 fragColor;
|
||||
|
||||
// This function determines the value for a single byte in the YUV stream
|
||||
float getByteValue(float byteOffset) {
|
||||
float width = u_resolution.x;
|
||||
float height = u_resolution.y;
|
||||
|
||||
float yPlaneSize = width * height;
|
||||
|
||||
if (byteOffset < yPlaneSize) {
|
||||
// This byte is in the luma plane. Find the corresponding pixel coordinates to sample from
|
||||
float y = floor(byteOffset / width);
|
||||
float x = mod(byteOffset, width);
|
||||
|
||||
// Add 0.5 to sample the center of the texel
|
||||
vec2 sampleCoord = (vec2(x, y) + 0.5) / u_resolution;
|
||||
|
||||
// The luma value is the alpha from the source texture
|
||||
return texture(u_sourceTexture, sampleCoord).a;
|
||||
} else {
|
||||
// Write a fixed value for chroma and beyond
|
||||
return 128.0 / 255.0;
|
||||
}
|
||||
}
|
||||
|
||||
void main() {
|
||||
// Each fragment writes 4 bytes (R, G, B, A)
|
||||
float pixelIndex = floor(gl_FragCoord.y) * u_resolution.x + floor(gl_FragCoord.x);
|
||||
float baseByteOffset = pixelIndex * 4.0;
|
||||
|
||||
vec4 result;
|
||||
for (int i = 0; i < 4; i++) {
|
||||
float currentByteOffset = baseByteOffset + float(i);
|
||||
result[i] = getByteValue(currentByteOffset);
|
||||
}
|
||||
|
||||
fragColor = result;
|
||||
}
|
||||
`);
|
||||
|
||||
const program = this.gl.createProgram();
|
||||
this.gl.attachShader(program, vertexShader);
|
||||
this.gl.attachShader(program, fragmentShader);
|
||||
this.gl.linkProgram(program);
|
||||
|
||||
return program;
|
||||
}
|
||||
|
||||
private createShader(type: number, source: string): WebGLShader {
|
||||
assert(this.gl);
|
||||
|
||||
const shader = this.gl.createShader(type)!;
|
||||
this.gl.shaderSource(shader, source);
|
||||
this.gl.compileShader(shader);
|
||||
if (!this.gl.getShaderParameter(shader, this.gl.COMPILE_STATUS)) {
|
||||
Logging._error('Shader compile error:', this.gl.getShaderInfoLog(shader));
|
||||
}
|
||||
return shader;
|
||||
}
|
||||
|
||||
private createVAO(): WebGLVertexArrayObject {
|
||||
assert(this.gl);
|
||||
assert(this.colorProgram);
|
||||
|
||||
const vao = this.gl.createVertexArray();
|
||||
this.gl.bindVertexArray(vao);
|
||||
|
||||
const vertices = new Float32Array([
|
||||
-1, -1, 0, 1,
|
||||
1, -1, 1, 1,
|
||||
-1, 1, 0, 0,
|
||||
1, 1, 1, 0,
|
||||
]);
|
||||
|
||||
const buffer = this.gl.createBuffer();
|
||||
this.gl.bindBuffer(this.gl.ARRAY_BUFFER, buffer);
|
||||
this.gl.bufferData(this.gl.ARRAY_BUFFER, vertices, this.gl.STATIC_DRAW);
|
||||
|
||||
const positionLocation = this.gl.getAttribLocation(this.colorProgram, 'a_position');
|
||||
const texCoordLocation = this.gl.getAttribLocation(this.colorProgram, 'a_texCoord');
|
||||
|
||||
this.gl.enableVertexAttribArray(positionLocation);
|
||||
this.gl.vertexAttribPointer(positionLocation, 2, this.gl.FLOAT, false, 16, 0);
|
||||
|
||||
this.gl.enableVertexAttribArray(texCoordLocation);
|
||||
this.gl.vertexAttribPointer(texCoordLocation, 2, this.gl.FLOAT, false, 16, 8);
|
||||
|
||||
return vao;
|
||||
}
|
||||
|
||||
private createTexture(): WebGLTexture {
|
||||
assert(this.gl);
|
||||
|
||||
const texture = this.gl.createTexture();
|
||||
|
||||
this.gl.bindTexture(this.gl.TEXTURE_2D, texture);
|
||||
this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_S, this.gl.CLAMP_TO_EDGE);
|
||||
this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_T, this.gl.CLAMP_TO_EDGE);
|
||||
this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MIN_FILTER, this.gl.LINEAR);
|
||||
this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MAG_FILTER, this.gl.LINEAR);
|
||||
|
||||
return texture;
|
||||
}
|
||||
|
||||
private runColorProgram(sourceFrame: VideoFrame) {
|
||||
assert(this.gl);
|
||||
assert(this.canvas);
|
||||
|
||||
this.gl.useProgram(this.colorProgram);
|
||||
this.gl.viewport(0, 0, this.canvas.width, this.canvas.height);
|
||||
this.gl.clear(this.gl.COLOR_BUFFER_BIT);
|
||||
this.gl.bindVertexArray(this.vao);
|
||||
this.gl.drawArrays(this.gl.TRIANGLE_STRIP, 0, 4);
|
||||
|
||||
return new VideoFrame(this.canvas, {
|
||||
timestamp: sourceFrame.timestamp,
|
||||
duration: sourceFrame.duration ?? undefined,
|
||||
alpha: 'discard',
|
||||
});
|
||||
}
|
||||
|
||||
private runAlphaProgram(sourceFrame: VideoFrame) {
|
||||
assert(this.gl);
|
||||
assert(this.canvas);
|
||||
|
||||
this.gl.useProgram(this.alphaProgram);
|
||||
this.gl.uniform2f(this.alphaResolutionLocation, this.canvas.width, this.canvas.height);
|
||||
|
||||
this.gl.viewport(0, 0, this.canvas.width, this.canvas.height);
|
||||
this.gl.clear(this.gl.COLOR_BUFFER_BIT);
|
||||
this.gl.bindVertexArray(this.vao);
|
||||
this.gl.drawArrays(this.gl.TRIANGLE_STRIP, 0, 4);
|
||||
|
||||
const { width, height } = this.canvas;
|
||||
|
||||
const chromaSamples = Math.ceil(width / 2) * Math.ceil(height / 2);
|
||||
const yuvSize = width * height + chromaSamples * 2;
|
||||
const requiredHeight = Math.ceil(yuvSize / (width * 4));
|
||||
|
||||
let yuv = new Uint8Array(4 * width * requiredHeight);
|
||||
this.gl.readPixels(0, 0, width, requiredHeight, this.gl.RGBA, this.gl.UNSIGNED_BYTE, yuv);
|
||||
yuv = yuv.subarray(0, yuvSize);
|
||||
|
||||
assert(yuv[width * height] === 128); // Where chroma data starts
|
||||
assert(yuv[yuv.length - 1] === 128); // Assert the YUV data has been fully written
|
||||
|
||||
// Defining this separately because TypeScript doesn't know `transfer` and I can't be bothered to do declaration
|
||||
// merging right now
|
||||
const init = {
|
||||
format: 'I420' as const,
|
||||
codedWidth: width,
|
||||
codedHeight: height,
|
||||
timestamp: sourceFrame.timestamp,
|
||||
duration: sourceFrame.duration ?? undefined,
|
||||
transfer: [yuv.buffer],
|
||||
};
|
||||
return new VideoFrame(yuv, init);
|
||||
}
|
||||
|
||||
private updateCpu(sourceFrame: VideoFrame): Promise<{ colorFrame: VideoFrame; alphaFrame: VideoFrame }> {
|
||||
split(sourceFrame: VideoFrame): Promise<{ colorFrame: VideoFrame; alphaFrame: VideoFrame }> {
|
||||
if (!this.worker) {
|
||||
if (!splitterWorkerUrl) {
|
||||
const blob = new Blob(
|
||||
[`(${colorAlphaSplitterWorkerCode.toString()})()`],
|
||||
{ type: 'application/javascript' },
|
||||
);
|
||||
const url = URL.createObjectURL(blob);
|
||||
this.worker = new Worker(url);
|
||||
URL.revokeObjectURL(url);
|
||||
splitterWorkerUrl = URL.createObjectURL(blob);
|
||||
}
|
||||
|
||||
this.worker = new Worker(splitterWorkerUrl);
|
||||
|
||||
this.worker.addEventListener('message', (event: MessageEvent<ColorAlphaSplitterWorkerResponse>) => {
|
||||
const data = event.data;
|
||||
@@ -1271,16 +966,14 @@ export class ColorAlphaSplitter {
|
||||
const pending = promiseWithResolvers<{ colorFrame: VideoFrame; alphaFrame: VideoFrame }>();
|
||||
this.pendingRequests.set(id, pending);
|
||||
this.worker.postMessage({ id, sourceFrame }, { transfer: [sourceFrame] });
|
||||
|
||||
return pending.promise;
|
||||
}
|
||||
|
||||
close() {
|
||||
this.gl?.getExtension('WEBGL_lose_context')?.loseContext();
|
||||
this.gl = null;
|
||||
this.canvas = null;
|
||||
|
||||
this.worker?.terminate();
|
||||
this.worker = null;
|
||||
|
||||
const error = new Error('Color/alpha splitter closed.');
|
||||
for (const pending of this.pendingRequests.values()) {
|
||||
pending.reject(error);
|
||||
|
||||
@@ -2,11 +2,11 @@ import { expect, test } from 'vitest';
|
||||
import { Input } from '../../src/input.js';
|
||||
import { BufferSource, UrlSource } from '../../src/source.js';
|
||||
import { ALL_FORMATS } from '../../src/input-format.js';
|
||||
import { CanvasSink, ColorAlphaMerger, EncodedPacketSink, VideoSampleSink } from '../../src/media-sink.js';
|
||||
import { CanvasSink, EncodedPacketSink, VideoSampleSink } from '../../src/media-sink.js';
|
||||
import { Output } from '../../src/output.js';
|
||||
import { WebMOutputFormat } from '../../src/output-format.js';
|
||||
import { BufferTarget } from '../../src/target.js';
|
||||
import { CanvasSource, ColorAlphaSplitter, VideoSampleSource } from '../../src/media-source.js';
|
||||
import { CanvasSource, VideoSampleSource } from '../../src/media-source.js';
|
||||
import { canEncodeVideo, QUALITY_HIGH } from '../../src/encode.js';
|
||||
import { VideoSample } from '../../src/sample.js';
|
||||
import { Conversion } from '../../src/conversion.js';
|
||||
@@ -39,15 +39,6 @@ test('Can decode transparent video', async () => {
|
||||
await decodeTransparentVideoTest();
|
||||
});
|
||||
|
||||
test('Can decode transparent video, forced CPU path', async () => {
|
||||
try {
|
||||
ColorAlphaMerger.forceCpu = true;
|
||||
await decodeTransparentVideoTest();
|
||||
} finally {
|
||||
ColorAlphaMerger.forceCpu = false;
|
||||
}
|
||||
});
|
||||
|
||||
test('Can decode faulty transparent video and behaves gracefully', async () => {
|
||||
using input = new Input({
|
||||
source: new UrlSource('/transparency-faulty.webm'),
|
||||
@@ -192,15 +183,6 @@ test('Can encode transparent video', async () => {
|
||||
await encodeTransparentVideoTest();
|
||||
});
|
||||
|
||||
test('Can encode transparent video, forced CPU path', async () => {
|
||||
try {
|
||||
ColorAlphaSplitter.forceCpu = true;
|
||||
await encodeTransparentVideoTest();
|
||||
} finally {
|
||||
ColorAlphaSplitter.forceCpu = false;
|
||||
}
|
||||
});
|
||||
|
||||
test('Can encode video with alternating transparency', async () => {
|
||||
// This test is already brutal when it comes to finding async race conditions, so run it thrice to be MORE brutal
|
||||
for (let j = 0; j < 3; j++) {
|
||||
|
||||
Reference in New Issue
Block a user