Fix async race condition in transparency code, reduce HEVC conversion duration

src/media-sink.ts

@@ -1036,8 +1036,6 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
 		if (!this.alphaDecoder) {
 			const alphaHandler = (frame: VideoFrame) => {
 				this.frameHandlerSerializer.call(async () => {
-					this.alphaDecoderQueueSize--;
-
 					if (this.colorQueue.length > 0) {
 						const colorFrame = this.colorQueue.shift();
 						assert(colorFrame !== undefined);
@@ -1064,6 +1062,8 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
 							this.alphaQueue.push(null);
 						}
 					}
+
+					this.alphaDecoderQueueSize--;
 				}).catch((error: Error) => this.onError(error));
 			};
 
@@ -1266,7 +1266,7 @@ let mergerGpuUnavailable = false;
 
 /** Utility class that merges together color and alpha information using simple WebGL 2 shaders. */
 export class ColorAlphaMerger {
-	static forceCpu = false;
+	static forceCpu = true;
 
 	canvas: OffscreenCanvas | HTMLCanvasElement | null = null;
 	private gl: WebGL2RenderingContext | null = null;

src/media-source.ts

@@ -877,7 +877,7 @@ let splitterGpuUnavailable = false;
 
 /** Utility class for splitting a composite frame into separate color and alpha components. */
 export class ColorAlphaSplitter {
-	static forceCpu = false;
+	static forceCpu = true;
 
 	canvas: OffscreenCanvas | HTMLCanvasElement | null = null;
 

test/browser/transparency.test.ts

@@ -202,73 +202,76 @@ test('Can encode transparent video, forced CPU path', async () => {
 });
 
 test('Can encode video with alternating transparency', async () => {
-	const output = new Output({
-		format: new WebMOutputFormat(),
-		target: new BufferTarget(),
-	});
-
-	const canvas1 = new OffscreenCanvas(640, 480);
-	const context1 = canvas1.getContext('2d', { alpha: true })!;
-	context1.fillStyle = '#ff000080';
-	context1.fillRect(0, 0, canvas1.width, canvas1.height);
-
-	const canvas2 = new OffscreenCanvas(640, 480);
-	const context2 = canvas2.getContext('2d', { alpha: false })!;
-	context2.fillStyle = '#0000ff';
-	context2.fillRect(0, 0, canvas2.width, canvas2.height);
-
-	const source = new VideoSampleSource({
-		codec: 'vp9',
-		bitrate: QUALITY_HIGH,
-		alpha: 'keep',
-	});
-	output.addVideoTrack(source);
-
-	await output.start();
-
-	for (let i = 0; i < 64; i++) {
-		using sample = new VideoSample(new Uint8Array(640 * 480 * 4), {
-			format: i % 2 ? 'RGBX' : 'RGBA',
-			codedWidth: 640,
-			codedHeight: 480,
-			timestamp: i,
-			duration: 1,
+	// This test is already brutal when it comes to finding async race conditions, so run it thrice to be MORE brutal
+	for (let j = 0; j < 3; j++) {
+		const output = new Output({
+			format: new WebMOutputFormat(),
+			target: new BufferTarget(),
 		});
-		await source.add(sample);
-	}
 
-	await output.finalize();
+		const canvas1 = new OffscreenCanvas(640, 480);
+		const context1 = canvas1.getContext('2d', { alpha: true })!;
+		context1.fillStyle = '#ff000080';
+		context1.fillRect(0, 0, canvas1.width, canvas1.height);
 
-	using input = new Input({
-		source: new BufferSource(output.target.buffer!),
-		formats: ALL_FORMATS,
-	});
+		const canvas2 = new OffscreenCanvas(640, 480);
+		const context2 = canvas2.getContext('2d', { alpha: false })!;
+		context2.fillStyle = '#0000ff';
+		context2.fillRect(0, 0, canvas2.width, canvas2.height);
 
-	const videoTrack = (await input.getPrimaryVideoTrack())!;
-	const packetSink = new EncodedPacketSink(videoTrack);
+		const source = new VideoSampleSource({
+			codec: 'vp9',
+			bitrate: QUALITY_HIGH,
+			alpha: 'keep',
+		});
+		output.addVideoTrack(source);
 
-	let i = 0;
-	for await (const packet of packetSink.packets()) {
-		if (i % 2) {
-			expect(packet.sideData.alpha).toBeUndefined();
-		} else {
-			expect(packet.sideData.alpha).toBeDefined();
+		await output.start();
+
+		for (let i = 0; i < 64; i++) {
+			using sample = new VideoSample(new Uint8Array(640 * 480 * 4), {
+				format: i % 2 ? 'RGBX' : 'RGBA',
+				codedWidth: 640,
+				codedHeight: 480,
+				timestamp: i,
+				duration: 1,
+			});
+			await source.add(sample);
 		}
 
-		i++;
-	}
+		await output.finalize();
 
-	const sampleSink = new VideoSampleSink(videoTrack);
+		using input = new Input({
+			source: new BufferSource(output.target.buffer!),
+			formats: ALL_FORMATS,
+		});
 
-	i = 0;
-	for await (using sample of sampleSink.samples()) {
-		if (i % 2) {
-			expect(sample.format).not.toContain('A');
-		} else {
-			expect(sample.format).toContain('A');
+		const videoTrack = (await input.getPrimaryVideoTrack())!;
+		const packetSink = new EncodedPacketSink(videoTrack);
+
+		let i = 0;
+		for await (const packet of packetSink.packets()) {
+			if (i % 2) {
+				expect(packet.sideData.alpha).toBeUndefined();
+			} else {
+				expect(packet.sideData.alpha).toBeDefined();
+			}
+
+			i++;
 		}
 
-		i++;
+		const sampleSink = new VideoSampleSink(videoTrack);
+
+		i = 0;
+		for await (using sample of sampleSink.samples()) {
+			if (i % 2) {
+				expect(sample.format).not.toContain('A');
+			} else {
+				expect(sample.format).toContain('A');
+			}
+
+			i++;
+		}
 	}
 });
 

test/node/server-extension.test.ts

@@ -593,7 +593,7 @@ describe('Video', async () => {
 	});
 
 	test('HEVC conversion roundtrip', { timeout: 20_000 }, async () => {
-		await conversionRoundtrip('hevc');
+		await conversionRoundtrip('hevc', 1); // GitHub is extremely slow on this one
 	});
 
 	test('VP8 conversion roundtrip', { timeout: 20_000 }, async () => {
@@ -608,7 +608,7 @@ describe('Video', async () => {
 		await conversionRoundtrip('av1');
 	});
 
-	const conversionRoundtrip = async (codec: VideoCodec) => {
+	const conversionRoundtrip = async (codec: VideoCodec, duration?: number) => {
 		using input = new Input({
 			source: new FilePathSource('./test/public/video.mp4'),
 			formats: ALL_FORMATS,
@@ -640,6 +640,9 @@ describe('Video', async () => {
 			}, audio: {
 				discard: true,
 			},
+			trim: {
+				end: duration,
+			},
 		});
 		await conversion.execute();