arcodange/mediabunny
1
0
Fork 0
mirror of https://github.com/arcodange-org/mediabunny.git synced 2026-09-29 11:53:52 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add CPU path for transparent encoding and decoding, add transparency support to FFmpeg-backed VP9 decoder and encoder

Browse Source
This commit is contained in:
Vanilagy
2026-05-11 21:33:29 +02:00
parent a89d715524
commit 1bf200894c
7 changed files with 888 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/browser/transparency.test.ts
+39 -13
View File
@@ -2,16 +2,16 @@ import { expect, test } from 'vitest';
import { Input } from '../../src/input.js';
import { BufferSource, UrlSource } from '../../src/source.js';
import { ALL_FORMATS } from '../../src/input-format.js';
import { CanvasSink, EncodedPacketSink, VideoSampleSink } from '../../src/media-sink.js';
import { CanvasSink, ColorAlphaMerger, EncodedPacketSink, VideoSampleSink } from '../../src/media-sink.js';
import { Output } from '../../src/output.js';
import { WebMOutputFormat } from '../../src/output-format.js';
import { BufferTarget } from '../../src/target.js';
import { CanvasSource, VideoSampleSource } from '../../src/media-source.js';
import { CanvasSource, ColorAlphaSplitter, VideoSampleSource } from '../../src/media-source.js';
import { canEncodeVideo, QUALITY_HIGH } from '../../src/encode.js';
import { VideoSample } from '../../src/sample.js';
import { Conversion } from '../../src/conversion.js';
test.skip('Can decode transparent video', async () => {
const decodeTransparentVideoTest = async () => {
using input = new Input({
source: new UrlSource('/transparency.webm'),
formats: ALL_FORMATS,
@@ -33,9 +33,22 @@ test.skip('Can decode transparent video', async () => {
const imageData = context.getImageData(0, 0, canvas.width, canvas.height);
expect(imageData.data[3]).toBeLessThan(255); // Check that there's actually transparent pixels
};
test('Can decode transparent video', async () => {
await decodeTransparentVideoTest();
});
test.skip('Can decode faulty transparent video and behaves gracefully', async () => {
test('Can decode transparent video, forced CPU path', async () => {
try {
ColorAlphaMerger.forceCpu = true;
await decodeTransparentVideoTest();
} finally {
ColorAlphaMerger.forceCpu = false;
}
});
test('Can decode faulty transparent video and behaves gracefully', async () => {
using input = new Input({
source: new UrlSource('/transparency-faulty.webm'),
formats: ALL_FORMATS,
@@ -55,7 +68,7 @@ test.skip('Can decode faulty transparent video and behaves gracefully', async ()
expect(secondSample.hasAlpha).toBe(false);
});
test.skip('Can extract transparent frames via CanvasSink', async () => {
test('Can extract transparent frames via CanvasSink', async () => {
using input = new Input({
source: new UrlSource('/transparency.webm'),
formats: ALL_FORMATS,
@@ -81,7 +94,7 @@ test.skip('Can extract transparent frames via CanvasSink', async () => {
expect(imageData.data[3]).toBe(255);
});
test.skip('Can encode transparent video', async () => {
const encodeTransparentVideoTest = async () => {
const output = new Output({
format: new WebMOutputFormat(),
target: new BufferTarget(),
@@ -172,10 +185,23 @@ test.skip('Can encode transparent video', async () => {
imageData = probeContext.getImageData(0, 0, probeCanvas.width, probeCanvas.height);
expect(imageData.data[3]).toBe(0); // Transparent
expect(imageData.data[4 * (2 * probeCanvas.width + 2) + 3]).toBe(0); // Transparent
};
test('Can encode transparent video', async () => {
await encodeTransparentVideoTest();
});
test.skip('Can encode video with alternating transparency', async () => {
test('Can encode transparent video, forced CPU path', async () => {
try {
ColorAlphaSplitter.forceCpu = true;
await encodeTransparentVideoTest();
} finally {
ColorAlphaSplitter.forceCpu = false;
}
});
test('Can encode video with alternating transparency', async () => {
const output = new Output({
format: new WebMOutputFormat(),
target: new BufferTarget(),
@@ -246,7 +272,7 @@ test.skip('Can encode video with alternating transparency', async () => {
}
});
test.skip('Can encode transparent video with odd dimensions', async () => {
test('Can encode transparent video with odd dimensions', async () => {
const output = new Output({
format: new WebMOutputFormat(),
target: new BufferTarget(),
@@ -269,12 +295,12 @@ test.skip('Can encode transparent video with odd dimensions', async () => {
await output.finalize();
});
test.skip('Positive encodability check with alpha', async () => {
test('Positive encodability check with alpha', async () => {
const result = await canEncodeVideo('vp9', { alpha: 'keep' });
expect(result).toBe(true);
});
test.skip('Can transmux transparent video, discards alpha by default', async () => {
test('Can transmux transparent video, discards alpha by default', async () => {
using input = new Input({
source: new UrlSource('/transparency.webm'),
formats: ALL_FORMATS,
@@ -303,7 +329,7 @@ test.skip('Can transmux transparent video, discards alpha by default', async ()
expect(sample.hasAlpha).toBe(false);
});
test.skip('Can transmux transparent video, can keep alpha', async () => {
test('Can transmux transparent video, can keep alpha', async () => {
using input = new Input({
source: new UrlSource('/transparency.webm'),
formats: ALL_FORMATS,
@@ -336,7 +362,7 @@ test.skip('Can transmux transparent video, can keep alpha', async () => {
expect(sample.hasAlpha).toBe(true);
});
test.skip('Can reencode transparent video, keeping alpha', async () => {
test('Can reencode transparent video, keeping alpha', async () => {
using input = new Input({
source: new UrlSource('/transparency.webm'),
formats: ALL_FORMATS,
test/node/server-extension.test.ts
+40 -2
View File
@@ -345,11 +345,12 @@ describe('Video', async () => {
});
});
test('VP9 encode and decode', async () => {
test('VP9 encode and decode, opaque', async () => {
await encodeDecodeTest('vp9', {}, async (packet, meta, i) => {
expect(packet.timestamp).toBe(i / 30);
expect(packet.duration).toBe(1 / 30);
expect(packet.type).toBe(i ? 'delta' : 'key');
expect(packet.sideData.alpha).toBeUndefined();
if (i === 0) {
expect(meta.decoderConfig).toBeDefined();
@@ -370,6 +371,36 @@ describe('Video', async () => {
});
});
test('VP9 encode and decode, transparent', async () => {
await encodeDecodeTest('vp9', { alpha: 'keep' }, async (packet, meta, i) => {
expect(packet.timestamp).toBe(i / 30);
expect(packet.duration).toBe(1 / 30);
expect(packet.type).toBe(i ? 'delta' : 'key');
expect(packet.sideData.alpha).toBeDefined();
expect(packet.sideData.alpha!.byteLength).toBeGreaterThan(0);
if (i === 0) {
expect(meta.decoderConfig).toBeDefined();
expect(meta.decoderConfig!.codec.startsWith('vp09.')).toBe(true);
expect(meta.decoderConfig!.description).toBeUndefined();
}
}, async (sample, i) => {
expect(sample.format).toBe('I420A');
expect(sample.codedWidth).toBe(1280);
expect(sample.codedHeight).toBe(720);
expect(sample.timestamp).toBe(i / 30);
expect(sample.duration).toBe(1 / 30);
const buf = new Uint8Array(sample.allocationSize({ format: 'RGBA' }));
await sample.copyTo(buf, { format: 'RGBA' });
expect(buf[0]).toBeGreaterThan(250); // Quasi-white
// Check transparency
expect(Math.abs(buf[3]! - 0x80)).toBeLessThan(3);
}, true);
});
test('AV1 encode and decode', async () => {
await encodeDecodeTest('av1', {}, async (packet, meta, i) => {
expect(packet.timestamp).toBe(i / 30);
@@ -434,6 +465,7 @@ describe('Video', async () => {
extraConfig: Partial<VideoEncoderConfig>,
onPacket: (packet: EncodedPacket, meta: EncodedVideoChunkMetadata, i: number) => Promise<void>,
onSample: (sample: VideoSample, i: number) => Promise<void>,
transparent = false,
) => {
const encoder = new NodeAvVideoEncoder();
// @ts-expect-error Readonly
@@ -460,9 +492,15 @@ describe('Video', async () => {
await encoder.init();
const data = new Uint8Array(1280 * 720 * 4).fill(0xff); // White
if (transparent) {
for (let i = 0; i < data.byteLength; i += 4) {
data[i + 3] = 0x80;
}
}
for (let i = 0; i < 10; i++) {
using sample = new VideoSample(data, {
format: 'RGBX',
format: transparent ? 'RGBA' : 'RGBX',
codedWidth: 1280,
codedHeight: 720,
timestamp: i / 30,