arcodange/mediabunny
1
0
Fork 0
mirror of https://github.com/arcodange-org/mediabunny.git synced 2026-09-27 10:53:50 +02:00
Code Issues Packages Projects Releases Wiki Activity

Change VideoSampleResource interface, validate return values, add more proper VideoSample.copyTo() logic, fix decoder/encoder color space issues

Browse Source
This commit is contained in:
Vanilagy
2026-05-08 19:37:41 +02:00
parent f5bd540863
commit ab9958234a
7 changed files with 831 additions and 654 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/browser/video-samples.test.ts
+181 -9
View File
@@ -154,7 +154,7 @@ test('copyTo and plane layouts', async () => {
expect(layout).toEqual([{
offset: 0,
stride: 1300 * 4,
stride: 1280 * 4,
}]);
using clone = sample.clone();
@@ -162,7 +162,7 @@ test('copyTo and plane layouts', async () => {
expect(clonedLayout).toEqual([{
offset: 0,
stride: 1300 * 4,
stride: 1280 * 4,
}]);
}
@@ -186,6 +186,184 @@ test('copyTo and plane layouts', async () => {
offset: 1280 * 720 + (1280 / 2) * (720 / 2),
stride: 1280 / 2,
}]);
const rgbLayout = await sample.copyTo(buffer, { format: 'RGBA' });
expect(rgbLayout).toEqual([{
offset: 0,
stride: 4 * 1280,
}]);
}
});
test('RGB conversion for ArrayBuffer-backed data', async () => {
// 2x2 RGBA image with distinct, easily-verifiable pixels
const src = new Uint8Array([
10, 20, 30, 255, 40, 50, 60, 255,
70, 80, 90, 255, 100, 110, 120, 255,
]);
{
// RGBA -> BGRA: R and B should swap
using sample = new VideoSample(src.slice(), {
timestamp: 0,
codedWidth: 2,
codedHeight: 2,
format: 'RGBA',
});
const dest = new Uint8Array(2 * 2 * 4);
const layout = await sample.copyTo(dest, { format: 'BGRA' });
expect(layout).toEqual([{ offset: 0, stride: 2 * 4 }]);
expect(Array.from(dest)).toEqual([
30, 20, 10, 255, 60, 50, 40, 255,
90, 80, 70, 255, 120, 110, 100, 255,
]);
}
{
// RGBA -> RGBA: no swap, bytes copied through unchanged
using sample = new VideoSample(src.slice(), {
timestamp: 0,
codedWidth: 2,
codedHeight: 2,
format: 'RGBA',
});
const dest = new Uint8Array(2 * 2 * 4);
await sample.copyTo(dest, { format: 'RGBA' });
expect(Array.from(dest)).toEqual(Array.from(src));
}
{
// RGBA -> BGRX: R and B should swap
using sample = new VideoSample(src.slice(), {
timestamp: 0,
codedWidth: 2,
codedHeight: 2,
format: 'RGBA',
});
const dest = new Uint8Array(2 * 2 * 4);
await sample.copyTo(dest, { format: 'BGRX' });
expect(Array.from(dest)).toEqual([
30, 20, 10, 255, 60, 50, 40, 255,
90, 80, 70, 255, 120, 110, 100, 255,
]);
}
{
// BGRA -> RGBA: R and B should swap
using sample = new VideoSample(src.slice(), {
timestamp: 0,
codedWidth: 2,
codedHeight: 2,
format: 'BGRA',
});
const dest = new Uint8Array(2 * 2 * 4);
await sample.copyTo(dest, { format: 'RGBA' });
expect(Array.from(dest)).toEqual([
30, 20, 10, 255, 60, 50, 40, 255,
90, 80, 70, 255, 120, 110, 100, 255,
]);
}
{
// BGRA -> BGRA: no swap
using sample = new VideoSample(src.slice(), {
timestamp: 0,
codedWidth: 2,
codedHeight: 2,
format: 'BGRA',
});
const dest = new Uint8Array(2 * 2 * 4);
await sample.copyTo(dest, { format: 'BGRA' });
expect(Array.from(dest)).toEqual(Array.from(src));
}
});
test('crop (rect option) for ArrayBuffer-backed data', async () => {
// 4x4 RGBA image where each pixel's R/G channels encode (x, y)
const width = 4;
const height = 4;
const src = new Uint8Array(width * height * 4);
for (let y = 0; y < height; y++) {
for (let x = 0; x < width; x++) {
const i = (y * width + x) * 4;
src[i] = x;
src[i + 1] = y;
src[i + 2] = 0;
src[i + 3] = 255;
}
}
{
// Crop a 2x2 region at (1, 1)
using sample = new VideoSample(src.slice(), {
timestamp: 0,
codedWidth: width,
codedHeight: height,
format: 'RGBA',
});
const size = sample.allocationSize({ rect: { x: 1, y: 1, width: 2, height: 2 } });
expect(size).toBe(2 * 2 * 4);
const dest = new Uint8Array(size);
const layout = await sample.copyTo(dest, { rect: { x: 1, y: 1, width: 2, height: 2 } });
expect(layout).toEqual([{ offset: 0, stride: 2 * 4 }]);
expect(Array.from(dest)).toEqual([
1, 1, 0, 255, 2, 1, 0, 255,
1, 2, 0, 255, 2, 2, 0, 255,
]);
}
{
// Crop an offset rect at the top-right corner
using sample = new VideoSample(src.slice(), {
timestamp: 0,
codedWidth: width,
codedHeight: height,
format: 'RGBA',
});
const dest = new Uint8Array(2 * 1 * 4);
const layout = await sample.copyTo(dest, { rect: { x: 2, y: 0, width: 2, height: 1 } });
expect(layout).toEqual([{ offset: 0, stride: 2 * 4 }]);
expect(Array.from(dest)).toEqual([
2, 0, 0, 255, 3, 0, 0, 255,
]);
}
{
// Crop combined with a custom destination stride (padding between rows)
using sample = new VideoSample(src.slice(), {
timestamp: 0,
codedWidth: width,
codedHeight: height,
format: 'RGBA',
});
const stride = 3 * 4; // wider than the crop, leaving trailing padding
const dest = new Uint8Array(stride * 2);
const layout = await sample.copyTo(dest, {
rect: { x: 0, y: 2, width: 2, height: 2 },
layout: [{ offset: 0, stride }],
});
expect(layout).toEqual([{ offset: 0, stride }]);
// Row 0 of crop (y=2): pixels (0,2) and (1,2), then 4 bytes of untouched padding
expect(Array.from(dest.subarray(0, 8))).toEqual([0, 2, 0, 255, 1, 2, 0, 255]);
expect(Array.from(dest.subarray(stride, stride + 8))).toEqual([0, 3, 0, 255, 1, 3, 0, 255]);
}
});
@@ -201,12 +379,7 @@ test('null format', async () => {
expect(() => sample.allocationSize()).toThrow('when format is null');
await expect(async () => sample.copyTo(new ArrayBuffer())).rejects.toThrow('when format is null');
// Even this throws :(
// See https://github.com/Vanilagy/mediabunny/issues/267
expect(() => sample.allocationSize({ format: 'RGBA' })).toThrow('when format is null');
// Uncomment this if the RGBA conversion works again:
/*
// BUT: See https://github.com/Vanilagy/mediabunny/issues/267
const size = sample.allocationSize({ format: 'RGBA' });
expect(size).toBe(1280 * 720 * 4);
const buffer = new ArrayBuffer(size);
@@ -223,5 +396,4 @@ test('null format', async () => {
await sample.copyTo(buffer, { format: 'RGBX' });
await sample.copyTo(buffer, { format: 'BGRA' });
await sample.copyTo(buffer, { format: 'BGRX' });
*/
});
test/node/server-extension.test.ts
+76 -59
View File
@@ -7,7 +7,8 @@ import { assert, toUint8Array } from '../../src/misc.js';
import { EncodedPacketSink, VideoSampleSink } from '../../src/media-sink.js';
import { NodeAvVideoDecoder } from '../../packages/server/src/video-decoder.js';
import { NodeAvVideoEncoder } from '../../packages/server/src/video-encoder.js';
import { VideoSample } from '../../src/sample.js';
import { NodeAvAudioDecoder } from '../../packages/server/src/audio-decoder.js';
import { AudioSample, VideoSample } from '../../src/sample.js';
import { buildVideoCodecString, VideoCodec } from '../../src/codec.js';
import { EncodedPacket } from '../../src/packet.js';
import {
@@ -122,12 +123,6 @@ describe('Video', async () => {
expect(meta.decoderConfig!.codec.startsWith('avc1.')).toBe(true);
expect(meta.decoderConfig!.codedWidth).toBe(1280);
expect(meta.decoderConfig!.codedHeight).toBe(720);
expect(meta.decoderConfig!.colorSpace).toEqual({
primaries: 'bt709',
transfer: 'iec61966-2-1',
matrix: 'rgb',
fullRange: true,
});
expect(meta.decoderConfig!.description).toBeDefined();
}
@@ -208,12 +203,6 @@ describe('Video', async () => {
expect(sample.codedHeight).toBe(720);
expect(sample.timestamp).toBe(i / 30);
expect(sample.duration).toBe(1 / 30);
expect(sample.colorSpace).toEqual({
primaries: 'bt709',
transfer: 'iec61966-2-1',
matrix: 'rgb',
fullRange: true,
});
const buf = new Uint8Array(sample.allocationSize({ format: 'RGBX' }));
await sample.copyTo(buf, { format: 'RGBX' });
@@ -248,12 +237,6 @@ describe('Video', async () => {
expect(sample.codedHeight).toBe(720);
expect(sample.timestamp).toBe(i / 30);
expect(sample.duration).toBe(1 / 30);
expect(sample.colorSpace).toEqual({
primaries: 'bt709',
transfer: 'iec61966-2-1',
matrix: 'rgb',
fullRange: true,
});
const buf = new Uint8Array(sample.allocationSize({ format: 'RGBX' }));
await sample.copyTo(buf, { format: 'RGBX' });
@@ -292,14 +275,6 @@ describe('Video', async () => {
expect(sample.timestamp).toBe(i / 30);
expect(sample.duration).toBe(1 / 30);
// Undefined, for some reason:
expect(sample.colorSpace).toEqual({
primaries: null,
transfer: null,
matrix: null,
fullRange: false,
});
const buf = new Uint8Array(sample.allocationSize({ format: 'RGBX' }));
await sample.copyTo(buf, { format: 'RGBX' });
@@ -336,14 +311,6 @@ describe('Video', async () => {
expect(sample.timestamp).toBe(i / 30);
expect(sample.duration).toBe(1 / 30);
// Undefined, for some reason:
expect(sample.colorSpace).toEqual({
primaries: null,
transfer: null,
matrix: null,
fullRange: false,
});
const buf = new Uint8Array(sample.allocationSize({ format: 'RGBX' }));
await sample.copyTo(buf, { format: 'RGBX' });
@@ -368,12 +335,6 @@ describe('Video', async () => {
expect(sample.codedHeight).toBe(720);
expect(sample.timestamp).toBe(i / 30);
expect(sample.duration).toBe(1 / 30);
expect(sample.colorSpace).toEqual({
primaries: 'bt709',
transfer: 'iec61966-2-1',
matrix: 'bt470bg',
fullRange: false,
});
const buf = new Uint8Array(sample.allocationSize({ format: 'RGBX' }));
await sample.copyTo(buf, { format: 'RGBX' });
@@ -399,12 +360,6 @@ describe('Video', async () => {
expect(sample.codedHeight).toBe(720);
expect(sample.timestamp).toBe(i / 30);
expect(sample.duration).toBe(1 / 30);
expect(sample.colorSpace).toEqual({
primaries: 'bt709',
transfer: 'iec61966-2-1',
matrix: null,
fullRange: false,
});
const buf = new Uint8Array(sample.allocationSize({ format: 'RGBX' }));
await sample.copyTo(buf, { format: 'RGBX' });
@@ -430,12 +385,6 @@ describe('Video', async () => {
expect(sample.codedHeight).toBe(720);
expect(sample.timestamp).toBe(i / 30);
expect(sample.duration).toBe(1 / 30);
expect(sample.colorSpace).toEqual({
primaries: 'bt709',
transfer: 'iec61966-2-1',
matrix: 'rgb',
fullRange: false,
});
const buf = new Uint8Array(sample.allocationSize({ format: 'RGBX' }));
await sample.copyTo(buf, { format: 'RGBX' });
@@ -668,12 +617,6 @@ describe('Video', async () => {
expect(sample.rotation).toBe(0);
expect(sample.timestamp).toBe(0);
expect(sample.duration).toBe(1 / 25);
expect(sample.colorSpace).toEqual({
primaries: null,
transfer: null,
matrix: 'bt470bg',
fullRange: true,
});
// Default expected YUV size
expect(sample.allocationSize()).toBe(1920 * 1080 * 1.5);
@@ -792,3 +735,77 @@ describe('Video', async () => {
]);
});
});
/*
describe('Audio', async () => {
test('Decoder lifecycle', async () => {
using input = new Input({
source: new FilePathSource('./test/public/trim-buck-bunny-ffmpeg.ts'),
formats: ALL_FORMATS,
});
const audioTrack = await input.getPrimaryAudioTrack();
assert(audioTrack);
const decoder = new NodeAvAudioDecoder();
// @ts-expect-error Readonly
decoder.codec = await audioTrack.getCodec();
// @ts-expect-error Readonly
decoder.config = await audioTrack.getDecoderConfig();
let sampleCount = 0;
const packetTimestamps: number[] = [];
// @ts-expect-error Readonly
decoder.onSample = (sample: AudioSample) => {
expect(sample.timestamp).toBe(packetTimestamps[sampleCount]);
if (sampleCount > 0) {
expect(sample.duration).toBeCloseTo(
packetTimestamps[sampleCount]! - packetTimestamps[sampleCount - 1]!,
);
}
sampleCount++;
sample.close();
};
await decoder.init();
const sink = new EncodedPacketSink(audioTrack);
let packetCount = 0;
for await (const packet of sink.packets()) {
packetTimestamps.push(packet.timestamp);
await decoder.decode(packet);
if (++packetCount === 10) {
break;
}
}
await decoder.flush();
expect(sampleCount).toBe(10);
// And, go again
sampleCount = 0;
packetTimestamps.length = 0;
packetCount = 0;
for await (const packet of sink.packets((await sink.getKeyPacket(5))!)) {
packetTimestamps.push(packet.timestamp);
await decoder.decode(packet);
if (++packetCount === 10) {
break;
}
}
await decoder.flush();
expect(sampleCount).toBe(10);
await decoder.close();
});
});
*/