Merge branch 'main' into v2

test/browser/flac.test.ts

@@ -19,7 +19,7 @@ test('can decode samples from a FLAC file', async () => {
 
 	const sink = new AudioSampleSink(track);
 
-	const sample = await sink.getSample(1);
+	using sample = await sink.getSample(1);
 	assert(sample);
 	expect(sample.timestamp).toBe(0.9287981859410431);
 });

test/browser/transparency.test.ts

@@ -21,7 +21,7 @@ test('Can decode transparent video', async () => {
 	expect(await videoTrack.canBeTransparent()).toBe(true);
 
 	const sink = new VideoSampleSink(videoTrack);
-	const sample = (await sink.getSample(0.5))!;
+	using sample = (await sink.getSample(0.5))!;
 
 	expect(sample.format).toContain('A'); // Probably RGBA
 	expect(sample.hasAlpha).toBe(true);
@@ -47,10 +47,10 @@ test('Can decode faulty transparent video and behaves gracefully', async () => {
 
 	const sink = new VideoSampleSink(videoTrack);
 
-	const startSample = (await sink.getSample(await videoTrack.getFirstTimestamp()))!;
+	using startSample = (await sink.getSample(await videoTrack.getFirstTimestamp()))!;
 	expect(startSample.format).toContain('A');
 
-	const secondSample = (await sink.getSample(secondKeyPacket.timestamp))!;
+	using secondSample = (await sink.getSample(secondKeyPacket.timestamp))!;
 	expect(secondSample.format).not.toContain('A'); // There was no alpha key frame for this one
 	expect(secondSample.hasAlpha).toBe(false);
 });
@@ -164,7 +164,7 @@ test('Can encode transparent video', async () => {
 
 	const sink = new VideoSampleSink(videoTrack);
 
-	const firstSample = (await sink.getSample(0))!;
+	using firstSample = (await sink.getSample(0))!;
 	expect(firstSample.format).toContain('A');
 
 	probeContext.clearRect(0, 0, probeCanvas.width, probeCanvas.height);
@@ -201,7 +201,7 @@ test('Can encode video with alternating transparency', async () => {
 	await output.start();
 
 	for (let i = 0; i < 64; i++) {
-		const sample = new VideoSample(new Uint8Array(640 * 480 * 4), {
+		using sample = new VideoSample(new Uint8Array(640 * 480 * 4), {
 			format: i % 2 ? 'RGBX' : 'RGBA',
 			codedWidth: 640,
 			codedHeight: 480,
@@ -235,7 +235,7 @@ test('Can encode video with alternating transparency', async () => {
 	const sampleSink = new VideoSampleSink(videoTrack);
 
 	i = 0;
-	for await (const sample of sampleSink.samples()) {
+	for await (using sample of sampleSink.samples()) {
 		if (i % 2) {
 			expect(sample.format).not.toContain('A');
 		} else {
@@ -299,7 +299,7 @@ test('Can transmux transparent video, discards alpha by default', async () => {
 	expect(await videoTrack.canBeTransparent()).toBe(false);
 
 	const sink = new VideoSampleSink(videoTrack);
-	const sample = (await sink.getSample(await videoTrack.getFirstTimestamp()))!;
+	using sample = (await sink.getSample(await videoTrack.getFirstTimestamp()))!;
 	expect(sample.hasAlpha).toBe(false);
 });
 
@@ -331,7 +331,7 @@ test('Can transmux transparent video, can keep alpha', async () => {
 	expect(await videoTrack.canBeTransparent()).toBe(true);
 
 	const sink = new VideoSampleSink(videoTrack);
-	const sample = (await sink.getSample(await videoTrack.getFirstTimestamp()))!;
+	using sample = (await sink.getSample(await videoTrack.getFirstTimestamp()))!;
 	expect(sample.format).toContain('A');
 	expect(sample.hasAlpha).toBe(true);
 });
@@ -370,7 +370,7 @@ test('Can reencode transparent video, keeping alpha', async () => {
 	expect(videoTrack.displayWidth).toBe(320);
 
 	const sink = new VideoSampleSink(videoTrack);
-	const sample = (await sink.getSample(await videoTrack.getFirstTimestamp()))!;
+	using sample = (await sink.getSample(await videoTrack.getFirstTimestamp()))!;
 	expect(sample.format).toContain('A');
 	expect(sample.hasAlpha).toBe(true);
 });

test/browser/video-samples.test.ts

@@ -0,0 +1,220 @@
+import { expect, test } from 'vitest';
+import { VideoSample } from '../../src/sample.js';
+
+test('allocationSize', async () => {
+	{
+		const canvas = new OffscreenCanvas(1280, 720);
+		canvas.getContext('2d');
+
+		using sample = new VideoSample(canvas, { timestamp: 0 });
+
+		const size1 = sample.allocationSize();
+		expect(size1).toBe(1280 * 720 * 4);
+
+		const size2 = sample.allocationSize({
+			layout: [{
+				offset: 0,
+				stride: 1300 * 4,
+			}],
+		});
+		expect(size2).toBe(1300 * 720 * 4);
+	}
+
+	{
+		const data = new Uint8Array(1280 * 720 * 4);
+		using sample = new VideoSample(data, {
+			timestamp: 0,
+			codedWidth: 1280,
+			codedHeight: 720,
+			format: 'RGBA',
+		});
+
+		const size1 = sample.allocationSize();
+		expect(size1).toBe(1280 * 720 * 4);
+
+		const size2 = sample.allocationSize({
+			layout: [{
+				offset: 0,
+				stride: 1300 * 4,
+			}],
+		});
+		expect(size2).toBe(1300 * 720 * 4);
+	}
+
+	{
+		const data = new Uint8Array(1280 * 720 * 1.5);
+		using sample = new VideoSample(data, {
+			timestamp: 0,
+			codedWidth: 1280,
+			codedHeight: 720,
+			format: 'I420',
+		});
+
+		const size1 = sample.allocationSize();
+		expect(size1).toBe(1280 * 720 * 1.5);
+
+		const size2 = sample.allocationSize({ format: 'RGBA' });
+		expect(size2).toBe(1280 * 720 * 4);
+
+		const size3 = sample.allocationSize({
+			format: 'RGBA',
+			layout: [{
+				offset: 0,
+				stride: 1300 * 4,
+			}],
+		});
+		expect(size3).toBe(1300 * 720 * 4);
+	}
+});
+
+test('copyTo and plane layouts', async () => {
+	const buffer = new ArrayBuffer(1e7);
+
+	{
+		const canvas = new OffscreenCanvas(1280, 720);
+		canvas.getContext('2d');
+
+		using sample = new VideoSample(canvas, { timestamp: 0 });
+		const layout = await sample.copyTo(buffer);
+
+		expect(layout).toEqual([{
+			offset: 0,
+			stride: 1280 * 4,
+		}]);
+	}
+
+	{
+		const canvas = new OffscreenCanvas(1280, 720);
+		canvas.getContext('2d');
+
+		using sample = new VideoSample(canvas, { timestamp: 0 });
+		const layout = await sample.copyTo(buffer, {
+			layout: [{
+				offset: 0,
+				stride: 1300 * 4,
+			}],
+		});
+
+		expect(layout).toEqual([{
+			offset: 0,
+			stride: 1300 * 4,
+		}]);
+	}
+
+	{
+		const data = new Uint8Array(1280 * 720 * 4);
+		using sample = new VideoSample(data, {
+			timestamp: 0,
+			codedWidth: 1280,
+			codedHeight: 720,
+			format: 'RGBA',
+		});
+		const layout = await sample.copyTo(buffer);
+
+		expect(layout).toEqual([{
+			offset: 0,
+			stride: 1280 * 4,
+		}]);
+	}
+
+	{
+		const data = new Uint8Array(1280 * 720 * 4);
+		using sample = new VideoSample(data, {
+			timestamp: 0,
+			codedWidth: 1280,
+			codedHeight: 720,
+			format: 'RGBA',
+		});
+		const layout = await sample.copyTo(buffer, {
+			layout: [{
+				offset: 0,
+				stride: 1300 * 4,
+			}],
+		});
+
+		expect(layout).toEqual([{
+			offset: 0,
+			stride: 1300 * 4,
+		}]);
+	}
+
+	{
+		const data = new Uint8Array(1300 * 720 * 4);
+		using sample = new VideoSample(data, {
+			timestamp: 0,
+			codedWidth: 1280,
+			codedHeight: 720,
+			format: 'RGBA',
+			layout: [{
+				offset: 0,
+				stride: 1300 * 4,
+			}],
+		});
+		const layout = await sample.copyTo(buffer);
+
+		expect(layout).toEqual([{
+			offset: 0,
+			stride: 1300 * 4,
+		}]);
+
+		using clone = sample.clone();
+		const clonedLayout = await clone.copyTo(buffer);
+
+		expect(clonedLayout).toEqual([{
+			offset: 0,
+			stride: 1300 * 4,
+		}]);
+	}
+
+	{
+		const data = new Uint8Array(1280 * 720 * 1.5);
+		using sample = new VideoSample(data, {
+			timestamp: 0,
+			codedWidth: 1280,
+			codedHeight: 720,
+			format: 'I420',
+		});
+		const layout = await sample.copyTo(buffer);
+
+		expect(layout).toEqual([{
+			offset: 0,
+			stride: 1280,
+		}, {
+			offset: 1280 * 720,
+			stride: 1280 / 2,
+		}, {
+			offset: 1280 * 720 + (1280 / 2) * (720 / 2),
+			stride: 1280 / 2,
+		}]);
+	}
+});
+
+test('null format', async () => {
+	const canvas = new OffscreenCanvas(1280, 720);
+	canvas.getContext('2d');
+
+	const frame = new VideoFrame(canvas, { timestamp: 0 });
+	using sample = new VideoSample(frame);
+	// @ts-expect-error Sybau
+	sample.format = null;
+
+	expect(() => sample.allocationSize()).toThrow('when format is null');
+	await expect(async () => sample.copyTo(new ArrayBuffer())).rejects.toThrow('when format is null');
+
+	const size = sample.allocationSize({ format: 'RGBA' });
+	expect(size).toBe(1280 * 720 * 4);
+	const buffer = new ArrayBuffer(size);
+	const layout = await sample.copyTo(buffer, { format: 'RGBA' });
+
+	expect(layout).toEqual([{
+		offset: 0,
+		stride: 1280 * 4,
+	}]);
+
+	sample.allocationSize({ format: 'RGBX' });
+	sample.allocationSize({ format: 'BGRA' });
+	sample.allocationSize({ format: 'BGRX' });
+	await sample.copyTo(buffer, { format: 'RGBX' });
+	await sample.copyTo(buffer, { format: 'BGRA' });
+	await sample.copyTo(buffer, { format: 'BGRX' });
+});

test/node/annex-b-conversion.test.ts

@@ -0,0 +1,54 @@
+import { expect, test } from 'vitest';
+import { Input } from '../../src/input.js';
+import { BufferSource, FilePathSource } from '../../src/source.js';
+import path from 'node:path';
+import { ALL_FORMATS } from '../../src/input-format.js';
+import { Output } from '../../src/output.js';
+import { Mp4OutputFormat } from '../../src/output-format.js';
+import { BufferTarget } from '../../src/target.js';
+import { Conversion } from '../../src/conversion.js';
+import { EncodedPacketSink } from '../../src/media-sink.js';
+import { extractAvcNalUnits } from '../../src/codec-data.js';
+
+const __dirname = new URL('.', import.meta.url).pathname;
+
+test('Annex B to length-prefixed conversion, MP4', async () => {
+	using originalInput = new Input({
+		source: new FilePathSource(path.join(__dirname, '..', 'public/annex-b-avc.mkv')),
+		formats: ALL_FORMATS,
+	});
+	const originalVideoTrack = (await originalInput.getPrimaryVideoTrack())!;
+	const originalDecoderConfig = (await originalVideoTrack.getDecoderConfig())!;
+	expect(originalDecoderConfig.description).toBeUndefined();
+	expect(originalVideoTrack.codec).toBe('avc');
+
+	const originalSink = new EncodedPacketSink(originalVideoTrack);
+	const originalFirstPacket = await originalSink.getFirstPacket();
+	expect([...originalFirstPacket!.data.slice(0, 4)]).toEqual([0, 0, 0, 1]);
+
+	const originalNalUnits = extractAvcNalUnits(originalFirstPacket!.data, originalDecoderConfig);
+
+	const output = new Output({
+		format: new Mp4OutputFormat(),
+		target: new BufferTarget(),
+	});
+
+	const conversion = await Conversion.init({ input: originalInput, output });
+	await conversion.execute();
+
+	using newInput = new Input({
+		source: new BufferSource(output.target.buffer!),
+		formats: ALL_FORMATS,
+	});
+	const newVideoTrack = (await newInput.getPrimaryVideoTrack())!;
+	const newDecoderConfig = (await newVideoTrack.getDecoderConfig())!;
+	expect(newDecoderConfig.description).toBeDefined();
+	expect(newVideoTrack.codec).toBe('avc');
+
+	const newSink = new EncodedPacketSink(newVideoTrack);
+	const newFirstPacket = await newSink.getFirstPacket();
+	expect([...newFirstPacket!.data.slice(0, 4)]).not.toEqual([0, 0, 0, 1]); // Successfully converted
+
+	const newNalUnits = extractAvcNalUnits(newFirstPacket!.data, newDecoderConfig);
+	expect(newNalUnits).toEqual(originalNalUnits); // Content is the same though
+});

test/node/disposition.test.ts

@@ -0,0 +1,92 @@
+import { expect, test } from 'vitest';
+import { Output } from '../../src/output.js';
+import { MkvOutputFormat } from '../../src/output-format.js';
+import { BufferTarget } from '../../src/target.js';
+import { EncodedVideoPacketSource } from '../../src/media-source.js';
+import { EncodedPacket } from '../../src/packet.js';
+import { Input } from '../../src/input.js';
+import { BufferSource } from '../../src/source.js';
+import { ALL_FORMATS } from '../../src/input-format.js';
+
+test('Default track disposition', async () => {
+	const output = new Output({
+		format: new MkvOutputFormat(),
+		target: new BufferTarget(),
+	});
+
+	const source = new EncodedVideoPacketSource('avc');
+	output.addVideoTrack(source);
+
+	await output.start();
+	await source.add(new EncodedPacket(new Uint8Array(1024), 'key', 0, 1), {
+		decoderConfig: {
+			codec: 'avc1.123456',
+			codedWidth: 1920,
+			codedHeight: 1080,
+		},
+	});
+
+	await output.finalize();
+
+	using input = new Input({
+		source: new BufferSource(output.target.buffer!),
+		formats: ALL_FORMATS,
+	});
+
+	const track = (await input.getPrimaryVideoTrack())!;
+
+	expect(track.disposition).toEqual({
+		default: true,
+		forced: false,
+		original: false,
+		hearingImpaired: false,
+		visuallyImpaired: false,
+		commentary: false,
+	});
+});
+
+test('Customized track disposition', async () => {
+	const output = new Output({
+		format: new MkvOutputFormat(),
+		target: new BufferTarget(),
+	});
+
+	const source = new EncodedVideoPacketSource('avc');
+	output.addVideoTrack(source, {
+		disposition: {
+			default: false,
+			forced: true,
+			original: true,
+			hearingImpaired: true,
+			visuallyImpaired: true,
+			commentary: true,
+		},
+	});
+
+	await output.start();
+	await source.add(new EncodedPacket(new Uint8Array(1024), 'key', 0, 1), {
+		decoderConfig: {
+			codec: 'avc1.123456',
+			codedWidth: 1920,
+			codedHeight: 1080,
+		},
+	});
+
+	await output.finalize();
+
+	using input = new Input({
+		source: new BufferSource(output.target.buffer!),
+		formats: ALL_FORMATS,
+	});
+
+	const track = (await input.getPrimaryVideoTrack())!;
+
+	expect(track.disposition).toEqual({
+		default: false,
+		forced: true,
+		original: true,
+		hearingImpaired: true,
+		visuallyImpaired: true,
+		commentary: true,
+	});
+});

test/node/metadata-tags.test.ts

@@ -15,7 +15,7 @@ import { EncodedPacket } from '../../src/packet.js';
 import { Input } from '../../src/input.js';
 import { BufferSource, FilePathSource } from '../../src/source.js';
 import { ALL_FORMATS } from '../../src/input-format.js';
-import { AttachedFile, MetadataTags } from '../../src/tags.js';
+import { AttachedFile, MetadataTags } from '../../src/metadata.js';
 import path from 'node:path';
 import { AudioCodec, buildAudioCodecString } from '../../src/codec.js';
 import { Conversion } from '../../src/conversion.js';

test/node/node-sample-creation.test.ts

@@ -0,0 +1,27 @@
+import { test } from 'vitest';
+import { AudioSample, VideoSample } from '../../src/sample.js';
+
+test('VideoSample creation from bytes', async () => {
+	const bytes = new Uint8Array(1024);
+	const sample = new VideoSample(bytes, {
+		codedWidth: 1280,
+		codedHeight: 720,
+		format: 'RGBA',
+		timestamp: 0,
+	});
+
+	sample.close();
+});
+
+test('AudioSample creation from bytes', async () => {
+	const bytes = new Uint8Array(1024);
+	const sample = new AudioSample({
+		data: bytes,
+		numberOfChannels: 2,
+		format: 's16',
+		sampleRate: 48000,
+		timestamp: 0,
+	});
+
+	sample.close();
+});