Add negative timestamp muxing support to Matroska

dev/mux.html

@@ -47,7 +47,7 @@
 	format = new Mediabunny.MkvOutputFormat();
 	format = new Mediabunny.MovOutputFormat();
 	format = new Mediabunny.Mp4OutputFormat({ fastStart: 'reserve' });
-	format = new Mediabunny.Mp4OutputFormat({  });
+	format = new Mediabunny.MkvOutputFormat({  });
 	let target = new Mediabunny.BufferTarget();
 
 	/*
@@ -197,7 +197,7 @@ Testing... <00:17.350>One... <00:18.125>Two...
 	const p = document.createElement('p');
 	document.body.append(p);
 
-	for (let i = 0; i < 1; i++) {
+	for (let i = 0; i < 100; i++) {
 		context.fillStyle = ['red', 'green', 'blue', 'yellow'][i % 4];
 		context.fillRect(canvas.width * Math.random(), canvas.height * Math.random(), canvas.width * Math.random(), canvas.height * Math.random());
 

src/matroska/matroska-muxer.ts

@@ -69,6 +69,7 @@ import { Writer } from '../writer';
 import { EncodedPacket } from '../packet';
 import { parseOpusIdentificationHeader } from '../codec-data';
 import { AttachedFile } from '../metadata';
+import { Logging } from '../logging';
 
 const MIN_CLUSTER_TIMESTAMP_MS = -(2 ** 15);
 const MAX_CLUSTER_TIMESTAMP_MS = 2 ** 15 - 1;
@@ -157,6 +158,7 @@ export class MatroskaMuxer extends Muxer {
 
 	private startTimestamp = Infinity;
 	private endTimestamp = -Infinity;
+	private warnedAboutTooNegativeTimestamp = false;
 
 	constructor(output: Output, format: MkvOutputFormat) {
 		super(output);
@@ -1191,6 +1193,15 @@ export class MatroskaMuxer extends Muxer {
 		const relativeTimestamp = msTimestamp - this.currentClusterStartMsTimestamp!;
 		if (relativeTimestamp < MIN_CLUSTER_TIMESTAMP_MS) {
 			// The block lies too far in the past, it's not representable within this cluster
+			if (!this.warnedAboutTooNegativeTimestamp) {
+				const formatName = this.format instanceof WebMOutputFormat ? 'WebM' : 'Matroska';
+				Logging._warn(
+					`Packets had to be discarded because their timestamp is too negative to represent in`
+					+ ` ${formatName}.`,
+				);
+				this.warnedAboutTooNegativeTimestamp = true;
+			}
+
 			return;
 		}
 
@@ -1253,6 +1264,8 @@ export class MatroskaMuxer extends Muxer {
 
 	/** Creates a new Cluster element to contain media chunks. */
 	private createNewCluster(msTimestamp: number) {
+		msTimestamp = Math.max(0, msTimestamp); // Cluster timestamps cannot be negative
+
 		if (this.currentCluster) {
 			this.finalizeCurrentCluster();
 		}
@@ -1313,7 +1326,7 @@ export class MatroskaMuxer extends Muxer {
 		for (const [msTimestamp, trackDatas] of groupedAndSortedByTimestamp) {
 			assert(this.cues);
 			(this.cues.data as EBML[]).push({ id: EBMLId.CuePoint, data: [
-				{ id: EBMLId.CueTime, data: msTimestamp },
+				{ id: EBMLId.CueTime, data: Math.max(0, msTimestamp) }, // CueTime is unsigned
 				// Create CueTrackPositions for each track that starts at this timestamp
 				...trackDatas.map((trackData) => {
 					return { id: EBMLId.CueTrackPositions, data: [

test/node/isobmff-muxer.test.ts

@@ -221,11 +221,11 @@ test('Non-zero start timestamp, fragmented MP4', async () => {
 });
 
 test('Negative start timestamps, regular MP4', async () => {
-	await testNegativeTimestampRoundTrip([-1, 0, 1, 2, 3], 1, false);
+	await testNegativeTimestampRoundTrip(Array.from({ length: 50 }, (_, index) => (index - 10) / 10), 0.1, false);
 });
 
 test('Negative start timestamps, fragmented MP4', async () => {
-	await testNegativeTimestampRoundTrip([-1, 0, 1, 2, 3], 1, true);
+	await testNegativeTimestampRoundTrip(Array.from({ length: 50 }, (_, index) => (index - 10) / 10), 0.1, true);
 });
 
 test('Wholly negative timestamps, regular MP4', async () => {
@@ -247,7 +247,7 @@ const testNegativeTimestampRoundTrip = async (
 	});
 
 	const source = new EncodedVideoPacketSource('vp8');
-	output.addVideoTrack(source);
+	output.addVideoTrack(source, { frameRate: 10 });
 
 	await output.start();
 
@@ -272,23 +272,33 @@ const testNegativeTimestampRoundTrip = async (
 
 	const track = await input.getPrimaryVideoTrack();
 	assert(track);
+	const sink = new EncodedPacketSink(track);
 
 	const outputPackets: EncodedPacket[] = [];
-	for await (const packet of new EncodedPacketSink(track).packets()) {
+	for await (const packet of sink.packets()) {
 		outputPackets.push(packet);
 	}
 
 	expect(outputPackets.map(packet => ({
-		data: packet.data,
-		type: packet.type,
 		timestamp: packet.timestamp,
 		duration: packet.duration,
 	}))).toEqual(inputPackets.map(packet => ({
-		data: packet.data,
-		type: packet.type,
 		timestamp: packet.timestamp,
 		duration: packet.duration,
 	})));
+
+	for (const inputPacket of inputPackets) {
+		const outputPacket = await sink.getPacket(inputPacket.timestamp);
+		assert(outputPacket);
+
+		expect({
+			timestamp: outputPacket.timestamp,
+			duration: outputPacket.duration,
+		}).toEqual({
+			timestamp: inputPacket.timestamp,
+			duration: inputPacket.duration,
+		});
+	}
 };
 
 test('PCM audio', async () => {

test/node/matroska-muxer.test.ts

@@ -9,6 +9,8 @@ import { BufferTarget } from '../../src/target.js';
 import { MkvOutputFormat } from '../../src/output-format.js';
 import { Conversion } from '../../src/conversion.js';
 import { assert } from '../../src/misc.js';
+import { EncodedVideoPacketSource } from '../../src/media-source.js';
+import { EncodedPacket } from '../../src/packet.js';
 
 const __dirname = new URL('.', import.meta.url).pathname;
 
@@ -61,3 +63,76 @@ test('Matroska muxer internally converts ADTS to AAC', async () => {
 
 	expect(count).toBe(4557);
 });
+
+test('Negative start timestamps', async () => {
+	await testNegativeTimestampRoundTrip(
+		Array.from({ length: 50 }, (_, index) => (index - 10) / 10),
+		0.1,
+		10,
+	);
+});
+
+test('Wholly negative timestamps', async () => {
+	await testNegativeTimestampRoundTrip([-1, -0.9, -0.8, -0.7, -0.6], 0.1, 10);
+});
+
+const testNegativeTimestampRoundTrip = async (timestamps: number[], duration: number, frameRate: number) => {
+	const output = new Output({
+		format: new MkvOutputFormat(),
+		target: new BufferTarget(),
+	});
+
+	const source = new EncodedVideoPacketSource('vp8');
+	output.addVideoTrack(source, { frameRate });
+
+	await output.start();
+
+	const meta = { decoderConfig: { codec: 'vp8', codedWidth: 1280, codedHeight: 720 } };
+	const inputPackets = timestamps.map((timestamp, index) => new EncodedPacket(
+		new Uint8Array(1024).fill(index),
+		'key',
+		timestamp,
+		duration,
+	));
+
+	for (let i = 0; i < inputPackets.length; i++) {
+		await source.add(inputPackets[i]!, i === 0 ? meta : undefined);
+	}
+
+	await output.finalize();
+
+	using input = new Input({
+		source: new BufferSource(output.target.buffer!),
+		formats: ALL_FORMATS,
+	});
+
+	const track = await input.getPrimaryVideoTrack();
+	assert(track);
+	const sink = new EncodedPacketSink(track);
+
+	const outputPackets: EncodedPacket[] = [];
+	for await (const packet of sink.packets()) {
+		outputPackets.push(packet);
+	}
+
+	expect(outputPackets.map(packet => ({
+		timestamp: packet.timestamp,
+		duration: packet.duration,
+	}))).toEqual(inputPackets.map(packet => ({
+		timestamp: packet.timestamp,
+		duration: packet.duration,
+	})));
+
+	for (const inputPacket of inputPackets) {
+		const outputPacket = await sink.getPacket(inputPacket.timestamp);
+		assert(outputPacket);
+
+		expect({
+			timestamp: outputPacket.timestamp,
+			duration: outputPacket.duration,
+		}).toEqual({
+			timestamp: inputPacket.timestamp,
+			duration: inputPacket.duration,
+		});
+	}
+};