Fix CTS=0 in fragmented fMP4 with multiple tracks (#317)

* Fix CTS=0 in fragmented fMP4 with multiple tracks

When muxing fragmented MP4 with both video and audio tracks,
compositionTimeOffset (CTS) was zero for all video samples, causing
B-frame content to display in decode order instead of presentation
order (visible judder).

Root cause: During finalize(), interleaveSamples(true) triggers
finalizeFragment() via the cross-track keyframe check in
addSampleToTrack. This writes the trun box before processTimestamps
has computed correct decodeTimestamp values, so CTS = PTS - DTS = 0.

Fix: Call processTimestamps() for all tracks at the start of
finalizeFragment(). This is safe because processTimestamps is a no-op
when the timestampProcessingQueue is empty.

Video-only fragmented muxing was unaffected because the single-track
case never triggers the cross-track keyframe check during
interleaveSamples.

Co-Authored-By: Claude Opus 4.6 <[email protected]>

* Fix the problem at the actual root

* Oopsie doopsie

---------

Co-authored-by: Claude Opus 4.6 <[email protected]>
Co-authored-by: Vanilagy <[email protected]>

dev/convert.html

@@ -26,7 +26,7 @@
 			chunked: true,
 			chunkSize: 2**20
 		});
-		const outputFormat = new Mediabunny.Mp4OutputFormat({});
+		const outputFormat = new Mediabunny.Mp4OutputFormat({fastStart: 'fragmented'});
 
 		const button = document.createElement('button');
 		button.textContent = 'Cancel';

dev/demux.html

@@ -18,10 +18,11 @@
 		});
 
 		const videoTrack = await input.getPrimaryVideoTrack();
-		const sink = new Mediabunny.VideoSampleSink(videoTrack);
+		const sink = new Mediabunny.EncodedPacketSink(videoTrack);
 
-		const sample = await sink.getSample(await videoTrack.getFirstTimestamp());
+		for await (const packet of sink.packets()) {
-		console.log(sample);
+			console.log(packet.timestamp);
+		}
 
 		/*
 		

src/isobmff/isobmff-muxer.ts

@@ -1245,11 +1245,13 @@ export class IsobmffMuxer extends Muxer {
 	override async onTrackClose(track: OutputTrack) {
 		const release = await this.mutex.acquire();
 
-		if (track.type === 'subtitle' && track.source._codec === 'webvtt') {
+		const trackData = this.trackDatas.find(x => x.track === track);
-			const trackData = this.trackDatas.find(x => x.track === track) as IsobmffSubtitleTrackData;
 		if (trackData) {
+			if (trackData.type === 'subtitle' && track.source._codec === 'webvtt') {
 				await this.processWebVTTCues(trackData, Infinity);
 			}
+
+			this.processTimestamps(trackData);
 		}
 
 		if (this.allTracksAreKnown()) {
@@ -1274,19 +1276,15 @@ export class IsobmffMuxer extends Muxer {
 			if (trackData.type === 'subtitle' && trackData.track.source._codec === 'webvtt') {
 				await this.processWebVTTCues(trackData, Infinity);
 			}
+
+			this.processTimestamps(trackData);
 		}
 
 		if (this.isFragmented) {
 			await this.interleaveSamples(true);
-
-			for (const trackData of this.trackDatas) {
-				this.processTimestamps(trackData);
-			}
-
 			await this.finalizeFragment(false); // Don't flush the last fragment as we will flush it with the mfra box
 		} else {
 			for (const trackData of this.trackDatas) {
-				this.processTimestamps(trackData);
 				await this.finalizeCurrentChunk(trackData);
 			}
 		}

test/node/isobmff-muxer.test.ts

@@ -61,3 +61,46 @@ test('ISOBMFF muxer internally converts ADTS to AAC', async () => {
 
 	expect(count).toBe(4557);
 });
+
+test('Fragmented fMP4 with video+audio preserves B-frame CTS', async () => {
+	using input = new Input({
+		source: new FilePathSource(path.join(__dirname, '../public/video.mp4')),
+		formats: ALL_FORMATS,
+	});
+
+	const videoTrack = await input.getPrimaryVideoTrack();
+	const audioTrack = await input.getPrimaryAudioTrack();
+	assert(videoTrack);
+	assert(audioTrack);
+
+	const originalVideoSink = new EncodedPacketSink(videoTrack);
+	const originalTimestamps: number[] = [];
+	for await (const packet of originalVideoSink.packets()) {
+		originalTimestamps.push(packet.timestamp);
+	}
+
+	const output = new Output({
+		format: new Mp4OutputFormat({ fastStart: 'fragmented' }),
+		target: new BufferTarget(),
+	});
+
+	const conversion = await Conversion.init({ input, output, showWarnings: false });
+	await conversion.execute();
+
+	using outputAsInput = new Input({
+		source: new BufferSource(output.target.buffer!),
+		formats: ALL_FORMATS,
+	});
+
+	const outputVideoTrack = await outputAsInput.getPrimaryVideoTrack();
+	assert(outputVideoTrack);
+
+	const videoSink = new EncodedPacketSink(outputVideoTrack);
+
+	const timestamps: number[] = [];
+	for await (const packet of videoSink.packets()) {
+		timestamps.push(packet.timestamp);
+	}
+
+	expect(timestamps).toEqual(originalTimestamps);
+});