From 6a11a1302c772bfb58c6975966478c869690cd46 Mon Sep 17 00:00:00 2001 From: Kyle Graehl Date: Fri, 6 Mar 2026 10:56:53 +0100 Subject: [PATCH] Fix CTS=0 in fragmented fMP4 with multiple tracks (#317) * Fix CTS=0 in fragmented fMP4 with multiple tracks When muxing fragmented MP4 with both video and audio tracks, compositionTimeOffset (CTS) was zero for all video samples, causing B-frame content to display in decode order instead of presentation order (visible judder). Root cause: During finalize(), interleaveSamples(true) triggers finalizeFragment() via the cross-track keyframe check in addSampleToTrack. This writes the trun box before processTimestamps has computed correct decodeTimestamp values, so CTS = PTS - DTS = 0. Fix: Call processTimestamps() for all tracks at the start of finalizeFragment(). This is safe because processTimestamps is a no-op when the timestampProcessingQueue is empty. Video-only fragmented muxing was unaffected because the single-track case never triggers the cross-track keyframe check during interleaveSamples. Co-Authored-By: Claude Opus 4.6 * Fix the problem at the actual root * Oopsie doopsie --------- Co-authored-by: Claude Opus 4.6 Co-authored-by: Vanilagy <1696106+Vanilagy@users.noreply.github.com> --- dev/convert.html | 2 +- dev/demux.html | 7 +++--- src/isobmff/isobmff-muxer.ts | 16 ++++++------ test/node/isobmff-muxer.test.ts | 43 +++++++++++++++++++++++++++++++++ 4 files changed, 55 insertions(+), 13 deletions(-) diff --git a/dev/convert.html b/dev/convert.html index 96c013a..9f18bfd 100644 --- a/dev/convert.html +++ b/dev/convert.html @@ -26,7 +26,7 @@ chunked: true, chunkSize: 2**20 }); - const outputFormat = new Mediabunny.Mp4OutputFormat({}); + const outputFormat = new Mediabunny.Mp4OutputFormat({fastStart: 'fragmented'}); const button = document.createElement('button'); button.textContent = 'Cancel'; diff --git a/dev/demux.html b/dev/demux.html index 233aae3..baa5541 100644 --- a/dev/demux.html +++ b/dev/demux.html @@ -18,10 +18,11 @@ }); const videoTrack = await input.getPrimaryVideoTrack(); - const sink = new Mediabunny.VideoSampleSink(videoTrack); + const sink = new Mediabunny.EncodedPacketSink(videoTrack); - const sample = await sink.getSample(await videoTrack.getFirstTimestamp()); - console.log(sample); + for await (const packet of sink.packets()) { + console.log(packet.timestamp); + } /* diff --git a/src/isobmff/isobmff-muxer.ts b/src/isobmff/isobmff-muxer.ts index 4256f53..b7a6158 100644 --- a/src/isobmff/isobmff-muxer.ts +++ b/src/isobmff/isobmff-muxer.ts @@ -1245,11 +1245,13 @@ export class IsobmffMuxer extends Muxer { override async onTrackClose(track: OutputTrack) { const release = await this.mutex.acquire(); - if (track.type === 'subtitle' && track.source._codec === 'webvtt') { - const trackData = this.trackDatas.find(x => x.track === track) as IsobmffSubtitleTrackData; - if (trackData) { + const trackData = this.trackDatas.find(x => x.track === track); + if (trackData) { + if (trackData.type === 'subtitle' && track.source._codec === 'webvtt') { await this.processWebVTTCues(trackData, Infinity); } + + this.processTimestamps(trackData); } if (this.allTracksAreKnown()) { @@ -1274,19 +1276,15 @@ export class IsobmffMuxer extends Muxer { if (trackData.type === 'subtitle' && trackData.track.source._codec === 'webvtt') { await this.processWebVTTCues(trackData, Infinity); } + + this.processTimestamps(trackData); } if (this.isFragmented) { await this.interleaveSamples(true); - - for (const trackData of this.trackDatas) { - this.processTimestamps(trackData); - } - await this.finalizeFragment(false); // Don't flush the last fragment as we will flush it with the mfra box } else { for (const trackData of this.trackDatas) { - this.processTimestamps(trackData); await this.finalizeCurrentChunk(trackData); } } diff --git a/test/node/isobmff-muxer.test.ts b/test/node/isobmff-muxer.test.ts index 55cee50..83d1d9e 100644 --- a/test/node/isobmff-muxer.test.ts +++ b/test/node/isobmff-muxer.test.ts @@ -61,3 +61,46 @@ test('ISOBMFF muxer internally converts ADTS to AAC', async () => { expect(count).toBe(4557); }); + +test('Fragmented fMP4 with video+audio preserves B-frame CTS', async () => { + using input = new Input({ + source: new FilePathSource(path.join(__dirname, '../public/video.mp4')), + formats: ALL_FORMATS, + }); + + const videoTrack = await input.getPrimaryVideoTrack(); + const audioTrack = await input.getPrimaryAudioTrack(); + assert(videoTrack); + assert(audioTrack); + + const originalVideoSink = new EncodedPacketSink(videoTrack); + const originalTimestamps: number[] = []; + for await (const packet of originalVideoSink.packets()) { + originalTimestamps.push(packet.timestamp); + } + + const output = new Output({ + format: new Mp4OutputFormat({ fastStart: 'fragmented' }), + target: new BufferTarget(), + }); + + const conversion = await Conversion.init({ input, output, showWarnings: false }); + await conversion.execute(); + + using outputAsInput = new Input({ + source: new BufferSource(output.target.buffer!), + formats: ALL_FORMATS, + }); + + const outputVideoTrack = await outputAsInput.getPrimaryVideoTrack(); + assert(outputVideoTrack); + + const videoSink = new EncodedPacketSink(outputVideoTrack); + + const timestamps: number[] = []; + for await (const packet of videoSink.packets()) { + timestamps.push(packet.timestamp); + } + + expect(timestamps).toEqual(originalTimestamps); +});