From f5a56e1f62b1ddc25633c2d642225ec62ca9a189 Mon Sep 17 00:00:00 2001 From: Vanilagy <1696106+Vanilagy@users.noreply.github.com> Date: Sat, 7 Jun 2025 18:54:19 +0200 Subject: [PATCH] Fix incorrectly processed timestamps for fragmented MP4 --- dev/convert.html | 8 ++++---- dev/demux.html | 18 +++++++++++++--- src/isobmff/isobmff-muxer.ts | 40 ++++++++++++++++++++++++++++++------ 3 files changed, 53 insertions(+), 13 deletions(-) diff --git a/dev/convert.html b/dev/convert.html index 94c7224..c49ed91 100644 --- a/dev/convert.html +++ b/dev/convert.html @@ -21,7 +21,7 @@ chunked: true, chunkSize: 2**20 }); - const outputFormat = new Metamuxer.Mp4OutputFormat(); + const outputFormat = new Metamuxer.Mp4OutputFormat({ fastStart: 'fragmented' }); const button = document.createElement('button'); button.textContent = 'Cancel'; @@ -41,7 +41,7 @@ //numberOfChannels: 1, //sampleRate: 4000 //discard: true - forceReencode: true, + //forceReencode: true, }, /* video: { @@ -72,7 +72,7 @@ //width: 1280, //discard: true, //width: 640 - forceReencode: true, + //forceReencode: true, //rotate: 90 //width: 720 ?? 2160, //height: 1280 ?? 3840, @@ -85,7 +85,7 @@ }, trim: { start: 0, - end: 40 + end: 3 }, }); console.log(conversion); diff --git a/dev/demux.html b/dev/demux.html index 9e503ee..84863bc 100644 --- a/dev/demux.html +++ b/dev/demux.html @@ -15,10 +15,17 @@ formats: Metamuxer.ALL_FORMATS, source }); - - let timestamps = []; + const videoTrack = await input.getPrimaryVideoTrack(); + const sink = new Metamuxer.EncodedPacketSink(videoTrack); + for await (const packet of sink.packets()) { + if (packet.timestamp >= 2) break; + console.log(packet); + } + + /* + let timestamps = []; const packetSink = new Metamuxer.EncodedPacketSink(videoTrack); for await (const packet of packetSink.packets()) { if (packet.timestamp >= 1) { @@ -71,22 +78,25 @@ 7.253333333333333, 7.253333333333333, ]; + */ - console.log(timestamps); //return; /* + console.log(timestamps); const sink = new Metamuxer.EncodedPacketSink(videoTrack); for await (const packet of sink.packets()) { console.log(packet.sequenceNumber); } */ + /* const sink = new Metamuxer.VideoSampleSink(videoTrack); for await (const sample of sink.samplesAtTimestamps(timestamps)) { console.log("wee", sample.timestamp); sample.close(); } console.log("don") + */ /* const sink = new Metamuxer.CanvasSink(videoTrack, { width: 320 }); @@ -100,6 +110,7 @@ + /* for (let i = 0; i < 40; i++) { const count = Math.floor(50 * Math.random()); const timestamps = Array.from({ length: count }, () => Math.random() * 20).sort((a, b) => a-b); @@ -112,6 +123,7 @@ } console.log("DONE!") + */ diff --git a/src/isobmff/isobmff-muxer.ts b/src/isobmff/isobmff-muxer.ts index 27537e9..234f349 100644 --- a/src/isobmff/isobmff-muxer.ts +++ b/src/isobmff/isobmff-muxer.ts @@ -37,6 +37,7 @@ export type Sample = { }; type Chunk = { + /** The lowest presentation timestamp in this chunk */ startTimestamp: number; samples: Sample[]; offset: number | null; @@ -592,7 +593,7 @@ export class IsobmffMuxer extends Muxer { return sample; } - private processTimestamps(trackData: IsobmffTrackData) { + private processTimestamps(trackData: IsobmffTrackData, nextSample?: Sample) { if (trackData.timestampProcessingQueue.length === 0) { return; } @@ -725,9 +726,29 @@ export class IsobmffMuxer extends Muxer { } trackData.timestampProcessingQueue.length = 0; + + assert(trackData.lastSample); + assert(trackData.lastTimescaleUnits !== null); + + if (nextSample !== undefined && trackData.lastSample.timescaleUnitsToNextSample === 0) { + assert(nextSample.type === 'key'); + + // Given the next sample, we can make a guess about the duration of the last sample. This avoids having + // the last sample's duration in each fragment be "0" for fragmented files. The guess we make here is + // actually correct most of the time, since typically, no delta frame with a lower timestamp follows the key + // frame (although it can happen). + const timescaleUnits = intoTimescale(nextSample.timestamp, trackData.timescale, false); + const delta = Math.round(timescaleUnits - trackData.lastTimescaleUnits); + trackData.lastSample.timescaleUnitsToNextSample = delta; + } } private async registerSample(trackData: IsobmffTrackData, sample: Sample) { + if (sample.type === 'key') { + this.processTimestamps(trackData, sample); + } + trackData.timestampProcessingQueue.push(sample); + if (this.isFragmented) { trackData.sampleQueue.push(sample); await this.interleaveSamples(); @@ -737,10 +758,6 @@ export class IsobmffMuxer extends Muxer { } private async addSampleToTrack(trackData: IsobmffTrackData, sample: Sample) { - if (sample.type === 'key') { - this.processTimestamps(trackData); - } - if (!this.isFragmented) { trackData.samples.push(sample); } @@ -749,6 +766,13 @@ export class IsobmffMuxer extends Muxer { if (!trackData.currentChunk) { beginNewChunk = true; } else { + // Timestamp don't need to be monotonic (think B-frames), so we may need to update the start timestamp of + // the chunk + trackData.currentChunk.startTimestamp = Math.min( + trackData.currentChunk.startTimestamp, + sample.timestamp, + ); + const currentChunkDuration = sample.timestamp - trackData.currentChunk.startTimestamp; if (this.isFragmented) { @@ -795,7 +819,6 @@ export class IsobmffMuxer extends Muxer { assert(trackData.currentChunk); trackData.currentChunk.samples.push(sample); - trackData.timestampProcessingQueue.push(sample); if (this.isFragmented) { this.maxWrittenTimestamp = Math.max(this.maxWrittenTimestamp, sample.timestamp); @@ -1003,6 +1026,11 @@ export class IsobmffMuxer extends Muxer { if (this.isFragmented) { await this.interleaveSamples(true); + + for (const trackData of this.trackDatas) { + this.processTimestamps(trackData); + } + await this.finalizeFragment(false); // Don't flush the last fragment as we will flush it with the mfra box } else { for (const trackData of this.trackDatas) {