Allow non-zero initial timestamps for ISOBMFF muxer

src/isobmff/isobmff-demuxer.ts

@@ -88,6 +88,11 @@ type SampleTable = {
 		presentationTimestamp: number;
 		sampleIndex: number;
 	}[] | null;
+	/**
+	 * Provides a fast map from sample index to index in the sorted presentation timestamps array - so, a fast map from
+	 * decode order to presentation order.
+	 */
+	presentationTimestampIndexMap: number[] | null;
 };
 type SampleTimingEntry = {
 	startIndex: number;
@@ -272,6 +277,7 @@ export class IsobmffDemuxer extends Demuxer {
 			chunkOffsets: [],
 			sampleToChunk: [],
 			presentationTimestamps: null,
+			presentationTimestampIndexMap: null,
 		};
 		internalTrack.sampleTable = sampleTable;
 
@@ -389,6 +395,11 @@ export class IsobmffDemuxer extends Demuxer {
 			}
 
 			sampleTable.presentationTimestamps.sort((a, b) => a.presentationTimestamp - b.presentationTimestamp);
+
+			sampleTable.presentationTimestampIndexMap = Array(sampleTable.presentationTimestamps.length).fill(-1);
+			for (let i = 0; i < sampleTable.presentationTimestamps.length; i++) {
+				sampleTable.presentationTimestampIndexMap[sampleTable.presentationTimestamps[i]!.sampleIndex] = i;
+			}
 		} else {
 			// If they're not defined, we can simply use the decode timestamps as presentation timestamps
 		}
@@ -1624,6 +1635,15 @@ export class IsobmffDemuxer extends Demuxer {
 					.map((x, i) => ({ presentationTimestamp: x.presentationTimestamp, sampleIndex: i }))
 					.sort((a, b) => a.presentationTimestamp - b.presentationTimestamp);
 
+				// Update sample durations based on presentation order
+				for (let i = 0; i < trackData.presentationTimestamps.length - 1; i++) {
+					const current = trackData.presentationTimestamps[i]!;
+					const next = trackData.presentationTimestamps[i + 1]!;
+
+					const duration = next.presentationTimestamp - current.presentationTimestamp;
+					trackData.samples[current.sampleIndex]!.duration = duration;
+				}
+
 				const firstSample = trackData.samples[trackData.presentationTimestamps[0]!.sampleIndex]!;
 				const lastSample = trackData.samples[last(trackData.presentationTimestamps)!.sampleIndex]!;
 
@@ -2319,9 +2339,23 @@ const getSampleInfo = (sampleTable: SampleTable, sampleIndex: number): SampleInf
 		}
 	}
 
+	let duration = timingEntry.delta;
+	if (sampleTable.presentationTimestamps) {
+		// In order to accurately compute the duration, we need to take the duration to the next sample in presentation
+		// order, not in decode order
+		const presentationIndex = sampleTable.presentationTimestampIndexMap![sampleIndex];
+		assert(presentationIndex !== undefined);
+
+		if (presentationIndex < sampleTable.presentationTimestamps.length - 1) {
+			const nextEntry = sampleTable.presentationTimestamps[presentationIndex + 1]!;
+			const nextPresentationTimestamp = nextEntry.presentationTimestamp;
+			duration = nextPresentationTimestamp - presentationTimestamp;
+		}
+	}
+
 	return {
 		presentationTimestamp,
-		duration: timingEntry.delta,
+		duration,
 		sampleOffset,
 		sampleSize,
 		chunkOffset,

src/isobmff/isobmff-muxer.ts

@@ -85,8 +85,6 @@ export const intoTimescale = (timeInSeconds: number, timescale: number, round =
 };
 
 export class IsobmffMuxer extends Muxer {
-	override timestampsMustStartAtZero = true;
-
 	private writer: Writer;
 	private boxWriter: IsobmffBoxWriter;
 	private fastStart: NonNullable<Mp4OutputFormatOptions['fastStart']>;
@@ -263,10 +261,6 @@ export class IsobmffMuxer extends Muxer {
 		this.trackDatas.push(newTrackData);
 		this.trackDatas.sort((a, b) => a.track.id - b.track.id);
 
-		// Subtitle cues don't need to start at 0, so let's register a timestamp at 0 to satisfy the
-		// "timestamps must start a zero" constraint
-		this.validateAndNormalizeTimestamp(track, 0, true);
-
 		return newTrackData;
 	}
 
@@ -446,8 +440,7 @@ export class IsobmffMuxer extends Muxer {
 			data,
 			size: data.byteLength,
 			type,
-			// Will be refined once the next sample comes in
-			timescaleUnitsToNextSample: intoTimescale(duration, trackData.timescale),
+			timescaleUnitsToNextSample: intoTimescale(duration, trackData.timescale), // Will be refined
 		};
 
 		return sample;
@@ -469,6 +462,12 @@ export class IsobmffMuxer extends Muxer {
 			// model it.
 			sample.decodeTimestamp = sortedTimestamps[i]!;
 
+			if (this.fastStart !== 'fragmented' && trackData.lastTimescaleUnits === null) {
+				// In non-fragmented files, the first decode timestamp is always zero. If the first presentation
+				// timestamp isn't zero, we'll simply use the composition time offset to achieve it.
+				sample.decodeTimestamp = 0;
+			}
+
 			const sampleCompositionTimeOffset
 				= intoTimescale(sample.timestamp - sample.decodeTimestamp, trackData.timescale);
 			const durationInTimescale = intoTimescale(sample.duration, trackData.timescale);
@@ -533,7 +532,8 @@ export class IsobmffMuxer extends Muxer {
 					}
 				}
 			} else {
-				trackData.lastTimescaleUnits = 0;
+				// Decode timestamp of the first sample
+				trackData.lastTimescaleUnits = intoTimescale(sample.decodeTimestamp, trackData.timescale, false);
 
 				if (this.fastStart !== 'fragmented') {
 					trackData.timeToSampleTable.push({
@@ -581,6 +581,10 @@ export class IsobmffMuxer extends Muxer {
 				// We can only finalize this fragment (and begin a new one) if we know that each track will be able to
 				// start the new one with a key frame.
 				const keyFrameQueuedEverywhere = this.trackDatas.every((otherTrackData) => {
+					if (otherTrackData.track.source._closed) {
+						return true;
+					}
+
 					if (trackData === otherTrackData) {
 						return sample.type === 'key';
 					}