Fix decoded frame timestamps (#49)

dev/demux.html

@@ -16,6 +16,14 @@
 			source
 		});
 
+		const videoTrack = await input.getPrimaryVideoTrack();
+		const sink = new Mediabunny.VideoSampleSink(videoTrack);
+
+		for await (const sample of sink.samples(0, 2)) {
+			console.log(sample.timestamp);
+			sample.close();
+		}
+
 		/*
 		const audioTrack = await input.getPrimaryAudioTrack();
 		const sink = new Mediabunny.EncodedPacketSink(audioTrack);

src/media-sink.ts

@@ -17,6 +17,7 @@ import {
 	CallSerializer,
 	getInt24,
 	getUint24,
+	isSafari,
 	last,
 	mapAsyncGenerator,
 	promiseWithResolvers,
@@ -750,7 +751,8 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
 	customDecoderCallSerializer = new CallSerializer();
 	customDecoderQueueSize = 0;
 
-	sampleQueue: VideoSample[] = [];
+	inputTimestamps: number[] = []; // Timestamps input into the decoder, sorted.
+	sampleQueue: VideoSample[] = []; // Safari-specific thing, check usage.
 
 	constructor(
 		onSample: (sample: VideoSample) => unknown,
@@ -762,28 +764,6 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
 	) {
 		super(onSample, onError);
 
-		const sampleHandler = (sample: VideoSample) => {
-			// For correct B-frame handling, we don't just hand over the frames directly but instead add them to a
-			// queue, because we want to ensure frames are emitted in presentation order. We flush the queue each time
-			// we receive a frame with a timestamp larger than the highest we've seen so far, as we can sure that is
-			// not a B-frame. Typically, WebCodecs automatically guarantees that frames are emitted in presentation
-			// order, but some browsers (Safari) don't always follow this rule.
-			if (this.sampleQueue.length > 0 && (sample.timestamp >= last(this.sampleQueue)!.timestamp)) {
-				for (const sample of this.sampleQueue) {
-					this.finalizeAndEmitSample(sample);
-				}
-
-				this.sampleQueue.length = 0;
-			}
-
-			const insertionIndex = binarySearchLessOrEqual(
-				this.sampleQueue,
-				sample.timestamp,
-				x => x.timestamp,
-			);
-			this.sampleQueue.splice(insertionIndex + 1, 0, sample);
-		};
-
 		const MatchingCustomDecoder = customVideoDecoders.find(x => x.supports(codec, decoderConfig));
 		if (MatchingCustomDecoder) {
 			// @ts-expect-error "Can't create instance of abstract class 🤓"
@@ -798,11 +778,50 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
 					throw new TypeError('The argument passed to onSample must be a VideoSample.');
 				}
 
-				sampleHandler(sample);
+				this.finalizeAndEmitSample(sample);
 			};
 
 			void this.customDecoderCallSerializer.call(() => this.customDecoder!.init());
 		} else {
+			// Specific handler for the WebCodecs VideoDecoder to iron out browser differences
+			const sampleHandler = (sample: VideoSample) => {
+				if (isSafari()) {
+					// For correct B-frame handling, we don't just hand over the frames directly but instead add them to
+					// a queue, because we want to ensure frames are emitted in presentation order. We flush the queue
+					// each time we receive a frame with a timestamp larger than the highest we've seen so far, as we
+					// can sure that is not a B-frame. Typically, WebCodecs automatically guarantees that frames are
+					// emitted in presentation order, but Safari doesn't always follow this rule.
+					if (this.sampleQueue.length > 0 && (sample.timestamp >= last(this.sampleQueue)!.timestamp)) {
+						for (const sample of this.sampleQueue) {
+							this.finalizeAndEmitSample(sample);
+						}
+
+						this.sampleQueue.length = 0;
+					}
+
+					const insertionIndex = binarySearchLessOrEqual(
+						this.sampleQueue,
+						sample.timestamp,
+						x => x.timestamp,
+					);
+					this.sampleQueue.splice(insertionIndex + 1, 0, sample);
+				} else {
+					// Assign it the next earliest timestamp from the input. We do this because browsers, by spec, are
+					// required to emit decoded frames in presentation order *while* retaining the timestamp of their
+					// originating EncodedVideoChunk. For files with B-frames but no out-of-order timestamps (like a
+					// missing ctts box, for example), this causes a mismatch. We therefore fix the timestamps and
+					// ensure they are sorted by doing this.
+					const timestamp = this.inputTimestamps.shift();
+
+					// There's no way we'd have more decoded frames than encoded packets we passed in. Actually, the
+					// correspondence should be 1:1.
+					assert(timestamp !== undefined);
+
+					sample.setTimestamp(timestamp);
+					this.finalizeAndEmitSample(sample);
+				}
+			};
+
 			this.decoder = new VideoDecoder({
 				output: frame => sampleHandler(new VideoSample(frame)),
 				error: onError,
@@ -837,6 +856,16 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
 				.then(() => this.customDecoderQueueSize--);
 		} else {
 			assert(this.decoder);
+
+			if (!isSafari()) {
+				const insertionIndex = binarySearchLessOrEqual(
+					this.inputTimestamps,
+					packet.timestamp,
+					x => x,
+				);
+				this.inputTimestamps.splice(insertionIndex + 1, 0, packet.timestamp);
+			}
+
 			this.decoder.decode(packet.toEncodedVideoChunk());
 		}
 	}
@@ -849,10 +878,13 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
 			await this.decoder.flush();
 		}
 
-		for (const sample of this.sampleQueue) {
-			this.finalizeAndEmitSample(sample);
+		if (isSafari()) {
+			for (const sample of this.sampleQueue) {
+				this.finalizeAndEmitSample(sample);
+			}
+
+			this.sampleQueue.length = 0;
 		}
-		this.sampleQueue.length = 0;
 	}
 
 	close() {

src/misc.ts

@@ -586,3 +586,21 @@ export class CallSerializer {
 		return this.currentPromise = this.currentPromise.then(fn);
 	}
 }
+
+let isSafariCache: boolean | null = null;
+export const isSafari = () => {
+	if (isSafariCache !== null) {
+		return isSafariCache;
+	}
+
+	const result = !!(
+		typeof navigator !== 'undefined'
+		&& navigator.vendor?.match(/apple/i)
+		&& !navigator.userAgent?.match(/crios/i)
+		&& !navigator.userAgent?.match(/fxios/i)
+		&& !navigator.userAgent?.match(/Opera|OPT\//)
+	);
+
+	isSafariCache = result;
+	return result;
+};