From 56ae97f99476f1a442dbb9a13a67d682f2a36dea Mon Sep 17 00:00:00 2001 From: Vanilagy <1696106+Vanilagy@users.noreply.github.com> Date: Sun, 10 Aug 2025 19:26:41 +0200 Subject: [PATCH] Fix decoded frame timestamps (#49) --- dev/demux.html | 8 +++++ src/media-sink.ts | 86 ++++++++++++++++++++++++++++++++--------------- src/misc.ts | 18 ++++++++++ 3 files changed, 85 insertions(+), 27 deletions(-) diff --git a/dev/demux.html b/dev/demux.html index f4357ab..73a6a79 100644 --- a/dev/demux.html +++ b/dev/demux.html @@ -16,6 +16,14 @@ source }); + const videoTrack = await input.getPrimaryVideoTrack(); + const sink = new Mediabunny.VideoSampleSink(videoTrack); + + for await (const sample of sink.samples(0, 2)) { + console.log(sample.timestamp); + sample.close(); + } + /* const audioTrack = await input.getPrimaryAudioTrack(); const sink = new Mediabunny.EncodedPacketSink(audioTrack); diff --git a/src/media-sink.ts b/src/media-sink.ts index 578acb3..09404a4 100644 --- a/src/media-sink.ts +++ b/src/media-sink.ts @@ -17,6 +17,7 @@ import { CallSerializer, getInt24, getUint24, + isSafari, last, mapAsyncGenerator, promiseWithResolvers, @@ -750,7 +751,8 @@ class VideoDecoderWrapper extends DecoderWrapper { customDecoderCallSerializer = new CallSerializer(); customDecoderQueueSize = 0; - sampleQueue: VideoSample[] = []; + inputTimestamps: number[] = []; // Timestamps input into the decoder, sorted. + sampleQueue: VideoSample[] = []; // Safari-specific thing, check usage. constructor( onSample: (sample: VideoSample) => unknown, @@ -762,28 +764,6 @@ class VideoDecoderWrapper extends DecoderWrapper { ) { super(onSample, onError); - const sampleHandler = (sample: VideoSample) => { - // For correct B-frame handling, we don't just hand over the frames directly but instead add them to a - // queue, because we want to ensure frames are emitted in presentation order. We flush the queue each time - // we receive a frame with a timestamp larger than the highest we've seen so far, as we can sure that is - // not a B-frame. Typically, WebCodecs automatically guarantees that frames are emitted in presentation - // order, but some browsers (Safari) don't always follow this rule. - if (this.sampleQueue.length > 0 && (sample.timestamp >= last(this.sampleQueue)!.timestamp)) { - for (const sample of this.sampleQueue) { - this.finalizeAndEmitSample(sample); - } - - this.sampleQueue.length = 0; - } - - const insertionIndex = binarySearchLessOrEqual( - this.sampleQueue, - sample.timestamp, - x => x.timestamp, - ); - this.sampleQueue.splice(insertionIndex + 1, 0, sample); - }; - const MatchingCustomDecoder = customVideoDecoders.find(x => x.supports(codec, decoderConfig)); if (MatchingCustomDecoder) { // @ts-expect-error "Can't create instance of abstract class 🤓" @@ -798,11 +778,50 @@ class VideoDecoderWrapper extends DecoderWrapper { throw new TypeError('The argument passed to onSample must be a VideoSample.'); } - sampleHandler(sample); + this.finalizeAndEmitSample(sample); }; void this.customDecoderCallSerializer.call(() => this.customDecoder!.init()); } else { + // Specific handler for the WebCodecs VideoDecoder to iron out browser differences + const sampleHandler = (sample: VideoSample) => { + if (isSafari()) { + // For correct B-frame handling, we don't just hand over the frames directly but instead add them to + // a queue, because we want to ensure frames are emitted in presentation order. We flush the queue + // each time we receive a frame with a timestamp larger than the highest we've seen so far, as we + // can sure that is not a B-frame. Typically, WebCodecs automatically guarantees that frames are + // emitted in presentation order, but Safari doesn't always follow this rule. + if (this.sampleQueue.length > 0 && (sample.timestamp >= last(this.sampleQueue)!.timestamp)) { + for (const sample of this.sampleQueue) { + this.finalizeAndEmitSample(sample); + } + + this.sampleQueue.length = 0; + } + + const insertionIndex = binarySearchLessOrEqual( + this.sampleQueue, + sample.timestamp, + x => x.timestamp, + ); + this.sampleQueue.splice(insertionIndex + 1, 0, sample); + } else { + // Assign it the next earliest timestamp from the input. We do this because browsers, by spec, are + // required to emit decoded frames in presentation order *while* retaining the timestamp of their + // originating EncodedVideoChunk. For files with B-frames but no out-of-order timestamps (like a + // missing ctts box, for example), this causes a mismatch. We therefore fix the timestamps and + // ensure they are sorted by doing this. + const timestamp = this.inputTimestamps.shift(); + + // There's no way we'd have more decoded frames than encoded packets we passed in. Actually, the + // correspondence should be 1:1. + assert(timestamp !== undefined); + + sample.setTimestamp(timestamp); + this.finalizeAndEmitSample(sample); + } + }; + this.decoder = new VideoDecoder({ output: frame => sampleHandler(new VideoSample(frame)), error: onError, @@ -837,6 +856,16 @@ class VideoDecoderWrapper extends DecoderWrapper { .then(() => this.customDecoderQueueSize--); } else { assert(this.decoder); + + if (!isSafari()) { + const insertionIndex = binarySearchLessOrEqual( + this.inputTimestamps, + packet.timestamp, + x => x, + ); + this.inputTimestamps.splice(insertionIndex + 1, 0, packet.timestamp); + } + this.decoder.decode(packet.toEncodedVideoChunk()); } } @@ -849,10 +878,13 @@ class VideoDecoderWrapper extends DecoderWrapper { await this.decoder.flush(); } - for (const sample of this.sampleQueue) { - this.finalizeAndEmitSample(sample); + if (isSafari()) { + for (const sample of this.sampleQueue) { + this.finalizeAndEmitSample(sample); + } + + this.sampleQueue.length = 0; } - this.sampleQueue.length = 0; } close() { diff --git a/src/misc.ts b/src/misc.ts index 18ddc10..6edc9ad 100644 --- a/src/misc.ts +++ b/src/misc.ts @@ -586,3 +586,21 @@ export class CallSerializer { return this.currentPromise = this.currentPromise.then(fn); } } + +let isSafariCache: boolean | null = null; +export const isSafari = () => { + if (isSafariCache !== null) { + return isSafariCache; + } + + const result = !!( + typeof navigator !== 'undefined' + && navigator.vendor?.match(/apple/i) + && !navigator.userAgent?.match(/crios/i) + && !navigator.userAgent?.match(/fxios/i) + && !navigator.userAgent?.match(/Opera|OPT\//) + ); + + isSafariCache = result; + return result; +};