diff --git a/dev/demux.html b/dev/demux.html
index f4357ab..73a6a79 100644
--- a/dev/demux.html
+++ b/dev/demux.html
@@ -16,6 +16,14 @@
source
});
+ const videoTrack = await input.getPrimaryVideoTrack();
+ const sink = new Mediabunny.VideoSampleSink(videoTrack);
+
+ for await (const sample of sink.samples(0, 2)) {
+ console.log(sample.timestamp);
+ sample.close();
+ }
+
/*
const audioTrack = await input.getPrimaryAudioTrack();
const sink = new Mediabunny.EncodedPacketSink(audioTrack);
diff --git a/src/media-sink.ts b/src/media-sink.ts
index 578acb3..09404a4 100644
--- a/src/media-sink.ts
+++ b/src/media-sink.ts
@@ -17,6 +17,7 @@ import {
CallSerializer,
getInt24,
getUint24,
+ isSafari,
last,
mapAsyncGenerator,
promiseWithResolvers,
@@ -750,7 +751,8 @@ class VideoDecoderWrapper extends DecoderWrapper {
customDecoderCallSerializer = new CallSerializer();
customDecoderQueueSize = 0;
- sampleQueue: VideoSample[] = [];
+ inputTimestamps: number[] = []; // Timestamps input into the decoder, sorted.
+ sampleQueue: VideoSample[] = []; // Safari-specific thing, check usage.
constructor(
onSample: (sample: VideoSample) => unknown,
@@ -762,28 +764,6 @@ class VideoDecoderWrapper extends DecoderWrapper {
) {
super(onSample, onError);
- const sampleHandler = (sample: VideoSample) => {
- // For correct B-frame handling, we don't just hand over the frames directly but instead add them to a
- // queue, because we want to ensure frames are emitted in presentation order. We flush the queue each time
- // we receive a frame with a timestamp larger than the highest we've seen so far, as we can sure that is
- // not a B-frame. Typically, WebCodecs automatically guarantees that frames are emitted in presentation
- // order, but some browsers (Safari) don't always follow this rule.
- if (this.sampleQueue.length > 0 && (sample.timestamp >= last(this.sampleQueue)!.timestamp)) {
- for (const sample of this.sampleQueue) {
- this.finalizeAndEmitSample(sample);
- }
-
- this.sampleQueue.length = 0;
- }
-
- const insertionIndex = binarySearchLessOrEqual(
- this.sampleQueue,
- sample.timestamp,
- x => x.timestamp,
- );
- this.sampleQueue.splice(insertionIndex + 1, 0, sample);
- };
-
const MatchingCustomDecoder = customVideoDecoders.find(x => x.supports(codec, decoderConfig));
if (MatchingCustomDecoder) {
// @ts-expect-error "Can't create instance of abstract class 🤓"
@@ -798,11 +778,50 @@ class VideoDecoderWrapper extends DecoderWrapper {
throw new TypeError('The argument passed to onSample must be a VideoSample.');
}
- sampleHandler(sample);
+ this.finalizeAndEmitSample(sample);
};
void this.customDecoderCallSerializer.call(() => this.customDecoder!.init());
} else {
+ // Specific handler for the WebCodecs VideoDecoder to iron out browser differences
+ const sampleHandler = (sample: VideoSample) => {
+ if (isSafari()) {
+ // For correct B-frame handling, we don't just hand over the frames directly but instead add them to
+ // a queue, because we want to ensure frames are emitted in presentation order. We flush the queue
+ // each time we receive a frame with a timestamp larger than the highest we've seen so far, as we
+ // can sure that is not a B-frame. Typically, WebCodecs automatically guarantees that frames are
+ // emitted in presentation order, but Safari doesn't always follow this rule.
+ if (this.sampleQueue.length > 0 && (sample.timestamp >= last(this.sampleQueue)!.timestamp)) {
+ for (const sample of this.sampleQueue) {
+ this.finalizeAndEmitSample(sample);
+ }
+
+ this.sampleQueue.length = 0;
+ }
+
+ const insertionIndex = binarySearchLessOrEqual(
+ this.sampleQueue,
+ sample.timestamp,
+ x => x.timestamp,
+ );
+ this.sampleQueue.splice(insertionIndex + 1, 0, sample);
+ } else {
+ // Assign it the next earliest timestamp from the input. We do this because browsers, by spec, are
+ // required to emit decoded frames in presentation order *while* retaining the timestamp of their
+ // originating EncodedVideoChunk. For files with B-frames but no out-of-order timestamps (like a
+ // missing ctts box, for example), this causes a mismatch. We therefore fix the timestamps and
+ // ensure they are sorted by doing this.
+ const timestamp = this.inputTimestamps.shift();
+
+ // There's no way we'd have more decoded frames than encoded packets we passed in. Actually, the
+ // correspondence should be 1:1.
+ assert(timestamp !== undefined);
+
+ sample.setTimestamp(timestamp);
+ this.finalizeAndEmitSample(sample);
+ }
+ };
+
this.decoder = new VideoDecoder({
output: frame => sampleHandler(new VideoSample(frame)),
error: onError,
@@ -837,6 +856,16 @@ class VideoDecoderWrapper extends DecoderWrapper {
.then(() => this.customDecoderQueueSize--);
} else {
assert(this.decoder);
+
+ if (!isSafari()) {
+ const insertionIndex = binarySearchLessOrEqual(
+ this.inputTimestamps,
+ packet.timestamp,
+ x => x,
+ );
+ this.inputTimestamps.splice(insertionIndex + 1, 0, packet.timestamp);
+ }
+
this.decoder.decode(packet.toEncodedVideoChunk());
}
}
@@ -849,10 +878,13 @@ class VideoDecoderWrapper extends DecoderWrapper {
await this.decoder.flush();
}
- for (const sample of this.sampleQueue) {
- this.finalizeAndEmitSample(sample);
+ if (isSafari()) {
+ for (const sample of this.sampleQueue) {
+ this.finalizeAndEmitSample(sample);
+ }
+
+ this.sampleQueue.length = 0;
}
- this.sampleQueue.length = 0;
}
close() {
diff --git a/src/misc.ts b/src/misc.ts
index 18ddc10..6edc9ad 100644
--- a/src/misc.ts
+++ b/src/misc.ts
@@ -586,3 +586,21 @@ export class CallSerializer {
return this.currentPromise = this.currentPromise.then(fn);
}
}
+
+let isSafariCache: boolean | null = null;
+export const isSafari = () => {
+ if (isSafariCache !== null) {
+ return isSafariCache;
+ }
+
+ const result = !!(
+ typeof navigator !== 'undefined'
+ && navigator.vendor?.match(/apple/i)
+ && !navigator.userAgent?.match(/crios/i)
+ && !navigator.userAgent?.match(/fxios/i)
+ && !navigator.userAgent?.match(/Opera|OPT\//)
+ );
+
+ isSafariCache = result;
+ return result;
+};