diff --git a/dev/index.html b/dev/mux.html
similarity index 100%
rename from dev/index.html
rename to dev/mux.html
diff --git a/src/custom-coder.ts b/src/custom-coder.ts
index e528387..f2794f2 100644
--- a/src/custom-coder.ts
+++ b/src/custom-coder.ts
@@ -2,75 +2,71 @@ import { AudioCodec, VideoCodec } from './codec';
import { EncodedAudioSample, EncodedVideoSample } from './sample';
/** @public */
-export class CustomVideoDecoder {
- constructor(
- public codec: VideoCodec,
- public config: VideoDecoderConfig,
- public onFrame: (frame: VideoFrame) => unknown,
- ) {}
+export abstract class CustomVideoDecoder {
+ codec!: VideoCodec;
+ config!: VideoDecoderConfig;
+ onFrame!: (frame: VideoFrame) => unknown;
// eslint-disable-next-line @typescript-eslint/no-unused-vars
static supports(codec: VideoCodec, config: VideoDecoderConfig): boolean {
return false;
}
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
- decode(sample: EncodedVideoSample): Promise | void {}
- flush(): Promise | void {}
+ abstract init(): void;
+ abstract decode(sample: EncodedVideoSample): Promise | void;
+ abstract flush(): Promise | void;
+ abstract close(): Promise | void;
}
/** @public */
-export class CustomAudioDecoder {
- constructor(
- public codec: AudioCodec,
- public config: AudioDecoderConfig,
- public onData: (data: AudioData) => unknown,
- ) {}
+export abstract class CustomAudioDecoder {
+ codec!: AudioCodec;
+ config!: AudioDecoderConfig;
+ onData!: (data: AudioData) => unknown;
// eslint-disable-next-line @typescript-eslint/no-unused-vars
static supports(codec: AudioCodec, config: AudioDecoderConfig): boolean {
return false;
}
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
- decode(sample: EncodedAudioSample): Promise | void {}
- flush(): Promise | void {}
+ abstract init(): void;
+ abstract decode(sample: EncodedAudioSample): Promise | void;
+ abstract flush(): Promise | void;
+ abstract close(): Promise | void;
}
/** @public */
-export class CustomVideoEncoder {
- constructor(
- public codec: VideoCodec,
- public config: VideoEncoderConfig,
- public onSample: (sample: EncodedVideoSample, meta?: EncodedVideoChunkMetadata) => unknown,
- ) {}
+export abstract class CustomVideoEncoder {
+ codec!: VideoCodec;
+ config!: VideoEncoderConfig;
+ onSample!: (sample: EncodedVideoSample, meta?: EncodedVideoChunkMetadata) => unknown;
// eslint-disable-next-line @typescript-eslint/no-unused-vars
static supports(codec: VideoCodec, config: VideoEncoderConfig): boolean {
return false;
}
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
- encode(videoFrame: VideoFrame, options: VideoEncoderEncodeOptions): Promise | void {}
- flush(): Promise | void {}
+ abstract init(): void;
+ abstract encode(videoFrame: VideoFrame, options: VideoEncoderEncodeOptions): Promise | void;
+ abstract flush(): Promise | void;
+ abstract close(): Promise | void;
}
/** @public */
-export class CustomAudioEncoder {
- constructor(
- public codec: AudioCodec,
- public config: AudioEncoderConfig,
- public onSample: (sample: EncodedAudioSample, meta?: EncodedAudioChunkMetadata) => unknown,
- ) {}
+export abstract class CustomAudioEncoder {
+ codec!: AudioCodec;
+ config!: AudioEncoderConfig;
+ onSample!: (sample: EncodedAudioSample, meta?: EncodedAudioChunkMetadata) => unknown;
// eslint-disable-next-line @typescript-eslint/no-unused-vars
static supports(codec: AudioCodec, config: AudioEncoderConfig): boolean {
return false;
}
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
- encode(audioData: AudioData): Promise | void {}
- flush(): Promise | void {}
+ abstract init(): void;
+ abstract encode(audioData: AudioData): Promise | void;
+ abstract flush(): Promise | void;
+ abstract close(): Promise | void;
}
export const customVideoDecoders: typeof CustomVideoDecoder[] = [];
diff --git a/src/isobmff/isobmff-demuxer.ts b/src/isobmff/isobmff-demuxer.ts
index 8985e8a..59732a2 100644
--- a/src/isobmff/isobmff-demuxer.ts
+++ b/src/isobmff/isobmff-demuxer.ts
@@ -1797,6 +1797,7 @@ abstract class IsobmffTrackBacking<
type: SampleType,
timestamp: number,
duration: number,
+ sequenceNumber: number
): Sample;
async getFirstSample(options: SampleRetrievalOptions) {
@@ -2067,6 +2068,7 @@ abstract class IsobmffTrackBacking<
sampleInfo.isKeyFrame ? 'key' : 'delta',
timestamp,
duration,
+ sampleIndex,
);
this.sampleToSampleIndex.set(sample, sampleIndex);
@@ -2103,6 +2105,7 @@ abstract class IsobmffTrackBacking<
fragmentSample.isKeyFrame ? 'key' : 'delta',
timestamp,
duration,
+ fragment.moofOffset + sampleIndex,
);
this.sampleToFragmentLocation.set(sample, { fragment, sampleIndex });
@@ -2362,8 +2365,9 @@ class IsobmffVideoTrackBacking extends IsobmffTrackBacking i
type: SampleType,
timestamp: number,
duration: number,
+ sequenceNumber: number,
) {
- return new EncodedVideoSample(data, type, timestamp, duration, byteLength);
+ return new EncodedVideoSample(data, type, timestamp, duration, sequenceNumber, byteLength);
}
}
diff --git a/src/matroska/matroska-demuxer.ts b/src/matroska/matroska-demuxer.ts
index d138599..0ad6903 100644
--- a/src/matroska/matroska-demuxer.ts
+++ b/src/matroska/matroska-demuxer.ts
@@ -1008,6 +1008,7 @@ abstract class MatroskaTrackBacking<
type: SampleType,
timestamp: number,
duration: number,
+ sequenceNumber: number,
): Sample;
async getFirstSample(options: SampleRetrievalOptions) {
@@ -1223,6 +1224,7 @@ abstract class MatroskaTrackBacking<
block.isKeyFrame ? 'key' : 'delta',
timestamp,
duration,
+ cluster.dataStartPos + blockIndex,
);
this.sampleToClusterLocation.set(sample, { cluster, blockIndex });
@@ -1506,8 +1508,9 @@ class MatroskaVideoTrackBacking extends MatroskaTrackBacking
type: SampleType,
timestamp: number,
duration: number,
+ sequenceNumber: number,
) {
- return new EncodedVideoSample(data, type, timestamp, duration, byteLength);
+ return new EncodedVideoSample(data, type, timestamp, duration, sequenceNumber, byteLength);
}
}
diff --git a/src/media-sink.ts b/src/media-sink.ts
index 83d23b6..3d1912a 100644
--- a/src/media-sink.ts
+++ b/src/media-sink.ts
@@ -4,8 +4,10 @@ import { InputAudioTrack, InputVideoTrack } from './input-track';
import {
AnyIterable,
assert,
+ binarySearchLessOrEqual,
getInt24,
getUint24,
+ last,
mapAsyncGenerator,
promiseWithResolvers,
toAsyncIterator,
@@ -338,6 +340,8 @@ export abstract class BaseMediaFrameSink<
onQueueDequeue();
onQueueNotEmpty();
+ lastFrame?.frame.close();
+
for (const frame of frameQueue) {
frame.frame.close();
}
@@ -443,30 +447,40 @@ export abstract class BaseMediaFrameSink<
continue;
}
- timestampsOfInterest.push(targetSample.timestamp);
+ if (lastSample && targetSample.sequenceNumber < lastSample.sequenceNumber) {
+ // We're going back in time with this one, let's flush and reset to an clean state
+ await decoder.flush();
+ timestampsOfInterest.length = 0;
+ }
if (
lastKeySample
- && keySample.timestamp === lastKeySample.timestamp
+ && keySample.sequenceNumber === lastKeySample.sequenceNumber
&& targetSample.timestamp >= lastSample!.timestamp
) {
assert(lastSample);
- if (targetSample.timestamp === lastSample.timestamp && timestampsOfInterest.length === 1) {
+ if (
+ targetSample.sequenceNumber === lastSample.sequenceNumber
+ && timestampsOfInterest.length === 0
+ ) {
// Special case: We have a repeat sample, but the frame for that sample has already been
// decoded. Therefore, we need to push the frame here instead of in the decoder callback.
if (lastUsedFrame) {
pushToQueue(this._duplicateFrame(lastUsedFrame));
}
- timestampsOfInterest.shift();
+ } else {
+ timestampsOfInterest.push(targetSample.timestamp);
}
} else {
+ // The key sample has changed
lastKeySample = keySample;
lastSample = keySample;
decoder.decode(keySample);
+ timestampsOfInterest.push(targetSample.timestamp);
}
- while (lastSample.timestamp !== targetSample.timestamp) {
+ while (lastSample.sequenceNumber < targetSample.sequenceNumber) {
const nextSample = await sampleSink.getNextSample(lastSample);
assert(nextSample);
@@ -475,7 +489,10 @@ export abstract class BaseMediaFrameSink<
}
}
- if (!terminated) await decoder.flush();
+ if (!terminated) {
+ await decoder.flush();
+ lastUsedFrame?.frame.close();
+ }
decoder.close();
decoderIsFlushed = true;
@@ -584,30 +601,48 @@ class VideoDecoderWrapper extends DecoderWrapper
lastCustomDecoderPromise = Promise.resolve();
customDecoderQueueSize = 0;
+ frameQueue: VideoFrame[] = [];
+
constructor(
onFrame: (frame: WrappedMediaFrame) => unknown,
onError: (error: DOMException) => unknown,
codec: VideoCodec,
decoderConfig: VideoDecoderConfig,
- timeResolution: number,
+ public timeResolution: number,
) {
super(onFrame, onError);
const frameHandler = (frame: VideoFrame) => {
- // Round the microsecond timestamps to the time resolution
- const timestamp = Math.round(frame.timestamp / 1e6 * timeResolution) / timeResolution;
- const duration = Math.round((frame.duration ?? 0) / 1e6 * timeResolution) / timeResolution;
+ // For correct B-frame handling, we don't just hand over the frames directly but instead add them to a
+ // queue, because we want to ensure frames are emitted in presentation order. We flush the queue each time
+ // we receive a frame with a timestamp larger than the highest we've seen so far, as we can sure that is
+ // not a B-frame. Typically, WebCodecs automatically guarantees that frames are emitted in presentation
+ // order, but some browsers (Safari) don't always follow this rule.
+ if (this.frameQueue.length > 0 && (frame.timestamp >= last(this.frameQueue)!.timestamp)) {
+ for (const frame of this.frameQueue) {
+ this.wrapAndEmitFrame(frame);
+ }
- onFrame({
- frame,
- timestamp,
- duration,
- });
+ this.frameQueue.length = 0;
+ }
+
+ const insertionIndex = binarySearchLessOrEqual(
+ this.frameQueue,
+ frame.timestamp,
+ x => x.timestamp,
+ );
+ this.frameQueue.splice(insertionIndex + 1, 0, frame);
};
const MatchingCustomDecoder = customVideoDecoders.find(x => x.supports(codec, decoderConfig));
if (MatchingCustomDecoder) {
- this.customDecoder = new MatchingCustomDecoder(codec, decoderConfig, frameHandler);
+ // @ts-expect-error "Can't create instance of abstract class 🤓"
+ this.customDecoder = new MatchingCustomDecoder() as CustomVideoDecoder;
+ this.customDecoder.codec = codec;
+ this.customDecoder.config = decoderConfig;
+ this.customDecoder.onFrame = frameHandler;
+
+ this.customDecoder.init();
} else {
this.decoder = new VideoDecoder({
output: frameHandler,
@@ -617,6 +652,18 @@ class VideoDecoderWrapper extends DecoderWrapper
}
}
+ wrapAndEmitFrame(frame: VideoFrame) {
+ // Round the microsecond timestamps to the time resolution
+ const timestamp = Math.round(frame.timestamp / 1e6 * this.timeResolution) / this.timeResolution;
+ const duration = Math.round((frame.duration ?? 0) / 1e6 * this.timeResolution) / this.timeResolution;
+
+ this.onFrame({
+ frame,
+ timestamp,
+ duration,
+ });
+ }
+
getDecodeQueueSize() {
if (this.customDecoder) {
return this.customDecoderQueueSize;
@@ -640,22 +687,32 @@ class VideoDecoderWrapper extends DecoderWrapper
}
}
- flush() {
+ async flush() {
if (this.customDecoder) {
- return this.lastCustomDecoderPromise.then(() => this.customDecoder!.flush());
+ await this.lastCustomDecoderPromise.then(() => this.customDecoder!.flush());
} else {
assert(this.decoder);
- return this.decoder.flush();
+ await this.decoder.flush();
}
+
+ for (const frame of this.frameQueue) {
+ this.wrapAndEmitFrame(frame);
+ }
+ this.frameQueue.length = 0;
}
close() {
if (this.customDecoder) {
- void this.lastCustomDecoderPromise.then(() => this.customDecoder!.flush());
+ void this.lastCustomDecoderPromise.then(() => this.customDecoder!.close());
} else {
assert(this.decoder);
this.decoder.close();
}
+
+ for (const frame of this.frameQueue) {
+ frame.close();
+ }
+ this.frameQueue.length = 0;
}
}
@@ -907,7 +964,13 @@ class AudioDecoderWrapper extends DecoderWrapper
const MatchingCustomDecoder = customAudioDecoders.find(x => x.supports(codec, decoderConfig));
if (MatchingCustomDecoder) {
- this.customDecoder = new MatchingCustomDecoder(codec, decoderConfig, dataHandler);
+ // @ts-expect-error "Can't create instance of abstract class 🤓"
+ this.customDecoder = new MatchingCustomDecoder() as CustomAudioDecoder;
+ this.customDecoder.codec = codec;
+ this.customDecoder.config = decoderConfig;
+ this.customDecoder.onData = dataHandler;
+
+ this.customDecoder.init();
} else {
this.decoder = new AudioDecoder({
output: dataHandler,
@@ -951,7 +1014,7 @@ class AudioDecoderWrapper extends DecoderWrapper
close() {
if (this.customDecoder) {
- void this.lastCustomDecoderPromise.then(() => this.customDecoder!.flush());
+ void this.lastCustomDecoderPromise.then(() => this.customDecoder!.close());
} else {
assert(this.decoder);
this.decoder.close();
diff --git a/src/media-source.ts b/src/media-source.ts
index f8d1748..4941d96 100644
--- a/src/media-source.ts
+++ b/src/media-source.ts
@@ -302,14 +302,16 @@ class VideoEncoderWrapper {
));
if (MatchingCustomEncoder) {
- this.customEncoder = new MatchingCustomEncoder(
- this.encodingConfig.codec,
- encoderConfig,
- (sample, meta) => {
- this.encodingConfig.onEncodedSample?.(sample, meta);
- void this.muxer!.addEncodedVideoSample(this.source._connectedTrack!, sample, meta);
- },
- );
+ // @ts-expect-error "Can't create instance of abstract class 🤓"
+ this.customEncoder = new MatchingCustomEncoder() as CustomVideoEncoder;
+ this.customEncoder.codec = this.encodingConfig.codec;
+ this.customEncoder.config = encoderConfig;
+ this.customEncoder.onSample = (sample, meta) => {
+ this.encodingConfig.onEncodedSample?.(sample, meta);
+ void this.muxer!.addEncodedVideoSample(this.source._connectedTrack!, sample, meta);
+ };
+
+ this.customEncoder.init();
} else {
if (typeof VideoEncoder === 'undefined') {
throw new Error('VideoEncoder is not supported by this browser.');
@@ -751,14 +753,16 @@ class AudioEncoderWrapper {
));
if (MatchingCustomEncoder) {
- this.customEncoder = new MatchingCustomEncoder(
- this.encodingConfig.codec,
- encoderConfig,
- (sample, meta) => {
- this.encodingConfig.onEncodedSample?.(sample, meta);
- void this.muxer!.addEncodedAudioSample(this.source._connectedTrack!, sample, meta);
- },
- );
+ // @ts-expect-error "Can't create instance of abstract class 🤓"
+ this.customEncoder = new MatchingCustomEncoder() as CustomAudioEncoder;
+ this.customEncoder.codec = this.encodingConfig.codec;
+ this.customEncoder.config = encoderConfig;
+ this.customEncoder.onSample = (sample, meta) => {
+ this.encodingConfig.onEncodedSample?.(sample, meta);
+ void this.muxer!.addEncodedAudioSample(this.source._connectedTrack!, sample, meta);
+ };
+
+ this.customEncoder.init();
} else if ((PCM_AUDIO_CODECS as readonly string[]).includes(this.encodingConfig.codec)) {
this.initPcmEncoder();
} else {
diff --git a/src/mp3/mp3-demuxer.ts b/src/mp3/mp3-demuxer.ts
index 1e32cdc..13a7655 100644
--- a/src/mp3/mp3-demuxer.ts
+++ b/src/mp3/mp3-demuxer.ts
@@ -179,6 +179,7 @@ class Mp3AudioTrackBacking implements InputAudioTrackBacking {
'key',
rawSample.timestamp,
rawSample.duration,
+ sampleIndex,
);
}
diff --git a/src/ogg/ogg-demuxer.ts b/src/ogg/ogg-demuxer.ts
index a787cf3..2859477 100644
--- a/src/ogg/ogg-demuxer.ts
+++ b/src/ogg/ogg-demuxer.ts
@@ -478,6 +478,7 @@ class OggAudioTrackBacking implements InputAudioTrackBacking {
'key',
Math.max(0, additional.timestampInSamples) / this.internalSampleRate,
durationInSamples / this.internalSampleRate,
+ packet.endPage.headerStartPos + packet.endSegmentIndex,
);
this.sampleToMetadata.set(sample, {
@@ -738,11 +739,12 @@ class OggAudioTrackBacking implements InputAudioTrackBacking {
}
endSegmentIndex = currentSegmentIndex - 1;
- const nextPosition = await this.demuxer.findNextPacketStart(reader, {
+ const pseudopacket: Packet = {
data: PLACEHOLDER_DATA,
endPage,
endSegmentIndex,
- });
+ };
+ const nextPosition = await this.demuxer.findNextPacketStart(reader, pseudopacket);
if (nextPosition) {
// Let's rewind a single step (packet) - this previous packet ensures that we'll correctly compute the
diff --git a/src/sample.ts b/src/sample.ts
index fdeb91e..3563120 100644
--- a/src/sample.ts
+++ b/src/sample.ts
@@ -10,6 +10,7 @@ export class EncodedVideoSample {
public readonly type: SampleType,
public readonly timestamp: number,
public readonly duration: number,
+ public readonly sequenceNumber = -1,
public readonly byteLength = data.byteLength,
) {
if (!(data instanceof Uint8Array)) {
@@ -58,19 +59,6 @@ export class EncodedVideoSample {
});
}
- is(otherSample: EncodedVideoSample) {
- if (!(otherSample instanceof EncodedVideoSample)) {
- throw new TypeError('otherSample must be an EncodedVideoSample.');
- }
-
- return (
- this.type === otherSample.type
- && this.timestamp === otherSample.timestamp
- && this.duration === otherSample.duration
- && this.byteLength === otherSample.byteLength
- );
- }
-
clone(options?: {
timestamp?: number;
duration?: number;
@@ -118,6 +106,7 @@ export class EncodedAudioSample {
public readonly type: SampleType,
public readonly timestamp: number,
public readonly duration: number,
+ public readonly sequenceNumber = -1,
public readonly byteLength = data.byteLength,
) {
if (!(data instanceof Uint8Array)) {
@@ -166,19 +155,6 @@ export class EncodedAudioSample {
});
}
- is(otherSample: EncodedAudioSample) {
- if (!(otherSample instanceof EncodedAudioSample)) {
- throw new TypeError('otherSample must be an EncodedAudioSample.');
- }
-
- return (
- this.type === otherSample.type
- && this.timestamp === otherSample.timestamp
- && this.duration === otherSample.duration
- && this.byteLength === otherSample.byteLength
- );
- }
-
clone(options?: {
timestamp?: number;
duration?: number;
diff --git a/src/wave/wave-demuxer.ts b/src/wave/wave-demuxer.ts
index ebf14aa..eec771a 100644
--- a/src/wave/wave-demuxer.ts
+++ b/src/wave/wave-demuxer.ts
@@ -274,6 +274,7 @@ class WaveAudioTrackBacking implements InputAudioTrackBacking {
'key',
timestamp,
duration,
+ sampleIndex,
);
}