Remove sample correlation logic in decoder pipeline

src/input-track.ts

@@ -11,6 +11,7 @@ export interface InputTrackBacking {
 	getFirstTimestamp(): Promise<number>;
 	computeDuration(): Promise<number>;
 	getLanguageCode(): Promise<string>;
+	getTimeResolution(): Promise<number>;
 }
 
 /** @public */
@@ -52,6 +53,10 @@ export abstract class InputTrack {
 	getLanguageCode() {
 		return this._backing.getLanguageCode();
 	}
+
+	getTimeResolution() {
+		return this._backing.getTimeResolution();
+	}
 }
 
 export interface InputVideoTrackBacking extends InputTrackBacking {

src/isobmff/isobmff-demuxer.ts

@@ -1743,6 +1743,10 @@ abstract class IsobmffTrackBacking<
 		return firstSample?.timestamp ?? 0;
 	}
 
+	async getTimeResolution() {
+		return this.internalTrack.timescale;
+	}
+
 	abstract createSample(
 		data: Uint8Array,
 		byteLength: number,

src/matroska/matroska-demuxer.ts

@@ -998,6 +998,10 @@ abstract class MatroskaTrackBacking<
 		return firstSample?.timestamp ?? 0;
 	}
 
+	async getTimeResolution() {
+		return this.internalTrack.segment.timestampFactor;
+	}
+
 	abstract createSample(
 		data: Uint8Array,
 		byteLength: number,

src/media-sink.ts

@@ -4,7 +4,6 @@ import { InputAudioTrack, InputVideoTrack } from './input-track';
 import {
 	AnyIterable,
 	assert,
-	binarySearchLessOrEqual,
 	getInt24,
 	getUint24,
 	mapAsyncGenerator,
@@ -140,9 +139,8 @@ export abstract class BaseSampleSink<Sample extends EncodedVideoSample | Encoded
 	}
 }
 
-export type WrappedMediaFrame<T extends VideoFrame | AudioData, S extends EncodedVideoSample | EncodedAudioSample> = {
+export type WrappedMediaFrame<T extends VideoFrame | AudioData> = {
 	frame: T;
-	sample: S;
 	timestamp: number;
 	duration: number;
 };
@@ -150,7 +148,7 @@ export type WrappedMediaFrame<T extends VideoFrame | AudioData, S extends Encode
 abstract class DecoderWrapper<
 	Sample extends EncodedVideoSample | EncodedAudioSample,
 	MediaFrame extends VideoFrame | AudioData,
-	WrappedFrame extends WrappedMediaFrame<MediaFrame, Sample> = WrappedMediaFrame<MediaFrame, Sample>,
+	WrappedFrame extends WrappedMediaFrame<MediaFrame> = WrappedMediaFrame<MediaFrame>,
 > {
 	constructor(
 		public onFrame: (frame: WrappedFrame) => unknown,
@@ -168,7 +166,7 @@ export abstract class BaseMediaFrameSink<
 	Sample extends EncodedVideoSample | EncodedAudioSample,
 	MediaFrame extends VideoFrame | AudioData,
 	/** @internal */
-	WrappedFrame extends WrappedMediaFrame<MediaFrame, Sample> = WrappedMediaFrame<MediaFrame, Sample>,
+	WrappedFrame extends WrappedMediaFrame<MediaFrame> = WrappedMediaFrame<MediaFrame>,
 > {
 	/** @internal */
 	abstract _createDecoder(
@@ -361,7 +359,7 @@ export abstract class BaseMediaFrameSink<
 	): AsyncGenerator<WrappedFrame | null, void, unknown> {
 		validateAnyIterable(timestamps);
 		const timestampIterator = toAsyncIterator(timestamps);
-		const samplesOfInterest: Sample[] = [];
+		const timestampsOfInterest: number[] = [];
 
 		const MAX_QUEUE_SIZE = 8;
 		const frameQueue: (WrappedFrame | null)[] = [];
@@ -395,11 +393,11 @@ export abstract class BaseMediaFrameSink<
 
 				let frameUsed = false;
 				while (
-					samplesOfInterest.length > 0
+					timestampsOfInterest.length > 0
-					&& samplesOfInterest[0]!.is(wrappedFrame.sample as EncodedVideoSample & EncodedAudioSample)
+					&& wrappedFrame.timestamp - timestampsOfInterest[0]! > -1e-10 // Give it a little epsilon
 				) {
 					pushToQueue(this._duplicateFrame(wrappedFrame));
-					samplesOfInterest.shift();
+					timestampsOfInterest.shift();
 					frameUsed = true;
 				}
 
@@ -445,7 +443,7 @@ export abstract class BaseMediaFrameSink<
 					continue;
 				}
 
-				samplesOfInterest.push(targetSample);
+				timestampsOfInterest.push(targetSample.timestamp);
 
 				if (
 					lastKeySample
@@ -454,13 +452,13 @@ export abstract class BaseMediaFrameSink<
 				) {
 					assert(lastSample);
 
-					if (targetSample.timestamp === lastSample.timestamp && samplesOfInterest.length === 1) {
+					if (targetSample.timestamp === lastSample.timestamp && timestampsOfInterest.length === 1) {
 						// Special case: We have a repeat sample, but the frame for that sample has already been
 						// decoded. Therefore, we need to push the frame here instead of in the decoder callback.
 						if (lastUsedFrame) {
 							pushToQueue(this._duplicateFrame(lastUsedFrame));
 						}
-						samplesOfInterest.shift();
+						timestampsOfInterest.shift();
 					}
 				} else {
 					lastKeySample = keySample;
@@ -581,32 +579,27 @@ export class EncodedVideoSampleSink extends BaseSampleSink<EncodedVideoSample> {
 
 class VideoDecoderWrapper extends DecoderWrapper<EncodedVideoSample, VideoFrame> {
 	decoder: VideoDecoder | null = null;
-	pendingSamples: EncodedVideoSample[] = [];
 
 	customDecoder: CustomVideoDecoder | null = null;
 	lastCustomDecoderPromise = Promise.resolve();
 	customDecoderQueueSize = 0;
 
 	constructor(
-		onFrame: (frame: WrappedMediaFrame<VideoFrame, EncodedVideoSample>) => unknown,
+		onFrame: (frame: WrappedMediaFrame<VideoFrame>) => unknown,
 		onError: (error: DOMException) => unknown,
 		codec: VideoCodec,
 		decoderConfig: VideoDecoderConfig,
+		timeResolution: number,
 	) {
 		super(onFrame, onError);
 
 		const frameHandler = (frame: VideoFrame) => {
-			const sample = this.pendingSamples.shift();
+			// Round the microsecond timestamps to the time resolution
-			assert(sample);
+			const timestamp = Math.round(frame.timestamp / 1e6 * timeResolution) / timeResolution;
-
+			const duration = Math.round((frame.duration ?? 0) / 1e6 * timeResolution) / timeResolution;
-			// Let's get these from the sample instead of the frame, as the frame has no innate timing info
-			// (unlike AudioData), so the sample will always be more accurate.
-			const timestamp = sample.timestamp;
-			const duration = sample.duration;
 
 			onFrame({
 				frame,
-				sample,
 				timestamp,
 				duration,
 			});
@@ -634,10 +627,6 @@ class VideoDecoderWrapper extends DecoderWrapper<EncodedVideoSample, VideoFrame>
 	}
 
 	decode(sample: EncodedVideoSample) {
-		// We know the decoder spits out frames in sorted order, so we need to insert the sample in the right place
-		const insertionIndex = binarySearchLessOrEqual(this.pendingSamples, sample.timestamp, x => x.timestamp);
-		this.pendingSamples.splice(insertionIndex + 1, 0, sample);
-
 		if (this.customDecoder) {
 			this.customDecoderQueueSize++;
 			this.lastCustomDecoderPromise = this.lastCustomDecoderPromise.then(() => {
@@ -694,7 +683,7 @@ export class VideoFrameSink extends BaseMediaFrameSink<EncodedVideoSample, Video
 
 	/** @internal */
 	async _createDecoder(
-		onFrame: (frame: WrappedMediaFrame<VideoFrame, EncodedVideoSample>) => unknown,
+		onFrame: (frame: WrappedMediaFrame<VideoFrame>) => unknown,
 		onError: (error: DOMException) => unknown,
 	) {
 		if (!(await this._videoTrack.canDecode())) {
@@ -706,9 +695,10 @@ export class VideoFrameSink extends BaseMediaFrameSink<EncodedVideoSample, Video
 
 		const codec = await this._videoTrack.getCodec();
 		const decoderConfig = await this._videoTrack.getDecoderConfig();
+		const timeResolution = await this._videoTrack.getTimeResolution();
 		assert(codec && decoderConfig);
 
-		return new VideoDecoderWrapper(onFrame, onError, codec, decoderConfig);
+		return new VideoDecoderWrapper(onFrame, onError, codec, decoderConfig, timeResolution);
 	}
 
 	/** @internal */
@@ -717,7 +707,7 @@ export class VideoFrameSink extends BaseMediaFrameSink<EncodedVideoSample, Video
 	}
 
 	/** @internal */
-	_wrappedFrameToWrappedVideoFrame(frame: WrappedMediaFrame<VideoFrame, EncodedVideoSample>): WrappedVideoFrame {
+	_wrappedFrameToWrappedVideoFrame(frame: WrappedMediaFrame<VideoFrame>): WrappedVideoFrame {
 		return {
 			frame: frame.frame,
 			timestamp: frame.timestamp,
@@ -888,31 +878,27 @@ export class EncodedAudioSampleSink extends BaseSampleSink<EncodedAudioSample> {
 
 class AudioDecoderWrapper extends DecoderWrapper<EncodedAudioSample, AudioData> {
 	decoder: AudioDecoder | null = null;
-	pendingSamples: EncodedAudioSample[] = [];
 
 	customDecoder: CustomAudioDecoder | null = null;
 	lastCustomDecoderPromise = Promise.resolve();
 	customDecoderQueueSize = 0;
 
 	constructor(
-		onData: (data: WrappedMediaFrame<AudioData, EncodedAudioSample>) => unknown,
+		onData: (data: WrappedMediaFrame<AudioData>) => unknown,
 		onError: (error: DOMException) => unknown,
 		codec: AudioCodec,
 		decoderConfig: AudioDecoderConfig,
+		timeResolution: number,
 	) {
 		super(onData, onError);
 
 		const dataHandler = (data: AudioData) => {
-			const sample = this.pendingSamples.shift();
+			// Round the microsecond timestamps to the time resolution
-			assert(sample);
+			const timestamp = Math.round(data.timestamp / 1e6 * timeResolution) / timeResolution;
-
+			const duration = Math.round(data.duration / 1e6 * timeResolution) / timeResolution;
-			// We use the timing information from the data instead of sample as it will be more accurate
-			const timestamp = Math.round(data.timestamp / 1e6 * decoderConfig.sampleRate) / decoderConfig.sampleRate;
-			const duration = Math.round(data.duration / 1e6 * decoderConfig.sampleRate) / decoderConfig.sampleRate;
 
 			onData({
 				frame: data,
-				sample,
 				timestamp,
 				duration,
 			});
@@ -940,10 +926,6 @@ class AudioDecoderWrapper extends DecoderWrapper<EncodedAudioSample, AudioData>
 	}
 
 	decode(sample: EncodedAudioSample) {
-		// We know the decoder spits out data in sorted order, so we need to insert the sample in the right place
-		const insertionIndex = binarySearchLessOrEqual(this.pendingSamples, sample.timestamp, x => x.timestamp);
-		this.pendingSamples.splice(insertionIndex + 1, 0, sample);
-
 		if (this.customDecoder) {
 			this.customDecoderQueueSize++;
 			this.lastCustomDecoderPromise = this.lastCustomDecoderPromise.then(() => {
@@ -993,7 +975,7 @@ class PcmAudioDecoderWrapper extends DecoderWrapper<EncodedAudioSample, AudioDat
 	currentTimestamp: number | null = null;
 
 	constructor(
-		onData: (data: WrappedMediaFrame<AudioData, EncodedAudioSample>) => unknown,
+		onData: (data: WrappedMediaFrame<AudioData>) => unknown,
 		onError: (error: DOMException) => unknown,
 		public decoderConfig: AudioDecoderConfig,
 	) {
@@ -1131,7 +1113,6 @@ class PcmAudioDecoderWrapper extends DecoderWrapper<EncodedAudioSample, AudioDat
 		// Since all other decoders are async, we'll make this one behave async as well
 		queueMicrotask(() => this.onFrame({
 			frame: audioData,
-			sample,
 			timestamp: preciseTimestamp,
 			duration: preciseDuration,
 		}));
@@ -1170,7 +1151,7 @@ export class AudioDataSink extends BaseMediaFrameSink<EncodedAudioSample, AudioD
 
 	/** @internal */
 	async _createDecoder(
-		onData: (data: WrappedMediaFrame<AudioData, EncodedAudioSample>) => unknown,
+		onData: (data: WrappedMediaFrame<AudioData>) => unknown,
 		onError: (error: DOMException) => unknown,
 	) {
 		if (!(await this._audioTrack.canDecode())) {
@@ -1187,12 +1168,13 @@ export class AudioDataSink extends BaseMediaFrameSink<EncodedAudioSample, AudioD
 		if ((PCM_AUDIO_CODECS as readonly string[]).includes(decoderConfig.codec)) {
 			return new PcmAudioDecoderWrapper(onData, onError, decoderConfig);
 		} else {
-			return new AudioDecoderWrapper(onData, onError, codec, decoderConfig);
+			const timeResolution = await this._audioTrack.getTimeResolution();
+			return new AudioDecoderWrapper(onData, onError, codec, decoderConfig, timeResolution);
 		}
 	}
 
 	/** @internal */
-	_wrappedFrameToWrappedAudioData(frame: WrappedMediaFrame<AudioData, EncodedAudioSample>): WrappedAudioData {
+	_wrappedFrameToWrappedAudioData(frame: WrappedMediaFrame<AudioData>): WrappedAudioData {
 		return {
 			data: frame.frame,
 			timestamp: frame.timestamp,

src/mp3/mp3-demuxer.ts

@@ -8,8 +8,6 @@ import { EncodedAudioSample, PLACEHOLDER_DATA } from '../sample';
 import { FrameHeader, getXingOffset, INFO, XING } from './mp3-misc';
 import { Mp3Reader } from './mp3-reader';
 
-const AUDIO_SAMPLES_PER_FRAME = 1152;
-
 type Sample = {
 	timestamp: number;
 	duration: number;
@@ -46,7 +44,7 @@ export class Mp3Demuxer extends Demuxer {
 				this.reader.pos += id3Tag.size;
 			}
 
-			let nextTimestamp = 0;
+			let nextTimestampInSamples = 0;
 
 			// Let's read all samples
 			while (true) {
@@ -71,16 +69,16 @@ export class Mp3Demuxer extends Demuxer {
 					this.firstFrameHeader = header;
 				}
 
-				const sampleDuration = AUDIO_SAMPLES_PER_FRAME / header.sampleRate;
+				const sampleDuration = header.audioSamplesInFrame / header.sampleRate;
 				const sample: Sample = {
-					timestamp: nextTimestamp,
+					timestamp: nextTimestampInSamples / header.sampleRate,
 					duration: sampleDuration,
 					dataStart: header.startPos,
 					dataSize: header.totalSize,
 				};
 
 				this.allSamples.push(sample);
-				nextTimestamp += sampleDuration;
+				nextTimestampInSamples += header.audioSamplesInFrame;
 			}
 
 			if (!this.firstFrameHeader) {
@@ -121,6 +119,11 @@ class Mp3AudioTrackBacking implements InputAudioTrackBacking {
 		return 0;
 	}
 
+	async getTimeResolution() {
+		assert(this.demuxer.firstFrameHeader);
+		return this.demuxer.firstFrameHeader.sampleRate / this.demuxer.firstFrameHeader.audioSamplesInFrame;
+	}
+
 	computeDuration() {
 		return this.demuxer.computeDuration();
 	}

src/mp3/mp3-misc.ts

@@ -44,6 +44,7 @@ export type FrameHeader = {
 	copyright: number;
 	original: number;
 	emphasis: number;
+	audioSamplesInFrame: number;
 };
 
 export const computeMp3FrameSize = (layer: number, bitrate: number, sampleRate: number, padding: number) => {
@@ -114,6 +115,19 @@ export const readFrameHeader = (word: number, reader: { pos: number; fileSize: n
 		return null;
 	}
 
+	let audioSamplesInFrame: number;
+	if (mpegVersionId === 3) {
+		audioSamplesInFrame = layer === 3 ? 384 : 1152;
+	} else {
+		if (layer === 3) {
+			audioSamplesInFrame = 384;
+		} else if (layer === 2) {
+			audioSamplesInFrame = 1152;
+		} else {
+			audioSamplesInFrame = 576;
+		}
+	}
+
 	return {
 		startPos: startPos,
 		totalSize: frameLength,
@@ -127,5 +141,6 @@ export const readFrameHeader = (word: number, reader: { pos: number; fileSize: n
 		copyright,
 		original,
 		emphasis,
+		audioSamplesInFrame,
 	};
 };

src/ogg/ogg-demuxer.ts

@@ -418,6 +418,10 @@ class OggAudioTrackBacking implements InputAudioTrackBacking {
 		return this.bitstream.sampleRate;
 	}
 
+	async getTimeResolution() {
+		return this.bitstream.sampleRate;
+	}
+
 	async getCodec() {
 		return this.bitstream.codec;
 	}

src/wave/wave-demuxer.ts

@@ -219,6 +219,11 @@ class WaveAudioTrackBacking implements InputAudioTrackBacking {
 		return this.demuxer.audioInfo.sampleRate;
 	}
 
+	async getTimeResolution() {
+		assert(this.demuxer.audioInfo);
+		return this.demuxer.audioInfo.sampleRate;
+	}
+
 	async getLanguageCode() {
 		return UNDETERMINED_LANGUAGE;
 	}