From 1d823805cfc0afb90bf4337ee4241facece10d7b Mon Sep 17 00:00:00 2001 From: Vanilagy <1696106+Vanilagy@users.noreply.github.com> Date: Sat, 8 Feb 2025 15:55:54 +0100 Subject: [PATCH] Remove sample correlation logic in decoder pipeline --- src/input-track.ts | 5 +++ src/isobmff/isobmff-demuxer.ts | 4 ++ src/matroska/matroska-demuxer.ts | 4 ++ src/media-sink.ts | 76 ++++++++++++-------------------- src/mp3/mp3-demuxer.ts | 15 ++++--- src/mp3/mp3-misc.ts | 15 +++++++ src/ogg/ogg-demuxer.ts | 4 ++ src/wave/wave-demuxer.ts | 5 +++ 8 files changed, 75 insertions(+), 53 deletions(-) diff --git a/src/input-track.ts b/src/input-track.ts index 9d512a6..5320f74 100644 --- a/src/input-track.ts +++ b/src/input-track.ts @@ -11,6 +11,7 @@ export interface InputTrackBacking { getFirstTimestamp(): Promise; computeDuration(): Promise; getLanguageCode(): Promise; + getTimeResolution(): Promise; } /** @public */ @@ -52,6 +53,10 @@ export abstract class InputTrack { getLanguageCode() { return this._backing.getLanguageCode(); } + + getTimeResolution() { + return this._backing.getTimeResolution(); + } } export interface InputVideoTrackBacking extends InputTrackBacking { diff --git a/src/isobmff/isobmff-demuxer.ts b/src/isobmff/isobmff-demuxer.ts index 653bd10..c00cc09 100644 --- a/src/isobmff/isobmff-demuxer.ts +++ b/src/isobmff/isobmff-demuxer.ts @@ -1743,6 +1743,10 @@ abstract class IsobmffTrackBacking< return firstSample?.timestamp ?? 0; } + async getTimeResolution() { + return this.internalTrack.timescale; + } + abstract createSample( data: Uint8Array, byteLength: number, diff --git a/src/matroska/matroska-demuxer.ts b/src/matroska/matroska-demuxer.ts index 566548b..12654a3 100644 --- a/src/matroska/matroska-demuxer.ts +++ b/src/matroska/matroska-demuxer.ts @@ -998,6 +998,10 @@ abstract class MatroskaTrackBacking< return firstSample?.timestamp ?? 0; } + async getTimeResolution() { + return this.internalTrack.segment.timestampFactor; + } + abstract createSample( data: Uint8Array, byteLength: number, diff --git a/src/media-sink.ts b/src/media-sink.ts index 3951137..16646bf 100644 --- a/src/media-sink.ts +++ b/src/media-sink.ts @@ -4,7 +4,6 @@ import { InputAudioTrack, InputVideoTrack } from './input-track'; import { AnyIterable, assert, - binarySearchLessOrEqual, getInt24, getUint24, mapAsyncGenerator, @@ -140,9 +139,8 @@ export abstract class BaseSampleSink = { +export type WrappedMediaFrame = { frame: T; - sample: S; timestamp: number; duration: number; }; @@ -150,7 +148,7 @@ export type WrappedMediaFrame = WrappedMediaFrame, + WrappedFrame extends WrappedMediaFrame = WrappedMediaFrame, > { constructor( public onFrame: (frame: WrappedFrame) => unknown, @@ -168,7 +166,7 @@ export abstract class BaseMediaFrameSink< Sample extends EncodedVideoSample | EncodedAudioSample, MediaFrame extends VideoFrame | AudioData, /** @internal */ - WrappedFrame extends WrappedMediaFrame = WrappedMediaFrame, + WrappedFrame extends WrappedMediaFrame = WrappedMediaFrame, > { /** @internal */ abstract _createDecoder( @@ -361,7 +359,7 @@ export abstract class BaseMediaFrameSink< ): AsyncGenerator { validateAnyIterable(timestamps); const timestampIterator = toAsyncIterator(timestamps); - const samplesOfInterest: Sample[] = []; + const timestampsOfInterest: number[] = []; const MAX_QUEUE_SIZE = 8; const frameQueue: (WrappedFrame | null)[] = []; @@ -395,11 +393,11 @@ export abstract class BaseMediaFrameSink< let frameUsed = false; while ( - samplesOfInterest.length > 0 - && samplesOfInterest[0]!.is(wrappedFrame.sample as EncodedVideoSample & EncodedAudioSample) + timestampsOfInterest.length > 0 + && wrappedFrame.timestamp - timestampsOfInterest[0]! > -1e-10 // Give it a little epsilon ) { pushToQueue(this._duplicateFrame(wrappedFrame)); - samplesOfInterest.shift(); + timestampsOfInterest.shift(); frameUsed = true; } @@ -445,7 +443,7 @@ export abstract class BaseMediaFrameSink< continue; } - samplesOfInterest.push(targetSample); + timestampsOfInterest.push(targetSample.timestamp); if ( lastKeySample @@ -454,13 +452,13 @@ export abstract class BaseMediaFrameSink< ) { assert(lastSample); - if (targetSample.timestamp === lastSample.timestamp && samplesOfInterest.length === 1) { + if (targetSample.timestamp === lastSample.timestamp && timestampsOfInterest.length === 1) { // Special case: We have a repeat sample, but the frame for that sample has already been // decoded. Therefore, we need to push the frame here instead of in the decoder callback. if (lastUsedFrame) { pushToQueue(this._duplicateFrame(lastUsedFrame)); } - samplesOfInterest.shift(); + timestampsOfInterest.shift(); } } else { lastKeySample = keySample; @@ -581,32 +579,27 @@ export class EncodedVideoSampleSink extends BaseSampleSink { class VideoDecoderWrapper extends DecoderWrapper { decoder: VideoDecoder | null = null; - pendingSamples: EncodedVideoSample[] = []; customDecoder: CustomVideoDecoder | null = null; lastCustomDecoderPromise = Promise.resolve(); customDecoderQueueSize = 0; constructor( - onFrame: (frame: WrappedMediaFrame) => unknown, + onFrame: (frame: WrappedMediaFrame) => unknown, onError: (error: DOMException) => unknown, codec: VideoCodec, decoderConfig: VideoDecoderConfig, + timeResolution: number, ) { super(onFrame, onError); const frameHandler = (frame: VideoFrame) => { - const sample = this.pendingSamples.shift(); - assert(sample); - - // Let's get these from the sample instead of the frame, as the frame has no innate timing info - // (unlike AudioData), so the sample will always be more accurate. - const timestamp = sample.timestamp; - const duration = sample.duration; + // Round the microsecond timestamps to the time resolution + const timestamp = Math.round(frame.timestamp / 1e6 * timeResolution) / timeResolution; + const duration = Math.round((frame.duration ?? 0) / 1e6 * timeResolution) / timeResolution; onFrame({ frame, - sample, timestamp, duration, }); @@ -634,10 +627,6 @@ class VideoDecoderWrapper extends DecoderWrapper } decode(sample: EncodedVideoSample) { - // We know the decoder spits out frames in sorted order, so we need to insert the sample in the right place - const insertionIndex = binarySearchLessOrEqual(this.pendingSamples, sample.timestamp, x => x.timestamp); - this.pendingSamples.splice(insertionIndex + 1, 0, sample); - if (this.customDecoder) { this.customDecoderQueueSize++; this.lastCustomDecoderPromise = this.lastCustomDecoderPromise.then(() => { @@ -694,7 +683,7 @@ export class VideoFrameSink extends BaseMediaFrameSink) => unknown, + onFrame: (frame: WrappedMediaFrame) => unknown, onError: (error: DOMException) => unknown, ) { if (!(await this._videoTrack.canDecode())) { @@ -706,9 +695,10 @@ export class VideoFrameSink extends BaseMediaFrameSink): WrappedVideoFrame { + _wrappedFrameToWrappedVideoFrame(frame: WrappedMediaFrame): WrappedVideoFrame { return { frame: frame.frame, timestamp: frame.timestamp, @@ -888,31 +878,27 @@ export class EncodedAudioSampleSink extends BaseSampleSink { class AudioDecoderWrapper extends DecoderWrapper { decoder: AudioDecoder | null = null; - pendingSamples: EncodedAudioSample[] = []; customDecoder: CustomAudioDecoder | null = null; lastCustomDecoderPromise = Promise.resolve(); customDecoderQueueSize = 0; constructor( - onData: (data: WrappedMediaFrame) => unknown, + onData: (data: WrappedMediaFrame) => unknown, onError: (error: DOMException) => unknown, codec: AudioCodec, decoderConfig: AudioDecoderConfig, + timeResolution: number, ) { super(onData, onError); const dataHandler = (data: AudioData) => { - const sample = this.pendingSamples.shift(); - assert(sample); - - // We use the timing information from the data instead of sample as it will be more accurate - const timestamp = Math.round(data.timestamp / 1e6 * decoderConfig.sampleRate) / decoderConfig.sampleRate; - const duration = Math.round(data.duration / 1e6 * decoderConfig.sampleRate) / decoderConfig.sampleRate; + // Round the microsecond timestamps to the time resolution + const timestamp = Math.round(data.timestamp / 1e6 * timeResolution) / timeResolution; + const duration = Math.round(data.duration / 1e6 * timeResolution) / timeResolution; onData({ frame: data, - sample, timestamp, duration, }); @@ -940,10 +926,6 @@ class AudioDecoderWrapper extends DecoderWrapper } decode(sample: EncodedAudioSample) { - // We know the decoder spits out data in sorted order, so we need to insert the sample in the right place - const insertionIndex = binarySearchLessOrEqual(this.pendingSamples, sample.timestamp, x => x.timestamp); - this.pendingSamples.splice(insertionIndex + 1, 0, sample); - if (this.customDecoder) { this.customDecoderQueueSize++; this.lastCustomDecoderPromise = this.lastCustomDecoderPromise.then(() => { @@ -993,7 +975,7 @@ class PcmAudioDecoderWrapper extends DecoderWrapper) => unknown, + onData: (data: WrappedMediaFrame) => unknown, onError: (error: DOMException) => unknown, public decoderConfig: AudioDecoderConfig, ) { @@ -1131,7 +1113,6 @@ class PcmAudioDecoderWrapper extends DecoderWrapper this.onFrame({ frame: audioData, - sample, timestamp: preciseTimestamp, duration: preciseDuration, })); @@ -1170,7 +1151,7 @@ export class AudioDataSink extends BaseMediaFrameSink) => unknown, + onData: (data: WrappedMediaFrame) => unknown, onError: (error: DOMException) => unknown, ) { if (!(await this._audioTrack.canDecode())) { @@ -1187,12 +1168,13 @@ export class AudioDataSink extends BaseMediaFrameSink): WrappedAudioData { + _wrappedFrameToWrappedAudioData(frame: WrappedMediaFrame): WrappedAudioData { return { data: frame.frame, timestamp: frame.timestamp, diff --git a/src/mp3/mp3-demuxer.ts b/src/mp3/mp3-demuxer.ts index 6997126..26ede3a 100644 --- a/src/mp3/mp3-demuxer.ts +++ b/src/mp3/mp3-demuxer.ts @@ -8,8 +8,6 @@ import { EncodedAudioSample, PLACEHOLDER_DATA } from '../sample'; import { FrameHeader, getXingOffset, INFO, XING } from './mp3-misc'; import { Mp3Reader } from './mp3-reader'; -const AUDIO_SAMPLES_PER_FRAME = 1152; - type Sample = { timestamp: number; duration: number; @@ -46,7 +44,7 @@ export class Mp3Demuxer extends Demuxer { this.reader.pos += id3Tag.size; } - let nextTimestamp = 0; + let nextTimestampInSamples = 0; // Let's read all samples while (true) { @@ -71,16 +69,16 @@ export class Mp3Demuxer extends Demuxer { this.firstFrameHeader = header; } - const sampleDuration = AUDIO_SAMPLES_PER_FRAME / header.sampleRate; + const sampleDuration = header.audioSamplesInFrame / header.sampleRate; const sample: Sample = { - timestamp: nextTimestamp, + timestamp: nextTimestampInSamples / header.sampleRate, duration: sampleDuration, dataStart: header.startPos, dataSize: header.totalSize, }; this.allSamples.push(sample); - nextTimestamp += sampleDuration; + nextTimestampInSamples += header.audioSamplesInFrame; } if (!this.firstFrameHeader) { @@ -121,6 +119,11 @@ class Mp3AudioTrackBacking implements InputAudioTrackBacking { return 0; } + async getTimeResolution() { + assert(this.demuxer.firstFrameHeader); + return this.demuxer.firstFrameHeader.sampleRate / this.demuxer.firstFrameHeader.audioSamplesInFrame; + } + computeDuration() { return this.demuxer.computeDuration(); } diff --git a/src/mp3/mp3-misc.ts b/src/mp3/mp3-misc.ts index 79f1056..367b972 100644 --- a/src/mp3/mp3-misc.ts +++ b/src/mp3/mp3-misc.ts @@ -44,6 +44,7 @@ export type FrameHeader = { copyright: number; original: number; emphasis: number; + audioSamplesInFrame: number; }; export const computeMp3FrameSize = (layer: number, bitrate: number, sampleRate: number, padding: number) => { @@ -114,6 +115,19 @@ export const readFrameHeader = (word: number, reader: { pos: number; fileSize: n return null; } + let audioSamplesInFrame: number; + if (mpegVersionId === 3) { + audioSamplesInFrame = layer === 3 ? 384 : 1152; + } else { + if (layer === 3) { + audioSamplesInFrame = 384; + } else if (layer === 2) { + audioSamplesInFrame = 1152; + } else { + audioSamplesInFrame = 576; + } + } + return { startPos: startPos, totalSize: frameLength, @@ -127,5 +141,6 @@ export const readFrameHeader = (word: number, reader: { pos: number; fileSize: n copyright, original, emphasis, + audioSamplesInFrame, }; }; diff --git a/src/ogg/ogg-demuxer.ts b/src/ogg/ogg-demuxer.ts index 14aa736..e6cbb16 100644 --- a/src/ogg/ogg-demuxer.ts +++ b/src/ogg/ogg-demuxer.ts @@ -418,6 +418,10 @@ class OggAudioTrackBacking implements InputAudioTrackBacking { return this.bitstream.sampleRate; } + async getTimeResolution() { + return this.bitstream.sampleRate; + } + async getCodec() { return this.bitstream.codec; } diff --git a/src/wave/wave-demuxer.ts b/src/wave/wave-demuxer.ts index ee3d9ae..db14bef 100644 --- a/src/wave/wave-demuxer.ts +++ b/src/wave/wave-demuxer.ts @@ -219,6 +219,11 @@ class WaveAudioTrackBacking implements InputAudioTrackBacking { return this.demuxer.audioInfo.sampleRate; } + async getTimeResolution() { + assert(this.demuxer.audioInfo); + return this.demuxer.audioInfo.sampleRate; + } + async getLanguageCode() { return UNDETERMINED_LANGUAGE; }