From 2dfb5959117b9c8e0ac4265c554eb99f15b1d4a9 Mon Sep 17 00:00:00 2001 From: Vanilagy <1696106+Vanilagy@users.noreply.github.com> Date: Sun, 12 Jan 2025 15:17:21 +0100 Subject: [PATCH] Improve lookup algorithm to handle misleading cue points, fix cluster start timestamp in Matroska muxer, improve cue points in Matroska muxer, & other misc improvements --- dev/player.html | 17 ++++- src/codec.ts | 8 +- src/input-track.ts | 1 + src/isobmff/isobmff-demuxer.ts | 45 ++++++----- src/isobmff/isobmff-muxer.ts | 46 ++++++----- src/matroska/matroska-demuxer.ts | 126 +++++++++++++++++++++++-------- src/matroska/matroska-muxer.ts | 66 +++++++++++----- 7 files changed, 216 insertions(+), 93 deletions(-) diff --git a/dev/player.html b/dev/player.html index 4c31360..5cc42fe 100644 --- a/dev/player.html +++ b/dev/player.html @@ -41,10 +41,23 @@ const input = new Metamuxer.Input({ source }); -console.log(await input.getMimeType()) +//console.log(await input.getMimeType()) let videoTrack = await input.getPrimaryVideoTrack(); -let audioTrack = await input.getPrimaryAudioTrack(); +let audioTrack = null && await input.getPrimaryAudioTrack(); + + +const sink = new Metamuxer.EncodedVideoSampleSink(videoTrack); +console.log(await sink.getSample(4.93)) +await new Promise(() => {}); + +for await (const sample of sink.samples()) { + console.log(sample); + await new Promise(resolve => setTimeout(resolve, 100)); +} + + + //console.log(await videoTrack.computeDuration()); //await new Promise(() => {}); diff --git a/src/codec.ts b/src/codec.ts index 25324a3..882dadb 100644 --- a/src/codec.ts +++ b/src/codec.ts @@ -573,7 +573,7 @@ export const extractAv1CodecInfoFromFrame = (data: Uint8Array): Av1CodecInfo | n let offset = 0; - const readLeb128 = (data: Uint8Array): number | null => { + const readLeb128 = () => { let value = 0; for (let i = 0; i < 8; i++) { @@ -617,7 +617,7 @@ export const extractAv1CodecInfoFromFrame = (data: Uint8Array): Av1CodecInfo | n // Read OBU size if present let obuSize: number; if (obuHasSizeField) { - const obuSizeValue = readLeb128(data); + const obuSizeValue = readLeb128(); if (obuSizeValue === null) return null; // It was invalid obuSize = obuSizeValue; } else { @@ -838,7 +838,9 @@ export const extractAudioCodecString = (trackInfo: { const { codec, codecDescription, aacCodecInfo } = trackInfo; if (codec === 'aac') { - assert(aacCodecInfo); + if (!aacCodecInfo) { + throw new TypeError('AAC codec info must be provided.'); + } if (aacCodecInfo.isMpeg2) { return 'mp4a.67'; diff --git a/src/input-track.ts b/src/input-track.ts index be12bf6..bc8d09a 100644 --- a/src/input-track.ts +++ b/src/input-track.ts @@ -22,6 +22,7 @@ export abstract class InputTrack { abstract getCodec(): Promise; abstract getCodecMimeType(): Promise; abstract canDecode(): Promise; + abstract computeSampleStats(): Promise; isVideoTrack(): this is InputVideoTrack { return this instanceof InputVideoTrack; diff --git a/src/isobmff/isobmff-demuxer.ts b/src/isobmff/isobmff-demuxer.ts index 93cdf90..3b39ffc 100644 --- a/src/isobmff/isobmff-demuxer.ts +++ b/src/isobmff/isobmff-demuxer.ts @@ -1792,6 +1792,7 @@ abstract class IsobmffTrackBacking< x => x.moofOffset, ); assert(fragmentIndex !== -1); + return { fragmentIndex, sampleIndex: 0, @@ -2053,15 +2054,11 @@ abstract class IsobmffTrackBacking< /** Looks for a sample in the fragments while trying to load as few fragments as possible to retrieve it. */ private async performFragmentedLookup( - // This function returns the best-matching sample that is currently loaded. Based on this information, we know - // which fragments we need to load to find the actual match. getBestMatch: () => { fragmentIndex: number; sampleIndex: number; correctSampleFound: boolean }, - // The timestamp with which we can search the lookup table searchTimestamp: number, - // The timestamp for which we know the correct sample will not come after it latestTimestamp: number, options: SampleRetrievalOptions, - ) { + ): Promise { const demuxer = this.internalTrack.demuxer; const release = await demuxer.fragmentLookupMutex.acquire(); // The algorithm requires exclusivity @@ -2080,20 +2077,18 @@ abstract class IsobmffTrackBacking< let bestFragmentIndex = fragmentIndex; let bestSampleIndex = sampleIndex; - let lookupEntry: FragmentLookupTableEntry | null = null; - if (this.internalTrack.fragmentLookupTable) { - // Search for a lookup entry; this way, we won't need to start searching from the start of the file - // but can jump right into the correct fragment (or at least nearby). - const index = binarySearchLessOrEqual( + // Search for a lookup entry; this way, we won't need to start searching from the start of the file + // but can jump right into the correct fragment (or at least nearby). + const lookupEntryIndex = this.internalTrack.fragmentLookupTable + ? binarySearchLessOrEqual( this.internalTrack.fragmentLookupTable, searchTimestamp, x => x.timestamp, - ); - - if (index !== -1) { - lookupEntry = this.internalTrack.fragmentLookupTable[index]!; - } - } + ) + : -1; + const lookupEntry = lookupEntryIndex !== -1 + ? this.internalTrack.fragmentLookupTable![lookupEntryIndex]! + : null; if (fragmentIndex === -1) { isobmffReader.pos = lookupEntry?.moofOffset ?? 0; @@ -2163,13 +2158,23 @@ abstract class IsobmffTrackBacking< isobmffReader.pos = startPos + boxInfo.totalSize; } - if (bestFragmentIndex !== -1) { + let result: Sample | null = null; + const bestFragment = bestFragmentIndex !== -1 ? this.internalTrack.fragments[bestFragmentIndex]! : null; + if (bestFragment) { // If we finished looping but didn't find a perfect match, still return the best match we found - const fragment = this.internalTrack.fragments[bestFragmentIndex]!; - return this.fetchSampleInFragment(fragment, bestSampleIndex, options); + result = await this.fetchSampleInFragment(bestFragment, bestSampleIndex, options); } - return null; + // Catch faulty lookup table entries + if (!result && lookupEntry && (!bestFragment || bestFragment.moofOffset < lookupEntry.moofOffset)) { + // The lookup table entry lied to us! We found a lookup entry but no fragment there that satisfied + // the match. In this case, let's search again but using the lookup entry before that. + const previousLookupEntry = this.internalTrack.fragmentLookupTable![lookupEntryIndex - 1]; + const newSearchTimestamp = previousLookupEntry?.timestamp ?? -Infinity; + return this.performFragmentedLookup(getBestMatch, newSearchTimestamp, latestTimestamp, options); + } + + return result; } finally { release(); } diff --git a/src/isobmff/isobmff-muxer.ts b/src/isobmff/isobmff-muxer.ts index 3b0d53c..a163477 100644 --- a/src/isobmff/isobmff-muxer.ts +++ b/src/isobmff/isobmff-muxer.ts @@ -101,6 +101,7 @@ export class IsobmffMuxer extends Muxer { private boxWriter: IsobmffBoxWriter; private isMov: boolean; private fastStart: NonNullable; + private isFragmented: boolean; private auxTarget = new BufferTarget(); private auxWriter = this.auxTarget._createWriter(); @@ -115,6 +116,8 @@ export class IsobmffMuxer extends Muxer { private finalizedChunks: Chunk[] = []; private nextFragmentNumber = 1; + // Only relevant for fragmented files, to make sure new fragments start with the highest timestamp seen so far + private maxWrittenTimestamp = -Infinity; constructor(output: Output, format: IsobmffOutputFormat) { super(output); @@ -128,8 +131,9 @@ export class IsobmffMuxer extends Muxer { // memory usage remains identical const fastStartDefault = this.writer instanceof BufferTargetWriter ? 'in-memory' : false; this.fastStart = format._options.fastStart ?? fastStartDefault; + this.isFragmented = this.fastStart === 'fragmented'; - if (this.fastStart === 'in-memory' || this.fastStart === 'fragmented') { + if (this.fastStart === 'in-memory' || this.isFragmented) { this.writer.ensureMonotonicity = true; } } @@ -143,14 +147,14 @@ export class IsobmffMuxer extends Muxer { this.boxWriter.writeBox(ftyp({ isMov: this.isMov, holdsAvc: holdsAvc, - fragmented: this.fastStart === 'fragmented', + fragmented: this.isFragmented, })); this.ftypSize = this.writer.getPos(); if (this.fastStart === 'in-memory') { this.mdat = mdat(false); - } else if (this.fastStart === 'fragmented') { + } else if (this.isFragmented) { // We write the moov box once we write out the first fragment to make sure we get the decoder configs } else { this.mdat = mdat(true); // Reserve large size by default, can refine this when finalizing. @@ -234,7 +238,7 @@ export class IsobmffMuxer extends Muxer { currentChunk: null, compactlyCodedChunkTable: [], requiresPcmTransformation: - this.fastStart !== 'fragmented' + !this.isFragmented && (PCM_CODECS as readonly string[]).includes(track.source._codec), }; @@ -545,7 +549,7 @@ export class IsobmffMuxer extends Muxer { // model it. sample.decodeTimestamp = sortedTimestamps[i]!; - if (this.fastStart !== 'fragmented' && trackData.lastTimescaleUnits === null) { + if (!this.isFragmented && trackData.lastTimescaleUnits === null) { // In non-fragmented files, the first decode timestamp is always zero. If the first presentation // timestamp isn't zero, we'll simply use the composition time offset to achieve it. sample.decodeTimestamp = 0; @@ -563,7 +567,7 @@ export class IsobmffMuxer extends Muxer { trackData.lastTimescaleUnits += delta; trackData.lastSample.timescaleUnitsToNextSample = delta; - if (this.fastStart !== 'fragmented') { + if (!this.isFragmented) { let lastTableEntry = last(trackData.timeToSampleTable); assert(lastTableEntry); @@ -618,7 +622,7 @@ export class IsobmffMuxer extends Muxer { // Decode timestamp of the first sample trackData.lastTimescaleUnits = intoTimescale(sample.decodeTimestamp, trackData.timescale, false); - if (this.fastStart !== 'fragmented') { + if (!this.isFragmented) { trackData.timeToSampleTable.push({ sampleCount: 1, sampleDelta: durationInTimescale, @@ -637,7 +641,7 @@ export class IsobmffMuxer extends Muxer { } private async registerSample(trackData: IsobmffTrackData, sample: Sample) { - if (this.fastStart === 'fragmented') { + if (this.isFragmented) { trackData.sampleQueue.push(sample); await this.interleaveSamples(); } else { @@ -650,7 +654,7 @@ export class IsobmffMuxer extends Muxer { this.processTimestamps(trackData); } - if (this.fastStart !== 'fragmented') { + if (!this.isFragmented) { trackData.samples.push(sample); } @@ -660,7 +664,7 @@ export class IsobmffMuxer extends Muxer { } else { const currentChunkDuration = sample.timestamp - trackData.currentChunk.startTimestamp; - if (this.fastStart === 'fragmented') { + if (this.isFragmented) { // We can only finalize this fragment (and begin a new one) if we know that each track will be able to // start the new one with a key frame. const keyFrameQueuedEverywhere = this.trackDatas.every((otherTrackData) => { @@ -676,7 +680,11 @@ export class IsobmffMuxer extends Muxer { return firstQueuedSample && firstQueuedSample.type === 'key'; }); - if (currentChunkDuration >= 1.0 && keyFrameQueuedEverywhere) { + if ( + currentChunkDuration >= 1.0 + && keyFrameQueuedEverywhere + && sample.timestamp > this.maxWrittenTimestamp + ) { beginNewChunk = true; await this.finalizeFragment(); } @@ -701,10 +709,14 @@ export class IsobmffMuxer extends Muxer { assert(trackData.currentChunk); trackData.currentChunk.samples.push(sample); trackData.timestampProcessingQueue.push(sample); + + if (this.isFragmented) { + this.maxWrittenTimestamp = Math.max(this.maxWrittenTimestamp, sample.timestamp); + } } private async finalizeCurrentChunk(trackData: IsobmffTrackData) { - assert(this.fastStart !== 'fragmented'); + assert(!this.isFragmented); if (!trackData.currentChunk) return; @@ -744,7 +756,7 @@ export class IsobmffMuxer extends Muxer { } private async interleaveSamples(isFinalCall = false) { - assert(this.fastStart === 'fragmented'); + assert(this.isFragmented); if (!isFinalCall) { for (const track of this.output._tracks) { @@ -780,7 +792,7 @@ export class IsobmffMuxer extends Muxer { } private async finalizeFragment(flushWriter = true) { - assert(this.fastStart === 'fragmented'); + assert(this.isFragmented); const fragmentNumber = this.nextFragmentNumber++; @@ -861,7 +873,7 @@ export class IsobmffMuxer extends Muxer { } } - if (this.fastStart === 'fragmented') { + if (this.isFragmented) { // Since a track is now closed, we may be able to write out chunks that were previously waiting await this.interleaveSamples(); } @@ -879,7 +891,7 @@ export class IsobmffMuxer extends Muxer { } } - if (this.fastStart === 'fragmented') { + if (this.isFragmented) { await this.interleaveSamples(true); await this.finalizeFragment(false); // Don't flush the last fragment as we will flush it with the mfra box } else { @@ -933,7 +945,7 @@ export class IsobmffMuxer extends Muxer { sample.data = null; } } - } else if (this.fastStart === 'fragmented') { + } else if (this.isFragmented) { // Append the mfra box to the end of the file for better random access const startPos = this.writer.getPos(); const mfraBox = mfra(this.trackDatas); diff --git a/src/matroska/matroska-demuxer.ts b/src/matroska/matroska-demuxer.ts index 2917f65..e81e45d 100644 --- a/src/matroska/matroska-demuxer.ts +++ b/src/matroska/matroska-demuxer.ts @@ -90,6 +90,7 @@ type ClusterBlock = { type CuePoint = { time: number; + trackId: number; clusterPosition: number; }; @@ -99,6 +100,7 @@ type InternalTrack = { segment: Segment; clusters: Cluster[]; clustersWithKeyFrame: Cluster[]; + cuePoints: CuePoint[]; isDefault: boolean; inputTrack: InputTrack | null; @@ -143,6 +145,7 @@ export class MatroskaDemuxer extends Demuxer { currentTrack: InternalTrack | null = null; currentCluster: Cluster | null = null; currentBlock: ClusterBlock | null = null; + currentCueTime: number | null = null; isWebM = false; @@ -336,8 +339,54 @@ export class MatroskaDemuxer extends Demuxer { // Put default tracks first this.currentSegment.tracks.sort((a, b) => Number(b.isDefault) - Number(a.isDefault)); - // Sort cue points by time - this.currentSegment.cuePoints.sort((a, b) => a.time - b.time); + // Sort cue points by cluster position (required for the next algorithm) + this.currentSegment.cuePoints.sort((a, b) => a.clusterPosition - b.clusterPosition); + + // Now, let's distribute the cue points to each track. Ideally, each track has their own cue point, but some + // Matroska files may only specify cue points for a single track. In this case, we still wanna use those cue + // points for all tracks. + const allTrackIds = this.currentSegment.tracks.map(x => x.id); + const remainingTrackIds = new Set(); + let lastClusterPosition: number | null = null; + let lastCuePoint: CuePoint | null = null; + + for (const cuePoint of this.currentSegment.cuePoints) { + if (cuePoint.clusterPosition !== lastClusterPosition) { + for (const id of remainingTrackIds) { + // These tracks didn't receive a cue point for the last cluster, so let's give them one + assert(lastCuePoint); + const track = this.currentSegment.tracks.find(x => x.id === id)!; + track.cuePoints.push(lastCuePoint); + } + + for (const id of allTrackIds) { + remainingTrackIds.add(id); + } + } + + lastCuePoint = cuePoint; + + if (!remainingTrackIds.has(cuePoint.trackId)) { + continue; + } + + const track = this.currentSegment.tracks.find(x => x.id === cuePoint.trackId)!; + track.cuePoints.push(cuePoint); + + remainingTrackIds.delete(cuePoint.trackId); + lastClusterPosition = cuePoint.clusterPosition; + } + + for (const id of remainingTrackIds) { + assert(lastCuePoint); + const track = this.currentSegment.tracks.find(x => x.id === id)!; + track.cuePoints.push(lastCuePoint); + } + + for (const track of this.currentSegment.tracks) { + // Sort cue points by time + track.cuePoints.sort((a, b) => a.time - b.time); + } this.currentSegment = null; } @@ -526,6 +575,8 @@ export class MatroskaDemuxer extends Demuxer { demuxer: this, clusters: [], clustersWithKeyFrame: [], + cuePoints: [], + isDefault: false, inputTrack: null, codecId: null, @@ -773,27 +824,32 @@ export class MatroskaDemuxer extends Demuxer { case EBMLId.CuePoint: { if (!this.currentSegment) break; - const cuePoint: CuePoint = { time: -1, clusterPosition: -1 }; + this.readContiguousElements(reader, size); + this.currentCueTime = null; + }; break; + + case EBMLId.CueTime: { + this.currentCueTime = reader.readUnsignedInt(size); + }; break; + + case EBMLId.CueTrackPositions: { + if (this.currentCueTime === null) break; + assert(this.currentSegment); + + const cuePoint: CuePoint = { time: this.currentCueTime, trackId: -1, clusterPosition: -1 }; this.currentSegment.cuePoints.push(cuePoint); this.readContiguousElements(reader, size); - if (cuePoint.time === -1 || cuePoint.clusterPosition === -1) { + if (cuePoint.trackId === -1 || cuePoint.clusterPosition === -1) { this.currentSegment.cuePoints.pop(); } }; break; - case EBMLId.CueTime: { + case EBMLId.CueTrack: { const lastCuePoint = this.currentSegment?.cuePoints[this.currentSegment.cuePoints.length - 1]; if (!lastCuePoint) break; - lastCuePoint.time = reader.readUnsignedInt(size); - }; break; - - case EBMLId.CueTrackPositions: { - const lastCuePoint = this.currentSegment?.cuePoints[this.currentSegment.cuePoints.length - 1]; - if (!lastCuePoint) break; - - this.readContiguousElements(reader, size); + lastCuePoint.trackId = reader.readUnsignedInt(size); }; break; case EBMLId.CueClusterPosition: { @@ -997,6 +1053,7 @@ abstract class MatroskaTrackBacking< x => x.elementStartPos, ); assert(clusterIndex !== -1); + return { clusterIndex, blockIndex: 0, @@ -1196,7 +1253,7 @@ abstract class MatroskaTrackBacking< // The timestamp for which we know the correct block will not come after it latestTimestamp: number, options: SampleRetrievalOptions, - ) { + ): Promise { const { demuxer, segment } = this.internalTrack; const release = await segment.clusterLookupMutex.acquire(); // The algorithm requires exclusivity @@ -1215,20 +1272,14 @@ abstract class MatroskaTrackBacking< let bestClusterIndex = clusterIndex; let bestBlockIndex = blockIndex; - let cuePoint: CuePoint | null = null; - if (segment.cuePoints.length > 0) { - // Search for a cue point; this way, we won't need to start searching from the start of the file - // but can jump right into the correct cluster (or at least nearby). - const index = binarySearchLessOrEqual( - segment.cuePoints, - searchTimestamp, - x => x.time, - ); - - if (index !== -1) { - cuePoint = segment.cuePoints[index]!; - } - } + // Search for a cue point; this way, we won't need to start searching from the start of the file + // but can jump right into the correct cluster (or at least nearby). + const cuePointIndex = binarySearchLessOrEqual( + this.internalTrack.cuePoints, + searchTimestamp, + x => x.time, + ); + const cuePoint = cuePointIndex !== -1 ? this.internalTrack.cuePoints[cuePointIndex]! : null; if (clusterIndex === -1) { metadataReader.pos = cuePoint?.clusterPosition ?? segment.clusterSeekStartPos; @@ -1299,13 +1350,23 @@ abstract class MatroskaTrackBacking< metadataReader.pos = dataStartPos + size; } - if (bestClusterIndex !== -1) { + let result: Sample | null = null; + const bestCluster = bestClusterIndex !== -1 ? this.internalTrack.clusters[bestClusterIndex]! : null; + if (bestCluster) { // If we finished looping but didn't find a perfect match, still return the best match we found - const cluster = this.internalTrack.clusters[bestClusterIndex]!; - return this.fetchSampleInCluster(cluster, bestBlockIndex, options); + result = await this.fetchSampleInCluster(bestCluster, bestBlockIndex, options); } - return null; + // Catch faulty cue points + if (!result && cuePoint && (!bestCluster || bestCluster.elementStartPos < cuePoint.clusterPosition)) { + // The cue point lied to us! We found a cue point but no cluster there that satisfied the match. In this + // case, let's search again but using the cue point before that. + const previousCuePoint = this.internalTrack.cuePoints[cuePointIndex - 1]; + const newSearchTimestamp = previousCuePoint?.time ?? -Infinity; + return this.performClusterLookup(getBestMatch, newSearchTimestamp, latestTimestamp, options); + } + + return result; } finally { release(); } @@ -1440,6 +1501,7 @@ class MatroskaAudioTrackBacking extends MatroskaTrackBacking * come before block B. The resulting array is one that is in decode order. */ const sortBlocksTopologically = (blocks: ClusterBlock[]) => { + return blocks; // temp // Based on "A fast and effective heuristic for the feedback arc set problem" by Peter Eades et al. const n = blocks.length; diff --git a/src/matroska/matroska-muxer.ts b/src/matroska/matroska-muxer.ts index 00b293a..0aac314 100644 --- a/src/matroska/matroska-muxer.ts +++ b/src/matroska/matroska-muxer.ts @@ -122,8 +122,11 @@ export class MatroskaMuxer extends Muxer { private cues: EBMLElement | null = null; private currentCluster: EBMLElement | null = null; - private currentClusterMsTimestamp: number | null = null; - private trackDatasInCurrentCluster = new Set(); + private currentClusterStartMsTimestamp: number | null = null; + private currentClusterMaxMsTimestamp: number | null = null; + private trackDatasInCurrentCluster = new Map(); private duration = 0; @@ -620,7 +623,7 @@ export class MatroskaMuxer extends Muxer { } const msTimestamp = Math.floor(1000 * chunk.timestamp); - // We can only finalize this fragment (and begin a new one) if we know that each track will be able to + // We can only finalize this cluster (and begin a new one) if we know that each track will be able to // start the new one with a key frame. const keyFrameQueuedEverywhere = this.trackDatas.every((otherTrackData) => { if (otherTrackData.track.source._closed) { @@ -637,12 +640,19 @@ export class MatroskaMuxer extends Muxer { if ( !this.currentCluster - || (keyFrameQueuedEverywhere && msTimestamp - this.currentClusterMsTimestamp! >= 1000) + || ( + keyFrameQueuedEverywhere + // This check is required because that means there is already a block with this timestamp in the + // CURRENT chunk, meaning that starting the next cluster at the same timestamp is forbidden (since the + // already-written block would belong into it instead). + && msTimestamp > this.currentClusterMaxMsTimestamp! + && msTimestamp - this.currentClusterStartMsTimestamp! >= 1000 + ) ) { this.createNewCluster(msTimestamp); } - const relativeTimestamp = msTimestamp - this.currentClusterMsTimestamp!; + const relativeTimestamp = msTimestamp - this.currentClusterStartMsTimestamp!; if (relativeTimestamp < 0) { // The chunk lies outside of the current cluster return; @@ -702,7 +712,12 @@ export class MatroskaMuxer extends Muxer { this.duration = Math.max(this.duration, msTimestamp + msDuration); trackData.lastWrittenMsTimestamp = msTimestamp; - this.trackDatasInCurrentCluster.add(trackData); + if (!this.trackDatasInCurrentCluster.has(trackData)) { + this.trackDatasInCurrentCluster.set(trackData, { + firstMsTimestamp: msTimestamp, + }); + } + this.currentClusterMaxMsTimestamp = Math.max(this.currentClusterMaxMsTimestamp!, msTimestamp); } /** Creates a new Cluster element to contain media chunks. */ @@ -726,7 +741,8 @@ export class MatroskaMuxer extends Muxer { }; this.ebmlWriter.writeEBML(this.currentCluster); - this.currentClusterMsTimestamp = msTimestamp; + this.currentClusterStartMsTimestamp = msTimestamp; + this.currentClusterMaxMsTimestamp = msTimestamp; this.trackDatasInCurrentCluster.clear(); } @@ -750,19 +766,31 @@ export class MatroskaMuxer extends Muxer { const clusterOffsetFromSegment = this.ebmlWriter.offsets.get(this.currentCluster)! - this.segmentDataOffset; - assert(this.cues); + // Group tracks by their first timestamp and create a CuePoint for each unique timestamp + const groupedByTimestamp = new Map(); + for (const [trackData, { firstMsTimestamp }] of this.trackDatasInCurrentCluster) { + if (!groupedByTimestamp.has(firstMsTimestamp)) { + groupedByTimestamp.set(firstMsTimestamp, []); + } + groupedByTimestamp.get(firstMsTimestamp)!.push(trackData); + } - // Add a CuePoint to the Cues element for better seeking - (this.cues.data as EBML[]).push({ id: EBMLId.CuePoint, data: [ - { id: EBMLId.CueTime, data: this.currentClusterMsTimestamp! }, - // We only write out cues for tracks that have at least one chunk in this cluster - ...[...this.trackDatasInCurrentCluster].map((trackData) => { - return { id: EBMLId.CueTrackPositions, data: [ - { id: EBMLId.CueTrack, data: trackData.track.id }, - { id: EBMLId.CueClusterPosition, data: clusterOffsetFromSegment }, - ] }; - }), - ] }); + const groupedAndSortedByTimestamp = [...groupedByTimestamp.entries()].sort((a, b) => a[0] - b[0]); + + // Add CuePoints to the Cues element for better seeking + for (const [msTimestamp, trackDatas] of groupedAndSortedByTimestamp) { + assert(this.cues); + (this.cues.data as EBML[]).push({ id: EBMLId.CuePoint, data: [ + { id: EBMLId.CueTime, data: msTimestamp }, + // Create CueTrackPositions for each track that starts at this timestamp + ...trackDatas.map((trackData) => { + return { id: EBMLId.CueTrackPositions, data: [ + { id: EBMLId.CueTrack, data: trackData.track.id }, + { id: EBMLId.CueClusterPosition, data: clusterOffsetFromSegment }, + ] }; + }), + ] }); + } } // eslint-disable-next-line @typescript-eslint/no-misused-promises