Improve lookup algorithm to handle misleading cue points, fix cluster start timestamp in Matroska muxer, improve cue points in Matroska muxer, & other misc improvements

dev/player.html

@@ -41,10 +41,23 @@ const input = new Metamuxer.Input({
 	source
 });
 
-console.log(await input.getMimeType())
+//console.log(await input.getMimeType())
 
 let videoTrack = await input.getPrimaryVideoTrack(); 
-let audioTrack = await input.getPrimaryAudioTrack();
+let audioTrack = null && await input.getPrimaryAudioTrack();
+
+
+const sink = new Metamuxer.EncodedVideoSampleSink(videoTrack);
+console.log(await sink.getSample(4.93))
+await new Promise(() => {});
+
+for await (const sample of sink.samples()) {
+	console.log(sample);
+	await new Promise(resolve => setTimeout(resolve, 100));
+}
+
+
+
 
 //console.log(await videoTrack.computeDuration());
 //await new Promise(() => {});

src/codec.ts

@@ -573,7 +573,7 @@ export const extractAv1CodecInfoFromFrame = (data: Uint8Array): Av1CodecInfo | n
 
 	let offset = 0;
 
-	const readLeb128 = (data: Uint8Array): number | null => {
+	const readLeb128 = () => {
 		let value = 0;
 
 		for (let i = 0; i < 8; i++) {
@@ -617,7 +617,7 @@ export const extractAv1CodecInfoFromFrame = (data: Uint8Array): Av1CodecInfo | n
 		// Read OBU size if present
 		let obuSize: number;
 		if (obuHasSizeField) {
-			const obuSizeValue = readLeb128(data);
+			const obuSizeValue = readLeb128();
 			if (obuSizeValue === null) return null; // It was invalid
 			obuSize = obuSizeValue;
 		} else {
@@ -838,7 +838,9 @@ export const extractAudioCodecString = (trackInfo: {
 	const { codec, codecDescription, aacCodecInfo } = trackInfo;
 
 	if (codec === 'aac') {
-		assert(aacCodecInfo);
+		if (!aacCodecInfo) {
+			throw new TypeError('AAC codec info must be provided.');
+		}
 
 		if (aacCodecInfo.isMpeg2) {
 			return 'mp4a.67';

src/input-track.ts

@@ -22,6 +22,7 @@ export abstract class InputTrack {
 	abstract getCodec(): Promise<MediaCodec | null>;
 	abstract getCodecMimeType(): Promise<string | null>;
 	abstract canDecode(): Promise<boolean>;
+	abstract computeSampleStats(): Promise<SampleStats>;
 
 	isVideoTrack(): this is InputVideoTrack {
 		return this instanceof InputVideoTrack;

src/isobmff/isobmff-demuxer.ts

@@ -1792,6 +1792,7 @@ abstract class IsobmffTrackBacking<
 									x => x.moofOffset,
 								);
 								assert(fragmentIndex !== -1);
+
 								return {
 									fragmentIndex,
 									sampleIndex: 0,
@@ -2053,15 +2054,11 @@ abstract class IsobmffTrackBacking<
 
 	/** Looks for a sample in the fragments while trying to load as few fragments as possible to retrieve it. */
 	private async performFragmentedLookup(
-		// This function returns the best-matching sample that is currently loaded. Based on this information, we know
-		// which fragments we need to load to find the actual match.
 		getBestMatch: () => { fragmentIndex: number; sampleIndex: number; correctSampleFound: boolean },
-		// The timestamp with which we can search the lookup table
 		searchTimestamp: number,
-		// The timestamp for which we know the correct sample will not come after it
 		latestTimestamp: number,
 		options: SampleRetrievalOptions,
-	) {
+	): Promise<Sample | null> {
 		const demuxer = this.internalTrack.demuxer;
 		const release = await demuxer.fragmentLookupMutex.acquire(); // The algorithm requires exclusivity
 
@@ -2080,20 +2077,18 @@ abstract class IsobmffTrackBacking<
 			let bestFragmentIndex = fragmentIndex;
 			let bestSampleIndex = sampleIndex;
 
-			let lookupEntry: FragmentLookupTableEntry | null = null;
-			if (this.internalTrack.fragmentLookupTable) {
-				// Search for a lookup entry; this way, we won't need to start searching from the start of the file
-				// but can jump right into the correct fragment (or at least nearby).
-				const index = binarySearchLessOrEqual(
+			// Search for a lookup entry; this way, we won't need to start searching from the start of the file
+			// but can jump right into the correct fragment (or at least nearby).
+			const lookupEntryIndex = this.internalTrack.fragmentLookupTable
+				? binarySearchLessOrEqual(
 					this.internalTrack.fragmentLookupTable,
 					searchTimestamp,
 					x => x.timestamp,
-				);
-
-				if (index !== -1) {
-					lookupEntry = this.internalTrack.fragmentLookupTable[index]!;
-				}
-			}
+				)
+				: -1;
+			const lookupEntry = lookupEntryIndex !== -1
+				? this.internalTrack.fragmentLookupTable![lookupEntryIndex]!
+				: null;
 
 			if (fragmentIndex === -1) {
 				isobmffReader.pos = lookupEntry?.moofOffset ?? 0;
@@ -2163,13 +2158,23 @@ abstract class IsobmffTrackBacking<
 				isobmffReader.pos = startPos + boxInfo.totalSize;
 			}
 
-			if (bestFragmentIndex !== -1) {
+			let result: Sample | null = null;
+			const bestFragment = bestFragmentIndex !== -1 ? this.internalTrack.fragments[bestFragmentIndex]! : null;
+			if (bestFragment) {
 				// If we finished looping but didn't find a perfect match, still return the best match we found
-				const fragment = this.internalTrack.fragments[bestFragmentIndex]!;
-				return this.fetchSampleInFragment(fragment, bestSampleIndex, options);
+				result = await this.fetchSampleInFragment(bestFragment, bestSampleIndex, options);
 			}
 
-			return null;
+			// Catch faulty lookup table entries
+			if (!result && lookupEntry && (!bestFragment || bestFragment.moofOffset < lookupEntry.moofOffset)) {
+				// The lookup table entry lied to us! We found a lookup entry but no fragment there that satisfied
+				// the match. In this case, let's search again but using the lookup entry before that.
+				const previousLookupEntry = this.internalTrack.fragmentLookupTable![lookupEntryIndex - 1];
+				const newSearchTimestamp = previousLookupEntry?.timestamp ?? -Infinity;
+				return this.performFragmentedLookup(getBestMatch, newSearchTimestamp, latestTimestamp, options);
+			}
+
+			return result;
 		} finally {
 			release();
 		}

src/isobmff/isobmff-muxer.ts

@@ -101,6 +101,7 @@ export class IsobmffMuxer extends Muxer {
 	private boxWriter: IsobmffBoxWriter;
 	private isMov: boolean;
 	private fastStart: NonNullable<IsobmffOutputFormatOptions['fastStart']>;
+	private isFragmented: boolean;
 
 	private auxTarget = new BufferTarget();
 	private auxWriter = this.auxTarget._createWriter();
@@ -115,6 +116,8 @@ export class IsobmffMuxer extends Muxer {
 	private finalizedChunks: Chunk[] = [];
 
 	private nextFragmentNumber = 1;
+	// Only relevant for fragmented files, to make sure new fragments start with the highest timestamp seen so far
+	private maxWrittenTimestamp = -Infinity;
 
 	constructor(output: Output, format: IsobmffOutputFormat) {
 		super(output);
@@ -128,8 +131,9 @@ export class IsobmffMuxer extends Muxer {
 		// memory usage remains identical
 		const fastStartDefault = this.writer instanceof BufferTargetWriter ? 'in-memory' : false;
 		this.fastStart = format._options.fastStart ?? fastStartDefault;
+		this.isFragmented = this.fastStart === 'fragmented';
 
-		if (this.fastStart === 'in-memory' || this.fastStart === 'fragmented') {
+		if (this.fastStart === 'in-memory' || this.isFragmented) {
 			this.writer.ensureMonotonicity = true;
 		}
 	}
@@ -143,14 +147,14 @@ export class IsobmffMuxer extends Muxer {
 		this.boxWriter.writeBox(ftyp({
 			isMov: this.isMov,
 			holdsAvc: holdsAvc,
-			fragmented: this.fastStart === 'fragmented',
+			fragmented: this.isFragmented,
 		}));
 
 		this.ftypSize = this.writer.getPos();
 
 		if (this.fastStart === 'in-memory') {
 			this.mdat = mdat(false);
-		} else if (this.fastStart === 'fragmented') {
+		} else if (this.isFragmented) {
 			// We write the moov box once we write out the first fragment to make sure we get the decoder configs
 		} else {
 			this.mdat = mdat(true); // Reserve large size by default, can refine this when finalizing.
@@ -234,7 +238,7 @@ export class IsobmffMuxer extends Muxer {
 			currentChunk: null,
 			compactlyCodedChunkTable: [],
 			requiresPcmTransformation:
-				this.fastStart !== 'fragmented'
+				!this.isFragmented
 				&& (PCM_CODECS as readonly string[]).includes(track.source._codec),
 		};
 
@@ -545,7 +549,7 @@ export class IsobmffMuxer extends Muxer {
 			// model it.
 			sample.decodeTimestamp = sortedTimestamps[i]!;
 
-			if (this.fastStart !== 'fragmented' && trackData.lastTimescaleUnits === null) {
+			if (!this.isFragmented && trackData.lastTimescaleUnits === null) {
 				// In non-fragmented files, the first decode timestamp is always zero. If the first presentation
 				// timestamp isn't zero, we'll simply use the composition time offset to achieve it.
 				sample.decodeTimestamp = 0;
@@ -563,7 +567,7 @@ export class IsobmffMuxer extends Muxer {
 				trackData.lastTimescaleUnits += delta;
 				trackData.lastSample.timescaleUnitsToNextSample = delta;
 
-				if (this.fastStart !== 'fragmented') {
+				if (!this.isFragmented) {
 					let lastTableEntry = last(trackData.timeToSampleTable);
 					assert(lastTableEntry);
 
@@ -618,7 +622,7 @@ export class IsobmffMuxer extends Muxer {
 				// Decode timestamp of the first sample
 				trackData.lastTimescaleUnits = intoTimescale(sample.decodeTimestamp, trackData.timescale, false);
 
-				if (this.fastStart !== 'fragmented') {
+				if (!this.isFragmented) {
 					trackData.timeToSampleTable.push({
 						sampleCount: 1,
 						sampleDelta: durationInTimescale,
@@ -637,7 +641,7 @@ export class IsobmffMuxer extends Muxer {
 	}
 
 	private async registerSample(trackData: IsobmffTrackData, sample: Sample) {
-		if (this.fastStart === 'fragmented') {
+		if (this.isFragmented) {
 			trackData.sampleQueue.push(sample);
 			await this.interleaveSamples();
 		} else {
@@ -650,7 +654,7 @@ export class IsobmffMuxer extends Muxer {
 			this.processTimestamps(trackData);
 		}
 
-		if (this.fastStart !== 'fragmented') {
+		if (!this.isFragmented) {
 			trackData.samples.push(sample);
 		}
 
@@ -660,7 +664,7 @@ export class IsobmffMuxer extends Muxer {
 		} else {
 			const currentChunkDuration = sample.timestamp - trackData.currentChunk.startTimestamp;
 
-			if (this.fastStart === 'fragmented') {
+			if (this.isFragmented) {
 				// We can only finalize this fragment (and begin a new one) if we know that each track will be able to
 				// start the new one with a key frame.
 				const keyFrameQueuedEverywhere = this.trackDatas.every((otherTrackData) => {
@@ -676,7 +680,11 @@ export class IsobmffMuxer extends Muxer {
 					return firstQueuedSample && firstQueuedSample.type === 'key';
 				});
 
-				if (currentChunkDuration >= 1.0 && keyFrameQueuedEverywhere) {
+				if (
+					currentChunkDuration >= 1.0
+					&& keyFrameQueuedEverywhere
+					&& sample.timestamp > this.maxWrittenTimestamp
+				) {
 					beginNewChunk = true;
 					await this.finalizeFragment();
 				}
@@ -701,10 +709,14 @@ export class IsobmffMuxer extends Muxer {
 		assert(trackData.currentChunk);
 		trackData.currentChunk.samples.push(sample);
 		trackData.timestampProcessingQueue.push(sample);
+
+		if (this.isFragmented) {
+			this.maxWrittenTimestamp = Math.max(this.maxWrittenTimestamp, sample.timestamp);
+		}
 	}
 
 	private async finalizeCurrentChunk(trackData: IsobmffTrackData) {
-		assert(this.fastStart !== 'fragmented');
+		assert(!this.isFragmented);
 
 		if (!trackData.currentChunk) return;
 
@@ -744,7 +756,7 @@ export class IsobmffMuxer extends Muxer {
 	}
 
 	private async interleaveSamples(isFinalCall = false) {
-		assert(this.fastStart === 'fragmented');
+		assert(this.isFragmented);
 
 		if (!isFinalCall) {
 			for (const track of this.output._tracks) {
@@ -780,7 +792,7 @@ export class IsobmffMuxer extends Muxer {
 	}
 
 	private async finalizeFragment(flushWriter = true) {
-		assert(this.fastStart === 'fragmented');
+		assert(this.isFragmented);
 
 		const fragmentNumber = this.nextFragmentNumber++;
 
@@ -861,7 +873,7 @@ export class IsobmffMuxer extends Muxer {
 			}
 		}
 
-		if (this.fastStart === 'fragmented') {
+		if (this.isFragmented) {
 			// Since a track is now closed, we may be able to write out chunks that were previously waiting
 			await this.interleaveSamples();
 		}
@@ -879,7 +891,7 @@ export class IsobmffMuxer extends Muxer {
 			}
 		}
 
-		if (this.fastStart === 'fragmented') {
+		if (this.isFragmented) {
 			await this.interleaveSamples(true);
 			await this.finalizeFragment(false); // Don't flush the last fragment as we will flush it with the mfra box
 		} else {
@@ -933,7 +945,7 @@ export class IsobmffMuxer extends Muxer {
 					sample.data = null;
 				}
 			}
-		} else if (this.fastStart === 'fragmented') {
+		} else if (this.isFragmented) {
 			// Append the mfra box to the end of the file for better random access
 			const startPos = this.writer.getPos();
 			const mfraBox = mfra(this.trackDatas);

src/matroska/matroska-demuxer.ts

@@ -90,6 +90,7 @@ type ClusterBlock = {
 
 type CuePoint = {
 	time: number;
+	trackId: number;
 	clusterPosition: number;
 };
 
@@ -99,6 +100,7 @@ type InternalTrack = {
 	segment: Segment;
 	clusters: Cluster[];
 	clustersWithKeyFrame: Cluster[];
+	cuePoints: CuePoint[];
 
 	isDefault: boolean;
 	inputTrack: InputTrack | null;
@@ -143,6 +145,7 @@ export class MatroskaDemuxer extends Demuxer {
 	currentTrack: InternalTrack | null = null;
 	currentCluster: Cluster | null = null;
 	currentBlock: ClusterBlock | null = null;
+	currentCueTime: number | null = null;
 
 	isWebM = false;
 
@@ -336,8 +339,54 @@ export class MatroskaDemuxer extends Demuxer {
 		// Put default tracks first
 		this.currentSegment.tracks.sort((a, b) => Number(b.isDefault) - Number(a.isDefault));
 
-		// Sort cue points by time
-		this.currentSegment.cuePoints.sort((a, b) => a.time - b.time);
+		// Sort cue points by cluster position (required for the next algorithm)
+		this.currentSegment.cuePoints.sort((a, b) => a.clusterPosition - b.clusterPosition);
+
+		// Now, let's distribute the cue points to each track. Ideally, each track has their own cue point, but some
+		// Matroska files may only specify cue points for a single track. In this case, we still wanna use those cue
+		// points for all tracks.
+		const allTrackIds = this.currentSegment.tracks.map(x => x.id);
+		const remainingTrackIds = new Set<number>();
+		let lastClusterPosition: number | null = null;
+		let lastCuePoint: CuePoint | null = null;
+
+		for (const cuePoint of this.currentSegment.cuePoints) {
+			if (cuePoint.clusterPosition !== lastClusterPosition) {
+				for (const id of remainingTrackIds) {
+					// These tracks didn't receive a cue point for the last cluster, so let's give them one
+					assert(lastCuePoint);
+					const track = this.currentSegment.tracks.find(x => x.id === id)!;
+					track.cuePoints.push(lastCuePoint);
+				}
+
+				for (const id of allTrackIds) {
+					remainingTrackIds.add(id);
+				}
+			}
+
+			lastCuePoint = cuePoint;
+
+			if (!remainingTrackIds.has(cuePoint.trackId)) {
+				continue;
+			}
+
+			const track = this.currentSegment.tracks.find(x => x.id === cuePoint.trackId)!;
+			track.cuePoints.push(cuePoint);
+
+			remainingTrackIds.delete(cuePoint.trackId);
+			lastClusterPosition = cuePoint.clusterPosition;
+		}
+
+		for (const id of remainingTrackIds) {
+			assert(lastCuePoint);
+			const track = this.currentSegment.tracks.find(x => x.id === id)!;
+			track.cuePoints.push(lastCuePoint);
+		}
+
+		for (const track of this.currentSegment.tracks) {
+			// Sort cue points by time
+			track.cuePoints.sort((a, b) => a.time - b.time);
+		}
 
 		this.currentSegment = null;
 	}
@@ -526,6 +575,8 @@ export class MatroskaDemuxer extends Demuxer {
 					demuxer: this,
 					clusters: [],
 					clustersWithKeyFrame: [],
+					cuePoints: [],
+
 					isDefault: false,
 					inputTrack: null,
 					codecId: null,
@@ -773,27 +824,32 @@ export class MatroskaDemuxer extends Demuxer {
 			case EBMLId.CuePoint: {
 				if (!this.currentSegment) break;
 
-				const cuePoint: CuePoint = { time: -1, clusterPosition: -1 };
+				this.readContiguousElements(reader, size);
+				this.currentCueTime = null;
+			}; break;
+
+			case EBMLId.CueTime: {
+				this.currentCueTime = reader.readUnsignedInt(size);
+			}; break;
+
+			case EBMLId.CueTrackPositions: {
+				if (this.currentCueTime === null) break;
+				assert(this.currentSegment);
+
+				const cuePoint: CuePoint = { time: this.currentCueTime, trackId: -1, clusterPosition: -1 };
 				this.currentSegment.cuePoints.push(cuePoint);
 				this.readContiguousElements(reader, size);
 
-				if (cuePoint.time === -1 || cuePoint.clusterPosition === -1) {
+				if (cuePoint.trackId === -1 || cuePoint.clusterPosition === -1) {
 					this.currentSegment.cuePoints.pop();
 				}
 			}; break;
 
-			case EBMLId.CueTime: {
+			case EBMLId.CueTrack: {
 				const lastCuePoint = this.currentSegment?.cuePoints[this.currentSegment.cuePoints.length - 1];
 				if (!lastCuePoint) break;
 
-				lastCuePoint.time = reader.readUnsignedInt(size);
-			}; break;
-
-			case EBMLId.CueTrackPositions: {
-				const lastCuePoint = this.currentSegment?.cuePoints[this.currentSegment.cuePoints.length - 1];
-				if (!lastCuePoint) break;
-
-				this.readContiguousElements(reader, size);
+				lastCuePoint.trackId = reader.readUnsignedInt(size);
 			}; break;
 
 			case EBMLId.CueClusterPosition: {
@@ -997,6 +1053,7 @@ abstract class MatroskaTrackBacking<
 								x => x.elementStartPos,
 							);
 							assert(clusterIndex !== -1);
+
 							return {
 								clusterIndex,
 								blockIndex: 0,
@@ -1196,7 +1253,7 @@ abstract class MatroskaTrackBacking<
 		// The timestamp for which we know the correct block will not come after it
 		latestTimestamp: number,
 		options: SampleRetrievalOptions,
-	) {
+	): Promise<Sample | null> {
 		const { demuxer, segment } = this.internalTrack;
 		const release = await segment.clusterLookupMutex.acquire(); // The algorithm requires exclusivity
 
@@ -1215,20 +1272,14 @@ abstract class MatroskaTrackBacking<
 			let bestClusterIndex = clusterIndex;
 			let bestBlockIndex = blockIndex;
 
-			let cuePoint: CuePoint | null = null;
-			if (segment.cuePoints.length > 0) {
-				// Search for a cue point; this way, we won't need to start searching from the start of the file
-				// but can jump right into the correct cluster (or at least nearby).
-				const index = binarySearchLessOrEqual(
-					segment.cuePoints,
-					searchTimestamp,
-					x => x.time,
-				);
-
-				if (index !== -1) {
-					cuePoint = segment.cuePoints[index]!;
-				}
-			}
+			// Search for a cue point; this way, we won't need to start searching from the start of the file
+			// but can jump right into the correct cluster (or at least nearby).
+			const cuePointIndex = binarySearchLessOrEqual(
+				this.internalTrack.cuePoints,
+				searchTimestamp,
+				x => x.time,
+			);
+			const cuePoint = cuePointIndex !== -1 ? this.internalTrack.cuePoints[cuePointIndex]! : null;
 
 			if (clusterIndex === -1) {
 				metadataReader.pos = cuePoint?.clusterPosition ?? segment.clusterSeekStartPos;
@@ -1299,13 +1350,23 @@ abstract class MatroskaTrackBacking<
 				metadataReader.pos = dataStartPos + size;
 			}
 
-			if (bestClusterIndex !== -1) {
+			let result: Sample | null = null;
+			const bestCluster = bestClusterIndex !== -1 ? this.internalTrack.clusters[bestClusterIndex]! : null;
+			if (bestCluster) {
 				// If we finished looping but didn't find a perfect match, still return the best match we found
-				const cluster = this.internalTrack.clusters[bestClusterIndex]!;
-				return this.fetchSampleInCluster(cluster, bestBlockIndex, options);
+				result = await this.fetchSampleInCluster(bestCluster, bestBlockIndex, options);
 			}
 
-			return null;
+			// Catch faulty cue points
+			if (!result && cuePoint && (!bestCluster || bestCluster.elementStartPos < cuePoint.clusterPosition)) {
+				// The cue point lied to us! We found a cue point but no cluster there that satisfied the match. In this
+				// case, let's search again but using the cue point before that.
+				const previousCuePoint = this.internalTrack.cuePoints[cuePointIndex - 1];
+				const newSearchTimestamp = previousCuePoint?.time ?? -Infinity;
+				return this.performClusterLookup(getBestMatch, newSearchTimestamp, latestTimestamp, options);
+			}
+
+			return result;
 		} finally {
 			release();
 		}
@@ -1440,6 +1501,7 @@ class MatroskaAudioTrackBacking extends MatroskaTrackBacking<EncodedAudioSample>
  * come before block B. The resulting array is one that is in decode order.
  */
 const sortBlocksTopologically = (blocks: ClusterBlock[]) => {
+	return blocks; // temp
 	// Based on "A fast and effective heuristic for the feedback arc set problem" by Peter Eades et al.
 
 	const n = blocks.length;

src/matroska/matroska-muxer.ts

@@ -122,8 +122,11 @@ export class MatroskaMuxer extends Muxer {
 	private cues: EBMLElement | null = null;
 
 	private currentCluster: EBMLElement | null = null;
-	private currentClusterMsTimestamp: number | null = null;
-	private trackDatasInCurrentCluster = new Set<MatroskaTrackData>();
+	private currentClusterStartMsTimestamp: number | null = null;
+	private currentClusterMaxMsTimestamp: number | null = null;
+	private trackDatasInCurrentCluster = new Map<MatroskaTrackData, {
+		firstMsTimestamp: number;
+	}>();
 
 	private duration = 0;
 
@@ -620,7 +623,7 @@ export class MatroskaMuxer extends Muxer {
 		}
 
 		const msTimestamp = Math.floor(1000 * chunk.timestamp);
-		// We can only finalize this fragment (and begin a new one) if we know that each track will be able to
+		// We can only finalize this cluster (and begin a new one) if we know that each track will be able to
 		// start the new one with a key frame.
 		const keyFrameQueuedEverywhere = this.trackDatas.every((otherTrackData) => {
 			if (otherTrackData.track.source._closed) {
@@ -637,12 +640,19 @@ export class MatroskaMuxer extends Muxer {
 
 		if (
 			!this.currentCluster
-			|| (keyFrameQueuedEverywhere && msTimestamp - this.currentClusterMsTimestamp! >= 1000)
+			|| (
+				keyFrameQueuedEverywhere
+				// This check is required because that means there is already a block with this timestamp in the
+				// CURRENT chunk, meaning that starting the next cluster at the same timestamp is forbidden (since the
+				// already-written block would belong into it instead).
+				&& msTimestamp > this.currentClusterMaxMsTimestamp!
+				&& msTimestamp - this.currentClusterStartMsTimestamp! >= 1000
+			)
 		) {
 			this.createNewCluster(msTimestamp);
 		}
 
-		const relativeTimestamp = msTimestamp - this.currentClusterMsTimestamp!;
+		const relativeTimestamp = msTimestamp - this.currentClusterStartMsTimestamp!;
 		if (relativeTimestamp < 0) {
 			// The chunk lies outside of the current cluster
 			return;
@@ -702,7 +712,12 @@ export class MatroskaMuxer extends Muxer {
 		this.duration = Math.max(this.duration, msTimestamp + msDuration);
 		trackData.lastWrittenMsTimestamp = msTimestamp;
 
-		this.trackDatasInCurrentCluster.add(trackData);
+		if (!this.trackDatasInCurrentCluster.has(trackData)) {
+			this.trackDatasInCurrentCluster.set(trackData, {
+				firstMsTimestamp: msTimestamp,
+			});
+		}
+		this.currentClusterMaxMsTimestamp = Math.max(this.currentClusterMaxMsTimestamp!, msTimestamp);
 	}
 
 	/** Creates a new Cluster element to contain media chunks. */
@@ -726,7 +741,8 @@ export class MatroskaMuxer extends Muxer {
 		};
 		this.ebmlWriter.writeEBML(this.currentCluster);
 
-		this.currentClusterMsTimestamp = msTimestamp;
+		this.currentClusterStartMsTimestamp = msTimestamp;
+		this.currentClusterMaxMsTimestamp = msTimestamp;
 		this.trackDatasInCurrentCluster.clear();
 	}
 
@@ -750,19 +766,31 @@ export class MatroskaMuxer extends Muxer {
 		const clusterOffsetFromSegment
 			= this.ebmlWriter.offsets.get(this.currentCluster)! - this.segmentDataOffset;
 
-		assert(this.cues);
+		// Group tracks by their first timestamp and create a CuePoint for each unique timestamp
+		const groupedByTimestamp = new Map<number, MatroskaTrackData[]>();
+		for (const [trackData, { firstMsTimestamp }] of this.trackDatasInCurrentCluster) {
+			if (!groupedByTimestamp.has(firstMsTimestamp)) {
+				groupedByTimestamp.set(firstMsTimestamp, []);
+			}
+			groupedByTimestamp.get(firstMsTimestamp)!.push(trackData);
+		}
 
-		// Add a CuePoint to the Cues element for better seeking
-		(this.cues.data as EBML[]).push({ id: EBMLId.CuePoint, data: [
-			{ id: EBMLId.CueTime, data: this.currentClusterMsTimestamp! },
-			// We only write out cues for tracks that have at least one chunk in this cluster
-			...[...this.trackDatasInCurrentCluster].map((trackData) => {
-				return { id: EBMLId.CueTrackPositions, data: [
-					{ id: EBMLId.CueTrack, data: trackData.track.id },
-					{ id: EBMLId.CueClusterPosition, data: clusterOffsetFromSegment },
-				] };
-			}),
-		] });
+		const groupedAndSortedByTimestamp = [...groupedByTimestamp.entries()].sort((a, b) => a[0] - b[0]);
+
+		// Add CuePoints to the Cues element for better seeking
+		for (const [msTimestamp, trackDatas] of groupedAndSortedByTimestamp) {
+			assert(this.cues);
+			(this.cues.data as EBML[]).push({ id: EBMLId.CuePoint, data: [
+				{ id: EBMLId.CueTime, data: msTimestamp },
+				// Create CueTrackPositions for each track that starts at this timestamp
+				...trackDatas.map((trackData) => {
+					return { id: EBMLId.CueTrackPositions, data: [
+						{ id: EBMLId.CueTrack, data: trackData.track.id },
+						{ id: EBMLId.CueClusterPosition, data: clusterOffsetFromSegment },
+					] };
+				}),
+			] });
+		}
 	}
 
 	// eslint-disable-next-line @typescript-eslint/no-misused-promises