Migrate fragmented MP4 reading to new packet system, add tests for it, add packet internal data, and various other fixes prob

dev/demux.html

@@ -11,11 +11,26 @@
 		const file = fileInput.files[0];
 		const input = new Mediabunny.Input({
 			formats: Mediabunny.ALL_FORMATS,
-			source: new Mediabunny.BlobSource(file),
+			source: new Mediabunny.BufferSource(await file.arrayBuffer()),
 		});
 
 		const videoTrack = await input.getPrimaryVideoTrack();
-		const cursor = await Mediabunny.VideoSampleCursor2.init(videoTrack);
+
+		const iters = 50;
+		let total = 0;
+
+		for (let i = 0; i < iters; i++) {
+			const start = performance.now();
+			let count = 0;
+			const reader = new Mediabunny.PacketCursor(videoTrack);
+			await reader.iterate(() => count++);
+
+			console.log(count)
+
+			total += performance.now() - start;
+		}
+
+		alert(total / iters);
 
 		/*
 		console.log(await cursor.seekTo(2.00));
@@ -27,6 +42,7 @@
 		//console.log(await cursor.seekTo(5.1))
 		//return;
 
+		/*
 		const mh = [
 			cursor.seekTo(2.05),
 			//cursor.seekTo(4.9),
@@ -61,6 +77,7 @@
 			//cursor.close(),
 			//cursor.seekTo(2.00),
 		];
+		*/
 
 		/*
 		for (const yo of mh) {
@@ -72,7 +89,7 @@
 		console.log("done")
 		*/
 
-		console.log(await Promise.all(mh));
+		//console.log(await Promise.all(mh));
 
 		//console.log(await cursor.seekTo(2.05));
 		//console.log(await cursor.seekTo(2.10));

src/cursors.ts

@@ -70,7 +70,7 @@ export class PacketReader<T extends InputTrack = InputTrack> {
 		const promise = this.track._backing.getFirstPacket(result, options);
 
 		if (result.pending) {
-			return (promise as Promise<Yo>).then(() => this.maybeVerifyPacketType(result.value, options));
+			return promise.then(() => this.maybeVerifyPacketType(result.value, options));
 		} else {
 			return this.maybeVerifyPacketType(result.value, options);
 		}
@@ -88,7 +88,7 @@ export class PacketReader<T extends InputTrack = InputTrack> {
 		const promise = this.track._backing.getPacket(result, timestamp, options);
 
 		if (result.pending) {
-			return (promise as Promise<Yo>).then(() => this.maybeVerifyPacketType(result.value, options));
+			return promise.then(() => this.maybeVerifyPacketType(result.value, options));
 		} else {
 			return this.maybeVerifyPacketType(result.value, options);
 		}
@@ -110,7 +110,7 @@ export class PacketReader<T extends InputTrack = InputTrack> {
 		const promise = this.track._backing.getKeyPacket(result, timestamp, options);
 
 		if (result.pending) {
-			return (promise as Promise<Yo>).then(() => result.value);
+			return promise.then(() => result.value);
 		} else {
 			return result.value;
 		}
@@ -152,7 +152,7 @@ export class PacketReader<T extends InputTrack = InputTrack> {
 		const promise = this.track._backing.getNextPacket(result, from, options);
 
 		if (result.pending) {
-			return (promise as Promise<Yo>).then(() => this.maybeVerifyPacketType(result.value, options));
+			return promise.then(() => this.maybeVerifyPacketType(result.value, options));
 		} else {
 			return this.maybeVerifyPacketType(result.value, options);
 		}
@@ -176,7 +176,7 @@ export class PacketReader<T extends InputTrack = InputTrack> {
 		const promise = this.track._backing.getNextKeyPacket(result, from, options);
 
 		if (result.pending) {
-			return (promise as Promise<Yo>).then(() => result.value);
+			return promise.then(() => result.value);
 		} else {
 			return result.value;
 		}
@@ -205,26 +205,28 @@ export class PacketReader<T extends InputTrack = InputTrack> {
 	}
 }
 
-export class PacketCursor {
-	reader: PacketReader;
+export class PacketCursor<T extends InputTrack = InputTrack> {
+	track: T;
 	current: EncodedPacket | null = null;
 
+	private _reader: PacketReader<T>;
 	private _options: PacketRetrievalOptions;
 	private _nextIsFirst = true;
 	private _callSerializer = new CallSerializer2();
 
-	constructor(reader: PacketReader, options: PacketRetrievalOptions = {}) {
-		if (!(reader instanceof PacketReader)) {
-			throw new TypeError('reader must be a PacketReader.');
+	constructor(track: T, options: PacketRetrievalOptions = {}) {
+		if (!(track instanceof InputTrack)) {
+			throw new TypeError('track must be an InputTrack.');
 		}
 		validatePacketRetrievalOptions(options);
 
-		this.reader = reader;
+		this.track = track;
+		this._reader = new PacketReader(track);
 		this._options = options;
 	}
 
 	private _seekToFirstDirect(): MaybePromise<EncodedPacket | null> {
-		const result = this.reader.readFirst(this._options);
+		const result = this._reader.readFirst(this._options);
 
 		const onPacket = (packet: EncodedPacket | null) => {
 			this._nextIsFirst = false;
@@ -246,7 +248,7 @@ export class PacketCursor {
 		validateTimestamp(timestamp);
 
 		return this._callSerializer.call(() => {
-			const result = this.reader.readAt(timestamp, this._options);
+			const result = this._reader.readAt(timestamp, this._options);
 
 			const onPacket = (packet: EncodedPacket | null) => {
 				this._nextIsFirst = !packet;
@@ -265,7 +267,7 @@ export class PacketCursor {
 		validateTimestamp(timestamp);
 
 		return this._callSerializer.call(() => {
-			const result = this.reader.readKeyAt(timestamp, this._options);
+			const result = this._reader.readKeyAt(timestamp, this._options);
 
 			const onPacket = (packet: EncodedPacket | null) => {
 				this._nextIsFirst = !packet;
@@ -290,7 +292,7 @@ export class PacketCursor {
 				return null;
 			}
 
-			const result = this.reader.readNext(this.current, this._options);
+			const result = this._reader.readNext(this.current, this._options);
 
 			const onPacket = (packet: EncodedPacket | null) => {
 				return this.current = packet;
@@ -314,7 +316,7 @@ export class PacketCursor {
 				return null;
 			}
 
-			const result = this.reader.readNextKey(this.current, this._options);
+			const result = this._reader.readNextKey(this.current, this._options);
 
 			const onPacket = (packet: EncodedPacket | null) => {
 				return this.current = packet;
@@ -494,7 +496,7 @@ export abstract class SampleCursor<
 	) {
 		this.track = track;
 		this._packetReader = new PacketReader(track);
-		this._packetCursor = new PacketCursor(this._packetReader);
+		this._packetCursor = new PacketCursor(track);
 		this._autoClose = options.autoClose ?? true;
 		this._transform = options.transform ?? (sample => sample as unknown as TransformedSample);
 

src/input-track.ts

@@ -144,37 +144,21 @@ export abstract class InputTrack {
 	 * may be positive or even negative. A negative starting timestamp means the track's timing has been offset. Samples
 	 * with a negative timestamp should not be presented.
 	 */
-	getFirstTimestamp() {
+	async getFirstTimestamp() {
 		const result = new ResultValue<EncodedPacket | null>();
-		const promise = this._backing.getFirstPacket(result, { metadataOnly: true });
+		await this._backing.getFirstPacket(result, { metadataOnly: true });
 
-		const getValue = () => {
-			const firstPacket = result.value;
-			return firstPacket?.timestamp ?? 0;
-		};
-
-		if (result.pending) {
-			return (promise).then(getValue);
-		} else {
-			return getValue();
-		}
+		const firstPacket = result.value;
+		return firstPacket?.timestamp ?? 0;
 	}
 
 	/** Returns the end timestamp of the last packet of this track, in seconds. */
-	computeDuration(): MaybePromise<number> {
+	async computeDuration() {
 		const result = new ResultValue<EncodedPacket | null>();
-		const promise = this._backing.getPacket(result, Infinity, { metadataOnly: true });
+		await this._backing.getPacket(result, Infinity, { metadataOnly: true });
 
-		const getValue = () => {
-			const lastPacket = result.value;
-			return (lastPacket?.timestamp ?? 0) + (lastPacket?.duration ?? 0);
-		};
-
-		if (result.pending) {
-			return (promise).then(getValue);
-		} else {
-			return getValue();
-		}
+		const lastPacket = result.value;
+		return (lastPacket?.timestamp ?? 0) + (lastPacket?.duration ?? 0);
 	}
 
 	/**

src/isobmff/isobmff-demuxer.ts

@@ -60,7 +60,6 @@ import {
 	roundIfAlmostInteger,
 	ResultValue,
 	Yo,
-	MaybePromise,
 } from '../misc';
 import { EncodedPacket, PLACEHOLDER_DATA } from '../packet';
 import { buildIsobmffMimeType } from './isobmff-misc';
@@ -263,12 +262,6 @@ export class IsobmffDemuxer extends Demuxer {
 		this.reader = input._reader;
 	}
 
-	override async computeDuration() {
-		const tracks = await this.getTracks();
-		const trackDurations = await Promise.all(tracks.map(x => x.computeDuration()));
-		return Math.max(0, ...trackDurations);
-	}
-
 	override async getTracks() {
 		await this.readMetadata();
 		return this.tracks.map(track => track.inputTrack!);
@@ -514,9 +507,9 @@ export class IsobmffDemuxer extends Demuxer {
 		return sampleTable;
 	}
 
-	async readFragment(startPos: number): Promise<Fragment> {
+	async readFragment(res: ResultValue<Fragment>, startPos: number): Promise<Yo> {
 		if (this.lastReadFragment?.moofOffset === startPos) {
-			return this.lastReadFragment;
+			return res.set(this.lastReadFragment);
 		}
 
 		let headerSlice = this.reader.requestSliceRange(startPos, MIN_BOX_HEADER_SIZE, MAX_BOX_HEADER_SIZE);
@@ -588,7 +581,7 @@ export class IsobmffDemuxer extends Demuxer {
 			}
 		}
 
-		return fragment;
+		return res.set(fragment);
 	}
 
 	readContiguousBoxes(slice: FileSlice) {
@@ -2323,12 +2316,6 @@ export class IsobmffDemuxer extends Demuxer {
 }
 
 abstract class IsobmffTrackBacking implements InputTrackBacking {
-	packetToSampleIndex = new WeakMap<EncodedPacket, number>();
-	packetToFragmentLocation = new WeakMap<EncodedPacket, {
-		fragment: Fragment;
-		sampleIndex: number;
-	}>();
-
 	constructor(public internalTrack: InternalTrack) {}
 
 	getId() {
@@ -2359,14 +2346,6 @@ abstract class IsobmffTrackBacking implements InputTrackBacking {
 		return this.internalTrack.disposition;
 	}
 
-	async computeDuration() {
-		const result = new ResultValue<EncodedPacket | null>();
-		await this.getPacket(result, Infinity, { metadataOnly: true });
-
-		const lastPacket = result.value;
-		return (lastPacket?.timestamp ?? 0) + (lastPacket?.duration ?? 0);
-	}
-
 	async getFirstTimestamp() {
 		const result = new ResultValue<EncodedPacket | null>();
 		await this.getFirstPacket(result, { metadataOnly: true });
@@ -2376,22 +2355,17 @@ abstract class IsobmffTrackBacking implements InputTrackBacking {
 	}
 
 	async getFirstPacket(res: ResultValue<EncodedPacket | null>, options: PacketRetrievalOptions): Promise<Yo> {
-		const promise = this.fetchPacketForSampleIndex(res, 0, options);
-		if (res.pending) await promise;
+		const result = new ResultValue<EncodedPacket | null>();
+		const promise = this.fetchPacketForSampleIndex(result, 0, options);
+		if (result.pending) await promise;
 
-		// const regularPacket = res.value;
-
-		// const regularPacket = await this.fetchPacketForSampleIndex(0, options);
-		if (res.value || !this.internalTrack.demuxer.isFragmented) {
+		if (result.value || !this.internalTrack.demuxer.isFragmented) {
 			// If there's a non-fragmented packet, always prefer that
-			return res.pass();
-			// return regularPacket;
+			return res.set(result.value);
 		}
 
-		throw new Error('bruh');
-
-		/*
 		return this.performFragmentedLookup(
+			res,
 			null,
 			(fragment) => {
 				const trackData = fragment.trackData.get(this.internalTrack.id);
@@ -2411,7 +2385,6 @@ abstract class IsobmffTrackBacking implements InputTrackBacking {
 			Infinity,
 			options,
 		);
-		*/
 	}
 
 	private mapTimestampIntoTimescale(timestamp: number) {
@@ -2429,20 +2402,14 @@ abstract class IsobmffTrackBacking implements InputTrackBacking {
 		const timestampInTimescale = this.mapTimestampIntoTimescale(timestamp);
 
 		const sampleTable = this.internalTrack.demuxer.getSampleTableForTrack(this.internalTrack);
-		const sampleIndex = getSampleIndexForTimestamp(sampleTable, timestampInTimescale);
-
-		const promise = this.fetchPacketForSampleIndex(res, sampleIndex, options);
-		if (res.pending) await promise;
-
 		if (!sampleTableIsEmpty(sampleTable) || !this.internalTrack.demuxer.isFragmented) {
 			// Prefer the non-fragmented packet
-			return res.pass();
+			const sampleIndex = getSampleIndexForTimestamp(sampleTable, timestampInTimescale);
+			return this.fetchPacketForSampleIndex(res, sampleIndex, options);
 		}
 
-		throw new Error('how');
-
-		/*
 		return this.performFragmentedLookup(
+			res,
 			null,
 			(fragment) => {
 				const trackData = fragment.trackData.get(this.internalTrack.id);
@@ -2465,7 +2432,6 @@ abstract class IsobmffTrackBacking implements InputTrackBacking {
 			timestampInTimescale,
 			options,
 		);
-		*/
 	}
 
 	async getNextPacket(
@@ -2473,30 +2439,31 @@ abstract class IsobmffTrackBacking implements InputTrackBacking {
 		packet: EncodedPacket,
 		options: PacketRetrievalOptions,
 	): Promise<Yo> {
-		const regularSampleIndex = packet.sampleIndex;// this.packetToSampleIndex.get(packet);
+		const isFromFragment = packet._internal !== undefined;
+		if (!isFromFragment) {
+			const sampleIndex = packet.sequenceNumber; // sequenceNumber = sampleIndex in this case
+			if (sampleIndex < 0) {
+				throw new Error('Packet was not created from this track.');
+			}
 
-		if (regularSampleIndex !== undefined) {
-			// Prefer the non-fragmented packet
-			return this.fetchPacketForSampleIndex(res, regularSampleIndex + 1, options);
+			return this.fetchPacketForSampleIndex(res, sampleIndex + 1, options);
 		}
 
-		throw new Error('This');
+		const moofOffset = packet._internal;
+		assert(typeof moofOffset === 'number');
 
-		/*
-		const locationInFragment = this.packetToFragmentLocation.get(packet);
-		if (locationInFragment === undefined) {
-			throw new Error('Packet was not created from this track.');
-		}
+		const sampleIndex = packet.sequenceNumber - moofOffset; // since sequenceNumber = moofOffset + sampleIndex
 
 		return this.performFragmentedLookup(
-			locationInFragment.fragment,
+			res,
+			moofOffset,
 			(fragment) => {
-				if (fragment === locationInFragment.fragment) {
+				if (fragment.moofOffset === moofOffset) {
 					const trackData = fragment.trackData.get(this.internalTrack.id)!;
-					if (locationInFragment.sampleIndex + 1 < trackData.samples.length) {
+					if (sampleIndex + 1 < trackData.samples.length) {
 						// We can simply take the next sample in the fragment
 						return {
-							sampleIndex: locationInFragment.sampleIndex + 1,
+							sampleIndex: sampleIndex + 1,
 							correctSampleFound: true,
 						};
 					}
@@ -2519,7 +2486,6 @@ abstract class IsobmffTrackBacking implements InputTrackBacking {
 			Infinity,
 			options,
 		);
-		*/
 	}
 
 	async getKeyPacket(
@@ -2530,20 +2496,14 @@ abstract class IsobmffTrackBacking implements InputTrackBacking {
 		const timestampInTimescale = this.mapTimestampIntoTimescale(timestamp);
 
 		const sampleTable = this.internalTrack.demuxer.getSampleTableForTrack(this.internalTrack);
-		const sampleIndex = getKeyframeSampleIndexForTimestamp(sampleTable, timestampInTimescale);
-		const promise = this.fetchPacketForSampleIndex(res, sampleIndex, options);
-		if (res.pending) await promise;
-
 		if (!sampleTableIsEmpty(sampleTable) || !this.internalTrack.demuxer.isFragmented) {
 			// Prefer the non-fragmented packet
-			return res.pass();
+			const sampleIndex = getKeyframeSampleIndexForTimestamp(sampleTable, timestampInTimescale);
+			return this.fetchPacketForSampleIndex(res, sampleIndex, options);
 		}
 
-		throw new Error('minusch');
-
-		/*
-
 		return this.performFragmentedLookup(
+			res,
 			null,
 			(fragment) => {
 				const trackData = fragment.trackData.get(this.internalTrack.id);
@@ -2565,7 +2525,6 @@ abstract class IsobmffTrackBacking implements InputTrackBacking {
 			timestampInTimescale,
 			options,
 		);
-		*/
 	}
 
 	async getNextKeyPacket(
@@ -2573,29 +2532,31 @@ abstract class IsobmffTrackBacking implements InputTrackBacking {
 		packet: EncodedPacket,
 		options: PacketRetrievalOptions,
 	): Promise<Yo> {
-		const regularSampleIndex = packet.sampleIndex;// this.packetToSampleIndex.get(packet);
-		if (regularSampleIndex !== undefined) {
-			// Prefer the non-fragmented packet
+		const isFromFragment = packet._internal !== undefined;
+		if (!isFromFragment) {
+			const sampleIndex = packet.sequenceNumber; // sequenceNumber = sampleIndex in this case
+			if (sampleIndex < 0) {
+				throw new Error('Packet was not created from this track.');
+			}
+
 			const sampleTable = this.internalTrack.demuxer.getSampleTableForTrack(this.internalTrack);
-			const nextKeyFrameSampleIndex = getNextKeyframeIndexForSample(sampleTable, regularSampleIndex);
+			const nextKeyFrameSampleIndex = getNextKeyframeIndexForSample(sampleTable, sampleIndex);
 			return this.fetchPacketForSampleIndex(res, nextKeyFrameSampleIndex, options);
 		}
 
-		const locationInFragment = this.packetToFragmentLocation.get(packet);
-		if (locationInFragment === undefined) {
-			throw new Error('Packet was not created from this track.');
-		}
+		const moofOffset = packet._internal;
+		assert(typeof moofOffset === 'number');
 
-		throw new Error('Sie sagen Mel');
+		const sampleIndex = packet.sequenceNumber - moofOffset; // since sequenceNumber = moofOffset + sampleIndex
 
-		/*
 		return this.performFragmentedLookup(
-			locationInFragment.fragment,
+			res,
+			moofOffset,
 			(fragment) => {
-				if (fragment === locationInFragment.fragment) {
+				if (fragment.moofOffset === moofOffset) {
 					const trackData = fragment.trackData.get(this.internalTrack.id)!;
 					const nextKeyFrameIndex = trackData.samples.findIndex(
-						(x, i) => x.isKeyFrame && i > locationInFragment.sampleIndex,
+						(x, i) => x.isKeyFrame && i > sampleIndex,
 					);
 
 					if (nextKeyFrameIndex !== -1) {
@@ -2627,7 +2588,6 @@ abstract class IsobmffTrackBacking implements InputTrackBacking {
 			Infinity,
 			options,
 		);
-		*/
 	}
 
 	private async fetchPacketForSampleIndex(
@@ -2671,15 +2631,17 @@ abstract class IsobmffTrackBacking implements InputTrackBacking {
 			sampleInfo.sampleSize,
 		);
 
-		packet.sampleIndex = sampleIndex;
-		// this.packetToSampleIndex.set(packet, sampleIndex);
-
 		return res.set(packet);
 	}
 
-	private async fetchPacketInFragment(fragment: Fragment, sampleIndex: number, options: PacketRetrievalOptions) {
+	private async fetchPacketInFragment(
+		res: ResultValue<EncodedPacket | null>,
+		fragment: Fragment,
+		sampleIndex: number,
+		options: PacketRetrievalOptions,
+	): Promise<Yo> {
 		if (sampleIndex === -1) {
-			return null;
+			return res.set(null);
 		}
 
 		const trackData = fragment.trackData.get(this.internalTrack.id)!;
@@ -2712,15 +2674,16 @@ abstract class IsobmffTrackBacking implements InputTrackBacking {
 			fragmentSample.byteSize,
 		);
 
-		this.packetToFragmentLocation.set(packet, { fragment, sampleIndex });
+		packet._internal = fragment.moofOffset;
 
-		return packet;
+		return res.set(packet);
 	}
 
 	/** Looks for a packet in the fragments while trying to load as few fragments as possible to retrieve it. */
 	private async performFragmentedLookup(
-		// The fragment where we start looking
-		startFragment: Fragment | null,
+		res: ResultValue<EncodedPacket | null>,
+		// The position where we'll start the lookup
+		startOffset: number | null,
 		// This function returns the best-matching sample in a given fragment
 		getMatchInFragment: (fragment: Fragment) => { sampleIndex: number; correctSampleFound: boolean },
 		// The timestamp with which we can search the lookup table
@@ -2728,20 +2691,25 @@ abstract class IsobmffTrackBacking implements InputTrackBacking {
 		// The timestamp for which we know the correct sample will not come after it
 		latestTimestamp: number,
 		options: PacketRetrievalOptions,
-	): Promise<EncodedPacket | null> {
+	): Promise<Yo> {
 		const demuxer = this.internalTrack.demuxer;
 
+		let currentPos = startOffset ?? 0;
 		let currentFragment: Fragment | null = null;
 		let bestFragment: Fragment | null = null;
 		let bestSampleIndex = -1;
 
-		if (startFragment) {
+		if (startOffset !== null && demuxer.lastReadFragment?.moofOffset === startOffset) {
+			const startFragment = demuxer.lastReadFragment;
 			const { sampleIndex, correctSampleFound } = getMatchInFragment(startFragment);
 
 			if (correctSampleFound) {
-				return this.fetchPacketInFragment(startFragment, sampleIndex, options);
+				return this.fetchPacketInFragment(res, startFragment, sampleIndex, options);
 			}
 
+			currentPos = startFragment.moofOffset + startFragment.moofSize; // Start reading from the next fragment
+			currentFragment = startFragment;
+
 			if (sampleIndex !== -1) {
 				bestFragment = startFragment;
 				bestSampleIndex = sampleIndex;
@@ -2773,18 +2741,10 @@ abstract class IsobmffTrackBacking implements InputTrackBacking {
 			positionCacheEntry?.moofOffset ?? 0,
 		) || null;
 
-		let currentPos: number;
-
-		if (!startFragment) {
-			currentPos = lookupEntryPosition ?? 0;
-		} else {
-			if (lookupEntryPosition === null || startFragment.moofOffset >= lookupEntryPosition) {
-				currentPos = startFragment.moofOffset + startFragment.moofSize;
-				currentFragment = startFragment;
-			} else {
-				// Use the lookup entry
-				currentPos = lookupEntryPosition;
-			}
+		if (lookupEntryPosition !== null && lookupEntryPosition > currentPos) {
+			// Use the lookup entry
+			currentPos = lookupEntryPosition;
+			currentFragment = null;
 		}
 
 		while (true) {
@@ -2808,10 +2768,15 @@ abstract class IsobmffTrackBacking implements InputTrackBacking {
 			}
 
 			if (boxInfo.name === 'moof') {
-				currentFragment = await demuxer.readFragment(boxStartPos);
+				const result = new ResultValue<Fragment>();
+				const promise = demuxer.readFragment(result, boxStartPos);
+				if (result.pending) await promise;
+
+				currentFragment = result.value;
+
 				const { sampleIndex, correctSampleFound } = getMatchInFragment(currentFragment);
 				if (correctSampleFound) {
-					return this.fetchPacketInFragment(currentFragment, sampleIndex, options);
+					return this.fetchPacketInFragment(res, currentFragment, sampleIndex, options);
 				}
 				if (sampleIndex !== -1) {
 					bestFragment = currentFragment;
@@ -2831,6 +2796,7 @@ abstract class IsobmffTrackBacking implements InputTrackBacking {
 
 			const newSearchTimestamp = previousLookupEntry?.timestamp ?? -Infinity;
 			return this.performFragmentedLookup(
+				res,
 				null,
 				getMatchInFragment,
 				newSearchTimestamp,
@@ -2841,10 +2807,10 @@ abstract class IsobmffTrackBacking implements InputTrackBacking {
 
 		if (bestFragment) {
 			// If we finished looping but didn't find a perfect match, still return the best match we found
-			return this.fetchPacketInFragment(bestFragment, bestSampleIndex, options);
+			return this.fetchPacketInFragment(res, bestFragment, bestSampleIndex, options);
 		}
 
-		return null;
+		return res.set(null);
 	}
 }
 
@@ -2892,11 +2858,22 @@ class IsobmffVideoTrackBacking extends IsobmffTrackBacking implements InputVideo
 		}
 
 		return this.decoderConfigPromise ??= (async (): Promise<VideoDecoderConfig> => {
+			let firstPacket: EncodedPacket | null = null;
+			const needsPacketForAdditionalInfo
+				= (this.internalTrack.info.codec === 'vp9' && !this.internalTrack.info.vp9CodecInfo)
+					|| (this.internalTrack.info.codec === 'av1' && !this.internalTrack.info.av1CodecInfo);
+
+			if (needsPacketForAdditionalInfo) {
+				const result = new ResultValue<EncodedPacket | null>();
+				const promise = this.getFirstPacket(result, {});
+				if (result.pending) await promise;
+
+				firstPacket = result.value;
+			}
+
 			if (this.internalTrack.info.codec === 'vp9' && !this.internalTrack.info.vp9CodecInfo) {
-				const firstPacket = await this.getFirstPacket({});
 				this.internalTrack.info.vp9CodecInfo = firstPacket && extractVp9CodecInfoFromPacket(firstPacket.data);
 			} else if (this.internalTrack.info.codec === 'av1' && !this.internalTrack.info.av1CodecInfo) {
-				const firstPacket = await this.getFirstPacket({});
 				this.internalTrack.info.av1CodecInfo = firstPacket && extractAv1CodecInfoFromPacket(firstPacket.data);
 			}
 

src/misc.ts

@@ -885,11 +885,6 @@ export class ResultValue<T> {
 		this.value = value;
 		this.pending = false;
 	}
-
-	// @ts-expect-error Just for the types
-	pass(): Yo {
-		assert(!this.pending);
-	}
 }
 
 export class AsyncMutex2 {

src/packet.ts

@@ -54,6 +54,12 @@ export class EncodedPacket {
 	/** Additional data carried with this packet. */
 	readonly sideData: EncodedPacketSideData;
 
+	/**
+	 * Data that demuxers can populate for whatever internal use they have.
+	 * @internal
+	 */
+	_internal: unknown = undefined;
+
 	/** Creates a new {@link EncodedPacket} from raw bytes and timing information. */
 	constructor(
 		/** The encoded data of this packet. */

test/node/demux-mp3.test.ts

@@ -1,95 +0,0 @@
-import { expect, test } from 'vitest';
-import { Input } from '../../src/input.js';
-import { BufferSource, FilePathSource } from '../../src/source.js';
-import path from 'node:path';
-import fs from 'node:fs';
-import { ALL_FORMATS, MP3 } from '../../src/input-format.js';
-import { PacketReader } from '../../src/cursors.js';
-import { InputAudioTrack } from '../../src/input-track.js';
-import { assert } from '../../src/misc.js';
-import { EncodedPacket } from '../../src/packet.js';
-
-const __dirname = new URL('.', import.meta.url).pathname;
-
-test('MP3 demuxing', async () => {
-	using input = new Input({
-		source: new FilePathSource(path.join(__dirname, '../public/AudacityTest1.mp3')),
-		formats: ALL_FORMATS,
-	});
-
-	expect(await input.getFormat()).toBe(MP3);
-	expect(await input.getMimeType()).toBe('audio/mpeg');
-
-	const tracks = await input.getTracks();
-	expect(tracks).toHaveLength(1);
-	const audioTrack = tracks[0] as InputAudioTrack;
-	expect(audioTrack).toBeInstanceOf(InputAudioTrack);
-
-	expect(audioTrack.numberOfChannels).toBe(1);
-	expect(audioTrack.sampleRate).toBe(44100);
-
-	const decoderConfig = await audioTrack.getDecoderConfig();
-	expect(decoderConfig!.codec).toBe('mp3');
-	expect(decoderConfig!.numberOfChannels).toBe(audioTrack.numberOfChannels);
-	expect(decoderConfig!.sampleRate).toBe(audioTrack.sampleRate);
-	expect(decoderConfig!.description).toBeUndefined();
-
-	const reader = new PacketReader(audioTrack);
-	const first = await reader.readFirst();
-	expect(first).not.toBe(null);
-	expect(first!.timestamp).toBe(0);
-	expect(first!.data[0]).toBe(255);
-
-	const next = await reader.readNext(first!);
-	expect(next).not.toBe(null);
-	expect(next!.timestamp).toBeCloseTo(first!.timestamp + first!.duration);
-	expect(next!.sequenceNumber).toBeGreaterThan(first!.sequenceNumber);
-
-	const nextKey = await reader.readNextKey(first!);
-	expect(nextKey).not.toBe(null);
-	expect(nextKey!.timestamp).toBe(next!.timestamp);
-
-	const seeked = await reader.readAt(1);
-	expect(seeked).not.toBe(null);
-	expect(seeked!.timestamp).toBeGreaterThan(0.9);
-	expect(seeked!.timestamp).toBeLessThanOrEqual(1);
-	expect(seeked!.sequenceNumber).toBeGreaterThan(next!.sequenceNumber);
-
-	const seekedKey = await reader.readKeyAt(1);
-	expect(seekedKey).not.toBe(null);
-	expect(seeked!.sequenceNumber).toBe(seeked!.sequenceNumber);
-
-	const last = await reader.readAt(Infinity);
-	expect(last).not.toBe(null);
-	expect(last!.sequenceNumber).toBeGreaterThan(seeked!.sequenceNumber);
-
-	const afterLast = await reader.readNext(last!);
-	expect(afterLast).toBe(null);
-
-	const firstTimestamp = await audioTrack.getFirstTimestamp();
-	expect(firstTimestamp).toBe(0);
-
-	const duration = await audioTrack.computeDuration();
-	expect(duration).toBe(last!.timestamp + last!.duration);
-});
-
-test('Sync reading', async () => {
-	using input = new Input({
-		source: new BufferSource(fs.readFileSync(path.join(__dirname, '../public/AudacityTest1.mp3'))),
-		formats: ALL_FORMATS,
-	});
-
-	const audioTrack = await input.getPrimaryAudioTrack();
-	assert(audioTrack);
-
-	const reader = new PacketReader(audioTrack);
-
-	let current = reader.readFirst() as EncodedPacket | null;
-	let count = 0;
-	while (current) {
-		current = reader.readNext(current) as EncodedPacket | null;
-		count++;
-	}
-
-	expect(count).toBe(475);
-});

test/node/demux.test.ts

@@ -0,0 +1,177 @@
+import { expect, test } from 'vitest';
+import { Input } from '../../src/input.js';
+import { BufferSource, FilePathSource } from '../../src/source.js';
+import path from 'node:path';
+import fs from 'node:fs';
+import { ALL_FORMATS, MP3, MP4, QTFF } from '../../src/input-format.js';
+import { PacketReader } from '../../src/cursors.js';
+import { InputAudioTrack, InputTrack } from '../../src/input-track.js';
+import { assert } from '../../src/misc.js';
+import { EncodedPacket } from '../../src/packet.js';
+
+const __dirname = new URL('.', import.meta.url).pathname;
+
+const testBasicPacketReading = async (track: InputTrack) => {
+	const reader = new PacketReader(track);
+	const first = await reader.readFirst();
+	expect(first).not.toBe(null);
+	expect(first!.timestamp).toBe(0);
+
+	const next = await reader.readNext(first!);
+	expect(next).not.toBe(null);
+	expect(next!.timestamp).toBeCloseTo(first!.timestamp + first!.duration);
+	expect(next!.sequenceNumber).toBeGreaterThan(first!.sequenceNumber);
+
+	const nextKey = await reader.readNextKey(first!);
+	expect(nextKey).not.toBe(null);
+	expect(nextKey!.type).toBe('key');
+	expect(nextKey!.timestamp).toBeGreaterThanOrEqual(next!.timestamp);
+
+	const seeked = await reader.readAt(1);
+	expect(seeked).not.toBe(null);
+	expect(seeked!.timestamp).toBeGreaterThan(0.9);
+	expect(seeked!.timestamp).toBeLessThanOrEqual(1);
+	expect(seeked!.sequenceNumber).toBeGreaterThan(next!.sequenceNumber);
+
+	const seekedKey = await reader.readKeyAt(1);
+	expect(seekedKey).not.toBe(null);
+	expect(seekedKey!.type).toBe('key');
+	expect(seekedKey!.sequenceNumber).toBeGreaterThanOrEqual(first!.sequenceNumber);
+	expect(seekedKey!.sequenceNumber).toBeLessThanOrEqual(seeked!.sequenceNumber);
+
+	const last = await reader.readAt(Infinity);
+	expect(last).not.toBe(null);
+	expect(last!.sequenceNumber).toBeGreaterThan(seeked!.sequenceNumber);
+
+	const afterLast = await reader.readNext(last!);
+	expect(afterLast).toBe(null);
+};
+
+test('Regular ISOBMFF demuxing', async () => {
+	using input = new Input({
+		source: new FilePathSource(path.join(__dirname, '../public/trim-buck-bunny.mov')),
+		formats: ALL_FORMATS,
+	});
+
+	expect(await input.getFormat()).toBe(QTFF);
+	expect(await input.getMimeType()).toBe('video/quicktime; codecs="avc1.4d4029, mp4a.40.2"');
+
+	const tracks = await input.getTracks();
+	expect(tracks).toHaveLength(2);
+
+	const videoTrack = await input.getPrimaryAudioTrack();
+	assert(videoTrack);
+
+	await testBasicPacketReading(videoTrack);
+
+	expect(await input.computeDuration()).toBeCloseTo(5.041666666666667);
+});
+
+test('Fragmented ISOBMFF demuxing', async () => {
+	using input = new Input({
+		source: new FilePathSource(path.join(__dirname, '../public/frag-buck-bunny.mp4')),
+		formats: ALL_FORMATS,
+	});
+
+	expect(await input.getFormat()).toBe(MP4);
+	expect(await input.getMimeType()).toBe('video/mp4; codecs="avc1.640014, mp4a.40.2"');
+
+	const tracks = await input.getTracks();
+	expect(tracks).toHaveLength(2);
+
+	const videoTrack = await input.getPrimaryAudioTrack();
+	assert(videoTrack);
+
+	await testBasicPacketReading(videoTrack);
+
+	expect(await input.computeDuration()).toBeCloseTo(5);
+});
+
+test('Regular ISOBMFF sync reading', async () => {
+	using input = new Input({
+		source: new BufferSource(fs.readFileSync(path.join(__dirname, '../public/trim-buck-bunny.mov'))),
+		formats: ALL_FORMATS,
+	});
+
+	const videoTrack = await input.getPrimaryVideoTrack();
+	assert(videoTrack);
+
+	const count = testSyncPacketReading(videoTrack);
+	expect(count).toBe(121);
+});
+
+test('Fragmented ISOBMFF sync reading', async () => {
+	using input = new Input({
+		source: new BufferSource(fs.readFileSync(path.join(__dirname, '../public/frag-buck-bunny.mp4'))),
+		formats: ALL_FORMATS,
+	});
+
+	const videoTrack = await input.getPrimaryVideoTrack();
+	assert(videoTrack);
+
+	const count = testSyncPacketReading(videoTrack);
+	expect(count).toBe(120);
+});
+
+test('MP3 demuxing', async () => {
+	using input = new Input({
+		source: new FilePathSource(path.join(__dirname, '../public/AudacityTest1.mp3')),
+		formats: ALL_FORMATS,
+	});
+
+	expect(await input.getFormat()).toBe(MP3);
+	expect(await input.getMimeType()).toBe('audio/mpeg');
+
+	const tracks = await input.getTracks();
+	expect(tracks).toHaveLength(1);
+	const audioTrack = tracks[0] as InputAudioTrack;
+	expect(audioTrack).toBeInstanceOf(InputAudioTrack);
+
+	expect(audioTrack.numberOfChannels).toBe(1);
+	expect(audioTrack.sampleRate).toBe(44100);
+
+	const decoderConfig = await audioTrack.getDecoderConfig();
+	expect(decoderConfig!.codec).toBe('mp3');
+	expect(decoderConfig!.numberOfChannels).toBe(audioTrack.numberOfChannels);
+	expect(decoderConfig!.sampleRate).toBe(audioTrack.sampleRate);
+	expect(decoderConfig!.description).toBeUndefined();
+
+	await testBasicPacketReading(audioTrack);
+
+	const firstTimestamp = await audioTrack.getFirstTimestamp();
+	expect(firstTimestamp).toBe(0);
+
+	const duration = await audioTrack.computeDuration();
+	expect(duration).toBeGreaterThan(0);
+});
+
+const testSyncPacketReading = (track: InputTrack) => {
+	const reader = new PacketReader(track);
+
+	let current = reader.readFirst() as EncodedPacket | null;
+	expect(current).toBeInstanceOf(EncodedPacket);
+
+	let count = 0;
+	while (current) {
+		current = reader.readNext(current) as EncodedPacket | null;
+		expect(current instanceof EncodedPacket || current === null).toBe(true);
+		count++;
+	}
+
+	expect(count).toBeGreaterThan(0);
+
+	return count;
+};
+
+test('MP3 sync reading', async () => {
+	using input = new Input({
+		source: new BufferSource(fs.readFileSync(path.join(__dirname, '../public/AudacityTest1.mp3'))),
+		formats: ALL_FORMATS,
+	});
+
+	const audioTrack = await input.getPrimaryAudioTrack();
+	assert(audioTrack);
+
+	const count = testSyncPacketReading(audioTrack);
+	expect(count).toBe(475);
+});

test/node/packet-reading.test.ts

@@ -64,8 +64,7 @@ test('Packet cursor seeking', async () => {
 	});
 
 	const videoTrack = (await input.getPrimaryVideoTrack())!;
-	const reader = new PacketReader(videoTrack);
-	const cursor = new PacketCursor(reader);
+	const cursor = new PacketCursor(videoTrack);
 
 	expect(cursor.current).toBe(null);
 
@@ -102,8 +101,7 @@ test('Packet cursor iteration', async () => {
 	});
 
 	const videoTrack = (await input.getPrimaryVideoTrack())!;
-	const reader = new PacketReader(videoTrack);
-	const cursor = new PacketCursor(reader);
+	const cursor = new PacketCursor(videoTrack);
 
 	const packet0 = (await cursor.seekToFirst())!;
 	expect(cursor.current!.timestamp).toBe(0);
@@ -211,7 +209,7 @@ test('Packet cursor iteration', async () => {
 
 	expect(total).toBe(121);
 
-	const cursor2 = new PacketCursor(reader);
+	const cursor2 = new PacketCursor(videoTrack);
 	const packet9 = (await cursor2.next())!; // Without any prior seeks
 	expect(packet9.sequenceNumber).toBe(packet0.sequenceNumber);
 });
@@ -224,7 +222,7 @@ test('Synchronous packet reading', async () => {
 
 	const videoTrack = (await input.getPrimaryVideoTrack())!;
 	const reader = new PacketReader(videoTrack);
-	const cursor = new PacketCursor(reader);
+	const cursor = new PacketCursor(videoTrack);
 
 	expect(reader.readFirst()).not.toBeInstanceOf(Promise);
 
@@ -249,8 +247,7 @@ test('Command queuing', async () => {
 	});
 
 	const videoTrack = (await input.getPrimaryVideoTrack())!;
-	const reader = new PacketReader(videoTrack);
-	const cursor = new PacketCursor(reader);
+	const cursor = new PacketCursor(videoTrack);
 
 	expect(cursor.waitUntilIdle()).toBe(null);