Fix fragmented MP4 files with a non-empty sample table

docs/index.md

@@ -98,6 +98,7 @@ const sponsors = {
 		{ image: 'https://avatars.githubusercontent.com/u/30229596', name: 'Pablo Bonilla', url: 'https://github.com/devPablo' },
 		{ image: 'https://avatars.githubusercontent.com/u/58149663', name: 'H7GhosT', url: 'https://github.com/H7GhosT' },
 		{ image: 'https://avatars.githubusercontent.com/u/91711202', name: 'ihasq', url: 'https://github.com/ihasq' },
+		{ image: 'https://avatars.githubusercontent.com/u/61233224', name: 'Allwhy', url: 'https://github.com/Allwhy' },
 	],
 };
 </script>

examples/file-compression/file-compression.ts

@@ -91,7 +91,7 @@ const compressFile = async (file: File) => {
 
 		// Display the final media file
 		videoElement.style.display = '';
-		videoElement.src = URL.createObjectURL(new Blob([output.target.buffer!]));
+		videoElement.src = URL.createObjectURL(new Blob([output.target.buffer!], { type: output.format.mimeType }));
 		void videoElement.play();
 
 		compressionFacts.style.display = '';

examples/metadata-extraction/metadata-extraction.ts

@@ -129,6 +129,8 @@ const renderObject = (object: Record<string, unknown>) => {
 				listItem.removeChild(loadingSpan);
 				listItem.appendChild(renderValue(resolvedValue));
 			}).catch((error) => {
+				console.error(error);
+
 				// Show the promise error
 				listItem.removeChild(loadingSpan);
 				const errorSpan = document.createElement('span');

examples/procedural-generation/procedural-generation.ts

@@ -180,7 +180,7 @@ const generateVideo = async () => {
 		videoInfo.style.display = '';
 
 		// Display and play the resulting media file
-		const videoBlob = new Blob([output.target.buffer!], { type: 'video/mp4' });
+		const videoBlob = new Blob([output.target.buffer!], { type: output.format.mimeType });
 		resultVideo.src = URL.createObjectURL(videoBlob);
 		void resultVideo.play();
 

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "mediabunny",
-  "version": "1.4.0",
+  "version": "1.4.4",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "mediabunny",
-      "version": "1.4.0",
+      "version": "1.4.4",
       "license": "MPL-2.0",
       "dependencies": {
         "@types/dom-mediacapture-transform": "^0.1.11",

package.json

@@ -1,7 +1,7 @@
 {
   "name": "mediabunny",
   "author": "Vanilagy",
-  "version": "1.4.0",
+  "version": "1.4.4",
   "description": "Pure TypeScript media toolkit for reading, writing, and converting media files, directly in the browser.",
   "type": "module",
   "main": "./dist/bundles/mediabunny.cjs",

src/isobmff/isobmff-demuxer.ts

@@ -1941,44 +1941,46 @@ abstract class IsobmffTrackBacking implements InputTrackBacking {
 	}
 
 	async getFirstPacket(options: PacketRetrievalOptions) {
-		if (this.internalTrack.demuxer.isFragmented) {
-			return this.performFragmentedLookup(
-				() => {
-					const startFragment = this.internalTrack.demuxer.fragments[0] ?? null;
-					if (startFragment?.isKnownToBeFirstFragment) {
-						// Walk from the very first fragment in the file until we find one with our track in it
-						let currentFragment: Fragment | null = startFragment;
-						while (currentFragment) {
-							const trackData = currentFragment.trackData.get(this.internalTrack.id);
-							if (trackData) {
-								return {
-									fragmentIndex: binarySearchExact(
-										this.internalTrack.fragments,
-										currentFragment.moofOffset,
-										x => x.moofOffset,
-									),
-									sampleIndex: 0,
-									correctSampleFound: true,
-								};
-							}
-
-							currentFragment = currentFragment.nextFragment;
-						}
-					}
-
-					return {
-						fragmentIndex: -1,
-						sampleIndex: -1,
-						correctSampleFound: false,
-					};
-				},
-				-Infinity, // Use -Infinity as a search timestamp to avoid using the lookup entries
-				Infinity,
-				options,
-			);
+		const regularPacket = await this.fetchPacketForSampleIndex(0, options);
+		if (regularPacket || !this.internalTrack.demuxer.isFragmented) {
+			// If there's a non-fragmented packet, always prefer that
+			return regularPacket;
 		}
 
-		return this.fetchPacketForSampleIndex(0, options);
+		return this.performFragmentedLookup(
+			() => {
+				const startFragment = this.internalTrack.demuxer.fragments[0] ?? null;
+				if (startFragment?.isKnownToBeFirstFragment) {
+					// Walk from the very first fragment in the file until we find one with our track in it
+					let currentFragment: Fragment | null = startFragment;
+					while (currentFragment) {
+						const trackData = currentFragment.trackData.get(this.internalTrack.id);
+						if (trackData) {
+							return {
+								fragmentIndex: binarySearchExact(
+									this.internalTrack.fragments,
+									currentFragment.moofOffset,
+									x => x.moofOffset,
+								),
+								sampleIndex: 0,
+								correctSampleFound: true,
+							};
+						}
+
+						currentFragment = currentFragment.nextFragment;
+					}
+				}
+
+				return {
+					fragmentIndex: -1,
+					sampleIndex: -1,
+					correctSampleFound: false,
+				};
+			},
+			-Infinity, // Use -Infinity as a search timestamp to avoid using the lookup entries
+			Infinity,
+			options,
+		);
 	}
 
 	private mapTimestampIntoTimescale(timestamp: number) {
@@ -1991,185 +1993,188 @@ abstract class IsobmffTrackBacking implements InputTrackBacking {
 	async getPacket(timestamp: number, options: PacketRetrievalOptions) {
 		const timestampInTimescale = this.mapTimestampIntoTimescale(timestamp);
 
-		if (this.internalTrack.demuxer.isFragmented) {
-			return this.performFragmentedLookup(
-				() => this.findSampleInFragmentsForTimestamp(timestampInTimescale),
-				timestampInTimescale,
-				timestampInTimescale,
-				options,
-			);
-		} else {
-			const sampleTable = this.internalTrack.demuxer.getSampleTableForTrack(this.internalTrack);
-			const sampleIndex = getSampleIndexForTimestamp(sampleTable, timestampInTimescale);
-			return this.fetchPacketForSampleIndex(sampleIndex, options);
+		const sampleTable = this.internalTrack.demuxer.getSampleTableForTrack(this.internalTrack);
+		const sampleIndex = getSampleIndexForTimestamp(sampleTable, timestampInTimescale);
+		const regularPacket = await this.fetchPacketForSampleIndex(sampleIndex, options);
+
+		if (!sampleTableIsEmpty(sampleTable) || !this.internalTrack.demuxer.isFragmented) {
+			// Prefer the non-fragmented packet
+			return regularPacket;
 		}
+
+		return this.performFragmentedLookup(
+			() => this.findSampleInFragmentsForTimestamp(timestampInTimescale),
+			timestampInTimescale,
+			timestampInTimescale,
+			options,
+		);
 	}
 
 	async getNextPacket(packet: EncodedPacket, options: PacketRetrievalOptions) {
-		if (this.internalTrack.demuxer.isFragmented) {
-			const locationInFragment = this.packetToFragmentLocation.get(packet);
-			if (locationInFragment === undefined) {
-				throw new Error('Packet was not created from this track.');
-			}
+		const regularSampleIndex = this.packetToSampleIndex.get(packet);
 
-			const trackData = locationInFragment.fragment.trackData.get(this.internalTrack.id)!;
-			const fragmentSample = trackData.samples[locationInFragment.sampleIndex]!;
-
-			const fragmentIndex = binarySearchExact(
-				this.internalTrack.fragments,
-				locationInFragment.fragment.moofOffset,
-				x => x.moofOffset,
-			);
-			assert(fragmentIndex !== -1);
-
-			return this.performFragmentedLookup(
-				() => {
-					if (locationInFragment.sampleIndex + 1 < trackData.samples.length) {
-						// We can simply take the next sample in the fragment
-						return {
-							fragmentIndex,
-							sampleIndex: locationInFragment.sampleIndex + 1,
-							correctSampleFound: true,
-						};
-					} else {
-						// Walk the list of fragments until we find the next fragment for this track
-						let currentFragment = locationInFragment.fragment;
-						while (currentFragment.nextFragment) {
-							currentFragment = currentFragment.nextFragment;
-
-							const trackData = currentFragment.trackData.get(this.internalTrack.id);
-							if (trackData) {
-								const fragmentIndex = binarySearchExact(
-									this.internalTrack.fragments,
-									currentFragment.moofOffset,
-									x => x.moofOffset,
-								);
-								assert(fragmentIndex !== -1);
-
-								return {
-									fragmentIndex,
-									sampleIndex: 0,
-									correctSampleFound: true,
-								};
-							}
-						}
-
-						return {
-							fragmentIndex,
-							sampleIndex: -1,
-							correctSampleFound: false,
-						};
-					}
-				},
-				fragmentSample.presentationTimestamp,
-				Infinity,
-				options,
-			);
+		if (regularSampleIndex !== undefined) {
+			// Prefer the non-fragmented packet
+			return this.fetchPacketForSampleIndex(regularSampleIndex + 1, options);
 		}
 
-		const sampleIndex = this.packetToSampleIndex.get(packet);
-		if (sampleIndex === undefined) {
+		const locationInFragment = this.packetToFragmentLocation.get(packet);
+		if (locationInFragment === undefined) {
 			throw new Error('Packet was not created from this track.');
 		}
-		return this.fetchPacketForSampleIndex(sampleIndex + 1, options);
+
+		const trackData = locationInFragment.fragment.trackData.get(this.internalTrack.id)!;
+		const fragmentSample = trackData.samples[locationInFragment.sampleIndex]!;
+
+		const fragmentIndex = binarySearchExact(
+			this.internalTrack.fragments,
+			locationInFragment.fragment.moofOffset,
+			x => x.moofOffset,
+		);
+		assert(fragmentIndex !== -1);
+
+		return this.performFragmentedLookup(
+			() => {
+				if (locationInFragment.sampleIndex + 1 < trackData.samples.length) {
+					// We can simply take the next sample in the fragment
+					return {
+						fragmentIndex,
+						sampleIndex: locationInFragment.sampleIndex + 1,
+						correctSampleFound: true,
+					};
+				} else {
+					// Walk the list of fragments until we find the next fragment for this track
+					let currentFragment = locationInFragment.fragment;
+					while (currentFragment.nextFragment) {
+						currentFragment = currentFragment.nextFragment;
+
+						const trackData = currentFragment.trackData.get(this.internalTrack.id);
+						if (trackData) {
+							const fragmentIndex = binarySearchExact(
+								this.internalTrack.fragments,
+								currentFragment.moofOffset,
+								x => x.moofOffset,
+							);
+							assert(fragmentIndex !== -1);
+
+							return {
+								fragmentIndex,
+								sampleIndex: 0,
+								correctSampleFound: true,
+							};
+						}
+					}
+
+					return {
+						fragmentIndex,
+						sampleIndex: -1,
+						correctSampleFound: false,
+					};
+				}
+			},
+			fragmentSample.presentationTimestamp,
+			Infinity,
+			options,
+		);
 	}
 
 	async getKeyPacket(timestamp: number, options: PacketRetrievalOptions) {
 		const timestampInTimescale = this.mapTimestampIntoTimescale(timestamp);
 
-		if (this.internalTrack.demuxer.isFragmented) {
-			return this.performFragmentedLookup(
-				() => this.findKeySampleInFragmentsForTimestamp(timestampInTimescale),
-				timestampInTimescale,
-				timestampInTimescale,
-				options,
-			);
-		}
-
 		const sampleTable = this.internalTrack.demuxer.getSampleTableForTrack(this.internalTrack);
 		const sampleIndex = getSampleIndexForTimestamp(sampleTable, timestampInTimescale);
 		const keyFrameSampleIndex = sampleIndex === -1
 			? -1
 			: getRelevantKeyframeIndexForSample(sampleTable, sampleIndex);
-		return this.fetchPacketForSampleIndex(keyFrameSampleIndex, options);
+		const regularPacket = await this.fetchPacketForSampleIndex(keyFrameSampleIndex, options);
+
+		if (!sampleTableIsEmpty(sampleTable) || !this.internalTrack.demuxer.isFragmented) {
+			// Prefer the non-fragmented packet
+			return regularPacket;
+		}
+
+		return this.performFragmentedLookup(
+			() => this.findKeySampleInFragmentsForTimestamp(timestampInTimescale),
+			timestampInTimescale,
+			timestampInTimescale,
+			options,
+		);
 	}
 
 	async getNextKeyPacket(packet: EncodedPacket, options: PacketRetrievalOptions) {
-		if (this.internalTrack.demuxer.isFragmented) {
-			const locationInFragment = this.packetToFragmentLocation.get(packet);
-			if (locationInFragment === undefined) {
-				throw new Error('Packet was not created from this track.');
-			}
-
-			const trackData = locationInFragment.fragment.trackData.get(this.internalTrack.id)!;
-			const fragmentSample = trackData.samples[locationInFragment.sampleIndex]!;
-
-			const fragmentIndex = binarySearchExact(
-				this.internalTrack.fragments,
-				locationInFragment.fragment.moofOffset,
-				x => x.moofOffset,
-			);
-			assert(fragmentIndex !== -1);
-
-			return this.performFragmentedLookup(
-				() => {
-					const nextKeyFrameIndex = trackData.samples.findIndex(
-						(x, i) => x.isKeyFrame && i > locationInFragment.sampleIndex,
-					);
-
-					if (nextKeyFrameIndex !== -1) {
-						// We can simply take the next key frame in the fragment
-						return {
-							fragmentIndex,
-							sampleIndex: nextKeyFrameIndex,
-							correctSampleFound: true,
-						};
-					} else {
-						// Walk the list of fragments until we find the next fragment for this track with a key frame
-						let currentFragment = locationInFragment.fragment;
-						while (currentFragment.nextFragment) {
-							currentFragment = currentFragment.nextFragment;
-
-							const trackData = currentFragment.trackData.get(this.internalTrack.id);
-							if (trackData && trackData.firstKeyFrameTimestamp !== null) {
-								const fragmentIndex = binarySearchExact(
-									this.internalTrack.fragments,
-									currentFragment.moofOffset,
-									x => x.moofOffset,
-								);
-								assert(fragmentIndex !== -1);
-
-								const keyFrameIndex = trackData.samples.findIndex(x => x.isKeyFrame);
-								assert(keyFrameIndex !== -1); // There must be one
-
-								return {
-									fragmentIndex,
-									sampleIndex: keyFrameIndex,
-									correctSampleFound: true,
-								};
-							}
-						}
-
-						return {
-							fragmentIndex,
-							sampleIndex: -1,
-							correctSampleFound: false,
-						};
-					}
-				},
-				fragmentSample.presentationTimestamp,
-				Infinity,
-				options,
-			);
+		const regularSampleIndex = this.packetToSampleIndex.get(packet);
+		if (regularSampleIndex !== undefined) {
+			// Prefer the non-fragmented packet
+			const sampleTable = this.internalTrack.demuxer.getSampleTableForTrack(this.internalTrack);
+			const nextKeyFrameSampleIndex = getNextKeyframeIndexForSample(sampleTable, regularSampleIndex);
+			return this.fetchPacketForSampleIndex(nextKeyFrameSampleIndex, options);
 		}
 
-		const sampleIndex = this.packetToSampleIndex.get(packet);
-		if (sampleIndex === undefined) {
+		const locationInFragment = this.packetToFragmentLocation.get(packet);
+		if (locationInFragment === undefined) {
 			throw new Error('Packet was not created from this track.');
 		}
-		const sampleTable = this.internalTrack.demuxer.getSampleTableForTrack(this.internalTrack);
-		const nextKeyFrameSampleIndex = getNextKeyframeIndexForSample(sampleTable, sampleIndex);
-		return this.fetchPacketForSampleIndex(nextKeyFrameSampleIndex, options);
+
+		const trackData = locationInFragment.fragment.trackData.get(this.internalTrack.id)!;
+		const fragmentSample = trackData.samples[locationInFragment.sampleIndex]!;
+
+		const fragmentIndex = binarySearchExact(
+			this.internalTrack.fragments,
+			locationInFragment.fragment.moofOffset,
+			x => x.moofOffset,
+		);
+		assert(fragmentIndex !== -1);
+
+		return this.performFragmentedLookup(
+			() => {
+				const nextKeyFrameIndex = trackData.samples.findIndex(
+					(x, i) => x.isKeyFrame && i > locationInFragment.sampleIndex,
+				);
+
+				if (nextKeyFrameIndex !== -1) {
+					// We can simply take the next key frame in the fragment
+					return {
+						fragmentIndex,
+						sampleIndex: nextKeyFrameIndex,
+						correctSampleFound: true,
+					};
+				} else {
+					// Walk the list of fragments until we find the next fragment for this track with a key frame
+					let currentFragment = locationInFragment.fragment;
+					while (currentFragment.nextFragment) {
+						currentFragment = currentFragment.nextFragment;
+
+						const trackData = currentFragment.trackData.get(this.internalTrack.id);
+						if (trackData && trackData.firstKeyFrameTimestamp !== null) {
+							const fragmentIndex = binarySearchExact(
+								this.internalTrack.fragments,
+								currentFragment.moofOffset,
+								x => x.moofOffset,
+							);
+							assert(fragmentIndex !== -1);
+
+							const keyFrameIndex = trackData.samples.findIndex(x => x.isKeyFrame);
+							assert(keyFrameIndex !== -1); // There must be one
+
+							return {
+								fragmentIndex,
+								sampleIndex: keyFrameIndex,
+								correctSampleFound: true,
+							};
+						}
+					}
+
+					return {
+						fragmentIndex,
+						sampleIndex: -1,
+						correctSampleFound: false,
+					};
+				}
+			},
+			fragmentSample.presentationTimestamp,
+			Infinity,
+			options,
+		);
 	}
 
 	private async fetchPacketForSampleIndex(sampleIndex: number, options: PacketRetrievalOptions) {
@@ -2701,3 +2706,7 @@ const extractRotationFromMatrix = (matrix: TransformationMatrix) => {
 	// Invert the rotation because matrices are post-multiplied in ISOBMFF
 	return -Math.atan2(sinTheta, cosTheta) * (180 / Math.PI);
 };
+
+const sampleTableIsEmpty = (sampleTable: SampleTable) => {
+	return sampleTable.sampleSizes.length === 0;
+};

src/matroska/matroska-demuxer.ts

@@ -220,13 +220,16 @@ export class MatroskaDemuxer extends Demuxer {
 
 			const fileSize = await this.input.source.getSize();
 
+			// Loop over all top-level elements in the file
 			while (this.metadataReader.pos <= fileSize - MIN_HEADER_SIZE) {
 				await this.metadataReader.reader.loadRange(
 					this.metadataReader.pos,
 					this.metadataReader.pos + MAX_HEADER_SIZE,
 				);
 
-				const { id, size } = this.metadataReader.readElementHeader();
+				const header = this.metadataReader.readElementHeader();
+				const id = header.id;
+				let size = header.size;
 				const startPos = this.metadataReader.pos;
 
 				if (id === EBMLId.EBML) {
@@ -242,6 +245,26 @@ export class MatroskaDemuxer extends Demuxer {
 						// and only segment
 						break;
 					}
+				} else if (id === EBMLId.Cluster) {
+					// Clusters are not a top-level element in Matroska, but some files contain a Segment whose size
+					// doesn't contain any of the clusters that follow it. In the case, we apply the following logic: if
+					// we find a top-level cluster, attribute it to the previous segment.
+
+					if (size === null) {
+						// Just in case this is one of those weird sizeless clusters, let's do our best and still try to
+						// determine its size.
+						const nextElementPos = await this.clusterReader.searchForNextElementId(
+							LEVEL_0_AND_1_EBML_IDS,
+							fileSize,
+						);
+						size = (nextElementPos ?? fileSize) - startPos;
+					}
+
+					const lastSegment = last(this.segments);
+					if (lastSegment) {
+						// Extend the previous segment's size
+						lastSegment.elementEndPos = startPos + size;
+					}
 				}
 
 				assertDefinedSize(size);
@@ -368,6 +391,13 @@ export class MatroskaDemuxer extends Demuxer {
 			this.readContiguousElements(this.metadataReader, size);
 		}
 
+		if (this.currentSegment.timestampScale === -1) {
+			// TimestampScale element is missing. Technically an invalid file, but let's default to the typical value,
+			// which is 1e6.
+			this.currentSegment.timestampScale = 1e6;
+			this.currentSegment.timestampFactor = 1e9 / 1e6;
+		}
+
 		// Put default tracks first
 		this.currentSegment.tracks.sort((a, b) => Number(b.isDefault) - Number(a.isDefault));
 

src/sample.ts

@@ -236,6 +236,7 @@ export class VideoSample {
 			return new VideoSample(this._data.clone(), {
 				timestamp: this.timestamp,
 				duration: this.duration,
+				rotation: this.rotation,
 			});
 		} else if (this._data instanceof Uint8Array) {
 			return new VideoSample(this._data.slice(), {
@@ -245,6 +246,7 @@ export class VideoSample {
 				timestamp: this.timestamp,
 				duration: this.duration,
 				colorSpace: this.colorSpace,
+				rotation: this.rotation,
 			});
 		} else {
 			return new VideoSample(this._data, {
@@ -254,6 +256,7 @@ export class VideoSample {
 				timestamp: this.timestamp,
 				duration: this.duration,
 				colorSpace: this.colorSpace,
+				rotation: this.rotation,
 			});
 		}
 	}