Fix fragmented MP4 files with a non-empty sample table

docs/index.md

@@ -98,6 +98,7 @@ const sponsors = {
 		{ image: 'https://avatars.githubusercontent.com/u/30229596', name: 'Pablo Bonilla', url: 'https://github.com/devPablo' },
 		{ image: 'https://avatars.githubusercontent.com/u/58149663', name: 'H7GhosT', url: 'https://github.com/H7GhosT' },
 		{ image: 'https://avatars.githubusercontent.com/u/91711202', name: 'ihasq', url: 'https://github.com/ihasq' },
+		{ image: 'https://avatars.githubusercontent.com/u/61233224', name: 'Allwhy', url: 'https://github.com/Allwhy' },
 	],
 };
 </script>

examples/file-compression/file-compression.ts

@@ -91,7 +91,7 @@ const compressFile = async (file: File) => {
 
 		// Display the final media file
 		videoElement.style.display = '';
-		videoElement.src = URL.createObjectURL(new Blob([output.target.buffer!]));
+		videoElement.src = URL.createObjectURL(new Blob([output.target.buffer!], { type: output.format.mimeType }));
 		void videoElement.play();
 
 		compressionFacts.style.display = '';

examples/metadata-extraction/metadata-extraction.ts

@@ -129,6 +129,8 @@ const renderObject = (object: Record<string, unknown>) => {
 				listItem.removeChild(loadingSpan);
 				listItem.appendChild(renderValue(resolvedValue));
 			}).catch((error) => {
+				console.error(error);
+
 				// Show the promise error
 				listItem.removeChild(loadingSpan);
 				const errorSpan = document.createElement('span');

examples/procedural-generation/procedural-generation.ts

@@ -180,7 +180,7 @@ const generateVideo = async () => {
 		videoInfo.style.display = '';
 
 		// Display and play the resulting media file
-		const videoBlob = new Blob([output.target.buffer!], { type: 'video/mp4' });
+		const videoBlob = new Blob([output.target.buffer!], { type: output.format.mimeType });
 		resultVideo.src = URL.createObjectURL(videoBlob);
 		void resultVideo.play();
 

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "mediabunny",
-  "version": "1.4.0",
+  "version": "1.4.4",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "mediabunny",
-      "version": "1.4.0",
+      "version": "1.4.4",
       "license": "MPL-2.0",
       "dependencies": {
         "@types/dom-mediacapture-transform": "^0.1.11",

package.json

@@ -1,7 +1,7 @@
 {
   "name": "mediabunny",
   "author": "Vanilagy",
-  "version": "1.4.0",
+  "version": "1.4.4",
   "description": "Pure TypeScript media toolkit for reading, writing, and converting media files, directly in the browser.",
   "type": "module",
   "main": "./dist/bundles/mediabunny.cjs",

src/isobmff/isobmff-demuxer.ts

@@ -1941,7 +1941,12 @@ abstract class IsobmffTrackBacking implements InputTrackBacking {
 	}
 
 	async getFirstPacket(options: PacketRetrievalOptions) {
-		if (this.internalTrack.demuxer.isFragmented) {
+		const regularPacket = await this.fetchPacketForSampleIndex(0, options);
+		if (regularPacket || !this.internalTrack.demuxer.isFragmented) {
+			// If there's a non-fragmented packet, always prefer that
+			return regularPacket;
+		}
+
 		return this.performFragmentedLookup(
 			() => {
 				const startFragment = this.internalTrack.demuxer.fragments[0] ?? null;
@@ -1978,9 +1983,6 @@ abstract class IsobmffTrackBacking implements InputTrackBacking {
 		);
 	}
 
-		return this.fetchPacketForSampleIndex(0, options);
-	}
-
 	private mapTimestampIntoTimescale(timestamp: number) {
 		// Do a little rounding to catch cases where the result is very close to an integer. If it is, it's likely
 		// that the number was originally an integer divided by the timescale. For stability, it's best
@@ -1991,22 +1993,31 @@ abstract class IsobmffTrackBacking implements InputTrackBacking {
 	async getPacket(timestamp: number, options: PacketRetrievalOptions) {
 		const timestampInTimescale = this.mapTimestampIntoTimescale(timestamp);
 
-		if (this.internalTrack.demuxer.isFragmented) {
+		const sampleTable = this.internalTrack.demuxer.getSampleTableForTrack(this.internalTrack);
+		const sampleIndex = getSampleIndexForTimestamp(sampleTable, timestampInTimescale);
+		const regularPacket = await this.fetchPacketForSampleIndex(sampleIndex, options);
+
+		if (!sampleTableIsEmpty(sampleTable) || !this.internalTrack.demuxer.isFragmented) {
+			// Prefer the non-fragmented packet
+			return regularPacket;
+		}
+
 		return this.performFragmentedLookup(
 			() => this.findSampleInFragmentsForTimestamp(timestampInTimescale),
 			timestampInTimescale,
 			timestampInTimescale,
 			options,
 		);
-		} else {
-			const sampleTable = this.internalTrack.demuxer.getSampleTableForTrack(this.internalTrack);
-			const sampleIndex = getSampleIndexForTimestamp(sampleTable, timestampInTimescale);
-			return this.fetchPacketForSampleIndex(sampleIndex, options);
-		}
 	}
 
 	async getNextPacket(packet: EncodedPacket, options: PacketRetrievalOptions) {
-		if (this.internalTrack.demuxer.isFragmented) {
+		const regularSampleIndex = this.packetToSampleIndex.get(packet);
+
+		if (regularSampleIndex !== undefined) {
+			// Prefer the non-fragmented packet
+			return this.fetchPacketForSampleIndex(regularSampleIndex + 1, options);
+		}
+
 		const locationInFragment = this.packetToFragmentLocation.get(packet);
 		if (locationInFragment === undefined) {
 			throw new Error('Packet was not created from this track.');
@@ -2067,17 +2078,21 @@ abstract class IsobmffTrackBacking implements InputTrackBacking {
 		);
 	}
 
-		const sampleIndex = this.packetToSampleIndex.get(packet);
-		if (sampleIndex === undefined) {
-			throw new Error('Packet was not created from this track.');
-		}
-		return this.fetchPacketForSampleIndex(sampleIndex + 1, options);
-	}
-
 	async getKeyPacket(timestamp: number, options: PacketRetrievalOptions) {
 		const timestampInTimescale = this.mapTimestampIntoTimescale(timestamp);
 
-		if (this.internalTrack.demuxer.isFragmented) {
+		const sampleTable = this.internalTrack.demuxer.getSampleTableForTrack(this.internalTrack);
+		const sampleIndex = getSampleIndexForTimestamp(sampleTable, timestampInTimescale);
+		const keyFrameSampleIndex = sampleIndex === -1
+			? -1
+			: getRelevantKeyframeIndexForSample(sampleTable, sampleIndex);
+		const regularPacket = await this.fetchPacketForSampleIndex(keyFrameSampleIndex, options);
+
+		if (!sampleTableIsEmpty(sampleTable) || !this.internalTrack.demuxer.isFragmented) {
+			// Prefer the non-fragmented packet
+			return regularPacket;
+		}
+
 		return this.performFragmentedLookup(
 			() => this.findKeySampleInFragmentsForTimestamp(timestampInTimescale),
 			timestampInTimescale,
@@ -2086,16 +2101,15 @@ abstract class IsobmffTrackBacking implements InputTrackBacking {
 		);
 	}
 
+	async getNextKeyPacket(packet: EncodedPacket, options: PacketRetrievalOptions) {
+		const regularSampleIndex = this.packetToSampleIndex.get(packet);
+		if (regularSampleIndex !== undefined) {
+			// Prefer the non-fragmented packet
 			const sampleTable = this.internalTrack.demuxer.getSampleTableForTrack(this.internalTrack);
-		const sampleIndex = getSampleIndexForTimestamp(sampleTable, timestampInTimescale);
+			const nextKeyFrameSampleIndex = getNextKeyframeIndexForSample(sampleTable, regularSampleIndex);
-		const keyFrameSampleIndex = sampleIndex === -1
+			return this.fetchPacketForSampleIndex(nextKeyFrameSampleIndex, options);
-			? -1
-			: getRelevantKeyframeIndexForSample(sampleTable, sampleIndex);
-		return this.fetchPacketForSampleIndex(keyFrameSampleIndex, options);
 		}
 
-	async getNextKeyPacket(packet: EncodedPacket, options: PacketRetrievalOptions) {
-		if (this.internalTrack.demuxer.isFragmented) {
 		const locationInFragment = this.packetToFragmentLocation.get(packet);
 		if (locationInFragment === undefined) {
 			throw new Error('Packet was not created from this track.');
@@ -2163,15 +2177,6 @@ abstract class IsobmffTrackBacking implements InputTrackBacking {
 		);
 	}
 
-		const sampleIndex = this.packetToSampleIndex.get(packet);
-		if (sampleIndex === undefined) {
-			throw new Error('Packet was not created from this track.');
-		}
-		const sampleTable = this.internalTrack.demuxer.getSampleTableForTrack(this.internalTrack);
-		const nextKeyFrameSampleIndex = getNextKeyframeIndexForSample(sampleTable, sampleIndex);
-		return this.fetchPacketForSampleIndex(nextKeyFrameSampleIndex, options);
-	}
-
 	private async fetchPacketForSampleIndex(sampleIndex: number, options: PacketRetrievalOptions) {
 		if (sampleIndex === -1) {
 			return null;
@@ -2701,3 +2706,7 @@ const extractRotationFromMatrix = (matrix: TransformationMatrix) => {
 	// Invert the rotation because matrices are post-multiplied in ISOBMFF
 	return -Math.atan2(sinTheta, cosTheta) * (180 / Math.PI);
 };
+
+const sampleTableIsEmpty = (sampleTable: SampleTable) => {
+	return sampleTable.sampleSizes.length === 0;
+};

src/matroska/matroska-demuxer.ts

@@ -220,13 +220,16 @@ export class MatroskaDemuxer extends Demuxer {
 
 			const fileSize = await this.input.source.getSize();
 
+			// Loop over all top-level elements in the file
 			while (this.metadataReader.pos <= fileSize - MIN_HEADER_SIZE) {
 				await this.metadataReader.reader.loadRange(
 					this.metadataReader.pos,
 					this.metadataReader.pos + MAX_HEADER_SIZE,
 				);
 
-				const { id, size } = this.metadataReader.readElementHeader();
+				const header = this.metadataReader.readElementHeader();
+				const id = header.id;
+				let size = header.size;
 				const startPos = this.metadataReader.pos;
 
 				if (id === EBMLId.EBML) {
@@ -242,6 +245,26 @@ export class MatroskaDemuxer extends Demuxer {
 						// and only segment
 						break;
 					}
+				} else if (id === EBMLId.Cluster) {
+					// Clusters are not a top-level element in Matroska, but some files contain a Segment whose size
+					// doesn't contain any of the clusters that follow it. In the case, we apply the following logic: if
+					// we find a top-level cluster, attribute it to the previous segment.
+
+					if (size === null) {
+						// Just in case this is one of those weird sizeless clusters, let's do our best and still try to
+						// determine its size.
+						const nextElementPos = await this.clusterReader.searchForNextElementId(
+							LEVEL_0_AND_1_EBML_IDS,
+							fileSize,
+						);
+						size = (nextElementPos ?? fileSize) - startPos;
+					}
+
+					const lastSegment = last(this.segments);
+					if (lastSegment) {
+						// Extend the previous segment's size
+						lastSegment.elementEndPos = startPos + size;
+					}
 				}
 
 				assertDefinedSize(size);
@@ -368,6 +391,13 @@ export class MatroskaDemuxer extends Demuxer {
 			this.readContiguousElements(this.metadataReader, size);
 		}
 
+		if (this.currentSegment.timestampScale === -1) {
+			// TimestampScale element is missing. Technically an invalid file, but let's default to the typical value,
+			// which is 1e6.
+			this.currentSegment.timestampScale = 1e6;
+			this.currentSegment.timestampFactor = 1e9 / 1e6;
+		}
+
 		// Put default tracks first
 		this.currentSegment.tracks.sort((a, b) => Number(b.isDefault) - Number(a.isDefault));
 

src/sample.ts

@@ -236,6 +236,7 @@ export class VideoSample {
 			return new VideoSample(this._data.clone(), {
 				timestamp: this.timestamp,
 				duration: this.duration,
+				rotation: this.rotation,
 			});
 		} else if (this._data instanceof Uint8Array) {
 			return new VideoSample(this._data.slice(), {
@@ -245,6 +246,7 @@ export class VideoSample {
 				timestamp: this.timestamp,
 				duration: this.duration,
 				colorSpace: this.colorSpace,
+				rotation: this.rotation,
 			});
 		} else {
 			return new VideoSample(this._data, {
@@ -254,6 +256,7 @@ export class VideoSample {
 				timestamp: this.timestamp,
 				duration: this.duration,
 				colorSpace: this.colorSpace,
+				rotation: this.rotation,
 			});
 		}
 	}