Make test more strict

Make Mediabunny more resilient against `data` boxes ending abruptly

README.md

@@ -200,7 +200,7 @@ npm run lint # ESLint
 
 npm run docs:generate # Generates API docs
 npm run docs:dev # Start docs development server
-npm run dev # Start examples development server
+npm run dev # Start examples development server, will run at http://localhost:5173/examples/[name]/
   
 npm run docs:build # Build docs and examples
 ```

dev/convert.html

@@ -115,7 +115,7 @@
 					return ctx.canvas;
 				},
 				*/
-				width: 300,
+				//width: 300,
 				//alpha: 'keep',
 				//width: 320,
 				//discard: true,

dev/demux.html

@@ -14,6 +14,24 @@
 			source: new Mediabunny.BlobSource(file),
 		});
 
+		const videoTrack = await input.getPrimaryVideoTrack();
+		const sink = new Mediabunny.VideoSampleSink(videoTrack);
+
+		console.log(await videoTrack.getDecoderConfig())
+
+		const sink2 = new Mediabunny.EncodedPacketSink(videoTrack);
+		for await (const packet of sink2.packets()) {
+			console.log(packet.data.join(', '))//, window.findNalUnitsInLengthPrefixed(packet.data, 4).map(x => window.extractNalUnitTypeForAvc(x)));
+			if (packet.timestamp > 1) break;
+		}
+
+		//return;
+
+		for await (const sample of sink.samples()) {
+			console.log(sample);
+		}
+
+		/*
 		let total = 0;
 		input.source.onread = (start, end) => {
 			total += end - start;
@@ -22,6 +40,7 @@
 
 		const videoTrack = await input.getPrimaryVideoTrack();
 		const sink = new Mediabunny.EncodedPacketSink(videoTrack);
+		*/
 
 		//console.log(await sink.getPacket(0));
 		/*
@@ -38,7 +57,7 @@
 
 
 
-		const stats = await videoTrack.computePacketStats();
+		//const stats = await videoTrack.computePacketStats();
 		
 		/*
 		const sink = new Mediabunny.EncodedPacketSink(videoTrack);
@@ -47,9 +66,9 @@
 		}
 		*/
 		//console.log(await videoTrack.computeDuration());
-		console.log("Done", stats, total, file.size)
+		//console.log("Done", stats, total, file.size)
 
-		console.log(input);
+		//console.log(input);
 
 		/*
 		const videoTrack = await input.getPrimaryVideoTrack();

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "mediabunny",
-  "version": "1.24.0",
+  "version": "1.24.3",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "mediabunny",
-      "version": "1.24.0",
+      "version": "1.24.3",
       "license": "MPL-2.0",
       "workspaces": [
         "packages/*"
@@ -7749,9 +7749,9 @@
       }
     },
     "node_modules/mediabunny": {
-      "version": "1.23.0",
+      "version": "1.24.2",
-      "resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.23.0.tgz",
+      "resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.24.2.tgz",
-      "integrity": "sha512-mWdhhdRquePfTgZ+18DLCFWmqwO6oGP/YudmRU8Puh+whMUJ3VSb9KMAHPl2utim66ixKdZw+DbmdqBfBxRp0A==",
+      "integrity": "sha512-+H2Jha7D8C92F1fysdN+nmBBAOgVV+pNv/Pxc+c//qo2E/idnv8MsZrft7VJsFE5WjK30ESkwW5ezM0oceFIkg==",
       "license": "MPL-2.0",
       "peer": true,
       "workspaces": [
@@ -12242,7 +12242,7 @@
     },
     "packages/mp3-encoder": {
       "name": "@mediabunny/mp3-encoder",
-      "version": "1.24.0",
+      "version": "1.24.3",
       "license": "MPL-2.0",
       "devDependencies": {
         "@types/emscripten": "^1.40.1"

package.json

@@ -1,7 +1,7 @@
 {
   "name": "mediabunny",
   "author": "Vanilagy",
-  "version": "1.24.0",
+  "version": "1.24.3",
   "description": "Pure TypeScript media toolkit for reading, writing, and converting media files, directly in the browser.",
   "type": "module",
   "workspaces": [

packages/mp3-encoder/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@mediabunny/mp3-encoder",
   "author": "Vanilagy",
-  "version": "1.24.0",
+  "version": "1.24.3",
   "description": "MP3 encoder extension for Mediabunny, based on LAME.",
   "main": "./dist/bundles/mediabunny-mp3-encoder.mjs",
   "module": "./dist/bundles/mediabunny-mp3-encoder.mjs",

src/conversion.ts

@@ -186,7 +186,7 @@ export type ConversionVideoOptions = {
 	 *
 	 * This function can also be used to manually resize frames. When doing so, you should signal the post-process
 	 * dimensions using the `processedWidth` and `processedHeight` fields, which enables the encoder to better know what
-	 * to expect.
+	 * to expect. If these fields aren't set, Mediabunny will assume you won't perform any resizing.
 	 */
 	process?: (sample: VideoSample) => MaybePromise<
 		CanvasImageSource | VideoSample | (CanvasImageSource | VideoSample)[] | null
@@ -229,7 +229,8 @@ export type ConversionAudioOptions = {
 	 *
 	 * This function can also be used to manually perform remixing or resampling. When doing so, you should signal the
 	 * post-process parameters using the `processedNumberOfChannels` and `processedSampleRate` fields, which enables the
-	 * encoder to better know what to expect.
+	 * encoder to better know what to expect. If these fields aren't set, Mediabunny will assume you won't perform
+	 * remixing or resampling.
 	 */
 	process?: (sample: AudioSample) => MaybePromise<
 		AudioSample | AudioSample[] | null
@@ -1350,7 +1351,6 @@ export class Conversion {
 				!encodableCodecs.some(codec => (NON_PCM_AUDIO_CODECS as readonly string[]).includes(codec))
 				&& audioCodecs.some(codec => (NON_PCM_AUDIO_CODECS as readonly string[]).includes(codec))
 				&& (numberOfChannels !== FALLBACK_NUMBER_OF_CHANNELS || sampleRate !== FALLBACK_SAMPLE_RATE)
-				&& !trackOptions.process
 			) {
 				// We could not find a compatible non-PCM codec despite the container supporting them. This can be
 				// caused by strange channel count or sample rate configurations. Therefore, let's try again but with

src/isobmff/isobmff-reader.ts

@@ -70,6 +70,11 @@ export const readDataBox = (slice: FileSlice) => {
 		return null;
 	}
 
+	if (slice.remainingLength < 8) {
+		// Box is too small
+		return null;
+	}
+
 	const typeIndicator = readU32Be(slice);
 	slice.skip(4); // Locale indicator
 	const data = readBytes(slice, header.contentSize - 8);

src/matroska/matroska-demuxer.ts

@@ -637,7 +637,10 @@ export class MatroskaDemuxer extends Demuxer {
 		this.currentCluster = cluster;
 
 		if (dataSlice) {
-			this.readContiguousElements(dataSlice);
+			// Read the children of the cluster, stopping early at level 0 or 1 EBML elements. We do this because some
+			// clusters have incorrect sizes that are too large
+			const endPos = this.readContiguousElements(dataSlice, LEVEL_0_AND_1_EBML_IDS);
+			cluster.elementEndPos = endPos;
 		}
 
 		for (const [, trackData] of cluster.trackData) {
@@ -913,24 +916,31 @@ export class MatroskaDemuxer extends Demuxer {
 		}
 	}
 
-	readContiguousElements(slice: FileSlice) {
+	readContiguousElements(slice: FileSlice, stopIds?: number[]) {
 		const startIndex = slice.filePos;
 
 		while (slice.filePos - startIndex <= slice.length - MIN_HEADER_SIZE) {
-			const foundElement = this.traverseElement(slice);
+			const startPos = slice.filePos;
+			const foundElement = this.traverseElement(slice, stopIds);
 
 			if (!foundElement) {
-				break;
+				return startPos;
 			}
 		}
+
+		return slice.filePos;
 	}
 
-	traverseElement(slice: FileSlice): boolean {
+	traverseElement(slice: FileSlice, stopIds?: number[]): boolean {
 		const header = readElementHeader(slice);
 		if (!header) {
 			return false;
 		}
 
+		if (stopIds && stopIds.includes(header.id)) {
+			return false;
+		}
+
 		const { id, size } = header;
 		const dataStartPos = slice.filePos;
 		assertDefinedSize(size);
@@ -2174,6 +2184,8 @@ abstract class MatroskaTrackBacking implements InputTrackBacking {
 
 			if (id === EBMLId.Cluster) {
 				currentCluster = await demuxer.readCluster(elementStartPos, segment);
+				// readCluster computes the proper size even if it's undefined in the header, so let's use that instead
+				size = currentCluster.elementEndPos - dataStartPos;
 
 				const { blockIndex, correctBlockFound } = getMatchInCluster(currentCluster);
 				if (correctBlockFound) {
@@ -2190,44 +2202,37 @@ abstract class MatroskaTrackBacking implements InputTrackBacking {
 				// Undefined element size (can happen in livestreamed files). In this case, we need to do some
 				// searching to determine the actual size of the element.
 
-				if (id === EBMLId.Cluster) {
+				assert(id !== EBMLId.Cluster); // Undefined cluster sizes are fixed further up
-					// The cluster should have already computed its length, we can just copy that result
-					assert(currentCluster);
-					size = currentCluster.elementEndPos - dataStartPos;
-				} else {
-					// Search for the next element at level 0 or 1
-					const nextElementPos = await searchForNextElementId(
-						demuxer.reader,
-						dataStartPos,
-						LEVEL_0_AND_1_EBML_IDS,
-						segment.elementEndPos,
-					);
 
-					size = nextElementPos.pos - dataStartPos;
+				// Search for the next element at level 0 or 1
-				}
+				const nextElementPos = await searchForNextElementId(
+					demuxer.reader,
+					dataStartPos,
+					LEVEL_0_AND_1_EBML_IDS,
+					segment.elementEndPos,
+				);
 
-				const endPos = dataStartPos + size;
+				size = nextElementPos.pos - dataStartPos;
-				if (segment.elementEndPos !== null && endPos > segment.elementEndPos - MIN_HEADER_SIZE) {
+			}
-					// No more elements fit in this segment
+
+			const endPos = dataStartPos + size;
+			if (segment.elementEndPos === null) {
+				// Check the next element. If it's a new segment, we know this segment ends here. The new
+				// segment is just ignored, since we're likely in a livestreamed file and thus only care about
+				// the first segment.
+
+				let slice = demuxer.reader.requestSliceRange(endPos, MIN_HEADER_SIZE, MAX_HEADER_SIZE);
+				if (slice instanceof Promise) slice = await slice;
+				if (!slice) break;
+
+				const elementId = readElementId(slice);
+				if (elementId === EBMLId.Segment) {
+					segment.elementEndPos = endPos; // We now know the segment's size
 					break;
-				} else {
-					// Check the next element. If it's a new segment, we know this segment ends here. The new
-					// segment is just ignored, since we're likely in a livestreamed file and thus only care about
-					// the first segment.
-
-					let slice = demuxer.reader.requestSliceRange(endPos, MIN_HEADER_SIZE, MAX_HEADER_SIZE);
-					if (slice instanceof Promise) slice = await slice;
-					if (!slice) break;
-
-					const elementId = readElementId(slice);
-					if (elementId === EBMLId.Segment) {
-						segment.elementEndPos = endPos;
-						break;
-					}
 				}
 			}
 
-			currentPos = dataStartPos + size;
+			currentPos = endPos;
 		}
 
 		// Catch faulty cue points

src/media-source.ts

@@ -1159,7 +1159,7 @@ export class MediaStreamVideoTrackSource extends VideoSource {
 					type: 'videoTrack',
 					trackId: this._workerTrackId,
 					track: this._track,
-				}, [this._track]);
+				});
 
 				this._workerListener = (event: MessageEvent) => {
 					const message = event.data as MediaStreamTrackProcessorWorkerMessage;
@@ -1978,17 +1978,19 @@ const mediaStreamTrackProcessorWorkerCode = () => {
 	});
 
 	const abortControllers = new Map<number, AbortController>();
-	const stoppedTracks = new Set<number>();
+	const activeTracks = new Map<number, MediaStreamVideoTrack>();
 
 	self.addEventListener('message', (event) => {
 		const message = event.data as MediaStreamTrackProcessorControllerMessage;
 
 		switch (message.type) {
 			case 'videoTrack': {
+				activeTracks.set(message.trackId, message.track);
+
 				const processor = new MediaStreamTrackProcessor({ track: message.track });
 				const consumer = new WritableStream<VideoFrame>({
 					write: (videoFrame) => {
-						if (stoppedTracks.has(message.trackId)) {
+						if (!activeTracks.has(message.trackId)) {
 							videoFrame.close();
 							return;
 						}
@@ -2026,7 +2028,9 @@ const mediaStreamTrackProcessorWorkerCode = () => {
 					abortControllers.delete(message.trackId);
 				}
 
-				stoppedTracks.add(message.trackId);
+				const track = activeTracks.get(message.trackId);
+				track?.stop();
+				activeTracks.delete(message.trackId);
 
 				sendMessage({
 					type: 'trackStopped',

src/source.ts

@@ -234,11 +234,19 @@ export class BlobSource extends Source {
 					break;
 				}
 
+				if (worker.aborted) {
+					break;
+				}
+
 				this.onread?.(worker.currentPos, worker.currentPos + value.length);
 				this._orchestrator.supplyWorkerData(worker, value);
 			} else {
 				const data = await this._blob.slice(worker.currentPos, worker.targetPos).arrayBuffer();
 
+				if (worker.aborted) {
+					break;
+				}
+
 				this.onread?.(worker.currentPos, worker.currentPos + data.byteLength);
 				this._orchestrator.supplyWorkerData(worker, new Uint8Array(data));
 			}
@@ -457,7 +465,7 @@ export class UrlSource extends Source {
 	/** @internal */
 	private async _runWorker(worker: ReadWorker) {
 		// The outer loop is for resuming a request if it dies mid-response
-		while (!worker.aborted) {
+		while (true) {
 			const existing = this._existingResponses.get(worker);
 			this._existingResponses.delete(worker);
 
@@ -534,6 +542,10 @@ export class UrlSource extends Source {
 					}
 				}
 
+				if (worker.aborted) {
+					break;
+				}
+
 				const { done, value } = readResult;
 
 				if (done) {
@@ -552,6 +564,10 @@ export class UrlSource extends Source {
 				this.onread?.(worker.currentPos, worker.currentPos + value.length);
 				this._orchestrator.supplyWorkerData(worker, value);
 			}
+
+			if (worker.aborted) {
+				break;
+			}
 		}
 
 		worker.running = false;
@@ -796,6 +812,10 @@ export class StreamSource extends Source {
 			let data = this._options.read(worker.currentPos, originalTargetPos);
 			if (data instanceof Promise) data = await data;
 
+			if (worker.aborted) {
+				break;
+			}
+
 			if (data instanceof Uint8Array) {
 				data = toUint8Array(data); // Normalize things like Node.js Buffer to Uint8Array
 
@@ -833,6 +853,10 @@ export class StreamSource extends Source {
 						throw new TypeError('ReadableStream returned by options.read must yield Uint8Array chunks.');
 					}
 
+					if (worker.aborted) {
+						break;
+					}
+
 					const data = toUint8Array(value); // Normalize things like Node.js Buffer to Uint8Array
 
 					this.onread?.(worker.currentPos, worker.currentPos + data.length);
@@ -1420,7 +1444,10 @@ class ReadOrchestrator {
 			currentPos: startPos,
 			targetPos,
 			running: false,
-			aborted: false,
+			// Due to async shenanigans, it can happen that workers are started after disposal. In this case, instead of
+			// simply not creating the worker, we allow it to run but immediately label it as aborted, so it can then
+			// shut itself down.
+			aborted: this.disposed,
 			pendingSlices: [],
 			age: this.nextAge++,
 		};
@@ -1473,10 +1500,7 @@ class ReadOrchestrator {
 
 	/** Called by a worker when it has read some data. */
 	supplyWorkerData(worker: ReadWorker, bytes: Uint8Array) {
-		if (this.disposed) {
+		assert(!worker.aborted);
-			// Writes may still come in after disposal, but we just ignore those
-			return;
-		}
 
 		const start = worker.currentPos;
 		const end = start + bytes.length;

test/browser/transparency.test.ts

@@ -139,17 +139,18 @@ test('Can encode transparent video', async () => {
 	probeContext.drawImage(video, 0, 0);
 
 	let imageData = probeContext.getImageData(0, 0, probeCanvas.width, probeCanvas.height);
-	expect(imageData.data[3]).lessThanOrEqual(2); // Transparent (within error)
+	const index1 = (100 + 100 * probeCanvas.width) * 4;
+	expect(imageData.data[index1 + 3]).toBe(0); // Transparent
 
 	const pos = { x: 300, y: 300 }; // Dead center in the red square
-	const index = (pos.x + pos.y * probeCanvas.width) * 4;
+	const index2 = (pos.x + pos.y * probeCanvas.width) * 4;
 
 	// Red (within error)
-	expect(imageData.data[index + 0]).greaterThanOrEqual(253);
+	expect(imageData.data[index2 + 0]).greaterThanOrEqual(253);
-	expect(imageData.data[index + 1]).lessThanOrEqual(2);
+	expect(imageData.data[index2 + 1]).lessThanOrEqual(2);
-	expect(imageData.data[index + 2]).lessThanOrEqual(2);
+	expect(imageData.data[index2 + 2]).lessThanOrEqual(2);
 
-	expect(imageData.data[index + 3]).greaterThanOrEqual(253); // Opaque (within error)
+	expect(imageData.data[index2 + 3]).toBe(255); // Opaque
 
 	// Let's also check it's read correctly by Mediabunny
 	using input = new Input({
@@ -169,7 +170,8 @@ test('Can encode transparent video', async () => {
 	firstSample.draw(probeContext, 0, 0);
 
 	imageData = probeContext.getImageData(0, 0, probeCanvas.width, probeCanvas.height);
-	expect(imageData.data[3]).lessThanOrEqual(2); // Transparent (within error)
+
+	expect(imageData.data[3]).toBe(0); // Transparent
 });
 
 test('Can encode video with alternating transparency', async () => {