Make test more strict

Make Mediabunny more resilient against `data` boxes ending abruptly

README.md

@@ -1,6 +1,3 @@
-> [!NOTE]
-> I'm on vacation until 18 October, so expect slow or no replies to issues during that time. 🏖️
-
 # Mediabunny - JavaScript media toolkit
 
 [![](https://img.shields.io/npm/v/mediabunny)](https://www.npmjs.com/package/mediabunny)
@@ -203,7 +200,7 @@ npm run lint # ESLint
 
 npm run docs:generate # Generates API docs
 npm run docs:dev # Start docs development server
-npm run dev # Start examples development server
-
+npm run dev # Start examples development server, will run at http://localhost:5173/examples/[name]/
+  
 npm run docs:build # Build docs and examples
 ```

dev/convert.html

@@ -56,6 +56,11 @@
 			output,
 			audio: (_, n) => ({
 				discard: n > 1,
+				/*
+				process: (sample) => {
+					return sample;
+				},
+				*/
 				//codec: 'pcm-s16',
 				//sampleRate: 16000,
 				//numberOfChannels: 1,
@@ -92,7 +97,25 @@
 			},
 			*/
 			video: () => ({
-				width: 300,
+				/*
+				process: (sample) => {
+					if (!ctx) {
+						// Create a canvas for image compositing
+						const canvas = new OffscreenCanvas(
+							sample.displayWidth,
+							sample.displayHeight,
+						);
+						ctx = canvas.getContext('2d');
+					}
+
+					ctx.clearRect(0, 0, ctx.canvas.width, ctx.canvas.height);
+					sample.draw(ctx, 0, 0);
+					ctx.drawImage(watermark, 32, 32);
+
+					return ctx.canvas;
+				},
+				*/
+				//width: 300,
 				//alpha: 'keep',
 				//width: 320,
 				//discard: true,

dev/demux.html

@@ -14,6 +14,24 @@
 			source: new Mediabunny.BlobSource(file),
 		});
 
+		const videoTrack = await input.getPrimaryVideoTrack();
+		const sink = new Mediabunny.VideoSampleSink(videoTrack);
+
+		console.log(await videoTrack.getDecoderConfig())
+
+		const sink2 = new Mediabunny.EncodedPacketSink(videoTrack);
+		for await (const packet of sink2.packets()) {
+			console.log(packet.data.join(', '))//, window.findNalUnitsInLengthPrefixed(packet.data, 4).map(x => window.extractNalUnitTypeForAvc(x)));
+			if (packet.timestamp > 1) break;
+		}
+
+		//return;
+
+		for await (const sample of sink.samples()) {
+			console.log(sample);
+		}
+
+		/*
 		let total = 0;
 		input.source.onread = (start, end) => {
 			total += end - start;
@@ -22,6 +40,7 @@
 
 		const videoTrack = await input.getPrimaryVideoTrack();
 		const sink = new Mediabunny.EncodedPacketSink(videoTrack);
+		*/
 
 		//console.log(await sink.getPacket(0));
 		/*
@@ -38,7 +57,7 @@
 
 
 
-		const stats = await videoTrack.computePacketStats();
+		//const stats = await videoTrack.computePacketStats();
 		
 		/*
 		const sink = new Mediabunny.EncodedPacketSink(videoTrack);
@@ -47,9 +66,9 @@
 		}
 		*/
 		//console.log(await videoTrack.computeDuration());
-		console.log("Done", stats, total, file.size)
+		//console.log("Done", stats, total, file.size)
 
-		console.log(input);
+		//console.log(input);
 
 		/*
 		const videoTrack = await input.getPrimaryVideoTrack();

docs/guide/converting-media-files.md

@@ -16,6 +16,7 @@ It has the following features:
 - Video transparency removal/preservation
 - Audio resampling
 - Audio up/downmixing
+- User-defined video & audio processing
 
 The conversion API was built to be simple, versatile and extremely performant.
 
@@ -126,7 +127,14 @@ type ConversionVideoOptions = {
 	alpha?: 'discard' | 'keep'; // Defaults to 'discard'
 	keyFrameInterval?: number;
 	forceTranscode?: boolean;
+	process?: (sample: VideoSample) => MaybePromise<
+		CanvasImageSource | VideoSample | (CanvasImageSource | VideoSample)[] | null
+	>;
+	processedWidth?: number;
+	processedHeight?: number;
 };
+
+type MaybePromise<T> = T | Promise<T>;
 ```
 
 For example, here we resize the video track to 720p:
@@ -185,6 +193,39 @@ Use the `keyFrameInterval` property to control the maximum interval in seconds b
 
 If you want to prevent direct copying of media data and force a transcoding step, use `forceTranscode: true`.
 
+### Processing video
+
+The `process` property can be used to define a custom video sample processing function, e.g. for [applying overlays](./quick-start#add-a-video-overlay), color transformations, or timestamp modifications. You are expected to perform this processing yourself, for example using the Canvas API.
+
+An example:
+```ts
+let ctx: CanvasRenderingContext2D | null = null;
+const conversion = await Conversion.init({
+	video: {
+		process: (sample) => {
+			if (!ctx) {
+				const canvas = new OffscreenCanvas(
+					sample.displayWidth,
+					sample.displayHeight,
+				);
+				ctx = canvas.getContext('2d')!;
+
+				// Convert the video to grayscale
+				ctx.filter = 'saturate(0)';
+			}
+			
+			sample.draw(ctx, 0, 0);
+
+			return ctx.canvas;
+		},
+	},
+});
+```
+
+The function is called for each input video sample after transformations and frame rate corrections. It must return a [`VideoSample`](./packets-and-samples#videosample), something that can convert to a `VideoSample`, an array of them, or `null` for dropping the frame.
+
+This function can also be used to manually resize frames. When doing so, you should signal the post-process dimensions using the `processedWidth` and `processedHeight` fields, which enables the encoder to better know what to expect.
+
 ## Audio options
 
 You can set the `audio` property in the conversion options to configure the converter's behavior for audio tracks. The options are:
@@ -196,7 +237,14 @@ type ConversionAudioOptions = {
 	numberOfChannels?: number;
 	sampleRate?: number;
 	forceTranscode?: boolean;
+	process?: (sample: AudioSample) => MaybePromise<
+		AudioSample | AudioSample[] | null
+	>;
+	processedNumberOfChannels?: number;
+	processedSampleRate?: number;
 };
+
+type MaybePromise<T> = T | Promise<T>;
 ```
 
 For example, here we convert the audio track to mono and set a specific sample rate:
@@ -233,6 +281,14 @@ Use the `bitrate` property to control the bitrate of the output audio. For examp
 
 If you want to prevent direct copying of media data and force a transcoding step, use `forceTranscode: true`.
 
+### Processing audio
+
+The `process` property can be used to define a custom audio sample processing function, e.g. for applying audio effects, transformations, or timestamp modifications. You are expected to perform this processing yourself.
+
+The function is called for each input audio sample after remixing and resampling. It must return an [`AudioSample`](./packets-and-samples#audiosample), an array of them, or `null` for dropping the sample.
+
+This function can also be used to manually perform remixing or resampling. When doing so, you should signal the post-process parameters using the `processedNumberOfChannels` and `processedSampleRate` fields, which enables the encoder to better know what to expect.
+
 ## Track-specific options
 
 You may want to configure your video and audio options differently depending on the specifics of the input track. Or, in case a media file has multiple video or audio tracks, you may want to discard only specific tracks or configure each track separately.

docs/guide/quick-start.md

@@ -578,4 +578,49 @@ await conversion.execute();
 
 ::: info
 - Check out the <a href="/examples/file-compression">File compression example</a> for this code in action.
-:::
+:::
+
+## Add a video overlay
+
+```ts
+import {
+	Input,
+	Output,
+	Conversion,
+} from 'mediabunny';
+
+// For example, let's load a watermark image
+const watermark = new Image();
+watermark.src = '/watermark.jpg';
+await new Promise(resolve => watermark.onload = resolve);
+
+const input = new Input(...);
+const output = new Output(...);
+
+let ctx: CanvasRenderingContext2D | null = null;
+const conversion = await Conversion.init({
+	input,
+	output,
+	video: {
+		process: (sample) => {
+			if (!ctx) {
+				// Create a canvas for image compositing
+				const canvas = new OffscreenCanvas(
+					sample.displayWidth,
+					sample.displayHeight,
+				);
+				ctx = canvas.getContext('2d')!;
+			}
+
+			ctx.clearRect(0, 0, ctx.canvas.width, ctx.canvas.height);
+			sample.draw(ctx, 0, 0);
+			ctx.drawImage(watermark, 32, 32);
+
+			return ctx.canvas;
+		},
+	},
+});
+
+await conversion.execute();
+// Conversion is complete
+```

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "mediabunny",
-  "version": "1.23.0",
+  "version": "1.24.3",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "mediabunny",
-      "version": "1.23.0",
+      "version": "1.24.3",
       "license": "MPL-2.0",
       "workspaces": [
         "packages/*"
@@ -7749,9 +7749,9 @@
       }
     },
     "node_modules/mediabunny": {
-      "version": "1.22.0",
-      "resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.22.0.tgz",
-      "integrity": "sha512-T8zsXwrRtKEAlITRizn0RiJc1GllVMw3Zcze5ACyluTy3jndUPhAM4SrwjR2SWgqqUa+4nlJ3MmyAi2XuRxwFg==",
+      "version": "1.24.2",
+      "resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.24.2.tgz",
+      "integrity": "sha512-+H2Jha7D8C92F1fysdN+nmBBAOgVV+pNv/Pxc+c//qo2E/idnv8MsZrft7VJsFE5WjK30ESkwW5ezM0oceFIkg==",
       "license": "MPL-2.0",
       "peer": true,
       "workspaces": [
@@ -12242,7 +12242,7 @@
     },
     "packages/mp3-encoder": {
       "name": "@mediabunny/mp3-encoder",
-      "version": "1.23.0",
+      "version": "1.24.3",
       "license": "MPL-2.0",
       "devDependencies": {
         "@types/emscripten": "^1.40.1"

package.json

@@ -1,7 +1,7 @@
 {
   "name": "mediabunny",
   "author": "Vanilagy",
-  "version": "1.23.0",
+  "version": "1.24.3",
   "description": "Pure TypeScript media toolkit for reading, writing, and converting media files, directly in the browser.",
   "type": "module",
   "workspaces": [

packages/mp3-encoder/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@mediabunny/mp3-encoder",
   "author": "Vanilagy",
-  "version": "1.23.0",
+  "version": "1.24.3",
   "description": "MP3 encoder extension for Mediabunny, based on LAME.",
   "main": "./dist/bundles/mediabunny-mp3-encoder.mjs",
   "module": "./dist/bundles/mediabunny-mp3-encoder.mjs",

src/conversion.ts

@@ -176,6 +176,31 @@ export type ConversionVideoOptions = {
 	keyFrameInterval?: number;
 	/** When `true`, video will always be re-encoded instead of directly copying over the encoded samples. */
 	forceTranscode?: boolean;
+	/**
+	 * Allows for custom user-defined processing of video frames, e.g. for applying overlays, color transformations, or
+	 * timestamp modifications. Will be called for each input video sample after transformations and frame rate
+	 * corrections.
+	 *
+	 * Must return a {@link VideoSample} or a `CanvasImageSource`, an array of them, or `null` for dropping the frame.
+	 * When non-timestamped data is returned, the timestamp and duration from the source sample will be used.
+	 *
+	 * This function can also be used to manually resize frames. When doing so, you should signal the post-process
+	 * dimensions using the `processedWidth` and `processedHeight` fields, which enables the encoder to better know what
+	 * to expect. If these fields aren't set, Mediabunny will assume you won't perform any resizing.
+	 */
+	process?: (sample: VideoSample) => MaybePromise<
+		CanvasImageSource | VideoSample | (CanvasImageSource | VideoSample)[] | null
+	>;
+	/**
+	 * An optional hint specifying the width of video samples returned by the `process` function, for better
+	 * encoder configuration.
+	 */
+	processedWidth?: number;
+	/**
+	 * An optional hint specifying the height of video samples returned by the `process` function, for better
+	 * encoder configuration.
+	 */
+	processedHeight?: number;
 };
 
 /**
@@ -196,6 +221,30 @@ export type ConversionAudioOptions = {
 	bitrate?: number | Quality;
 	/** When `true`, audio will always be re-encoded instead of directly copying over the encoded samples. */
 	forceTranscode?: boolean;
+	/**
+	 * Allows for custom user-defined processing of audio samples, e.g. for applying audio effects, transformations, or
+	 * timestamp modifications. Will be called for each input audio sample after remixing and resampling.
+	 *
+	 * Must return an {@link AudioSample}, an array of them, or `null` for dropping the sample.
+	 *
+	 * This function can also be used to manually perform remixing or resampling. When doing so, you should signal the
+	 * post-process parameters using the `processedNumberOfChannels` and `processedSampleRate` fields, which enables the
+	 * encoder to better know what to expect. If these fields aren't set, Mediabunny will assume you won't perform
+	 * remixing or resampling.
+	 */
+	process?: (sample: AudioSample) => MaybePromise<
+		AudioSample | AudioSample[] | null
+	>;
+	/**
+	 * An optional hint specifying the channel count of audio samples returned by the `process` function, for better
+	 * encoder configuration.
+	 */
+	processedNumberOfChannels?: number;
+	/**
+	 * An optional hint specifying the sample rate of audio samples returned by the `process` function, for better
+	 * encoder configuration.
+	 */
+	processedSampleRate?: number;
 };
 
 const validateVideoOptions = (videoOptions: ConversionVideoOptions | undefined) => {
@@ -264,7 +313,22 @@ const validateVideoOptions = (videoOptions: ConversionVideoOptions | undefined)
 		videoOptions?.keyFrameInterval !== undefined
 		&& (!Number.isFinite(videoOptions.keyFrameInterval) || videoOptions.keyFrameInterval < 0)
 	) {
-		throw new TypeError('config.keyFrameInterval, when provided, must be a non-negative number.');
+		throw new TypeError('options.video.keyFrameInterval, when provided, must be a non-negative number.');
+	}
+	if (videoOptions?.process !== undefined && typeof videoOptions.process !== 'function') {
+		throw new TypeError('options.video.process, when provided, must be a function.');
+	}
+	if (
+		videoOptions?.processedWidth !== undefined
+		&& (!Number.isInteger(videoOptions.processedWidth) || videoOptions.processedWidth <= 0)
+	) {
+		throw new TypeError('options.video.processedWidth, when provided, must be a positive integer.');
+	}
+	if (
+		videoOptions?.processedHeight !== undefined
+		&& (!Number.isInteger(videoOptions.processedHeight) || videoOptions.processedHeight <= 0)
+	) {
+		throw new TypeError('options.video.processedHeight, when provided, must be a positive integer.');
 	}
 };
 
@@ -302,6 +366,21 @@ const validateAudioOptions = (audioOptions: ConversionAudioOptions | undefined)
 	) {
 		throw new TypeError('options.audio.sampleRate, when provided, must be a positive integer.');
 	}
+	if (audioOptions?.process !== undefined && typeof audioOptions.process !== 'function') {
+		throw new TypeError('options.audio.process, when provided, must be a function.');
+	}
+	if (
+		audioOptions?.processedNumberOfChannels !== undefined
+		&& (!Number.isInteger(audioOptions.processedNumberOfChannels) || audioOptions.processedNumberOfChannels <= 0)
+	) {
+		throw new TypeError('options.audio.processedNumberOfChannels, when provided, must be a positive integer.');
+	}
+	if (
+		audioOptions?.processedSampleRate !== undefined
+		&& (!Number.isInteger(audioOptions.processedSampleRate) || audioOptions.processedSampleRate <= 0)
+	) {
+		throw new TypeError('options.audio.processedSampleRate, when provided, must be a positive integer.');
+	}
 };
 
 const FALLBACK_NUMBER_OF_CHANNELS = 2;
@@ -790,7 +869,8 @@ export class Conversion {
 			|| this._startTimestamp > 0
 			|| firstTimestamp < 0
 			|| !!trackOptions.frameRate
-			|| trackOptions.keyFrameInterval !== undefined;
+			|| trackOptions.keyFrameInterval !== undefined
+			|| trackOptions.process !== undefined;
 		let needsRerender = width !== originalWidth
 			|| height !== originalHeight
 			|| (totalRotation !== 0 && !outputSupportsRotation)
@@ -822,10 +902,6 @@ export class Conversion {
 					: undefined;
 
 				for await (const packet of sink.packets(undefined, endPacket, { verifyKeyPackets: true })) {
-					if (this._synchronizer.shouldWait(track.id, packet.timestamp)) {
-						await this._synchronizer.wait(packet.timestamp);
-					}
-
 					if (this._canceled) {
 						return;
 					}
@@ -836,8 +912,12 @@ export class Conversion {
 						delete packet.sideData.alphaByteLength;
 					}
 
+					this._reportProgress(track.id, packet.timestamp);
 					await source.add(packet, meta);
-					this._reportProgress(track.id, packet.timestamp + packet.duration);
+
+					if (this._synchronizer.shouldWait(track.id, packet.timestamp)) {
+						await this._synchronizer.wait(packet.timestamp);
+					}
 				}
 
 				source.close();
@@ -861,7 +941,15 @@ export class Conversion {
 
 			const bitrate = trackOptions.bitrate ?? QUALITY_HIGH;
 
-			const encodableCodec = await getFirstEncodableVideoCodec(videoCodecs, { width, height, bitrate });
+			const encodableCodec = await getFirstEncodableVideoCodec(videoCodecs, {
+				width: trackOptions.process && trackOptions.processedWidth
+					? trackOptions.processedWidth
+					: width,
+				height: trackOptions.process && trackOptions.processedHeight
+					? trackOptions.processedHeight
+					: height,
+				bitrate,
+			});
 			if (!encodableCodec) {
 				this.discardedTracks.push({
 					track,
@@ -876,7 +964,6 @@ export class Conversion {
 				keyFrameInterval: trackOptions.keyFrameInterval,
 				sizeChangeBehavior: trackOptions.fit ?? 'passThrough',
 				alpha,
-				onEncodedPacket: sample => this._reportProgress(track.id, sample.timestamp + sample.duration),
 			};
 
 			const source = new VideoSampleSource(encodingConfig);
@@ -889,7 +976,7 @@ export class Conversion {
 				// back to the rerender path.
 				//
 				// Creating a new temporary Output is sort of hacky, but due to a lack of an isolated encoder API right
-				// now, this is the simplest way. Will refactor in the future!
+				// now, this is the simplest way. Will refactor in the future! TODO
 
 				const tempOutput = new Output({
 					format: new Mp4OutputFormat(), // Supports all video codecs
@@ -951,15 +1038,11 @@ export class Conversion {
 								timestamp: lastCanvasTimestamp! + i / frameRate,
 								duration: 1 / frameRate,
 							});
-							await source.add(sample);
+							await this._registerVideoSample(track, trackOptions, source, sample);
 						}
 					};
 
 					for await (const { canvas, timestamp, duration } of iterator) {
-						if (this._synchronizer.shouldWait(track.id, timestamp)) {
-							await this._synchronizer.wait(timestamp);
-						}
-
 						if (this._canceled) {
 							return;
 						}
@@ -991,8 +1074,7 @@ export class Conversion {
 							timestamp: adjustedSampleTimestamp,
 							duration: frameRate !== undefined ? 1 / frameRate : duration,
 						});
-
-						await source.add(sample);
+						await this._registerVideoSample(track, trackOptions, source, sample);
 
 						if (frameRate !== undefined) {
 							lastCanvas = canvas;
@@ -1034,17 +1116,13 @@ export class Conversion {
 						for (let i = 1; i < frameDifference; i++) {
 							lastSample.setTimestamp(lastSampleTimestamp! + i / frameRate);
 							lastSample.setDuration(1 / frameRate);
-							await source.add(lastSample);
+							await this._registerVideoSample(track, trackOptions, source, lastSample);
 						}
 
 						lastSample.close();
 					};
 
 					for await (const sample of sink.samples(this._startTimestamp, this._endTimestamp)) {
-						if (this._synchronizer.shouldWait(track.id, sample.timestamp)) {
-							await this._synchronizer.wait(sample.timestamp);
-						}
-
 						if (this._canceled) {
 							lastSample?.close();
 							return;
@@ -1076,7 +1154,7 @@ export class Conversion {
 						}
 
 						sample.setTimestamp(adjustedSampleTimestamp);
-						await source.add(sample);
+						await this._registerVideoSample(track, trackOptions, source, sample);
 
 						if (frameRate !== undefined) {
 							lastSample = sample;
@@ -1113,6 +1191,67 @@ export class Conversion {
 		this.utilizedTracks.push(track);
 	}
 
+	/** @internal */
+	async _registerVideoSample(
+		track: InputVideoTrack,
+		trackOptions: ConversionVideoOptions,
+		source: VideoSampleSource,
+		sample: VideoSample,
+	) {
+		if (this._canceled) {
+			return;
+		}
+
+		this._reportProgress(track.id, sample.timestamp);
+
+		let finalSamples: VideoSample[];
+		if (!trackOptions.process) {
+			finalSamples = [sample];
+		} else {
+			let processed = trackOptions.process(sample);
+			if (processed instanceof Promise) processed = await processed;
+
+			if (!Array.isArray(processed)) {
+				processed = processed === null ? [] : [processed];
+			}
+
+			finalSamples = processed.map((x) => {
+				if (x instanceof VideoSample) {
+					return x;
+				}
+
+				if (typeof VideoFrame !== 'undefined' && x instanceof VideoFrame) {
+					return new VideoSample(x);
+				}
+
+				// Calling the VideoSample constructor here will automatically handle input validation for us
+				// (it throws for any non-legal argument).
+				return new VideoSample(x, {
+					timestamp: sample.timestamp,
+					duration: sample.duration,
+				});
+			});
+		}
+
+		for (const finalSample of finalSamples) {
+			if (this._canceled) {
+				break;
+			}
+
+			await source.add(finalSample);
+
+			if (this._synchronizer.shouldWait(track.id, finalSample.timestamp)) {
+				await this._synchronizer.wait(finalSample.timestamp);
+			}
+		}
+
+		for (const finalSample of finalSamples) {
+			if (finalSample !== sample) {
+				finalSample.close();
+			}
+		}
+	}
+
 	/** @internal */
 	async _processAudioTrack(track: InputAudioTrack, trackOptions: ConversionAudioOptions) {
 		const sourceCodec = track.codec;
@@ -1145,6 +1284,7 @@ export class Conversion {
 			&& !needsResample
 			&& audioCodecs.includes(sourceCodec)
 			&& (!trackOptions.codec || trackOptions.codec === sourceCodec)
+			&& !trackOptions.process
 		) {
 			// Fast path, we can simply copy over the encoded packets
 
@@ -1162,16 +1302,16 @@ export class Conversion {
 					: undefined;
 
 				for await (const packet of sink.packets(undefined, endPacket)) {
-					if (this._synchronizer.shouldWait(track.id, packet.timestamp)) {
-						await this._synchronizer.wait(packet.timestamp);
-					}
-
 					if (this._canceled) {
 						return;
 					}
 
+					this._reportProgress(track.id, packet.timestamp);
 					await source.add(packet, meta);
-					this._reportProgress(track.id, packet.timestamp + packet.duration);
+
+					if (this._synchronizer.shouldWait(track.id, packet.timestamp)) {
+						await this._synchronizer.wait(packet.timestamp);
+					}
 				}
 
 				source.close();
@@ -1198,8 +1338,12 @@ export class Conversion {
 			const bitrate = trackOptions.bitrate ?? QUALITY_HIGH;
 
 			const encodableCodecs = await getEncodableAudioCodecs(audioCodecs, {
-				numberOfChannels,
-				sampleRate,
+				numberOfChannels: trackOptions.process && trackOptions.processedNumberOfChannels
+					? trackOptions.processedNumberOfChannels
+					: numberOfChannels,
+				sampleRate: trackOptions.process && trackOptions.processedSampleRate
+					? trackOptions.processedSampleRate
+					: sampleRate,
 				bitrate,
 			});
 
@@ -1240,12 +1384,18 @@ export class Conversion {
 			}
 
 			if (needsResample) {
-				audioSource = this._resampleAudio(track, codecOfChoice, numberOfChannels, sampleRate, bitrate);
+				audioSource = this._resampleAudio(
+					track,
+					trackOptions,
+					codecOfChoice,
+					numberOfChannels,
+					sampleRate,
+					bitrate,
+				);
 			} else {
 				const source = new AudioSampleSource({
 					codec: codecOfChoice,
 					bitrate,
-					onEncodedPacket: packet => this._reportProgress(track.id, packet.timestamp + packet.duration),
 				});
 				audioSource = source;
 
@@ -1254,15 +1404,11 @@ export class Conversion {
 
 					const sink = new AudioSampleSink(track);
 					for await (const sample of sink.samples(undefined, this._endTimestamp)) {
-						if (this._synchronizer.shouldWait(track.id, sample.timestamp)) {
-							await this._synchronizer.wait(sample.timestamp);
-						}
-
 						if (this._canceled) {
 							return;
 						}
 
-						await source.add(sample);
+						await this._registerAudioSample(track, trackOptions, source, sample);
 						sample.close();
 					}
 
@@ -1283,9 +1429,62 @@ export class Conversion {
 		this.utilizedTracks.push(track);
 	}
 
+	/** @internal */
+	async _registerAudioSample(
+		track: InputAudioTrack,
+		trackOptions: ConversionAudioOptions,
+		source: AudioSampleSource,
+		sample: AudioSample,
+	) {
+		if (this._canceled) {
+			return;
+		}
+
+		this._reportProgress(track.id, sample.timestamp);
+
+		let finalSamples: AudioSample[];
+		if (!trackOptions.process) {
+			finalSamples = [sample];
+		} else {
+			let processed = trackOptions.process(sample);
+			if (processed instanceof Promise) processed = await processed;
+
+			if (!Array.isArray(processed)) {
+				processed = processed === null ? [] : [processed];
+			}
+
+			if (!processed.every(x => x instanceof AudioSample)) {
+				throw new TypeError(
+					'The audio process function must return an AudioSample, null, or an array of AudioSamples.',
+				);
+			}
+
+			finalSamples = processed;
+		}
+
+		for (const finalSample of finalSamples) {
+			if (this._canceled) {
+				break;
+			}
+
+			await source.add(finalSample);
+
+			if (this._synchronizer.shouldWait(track.id, finalSample.timestamp)) {
+				await this._synchronizer.wait(finalSample.timestamp);
+			}
+		}
+
+		for (const finalSample of finalSamples) {
+			if (finalSample !== sample) {
+				finalSample.close();
+			}
+		}
+	}
+
 	/** @internal */
 	_resampleAudio(
 		track: InputAudioTrack,
+		trackOptions: ConversionAudioOptions,
 		codec: AudioCodec,
 		targetNumberOfChannels: number,
 		targetSampleRate: number,
@@ -1294,7 +1493,6 @@ export class Conversion {
 		const source = new AudioSampleSource({
 			codec,
 			bitrate,
-			onEncodedPacket: packet => this._reportProgress(track.id, packet.timestamp + packet.duration),
 		});
 
 		this._trackPromises.push((async () => {
@@ -1305,17 +1503,13 @@ export class Conversion {
 				targetSampleRate,
 				startTime: this._startTimestamp,
 				endTime: this._endTimestamp,
-				onSample: sample => source.add(sample),
+				onSample: sample => this._registerAudioSample(track, trackOptions, source, sample),
 			});
 
 			const sink = new AudioSampleSink(track);
 			const iterator = sink.samples(this._startTimestamp, this._endTimestamp);
 
 			for await (const sample of iterator) {
-				if (this._synchronizer.shouldWait(track.id, sample.timestamp)) {
-					await this._synchronizer.wait(sample.timestamp);
-				}
-
 				if (this._canceled) {
 					return;
 				}

src/isobmff/isobmff-reader.ts

@@ -70,6 +70,11 @@ export const readDataBox = (slice: FileSlice) => {
 		return null;
 	}
 
+	if (slice.remainingLength < 8) {
+		// Box is too small
+		return null;
+	}
+
 	const typeIndicator = readU32Be(slice);
 	slice.skip(4); // Locale indicator
 	const data = readBytes(slice, header.contentSize - 8);

src/matroska/matroska-demuxer.ts

@@ -637,7 +637,10 @@ export class MatroskaDemuxer extends Demuxer {
 		this.currentCluster = cluster;
 
 		if (dataSlice) {
-			this.readContiguousElements(dataSlice);
+			// Read the children of the cluster, stopping early at level 0 or 1 EBML elements. We do this because some
+			// clusters have incorrect sizes that are too large
+			const endPos = this.readContiguousElements(dataSlice, LEVEL_0_AND_1_EBML_IDS);
+			cluster.elementEndPos = endPos;
 		}
 
 		for (const [, trackData] of cluster.trackData) {
@@ -913,24 +916,31 @@ export class MatroskaDemuxer extends Demuxer {
 		}
 	}
 
-	readContiguousElements(slice: FileSlice) {
+	readContiguousElements(slice: FileSlice, stopIds?: number[]) {
 		const startIndex = slice.filePos;
 
 		while (slice.filePos - startIndex <= slice.length - MIN_HEADER_SIZE) {
-			const foundElement = this.traverseElement(slice);
+			const startPos = slice.filePos;
+			const foundElement = this.traverseElement(slice, stopIds);
 
 			if (!foundElement) {
-				break;
+				return startPos;
 			}
 		}
+
+		return slice.filePos;
 	}
 
-	traverseElement(slice: FileSlice): boolean {
+	traverseElement(slice: FileSlice, stopIds?: number[]): boolean {
 		const header = readElementHeader(slice);
 		if (!header) {
 			return false;
 		}
 
+		if (stopIds && stopIds.includes(header.id)) {
+			return false;
+		}
+
 		const { id, size } = header;
 		const dataStartPos = slice.filePos;
 		assertDefinedSize(size);
@@ -2174,6 +2184,8 @@ abstract class MatroskaTrackBacking implements InputTrackBacking {
 
 			if (id === EBMLId.Cluster) {
 				currentCluster = await demuxer.readCluster(elementStartPos, segment);
+				// readCluster computes the proper size even if it's undefined in the header, so let's use that instead
+				size = currentCluster.elementEndPos - dataStartPos;
 
 				const { blockIndex, correctBlockFound } = getMatchInCluster(currentCluster);
 				if (correctBlockFound) {
@@ -2190,44 +2202,37 @@ abstract class MatroskaTrackBacking implements InputTrackBacking {
 				// Undefined element size (can happen in livestreamed files). In this case, we need to do some
 				// searching to determine the actual size of the element.
 
-				if (id === EBMLId.Cluster) {
-					// The cluster should have already computed its length, we can just copy that result
-					assert(currentCluster);
-					size = currentCluster.elementEndPos - dataStartPos;
-				} else {
-					// Search for the next element at level 0 or 1
-					const nextElementPos = await searchForNextElementId(
-						demuxer.reader,
-						dataStartPos,
-						LEVEL_0_AND_1_EBML_IDS,
-						segment.elementEndPos,
-					);
+				assert(id !== EBMLId.Cluster); // Undefined cluster sizes are fixed further up
 
-					size = nextElementPos.pos - dataStartPos;
-				}
+				// Search for the next element at level 0 or 1
+				const nextElementPos = await searchForNextElementId(
+					demuxer.reader,
+					dataStartPos,
+					LEVEL_0_AND_1_EBML_IDS,
+					segment.elementEndPos,
+				);
 
-				const endPos = dataStartPos + size;
-				if (segment.elementEndPos !== null && endPos > segment.elementEndPos - MIN_HEADER_SIZE) {
-					// No more elements fit in this segment
+				size = nextElementPos.pos - dataStartPos;
+			}
+
+			const endPos = dataStartPos + size;
+			if (segment.elementEndPos === null) {
+				// Check the next element. If it's a new segment, we know this segment ends here. The new
+				// segment is just ignored, since we're likely in a livestreamed file and thus only care about
+				// the first segment.
+
+				let slice = demuxer.reader.requestSliceRange(endPos, MIN_HEADER_SIZE, MAX_HEADER_SIZE);
+				if (slice instanceof Promise) slice = await slice;
+				if (!slice) break;
+
+				const elementId = readElementId(slice);
+				if (elementId === EBMLId.Segment) {
+					segment.elementEndPos = endPos; // We now know the segment's size
 					break;
-				} else {
-					// Check the next element. If it's a new segment, we know this segment ends here. The new
-					// segment is just ignored, since we're likely in a livestreamed file and thus only care about
-					// the first segment.
-
-					let slice = demuxer.reader.requestSliceRange(endPos, MIN_HEADER_SIZE, MAX_HEADER_SIZE);
-					if (slice instanceof Promise) slice = await slice;
-					if (!slice) break;
-
-					const elementId = readElementId(slice);
-					if (elementId === EBMLId.Segment) {
-						segment.elementEndPos = endPos;
-						break;
-					}
 				}
 			}
 
-			currentPos = dataStartPos + size;
+			currentPos = endPos;
 		}
 
 		// Catch faulty cue points

src/media-source.ts

@@ -1159,7 +1159,7 @@ export class MediaStreamVideoTrackSource extends VideoSource {
 					type: 'videoTrack',
 					trackId: this._workerTrackId,
 					track: this._track,
-				}, [this._track]);
+				});
 
 				this._workerListener = (event: MessageEvent) => {
 					const message = event.data as MediaStreamTrackProcessorWorkerMessage;
@@ -1978,17 +1978,19 @@ const mediaStreamTrackProcessorWorkerCode = () => {
 	});
 
 	const abortControllers = new Map<number, AbortController>();
-	const stoppedTracks = new Set<number>();
+	const activeTracks = new Map<number, MediaStreamVideoTrack>();
 
 	self.addEventListener('message', (event) => {
 		const message = event.data as MediaStreamTrackProcessorControllerMessage;
 
 		switch (message.type) {
 			case 'videoTrack': {
+				activeTracks.set(message.trackId, message.track);
+
 				const processor = new MediaStreamTrackProcessor({ track: message.track });
 				const consumer = new WritableStream<VideoFrame>({
 					write: (videoFrame) => {
-						if (stoppedTracks.has(message.trackId)) {
+						if (!activeTracks.has(message.trackId)) {
 							videoFrame.close();
 							return;
 						}
@@ -2026,7 +2028,9 @@ const mediaStreamTrackProcessorWorkerCode = () => {
 					abortControllers.delete(message.trackId);
 				}
 
-				stoppedTracks.add(message.trackId);
+				const track = activeTracks.get(message.trackId);
+				track?.stop();
+				activeTracks.delete(message.trackId);
 
 				sendMessage({
 					type: 'trackStopped',

src/misc.ts

@@ -208,7 +208,7 @@ export const TRANSFER_CHARACTERISTICS_MAP = {
 	'smpte170m': 6, // SMPTE 170M
 	'linear': 8, // Linear transfer characteristics
 	'iec61966-2-1': 13, // IEC 61966-2-1
-	'pg': 16, // Rec. ITU-R BT.2100-2 perceptual quantization (PQ) system
+	'pq': 16, // Rec. ITU-R BT.2100-2 perceptual quantization (PQ) system
 	'hlg': 18, // Rec. ITU-R BT.2100-2 hybrid loggamma (HLG) system
 };
 export const TRANSFER_CHARACTERISTICS_MAP_INVERSE = invertObject(TRANSFER_CHARACTERISTICS_MAP);

src/source.ts

@@ -234,11 +234,19 @@ export class BlobSource extends Source {
 					break;
 				}
 
+				if (worker.aborted) {
+					break;
+				}
+
 				this.onread?.(worker.currentPos, worker.currentPos + value.length);
 				this._orchestrator.supplyWorkerData(worker, value);
 			} else {
 				const data = await this._blob.slice(worker.currentPos, worker.targetPos).arrayBuffer();
 
+				if (worker.aborted) {
+					break;
+				}
+
 				this.onread?.(worker.currentPos, worker.currentPos + data.byteLength);
 				this._orchestrator.supplyWorkerData(worker, new Uint8Array(data));
 			}
@@ -457,7 +465,7 @@ export class UrlSource extends Source {
 	/** @internal */
 	private async _runWorker(worker: ReadWorker) {
 		// The outer loop is for resuming a request if it dies mid-response
-		while (!worker.aborted) {
+		while (true) {
 			const existing = this._existingResponses.get(worker);
 			this._existingResponses.delete(worker);
 
@@ -534,6 +542,10 @@ export class UrlSource extends Source {
 					}
 				}
 
+				if (worker.aborted) {
+					break;
+				}
+
 				const { done, value } = readResult;
 
 				if (done) {
@@ -552,6 +564,10 @@ export class UrlSource extends Source {
 				this.onread?.(worker.currentPos, worker.currentPos + value.length);
 				this._orchestrator.supplyWorkerData(worker, value);
 			}
+
+			if (worker.aborted) {
+				break;
+			}
 		}
 
 		worker.running = false;
@@ -796,6 +812,10 @@ export class StreamSource extends Source {
 			let data = this._options.read(worker.currentPos, originalTargetPos);
 			if (data instanceof Promise) data = await data;
 
+			if (worker.aborted) {
+				break;
+			}
+
 			if (data instanceof Uint8Array) {
 				data = toUint8Array(data); // Normalize things like Node.js Buffer to Uint8Array
 
@@ -833,6 +853,10 @@ export class StreamSource extends Source {
 						throw new TypeError('ReadableStream returned by options.read must yield Uint8Array chunks.');
 					}
 
+					if (worker.aborted) {
+						break;
+					}
+
 					const data = toUint8Array(value); // Normalize things like Node.js Buffer to Uint8Array
 
 					this.onread?.(worker.currentPos, worker.currentPos + data.length);
@@ -1420,7 +1444,10 @@ class ReadOrchestrator {
 			currentPos: startPos,
 			targetPos,
 			running: false,
-			aborted: false,
+			// Due to async shenanigans, it can happen that workers are started after disposal. In this case, instead of
+			// simply not creating the worker, we allow it to run but immediately label it as aborted, so it can then
+			// shut itself down.
+			aborted: this.disposed,
 			pendingSlices: [],
 			age: this.nextAge++,
 		};
@@ -1473,10 +1500,7 @@ class ReadOrchestrator {
 
 	/** Called by a worker when it has read some data. */
 	supplyWorkerData(worker: ReadWorker, bytes: Uint8Array) {
-		if (this.disposed) {
-			// Writes may still come in after disposal, but we just ignore those
-			return;
-		}
+		assert(!worker.aborted);
 
 		const start = worker.currentPos;
 		const end = start + bytes.length;

test/browser/transparency.test.ts

@@ -139,17 +139,18 @@ test('Can encode transparent video', async () => {
 	probeContext.drawImage(video, 0, 0);
 
 	let imageData = probeContext.getImageData(0, 0, probeCanvas.width, probeCanvas.height);
-	expect(imageData.data[3]).lessThanOrEqual(2); // Transparent (within error)
+	const index1 = (100 + 100 * probeCanvas.width) * 4;
+	expect(imageData.data[index1 + 3]).toBe(0); // Transparent
 
 	const pos = { x: 300, y: 300 }; // Dead center in the red square
-	const index = (pos.x + pos.y * probeCanvas.width) * 4;
+	const index2 = (pos.x + pos.y * probeCanvas.width) * 4;
 
 	// Red (within error)
-	expect(imageData.data[index + 0]).greaterThanOrEqual(253);
-	expect(imageData.data[index + 1]).lessThanOrEqual(2);
-	expect(imageData.data[index + 2]).lessThanOrEqual(2);
+	expect(imageData.data[index2 + 0]).greaterThanOrEqual(253);
+	expect(imageData.data[index2 + 1]).lessThanOrEqual(2);
+	expect(imageData.data[index2 + 2]).lessThanOrEqual(2);
 
-	expect(imageData.data[index + 3]).greaterThanOrEqual(253); // Opaque (within error)
+	expect(imageData.data[index2 + 3]).toBe(255); // Opaque
 
 	// Let's also check it's read correctly by Mediabunny
 	using input = new Input({
@@ -169,7 +170,8 @@ test('Can encode transparent video', async () => {
 	firstSample.draw(probeContext, 0, 0);
 
 	imageData = probeContext.getImageData(0, 0, probeCanvas.width, probeCanvas.height);
-	expect(imageData.data[3]).lessThanOrEqual(2); // Transparent (within error)
+
+	expect(imageData.data[3]).toBe(0); // Transparent
 });
 
 test('Can encode video with alternating transparency', async () => {