Provide "in-band" encoding errors

dev/convert.html

@@ -7,9 +7,9 @@
 	fileInput.type = 'file';
 	document.body.append(fileInput);
 
-	const progress = document.createElement('progress');	
+	const progressElement = document.createElement('progress');	
-	progress.max = 1;
+	progressElement.max = 1;
-	document.body.append(progress);
+	document.body.append(progressElement);
 
 	fileInput.addEventListener('change', async () => {
 		const file = fileInput.files[0];
@@ -21,7 +21,7 @@
 			chunked: true,
 			chunkSize: 2**20
 		});
-		const outputFormat = new Metamuxer.WavOutputFormat();
+		const outputFormat = new Metamuxer.Mp4OutputFormat();
 
 		const button = document.createElement('button');
 		button.textContent = 'Cancel';
@@ -38,10 +38,10 @@
 				target
 			}),
 			audio: {
-				numberOfChannels: 1,
+				//numberOfChannels: 1,
-				sampleRate: 4000
+				//sampleRate: 4000
 				//discard: true
-				//forceReencode: true,
+				forceReencode: true,
 			},
 			/*
 			video: {
@@ -68,11 +68,11 @@
 			},
 			*/
 			video: {
-				discard: true,
+				//discard: true,
-				width: 1280,
+				//width: 1280,
 				//discard: true,
 				//width: 640
-				//forceReencode: true,
+				forceReencode: true,
 				//rotate: 90
 				//width: 720 ?? 2160,
 				//height: 1280 ?? 3840,
@@ -87,14 +87,16 @@
 				start: 0,
 				end: 40
 			},
-			computeProgress: true
 		});
 		console.log(conversion);
 
-		function updateProgress() {
+		let progress = 0;
-			progress.value = conversion.progress;
+		conversion.onProgress = newProgress => progress = newProgress;
 
-			if (conversion.progress === 1) {
+		function updateProgress() {
+			progressElement.value = progress;
+
+			if (progress === 1) {
 				return;
 			}
 

dev/mux.html

@@ -115,7 +115,7 @@
 	*/
 
 	let videoSource = new Metamuxer.CanvasSource(canvas, {
-		codec: 'hevc',
+		codec: 'avc',
 		//fullCodecString: 'avc1.42001f',
 		bitrate: 1e6
 	});
@@ -206,5 +206,5 @@ Testing... <00:17.350>One... <00:18.125>Two...
 	await output.finalize();
 
 	console.log(target);
-	download(new Blob([target.buffer]), 'test' + format.fileExtension);
+	//download(new Blob([target.buffer]), 'test' + format.fileExtension);
 </script>

docs/guide/media-sources.md

@@ -59,9 +59,6 @@ type VideoEncodingConfig = {
 		packet: EncodedPacket,
 		meta: EncodedVideoChunkMetadata | undefined
 	) => unknown;
-	onEncoderError?: (
-		error: Error
-	) => unknown;
 	onEncoderConfig?: (
 		config: AudioEncoderConfig
 	) => unknown;
@@ -73,7 +70,6 @@ type VideoEncodingConfig = {
 - `keyFrameInterval`: The maximum interval in seconds between two adjacent key frames. Defaults to 5 seconds. More frequent key frames improve seeking behavior but increase file size. When using multiple video tracks, this value should be set to the same value for all tracks.
 - `fullCodecString`: Allows you to optionally specify the full codec string used by the video encoder, as specified in the [WebCodecs Codec Registry](https://www.w3.org/TR/webcodecs-codec-registry/). For example, you may set it to `'avc1.42001f'` when using AVC. Keep in mind that the codec string must still match the codec specified in `codec`. If you don't set this field, a codec string will be generated automatically.
 - `onEncodedPacket`: Called for each successfully encoded packet. Useful for determining encoding progress.
-- `onEncoderError`: Called when an error occurs within [VideoEncoder](https://developer.mozilla.org/en-US/docs/Web/API/VideoEncoder).
 - `onEncoderConfig`: Called when the internal encoder config, as used by the WebCodecs API, is created. You can use this to introspect the full codec string.
 
 ### Audio encoding config
@@ -89,9 +85,6 @@ type AudioEncodingConfig = {
 		packet: EncodedPacket,
 		meta: EncodedAudioChunkMetadata | undefined
 	) => unknown;
-	onEncoderError?: (
-		error: Error
-	) => unknown;
 	onEncoderConfig?: (
 		config: AudioEncoderConfig
 	) => unknown;
@@ -101,7 +94,6 @@ type AudioEncodingConfig = {
 - `bitrate`: The target number of bits per second. Alternatively, this can be a [subjective quality](#subjective-qualities).
 - `fullCodecString`: Allows you to optionally specify the full codec string used by the audio encoder, as specified in the [WebCodecs Codec Registry](https://www.w3.org/TR/webcodecs-codec-registry/). For example, you may set it to `'mp4a.40.2'` when using AAC. Keep in mind that the codec string must still match the codec specified in `codec`. If you don't set this field, a codec string will be generated automatically.
 - `onEncodedPacket`: Called for each successfully encoded packet. Useful for determining encoding progress.	
-- `onEncoderError`: Called when an error occurs within [AudioEncoder](https://developer.mozilla.org/en-US/docs/Web/API/AudioEncoder).	
 - `onEncoderConfig`: Called when the internal encoder config, as used by the WebCodecs API, is created. You can use this to introspect the full codec string.
 
 ### Subjective qualities

src/codec.ts

@@ -756,7 +756,7 @@ export class Quality {
 	/** @internal */
 	_toAudioBitrate(codec: AudioCodec) {
 		if ((PCM_AUDIO_CODECS as readonly string[]).includes(codec) || codec === 'flac') {
-			return -1;
+			return undefined;
 		}
 
 		const baseRates = {

src/conversion.ts

@@ -1,8 +1,8 @@
 import {
 	AUDIO_CODECS,
 	AudioCodec,
+	getFirstEncodableVideoCodec,
 	getEncodableAudioCodecs,
-	getEncodableVideoCodecs,
 	NON_PCM_AUDIO_CODECS,
 	Quality,
 	QUALITY_HIGH,
@@ -520,8 +520,8 @@ export class Conversion {
 
 			const bitrate = this._options.video?.bitrate ?? QUALITY_HIGH;
 
-			const encodableCodecs = await getEncodableVideoCodecs(videoCodecs, { width, height, bitrate });
+			const encodableCodec = await getFirstEncodableVideoCodec(videoCodecs, { width, height, bitrate });
-			if (encodableCodecs.length === 0) {
+			if (!encodableCodec) {
 				this.discardedTracks.push({
 					track,
 					reason: 'no_encodable_target_codec',
@@ -530,7 +530,7 @@ export class Conversion {
 			}
 
 			const encodingConfig: VideoEncodingConfig = {
-				codec: encodableCodecs[0]!,
+				codec: encodableCodec,
 				bitrate,
 				onEncodedPacket: sample => this._reportProgress(track.id, sample.timestamp + sample.duration),
 			};
@@ -714,13 +714,12 @@ export class Conversion {
 					bitrate,
 				});
 
-				if (
+				const nonPcmCodec = encodableCodecsWithDefaultParams
-					encodableCodecsWithDefaultParams
+					.find(codec => (NON_PCM_AUDIO_CODECS as readonly string[]).includes(codec));
-						.some(codec => (NON_PCM_AUDIO_CODECS as readonly string[]).includes(codec))
+				if (nonPcmCodec) {
-				) {
 					// We are able to encode using a non-PCM codec, but it'll require resampling
 					needsResample = true;
-					codecOfChoice = encodableCodecsWithDefaultParams[0]!;
+					codecOfChoice = nonPcmCodec;
 					numberOfChannels = FALLBACK_NUMBER_OF_CHANNELS;
 					sampleRate = FALLBACK_SAMPLE_RATE;
 				}

src/media-source.ts

@@ -201,8 +201,6 @@ export type VideoEncodingConfig = {
 
 	/** Called for each successfully encoded packet. Both the packet and the encoding metadata are passed. */
 	onEncodedPacket?: (packet: EncodedPacket, meta: EncodedVideoChunkMetadata | undefined) => unknown;
-	/** Called when an error occurs within VideoEncoder. */
-	onEncoderError?: (error: Error) => unknown;
 	/** Called when the internal encoder config, as used by the WebCodecs API, is created. */
 	onEncoderConfig?: (config: VideoEncoderConfig) => unknown;
 };
@@ -238,9 +236,6 @@ const validateVideoEncodingConfig = (config: VideoEncodingConfig) => {
 	if (config.onEncodedPacket !== undefined && typeof config.onEncodedPacket !== 'function') {
 		throw new TypeError('config.onEncodedChunk, when provided, must be a function.');
 	}
-	if (config.onEncoderError !== undefined && typeof config.onEncoderError !== 'function') {
-		throw new TypeError('config.onEncodingError, when provided, must be a function.');
-	}
 	if (config.onEncoderConfig !== undefined && typeof config.onEncoderConfig !== 'function') {
 		throw new TypeError('config.onEncoderConfig, when provided, must be a function.');
 	}
@@ -259,9 +254,17 @@ class VideoEncoderWrapper {
 	private customEncoderCallSerializer = new CallSerializer();
 	private customEncoderQueueSize = 0;
 
+	/**
+	 * Encoders typically throw their errors "out of band", meaning asynchronously in some other execution context.
+	 * However, we want to surface these errors to the user within the normal control flow, so they don't go uncaught.
+	 * So, we keep track of the encoder error and throw it as soon as we get the chance.
+	 */
+	private encoderError: Error | null = null;
+
 	constructor(private source: VideoSource, private encodingConfig: VideoEncodingConfig) {}
 
 	async add(videoSample: VideoSample, shouldClose: boolean, encodeOptions?: VideoEncoderEncodeOptions) {
+		this.checkForEncoderError();
 		this.source._ensureValidAdd();
 
 		// Ensure video sample size remains constant
@@ -316,6 +319,9 @@ class VideoEncoderWrapper {
 					if (shouldClose) {
 						videoSample.close();
 					}
+				})
+				.catch((error: Error) => {
+					this.encoderError ??= error;
 				});
 
 			if (this.customEncoderQueueSize >= 4) {
@@ -411,7 +417,9 @@ class VideoEncoderWrapper {
 						this.encodingConfig.onEncodedPacket?.(packet, meta);
 						void this.muxer!.addEncodedVideoPacket(this.source._connectedTrack!, packet, meta);
 					},
-					error: this.encodingConfig.onEncoderError ?? (error => console.error('VideoEncoder error:', error)),
+					error: (error) => {
+						this.encoderError ??= error;
+					},
 				});
 				this.encoder.configure(encoderConfig);
 			}
@@ -424,6 +432,8 @@ class VideoEncoderWrapper {
 	}
 
 	async flushAndClose() {
+		this.checkForEncoderError();
+
 		if (this.customEncoder) {
 			void this.customEncoderCallSerializer.call(() => this.customEncoder!.flush());
 			await this.customEncoderCallSerializer.call(() => this.customEncoder!.close());
@@ -431,6 +441,8 @@ class VideoEncoderWrapper {
 			await this.encoder.flush();
 			this.encoder.close();
 		}
+
+		this.checkForEncoderError();
 	}
 
 	getQueueSize() {
@@ -440,6 +452,13 @@ class VideoEncoderWrapper {
 			return this.encoder?.encodeQueueSize ?? 0;
 		}
 	}
+
+	checkForEncoderError() {
+		if (this.encoderError) {
+			this.encoderError.stack = new Error().stack; // Provide a more useful stack trace
+			throw this.encoderError;
+		}
+	}
 }
 
 /**
@@ -581,7 +600,11 @@ export class MediaStreamVideoTrackSource extends VideoSource {
 					return;
 				}
 
-				void this._encoder.add(new VideoSample(videoFrame), true);
+				void this._encoder.add(new VideoSample(videoFrame), true)
+					.catch((error) => {
+						this._abortController?.abort();
+						throw error;
+					});
 			},
 		});
 
@@ -681,8 +704,6 @@ export type AudioEncodingConfig = {
 
 	/** Called for each successfully encoded packet. Both the packet and the encoding metadata are passed. */
 	onEncodedPacket?: (packet: EncodedPacket, meta: EncodedAudioChunkMetadata | undefined) => unknown;
-	/** Called when an error occurs within AudioEncoder. */
-	onEncoderError?: (error: Error) => unknown;
 	/** Called when the internal encoder config, as used by the WebCodecs API, is created. */
 	onEncoderConfig?: (config: AudioEncoderConfig) => unknown;
 };
@@ -719,9 +740,6 @@ const validateAudioEncodingConfig = (config: AudioEncodingConfig) => {
 	if (config.onEncodedPacket !== undefined && typeof config.onEncodedPacket !== 'function') {
 		throw new TypeError('config.onEncodedChunk, when provided, must be a function.');
 	}
-	if (config.onEncoderError !== undefined && typeof config.onEncoderError !== 'function') {
-		throw new TypeError('config.onEncodingError, when provided, must be a function.');
-	}
 	if (config.onEncoderConfig !== undefined && typeof config.onEncoderConfig !== 'function') {
 		throw new TypeError('config.onEncoderConfig, when provided, must be a function.');
 	}
@@ -743,9 +761,17 @@ class AudioEncoderWrapper {
 	private customEncoderCallSerializer = new CallSerializer();
 	private customEncoderQueueSize = 0;
 
+	/**
+	 * Encoders typically throw their errors "out of band", meaning asynchronously in some other execution context.
+	 * However, we want to surface these errors to the user within the normal control flow, so they don't go uncaught.
+	 * So, we keep track of the encoder error and throw it as soon as we get the chance.
+	 */
+	private encoderError: Error | null = null;
+
 	constructor(private source: AudioSource, private encodingConfig: AudioEncodingConfig) {}
 
 	async add(audioSample: AudioSample, shouldClose: boolean) {
+		this.checkForEncoderError();
 		this.source._ensureValidAdd();
 
 		// Ensure audio parameters remain constant
@@ -790,6 +816,9 @@ class AudioEncoderWrapper {
 					if (shouldClose) {
 						audioSample.close();
 					}
+				})
+				.catch((error: Error) => {
+					this.encoderError ??= error;
 				});
 
 			if (this.customEncoderQueueSize >= 4) {
@@ -957,7 +986,9 @@ class AudioEncoderWrapper {
 						this.encodingConfig.onEncodedPacket?.(packet, meta);
 						void this.muxer!.addEncodedAudioPacket(this.source._connectedTrack!, packet, meta);
 					},
-					error: this.encodingConfig.onEncoderError ?? (error => console.error('AudioEncoder error:', error)),
+					error: (error) => {
+						this.encoderError ??= error;
+					},
 				});
 				this.encoder.configure(encoderConfig);
 			}
@@ -1051,6 +1082,8 @@ class AudioEncoderWrapper {
 	}
 
 	async flushAndClose() {
+		this.checkForEncoderError();
+
 		if (this.customEncoder) {
 			void this.customEncoderCallSerializer.call(() => this.customEncoder!.flush());
 			await this.customEncoderCallSerializer.call(() => this.customEncoder!.close());
@@ -1058,6 +1091,8 @@ class AudioEncoderWrapper {
 			await this.encoder.flush();
 			this.encoder.close();
 		}
+
+		this.checkForEncoderError();
 	}
 
 	getQueueSize() {
@@ -1069,6 +1104,13 @@ class AudioEncoderWrapper {
 			return this.encoder?.encodeQueueSize ?? 0;
 		}
 	}
+
+	checkForEncoderError() {
+		if (this.encoderError) {
+			this.encoderError.stack = new Error().stack; // Provide a more useful stack trace
+			throw this.encoderError;
+		}
+	}
 }
 
 /**
@@ -1234,7 +1276,11 @@ export class MediaStreamAudioTrackSource extends AudioSource {
 					return;
 				}
 
-				void this._encoder.add(new AudioSample(audioData), true);
+				void this._encoder.add(new AudioSample(audioData), true)
+					.catch((error) => {
+						this._abortController?.abort();
+						throw error;
+					});
 			},
 		});