Probe encoder support with sample (fixes #90), improve encoder error handling

dev/convert.html

@@ -18,15 +18,13 @@
 		const file = fileInput.files[0];
 
 		const source = new Mediabunny.BlobSource(file);
-		const target = new Mediabunny.NullTarget() ?? new Mediabunny.BufferTarget() ?? new Mediabunny.StreamTarget(new WritableStream({
+		const target = new Mediabunny.BufferTarget() ?? new Mediabunny.StreamTarget(new WritableStream({
 			write: console.log
 		}), {
 			chunked: true,
 			chunkSize: 2**20
 		});
-		const outputFormat = new Mediabunny.Mp4OutputFormat({
+		const outputFormat = new Mediabunny.Mp4OutputFormat({});
-			onMoov: console.log
-		});
 
 		const button = document.createElement('button');
 		button.textContent = 'Cancel';
@@ -76,7 +74,8 @@
 			},
 			*/
 			video: () => ({
-				//codec: 'avc',
+				forceTranscode: true,
+				codec: 'avc',
 				//fit: 'contain',
 				//frameRate: 27.123,
 				//width: 320,
@@ -98,8 +97,8 @@
 				//height: 100,
 			}),
 			trim: {
-				//start: 0,
+				start: 0,
-				//end: 20
+				end: 5
 			},
 		});
 		console.log(conversion);

src/conversion.ts

@@ -46,7 +46,9 @@ import {
 	Rotation,
 } from './misc';
 import { Output, TrackType } from './output';
+import { Mp4OutputFormat } from './output-format';
 import { AudioSample, VideoSample } from './sample';
+import { NullTarget } from './target';
 
 /**
  * The options for media file conversion.
@@ -582,7 +584,7 @@ export class Conversion {
 			|| this._startTimestamp > 0
 			|| firstTimestamp < 0
 			|| !!trackOptions.frameRate;
-		const needsRerender = width !== originalWidth
+		let needsRerender = width !== originalWidth
 			|| height !== originalHeight
 			|| (totalRotation !== 0 && !outputSupportsRotation);
 
@@ -662,6 +664,43 @@ export class Conversion {
 			const source = new VideoSampleSource(encodingConfig);
 			videoSource = source;
 
+			if (!needsRerender) {
+				// If we're directly passing decoded samples back to the encoder, sometimes the encoder may error due
+				// to lack of support of certain video frame formats, like when HDR is at play. To check for this, we
+				// first try to pass a single frame to the encoder to see how it behaves. If it throws, we then fall
+				// back to the rerender path.
+				//
+				// Creating a new temporary Output is sort of hacky, but due to a lack of an isolated encoder API right
+				// now, this is the simplest way. Will refactor in the future!
+
+				const tempOutput = new Output({
+					format: new Mp4OutputFormat(), // Supports all video codecs
+					target: new NullTarget(),
+				});
+
+				const tempSource = new VideoSampleSource(encodingConfig);
+				tempOutput.addVideoTrack(tempSource);
+
+				await tempOutput.start();
+
+				const sink = new VideoSampleSink(track);
+				const firstSample = await sink.getSample(this._startTimestamp);
+
+				if (firstSample) {
+					try {
+						await tempSource.add(firstSample);
+						firstSample.close();
+						await tempOutput.finalize();
+					} catch (error) {
+						console.info('Error when probing encoder support. Falling back to rerender path.', error);
+						needsRerender = true;
+						void tempOutput.cancel();
+					}
+				} else {
+					await tempOutput.cancel();
+				}
+			}
+
 			if (needsRerender) {
 				this._trackPromises.push((async () => {
 					await this._started;
@@ -1084,13 +1123,8 @@ export class Conversion {
 
 		this._maxTimestamps.set(trackId, Math.max(endTimestamp, this._maxTimestamps.get(trackId) ?? -Infinity));
 
-		let totalTimestamps = 0;
+		const minTimestamp = Math.min(...this._maxTimestamps.values());
-		for (const [, timestamp] of this._maxTimestamps) {
+		const newProgress = clamp(minTimestamp / this._totalDuration, 0, 1);
-			totalTimestamps += timestamp;
-		}
-
-		const averageTimestamp = totalTimestamps / this._totalTrackCount;
-		const newProgress = clamp(averageTimestamp / this._totalDuration, 0, 1);
 
 		if (newProgress !== this._lastProgress) {
 			this._lastProgress = newProgress;

src/media-source.ts

@@ -353,6 +353,7 @@ class VideoEncoderWrapper {
 			return;
 		}
 
+		const encoderError = new Error();
 		return this.ensureEncoderPromise = (async () => {
 			const encoderConfig = buildVideoEncoderConfig({
 				width: videoSample.codedWidth,
@@ -411,7 +412,7 @@ class VideoEncoderWrapper {
 						void this.muxer!.addEncodedVideoPacket(this.source._connectedTrack!, packet, meta);
 					},
 					error: (error) => {
-						error.stack = new Error().stack; // Provide a more useful stack trace
+						error.stack = encoderError.stack; // Provide a more useful stack trace
 						this.encoderError ??= error;
 					},
 				});
@@ -426,7 +427,7 @@ class VideoEncoderWrapper {
 	}
 
 	async flushAndClose(forceClose: boolean) {
-		this.checkForEncoderError();
+		if (!forceClose) this.checkForEncoderError();
 
 		if (this.customEncoder) {
 			if (!forceClose) {
@@ -439,10 +440,12 @@ class VideoEncoderWrapper {
 				await this.encoder.flush();
 			}
 
-			this.encoder.close();
+			if (this.encoder.state !== 'closed') {
+				this.encoder.close();
+			}
 		}
 
-		this.checkForEncoderError();
+		if (!forceClose) this.checkForEncoderError();
 	}
 
 	getQueueSize() {
@@ -455,6 +458,7 @@ class VideoEncoderWrapper {
 
 	checkForEncoderError() {
 		if (this.encoderError) {
+			this.encoderError.stack = new Error().stack; // Provide an even more useful stack trace
 			throw this.encoderError;
 		}
 	}
@@ -990,6 +994,7 @@ class AudioEncoderWrapper {
 			return;
 		}
 
+		const encoderError = new Error();
 		return this.ensureEncoderPromise = (async () => {
 			const { numberOfChannels, sampleRate } = audioSample;
 
@@ -1050,7 +1055,7 @@ class AudioEncoderWrapper {
 						void this.muxer!.addEncodedAudioPacket(this.source._connectedTrack!, packet, meta);
 					},
 					error: (error) => {
-						error.stack = new Error().stack; // Provide a more useful stack trace
+						error.stack = encoderError.stack; // Provide a more useful stack trace
 						this.encoderError ??= error;
 					},
 				});
@@ -1158,7 +1163,7 @@ class AudioEncoderWrapper {
 	}
 
 	async flushAndClose(forceClose: boolean) {
-		this.checkForEncoderError();
+		if (!forceClose) this.checkForEncoderError();
 
 		if (this.customEncoder) {
 			if (!forceClose) {
@@ -1171,10 +1176,12 @@ class AudioEncoderWrapper {
 				await this.encoder.flush();
 			}
 
-			this.encoder.close();
+			if (this.encoder.state !== 'closed') {
+				this.encoder.close();
+			}
 		}
 
-		this.checkForEncoderError();
+		if (!forceClose) this.checkForEncoderError();
 	}
 
 	getQueueSize() {
@@ -1189,6 +1196,7 @@ class AudioEncoderWrapper {
 
 	checkForEncoderError() {
 		if (this.encoderError) {
+			this.encoderError.stack = new Error().stack; // Provide an even more useful stack trace
 			throw this.encoderError;
 		}
 	}