Big realtime playback refactor

- Added Worker- or AudioContext-based fallbacks in case MediaStreamTrackProcessor is not available
- Fixed fMP4-streamed files not playing in Safari
- Added better error handling for MediaStream sources
- Improved the live recording example to be more robust

examples/file-compression/file-compression.ts

@@ -98,6 +98,10 @@ const compressFile = async (file: File) => {
 		compressionFacts.textContent
 			= `${(output.target.buffer!.byteLength / file.size * 100).toPrecision(3)}% of original size`;
 	} catch (error) {
+		console.error(error);
+
+		await currentConversion?.cancel();
+
 		errorElement.textContent = String(error);
 		clearInterval(currentIntervalId);
 

examples/live-recording/index.html

@@ -22,6 +22,7 @@
 		<hr class="w-full max-w-96 my-4 border-zinc-300 dark:border-zinc-700" style="display: none;">
 
 		<p id="error-element" class="text-red-500"></p>
+		<p id="warning-element" class="text-amber-500"></p>
 
 		<div class="flex gap-4" id="main-container" style="display: none;">
 			<div class="flex flex-col items-center">

examples/live-recording/live-recording.ts

@@ -1,4 +1,5 @@
 import {
+	canEncodeAudio,
 	CanvasSource,
 	MediaStreamAudioTrackSource,
 	Mp4OutputFormat,
@@ -13,6 +14,7 @@ const mainContainer = document.querySelector('#main-container') as HTMLDivElemen
 const videoElement = document.querySelector('video') as HTMLVideoElement;
 const downloadButton = document.querySelector('#download-button') as HTMLAnchorElement;
 const errorElement = document.querySelector('#error-element') as HTMLParagraphElement;
+const warningElement = document.querySelector('#warning-element') as HTMLParagraphElement;
 
 const canvas = document.querySelector('canvas') as HTMLCanvasElement;
 const context = canvas.getContext('2d', { alpha: false, desynchronized: true })!;
@@ -39,19 +41,29 @@ const startRecording = async () => {
 		videoElement.src = '';
 		downloadButton.style.display = 'none';
 		errorElement.textContent = '';
+		warningElement.textContent = '';
 
 		// Paint a white background to the canvas
 		context.fillStyle = 'white';
 		context.fillRect(0, 0, canvas.width, canvas.height);
 
-		// Get user microphone
-		mediaStream = await navigator.mediaDevices.getUserMedia({ audio: true });
+		const audioIsEncodable = await canEncodeAudio('opus', {
+			bitrate: QUALITY_MEDIUM,
+		});
+
+		let audioTrack: MediaStreamAudioTrack | null = null;
+		if (audioIsEncodable) {
+			// Get user microphone
+			mediaStream = await navigator.mediaDevices.getUserMedia({ audio: true });
+			audioTrack = mediaStream.getAudioTracks()[0] ?? null;
+		} else {
+			warningElement.textContent
+				= 'Audio is not yet encodable by your browser, so the audio track has been omitted.';
+		}
 
 		horizontalRule.style.display = '';
 		mainContainer.style.display = '';
 
-		const audioTrack = mediaStream.getAudioTracks()[0];
-
 		// Create a new output file
 		output = new Output({
 			// We're using fragmented MP4 here; streamable WebM would also work
@@ -99,6 +111,8 @@ const startRecording = async () => {
 				codec: 'opus',
 				bitrate: QUALITY_MEDIUM,
 			});
+			audioSource.errorPromise.catch(cancelRecording); // Make sure errors are bubbled up
+
 			output.addAudioTrack(audioSource);
 		}
 
@@ -108,9 +122,9 @@ const startRecording = async () => {
 		readyForMoreFrames = true;
 		lastFrameNumber = -1;
 
-		// Start the video frame capture loop
-		void addVideoFrame();
-		videoCaptureInterval = window.setInterval(() => void addVideoFrame(), 1000 / frameRate);
+		// Start the video frame capture loop, making sure errors are caught
+		void addVideoFrame().catch(cancelRecording);
+		videoCaptureInterval = window.setInterval(() => void addVideoFrame().catch(cancelRecording), 1000 / frameRate);
 
 		const mimeType = await output.getMimeType();
 		sourceBuffer = mediaSource.addSourceBuffer(mimeType);
@@ -121,21 +135,35 @@ const startRecording = async () => {
 		toggleRecordingButton.textContent = 'Stop recording';
 		toggleRecordingButton.disabled = false;
 	} catch (error) {
-		errorElement.textContent = String(error);
-
-		mainContainer.style.display = 'none';
-		toggleRecordingButton.textContent = 'Start recording';
-		toggleRecordingButton.disabled = false;
-		recording = false;
+		await cancelRecording(error);
 	}
 };
 
+const cancelRecording = async (error: unknown) => {
+	if (!recording) {
+		return; // Already canceled
+	}
+
+	console.error(error);
+
+	errorElement.textContent = String(error);
+
+	clearInterval(videoCaptureInterval);
+	mainContainer.style.display = 'none';
+	toggleRecordingButton.textContent = 'Start recording';
+	toggleRecordingButton.disabled = false;
+	recording = false;
+	await output?.cancel();
+
+	mediaStream?.getTracks().forEach(track => track.stop());
+};
+
 const stopRecording = async () => {
 	toggleRecordingButton.textContent = 'Stopping...';
 	toggleRecordingButton.disabled = true;
 
 	clearInterval(videoCaptureInterval);
-	mediaStream.getTracks().forEach(track => track.stop());
+	mediaStream?.getTracks().forEach(track => track.stop());
 
 	await output.finalize();
 

examples/media-player/media-player.ts

@@ -175,8 +175,10 @@ const initMediaPlayer = async (file: File) => {
 			controlsElement.style.opacity = '1';
 			playerContainer.style.cursor = '';
 		}
-	} catch (e) {
-		errorElement.textContent = String(e);
+	} catch (error) {
+		console.error(error);
+
+		errorElement.textContent = String(error);
 		playerContainer.style.display = 'none';
 	}
 };

examples/procedural-generation/procedural-generation.ts

@@ -7,6 +7,7 @@ import {
 	QUALITY_HIGH,
 	getFirstEncodableAudioCodec,
 	getFirstEncodableVideoCodec,
+	OutputFormat,
 } from 'mediabunny';
 
 const durationSlider = document.querySelector('#duration-slider') as HTMLInputElement;
@@ -57,6 +58,8 @@ let currentScaleIndex = 0;
 let collisionCount = 0;
 let collisionsPerScale = 0;
 
+let output: Output<OutputFormat, BufferTarget>;
+
 /** === MAIN VIDEO FILE GENERATION LOGIC === */
 
 const generateVideo = async () => {
@@ -82,7 +85,7 @@ const generateVideo = async () => {
 		initScene(duration);
 
 		// Create a new output file
-		const output = new Output({
+		output = new Output({
 			target: new BufferTarget(), // Stored in memory
 			format: new Mp4OutputFormat(),
 		});
@@ -184,6 +187,10 @@ const generateVideo = async () => {
 		const fileSizeMiB = (videoBlob.size / (1024 * 1024)).toPrecision(3);
 		videoInfo.textContent = `File size: ${fileSizeMiB} MiB`;
 	} catch (error) {
+		console.error(error);
+
+		await output?.cancel();
+
 		clearInterval(progressInterval);
 		errorElement.textContent = String(error);
 		progressBarContainer.style.display = 'none';

examples/thumbnail-generation/thumbnail-generation.ts

@@ -96,8 +96,10 @@ const generateThumbnails = async (file: File) => {
 
 			i++;
 		}
-	} catch (e) {
-		errorElement.textContent = String(e);
+	} catch (error) {
+		console.error(error);
+
+		errorElement.textContent = String(error);
 		thumbnailContainer.innerHTML = '';
 	}
 };