mediabunny/examples/media-player/media-player.ts

import {
	ALL_FORMATS,
	AudioBufferSink,
	BlobSource,
	CanvasSink,
	Input,
	WrappedAudioBuffer,
	WrappedCanvas,
} from 'mediabunny';

const selectMediaButton = document.querySelector('button') as HTMLButtonElement;
const fileNameElement = document.querySelector('#file-name') as HTMLParagraphElement;
const horizontalRule = document.querySelector('hr') as HTMLHRElement;
const playerContainer = document.querySelector('#player') as HTMLDivElement;
const canvas = document.querySelector('canvas') as HTMLCanvasElement;
const controlsElement = document.querySelector('#controls') as HTMLDivElement;
const playButton = document.querySelector('#play-button') as HTMLButtonElement;
const playIcon = document.querySelector('#play-icon') as HTMLSpanElement;
const pauseIcon = document.querySelector('#pause-icon') as HTMLSpanElement;
const currentTimeElement = document.querySelector('#current-time') as HTMLSpanElement;
const durationElement = document.querySelector('#duration') as HTMLSpanElement;
const progressBarContainer = document.querySelector('#progress-bar-container') as HTMLDivElement;
const progressBar = document.querySelector('#progress-bar') as HTMLDivElement;
const volumeBarContainer = document.querySelector('#volume-bar-container') as HTMLDivElement;
const volumeBar = document.querySelector('#volume-bar') as HTMLDivElement;
const volumeIconWrapper = document.querySelector('#volume-icon-wrapper') as HTMLDivElement;
const volumeButton = document.querySelector('#volume-button') as HTMLButtonElement;
const fullscreenButton = document.querySelector('#fullscreen-button') as HTMLButtonElement;
const errorElement = document.querySelector('#error-element') as HTMLDivElement;

const context = canvas.getContext('2d', { alpha: false, desynchronized: true })!;

let audioContext: AudioContext | null = null;
let gainNode: GainNode | null = null;

let fileLoaded = false;
let videoSink: CanvasSink | null = null;
let audioSink: AudioBufferSink | null = null;

let totalDuration = 0;
/** The value of the audio context's currentTime the moment the playback was started. */
let audioContextStartTime: number | null = null;
let playing = false;
/** The timestamp within the media file when the playback was started. */
let playbackTimeAtStart = 0;

let videoFrameIterator: AsyncGenerator<WrappedCanvas, void, unknown> | null = null;
let audioBufferIterator: AsyncGenerator<WrappedAudioBuffer, void, unknown> | null = null;
let nextFrame: WrappedCanvas | null = null;
const queuedAudioNodes: Set<AudioBufferSourceNode> = new Set();

/**
 * Used to prevent async race conditions. When seekId is incremented, already-running async functions will be prevented
 * from having an effect.
 */
let asyncId = 0;

let draggingProgressBar = false;
let volume = 0.7;
let draggingVolumeBar = false;
let volumeMuted = false;

/** === INIT LOGIC === */

const initMediaPlayer = async (file: File) => {
	try {
		// First, dispose any ongoing playback:

		if (playing) {
			pause();
		}

		void videoFrameIterator?.return();
		void audioBufferIterator?.return();
		asyncId++;

		fileLoaded = false;
		fileNameElement.textContent = file.name;
		horizontalRule.style.display = '';
		playerContainer.style.display = 'none';

		// Create an Input from the file
		const input = new Input({
			source: new BlobSource(file),
			formats: ALL_FORMATS,
		});

		playbackTimeAtStart = 0;
		totalDuration = await input.computeDuration();
		durationElement.textContent = formatSeconds(totalDuration);

		let videoTrack = await input.getPrimaryVideoTrack();
		let audioTrack = await input.getPrimaryAudioTrack();

		if (!(await videoTrack?.canDecode())) {
			// We can't decode the video track, so treat it like there is no video track
			videoTrack = null;
		}
		if (!(await audioTrack?.canDecode())) {
			// We can't decode the audio track, so treat it like there is no audio track
			audioTrack = null;
		}

		if (!videoTrack && !audioTrack) {
			throw new Error('Media file has no playable video or audio track.');
		}

		// We must create the audio context with the matching sample rate for correct acoustic results
		// (especially for low-sample rate files)
		audioContext = new AudioContext({ sampleRate: audioTrack?.sampleRate });
		gainNode = audioContext.createGain();
		gainNode.connect(audioContext.destination);
		updateVolume();

		// For video, let's use a CanvasSink as it handles rotation and closing video samples for us.
		// Pool size of 2: We'll only ever have the current and the next frame around, so we only need two canvases.
		videoSink = videoTrack && new CanvasSink(videoTrack, { poolSize: 2 });
		// For audio, we'll use an AudioBufferSink to directly retrieve AudioBuffers compatible with the Web Audio API
		audioSink = audioTrack && new AudioBufferSink(audioTrack);

		// Show the canvas if there's a video track, otherwise hide it
		if (videoTrack) {
			canvas.style.display = '';
			canvas.width = videoTrack.displayWidth;
			canvas.height = videoTrack.displayHeight;
		} else {
			canvas.style.display = 'none';
		}

		// Show volume controls if there's an audio track, otherwise hide them
		if (audioTrack) {
			volumeButton.style.display = '';
			volumeBarContainer.style.display = '';
		} else {
			volumeButton.style.display = 'none';
			volumeBarContainer.style.display = 'none';
		}

		fileLoaded = true;

		await startVideoIterator();
		await play();

		playerContainer.style.display = '';

		if (!videoSink) {
			// If there's only an audio track, always show the controls
			controlsElement.style.opacity = '1';
			playerContainer.style.cursor = '';
		}
	} catch (e) {
		errorElement.textContent = String(e);
		playerContainer.style.display = 'none';
	}
};

/** === VIDEO RENDERING LOGIC === */

/** Creates a new video frame iterator and renders the first video frame. */
const startVideoIterator = async () => {
	if (!videoSink) {
		return;
	}

	asyncId++;

	await videoFrameIterator?.return(); // Dispose of the current iterator

	// Create a new iterator
	videoFrameIterator = videoSink.canvases(getPlaybackTime());

	// Get the first two frames
	const firstFrame = (await videoFrameIterator.next()).value ?? null;
	const secondFrame = (await videoFrameIterator.next()).value ?? null;

	nextFrame = secondFrame;

	if (firstFrame) {
		// Draw the first frame
		context.drawImage(firstFrame.canvas, 0, 0);
	}
};

/** Runs every frame; updates the canvas if necessary. */
const render = (requestFrame = true) => {
	if (fileLoaded) {
		const playbackTime = getPlaybackTime();
		if (playbackTime >= totalDuration) {
			// Pause playback once the end is reached
			pause();
			playbackTimeAtStart = totalDuration;
		}

		// Check if the current playback time has caught up to the next frame
		if (nextFrame && nextFrame.timestamp <= playbackTime) {
			context.drawImage(nextFrame.canvas, 0, 0);
			nextFrame = null;

			// Request the next frame
			void updateNextFrame();
		}

		if (!draggingProgressBar) {
			updateProgressBarTime(playbackTime);
		}
	}

	if (requestFrame) {
		requestAnimationFrame(() => render());
	}
};
render();

// Also call the render function on an interval to make sure the video keeps updating even if the tab isn't visible
setInterval(() => render(false), 500);

/** Iterates over the video frame iterator until it finds a video frame in the future. */
const updateNextFrame = async () => {
	const currentAsyncId = asyncId;

	// We have a loop here because we may need to iterate over multiple frames until we reach a frame in the future
	while (true) {
		const newNextFrame = (await videoFrameIterator!.next()).value ?? null;
		if (!newNextFrame) {
			break;
		}

		if (currentAsyncId !== asyncId) {
			break;
		}

		const playbackTime = getPlaybackTime();
		if (newNextFrame.timestamp <= playbackTime) {
			// Draw it immediately
			context.drawImage(newNextFrame.canvas, 0, 0);
		} else {
			// Save it for later
			nextFrame = newNextFrame;
			break;
		}
	}
};

/** === AUDIO PLAYBACK LOGIC === */

/** Loops over the audio buffer iterator, scheduling the audio to be played in the audio context. */
const runAudioIterator = async () => {
	if (!audioSink) {
		return;
	}

	// To play back audio, we loop over all audio chunks (typically very short) of the file and play them at the correct
	// timestamp. The result is a continuous, uninteruppted audio signal.
	for await (const { buffer, timestamp } of audioBufferIterator!) {
		const node = audioContext!.createBufferSource();
		node.buffer = buffer;
		node.connect(gainNode!);

		const startTimestamp = audioContextStartTime! + timestamp - playbackTimeAtStart;

		// Two cases: Either, the audio starts in the future or in the past
		if (startTimestamp >= audioContext!.currentTime) {
			// If the audio starts in the future, easy, we just schedule it
			node.start(startTimestamp);
		} else {
			// If it starts in the past, then let's only play the audible section that remains from here on out
			node.start(audioContext!.currentTime, audioContext!.currentTime - startTimestamp);
		}

		queuedAudioNodes.add(node);
		node.onended = () => {
			queuedAudioNodes.delete(node);
		};

		// If we're more than a second ahead of the current playback time, let's slow down the loop until time has
		// passed.
		if (timestamp - getPlaybackTime() >= 1) {
			await new Promise<void>((resolve) => {
				const id = setInterval(() => {
					if (timestamp - getPlaybackTime() < 1) {
						clearInterval(id);
						resolve();
					}
				}, 100);
			});
		}
	}
};

/** === PLAYBACK CONTROL LOGIC === */

/** Returns the current playback time in the media file. */
const getPlaybackTime = () => {
	if (playing) {
		// To ensure perfect audio-video sync, we always use the audio context's clock to determine playback time, even
		// when there is no audio track.
		return audioContext!.currentTime - audioContextStartTime! + playbackTimeAtStart;
	} else {
		return playbackTimeAtStart;
	}
};

const play = async () => {
	if (getPlaybackTime() === totalDuration) {
		// If we're at the end, let's snap back to the start
		playbackTimeAtStart = 0;
		await startVideoIterator();
	}

	audioContextStartTime = audioContext!.currentTime;
	playing = true;

	if (audioSink) {
		// Start the audio iterator
		void audioBufferIterator?.return();
		audioBufferIterator = audioSink?.buffers(getPlaybackTime());
		void runAudioIterator();
	}

	playIcon.style.display = 'none';
	pauseIcon.style.display = '';
};

const pause = () => {
	playbackTimeAtStart = getPlaybackTime();
	playing = false;
	void audioBufferIterator?.return(); // This stops any for-loops that are iterating the iterator
	audioBufferIterator = null;

	// Stop all audio nodes that were already queued to play
	for (const node of queuedAudioNodes) {
		node.stop();
	}
	queuedAudioNodes.clear();

	playIcon.style.display = '';
	pauseIcon.style.display = 'none';
};

const togglePlay = () => {
	if (playing) {
		pause();
	} else {
		void play();
	}
};

const seekToTime = async (seconds: number) => {
	updateProgressBarTime(seconds);

	const wasPlaying = playing;

	if (wasPlaying) {
		pause();
	}

	playbackTimeAtStart = seconds;

	await startVideoIterator();

	if (wasPlaying && playbackTimeAtStart < totalDuration) {
		void play();
	}
};

/** === PROGRESS BAR LOGIC === */

const updateProgressBarTime = (seconds: number) => {
	currentTimeElement.textContent = formatSeconds(seconds);
	progressBar.style.width = `${(seconds / totalDuration) * 100}%`;
};

progressBarContainer.addEventListener('pointerdown', (event) => {
	draggingProgressBar = true;

	const rect = progressBarContainer.getBoundingClientRect();
	const completion = Math.max(Math.min((event.clientX - rect.left) / rect.width, 1), 0);
	updateProgressBarTime(completion * totalDuration);

	window.addEventListener('pointerup', (event) => {
		draggingProgressBar = false;

		const rect = progressBarContainer.getBoundingClientRect();
		const completion = Math.max(Math.min((event.clientX - rect.left) / rect.width, 1), 0);
		const newTime = completion * totalDuration;

		void seekToTime(newTime);
	}, { once: true });
});

window.addEventListener('pointermove', (event) => {
	if (draggingProgressBar) {
		const rect = progressBarContainer.getBoundingClientRect();
		const completion = Math.max(Math.min((event.clientX - rect.left) / rect.width, 1), 0);
		updateProgressBarTime(completion * totalDuration);
	}
});

/** === VOLUME CONTROL LOGIC === */

const updateVolume = () => {
	const actualVolume = volumeMuted ? 0 : volume;

	volumeBar.style.width = `${actualVolume * 100}%`;
	gainNode!.gain.value = actualVolume ** 2; // Quadratic for more fine-grained control

	const iconNumber = volumeMuted ? 0 : Math.ceil(1 + 3 * volume);
	for (let i = 0; i < volumeIconWrapper.children.length; i++) {
		const icon = volumeIconWrapper.children[i] as HTMLImageElement;
		icon.style.display = i === iconNumber ? '' : 'none';
	}
};

volumeBarContainer.addEventListener('pointerdown', (event) => {
	draggingVolumeBar = true;

	const rect = volumeBarContainer.getBoundingClientRect();
	volume = Math.max(Math.min((event.clientX - rect.left) / rect.width, 1), 0);
	volumeMuted = false;
	updateVolume();

	window.addEventListener('pointerup', (event) => {
		draggingVolumeBar = false;

		const rect = volumeBarContainer.getBoundingClientRect();
		volume = Math.max(Math.min((event.clientX - rect.left) / rect.width, 1), 0);
		updateVolume();
	}, { once: true });
});

volumeButton.addEventListener('click', () => {
	volumeMuted = !volumeMuted;
	updateVolume();
});

window.addEventListener('pointermove', (event) => {
	if (draggingVolumeBar) {
		const rect = volumeBarContainer.getBoundingClientRect();
		volume = Math.max(Math.min((event.clientX - rect.left) / rect.width, 1), 0);
		updateVolume();
	}
});

/** === CONTROL UI LOGIC === */

const showControlsTemporarily = () => {
	if (!videoSink) {
		// Shouldn't run if there's only an audio track
		return;
	}

	controlsElement.style.opacity = '1';
	playerContainer.style.cursor = '';

	clearTimeout(hideControlsTimeout);
	hideControlsTimeout = window.setTimeout(() => {
		controlsElement.style.opacity = '0';
		playerContainer.style.cursor = 'none';
	}, 2000);
};

let hideControlsTimeout = -1;
playerContainer.addEventListener('pointermove', () => {
	showControlsTemporarily();
});
playerContainer.addEventListener('pointerleave', () => {
	if (!videoSink) {
		// Shouldn't run if there's only an audio track
		return;
	}

	controlsElement.style.opacity = '0';
	clearTimeout(hideControlsTimeout);
});

/** === EVENT LISTENERS === */

playButton.addEventListener('click', togglePlay);
window.addEventListener('keydown', (e) => {
	if (!fileLoaded) {
		return;
	}

	if (e.code === 'Space' || e.code === 'KeyK') {
		togglePlay();
	} else if (e.code === 'KeyF') {
		fullscreenButton.click();
	} else if (e.code === 'ArrowLeft') {
		const newTime = Math.max(getPlaybackTime() - 5, 0);
		void seekToTime(newTime);
	} else if (e.code === 'ArrowRight') {
		const newTime = Math.min(getPlaybackTime() + 5, totalDuration);
		void seekToTime(newTime);
	} else if (e.code === 'KeyM') {
		volumeButton.click();
	} else {
		return;
	}

	showControlsTemporarily();
	e.preventDefault();
});

fullscreenButton.addEventListener('click', () => {
	if (document.fullscreenElement) {
		void document.exitFullscreen();
	} else {
		playerContainer.requestFullscreen().catch((e) => {
			console.error('Failed to enter fullscreen mode:', e);
		});
	}
});

playerContainer.addEventListener('click', () => {
	togglePlay();
});
controlsElement.addEventListener('click', (event) => {
	// Make sure this does NOT toggle play
	event.stopPropagation();
});

/** === UTILS === */

const formatSeconds = (seconds: number) => {
	seconds = Math.round(seconds * 1000) / 1000; // Round to milliseconds

	const hours = Math.floor(seconds / 3600);
	const minutes = Math.floor((seconds % 3600) / 60);
	const remainingSeconds = Math.floor(seconds % 60);
	const millisecs = Math.floor(1000 * seconds % 1000).toString().padStart(3, '0');

	if (hours > 0) {
		return `${hours}:${minutes.toString().padStart(2, '0')}`
			+ `:${remainingSeconds.toString().padStart(2, '0')}.${millisecs}`;
	}
	return `${minutes.toString().padStart(2, '0')}:${remainingSeconds.toString().padStart(2, '0')}.${millisecs}`;
};

/** === FILE SELECTION LOGIC === */

selectMediaButton.addEventListener('click', () => {
	const fileInput = document.createElement('input');
	fileInput.type = 'file';
	fileInput.addEventListener('change', () => {
		const file = fileInput.files?.[0];
		if (!file) {
			return;
		}

		void initMediaPlayer(file);
	});

	fileInput.click();
});

document.addEventListener('dragover', (event) => {
	event.preventDefault();
	event.dataTransfer!.dropEffect = 'copy';
});

document.addEventListener('drop', (event) => {
	event.preventDefault();
	const files = event.dataTransfer?.files;
	const file = files && files.length > 0 ? files[0] : undefined;
	if (file) {
		void initMediaPlayer(file);
	}
});