mediabunny/examples/media-player/media-player.ts

import {
	ALL_FORMATS,
	AudioBufferSink,
	BlobSource,
	CanvasSink,
	Input,
	UrlSource,
	WrappedAudioBuffer,
	WrappedCanvas,
} from 'mediabunny';

import SampleFileUrl from '../../docs/assets/big-buck-bunny-trimmed.mp4';
(document.querySelector('#sample-file-download') as HTMLAnchorElement).href = SampleFileUrl;

const selectMediaButton = document.querySelector('#select-file') as HTMLButtonElement;
const loadUrlButton = document.querySelector('#load-url') as HTMLButtonElement;
const fileNameElement = document.querySelector('#file-name') as HTMLParagraphElement;
const horizontalRule = document.querySelector('hr') as HTMLHRElement;
const loadingElement = document.querySelector('#loading-element') as HTMLParagraphElement;
const playerContainer = document.querySelector('#player') as HTMLDivElement;
const canvas = document.querySelector('canvas') as HTMLCanvasElement;
const controlsElement = document.querySelector('#controls') as HTMLDivElement;
const playButton = document.querySelector('#play-button') as HTMLButtonElement;
const playIcon = document.querySelector('#play-icon') as HTMLSpanElement;
const pauseIcon = document.querySelector('#pause-icon') as HTMLSpanElement;
const currentTimeElement = document.querySelector('#current-time') as HTMLSpanElement;
const durationElement = document.querySelector('#duration') as HTMLSpanElement;
const progressBarContainer = document.querySelector('#progress-bar-container') as HTMLDivElement;
const progressBar = document.querySelector('#progress-bar') as HTMLDivElement;
const volumeBarContainer = document.querySelector('#volume-bar-container') as HTMLDivElement;
const volumeBar = document.querySelector('#volume-bar') as HTMLDivElement;
const volumeIconWrapper = document.querySelector('#volume-icon-wrapper') as HTMLDivElement;
const volumeButton = document.querySelector('#volume-button') as HTMLButtonElement;
const liveDot = document.querySelector('#live-dot') as HTMLButtonElement;
const fullscreenButton = document.querySelector('#fullscreen-button') as HTMLButtonElement;
const errorElement = document.querySelector('#error-element') as HTMLDivElement;
const warningElement = document.querySelector('#warning-element') as HTMLDivElement;

const context = canvas.getContext('2d')!;

let audioContext: AudioContext | null = null;
let gainNode: GainNode | null = null;

let fileLoaded = false;
let videoSink: CanvasSink | null = null;
let audioSink: AudioBufferSink | null = null;

let firstTimestamp = 0;
let endTimestamp = 0;
let isRelativeToUnixEpoch = false;
/** The value of the audio context's currentTime the moment the playback was started. */
let audioContextStartTime: number | null = null;
let playing = false;
/** The timestamp within the media file when the playback was started. */
let playbackTimeAtStart = 0;

let videoFrameIterator: AsyncGenerator<WrappedCanvas, void, unknown> | null = null;
let audioBufferIterator: AsyncGenerator<WrappedAudioBuffer, void, unknown> | null = null;
let nextFrame: WrappedCanvas | null = null;
const queuedAudioNodes: Set<AudioBufferSourceNode> = new Set();

/**
 * Used to prevent async race conditions. When seekId is incremented, already-running async functions will be prevented
 * from having an effect.
 */
let asyncId = 0;

let liveRefreshIntervalId = -1;

let draggingProgressBar = false;
let volume = 0.7;
let draggingVolumeBar = false;
let volumeMuted = false;

/** === INIT LOGIC === */

const initMediaPlayer = async (resource: File | string) => {
	try {
		// First, dispose any ongoing playback:

		if (playing) {
			pause();
		}

		void videoFrameIterator?.return();
		void audioBufferIterator?.return();

		asyncId++;

		fileLoaded = false;
		fileNameElement.textContent = resource instanceof File ? resource.name : resource;
		horizontalRule.style.display = '';
		loadingElement.style.display = '';
		playerContainer.style.display = 'none';
		errorElement.textContent = '';
		warningElement.textContent = '';
		liveDot.style.display = 'none';
		clearTimeout(liveRefreshIntervalId);

		// Create an Input from the resource
		const input = new Input({
			source: typeof resource === 'string'
				? new UrlSource(resource)
				: new BlobSource(resource),
			formats: ALL_FORMATS, // Accept all formats
		});

		let videoTrack = await input.getPrimaryVideoTrack();
		let audioTrack = await input.getPrimaryAudioTrack();

		const tracks = [videoTrack, audioTrack].filter(t => t !== null);

		firstTimestamp = Math.max(
			await input.getFirstTimestamp(tracks),
			0,
		);
		endTimestamp = await input.getDurationFromMetadata(tracks, { skipLiveWait: true })
			?? await input.computeDuration(tracks, { skipLiveWait: true });
		isRelativeToUnixEpoch = (await Promise.all(tracks.map(t => t.isRelativeToUnixEpoch()))).some(Boolean);
		playbackTimeAtStart = firstTimestamp;

		// Configure the time display elements accordingly
		const timestampFontSize = isRelativeToUnixEpoch ? '12px' : '';
		const timestampWhiteSpace = isRelativeToUnixEpoch ? 'pre' : '';
		const timestampTextAlign = isRelativeToUnixEpoch ? 'center' : '';
		currentTimeElement.style.fontSize = timestampFontSize;
		currentTimeElement.style.whiteSpace = timestampWhiteSpace;
		currentTimeElement.style.textAlign = timestampTextAlign;
		durationElement.style.fontSize = timestampFontSize;
		durationElement.style.whiteSpace = timestampWhiteSpace;
		durationElement.style.textAlign = timestampTextAlign;
		durationElement.textContent = formatTimestamp(endTimestamp);

		let problemMessage = '';

		if (videoTrack) {
			if (await videoTrack.getCodec() === null) {
				problemMessage += 'Unsupported video codec. ';
				videoTrack = null;
			} else if (!(await videoTrack.canDecode())) {
				problemMessage += 'Unable to decode the video track. ';
				videoTrack = null;
			}
		}

		if (audioTrack) {
			if (await audioTrack.getCodec() === null) {
				problemMessage += 'Unsupported audio codec. ';
				audioTrack = null;
			} else if (!(await audioTrack.canDecode())) {
				problemMessage += 'Unable to decode the audio track. ';
				audioTrack = null;
			}
		}

		if (!videoTrack && !audioTrack) {
			if (!problemMessage) {
				problemMessage = 'No audio or video track found.';
			}

			throw new Error(problemMessage);
		}

		if (problemMessage) {
			warningElement.textContent = problemMessage;
		}

		// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access
		const AudioContext = window.AudioContext || (window as any).webkitAudioContext;

		// We must create the audio context with the matching sample rate for correct acoustic results
		// (especially for low-sample rate files)
		audioContext = new AudioContext({ sampleRate: await audioTrack?.getSampleRate() });
		gainNode = audioContext.createGain();
		gainNode.connect(audioContext.destination);
		updateVolume();

		const videoCanBeTransparent = videoTrack
			? await videoTrack.canBeTransparent()
			: false;

		playerContainer.style.background = videoCanBeTransparent ? 'transparent' : '';

		// For video, let's use a CanvasSink as it handles rotation and closing video samples for us.
		// Pool size of 2: We'll only ever have the current and the next frame around, so we only need two canvases.
		videoSink = videoTrack && new CanvasSink(videoTrack, {
			poolSize: 2,
			fit: 'contain', // In case the video changes dimensions over time
			alpha: videoCanBeTransparent,
		});
		// For audio, we'll use an AudioBufferSink to directly retrieve AudioBuffers compatible with the Web Audio API
		audioSink = audioTrack && new AudioBufferSink(audioTrack);

		// Show the canvas if there's a video track, otherwise hide it
		if (videoTrack) {
			canvas.style.display = '';
			canvas.width = await videoTrack.getDisplayWidth();
			canvas.height = await videoTrack.getDisplayHeight();
		} else {
			canvas.style.display = 'none';
		}

		// Show volume controls if there's an audio track, otherwise hide them
		if (audioTrack) {
			volumeButton.style.display = '';
			volumeBarContainer.style.display = '';
		} else {
			volumeButton.style.display = 'none';
			volumeBarContainer.style.display = 'none';
		}

		fileLoaded = true;

		await startVideoIterator();

		if (audioContext.state === 'running') {
			// Start playback automatically if the audio context permits
			await play();
		}

		loadingElement.style.display = 'none';
		playerContainer.style.display = '';

		if (!videoSink) {
			// If there's only an audio track, always show the controls
			controlsElement.style.opacity = '1';
			controlsElement.style.pointerEvents = '';
			playerContainer.style.cursor = '';
		}

		const refreshIntervals = await Promise.all(tracks.map(t => t.getLiveRefreshInterval()));
		const nonNullIntervals = refreshIntervals.filter(x => x !== null);

		if (nonNullIntervals.length > 0) {
			// At least one track is live! This means that we'll need to continually refresh the end timestamp of the
			// media to allow continuous live playback.

			const interval = Math.min(...nonNullIntervals);
			liveDot.style.display = '';
			liveDot.onclick = () => {
				void seekToTime(endTimestamp - interval * 1.5);
			};

			const scheduleLiveRefresh = () => {
				// eslint-disable-next-line @typescript-eslint/no-misused-promises
				liveRefreshIntervalId = window.setTimeout(async () => {
					endTimestamp = await input.getDurationFromMetadata(tracks, { skipLiveWait: true })
						?? await input.computeDuration(tracks, { skipLiveWait: true });
					durationElement.textContent = formatTimestamp(endTimestamp);

					// Check if we're still live
					const stillLive = await Promise.all(tracks.map(t => t.isLive()));
					if (stillLive.every(live => !live)) {
						liveDot.style.display = 'none';
					} else {
						scheduleLiveRefresh();
					}
				}, interval * 1000);
			};
			scheduleLiveRefresh();
		}
	} catch (error) {
		console.error(error);

		errorElement.textContent = String(error);
		loadingElement.style.display = 'none';
		playerContainer.style.display = 'none';
	}
};

/** === VIDEO RENDERING LOGIC === */

/** Creates a new video frame iterator and renders the first video frame. */
const startVideoIterator = async () => {
	if (!videoSink) {
		return;
	}

	asyncId++;

	await videoFrameIterator?.return(); // Dispose of the current iterator

	// Create a new iterator
	videoFrameIterator = videoSink.canvases(getPlaybackTime());

	// Get the first two frames
	const firstFrame = (await videoFrameIterator.next()).value ?? null;
	const secondFrame = (await videoFrameIterator.next()).value ?? null;

	nextFrame = secondFrame;

	if (firstFrame) {
		// Draw the first frame
		context.clearRect(0, 0, canvas.width, canvas.height);
		context.drawImage(firstFrame.canvas, 0, 0);
	}
};

/** Runs every frame; updates the canvas if necessary. */
const render = (requestFrame = true) => {
	if (fileLoaded) {
		const playbackTime = getPlaybackTime();
		if (playbackTime >= endTimestamp) {
			// Pause playback once the end is reached
			pause();
			playbackTimeAtStart = endTimestamp;
		}

		// Check if the current playback time has caught up to the next frame
		if (nextFrame && nextFrame.timestamp <= playbackTime) {
			context.clearRect(0, 0, canvas.width, canvas.height);
			context.drawImage(nextFrame.canvas, 0, 0);
			nextFrame = null;

			// Request the next frame
			void updateNextFrame();
		}

		if (!draggingProgressBar) {
			updateProgressBarTime(playbackTime);
		}
	}

	if (requestFrame) {
		requestAnimationFrame(() => render());
	}
};
render();

// Also call the render function on an interval to make sure the video keeps updating even if the tab isn't visible
setInterval(() => render(false), 500);

/** Iterates over the video frame iterator until it finds a video frame in the future. */
const updateNextFrame = async () => {
	const currentAsyncId = asyncId;

	// We have a loop here because we may need to iterate over multiple frames until we reach a frame in the future
	while (true) {
		const newNextFrame = (await videoFrameIterator!.next()).value ?? null;
		if (!newNextFrame) {
			break;
		}

		if (currentAsyncId !== asyncId) {
			break;
		}

		const playbackTime = getPlaybackTime();
		if (newNextFrame.timestamp <= playbackTime) {
			// Draw it immediately
			context.clearRect(0, 0, canvas.width, canvas.height);
			context.drawImage(newNextFrame.canvas, 0, 0);
		} else {
			// Save it for later
			nextFrame = newNextFrame;
			break;
		}
	}
};

/** === AUDIO PLAYBACK LOGIC === */

/** Loops over the audio buffer iterator, scheduling the audio to be played in the audio context. */
const runAudioIterator = async () => {
	if (!audioSink) {
		return;
	}

	// To play back audio, we loop over all audio chunks (typically very short) of the file and play them at the correct
	// timestamp. The result is a continuous, uninterrupted audio signal.
	for await (const { buffer, timestamp } of audioBufferIterator!) {
		const node = audioContext!.createBufferSource();
		node.buffer = buffer;
		node.connect(gainNode!);

		let startTimestamp = audioContextStartTime! + timestamp - playbackTimeAtStart;
		// Round timestamp to the context's sample boundaries to prevent subsample audio glitches
		startTimestamp = Math.round(audioContext!.sampleRate * startTimestamp) / audioContext!.sampleRate;

		// Two cases: Either, the audio starts in the future or in the past
		if (startTimestamp >= audioContext!.currentTime) {
			// If the audio starts in the future, easy, we just schedule it
			node.start(startTimestamp);
		} else {
			// If it starts in the past, then let's only play the audible section that remains from here on out
			node.start(audioContext!.currentTime, audioContext!.currentTime - startTimestamp);
		}

		queuedAudioNodes.add(node);
		node.onended = () => {
			queuedAudioNodes.delete(node);
		};

		// If we're more than a second ahead of the current playback time, let's slow down the loop until time has
		// passed.
		if (timestamp - getPlaybackTime() >= 1) {
			await new Promise<void>((resolve) => {
				const id = setInterval(() => {
					if (timestamp - getPlaybackTime() < 1) {
						clearInterval(id);
						resolve();
					}
				}, 100);
			});
		}
	}
};

/** === PLAYBACK CONTROL LOGIC === */

/** Returns the current playback time in the media file. */
const getPlaybackTime = () => {
	if (playing) {
		// To ensure perfect audio-video sync, we always use the audio context's clock to determine playback time, even
		// when there is no audio track.
		return audioContext!.currentTime - audioContextStartTime! + playbackTimeAtStart;
	} else {
		return playbackTimeAtStart;
	}
};

const play = async () => {
	if (audioContext!.state === 'suspended') {
		await audioContext!.resume();
	}

	if (getPlaybackTime() === endTimestamp) {
		// If we're at the end, let's snap back to the start
		playbackTimeAtStart = firstTimestamp;
		await startVideoIterator();
	}

	audioContextStartTime = audioContext!.currentTime;
	playing = true;

	if (audioSink) {
		// Start the audio iterator
		void audioBufferIterator?.return();
		audioBufferIterator = audioSink?.buffers(getPlaybackTime());
		void runAudioIterator();
	}

	playIcon.style.display = 'none';
	pauseIcon.style.display = '';
};

const pause = () => {
	playbackTimeAtStart = getPlaybackTime();
	playing = false;
	void audioBufferIterator?.return(); // This stops any for-loops that are iterating the iterator
	audioBufferIterator = null;

	// Stop all audio nodes that were already queued to play
	for (const node of queuedAudioNodes) {
		node.stop();
	}
	queuedAudioNodes.clear();

	playIcon.style.display = '';
	pauseIcon.style.display = 'none';
};

const togglePlay = () => {
	if (playing) {
		pause();
	} else {
		void play();
	}
};

const seekToTime = async (seconds: number) => {
	updateProgressBarTime(seconds);

	const wasPlaying = playing;

	if (wasPlaying) {
		pause();
	}

	playbackTimeAtStart = seconds;

	await startVideoIterator();

	if (wasPlaying && playbackTimeAtStart < endTimestamp) {
		void play();
	}
};

/** === PROGRESS BAR LOGIC === */

const updateProgressBarTime = (seconds: number) => {
	currentTimeElement.textContent = formatTimestamp(seconds);
	progressBar.style.width = `${((seconds - firstTimestamp) / (endTimestamp - firstTimestamp)) * 100}%`;
};

progressBarContainer.addEventListener('pointerdown', (event) => {
	draggingProgressBar = true;
	progressBarContainer.setPointerCapture(event.pointerId);

	const rect = progressBarContainer.getBoundingClientRect();
	const completion = Math.max(Math.min((event.clientX - rect.left) / rect.width, 1), 0);
	updateProgressBarTime(firstTimestamp + completion * (endTimestamp - firstTimestamp));

	clearTimeout(hideControlsTimeout);

	window.addEventListener('pointerup', (event) => {
		draggingProgressBar = false;
		progressBarContainer.releasePointerCapture(event.pointerId);

		const rect = progressBarContainer.getBoundingClientRect();
		const completion = Math.max(Math.min((event.clientX - rect.left) / rect.width, 1), 0);
		const newTime = firstTimestamp + completion * (endTimestamp - firstTimestamp);

		void seekToTime(newTime);
		showControlsTemporarily();
	}, { once: true });
});

progressBarContainer.addEventListener('pointermove', (event) => {
	if (draggingProgressBar) {
		const rect = progressBarContainer.getBoundingClientRect();
		const completion = Math.max(Math.min((event.clientX - rect.left) / rect.width, 1), 0);
		updateProgressBarTime(firstTimestamp + completion * (endTimestamp - firstTimestamp));
	}
});

/** === VOLUME CONTROL LOGIC === */

const updateVolume = () => {
	const actualVolume = volumeMuted ? 0 : volume;

	volumeBar.style.width = `${actualVolume * 100}%`;
	gainNode!.gain.value = actualVolume ** 2; // Quadratic for more fine-grained control

	const iconNumber = volumeMuted ? 0 : Math.ceil(1 + 3 * volume);
	for (let i = 0; i < volumeIconWrapper.children.length; i++) {
		const icon = volumeIconWrapper.children[i] as HTMLImageElement;
		icon.style.display = i === iconNumber ? '' : 'none';
	}
};

volumeBarContainer.addEventListener('pointerdown', (event) => {
	draggingVolumeBar = true;
	volumeBarContainer.setPointerCapture(event.pointerId);

	const rect = volumeBarContainer.getBoundingClientRect();
	volume = Math.max(Math.min((event.clientX - rect.left) / rect.width, 1), 0);
	volumeMuted = false;
	updateVolume();

	clearTimeout(hideControlsTimeout);

	window.addEventListener('pointerup', (event) => {
		draggingVolumeBar = false;
		volumeBarContainer.releasePointerCapture(event.pointerId);

		const rect = volumeBarContainer.getBoundingClientRect();
		volume = Math.max(Math.min((event.clientX - rect.left) / rect.width, 1), 0);
		updateVolume();

		showControlsTemporarily();
	}, { once: true });
});

volumeButton.addEventListener('click', () => {
	volumeMuted = !volumeMuted;
	updateVolume();
});

volumeBarContainer.addEventListener('pointermove', (event) => {
	if (draggingVolumeBar) {
		const rect = volumeBarContainer.getBoundingClientRect();
		volume = Math.max(Math.min((event.clientX - rect.left) / rect.width, 1), 0);
		updateVolume();
	}
});

/** === CONTROL UI LOGIC === */

const showControlsTemporarily = () => {
	if (!videoSink) {
		// Shouldn't run if there's only an audio track
		return;
	}

	controlsElement.style.opacity = '1';
	controlsElement.style.pointerEvents = '';
	playerContainer.style.cursor = '';

	clearTimeout(hideControlsTimeout);
	hideControlsTimeout = window.setTimeout(() => {
		if (draggingProgressBar) {
			return;
		}

		hideControls();
		playerContainer.style.cursor = 'none';
	}, 2000);
};

const hideControls = () => {
	controlsElement.style.opacity = '0';
	controlsElement.style.pointerEvents = 'none';
};
hideControls();

let hideControlsTimeout = -1;
playerContainer.addEventListener('pointermove', (event) => {
	if (event.pointerType !== 'touch') {
		showControlsTemporarily();
	}
});
playerContainer.addEventListener('pointerleave', (event) => {
	if (!videoSink) {
		// Shouldn't run if there's only an audio track
		return;
	}

	if (draggingProgressBar || draggingVolumeBar || event.pointerType === 'touch') {
		return;
	}

	hideControls();
	clearTimeout(hideControlsTimeout);
});

/** === EVENT LISTENERS === */

playButton.addEventListener('click', togglePlay);
window.addEventListener('keydown', (e) => {
	if (!fileLoaded) {
		return;
	}

	if (e.code === 'Space' || e.code === 'KeyK') {
		togglePlay();
	} else if (e.code === 'KeyF') {
		fullscreenButton.click();
	} else if (e.code === 'ArrowLeft') {
		const newTime = Math.max(getPlaybackTime() - 5, firstTimestamp);
		void seekToTime(newTime);
	} else if (e.code === 'ArrowRight') {
		const newTime = Math.min(getPlaybackTime() + 5, endTimestamp);
		void seekToTime(newTime);
	} else if (e.code === 'KeyM') {
		volumeButton.click();
	} else {
		return;
	}

	showControlsTemporarily();
	e.preventDefault();
});

fullscreenButton.addEventListener('click', () => {
	if (document.fullscreenElement) {
		void document.exitFullscreen();
	} else {
		playerContainer.requestFullscreen().catch((e) => {
			console.error('Failed to enter fullscreen mode:', e);
		});
	}
});

// I'm sorry for this
const isTouchDevice = () => {
	return 'ontouchstart' in window;
};

playerContainer.addEventListener('click', () => {
	if (isTouchDevice()) {
		if (controlsElement.style.opacity === '1') {
			hideControls();
		} else {
			showControlsTemporarily();
		}
	} else {
		togglePlay();
	}
});
controlsElement.addEventListener('click', (event) => {
	// Make sure this does NOT toggle play
	event.stopPropagation();
	showControlsTemporarily();
});

/** === UTILS === */

const formatTimestamp = (seconds: number) => {
	if (isRelativeToUnixEpoch) {
		const iso = new Date(seconds * 1000).toISOString();
		return iso.replace('T', '\n');
	}

	return formatSeconds(seconds);
};

const formatSeconds = (seconds: number) => {
	const showMilliseconds = window.innerWidth >= 640;

	seconds = Math.round(seconds * 1000) / 1000; // Round to milliseconds

	const hours = Math.floor(seconds / 3600);
	const minutes = Math.floor((seconds % 3600) / 60);
	const remainingSeconds = Math.floor(seconds % 60);
	const millisecs = Math.floor(1000 * seconds % 1000).toString().padStart(3, '0');

	let result: string;
	if (hours > 0) {
		result = `${hours}:${minutes.toString().padStart(2, '0')}`
			+ `:${remainingSeconds.toString().padStart(2, '0')}`;
	} else {
		result = `${minutes.toString().padStart(2, '0')}:${remainingSeconds.toString().padStart(2, '0')}`;
	}

	if (showMilliseconds) {
		result += `.${millisecs}`;
	}

	return result;
};

window.addEventListener('resize', () => {
	if (endTimestamp) {
		updateProgressBarTime(getPlaybackTime());
		durationElement.textContent = formatTimestamp(endTimestamp);
	}
});

/** === FILE SELECTION LOGIC === */

selectMediaButton.addEventListener('click', () => {
	const fileInput = document.createElement('input');
	fileInput.type = 'file';
	fileInput.accept = 'video/*,video/x-matroska,video/mp2t,.ts,audio/*,audio/aac';
	fileInput.addEventListener('change', () => {
		const file = fileInput.files?.[0];
		if (!file) {
			return;
		}

		void initMediaPlayer(file);
	});

	fileInput.click();
});

loadUrlButton.addEventListener('click', () => {
	const url = prompt(
		'Please enter a URL of a media file. Note that it must be HTTPS and support cross-origin requests, so have the'
		+ ' right CORS headers set.',
		'https://mediabunny.dev/big-buck-bunny.mp4',
	);
	if (!url) {
		return;
	}

	void initMediaPlayer(url);
});

document.addEventListener('dragover', (event) => {
	event.preventDefault();
	event.dataTransfer!.dropEffect = 'copy';
});

document.addEventListener('drop', (event) => {
	event.preventDefault();
	const files = event.dataTransfer?.files;
	const file = files && files.length > 0 ? files[0] : undefined;
	if (file) {
		void initMediaPlayer(file);
	}
});