Add support for reading and writing RF64 WAVE files

dev/convert.html

@@ -21,7 +21,7 @@
 			chunked: true,
 			chunkSize: 2**20
 		});
-		const outputFormat = new Mediabunny.Mp4OutputFormat();
+		const outputFormat = new Mediabunny.WavOutputFormat({ large: true });
 
 		const button = document.createElement('button');
 		button.textContent = 'Cancel';
@@ -38,8 +38,8 @@
 				target
 			}),
 			audio: {
-				codec: 'opus',
-				bitrate: 128000,
+				//codec: 'opus',
+				//bitrate: 128000,
 				//numberOfChannels: 1,
 				//sampleRate: 4000
 				//discard: true

dev/demux.html

@@ -15,14 +15,21 @@
 			formats: Mediabunny.ALL_FORMATS,
 			source
 		});
+
+		const canvas = document.createElement('canvas');
+		canvas.width = 1920;
+		canvas.height = 1080;
+		canvas.style.background = 'ghostwhite';
+		const ctx = canvas.getContext('2d');
+		document.body.append(canvas);
 		
 		const videoTrack = await input.getPrimaryVideoTrack();
+		const sink = new Mediabunny.VideoSampleSink(videoTrack);
 
-		const sink = new Mediabunny.EncodedPacketSink(videoTrack);
-		for await (const packet of sink.packets()) {
-			if (packet.timestamp >= 2) break;
-			console.log(packet);
-		}
+		const sample = await sink.getSample(3);
+		console.log(sample);
+
+		sample.draw(ctx, 1500, 500, 50, 50, 0, 0);
 
 		/*
 		let timestamps = [];

docs/.vitepress/config.mts

@@ -4,11 +4,13 @@ import tailwindcss from '@tailwindcss/vite';
 import llmstxt from 'vitepress-plugin-llms';
 import { HeadConfig } from 'vitepress';
 
+const DESCRIPTION = 'A JavaScript library for reading, writing, and converting media files. Directly in the browser,'
+	+ ' and faster than anybunny else.';
+
 // https://vitepress.dev/reference/site-config
 export default withMermaid({
 	title: 'Mediabunny',
-	description: 'A JavaScript library for reading, writing, and converting media files. Directly in the browser, and'
-		+ ' faster than anybunny else.',
+	description: DESCRIPTION,
 	cleanUrls: true,
 	head: [
 		['link', { rel: 'icon', type: 'image/png', href: '/mediabunny-logo.png' }],
@@ -16,9 +18,13 @@ export default withMermaid({
 		['meta', { property: 'og:type', content: 'website' }],
 		['meta', { property: 'og:site_name', content: 'Mediabunny' }],
 		['meta', { property: 'og:url', content: 'https://mediabunny.dev/' }],
-		['meta', { property: 'og:image', content: '/mediabunny-og-image.png' }],
+		['meta', { property: 'og:image', content: 'https://mediabunny.dev/mediabunny-og-image.png' }],
 		['meta', { property: 'og:locale', content: 'en-US' }],
+		['meta', { property: 'og:description', content: DESCRIPTION }],
+		['meta', { name: 'twitter:image', content: 'https://mediabunny.dev/mediabunny-og-image.png' }],
 		['meta', { name: 'twitter:card', content: 'summary_large_image' }],
+		['meta', { name: 'twitter:site', content: '@vanilagy' }],
+		['meta', { name: 'twitter:description', content: DESCRIPTION }],
 	],
 	themeConfig: {
 		logo: '/mediabunny-logo.svg',
@@ -114,6 +120,7 @@ export default withMermaid({
 
 		((pageData.frontmatter['head'] ??= []) as HeadConfig[]).push(
 			['meta', { property: 'og:title', content: title }],
+			['meta', { property: 'twitter:title', content: title }],
 		);
 	},
 });

docs/guide/output-formats.md

@@ -227,8 +227,11 @@ const output = new Output({
 The following options are available:
 ```ts
 type WavOutputFormatOptions = {
+	large?: boolean;
 	onHeader?: (data: Uint8Array, position: number) => unknown;
 };
 ```
+- `large`\
+	When enabled, an RF64 file be written, allowing for file sizes to exceed 4 GiB, which is otherwise not possible for regular WAVE files.
 - `onHeader`\
 	Will be called once the file header is written. The header consists of the RIFF header, the format chunk, and the start of the data chunk (with a placeholder size of 0).

docs/guide/packets-and-samples.md

@@ -309,17 +309,29 @@ The `VideoFrame` returned by this method **must** be closed separately from the
 
 ---
 
-It's also common to draw video samples to a `<canvas>` element or an `OffscreenCanvas`. For this, you can use the following method:
+It's also common to draw video samples to a `<canvas>` element or an `OffscreenCanvas`. For this, you can use the following methods:
 ```ts
 draw(
 	context: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D,
 	dx: number,
 	dy: number,
-	dWidth?: number,
-	dHeight?: number,
+	dWidth?: number, // defaults to displayWidth
+	dHeight?: number, // defaults to displayHeight
+): void;
+
+draw(
+	context: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D,
+	sx: number,
+	sy: number,
+	sWidth: number,
+	sHeight: number,
+	dx: number,
+	dy: number,
+	dWidth?: number, // defaults to sWidth
+	dHeight?: number, // defaults to sHeight
 ): void;
 ```
-This method is similar to [drawImage](https://developer.mozilla.org/en-US/docs/Web/API/CanvasRenderingContext2D/drawImage) and paints the video frame at the given position with the given dimensions. If the dimensions aren't specified, they will default to the display dimensions. This method will automatically draw the frame with the correct rotation based on its `rotation` property.
+These methods behave like [drawImage](https://developer.mozilla.org/en-US/docs/Web/API/CanvasRenderingContext2D/drawImage) and paint the video frame at the given position with the given dimensions. This method will automatically draw the frame with the correct rotation based on its `rotation` property.
 
 If you want to draw the raw underlying image to a canvas directly (without respecting the rotation metadata), then you can use the following method:
 ```ts

docs/index.md

@@ -95,6 +95,7 @@ const sponsors = {
 		{ image: 'https://avatars.githubusercontent.com/u/84167135', name: 'Memenome', url: 'https://github.com/memenome' },
 		{ image: 'https://avatars.githubusercontent.com/u/30229596', name: 'Pablo Bonilla', url: 'https://github.com/devPablo' },
 		{ image: 'https://avatars.githubusercontent.com/u/58149663', name: 'H7GhosT', url: 'https://github.com/H7GhosT' },
+		{ image: 'https://avatars.githubusercontent.com/u/91711202', name: 'ihasq', url: 'https://github.com/ihasq' },
 	],
 };
 </script>

examples/live-recording/live-recording.ts

@@ -85,7 +85,7 @@ const startRecording = async () => {
 
 		// Add the video track, with the canvas as the source
 		videoSource = new CanvasSource(canvas, {
-			codec: 'vp9',
+			codec: 'avc',
 			bitrate: QUALITY_MEDIUM,
 			keyFrameInterval: 0.5,
 			latencyMode: 'realtime', // Allow the encoder to skip frames to keep up with real-time constraints

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "mediabunny",
-  "version": "1.0.6",
+  "version": "1.2.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "mediabunny",
-      "version": "1.0.6",
+      "version": "1.2.0",
       "license": "MPL-2.0",
       "dependencies": {
         "@types/dom-mediacapture-transform": "^0.1.11",

package.json

@@ -1,7 +1,7 @@
 {
   "name": "mediabunny",
   "author": "Vanilagy",
-  "version": "1.0.6",
+  "version": "1.2.0",
   "description": "Pure TypeScript media toolkit for reading, writing, and converting media files, directly in the browser.",
   "type": "module",
   "main": "./dist/bundles/mediabunny.cjs",

src/input-format.ts

@@ -291,7 +291,7 @@ export class WaveInputFormat extends InputFormat {
 
 		const riffReader = new RiffReader(input._mainReader);
 		const riffType = riffReader.readAscii(4);
-		if (riffType !== 'RIFF' && riffType !== 'RIFX') {
+		if (riffType !== 'RIFF' && riffType !== 'RIFX' && riffType !== 'RF64') {
 			return false;
 		}
 

src/matroska/matroska-demuxer.ts

@@ -220,7 +220,7 @@ export class MatroskaDemuxer extends Demuxer {
 
 			const fileSize = await this.input.source.getSize();
 
-			while (this.metadataReader.pos < fileSize - MIN_HEADER_SIZE) {
+			while (this.metadataReader.pos <= fileSize - MIN_HEADER_SIZE) {
 				await this.metadataReader.reader.loadRange(
 					this.metadataReader.pos,
 					this.metadataReader.pos + MAX_HEADER_SIZE,
@@ -557,7 +557,7 @@ export class MatroskaDemuxer extends Demuxer {
 	readContiguousElements(reader: EBMLReader, totalSize: number) {
 		const startIndex = reader.pos;
 
-		while (reader.pos - startIndex < totalSize) {
+		while (reader.pos - startIndex <= totalSize - MIN_HEADER_SIZE) {
 			this.traverseElement(reader);
 		}
 	}
@@ -1468,7 +1468,7 @@ abstract class MatroskaTrackBacking implements InputTrackBacking {
 					}
 
 					const endPos = dataStartPos + size;
-					if (endPos >= segment.elementEndPos - MIN_HEADER_SIZE) {
+					if (endPos > segment.elementEndPos - MIN_HEADER_SIZE) {
 						// No more elements fit in this segment
 						break;
 					} else {

src/media-sink.ts

@@ -737,6 +737,7 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
 		// Round the timestamps to the time resolution
 		sample.setTimestamp(Math.round(sample.timestamp * this.timeResolution) / this.timeResolution);
 		sample.setDuration(Math.round(sample.duration * this.timeResolution) / this.timeResolution);
+		sample.setRotation(this.rotation);
 
 		this.onSample(sample);
 	}

src/output-format.ts

@@ -546,6 +546,12 @@ export class Mp3OutputFormat extends OutputFormat {
  * @public
  */
 export type WavOutputFormatOptions = {
+	/**
+	 * When enabled, an RF64 file be written, allowing for file sizes to exceed 4 GiB, which is otherwise not possible
+	 * for regular WAVE files.
+	 */
+	large?: boolean;
+
 	/**
 	 * Will be called once the file header is written. The header consists of the RIFF header, the format chunk, and the
 	 * start of the data chunk (with a placeholder size of 0).
@@ -565,6 +571,9 @@ export class WavOutputFormat extends OutputFormat {
 		if (!options || typeof options !== 'object') {
 			throw new TypeError('options must be an object.');
 		}
+		if (options.large !== undefined && typeof options.large !== 'boolean') {
+			throw new TypeError('options.large, when provided, must be a boolean.');
+		}
 		if (options.onHeader !== undefined && typeof options.onHeader !== 'function') {
 			throw new TypeError('options.onHeader, when provided, must be a function.');
 		}

src/sample.ts

@@ -366,9 +366,79 @@ export class VideoSample {
 		context: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D,
 		dx: number,
 		dy: number,
-		dWidth: number = this.displayWidth,
-		dHeight: number = this.displayHeight,
+		dWidth?: number,
+		dHeight?: number,
+	): void;
+	/**
+	 * Draws the video sample to a 2D canvas context. Rotation metadata will be taken into account.
+	 *
+	 * @param sx - The x-coordinate of the top left corner of the sub-rectangle of the source image to draw into the
+	 * destination context.
+	 * @param sy - The y-coordinate of the top left corner of the sub-rectangle of the source image to draw into the
+	 * destination context.
+	 * @param sWidth - The width of the sub-rectangle of the source image to draw into the destination context.
+	 * @param sHeight - The height of the sub-rectangle of the source image to draw into the destination context.
+	 * @param dx - The x-coordinate in the destination canvas at which to place the top-left corner of the source image.
+	 * @param dy - The y-coordinate in the destination canvas at which to place the top-left corner of the source image.
+	 * @param dWidth - The width in pixels with which to draw the image in the destination canvas.
+	 * @param dHeight - The height in pixels with which to draw the image in the destination canvas.
+	 */
+	draw(
+		context: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D,
+		sx: number,
+		sy: number,
+		sWidth: number,
+		sHeight: number,
+		dx: number,
+		dy: number,
+		dWidth?: number,
+		dHeight?: number,
+	): void;
+	draw(
+		context: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D,
+		arg1: number,
+		arg2: number,
+		arg3?: number,
+		arg4?: number,
+		arg5?: number,
+		arg6?: number,
+		arg7?: number,
+		arg8?: number,
 	) {
+		let sx = 0;
+		let sy = 0;
+		let sWidth = this.displayWidth;
+		let sHeight = this.displayHeight;
+		let dx = 0;
+		let dy = 0;
+		let dWidth = this.displayWidth;
+		let dHeight = this.displayHeight;
+
+		if (arg5 !== undefined) {
+			sx = arg1!;
+			sy = arg2!;
+			sWidth = arg3!;
+			sHeight = arg4!;
+			dx = arg5;
+			dy = arg6!;
+
+			if (arg7 !== undefined) {
+				dWidth = arg7;
+				dHeight = arg8!;
+			} else {
+				dWidth = sWidth;
+				dHeight = sHeight;
+			}
+		} else {
+			dx = arg1;
+			dy = arg2;
+
+			if (arg3 !== undefined) {
+				dWidth = arg3;
+				dHeight = arg4!;
+			}
+		}
+
 		if (!(
 			(typeof CanvasRenderingContext2D !== 'undefined' && context instanceof CanvasRenderingContext2D)
 			|| (
@@ -378,6 +448,18 @@ export class VideoSample {
 		)) {
 			throw new TypeError('context must be a CanvasRenderingContext2D or OffscreenCanvasRenderingContext2D.');
 		}
+		if (!Number.isFinite(sx)) {
+			throw new TypeError('sx must be a number.');
+		}
+		if (!Number.isFinite(sy)) {
+			throw new TypeError('sy must be a number.');
+		}
+		if (!Number.isFinite(sWidth) || sWidth < 0) {
+			throw new TypeError('sWidth must be a non-negative number.');
+		}
+		if (!Number.isFinite(sHeight) || sHeight < 0) {
+			throw new TypeError('sHeight must be a non-negative number.');
+		}
 		if (!Number.isFinite(dx)) {
 			throw new TypeError('dx must be a number.');
 		}
@@ -395,6 +477,29 @@ export class VideoSample {
 			throw new Error('VideoSample is closed.');
 		}
 
+		// The provided sx,sy,sWidth,sHeight refer to the final rotated image, but that's not actually how the image is
+		// stored. Therefore, we must map these back onto the original, pre-rotation image.
+		if (this.rotation === 90) {
+			[sx, sy, sWidth, sHeight] = [
+				sy,
+				this.codedHeight - sx - sWidth,
+				sHeight,
+				sWidth,
+			];
+		} else if (this.rotation === 180) {
+			[sx, sy] = [
+				this.codedWidth - sx - sWidth,
+				this.codedHeight - sy - sHeight,
+			];
+		} else if (this.rotation === 270) {
+			[sx, sy, sWidth, sHeight] = [
+				this.codedWidth - sy - sHeight,
+				sx,
+				sHeight,
+				sWidth,
+			];
+		}
+
 		const source = this.toCanvasImageSource();
 
 		context.save();
@@ -412,6 +517,10 @@ export class VideoSample {
 
 		context.drawImage(
 			source,
+			sx,
+			sy,
+			sWidth,
+			sHeight,
 			-dWidth / 2,
 			-dHeight / 2,
 			dWidth,

src/wave/riff-reader.ts

@@ -35,6 +35,21 @@ export class RiffReader {
 		return view.getUint32(offset, this.littleEndian);
 	}
 
+	readU64() {
+		let low: number;
+		let high: number;
+
+		if (this.littleEndian) {
+			low = this.readU32();
+			high = this.readU32();
+		} else {
+			high = this.readU32();
+			low = this.readU32();
+		}
+
+		return high * 0x100000000 + low;
+	}
+
 	readAscii(length: number) {
 		const { view, offset } = this.reader.getViewAndOffset(this.pos, this.pos + length);
 		this.pos += length;

src/wave/riff-writer.ts

@@ -14,14 +14,20 @@ export class RiffWriter {
 
 	constructor(private writer: Writer) {}
 
+	writeU16(value: number) {
+		this.helperView.setUint16(0, value, true);
+		this.writer.write(this.helper.subarray(0, 2));
+	}
+
 	writeU32(value: number) {
 		this.helperView.setUint32(0, value, true);
 		this.writer.write(this.helper.subarray(0, 4));
 	}
 
-	writeU16(value: number) {
-		this.helperView.setUint16(0, value, true);
-		this.writer.write(this.helper.subarray(0, 2));
+	writeU64(value: number) {
+		this.helperView.setUint32(0, value, true);
+		this.helperView.setUint32(4, Math.floor(value / 2 ** 32), true);
+		this.writer.write(this.helper);
 	}
 
 	writeAscii(text: string) {

src/wave/wave-demuxer.ts

@@ -53,9 +53,13 @@ export class WaveDemuxer extends Demuxer {
 			const actualFileSize = await this.metadataReader.reader.source.getSize();
 
 			const riffType = this.metadataReader.readAscii(4);
-			this.metadataReader.littleEndian = riffType === 'RIFF';
+			this.metadataReader.littleEndian = riffType !== 'RIFX';
 
-			const totalFileSize = Math.min(this.metadataReader.readU32() + 8, actualFileSize);
+			const isRf64 = riffType === 'RF64';
+
+			const outerChunkSize = this.metadataReader.readU32();
+
+			let totalFileSize = isRf64 ? actualFileSize : Math.min(outerChunkSize + 8, actualFileSize);
 			const format = this.metadataReader.readAscii(4);
 
 			if (format !== 'WAVE') {
@@ -63,6 +67,9 @@ export class WaveDemuxer extends Demuxer {
 			}
 
 			this.metadataReader.pos = 12;
+			let chunksRead = 0;
+			let dataChunkSize: number | null = null;
+
 			while (this.metadataReader.pos < totalFileSize) {
 				await this.metadataReader.reader.loadRange(this.metadataReader.pos, this.metadataReader.pos + 8);
 
@@ -70,14 +77,28 @@ export class WaveDemuxer extends Demuxer {
 				const chunkSize = this.metadataReader.readU32();
 				const startPos = this.metadataReader.pos;
 
+				if (isRf64 && chunksRead === 0 && chunkId !== 'ds64') {
+					throw new Error('Invalid RF64 file: First chunk must be "ds64".');
+				}
+
 				if (chunkId === 'fmt ') {
 					await this.parseFmtChunk(chunkSize);
 				} else if (chunkId === 'data') {
+					dataChunkSize ??= chunkSize;
+
 					this.dataStart = this.metadataReader.pos;
-					this.dataSize = Math.min(chunkSize, totalFileSize - this.dataStart);
+					this.dataSize = Math.min(dataChunkSize, totalFileSize - this.dataStart);
+				} else if (chunkId === 'ds64') {
+					// File and data chunk sizes are defined in here instead
+
+					const riffChunkSize = this.metadataReader.readU64();
+					dataChunkSize = this.metadataReader.readU64();
+
+					totalFileSize = Math.min(riffChunkSize + 8, actualFileSize);
 				}
 
 				this.metadataReader.pos = startPos + chunkSize + (chunkSize & 1); // Handle padding
+				chunksRead++;
 			}
 
 			if (!this.audioInfo) {

src/wave/wave-muxer.ts

@@ -18,10 +18,13 @@ import { assert } from '../misc';
 
 export class WaveMuxer extends Muxer {
 	private format: WavOutputFormat;
+	private isRf64: boolean;
 	private writer: Writer;
 	private riffWriter: RiffWriter;
 	private headerWritten = false;
 	private dataSize = 0;
+	private sampleRate: number | null = null;
+	private sampleCount = 0;
 
 	constructor(output: Output, format: WavOutputFormat) {
 		super(output);
@@ -29,6 +32,7 @@ export class WaveMuxer extends Muxer {
 		this.format = format;
 		this.writer = output._writer;
 		this.riffWriter = new RiffWriter(output._writer);
+		this.isRf64 = !!format._options.large;
 	}
 
 	async start() {
@@ -58,13 +62,22 @@ export class WaveMuxer extends Muxer {
 				assert(meta.decoderConfig);
 
 				this.writeHeader(track, meta.decoderConfig);
+				this.sampleRate = meta.decoderConfig.sampleRate;
 				this.headerWritten = true;
 			}
 
 			this.validateAndNormalizeTimestamp(track, packet.timestamp, packet.type === 'key');
 
+			if (!this.isRf64 && this.writer.getPos() + packet.data.byteLength >= 2 ** 32) {
+				throw new Error(
+					'Adding more audio data would exceed the maximum RIFF size of 4 GiB. To write larger files, use'
+					+ ' RF64 by setting `large: true` in the WavOutputFormatOptions.',
+				);
+			}
+
 			this.writer.write(packet.data);
 			this.dataSize += packet.data.byteLength;
+			this.sampleCount += Math.round(packet.duration * this.sampleRate!);
 
 			await this.writer.flush();
 		} finally {
@@ -101,10 +114,26 @@ export class WaveMuxer extends Muxer {
 		const blockSize = pcmInfo.sampleSize * channels;
 
 		// RIFF header
-		this.riffWriter.writeAscii('RIFF');
-		this.riffWriter.writeU32(0); // File size placeholder
+		this.riffWriter.writeAscii(this.isRf64 ? 'RF64' : 'RIFF');
+
+		if (this.isRf64) {
+			this.riffWriter.writeU32(0xffffffff); // Not used in RF64
+		} else {
+			this.riffWriter.writeU32(0); // File size placeholder
+		}
+
 		this.riffWriter.writeAscii('WAVE');
 
+		if (this.isRf64) {
+			this.riffWriter.writeAscii('ds64');
+			this.riffWriter.writeU32(28); // Chunk size
+			this.riffWriter.writeU64(0); // RIFF size placeholder
+			this.riffWriter.writeU64(0); // Data size placeholder
+			this.riffWriter.writeU64(0); // Sample count placeholder
+			this.riffWriter.writeU32(0); // Table length
+			// Empty table
+		}
+
 		// fmt chunk
 		this.riffWriter.writeAscii('fmt ');
 		this.riffWriter.writeU32(16); // Chunk size
@@ -117,7 +146,12 @@ export class WaveMuxer extends Muxer {
 
 		// data chunk
 		this.riffWriter.writeAscii('data');
-		this.riffWriter.writeU32(0); // Data size placeholder
+
+		if (this.isRf64) {
+			this.riffWriter.writeU32(0xffffffff); // Not used in RF64
+		} else {
+			this.riffWriter.writeU32(0); // Data size placeholder
+		}
 
 		if (this.format._options.onHeader) {
 			const { data, start } = this.writer.stopTrackingWrites();
@@ -130,13 +164,27 @@ export class WaveMuxer extends Muxer {
 
 		const endPos = this.writer.getPos();
 
-		// Write file size
-		this.writer.seek(4);
-		this.riffWriter.writeU32(this.dataSize + 36); // File size - 8
+		if (this.isRf64) {
+			// Write riff size
+			this.writer.seek(20);
+			this.riffWriter.writeU64(endPos - 8);
 
-		// Write data chunk size
-		this.writer.seek(40);
-		this.riffWriter.writeU32(this.dataSize);
+			// Write data size
+			this.writer.seek(28);
+			this.riffWriter.writeU64(this.dataSize);
+
+			// Write sample count
+			this.writer.seek(36);
+			this.riffWriter.writeU64(this.sampleCount);
+		} else {
+			// Write file size
+			this.writer.seek(4);
+			this.riffWriter.writeU32(endPos - 8);
+
+			// Write data chunk size
+			this.writer.seek(40);
+			this.riffWriter.writeU32(this.dataSize);
+		}
 
 		this.writer.seek(endPos);