Allow all Matroska AttachedFiles to be read & written via metadata tags

dev/demux.html

@@ -14,6 +14,9 @@
 			source: new Mediabunny.BlobSource(file),
 		});
 
+		console.log(await input.getMetadataTags());
+
+		/*
 		console.log(await input.computeDuration());
 		return;
 
@@ -25,6 +28,7 @@
 		}
 
 		console.log("Done")
+		*/
 
 
 		/*

src/index.ts

@@ -197,6 +197,7 @@ export {
 	MetadataTags,
 	AttachedImage,
 	RichImageData,
+	AttachedFile,
 } from './tags';
 
 // 🐡🦔

src/matroska/ebml.ts

@@ -16,6 +16,7 @@ export interface EBMLElement {
 	size?: number;
 	data:
 		| number
+		| bigint
 		| string
 		| Uint8Array
 		| EBMLFloat32
@@ -198,6 +199,26 @@ export const measureUnsignedInt = (value: number) => {
 	}
 };
 
+export const measureUnsignedBigInt = (value: bigint) => {
+	if (value < (1n << 8n)) {
+		return 1;
+	} else if (value < (1n << 16n)) {
+		return 2;
+	} else if (value < (1n << 24n)) {
+		return 3;
+	} else if (value < (1n << 32n)) {
+		return 4;
+	} else if (value < (1n << 40n)) {
+		return 5;
+	} else if (value < (1n << 48n)) {
+		return 6;
+	} else if (value < (1n << 56n)) {
+		return 7;
+	} else {
+		return 8;
+	}
+};
+
 export const measureSignedInt = (value: number) => {
 	if (value >= -(1 << 6) && value < (1 << 6)) {
 		return 1;
@@ -276,16 +297,16 @@ export class EBMLWriter {
 			// eslint-disable-next-line no-fallthrough
 			case 5:
 				this.helperView.setUint8(pos++, (value / 2 ** 32) | 0);
-				// eslint-disable-next-line no-fallthrough
+			// eslint-disable-next-line no-fallthrough
 			case 4:
 				this.helperView.setUint8(pos++, value >> 24);
-				// eslint-disable-next-line no-fallthrough
+			// eslint-disable-next-line no-fallthrough
 			case 3:
 				this.helperView.setUint8(pos++, value >> 16);
-				// eslint-disable-next-line no-fallthrough
+			// eslint-disable-next-line no-fallthrough
 			case 2:
 				this.helperView.setUint8(pos++, value >> 8);
-				// eslint-disable-next-line no-fallthrough
+			// eslint-disable-next-line no-fallthrough
 			case 1:
 				this.helperView.setUint8(pos++, value);
 				break;
@@ -296,6 +317,16 @@ export class EBMLWriter {
 		this.writer.write(this.helper.subarray(0, pos));
 	}
 
+	writeUnsignedBigInt(value: bigint, width = measureUnsignedBigInt(value)) {
+		let pos = 0;
+
+		for (let i = width - 1; i >= 0; i--) {
+			this.helperView.setUint8(pos++, Number((value >> BigInt(i * 8)) & 0xffn));
+		}
+
+		this.writer.write(this.helper.subarray(0, pos));
+	}
+
 	writeSignedInt(value: number, width = measureSignedInt(value)) {
 		if (value < 0) {
 			// Two's complement stuff
@@ -398,6 +429,10 @@ export class EBMLWriter {
 				const size = data.size ?? measureUnsignedInt(data.data);
 				this.writeVarInt(size);
 				this.writeUnsignedInt(data.data, size);
+			} else if (typeof data.data === 'bigint') {
+				const size = data.size ?? measureUnsignedBigInt(data.data);
+				this.writeVarInt(size);
+				this.writeUnsignedBigInt(data.data, size);
 			} else if (typeof data.data === 'string') {
 				this.writeVarInt(data.data.length);
 				this.writeAsciiString(data.data);
@@ -491,6 +526,21 @@ export const readUnsignedInt = (slice: FileSlice, width: number) => {
 	return value;
 };
 
+export const readUnsignedBigInt = (slice: FileSlice, width: number) => {
+	if (width < 1) {
+		throw new Error('Bad unsigned int size ' + width);
+	}
+
+	let value = 0n;
+
+	for (let i = 0; i < width; i++) {
+		value <<= 8n;
+		value += BigInt(readU8(slice));
+	}
+
+	return value;
+};
+
 export const readSignedInt = (slice: FileSlice, width: number) => {
 	let value = readUnsignedInt(slice, width);
 

src/matroska/matroska-demuxer.ts

@@ -31,7 +31,7 @@ import {
 	InputVideoTrack,
 	InputVideoTrackBacking,
 } from '../input-track';
-import { MetadataTags } from '../tags';
+import { AttachedFile, MetadataTags } from '../tags';
 import { PacketRetrievalOptions } from '../media-sink';
 import {
 	assert,
@@ -69,6 +69,7 @@ import {
 	readVarInt,
 	resync,
 	searchForNextElementId,
+	readUnsignedBigInt,
 } from './ebml';
 import { buildMatroskaMimeType } from './matroska-misc';
 import { FileSlice, readBytes, Reader, readI16Be, readU8 } from '../reader';
@@ -240,6 +241,7 @@ export class MatroskaDemuxer extends Demuxer {
 	currentTagTargetIsMovie: boolean = true;
 	currentSimpleTagName: string | null = null;
 	currentAttachedFile: {
+		fileUid: bigint | null;
 		fileName: string | null;
 		fileMediaType: string | null;
 		fileData: Uint8Array | null;
@@ -1478,6 +1480,7 @@ export class MatroskaDemuxer extends Demuxer {
 				if (!this.currentSegment) break;
 
 				this.currentAttachedFile = {
+					fileUid: null,
 					fileName: null,
 					fileMediaType: null,
 					fileData: null,
@@ -1486,6 +1489,19 @@ export class MatroskaDemuxer extends Demuxer {
 
 				this.readContiguousElements(slice.slice(dataStartPos, size));
 
+				const tags = this.currentSegment.metadataTags;
+
+				if (this.currentAttachedFile.fileUid && this.currentAttachedFile.fileData) {
+					// All attached files get surfaced in the `raw` metadata tags
+					tags.raw ??= {};
+					tags.raw[this.currentAttachedFile.fileUid.toString()] = new AttachedFile(
+						this.currentAttachedFile.fileData,
+						this.currentAttachedFile.fileMediaType ?? undefined,
+						this.currentAttachedFile.fileName ?? undefined,
+						this.currentAttachedFile.fileDescription ?? undefined,
+					);
+				}
+
 				// Only process image attachments
 				if (this.currentAttachedFile.fileMediaType?.startsWith('image/') && this.currentAttachedFile.fileData) {
 					const fileName = this.currentAttachedFile.fileName;
@@ -1500,8 +1516,8 @@ export class MatroskaDemuxer extends Demuxer {
 						}
 					}
 
-					this.currentSegment.metadataTags.images ??= [];
+					tags.images ??= [];
-					this.currentSegment.metadataTags.images.push({
+					tags.images.push({
 						data: this.currentAttachedFile.fileData,
 						mimeType: this.currentAttachedFile.fileMediaType,
 						kind,
@@ -1513,6 +1529,12 @@ export class MatroskaDemuxer extends Demuxer {
 				this.currentAttachedFile = null;
 			}; break;
 
+			case EBMLId.FileUID: {
+				if (!this.currentAttachedFile) break;
+
+				this.currentAttachedFile.fileUid = readUnsignedBigInt(slice, size);
+			}; break;
+
 			case EBMLId.FileName: {
 				if (!this.currentAttachedFile) break;
 

src/matroska/matroska-muxer.ts

@@ -22,6 +22,7 @@ import {
 	roundToMultiple,
 	textEncoder,
 	toUint8Array,
+	uint8ArraysAreEqual,
 	writeBits,
 } from '../misc';
 import {
@@ -62,6 +63,7 @@ import { Muxer } from '../muxer';
 import { Writer } from '../writer';
 import { EncodedPacket } from '../packet';
 import { parseOpusIdentificationHeader } from '../codec-data';
+import { AttachedFile } from '../tags';
 
 const MIN_CLUSTER_TIMESTAMP_MS = -(2 ** 15);
 const MAX_CLUSTER_TIMESTAMP_MS = 2 ** 15 - 1;
@@ -537,13 +539,12 @@ export class MatroskaMuxer extends Muxer {
 
 	private maybeCreateAttachments() {
 		const metadataTags = this.output._metadataTags;
-		if (!metadataTags.images || metadataTags.images.length === 0) {
+		const elements: EBMLElement[] = [];
-			return;
-		}
 
 		const existingFileUids = new Set<number>();
+		const images = metadataTags.images ?? [];
 
-		this.attachmentsElement = { id: EBMLId.Attachments, data: metadataTags.images.map((image): EBMLElement => {
+		for (const image of images) {
 			let imageName = image.name;
 			if (imageName === undefined) {
 				const baseName = image.kind === 'coverFront' ? 'cover' : image.kind === 'coverBack' ? 'back' : 'image';
@@ -561,7 +562,7 @@ export class MatroskaMuxer extends Muxer {
 
 			existingFileUids.add(fileUid);
 
-			return {
+			elements.push({
 				id: EBMLId.AttachedFile,
 				data: [
 					image.description !== undefined
@@ -572,8 +573,45 @@ export class MatroskaMuxer extends Muxer {
 					{ id: EBMLId.FileData, data: image.data },
 					{ id: EBMLId.FileUID, data: fileUid },
 				],
-			};
+			});
-		}) };
+		}
+
+		// Add all AttachedFiles from the raw metadata
+		for (const [key, value] of Object.entries(metadataTags.raw ?? {})) {
+			if (!(value instanceof AttachedFile)) {
+				continue;
+			}
+
+			const keyIsNumeric = /^\d+$/.test(key);
+			if (!keyIsNumeric) {
+				continue;
+			}
+
+			if (images.find(x => x.mimeType === value.mimeType && uint8ArraysAreEqual(x.data, value.data))) {
+				// This attached file has very likely already been added as an image above
+				// (happens when remuxing Matroska)
+				continue;
+			}
+
+			elements.push({
+				id: EBMLId.AttachedFile,
+				data: [
+					value.description !== undefined
+						? { id: EBMLId.FileDescription, data: new EBMLUnicodeString(value.description) }
+						: null,
+					{ id: EBMLId.FileName, data: new EBMLUnicodeString(value.name ?? '') },
+					{ id: EBMLId.FileMediaType, data: value.mimeType ?? '' },
+					{ id: EBMLId.FileData, data: value.data },
+					{ id: EBMLId.FileUID, data: BigInt(key) },
+				],
+			});
+		}
+
+		if (elements.length === 0) {
+			return;
+		}
+
+		this.attachmentsElement = { id: EBMLId.Attachments, data: elements };
 	}
 
 	private createSegment() {

src/misc.ts

@@ -765,3 +765,17 @@ export const bytesToBase64 = (bytes: Uint8Array) => {
 
 	return btoa(string);
 };
+
+export const uint8ArraysAreEqual = (a: Uint8Array, b: Uint8Array) => {
+	if (a.length !== b.length) {
+		return false;
+	}
+
+	for (let i = 0; i < a.length; i++) {
+		if (a[i] !== b[i]) {
+			return false;
+		}
+	}
+
+	return true;
+};

src/tags.ts

@@ -16,9 +16,9 @@
  * - For MP3 files, the metadata refers to the ID3v2 or ID3v1 tags.
  * - For Ogg files, there is no global metadata so instead, the metadata refers to the combined metadata of all tracks,
  * in Vorbis-style comment headers.
- * - For FLAC files, the metadata lives in Vorbis style in the Vorbis comment block.
  * - For WAVE files, the metadata refers to the chunks within the RIFF INFO chunk.
  * - For ADTS files, there is no metadata.
+ * - For FLAC files, the metadata lives in Vorbis style in the Vorbis comment block.
  *
  * @group Metadata tags
  * @public
@@ -66,6 +66,8 @@ export type MetadataTags = {
 	 * Additionally, any atoms within the `'udta'` atom are dumped into here, however with unknown internal format
 	 * (`Uint8Array`).
 	 * - Matroska: `SimpleTag` elements whose target is 50 (MOVIE), either containing string or `Uint8Array` values.
+	 * Additionally, all attached files (such as font files) are included here, where the key corresponds to the FileUID
+	 * and the value is an {@link AttachedFile}.
 	 * - MP3: The ID3v2 tags, or a single `'TAG'` key with the contents of the ID3v1 tag.
 	 * - Ogg: The key-value string pairs from the Vorbis-style comment header (see RFC 7845, Section 5.2).
 	 * Additionally, the `'vendor'` key refers to the vendor string within this header.
@@ -73,7 +75,7 @@ export type MetadataTags = {
 	 * - FLAC: The key-value string pairs from the vorbis metadata block (see RFC 9639, Section D.2.3).
 	 * Additionally, the `'vendor'` key refers to the vendor string within this header.
 	*/
-	raw?: Record<string, string | Uint8Array | RichImageData | null>;
+	raw?: Record<string, string | Uint8Array | RichImageData | AttachedFile | null>;
 };
 
 /**
@@ -91,7 +93,7 @@ export type AttachedImage = {
 	kind: 'coverFront' | 'coverBack' | 'unknown';
 	/** The name of the image file. */
 	name?: string;
-	/** A short description of the image. */
+	/** A description of the image. */
 	description?: string;
 };
 
@@ -108,9 +110,49 @@ export class RichImageData {
 		public data: Uint8Array,
 		/** An RFC 6838 MIME type (e.g. image/jpeg, image/png, etc.) */
 		public mimeType: string,
-	) {}
+	) {
+		if (!(data instanceof Uint8Array)) {
+			throw new TypeError('data must be a Uint8Array.');
+		}
+		if (typeof mimeType !== 'string') {
+			throw new TypeError('mimeType must be a string.');
+		}
+	}
 }
 
+/**
+ * A file attached to a media file.
+ *
+ * @group Metadata tags
+ * @public
+ */
+export class AttachedFile {
+	/** Creates a new {@link AttachedFile}. */
+	constructor(
+		/** The raw file data. */
+		public data: Uint8Array,
+		/** An RFC 6838 MIME type (e.g. image/jpeg, image/png, font/ttf, etc.) */
+		public mimeType?: string,
+		/** The name of the file. */
+		public name?: string,
+		/** A description of the file. */
+		public description?: string,
+	) {
+		if (!(data instanceof Uint8Array)) {
+			throw new TypeError('data must be a Uint8Array.');
+		}
+		if (mimeType !== undefined && typeof mimeType !== 'string') {
+			throw new TypeError('mimeType, when provided, must be a string.');
+		}
+		if (name !== undefined && typeof name !== 'string') {
+			throw new TypeError('name, when provided, must be a string.');
+		}
+		if (description !== undefined && typeof description !== 'string') {
+			throw new TypeError('description, when provided, must be a string.');
+		}
+	}
+};
+
 export const validateMetadataTags = (tags: MetadataTags) => {
 	if (!tags || typeof tags !== 'object') {
 		throw new TypeError('tags must be an object.');
@@ -190,9 +232,10 @@ export const validateMetadataTags = (tags: MetadataTags) => {
 				&& typeof value !== 'string'
 				&& !(value instanceof Uint8Array)
 				&& !(value instanceof RichImageData)
+				&& !(value instanceof AttachedFile)
 			) {
 				throw new TypeError(
-					'Each value in tags.raw must be a string, Uint8Array, RichImageData, or null.',
+					'Each value in tags.raw must be a string, Uint8Array, RichImageData, AttachedFile, or null.',
 				);
 			}
 		}

test/node/metadata-tags.test.ts

@@ -15,7 +15,7 @@ import { EncodedPacket } from '../../src/packet.js';
 import { Input } from '../../src/input.js';
 import { BufferSource, FilePathSource } from '../../src/source.js';
 import { ALL_FORMATS } from '../../src/input-format.js';
-import { MetadataTags } from '../../src/tags.js';
+import { AttachedFile, MetadataTags } from '../../src/tags.js';
 import path from 'node:path';
 import { AudioCodec, buildAudioCodecString } from '../../src/codec.js';
 import { Conversion } from '../../src/conversion.js';
@@ -207,7 +207,21 @@ test('Read and write metadata, Matroska', async () => {
 	output.setMetadataTags({
 		...songMetadata,
 		raw: {
-			CUSTOM: 'Levels',
+			'CUSTOM': 'Levels',
+			// This number cannot be represented by f64, so this tests correct BigInt usage
+			'9007199254740993': new AttachedFile(
+				new Uint8Array(20),
+				'font/ttf',
+				'Heya',
+				'A font file',
+			),
+			// And this one we don't expect to get attached again, since it mirrors the image we're also attaching
+			'987654': new AttachedFile(
+				songMetadata.images![0]!.data,
+				songMetadata.images![0]!.mimeType,
+				songMetadata.images![0]!.name,
+				songMetadata.images![0]!.description,
+			),
 		},
 	});
 
@@ -246,6 +260,19 @@ test('Read and write metadata, Matroska', async () => {
 	expect(readTags.raw!['TITLE']).toBe(songMetadata.title);
 	expect(readTags.raw!['PART_NUMBER']).toBe('13/14');
 	expect(readTags.raw!['CUSTOM']).toBe('Levels');
+
+	const files = Object.entries(readTags.raw!).filter(x => x[1] instanceof AttachedFile);
+	expect(files).toHaveLength(2);
+
+	const imageFile = files.find(x => (x[1] as AttachedFile).mimeType === 'image/jpeg');
+	expect((imageFile![1] as AttachedFile).data).toEqual(songMetadata.images![0]!.data);
+
+	const secondFile = files.find(x => (x[1] as AttachedFile).mimeType === 'font/ttf');
+	expect(secondFile![0]).toBe('9007199254740993');
+	expect((secondFile![1] as AttachedFile).data).toHaveLength(20);
+	expect((secondFile![1] as AttachedFile).mimeType).toBe('font/ttf');
+	expect((secondFile![1] as AttachedFile).name).toBe('Heya');
+	expect((secondFile![1] as AttachedFile).description).toBe('A font file');
 });
 
 test('Read and write metadata, MP3', async () => {