feat: Add support for custom MP4 metadata with mdta format (#132)

* feat: Add support for custom MP4 metadata with mdta format

- Add metadataFormat option ('mdir' | 'mdta') to IsobmffOutputFormatOptions
- Implement complete mdta format with keys box generation
- Enable custom metadata fields readable by ffmpeg/ffprobe
- Maintain backward compatibility with default mdir format

* Refactor ISOBMFF metadata tag writing logic

* Autism

---------

Co-authored-by: tanfu <[email protected]>
Co-authored-by: Vanilagy <[email protected]>

dev/convert.html

@@ -24,7 +24,9 @@
 			chunked: true,
 			chunkSize: 2**20
 		});
-		const outputFormat = new Mediabunny.FlacOutputFormat({});
+		const outputFormat = new Mediabunny.Mp4OutputFormat({
+			metadataFormat: 'mdta',
+		});
 
 		const button = document.createElement('button');
 		button.textContent = 'Cancel';
@@ -92,7 +94,7 @@
 			},
 			*/
 			video: () => ({
-				discard: true,
+				//discard: true,
 				//discard: true,
 				//crop: {
 				//	left: 0,
@@ -126,7 +128,7 @@
 				//width: 200,
 				//height: 100,
 			}),
-			tags: {} ?? {
+			tags: {
 				title: 'Bigggy',
 				artist: 'Buck Bunny',
 				images: [{

dev/demux.html

@@ -14,6 +14,10 @@
 			source: new Mediabunny.BlobSource(file),
 		});
 
+		console.log(await input.getMetadataTags());
+
+		/*
+
 		const videoTrack = await input.getPrimaryVideoTrack();
 		const sink = new Mediabunny.VideoSampleSink(videoTrack);
 		const sink2 = new Mediabunny.EncodedPacketSink(videoTrack);
@@ -37,6 +41,7 @@
 				break;
 			}
 		}
+		*/
 
 		/*
 		const sink = new Mediabunny.EncodedPacketSink(videoTrack);

docs/guide/output-formats.md

@@ -64,6 +64,7 @@ The following options are available:
 type IsobmffOutputFormatOptions = {
 	fastStart?: false | 'in-memory' | 'reserve' | 'fragmented';
 	minimumFragmentDuration?: number;
+	metadataFormat?: 'mdir' | 'mdta' | 'udta' | 'auto';
 
 	onFtyp?: (data: Uint8Array, position: number) => unknown;
 	onMoov?: (data: Uint8Array, position: number) => unknown;
@@ -94,6 +95,12 @@ type IsobmffOutputFormatOptions = {
 		The default option; it behaves like `'in-memory'` when using [`BufferTarget`](./writing-media-files#buffertarget) and like `false` otherwise.
 - `minimumFragmentDuration`\
 	Only relevant when `fastStart` is `'fragmented'`. Sets the minimum duration in seconds a fragment must have to be finalized and written to the file. Defaults to 1 second.
+- `metadataFormat`\
+	The metadata format to use for writing metadata tags.
+	- `'auto'` (default): Behaves like `'mdir'` for MP4 and like `'udta'` for QuickTime, matching FFmpeg's default behavior.
+	- `'mdir'`: Write tags into `moov/udta/meta` using the 'mdir' handler format.
+	- `'mdta'`: Write tags into `moov/udta/meta` using the 'mdta' handler format, equivalent to FFmpeg's `use_metadata_tags` flag. This allows for custom keys of arbitrary length.
+	- `'udta'`: Write tags directly into `moov/udta`.
 - `onFtyp`\
 	Will be called once the ftyp (File Type) box of the output file has been written.
 - `onMoov`\

src/isobmff/isobmff-boxes.ts

@@ -1274,14 +1274,18 @@ export const vtta = (notes: string) => box('vtta', [...textEncoder.encode(notes)
 const udta = (muxer: IsobmffMuxer) => {
 	const boxes: Box[] = [];
 
+	const metadataFormat = muxer.format._options.metadataFormat ?? 'auto';
+	const metadataTags = muxer.output._metadataTags;
+
 	// Depending on the format, metadata tags are written differently
-	if (muxer.isQuickTime) {
+	if (metadataFormat === 'mdir' || (metadataFormat === 'auto' && !muxer.isQuickTime)) {
+		const metaBox = metaMdir(metadataTags);
+		if (metaBox) boxes.push(metaBox);
+	} else if (metadataFormat === 'mdta') {
+		const metaBox = metaMdta(metadataTags);
+		if (metaBox) boxes.push(metaBox);
+	} else if (metadataFormat === 'udta' || (metadataFormat === 'auto' && muxer.isQuickTime)) {
 		addQuickTimeMetadataTagBoxes(boxes, muxer.output._metadataTags);
-	} else {
-		const metaBox = meta(muxer.output._metadataTags);
-		if (metaBox) {
-			boxes.push(metaBox);
-		}
 	}
 
 	if (boxes.length === 0) {
@@ -1381,9 +1385,14 @@ const DATA_BOX_MIME_TYPE_MAP: Record<string, number> = {
 	'image/bmp': 27,
 };
 
-/** Metadata Box */
+/**
-const meta = (tags: MetadataTags) => {
+ * Generates key-value metadata for inclusion in the "meta" box.
-	const boxes: Box[] = [];
+ */
+const generateMetadataPairs = (tags: MetadataTags, isMdta: boolean) => {
+	const pairs: {
+		key: string;
+		value: Box;
+	}[] = [];
 
 	// https://exiftool.org/TagNames/QuickTime.html (QuickTime ItemList Tags)
 	// This is the metadata format used for MP4 files
@@ -1391,39 +1400,42 @@ const meta = (tags: MetadataTags) => {
 	for (const { key, value } of keyValueIterator(tags)) {
 		switch (key) {
 			case 'title': {
-				boxes.push(metadataTagStringBoxLong('©nam', value));
+				pairs.push({ key: isMdta ? 'title' : '©nam', value: dataStringBoxLong(value) });
 			}; break;
 
 			case 'description': {
-				boxes.push(metadataTagStringBoxLong('©des', value));
+				pairs.push({ key: isMdta ? 'description' : '©des', value: dataStringBoxLong(value) });
 			}; break;
 
 			case 'artist': {
-				boxes.push(metadataTagStringBoxLong('©ART', value));
+				pairs.push({ key: isMdta ? 'artist' : '©ART', value: dataStringBoxLong(value) });
 			}; break;
 
 			case 'album': {
-				boxes.push(metadataTagStringBoxLong('©alb', value));
+				pairs.push({ key: isMdta ? 'album' : '©alb', value: dataStringBoxLong(value) });
 			}; break;
 
 			case 'albumArtist': {
-				boxes.push(metadataTagStringBoxLong('aART', value));
+				pairs.push({ key: isMdta ? 'album_artist' : 'aART', value: dataStringBoxLong(value) });
 			}; break;
 
 			case 'comment': {
-				boxes.push(metadataTagStringBoxLong('©cmt', value));
+				pairs.push({ key: isMdta ? 'comment' : '©cmt', value: dataStringBoxLong(value) });
 			}; break;
 
 			case 'genre': {
-				boxes.push(metadataTagStringBoxLong('©gen', value));
+				pairs.push({ key: isMdta ? 'genre' : '©gen', value: dataStringBoxLong(value) });
 			}; break;
 
 			case 'lyrics': {
-				boxes.push(metadataTagStringBoxLong('©lyr', value));
+				pairs.push({ key: isMdta ? 'lyrics' : '©lyr', value: dataStringBoxLong(value) });
 			}; break;
 
 			case 'date': {
-				boxes.push(metadataTagStringBoxLong('©day', value.toISOString().slice(0, 10)));
+				pairs.push({
+					key: isMdta ? 'date' : '©day',
+					value: dataStringBoxLong(value.toISOString().slice(0, 10)),
+				});
 			}; break;
 
 			case 'images': {
@@ -1432,40 +1444,45 @@ const meta = (tags: MetadataTags) => {
 						continue;
 					}
 
-					boxes.push(box('covr', undefined, [
+					pairs.push({ key: 'covr', value: box('data', [
-						box('data', [
+						u32(DATA_BOX_MIME_TYPE_MAP[image.mimeType] ?? 0), // Type indicator
-							u32(DATA_BOX_MIME_TYPE_MAP[image.mimeType] ?? 0), // Type indicator
+						u32(0), // Locale indicator
-							u32(0), // Locale indicator
+						Array.from(image.data), // Kinda slow, hopefully temp
-							Array.from(image.data), // Kinda slow, hopefully temp
+					]) });
-						]),
-					]));
 				}
 			}; break;
 
 			case 'trackNumber': {
-				boxes.push(box('trkn', undefined, [
+				if (isMdta) {
-					box('data', [
+					const string = tags.tracksTotal !== undefined
+						? `${value}/${tags.tracksTotal}`
+						: value.toString();
+
+					pairs.push({ key: 'track', value: dataStringBoxLong(string) });
+				} else {
+					pairs.push({ key: 'trkn', value: box('data', [
 						u32(0), // 8 bytes empty
 						u32(0),
 						u16(0), // Empty
 						u16(value),
 						u16(tags.tracksTotal ?? 0),
 						u16(0), // Empty
-					]),
+					]) });
-				]));
+				}
 			}; break;
 
 			case 'discNumber': {
-				boxes.push(box('disc', undefined, [
+				if (!isMdta) {
-					box('data', [
+					// Only written for mdir
+					pairs.push({ key: 'disc', value: box('data', [
 						u32(0), // 8 bytes empty
 						u32(0),
 						u16(0), // Empty
 						u16(value),
 						u16(tags.discsTotal ?? 0),
 						u16(0), // Empty
-					]),
+					]) });
-				]));
+				}
 			}; break;
 
 			case 'tracksTotal':
@@ -1484,49 +1501,74 @@ const meta = (tags: MetadataTags) => {
 	if (tags.raw) {
 		for (const key in tags.raw) {
 			const value = tags.raw[key];
-			if (value == null || key.length !== 4 || boxes.some(x => x.type === key)) {
+			if (value == null || (!isMdta && key.length !== 4) || pairs.some(x => x.key === key)) {
 				continue;
 			}
 
 			if (typeof value === 'string') {
-				boxes.push(metadataTagStringBoxLong(key, value));
+				pairs.push({ key, value: dataStringBoxLong(value) });
 			} else if (value instanceof Uint8Array) {
-				boxes.push(box(key, undefined, [
+				pairs.push({ key, value: box('data', [
-					box('data', [
+					u32(0), // Type indicator
-						u32(0), // Type indicator
+					u32(0), // Locale indicator
-						u32(0), // Locale indicator
+					Array.from(value),
-						Array.from(value),
+				]) });
-					]),
-				]));
 			} else if (value instanceof RichImageData) {
-				boxes.push(box(key, undefined, [
+				pairs.push({ key, value: box('data', [
-					box('data', [
+					u32(DATA_BOX_MIME_TYPE_MAP[value.mimeType] ?? 0), // Type indicator
-						u32(DATA_BOX_MIME_TYPE_MAP[value.mimeType] ?? 0), // Type indicator
+					u32(0), // Locale indicator
-						u32(0), // Locale indicator
+					Array.from(value.data), // Kinda slow, hopefully temp
-						Array.from(value.data), // Kinda slow, hopefully temp
+				]) });
-					]),
-				]));
 			}
 		}
 	}
 
-	if (boxes.length === 0) {
+	return pairs;
+};
+
+/** Metadata Box (mdir format) */
+const metaMdir = (tags: MetadataTags) => {
+	const pairs = generateMetadataPairs(tags, false);
+
+	if (pairs.length === 0) {
 		return null;
 	}
 
+	// fullBox format
 	return fullBox('meta', 0, 0, undefined, [
-		hdlr(false, 'mdir', '', 'appl'),
+		hdlr(false, 'mdir', '', 'appl'), // mdir handler
-		box('ilst', undefined, boxes),
+		box('ilst', undefined, pairs.map(pair => box(pair.key, undefined, [pair.value]))), // Item list without keys box
 	]);
 };
 
-const metadataTagStringBoxLong = (name: string, value: string) => {
+/** Metadata Box (mdta format with keys box) */
-	return box(name, undefined, [
+const metaMdta = (tags: MetadataTags) => {
-		box('data', [
+	const pairs = generateMetadataPairs(tags, true);
-			u32(1), // Type indicator (UTF-8)
+
-			u32(0), // Locale indicator
+	if (pairs.length === 0) {
-			...textEncoder.encode(value),
+		return null;
-		]),
+	}
+
+	// box without version and flags
+	return box('meta', undefined, [
+		hdlr(false, 'mdta', ''), // mdta handler
+		fullBox('keys', 0, 0, [
+			u32(pairs.length),
+		], pairs.map(pair => box('mdta', [ // Hacky since these aren't boxes technically, but if not box why box-shaped?
+			...textEncoder.encode(pair.key),
+		]))),
+		box('ilst', undefined, pairs.map((pair, i) => {
+			const boxName = String.fromCharCode(...u32(i + 1));
+			return box(boxName, undefined, [pair.value]);
+		})),
+	]);
+};
+
+const dataStringBoxLong = (value: string) => {
+	return box('data', [
+		u32(1), // Type indicator (UTF-8)
+		u32(0), // Locale indicator
+		...textEncoder.encode(value),
 	]);
 };
 

src/isobmff/isobmff-demuxer.ts

@@ -2323,7 +2323,7 @@ export class IsobmffDemuxer extends Demuxer {
 						}; break;
 
 						case 'trkn': {
-							if (data instanceof Uint8Array) {
+							if (data instanceof Uint8Array && data.length >= 6) {
 								const view = toDataView(data);
 
 								const trackNumber = view.getUint16(2, false);
@@ -2340,7 +2340,7 @@ export class IsobmffDemuxer extends Demuxer {
 
 						case 'disc':
 						case 'disk': {
-							if (data instanceof Uint8Array) {
+							if (data instanceof Uint8Array && data.length >= 6) {
 								const view = toDataView(data);
 
 								const discNumber = view.getUint16(2, false);

src/isobmff/isobmff-muxer.ts

@@ -138,7 +138,7 @@ export const intoTimescale = (timeInSeconds: number, timescale: number, round =
 };
 
 export class IsobmffMuxer extends Muxer {
-	private format: IsobmffOutputFormat;
+	format: IsobmffOutputFormat;
 	private writer: Writer;
 	private boxWriter: IsobmffBoxWriter;
 	private fastStart: NonNullable<IsobmffOutputFormatOptions['fastStart']>;

src/output-format.ts

@@ -140,6 +140,18 @@ export type IsobmffOutputFormatOptions = {
 	 */
 	minimumFragmentDuration?: number;
 
+	/**
+	 * The metadata format to use for writing metadata tags.
+	 *
+	 * - `'auto'` (default): Behaves like `'mdir'` for MP4 and like `'udta'` for QuickTime, matching FFmpeg's default
+	 * behavior.
+	 * - `'mdir'`: Write tags into `moov/udta/meta` using the 'mdir' handler format.
+	 * - `'mdta'`: Write tags into `moov/udta/meta` using the 'mdta' handler format, equivalent to FFmpeg's
+	 * `use_metadata_tags` flag. This allows for custom keys of arbitrary length.
+	 * - `'udta'`: Write tags directly into `moov/udta`.
+	 */
+	metadataFormat?: 'auto' | 'mdir' | 'mdta' | 'udta';
+
 	/**
 	 * Will be called once the ftyp (File Type) box of the output file has been written.
 	 *
@@ -216,6 +228,14 @@ export abstract class IsobmffOutputFormat extends OutputFormat {
 		if (options.onMoof !== undefined && typeof options.onMoof !== 'function') {
 			throw new TypeError('options.onMoof, when provided, must be a function.');
 		}
+		if (
+			options.metadataFormat !== undefined
+			&& !['mdir', 'mdta', 'udta', 'auto'].includes(options.metadataFormat)
+		) {
+			throw new TypeError(
+				'options.metadataFormat, when provided, must be either \'auto\', \'mdir\', \'mdta\', or \'udta\'.',
+			);
+		}
 
 		super();