diff --git a/dev/convert.html b/dev/convert.html index 5558bab..a50cbd5 100644 --- a/dev/convert.html +++ b/dev/convert.html @@ -24,23 +24,57 @@ chunked: true, chunkSize: 2**20 }); - const outputFormat = new Mediabunny.Mp4OutputFormat({}); + const outputFormat = new Mediabunny.WavOutputFormat({}); const button = document.createElement('button'); button.textContent = 'Cancel'; button.onclick = () => conversion.cancel(); document.body.append(button); + const yo = document.createElement('canvas'); + yo.width = 512; + yo.height = 512; + const context = yo.getContext('2d', { alpha: false }); + context.fillStyle = 'red'; + context.fillRect(100, 100, 300, 300); + const blob = await new Promise(resolve => yo.toBlob(resolve, 'image/jpeg', 0.92)); + const blobData = new Uint8Array(await blob.arrayBuffer()); + + const mh = new Blob([blobData], { type: 'image/jpeg' }); + console.log(URL.createObjectURL(mh)) + + const output = new Mediabunny.Output({ + format: outputFormat, + target + }); + output.setMetadataTags({ + title: 'Bigggy', + artist: 'Buck Bunny', + images: [{ + data: blobData, + kind: 'coverFront', + mimeType: 'image/jpeg' + }], + trackNumber: 4, + tracksTotal: 10, + discNumber: 5, + discNumberMax: 8, + lyrics: "There's no way\nThat it's not going there", + raw: { + 'ENCODER': 'Mediabunny epic own encoder' + } + }); + const conversion = await Mediabunny.Conversion.init({ input: new Mediabunny.Input({ formats: Mediabunny.ALL_FORMATS, source }), - output: new Mediabunny.Output({ - format: outputFormat, - target - }), + output, audio: { + codec: 'pcm-s16', + //sampleRate: 16000, + //numberOfChannels: 1, //discard: true, //codec: 'opus', //bitrate: 128000, @@ -74,8 +108,9 @@ }, */ video: () => ({ - forceTranscode: true, - codec: 'avc', + //discard: true, + //forceTranscode: true, + //codec: 'avc', //fit: 'contain', //frameRate: 27.123, //width: 320, @@ -98,7 +133,7 @@ }), trim: { start: 0, - end: 5 + end: 10 }, }); console.log(conversion); diff --git a/dev/demux.html b/dev/demux.html index 9504e3c..b844f7d 100644 --- a/dev/demux.html +++ b/dev/demux.html @@ -8,6 +8,39 @@ document.body.append(fileInput); fileInput.addEventListener('change', async () => { + const file = fileInput.files[0]; + const input = new Mediabunny.Input({ + formats: Mediabunny.ALL_FORMATS, + source: new Mediabunny.BlobSource(file), + }); + + const audioTrack = await input.getPrimaryAudioTrack(); + const sink = new Mediabunny.EncodedPacketSink(audioTrack); + + for await (const packet of sink.packets()) { + console.log(packet); + } + + /* + const sink = new Mediabunny.EncodedPacketSink(videoTrack); + + for await (const packet of sink.packets()) { + console.log(packet) + } + */ + + /* + for await (const sample of sink.samples(0.99)) { + console.log(sample); + + if (sample.timestamp >= 2) { + break; + } + + } + */ + //console.log(await sink.getSample(1)) + /* const screenStream = await navigator.mediaDevices.getDisplayMedia({ video: true }); const micStream = await navigator.mediaDevices.getUserMedia({ audio: true }); diff --git a/docs/api-config.json b/docs/api-config.json index e160b55..1cd9433 100644 --- a/docs/api-config.json +++ b/docs/api-config.json @@ -13,6 +13,7 @@ "Media sinks": "Methods for extracting media data from input files.", "Media sources": "Methods for adding media data to output files.", "Conversion": "A simple API for converting and transforming media files.", + "Metadata tags": "Descriptive metadata tags attached to media files.", "Codecs": "Codecs understood by Mediabunny.", "Encoding": "Encoder configuration and encodability checks.", "Custom coders": "API for adding custom encoders and decoders.", diff --git a/docs/guide/converting-media-files.md b/docs/guide/converting-media-files.md index 20976a9..8f900fa 100644 --- a/docs/guide/converting-media-files.md +++ b/docs/guide/converting-media-files.md @@ -286,6 +286,44 @@ In this case, the output will be 15 seconds long. If only `start` is set, the clip will run until the end of the input file. If only `end` is set, the clip will start at the beginning of the input file. +## Metadata tags + +By default, any [descriptive metadata tags](../api/MetadataTags.md) of the input will be copied to the output. If you want to further control the metadata tags written to the output, you can use the `tags` options: + +```ts +// Set your own metadata: +const conversion = await Conversion.init({ + // ... + tags: () => ({ + title: 're:Turning', + artist: 'Alexander Panos', + }), + // ... +}); + +// Or, augment the input's metadata: +const conversion = await Conversion.init({ + // ... + tags: inputTags => ({ + ...inputTags, // Keep the existing metadata + images: [{ // And add cover art + data: new Uint8Array(...), + mimeType: 'image/jpeg', + kind: 'coverFront', + }], + comment: undefined, // And remove any comments + }), + // ... +}); + +// Or, remove all metadata +const conversion = await Conversion.init({ + // ... + tags: () => ({}), + // ... +}); +``` + ## Discarded tracks If an input track is excluded from the output file, it is considered *discarded*. The list of discarded tracks can be accessed after initializing a `Conversion`: diff --git a/docs/guide/reading-media-files.md b/docs/guide/reading-media-files.md index d9af1c8..dc8fd42 100644 --- a/docs/guide/reading-media-files.md +++ b/docs/guide/reading-media-files.md @@ -60,6 +60,12 @@ await input.computeDuration(); // => 1905.4615 ``` More specifically, the duration is defined as the maximum end timestamp across all tracks. +Mediabunny also lets you read descriptive metadata tags from media files, such as title, artist, or cover art: +```ts +await input.getMetadataTags(); // => MetadataTags +``` +For more info, see [`MetadataTags`](../api/MetadataTags). + ## Reading track metadata You can extract the list of all media tracks in the file like so: diff --git a/docs/guide/writing-media-files.md b/docs/guide/writing-media-files.md index 5e58081..c70a4b2 100644 --- a/docs/guide/writing-media-files.md +++ b/docs/guide/writing-media-files.md @@ -106,6 +106,25 @@ output.addAudioTrack(audioSource); Adding tracks to an `Output` will throw if the track is not compatible with the output format. Be sure to respect the [properties](./output-formats#format-properties) of the output format when adding tracks. ::: +## Setting metadata tags + +Mediabunny lets you write additional descriptive metadata tags to an output file, such as title, artist, or cover art: + +```ts +output.setMetadataTags({ + title: 'Big Buck Bunny', + artist: 'Blender Foundation', + date: new Date('2008-05-20'), + images: [{ + data: new Uint8Array([...]), + mimeType: 'image/jpeg', + kind: 'coverFront', + }], +}); +``` + +For more info on which tags you can write, see [`MetadataTags`](../api/MetadataTags). + ## Starting an output After all tracks have been added to the `Output`, you need to *start* it. Starting an output spins up the writing process, allowing you to now start sending media data to the output file. It also prevents you from adding any new tracks to it. diff --git a/docs/index.md b/docs/index.md index 74fe63b..418e426 100644 --- a/docs/index.md +++ b/docs/index.md @@ -105,6 +105,7 @@ const sponsors = { { image: 'https://avatars.githubusercontent.com/u/9549394', name: 'studnitz', url: 'https://github.com/studnitz' }, { image: 'https://avatars.githubusercontent.com/u/504909', name: 'Hirbod', url: 'https://github.com/hirbod' }, { image: 'https://avatars.githubusercontent.com/u/30229596', name: 'Pablo Bonilla', url: 'https://github.com/devPablo' }, + { image: 'https://avatars.githubusercontent.com/u/38181164', name: 'wcw', url: 'https://github.com/asd55667' }, { image: 'https://avatars.githubusercontent.com/u/63088713', name: 'taf2000', url: 'https://github.com/taf2000' }, { image: 'https://avatars.githubusercontent.com/u/58149663', name: 'H7GhosT', url: 'https://github.com/H7GhosT' }, { image: 'https://avatars.githubusercontent.com/u/91711202', name: 'ihasq', url: 'https://github.com/ihasq' }, diff --git a/examples/metadata-extraction/index.html b/examples/metadata-extraction/index.html index e6deed9..64e4c0f 100644 --- a/examples/metadata-extraction/index.html +++ b/examples/metadata-extraction/index.html @@ -57,7 +57,7 @@ @reference "../base.css"; ul { - @apply py-1 px-4 bg-gray-500/10 rounded-lg; + @apply py-1 px-4 bg-gray-500/10 rounded-lg max-w-lg; } b { diff --git a/examples/metadata-extraction/metadata-extraction.ts b/examples/metadata-extraction/metadata-extraction.ts index 8cef42c..a071e95 100644 --- a/examples/metadata-extraction/metadata-extraction.ts +++ b/examples/metadata-extraction/metadata-extraction.ts @@ -80,6 +80,37 @@ const extractMetadata = (resource: File | string) => { : {} ), }))), + 'Metadata tags': input.getMetadataTags().then((tags) => { + const result = { + 'Title': tags.title, + 'Description': tags.description, + 'Artist': tags.artist, + 'Album': tags.album, + 'Album artist': tags.albumArtist, + 'Track number': tags.trackNumber, + 'Tracks total': tags.tracksTotal, + 'Disc number': tags.discNumber, + 'Discs total': tags.discsTotal, + 'Genre': tags.genre, + 'Date': tags.date?.toISOString().slice(0, 10), + 'Lyrics': tags.lyrics, + 'Comment': tags.comment, + 'Images': tags.images?.map((image) => { + const blob = new Blob([image.data], { type: image.mimeType }); + const element = new Image(); + element.src = URL.createObjectURL(blob); + + return element; + }), + 'Raw tag count': tags.raw && Object.keys(tags.raw).length, + }; + + if (Object.values(result).some(x => x !== undefined)) { + return result; + } else { + return undefined; + } + }), }; fileNameElement.textContent = resource instanceof File ? resource.name : resource; @@ -93,7 +124,9 @@ const extractMetadata = (resource: File | string) => { // Creates an HTML element to display any given value const renderValue = (value: unknown) => { - if (Array.isArray(value)) { + if (value instanceof HTMLElement) { + return value; + } else if (Array.isArray(value)) { const arrayAsObject: Record = Object.fromEntries( value.map((item, index) => [(index + 1).toString(), item]), ); @@ -116,6 +149,7 @@ const renderObject = (object: Record) => { for (const key of keys) { const value = object[key]; const listItem = document.createElement('li'); + listItem.style.wordBreak = 'break-word'; const keySpan = document.createElement('b'); keySpan.textContent = `${key}: `; @@ -132,6 +166,10 @@ const renderObject = (object: Record) => { // Replace the loading text with the resolved value listItem.removeChild(loadingSpan); listItem.appendChild(renderValue(resolvedValue)); + + if (resolvedValue === undefined) { + listElement.removeChild(listItem); + } }).catch((error) => { console.error(error); @@ -146,7 +184,9 @@ const renderObject = (object: Record) => { listItem.appendChild(renderValue(value)); } - listElement.appendChild(listItem); + if (value !== undefined) { + listElement.appendChild(listItem); + } } return listElement; diff --git a/examples/thumbnail-generation/thumbnail-generation.ts b/examples/thumbnail-generation/thumbnail-generation.ts index b71b973..ea7716c 100644 --- a/examples/thumbnail-generation/thumbnail-generation.ts +++ b/examples/thumbnail-generation/thumbnail-generation.ts @@ -96,6 +96,13 @@ const generateThumbnails = async (resource: File | string) => { timestampElement.className = 'absolute bottom-0 right-0 bg-black/30 text-white px-1 py-0.5 text-[11px] rounded-tl-lg'; container.append(timestampElement); + } else { + // Add something to indicate that the thumbnail is missing + const p = document.createElement('p'); + p.textContent = '?'; + p.className = 'absolute inset-0 flex items-center justify-center text-3xl opacity-50'; + + container.append(p); } i++; diff --git a/package-lock.json b/package-lock.json index 80fe289..72f8bfe 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "mediabunny", - "version": "1.13.3", + "version": "1.14.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "mediabunny", - "version": "1.13.3", + "version": "1.14.4", "license": "MPL-2.0", "workspaces": [ "packages/*" @@ -7749,9 +7749,9 @@ } }, "node_modules/mediabunny": { - "version": "1.13.2", - "resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.13.2.tgz", - "integrity": "sha512-quUwxiyA76+iAh2REQAFifHNthgVqzET37ys6v+GC8ljJpOXuP/g/XNsITc+/6r18gGuYQyF/9C4T/CeiTf7gA==", + "version": "1.14.3", + "resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.14.3.tgz", + "integrity": "sha512-kCvieRo6X1QDcdWLjn7o2BY/VCDeyU9nNGBVjOIOiWPoTtIekHR+viKAYaZafLEm0poBv+O2PwttL37PaSo/kA==", "license": "MPL-2.0", "peer": true, "workspaces": [ @@ -12242,7 +12242,7 @@ }, "packages/mp3-encoder": { "name": "@mediabunny/mp3-encoder", - "version": "1.13.3", + "version": "1.14.4", "license": "MPL-2.0", "devDependencies": { "@types/emscripten": "^1.40.1" diff --git a/package.json b/package.json index 01f6d9a..0ffae19 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "mediabunny", "author": "Vanilagy", - "version": "1.13.3", + "version": "1.14.4", "description": "Pure TypeScript media toolkit for reading, writing, and converting media files, directly in the browser.", "type": "module", "workspaces": [ diff --git a/packages/mp3-encoder/package.json b/packages/mp3-encoder/package.json index 8a85fe7..cc977a9 100644 --- a/packages/mp3-encoder/package.json +++ b/packages/mp3-encoder/package.json @@ -1,7 +1,7 @@ { "name": "@mediabunny/mp3-encoder", "author": "Vanilagy", - "version": "1.13.3", + "version": "1.14.4", "description": "MP3 encoder extension for Mediabunny, based on LAME.", "main": "./dist/bundles/mediabunny-mp3-encoder.mjs", "module": "./dist/bundles/mediabunny-mp3-encoder.mjs", diff --git a/scripts/generate-api-docs.ts b/scripts/generate-api-docs.ts index b2291c3..392a61f 100644 --- a/scripts/generate-api-docs.ts +++ b/scripts/generate-api-docs.ts @@ -1,6 +1,6 @@ -// This script has been 100% vibe-coded with Claude, meaning I literally haven't looked at any of the code. It's -// probably a mess, but it solves a one-off problem where only the output matters, and the output is indeed good, which -// is the point of a custom script for this: full, precise control. +// This script has been 100% vibe-coded with Claude (and Gemini!), meaning I literally haven't looked at any of the +// code. It's probably a mess, but it solves a one-off problem where only the output matters, and the output is indeed +// good, which is the point of a custom script for this: full, precise control. /* eslint-disable @typescript-eslint/restrict-template-expressions */ /* eslint-disable @typescript-eslint/no-unused-vars */ @@ -386,7 +386,34 @@ const generateDocs = (entryFiles: string[], apiConfigFile: string) => { } }); - // Phase 2: Generate documentation for each symbol + // Data structures for "Used by" feature + const usedByReferences = new Map>(); + const generatedDocs = new Map(); + + const addUsage = ( + used: string, + user: string, + context: string, + type: 'constructor' | 'method' | 'property' | 'extends' | 'type_param' | 'type_alias' | 'variable' | 'function', + ) => { + // No self-references + if (used === user) return; + + if (!usedByReferences.has(used)) { + usedByReferences.set(used, new Set()); + } + const usageSet = usedByReferences.get(used)!; + + // Check for duplicates before adding + for (const item of usageSet) { + if (item.user === user && item.context === context && item.type === type) { + return; + } + } + usageSet.add({ user, context, type }); + }; + + // Phase 2: Generate documentation for each symbol (and collect usage data) allSymbols.forEach((exportSymbol) => { const declaration = exportSymbol.valueDeclaration || exportSymbol.declarations?.[0]; if (!declaration) return; @@ -492,9 +519,10 @@ const generateDocs = (entryFiles: string[], apiConfigFile: string) => { const allReferences = filterToExportedTypes([...new Set(allTypeStrings.flatMap(findAllTypeReferences))], variableName); markdown += formatReferences(allReferences); - const outputPath = path.join(outputDir, `${variableName}.md`); - fs.writeFileSync(outputPath, markdown); - console.log(`Generated: ${outputPath}`); + // In tandem: update usage map + allReferences.forEach(ref => addUsage(ref, variableName, variableName, 'function')); + + generatedDocs.set(variableName, markdown); } } else { // Handle regular variables @@ -507,9 +535,10 @@ const generateDocs = (entryFiles: string[], apiConfigFile: string) => { const references = filterToExportedTypes(findAllTypeReferences(variableValue), variableName); markdown += formatReferences(references); - const outputPath = path.join(outputDir, `${variableName}.md`); - fs.writeFileSync(outputPath, markdown); - console.log(`Generated: ${outputPath}`); + // In tandem: update usage map + references.forEach(ref => addUsage(ref, variableName, variableName, 'variable')); + + generatedDocs.set(variableName, markdown); } return; @@ -643,6 +672,9 @@ const generateDocs = (entryFiles: string[], apiConfigFile: string) => { if (typeParamReferencesText) { typeParameters += typeParamReferencesText; } + + // In tandem: update usage map + typeParamReferences.forEach(ref => addUsage(ref, className, className, 'type_param')); } // Check for extends clause (only for classes/interfaces) @@ -653,6 +685,11 @@ const generateDocs = (entryFiles: string[], apiConfigFile: string) => { if (extendsClauseNode && extendsClauseNode.types[0]) { const superClassName = extendsClauseNode.types[0].expression.getText(); extendsClause = `\n\n**Extends:** [\`${superClassName}\`](./${superClassName}.md)\n`; + + // In tandem: update usage map + if (exportedTypes.has(superClassName)) { + addUsage(superClassName, className, className, 'extends'); + } } } @@ -835,6 +872,9 @@ const generateDocs = (entryFiles: string[], apiConfigFile: string) => { const references = filterToExportedTypes(findAllTypeReferences(paramType), className); const referencesText = formatReferences(references); + // In tandem: update usage map + references.forEach(ref => addUsage(ref, className, paramName, 'property')); + const propertyContent = `### \`${paramName}\`\n\n\`\`\`ts\n${propertyDef}\n\`\`\`${desc ? `\n\n${desc}` : ''}${referencesText}`; properties.push(propertyContent); }); @@ -905,6 +945,9 @@ const generateDocs = (entryFiles: string[], apiConfigFile: string) => { const overloadReferences = filterToExportedTypes([...new Set(ctorTypeStrings.flatMap(findAllTypeReferences))], className); constructorBlock += formatReferences(overloadReferences, linkedTypes); + // In tandem: update usage map + overloadReferences.forEach(ref => addUsage(ref, className, className, 'constructor')); + constructorBlocks.push(constructorBlock); }); @@ -950,6 +993,9 @@ const generateDocs = (entryFiles: string[], apiConfigFile: string) => { const references = filterToExportedTypes(findAllTypeReferences(type), className); const referencesText = formatReferences(references, linkedTypes); + // In tandem: update usage map + references.forEach(ref => addUsage(ref, className, name, 'property')); + // Check if this is an event handler (starts with "on" and can be a function) const isEventHandler = name.startsWith('on') && ( type.includes('=>') @@ -993,6 +1039,9 @@ const generateDocs = (entryFiles: string[], apiConfigFile: string) => { const references = filterToExportedTypes(findAllTypeReferences(type), className); const referencesText = formatReferences(references, linkedTypes); + // In tandem: update usage map + references.forEach(ref => addUsage(ref, className, name, 'property')); + const inheritedBadge = ''; properties.push(`### \`${name}\`${inheritedBadge}\n\n\`\`\`ts\n${accessorDef}\n\`\`\`${desc ? `\n\n${desc}` : ''}${referencesText}`); } else if (ts.isSetAccessorDeclaration(member) && member.name) { @@ -1013,6 +1062,9 @@ const generateDocs = (entryFiles: string[], apiConfigFile: string) => { const references = filterToExportedTypes(findAllTypeReferences(paramType), className); const referencesText = formatReferences(references, linkedTypes); + // In tandem: update usage map + references.forEach(ref => addUsage(ref, className, name, 'property')); + const inheritedBadge = ''; properties.push(`### \`${name}\`${inheritedBadge}\n\n\`\`\`ts\n${accessorDef}\n\`\`\`${desc ? `\n\n${desc}` : ''}${referencesText}`); } else if ((ts.isMethodDeclaration(member) || ts.isMethodSignature(member)) && member.name) { @@ -1106,6 +1158,9 @@ const generateDocs = (entryFiles: string[], apiConfigFile: string) => { const allReferences = filterToExportedTypes([...new Set(allTypeStrings.flatMap(findAllTypeReferences))], className); methodContent += formatReferences(allReferences, linkedTypes); + // In tandem: update usage map + allReferences.forEach(ref => addUsage(ref, className, name, 'method')); + if (isStatic) { staticMethods.push(methodContent); } else { @@ -1174,6 +1229,9 @@ const generateDocs = (entryFiles: string[], apiConfigFile: string) => { const references = filterToExportedTypes(findAllTypeReferences(cleanedType), className); const referencesText = formatReferences(references); + // In tandem: update usage map + references.forEach(ref => addUsage(ref, className, propName, 'property')); + // Check if this is an event handler (starts with "on" and can be a function) const isEventHandler = propName.startsWith('on') && ( cleanedType.includes('=>') @@ -1252,19 +1310,39 @@ const generateDocs = (entryFiles: string[], apiConfigFile: string) => { // Add subclasses section for classes that have subclasses if (ts.isClassDeclaration(declaration) && classHierarchy.has(className)) { - const subclasses = classHierarchy.get(className)!; - // Sort by definition order instead of alphabetically - subclasses.sort((a, b) => { - const orderA = symbolOrderMap.get(a); - const orderB = symbolOrderMap.get(b); - if (orderA === undefined) throw new Error(`Symbol '${a}' not found in entry files export order`); - if (orderB === undefined) throw new Error(`Symbol '${b}' not found in entry files export order`); - return orderA - orderB; - }); - markdown += `\n## Subclasses\n\n`; - subclasses.forEach((sub) => { - markdown += `- [\`${sub}\`](./${sub}.md)\n`; - }); + // Recursively build hierarchical list + const buildHierarchicalList = (parentClass: string, depth = 0, visited = new Set()): string => { + if (visited.has(parentClass)) return ''; // Prevent infinite loops + visited.add(parentClass); + + const directChildren = classHierarchy.get(parentClass) || []; + if (directChildren.length === 0) return ''; + + // Sort children by definition order + const sortedChildren = [...directChildren].sort((a, b) => { + const orderA = symbolOrderMap.get(a); + const orderB = symbolOrderMap.get(b); + if (orderA === undefined) throw new Error(`Symbol '${a}' not found in entry files export order`); + if (orderB === undefined) throw new Error(`Symbol '${b}' not found in entry files export order`); + return orderA - orderB; + }); + + let result = ''; + const indent = ' '.repeat(depth); + + for (const child of sortedChildren) { + result += `${indent}- [\`${child}\`](./${child}.md)\n`; + // Recursively add children of this child + result += buildHierarchicalList(child, depth + 1, visited); + } + + return result; + }; + + const hierarchicalList = buildHierarchicalList(className); + if (hierarchicalList) { + markdown += `\n## Subclasses\n\n${hierarchicalList}`; + } } // Add instances section for classes that have instances @@ -1284,6 +1362,11 @@ const generateDocs = (entryFiles: string[], apiConfigFile: string) => { }); } + // Add placeholder for "Used by" section for classes and interfaces + if (ts.isClassDeclaration(declaration) || ts.isInterfaceDeclaration(declaration)) { + markdown += '\n\n'; + } + // Add type definition for type aliases if (ts.isTypeAliasDeclaration(declaration) && declaration.type) { const resolvedType = typeChecker.getTypeAtLocation(declaration); @@ -1346,7 +1429,13 @@ const generateDocs = (entryFiles: string[], apiConfigFile: string) => { const typeReferences = filterToExportedTypes([...new Set(allTypeRefs)], className); const typeReferencesText = formatReferences(typeReferences); + // In tandem: update usage map + typeReferences.forEach(ref => addUsage(ref, className, className, 'type_alias')); + markdown += `\n\`\`\`ts\n${typeDefinition}\n\`\`\`${typeReferencesText}`; + + // Add placeholder for "Used by" section for type aliases + markdown += '\n\n'; } if (typeParameters) { @@ -1372,12 +1461,89 @@ const generateDocs = (entryFiles: string[], apiConfigFile: string) => { if (methods.length > 0) { markdown += `\n## Methods\n\n${methods.join('\n\n')}\n`; } - const outputPath = path.join(outputDir, `${className}.md`); - fs.writeFileSync(outputPath, markdown); - console.log(`Generated: ${outputPath}`); + + generatedDocs.set(className, markdown); } }); + // Phase 3: Assemble final docs with "Used by" sections and write files + generatedDocs.forEach((markdown, symbolName) => { + const usages = usedByReferences.get(symbolName); + let usedByMarkdown = ''; + + if (usages && usages.size > 0) { + const symbolSubclasses = classHierarchy.get(symbolName) || []; + + const usedByLines = [...usages] + .filter((usage) => { + // Filter out subclasses from "Used by" since they already appear in "Subclasses" section + if (usage.type === 'extends' && symbolSubclasses.includes(usage.user)) { + return false; + } + + // Filter out top-level type references when there are more specific contexts available + // This prevents redundancy where both "TypeName" and "TypeName.property" appear + if (usage.type === 'type_alias' || usage.type === 'type_param') { + // Check if there are more specific usages from the same user (property, method, constructor, etc.) + const hasMoreSpecificUsage = [...usages].some(otherUsage => + otherUsage.user === usage.user + && otherUsage.type !== 'type_alias' + && otherUsage.type !== 'type_param' + && otherUsage.type !== 'extends', + ); + if (hasMoreSpecificUsage) { + return false; + } + } + + return true; + }) + .map((usage) => { + let displayText = ''; + let link = ''; + + switch (usage.type) { + case 'constructor': + displayText = `new ${usage.user}()`; + link = `./${usage.user}.md#constructor`; + break; + case 'method': + displayText = `${usage.user}.${usage.context}()`; + link = `./${usage.user}.md#${usage.context.toLowerCase()}`; + break; + case 'property': + case 'variable': + displayText = `${usage.user}.${usage.context}`; + link = `./${usage.user}.md#${usage.context.toLowerCase()}`; + break; + case 'function': + displayText = `${usage.user}()`; + link = `./${usage.user}.md`; + break; + case 'extends': + case 'type_param': + case 'type_alias': + displayText = usage.user; + link = `./${usage.user}.md`; + break; + } + return { text: `[\`${displayText}\`](${link})`, sortKey: displayText.toLowerCase() }; + }); + + if (usedByLines.length > 0) { + // Sort alphabetically by display text + usedByLines.sort((a, b) => a.sortKey.localeCompare(b.sortKey)); + const listItems = usedByLines.map(item => `- ${item.text}`).join('\n'); + usedByMarkdown = `\n## Used by\n\n${listItems}\n`; + } + } + + const finalMarkdown = markdown.replace('', usedByMarkdown); + const outputPath = path.join(outputDir, `${symbolName}.md`); + fs.writeFileSync(outputPath, finalMarkdown); + console.log(`Generated: ${outputPath}`); + }); + // Generate index.md with all exported symbols grouped by group const entriesByGroup = new Map>(); diff --git a/shared/mp3-misc.ts b/shared/mp3-misc.ts index 67358f9..433771b 100644 --- a/shared/mp3-misc.ts +++ b/shared/mp3-misc.ts @@ -158,3 +158,32 @@ export const readFrameHeader = (word: number, remainingBytes: number | null): { bytesAdvanced: 1, }; }; + +export const encodeSynchsafe = (unsynchsafed: number) => { + let mask = 0x7f; + let synchsafed = 0; + let unsynchsafedRest = unsynchsafed; + + while ((mask ^ 0x7fffffff) !== 0) { + synchsafed = unsynchsafedRest & ~mask; + synchsafed <<= 1; + synchsafed |= unsynchsafedRest & mask; + mask = ((mask + 1) << 8) - 1; + unsynchsafedRest = synchsafed; + } + + return synchsafed; +}; + +export const decodeSynchsafe = (synchsafed: number) => { + let mask = 0x7f000000; + let unsynchsafed = 0; + + while (mask !== 0) { + unsynchsafed >>= 1; + unsynchsafed |= synchsafed & mask; + mask >>= 8; + } + + return unsynchsafed; +}; diff --git a/src/adts/adts-demuxer.ts b/src/adts/adts-demuxer.ts index 794160c..5985df0 100644 --- a/src/adts/adts-demuxer.ts +++ b/src/adts/adts-demuxer.ts @@ -125,6 +125,10 @@ export class AdtsDemuxer extends Demuxer { return track.computeDuration(); } + + async getMetadataTags() { + return {}; // No tags in this one + } } class AdtsAudioTrackBacking implements InputAudioTrackBacking { diff --git a/src/codec-data.ts b/src/codec-data.ts index d315199..895569f 100644 --- a/src/codec-data.ts +++ b/src/codec-data.ts @@ -26,8 +26,27 @@ import { EncodedPacket, PacketType } from './packet'; // Rec. ITU-T H.265 // https://stackoverflow.com/questions/24884827 +export enum AvcNalUnitType { + IDR = 5, + SPS = 7, + PPS = 8, + SPS_EXT = 13, +} + +export enum HevcNalUnitType { + RASL_N = 8, + RASL_R = 9, + BLA_W_LP = 16, + RSV_IRAP_VCL23 = 23, + VPS_NUT = 32, + SPS_NUT = 33, + PPS_NUT = 34, + PREFIX_SEI_NUT = 39, + SUFFIX_SEI_NUT = 40, +} + /** Finds all NAL units in an AVC packet in Annex B format. */ -const findNalUnitsInAnnexB = (packetData: Uint8Array) => { +export const findNalUnitsInAnnexB = (packetData: Uint8Array) => { const nalUnits: Uint8Array[] = []; let i = 0; @@ -184,6 +203,21 @@ export type AvcDecoderConfigurationRecord = { sequenceParameterSetExt: Uint8Array[] | null; }; +export const extractAvcNalUnits = (packetData: Uint8Array, decoderConfig: VideoDecoderConfig) => { + if (decoderConfig.description) { + // Stream is length-prefixed. Let's extract the size of the length prefix from the decoder config + + const bytes = toUint8Array(decoderConfig.description); + const lengthSizeMinusOne = bytes[4]! & 0b11; + const lengthSize = (lengthSizeMinusOne + 1) as 1 | 2 | 3 | 4; + + return findNalUnitsInLengthPrefixed(packetData, lengthSize); + } else { + // Stream is in Annex B format + return findNalUnitsInAnnexB(packetData); + } +}; + const extractNalUnitTypeForAvc = (data: Uint8Array) => { return data[0]! & 0x1F; }; @@ -193,9 +227,9 @@ export const extractAvcDecoderConfigurationRecord = (packetData: Uint8Array) => try { const nalUnits = findNalUnitsInAnnexB(packetData); - const spsUnits = nalUnits.filter(unit => extractNalUnitTypeForAvc(unit) === 7); - const ppsUnits = nalUnits.filter(unit => extractNalUnitTypeForAvc(unit) === 8); - const spsExtUnits = nalUnits.filter(unit => extractNalUnitTypeForAvc(unit) === 13); + const spsUnits = nalUnits.filter(unit => extractNalUnitTypeForAvc(unit) === AvcNalUnitType.SPS); + const ppsUnits = nalUnits.filter(unit => extractNalUnitTypeForAvc(unit) === AvcNalUnitType.PPS); + const spsExtUnits = nalUnits.filter(unit => extractNalUnitTypeForAvc(unit) === AvcNalUnitType.SPS_EXT); if (spsUnits.length === 0) { return null; @@ -337,12 +371,6 @@ export const serializeAvcDecoderConfigurationRecord = (record: AvcDecoderConfigu return new Uint8Array(bytes); }; -const NALU_TYPE_VPS = 32; -const NALU_TYPE_SPS = 33; -const NALU_TYPE_PPS = 34; -const NALU_TYPE_SEI_PREFIX = 39; -const NALU_TYPE_SEI_SUFFIX = 40; - // Data specified in ISO 14496-15 export type HevcDecoderConfigurationRecord = { configurationVersion: number; @@ -369,7 +397,22 @@ export type HevcDecoderConfigurationRecord = { }[]; }; -const extractNalUnitTypeForHevc = (data: Uint8Array) => { +export const extractHevcNalUnits = (packetData: Uint8Array, decoderConfig: VideoDecoderConfig) => { + if (decoderConfig.description) { + // Stream is length-prefixed. Let's extract the size of the length prefix from the decoder config + + const bytes = toUint8Array(decoderConfig.description); + const lengthSizeMinusOne = bytes[21]! & 0b11; + const lengthSize = (lengthSizeMinusOne + 1) as 1 | 2 | 3 | 4; + + return findNalUnitsInLengthPrefixed(packetData, lengthSize); + } else { + // Stream is in Annex B format + return findNalUnitsInAnnexB(packetData); + } +}; + +export const extractNalUnitTypeForHevc = (data: Uint8Array) => { return (data[0]! >> 1) & 0x3F; }; @@ -380,12 +423,12 @@ export const extractHevcDecoderConfigurationRecord = ( try { const nalUnits = findNalUnitsInAnnexB(packetData); - const vpsUnits = nalUnits.filter(unit => extractNalUnitTypeForHevc(unit) === NALU_TYPE_VPS); - const spsUnits = nalUnits.filter(unit => extractNalUnitTypeForHevc(unit) === NALU_TYPE_SPS); - const ppsUnits = nalUnits.filter(unit => extractNalUnitTypeForHevc(unit) === NALU_TYPE_PPS); + const vpsUnits = nalUnits.filter(unit => extractNalUnitTypeForHevc(unit) === HevcNalUnitType.VPS_NUT); + const spsUnits = nalUnits.filter(unit => extractNalUnitTypeForHevc(unit) === HevcNalUnitType.SPS_NUT); + const ppsUnits = nalUnits.filter(unit => extractNalUnitTypeForHevc(unit) === HevcNalUnitType.PPS_NUT); const seiUnits = nalUnits.filter( - unit => extractNalUnitTypeForHevc(unit) === NALU_TYPE_SEI_PREFIX - || extractNalUnitTypeForHevc(unit) === NALU_TYPE_SEI_SUFFIX, + unit => extractNalUnitTypeForHevc(unit) === HevcNalUnitType.PREFIX_SEI_NUT + || extractNalUnitTypeForHevc(unit) === HevcNalUnitType.SUFFIX_SEI_NUT, ); if (spsUnits.length === 0 || ppsUnits.length === 0) return null; @@ -521,7 +564,7 @@ export const extractHevcDecoderConfigurationRecord = ( ? [ { arrayCompleteness: 1, - nalUnitType: NALU_TYPE_VPS, + nalUnitType: HevcNalUnitType.VPS_NUT, nalUnits: vpsUnits, }, ] @@ -530,7 +573,7 @@ export const extractHevcDecoderConfigurationRecord = ( ? [ { arrayCompleteness: 1, - nalUnitType: NALU_TYPE_SPS, + nalUnitType: HevcNalUnitType.SPS_NUT, nalUnits: spsUnits, }, ] @@ -539,7 +582,7 @@ export const extractHevcDecoderConfigurationRecord = ( ? [ { arrayCompleteness: 1, - nalUnitType: NALU_TYPE_PPS, + nalUnitType: HevcNalUnitType.PPS_NUT, nalUnits: ppsUnits, }, ] @@ -1440,22 +1483,9 @@ export const determineVideoPacketType = async ( const decoderConfig = await videoTrack.getDecoderConfig(); assert(decoderConfig); - let nalUnits: Uint8Array[]; + const nalUnits = extractAvcNalUnits(packet.data, decoderConfig); + const isKeyframe = nalUnits.some(x => extractNalUnitTypeForAvc(x) === AvcNalUnitType.IDR); - if (decoderConfig.description) { - // Stream is length-prefixed. Let's extract the size of the length prefix from the decoder config - - const bytes = toUint8Array(decoderConfig.description); - const lengthSizeMinusOne = bytes[4]! & 0b11; - const lengthSize = (lengthSizeMinusOne + 1) as 1 | 2 | 3 | 4; - - nalUnits = findNalUnitsInLengthPrefixed(packet.data, lengthSize); - } else { - // Stream is in Annex B format - nalUnits = findNalUnitsInAnnexB(packet.data); - } - - const isKeyframe = nalUnits.some(x => extractNalUnitTypeForAvc(x) === 5); return isKeyframe ? 'key' : 'delta'; }; @@ -1463,25 +1493,12 @@ export const determineVideoPacketType = async ( const decoderConfig = await videoTrack.getDecoderConfig(); assert(decoderConfig); - let nalUnits: Uint8Array[]; - - if (decoderConfig.description) { - // Stream is length-prefixed. Let's extract the size of the length prefix from the decoder config - - const bytes = toUint8Array(decoderConfig.description); - const lengthSizeMinusOne = bytes[21]! & 0b11; - const lengthSize = (lengthSizeMinusOne + 1) as 1 | 2 | 3 | 4; - - nalUnits = findNalUnitsInLengthPrefixed(packet.data, lengthSize); - } else { - // Stream is in Annex B format - nalUnits = findNalUnitsInAnnexB(packet.data); - } - + const nalUnits = extractHevcNalUnits(packet.data, decoderConfig); const isKeyframe = nalUnits.some((x) => { const type = extractNalUnitTypeForHevc(x); - return 16 <= type && type <= 23; + return HevcNalUnitType.BLA_W_LP <= type && type <= HevcNalUnitType.RSV_IRAP_VCL23; }); + return isKeyframe ? 'key' : 'delta'; }; diff --git a/src/codec.ts b/src/codec.ts index 119f7cb..2e90cfd 100644 --- a/src/codec.ts +++ b/src/codec.ts @@ -621,7 +621,7 @@ export const parseAacAudioSpecificConfig = (bytes: Uint8Array | null): AacAudioS }; }; -export const OPUS_INTERNAL_SAMPLE_RATE = 48000; +export const OPUS_SAMPLE_RATE = 48_000; const PCM_CODEC_REGEX = /^pcm-([usf])(\d+)+(be)?$/; diff --git a/src/conversion.ts b/src/conversion.ts index fd2832b..bc5e980 100644 --- a/src/conversion.ts +++ b/src/conversion.ts @@ -48,6 +48,7 @@ import { import { Output, TrackType } from './output'; import { Mp4OutputFormat } from './output-format'; import { AudioSample, VideoSample } from './sample'; +import { MetadataTags, validateMetadataTags } from './tags'; import { NullTarget } from './target'; /** @@ -86,6 +87,15 @@ export type ConversionOptions = { /** The time in the input file in seconds at which the output file should end. Must be greater than `start`. */ end: number; }; + + /** + * A callback that returns or resolves to the descriptive metadata tags that should be written to the output file. + * As input, this function will be passed the tags of the input file, allowing you to modify, augment or extend + * them. + * + * If no function is set, the input's metadata tags will be copied to the output. + */ + tags?: (inputTags: MetadataTags) => MaybePromise; }; /** @@ -433,6 +443,9 @@ export class Conversion { && options.trim.start >= options.trim.end) { throw new TypeError('options.trim.start must be less than options.trim.end.'); } + if (options.tags !== undefined && typeof options.tags !== 'function') { + throw new TypeError('options.tags, when provided, must be a function.'); + } this._options = options; this.input = options.input; @@ -516,6 +529,32 @@ export class Conversion { // Let's give the user a notice/warning about discarded tracks so they aren't confused console.warn('Some tracks had to be discarded from the conversion:', unintentionallyDiscardedTracks); } + + // Now, let's deal with metadata tags + + const inputTags = await this.input.getMetadataTags(); + let outputTags: MetadataTags; + + if (this._options.tags) { + const result = await this._options.tags(inputTags); + validateMetadataTags(result); + + outputTags = result; + } else { + outputTags = inputTags; + } + + // Somewhat dirty but pragmatic + const inputAndOutputFormatMatch = (await this.input.getFormat()).mimeType === this.output.format.mimeType; + const rawTagsAreUnchanged = inputTags.raw === outputTags.raw; + + if (inputTags.raw && rawTagsAreUnchanged && !inputAndOutputFormatMatch) { + // If the input and output formats aren't the same, copying over raw metadata tags makes no sense and only + // results in junk tags, so let's cut them out. + delete outputTags.raw; + } + + this.output.setMetadataTags(outputTags); } /** Executes the conversion process. Resolves once conversion is complete. */ @@ -529,9 +568,14 @@ export class Conversion { if (this.onProgress) { this._computeProgress = true; this._totalDuration = Math.min( - await this.input.computeDuration() - this._startTimestamp, + (await this.input.computeDuration()) - this._startTimestamp, this._endTimestamp - this._startTimestamp, ); + + for (const track of this.utilizedTracks) { + this._maxTimestamps.set(track.id, 0); + } + this.onProgress?.(0); } @@ -723,7 +767,7 @@ export class Conversion { await tempOutput.start(); const sink = new VideoSampleSink(track); - const firstSample = await sink.getSample(this._startTimestamp); + const firstSample = await sink.getSample(firstTimestamp); // Let's just use the first sample if (firstSample) { try { @@ -1121,8 +1165,6 @@ export class Conversion { await this._started; const resampler = new AudioResampler({ - sourceNumberOfChannels: track.numberOfChannels, - sourceSampleRate: track.sampleRate, targetNumberOfChannels, targetSampleRate, startTime: this._startTimestamp, @@ -1161,7 +1203,10 @@ export class Conversion { } assert(this._totalDuration !== null); - this._maxTimestamps.set(trackId, Math.max(endTimestamp, this._maxTimestamps.get(trackId) ?? -Infinity)); + this._maxTimestamps.set( + trackId, + Math.max(endTimestamp, this._maxTimestamps.get(trackId)!), + ); const minTimestamp = Math.min(...this._maxTimestamps.values()); const newProgress = clamp(minTimestamp / this._totalDuration, 0, 1); @@ -1239,9 +1284,9 @@ class TrackSynchronizer { * OfflineAudioContext. */ export class AudioResampler { - sourceSampleRate: number; + sourceSampleRate: number | null = null; targetSampleRate: number; - sourceNumberOfChannels: number; + sourceNumberOfChannels: number | null = null; targetNumberOfChannels: number; startTime: number; endTime: number; @@ -1255,20 +1300,16 @@ export class AudioResampler { /** The highest index written to in the current buffer */ maxWrittenFrame: number; channelMixer!: (sourceData: Float32Array, sourceFrameIndex: number, targetChannelIndex: number) => number; - tempSourceBuffer: Float32Array; + tempSourceBuffer!: Float32Array; constructor(options: { - sourceSampleRate: number; targetSampleRate: number; - sourceNumberOfChannels: number; targetNumberOfChannels: number; startTime: number; endTime: number; onSample: (sample: AudioSample) => Promise; }) { - this.sourceSampleRate = options.sourceSampleRate; this.targetSampleRate = options.targetSampleRate; - this.sourceNumberOfChannels = options.sourceNumberOfChannels; this.targetNumberOfChannels = options.targetNumberOfChannels; this.startTime = options.startTime; this.endTime = options.endTime; @@ -1280,17 +1321,14 @@ export class AudioResampler { this.outputBuffer = new Float32Array(this.bufferSizeInSamples); this.bufferStartFrame = 0; this.maxWrittenFrame = -1; - - this.setupChannelMixer(); - - // Pre-allocate temporary buffer for source data - this.tempSourceBuffer = new Float32Array(this.sourceSampleRate * this.sourceNumberOfChannels); } /** * Sets up the channel mixer to handle up/downmixing in the case where input and output channel counts don't match. */ - setupChannelMixer(): void { + doChannelMixerSetup(): void { + assert(this.sourceNumberOfChannels !== null); + const sourceNum = this.sourceNumberOfChannels; const targetNum = this.targetNumberOfChannels; @@ -1408,8 +1446,17 @@ export class AudioResampler { } async add(audioSample: AudioSample) { - if (!audioSample || audioSample._closed) { - return; + if (this.sourceSampleRate === null) { + // This is the first sample, so let's init the missing data. Initting the sample rate from the decoded + // sample is more reliable than using the file's metadata, because decoders are free to emit any sample rate + // they see fit. + this.sourceSampleRate = audioSample.sampleRate; + this.sourceNumberOfChannels = audioSample.numberOfChannels; + + // Pre-allocate temporary buffer for source data + this.tempSourceBuffer = new Float32Array(this.sourceSampleRate * this.sourceNumberOfChannels); + + this.doChannelMixerSetup(); } const requiredSamples = audioSample.numberOfFrames * audioSample.numberOfChannels; diff --git a/src/demuxer.ts b/src/demuxer.ts index b47d0c1..f654dab 100644 --- a/src/demuxer.ts +++ b/src/demuxer.ts @@ -8,6 +8,7 @@ import { Input } from './input'; import { InputTrack } from './input-track'; +import { MetadataTags } from './tags'; export abstract class Demuxer { input: Input; @@ -19,4 +20,5 @@ export abstract class Demuxer { abstract computeDuration(): Promise; abstract getTracks(): Promise; abstract getMimeType(): Promise; + abstract getMetadataTags(): Promise; } diff --git a/src/index.ts b/src/index.ts index 9bd1991..c7bbedd 100644 --- a/src/index.ts +++ b/src/index.ts @@ -188,5 +188,10 @@ export { registerDecoder, registerEncoder, } from './custom-coder'; +export { + MetadataTags, + AttachedImage, + RichImageData, +} from './tags'; // 🐡🦔 diff --git a/src/input-format.ts b/src/input-format.ts index 72f2ffc..01aa838 100644 --- a/src/input-format.ts +++ b/src/input-format.ts @@ -22,7 +22,7 @@ import { import { MatroskaDemuxer } from './matroska/matroska-demuxer'; import { Mp3Demuxer } from './mp3/mp3-demuxer'; import { FRAME_HEADER_SIZE } from '../shared/mp3-misc'; -import { readId3, readNextFrameHeader } from './mp3/mp3-reader'; +import { ID3_V2_HEADER_SIZE, readId3V2Header, readNextFrameHeader } from './mp3/mp3-reader'; import { OggDemuxer } from './ogg/ogg-demuxer'; import { WaveDemuxer } from './wave/wave-demuxer'; import { MAX_FRAME_HEADER_SIZE, MIN_FRAME_HEADER_SIZE, readFrameHeader } from './adts/adts-reader'; @@ -262,11 +262,20 @@ export class Mp3InputFormat extends InputFormat { if (!slice) return false; let currentPos = 0; + let id3V2HeaderFound = false; - const id3Tag = readId3(slice); + while (true) { + let slice = input._reader.requestSlice(currentPos, ID3_V2_HEADER_SIZE); + if (slice instanceof Promise) slice = await slice; + if (!slice) break; - if (id3Tag) { - currentPos = slice.filePos + id3Tag.size; + const id3V2Header = readId3V2Header(slice); + if (!id3V2Header) { + break; + } + + id3V2HeaderFound = true; + currentPos = slice.filePos + id3V2Header.size; } const firstResult = await readNextFrameHeader(input._reader, currentPos, currentPos + 4096); @@ -274,8 +283,8 @@ export class Mp3InputFormat extends InputFormat { return false; } - if (id3Tag) { - // If there was an ID3 tag at the start, we can be pretty sure this is MP3 by now + if (id3V2HeaderFound) { + // If there was an ID3v2 tag at the start, we can be pretty sure this is MP3 by now return true; } diff --git a/src/input-track.ts b/src/input-track.ts index b093b0d..7f3e56e 100644 --- a/src/input-track.ts +++ b/src/input-track.ts @@ -258,7 +258,8 @@ export class InputVideoTrack extends InputTrack { /** * Returns the [decoder configuration](https://www.w3.org/TR/webcodecs/#video-decoder-config) for decoding the - * track's packets using a VideoDecoder. Returns null if the track's codec is unknown. + * track's packets using a [`VideoDecoder`](https://developer.mozilla.org/en-US/docs/Web/API/VideoDecoder). Returns + * null if the track's codec is unknown. */ getDecoderConfig() { return this._backing.getDecoderConfig(); @@ -354,7 +355,8 @@ export class InputAudioTrack extends InputTrack { /** * Returns the [decoder configuration](https://www.w3.org/TR/webcodecs/#audio-decoder-config) for decoding the - * track's packets using an AudioDecoder. Returns null if the track's codec is unknown. + * track's packets using an [`AudioDecoder`](https://developer.mozilla.org/en-US/docs/Web/API/AudioDecoder). Returns + * null if the track's codec is unknown. */ getDecoderConfig() { return this._backing.getDecoderConfig(); diff --git a/src/input.ts b/src/input.ts index 4d815e5..1b9a0c0 100644 --- a/src/input.ts +++ b/src/input.ts @@ -141,4 +141,10 @@ export class Input { const demuxer = await this._getDemuxer(); return demuxer.getMimeType(); } + + /** Returns descriptive metadata tags about the media file, such as title, author, date, or cover art. */ + async getMetadataTags() { + const demuxer = await this._getDemuxer(); + return demuxer.getMetadataTags(); + } } diff --git a/src/isobmff/isobmff-boxes.ts b/src/isobmff/isobmff-boxes.ts index e25824a..c15b05f 100644 --- a/src/isobmff/isobmff-boxes.ts +++ b/src/isobmff/isobmff-boxes.ts @@ -18,6 +18,8 @@ import { MATRIX_COEFFICIENTS_MAP, colorSpaceIsComplete, UNDETERMINED_LANGUAGE, + assertNever, + keyValueIterator, } from '../misc'; import { AudioCodec, @@ -35,12 +37,14 @@ import { GLOBAL_TIMESCALE, intoTimescale, IsobmffAudioTrackData, + IsobmffMuxer, IsobmffSubtitleTrackData, IsobmffTrackData, IsobmffVideoTrackData, Sample, } from './isobmff-muxer'; import { parseOpusIdentificationHeader } from '../codec-data'; +import { MetadataTags, RichImageData } from '../tags'; export class IsobmffBoxWriter { private helper = new Uint8Array(8); @@ -331,13 +335,13 @@ export const mdat = (reserveLargeSize: boolean): Box => ({ type: 'mdat', largeSi * an application to interpret the sample data that is stored elsewhere. */ export const moov = ( - trackDatas: IsobmffTrackData[], - creationTime: number, + muxer: IsobmffMuxer, fragmented = false, ) => box('moov', undefined, [ - mvhd(creationTime, trackDatas), - ...trackDatas.map(x => trak(x, creationTime)), - fragmented ? mvex(trackDatas) : null, + mvhd(muxer.creationTime, muxer.trackDatas), + ...muxer.trackDatas.map(x => trak(x, muxer.creationTime)), + fragmented ? mvex(muxer.trackDatas) : null, + udta(muxer), ]); /** Movie Header Box: Used to specify the characteristics of the entire movie, such as timescale and duration. */ @@ -387,7 +391,7 @@ export const trak = (trackData: IsobmffTrackData, creationTime: number) => { mdia(trackData, creationTime), trackMetadata.name !== undefined ? box('udta', undefined, [ - box('©nam', [ + box('name', [ // VLC (and Mediabunny) also recognize ©nam ...textEncoder.encode(trackMetadata.name), ]), ]) @@ -395,57 +399,6 @@ export const trak = (trackData: IsobmffTrackData, creationTime: number) => { ]); }; -/* -const meta = (trackData: IsobmffTrackData) => { - const trackMetadata = getTrackMetadata(trackData); - - if (trackData.muxer.isQuickTime) { - const keyMap: Record = { - name: 'com.apple.quicktime.title', - }; - - return box('meta', undefined, [ - hdlr(false, 'mdta', ''), - fullBox('keys', 0, 0, [ - u32(Object.keys(trackMetadata).length), - ], Object.keys(trackMetadata).map(key => - box('mdta', [ - ascii(keyMap[key as keyof IsobmffMetadata]), // Key name - ]), - )), - box('ilst', undefined, Object.values(trackMetadata) - .map((value, i) => box(u32(i + 1).map(x => String.fromCharCode(x)).join(''), undefined, [ - data(value), - ]))), - ]); - } else { - const keyMap: Record = { - name: '©nam', - }; - - return fullBox('meta', 0, 0, undefined, [ - hdlr(false, 'mdir', ''), - box('ilst', undefined, Object.entries(trackMetadata) - .map(([key, value]) => box(keyMap[key as keyof IsobmffMetadata], undefined, [ - data(value), - ]))), - ]); - } -}; - -const data = (value: unknown) => { - if (typeof value === 'string') { - return box('data', [ - u32(1), // Type indicator (UTF-8) - u32(0), // Locale indicator - ...textEncoder.encode(value), - ]); - } - - throw new Error('Unhandled data type.'); -}; -*/ - /** Track Header Box: Specifies the characteristics of a single track within a movie. */ export const tkhd = ( trackData: IsobmffTrackData, @@ -506,18 +459,12 @@ export const mdhd = ( const needsU64 = !isU32(creationTime) || !isU32(localDuration); const u32OrU64 = needsU64 ? u64 : u32; - let language = 0; - for (const character of (trackData.track.metadata.languageCode ?? UNDETERMINED_LANGUAGE)) { - language <<= 5; - language += character.charCodeAt(0) - 0x60; - } - return fullBox('mdhd', +needsU64, 0, [ u32OrU64(creationTime), // Creation time u32OrU64(creationTime), // Modification time u32(trackData.timescale), // Timescale u32OrU64(localDuration), // Duration - u16(language), // Language + u16(getLanguageCodeInt(trackData.track.metadata.languageCode ?? UNDETERMINED_LANGUAGE)), // Language u16(0), // Quality ]); }; @@ -535,10 +482,15 @@ const TRACK_TYPE_TO_HANDLER_NAME: Record = { }; /** Handler Reference Box. */ -export const hdlr = (hasComponentType: boolean, handlerType: string, name: string) => fullBox('hdlr', 0, 0, [ +export const hdlr = ( + hasComponentType: boolean, + handlerType: string, + name: string, + manufacturer = '\0\0\0\0', +) => fullBox('hdlr', 0, 0, [ hasComponentType ? ascii('mhlr') : u32(0), // Component type ascii(handlerType), // Component subtype - u32(0), // Component manufacturer + ascii(manufacturer), // Component manufacturer u32(0), // Component flags u32(0), // Component flags mask ascii(name, true), // Component name @@ -1318,6 +1270,266 @@ export const vttc = ( /** VTT Additional Text Box */ export const vtta = (notes: string) => box('vtta', [...textEncoder.encode(notes)]); +/** User Data Box */ +const udta = (muxer: IsobmffMuxer) => { + const boxes: Box[] = []; + + // Depending on the format, metadata tags are written differently + if (muxer.isQuickTime) { + addQuickTimeMetadataTagBoxes(boxes, muxer.output._metadataTags); + } else { + const metaBox = meta(muxer.output._metadataTags); + if (metaBox) { + boxes.push(metaBox); + } + } + + if (boxes.length === 0) { + return null; + } + + return box('udta', undefined, boxes); +}; + +const addQuickTimeMetadataTagBoxes = (boxes: Box[], tags: MetadataTags) => { + // https://exiftool.org/TagNames/QuickTime.html (QuickTime UserData Tags) + // For QuickTime files, metadata tags are dumped into the udta box + + for (const { key, value } of keyValueIterator(tags)) { + switch (key) { + case 'title': { + boxes.push(metadataTagStringBoxShort('©nam', value)); + }; break; + + case 'description': { + boxes.push(metadataTagStringBoxShort('©des', value)); + }; break; + + case 'artist': { + boxes.push(metadataTagStringBoxShort('©ART', value)); + }; break; + + case 'album': { + boxes.push(metadataTagStringBoxShort('©alb', value)); + }; break; + + case 'albumArtist': { + boxes.push(metadataTagStringBoxShort('albr', value)); + }; break; + + case 'genre': { + boxes.push(metadataTagStringBoxShort('©gen', value)); + }; break; + + case 'date': { + boxes.push(metadataTagStringBoxShort('©day', value.toISOString().slice(0, 10))); + }; break; + + case 'comment': { + boxes.push(metadataTagStringBoxShort('©cmt', value)); + }; break; + + case 'lyrics': { + boxes.push(metadataTagStringBoxShort('©lyr', value)); + }; break; + + case 'raw': { + // Handled later + }; break; + + case 'discNumber': + case 'discsTotal': + case 'trackNumber': + case 'tracksTotal': + case 'images': { + // Not written for QuickTime (common Apple L) + }; break; + + default: assertNever(key); + } + } + + if (tags.raw) { + for (const key in tags.raw) { + const value = tags.raw[key]; + if (value == null || key.length !== 4 || boxes.some(x => x.type === key)) { + continue; + } + + if (typeof value === 'string') { + boxes.push(metadataTagStringBoxShort(key, value)); + } else if (value instanceof Uint8Array) { + boxes.push(box(key, Array.from(value))); + } + } + } +}; + +const metadataTagStringBoxShort = (name: string, value: string) => { + const encoded = textEncoder.encode(value); + + return box(name, [ + u16(encoded.length), + u16(getLanguageCodeInt('und')), + Array.from(encoded), + ]); +}; + +const DATA_BOX_MIME_TYPE_MAP: Record = { + 'image/jpeg': 13, + 'image/png': 14, + 'image/bmp': 27, +}; + +/** Metadata Box */ +const meta = (tags: MetadataTags) => { + const boxes: Box[] = []; + + // https://exiftool.org/TagNames/QuickTime.html (QuickTime ItemList Tags) + // This is the metadata format used for MP4 files + + for (const { key, value } of keyValueIterator(tags)) { + switch (key) { + case 'title': { + boxes.push(metadataTagStringBoxLong('©nam', value)); + }; break; + + case 'description': { + boxes.push(metadataTagStringBoxLong('©des', value)); + }; break; + + case 'artist': { + boxes.push(metadataTagStringBoxLong('©ART', value)); + }; break; + + case 'album': { + boxes.push(metadataTagStringBoxLong('©alb', value)); + }; break; + + case 'albumArtist': { + boxes.push(metadataTagStringBoxLong('aART', value)); + }; break; + + case 'comment': { + boxes.push(metadataTagStringBoxLong('©cmt', value)); + }; break; + + case 'genre': { + boxes.push(metadataTagStringBoxLong('©gen', value)); + }; break; + + case 'lyrics': { + boxes.push(metadataTagStringBoxLong('©lyr', value)); + }; break; + + case 'date': { + boxes.push(metadataTagStringBoxLong('©day', value.toISOString().slice(0, 10))); + }; break; + + case 'images': { + for (const image of value) { + if (image.kind !== 'coverFront') { + continue; + } + + boxes.push(box('covr', undefined, [ + box('data', [ + u32(DATA_BOX_MIME_TYPE_MAP[image.mimeType] ?? 0), // Type indicator + u32(0), // Locale indicator + Array.from(image.data), // Kinda slow, hopefully temp + ]), + ])); + } + }; break; + + case 'trackNumber': { + boxes.push(box('trkn', undefined, [ + box('data', [ + u32(0), // 8 bytes empty + u32(0), + u16(0), // Empty + u16(value), + u16(tags.tracksTotal ?? 0), + u16(0), // Empty + ]), + ])); + }; break; + + case 'discNumber': { + boxes.push(box('disc', undefined, [ + box('data', [ + u32(0), // 8 bytes empty + u32(0), + u16(0), // Empty + u16(value), + u16(tags.discsTotal ?? 0), + u16(0), // Empty + ]), + ])); + }; break; + + case 'tracksTotal': + case 'discsTotal':{ + // These are included with 'trackNumber' and 'discNumber' respectively + }; break; + + case 'raw': { + // Handled later + }; break; + + default: assertNever(key); + } + } + + if (tags.raw) { + for (const key in tags.raw) { + const value = tags.raw[key]; + if (value == null || key.length !== 4 || boxes.some(x => x.type === key)) { + continue; + } + + if (typeof value === 'string') { + boxes.push(metadataTagStringBoxLong(key, value)); + } else if (value instanceof Uint8Array) { + boxes.push(box(key, undefined, [ + box('data', [ + u32(0), // Type indicator + u32(0), // Locale indicator + Array.from(value), + ]), + ])); + } else if (value instanceof RichImageData) { + boxes.push(box(key, undefined, [ + box('data', [ + u32(DATA_BOX_MIME_TYPE_MAP[value.mimeType] ?? 0), // Type indicator + u32(0), // Locale indicator + Array.from(value.data), // Kinda slow, hopefully temp + ]), + ])); + } + } + } + + if (boxes.length === 0) { + return null; + } + + return fullBox('meta', 0, 0, undefined, [ + hdlr(false, 'mdir', '', 'appl'), + box('ilst', undefined, boxes), + ]); +}; + +const metadataTagStringBoxLong = (name: string, value: string) => { + return box(name, undefined, [ + box('data', [ + u32(1), // Type indicator (UTF-8) + u32(0), // Locale indicator + ...textEncoder.encode(value), + ]), + ]); +}; + const VIDEO_CODEC_TO_BOX_NAME: Record = { avc: 'avc1', hevc: 'hvc1', @@ -1426,3 +1638,15 @@ const SUBTITLE_CODEC_TO_CONFIGURATION_BOX: Record< > = { webvtt: vttC, }; + +const getLanguageCodeInt = (code: string) => { + assert(code.length === 3); ; + + let language = 0; + for (let i = 0; i < 3; i++) { + language <<= 5; + language += code.charCodeAt(i) - 0x60; + } + + return language; +}; diff --git a/src/isobmff/isobmff-demuxer.ts b/src/isobmff/isobmff-demuxer.ts index f43c784..84fce37 100644 --- a/src/isobmff/isobmff-demuxer.ts +++ b/src/isobmff/isobmff-demuxer.ts @@ -12,6 +12,7 @@ import { extractAudioCodecString, extractVideoCodecString, MediaCodec, + OPUS_SAMPLE_RATE, parseAacAudioSpecificConfig, parsePcmCodec, PCM_AUDIO_CODECS, @@ -57,6 +58,7 @@ import { TransformationMatrix, TRANSFER_CHARACTERISTICS_MAP_INVERSE, UNDETERMINED_LANGUAGE, + toDataView, } from '../misc'; import { EncodedPacket, PLACEHOLDER_DATA } from '../packet'; import { buildIsobmffMimeType } from './isobmff-misc'; @@ -64,9 +66,11 @@ import { MAX_BOX_HEADER_SIZE, MIN_BOX_HEADER_SIZE, readBoxHeader, + readDataBox, readFixed_16_16, readFixed_2_30, readIsomVariableInteger, + readMetadataStringShort, } from './isobmff-reader'; import { FileSlice, @@ -83,6 +87,20 @@ import { readU8, readAscii, } from '../reader'; +import { MetadataTags, RichImageData } from '../tags'; + +// https://exiftool.org/TagNames/QuickTime.html +const UDTA_STRING_KEYS = new Set([ + '@day', '@mak', '@mod', '@swr', '@xyz', 'CAME', 'CNCV', 'CNFV', 'CNMN', 'FIRM', 'FOV\0', 'GoPr', 'LENS', 'PXMN', + 'SIGM', 'SNum', 'TAGS', 'albm', 'albr', 'angl', 'auth', 'ccid', 'cdis', 'clfn', 'clid', 'clsf', 'cmid', 'cmnm', + 'coll', 'cprt', 'cver', 'cvru', 'date', 'dscp', 'fsid', 'gnre', 'hinv', 'icnu', 'info', 'infu', 'kgtt', 'loci', + 'lrcu', 'mcvr', 'name', 'perf', 'pmcc', 'reel', 'rtng', 'scen', 'shot', 'slno', 'thmb', 'titl', 'tnam', 'urat', + 'uuid', 'vndr', 'yrrc', '©ART', '©TIM', '©TSC', '©TSZ', '©alb', '©arg', '©ark', '©cmt', '©cok', '©com', '©cpy', + '©day', '©dir', '©ed1', '©ed2', '©ed3', '©ed4', '©ed5', '©ed6', '©ed7', '©ed8', '©ed9', '©enc', '©fmt', '©fpt', + '©frl', '©fyw', '©gen', '©gpt', '©grl', '©grp', '©gyw', '©inf', '©isr', '©lab', '©lal', '©lyr', '©mak', '©mal', + '©mdl', '©mod', '©nam', '©pdk', '©phg', '©prd', '©prf', '©prk', '©prl', '©req', '©snk', '©snm', '©src', '©swf', + '©swk', '©swr', '©too', '©trk', '©wrt', '©xsp', '©xyz', '©ysp', '©zsp', +]); type InternalTrack = { id: number; @@ -232,6 +250,8 @@ export class IsobmffDemuxer extends Demuxer { movieTimescale = -1; movieDurationInTimescale = -1; isQuickTime = false; + metadataTags: MetadataTags = {}; + currentMetadataKeys: Map | null = null; isFragmented = false; fragmentTrackDefaults: FragmentTrackDefaults[] = []; @@ -269,6 +289,11 @@ export class IsobmffDemuxer extends Demuxer { }); } + async getMetadataTags() { + await this.readMetadata(); + return this.metadataTags; + } + readMetadata() { return this.metadataPromise ??= (async () => { let currentPos = 0; @@ -605,6 +630,22 @@ export class IsobmffDemuxer extends Demuxer { } } + // eslint-disable-next-line @stylistic/generator-star-spacing + *iterateContiguousBoxes(slice: FileSlice) { + const startIndex = slice.filePos; + + while (slice.filePos - startIndex <= slice.length - MIN_BOX_HEADER_SIZE) { + const startPos = slice.filePos; + const boxInfo = readBoxHeader(slice); + if (!boxInfo) { + break; + } + + yield { boxInfo, slice }; + slice.filePos = startPos + boxInfo.totalSize; + } + } + traverseBox(slice: FileSlice): boolean { const startPos = slice.filePos; const boxInfo = readBoxHeader(slice); @@ -620,8 +661,7 @@ export class IsobmffDemuxer extends Demuxer { case 'minf': case 'dinf': case 'mfra': - case 'edts': - case 'udta': { + case 'edts': { this.readContiguousBoxes(slice.slice(contentStartPos, boxInfo.contentSize)); }; break; @@ -683,7 +723,9 @@ export class IsobmffDemuxer extends Demuxer { case 'tkhd': { const track = this.currentTrack; - assert(track); + if (!track) { + break; + } const version = readU8(slice); const flags = readU24Be(slice); @@ -729,7 +771,9 @@ export class IsobmffDemuxer extends Demuxer { case 'elst': { const track = this.currentTrack; - assert(track); + if (!track) { + break; + } const version = readU8(slice); slice.skip(3); // Flags @@ -777,7 +821,9 @@ export class IsobmffDemuxer extends Demuxer { case 'mdhd': { const track = this.currentTrack; - assert(track); + if (!track) { + break; + } const version = readU8(slice); slice.skip(3); // Flags @@ -811,7 +857,9 @@ export class IsobmffDemuxer extends Demuxer { case 'hdlr': { const track = this.currentTrack; - assert(track); + if (!track) { + break; + } slice.skip(8); // Version + flags + pre-defined const handlerType = readAscii(slice, 4); @@ -843,7 +891,9 @@ export class IsobmffDemuxer extends Demuxer { case 'stbl': { const track = this.currentTrack; - assert(track); + if (!track) { + break; + } track.sampleTableByteOffset = startPos; @@ -852,7 +902,9 @@ export class IsobmffDemuxer extends Demuxer { case 'stsd': { const track = this.currentTrack; - assert(track); + if (!track) { + break; + } if (track.info === null || track.sampleTable) { break; @@ -998,6 +1050,10 @@ export class IsobmffDemuxer extends Demuxer { } } + if (track.info.codec === 'opus') { + sampleRate = OPUS_SAMPLE_RATE; // Always the same + } + track.info.numberOfChannels = channelCount; track.info.sampleRate = sampleRate; @@ -1048,21 +1104,30 @@ export class IsobmffDemuxer extends Demuxer { case 'avcC': { const track = this.currentTrack; - assert(track && track.info); + if (!track) { + break; + } + assert(track.info); track.info.codecDescription = readBytes(slice, boxInfo.contentSize); }; break; case 'hvcC': { const track = this.currentTrack; - assert(track && track.info); + if (!track) { + break; + } + assert(track.info); track.info.codecDescription = readBytes(slice, boxInfo.contentSize); }; break; case 'vpcC': { const track = this.currentTrack; - assert(track && track.info?.type === 'video'); + if (!track) { + break; + } + assert(track.info?.type === 'video'); slice.skip(4); // Version + flags @@ -1090,7 +1155,10 @@ export class IsobmffDemuxer extends Demuxer { case 'av1C': { const track = this.currentTrack; - assert(track && track.info?.type === 'video'); + if (!track) { + break; + } + assert(track.info?.type === 'video'); slice.skip(1); // Marker + version @@ -1124,7 +1192,10 @@ export class IsobmffDemuxer extends Demuxer { case 'colr': { const track = this.currentTrack; - assert(track && track.info?.type === 'video'); + if (!track) { + break; + } + assert(track.info?.type === 'video'); const colourType = readAscii(slice, 4); if (colourType !== 'nclx') { @@ -1150,7 +1221,10 @@ export class IsobmffDemuxer extends Demuxer { case 'esds': { const track = this.currentTrack; - assert(track && track.info?.type === 'audio'); + if (!track) { + break; + } + assert(track.info?.type === 'audio'); slice.skip(4); // Version + flags @@ -1224,7 +1298,10 @@ export class IsobmffDemuxer extends Demuxer { case 'enda': { const track = this.currentTrack; - assert(track && track.info?.type === 'audio'); + if (!track) { + break; + } + assert(track.info?.type === 'audio'); const littleEndian = readU16Be(slice) & 0xff; // 0xff is from FFmpeg @@ -1245,7 +1322,10 @@ export class IsobmffDemuxer extends Demuxer { case 'pcmC': { const track = this.currentTrack; - assert(track && track.info?.type === 'audio'); + if (!track) { + break; + } + assert(track.info?.type === 'audio'); slice.skip(1 + 3); // Version + flags @@ -1310,7 +1390,10 @@ export class IsobmffDemuxer extends Demuxer { case 'dOps': { // Used for Opus audio const track = this.currentTrack; - assert(track && track.info?.type === 'audio'); + if (!track) { + break; + } + assert(track.info?.type === 'audio'); slice.skip(1); // Version @@ -1343,12 +1426,15 @@ export class IsobmffDemuxer extends Demuxer { track.info.codecDescription = description; track.info.numberOfChannels = outputChannelCount; - track.info.sampleRate = inputSampleRate; + // Don't copy the input sample rate, irrelevant, and output sample rate is fixed }; break; case 'dfLa': { // Used for FLAC audio const track = this.currentTrack; - assert(track && track.info?.type === 'audio'); + if (!track) { + break; + } + assert(track.info?.type === 'audio'); slice.skip(4); // Version + flags @@ -1402,7 +1488,9 @@ export class IsobmffDemuxer extends Demuxer { case 'stts': { const track = this.currentTrack; - assert(track); + if (!track) { + break; + } if (!track.sampleTable) { break; @@ -1433,7 +1521,9 @@ export class IsobmffDemuxer extends Demuxer { case 'ctts': { const track = this.currentTrack; - assert(track); + if (!track) { + break; + } if (!track.sampleTable) { break; @@ -1460,7 +1550,9 @@ export class IsobmffDemuxer extends Demuxer { case 'stsz': { const track = this.currentTrack; - assert(track); + if (!track) { + break; + } if (!track.sampleTable) { break; @@ -1483,7 +1575,9 @@ export class IsobmffDemuxer extends Demuxer { case 'stz2': { const track = this.currentTrack; - assert(track); + if (!track) { + break; + } if (!track.sampleTable) { break; @@ -1506,7 +1600,9 @@ export class IsobmffDemuxer extends Demuxer { case 'stss': { const track = this.currentTrack; - assert(track); + if (!track) { + break; + } if (!track.sampleTable) { break; @@ -1531,7 +1627,9 @@ export class IsobmffDemuxer extends Demuxer { case 'stsc': { const track = this.currentTrack; - assert(track); + if (!track) { + break; + } if (!track.sampleTable) { break; @@ -1569,7 +1667,9 @@ export class IsobmffDemuxer extends Demuxer { case 'stco': { const track = this.currentTrack; - assert(track); + if (!track) { + break; + } if (!track.sampleTable) { break; @@ -1587,7 +1687,9 @@ export class IsobmffDemuxer extends Demuxer { case 'co64': { const track = this.currentTrack; - assert(track); + if (!track) { + break; + } if (!track.sampleTable) { break; @@ -1950,14 +2052,294 @@ export class IsobmffDemuxer extends Demuxer { this.currentFragment.implicitBaseDataOffset = currentOffset; }; break; - // These appear in udta: - case '©nam': - case 'name': { - if (!this.currentTrack) { + // Metadata section + // https://exiftool.org/TagNames/QuickTime.html + // https://mp4workshop.com/about + + case 'udta': { // Contains either movie metadata or track metadata + const iterator = this.iterateContiguousBoxes(slice.slice(contentStartPos, boxInfo.contentSize)); + + for (const { boxInfo, slice } of iterator) { + if (boxInfo.name !== 'meta' && !this.currentTrack) { + const startPos = slice.filePos; + this.metadataTags.raw ??= {}; + + if (UDTA_STRING_KEYS.has(boxInfo.name)) { + this.metadataTags.raw[boxInfo.name] ??= readMetadataStringShort(slice); + } else { + this.metadataTags.raw[boxInfo.name] ??= readBytes(slice, boxInfo.contentSize); + } + + slice.filePos = startPos; + } + + switch (boxInfo.name) { + case 'meta': { + slice.skip(-boxInfo.headerSize); + this.traverseBox(slice); + }; break; + + case '©nam': + case 'name': { + if (this.currentTrack) { + this.currentTrack.name = textDecoder.decode(readBytes(slice, boxInfo.contentSize)); + } else { + this.metadataTags.title ??= readMetadataStringShort(slice); + } + }; break; + + case '©des': { + if (!this.currentTrack) { + this.metadataTags.description ??= readMetadataStringShort(slice); + } + }; break; + + case '©ART': { + if (!this.currentTrack) { + this.metadataTags.artist ??= readMetadataStringShort(slice); + } + }; break; + + case '©alb': { + if (!this.currentTrack) { + this.metadataTags.album ??= readMetadataStringShort(slice); + } + }; break; + + case 'albr': { + if (!this.currentTrack) { + this.metadataTags.albumArtist ??= readMetadataStringShort(slice); + } + }; break; + + case '©gen': { + if (!this.currentTrack) { + this.metadataTags.genre ??= readMetadataStringShort(slice); + } + }; break; + + case '©day': { + if (!this.currentTrack) { + const date = new Date(readMetadataStringShort(slice)); + if (!Number.isNaN(date.getTime())) { + this.metadataTags.date ??= date; + } + } + }; break; + + case '©cmt': { + if (!this.currentTrack) { + this.metadataTags.comment ??= readMetadataStringShort(slice); + } + }; break; + + case '©lyr': { + if (!this.currentTrack) { + this.metadataTags.lyrics ??= readMetadataStringShort(slice); + } + }; break; + } + } + }; break; + + case 'meta': { + if (this.currentTrack) { + break; // Only care about movie-level metadata for now + } + + // The 'meta' box comes in two flavors, one with flags/version and one without. To know which is which, + // let's read the next 4 bytes, which are either the version or the size of the first subbox. + const word = readU32Be(slice); + const isQuickTime = word !== 0; + + this.currentMetadataKeys = new Map(); + + if (isQuickTime) { + this.readContiguousBoxes(slice.slice(contentStartPos, boxInfo.contentSize)); + } else { + this.readContiguousBoxes(slice.slice(contentStartPos + 4, boxInfo.contentSize - 4)); + } + + this.currentMetadataKeys = null; + }; break; + + case 'keys': { + if (!this.currentMetadataKeys) { break; } - this.currentTrack.name = textDecoder.decode(readBytes(slice, boxInfo.contentSize)); + slice.skip(4); // Version + flags + + const entryCount = readU32Be(slice); + + for (let i = 0; i < entryCount; i++) { + const keySize = readU32Be(slice); + slice.skip(4); // Key namespace + const keyName = textDecoder.decode(readBytes(slice, keySize - 8)); + + this.currentMetadataKeys.set(i + 1, keyName); + } + }; break; + + case 'ilst': { + if (!this.currentMetadataKeys) { + break; + } + + const iterator = this.iterateContiguousBoxes(slice.slice(contentStartPos, boxInfo.contentSize)); + + for (const { boxInfo, slice } of iterator) { + let metadataKey = boxInfo.name; + + // Interpret the box name as a u32be + const nameAsNumber = (metadataKey.charCodeAt(0) << 24) + + (metadataKey.charCodeAt(1) << 16) + + (metadataKey.charCodeAt(2) << 8) + + metadataKey.charCodeAt(3); + + if (this.currentMetadataKeys.has(nameAsNumber)) { + // An entry exists for this number + metadataKey = this.currentMetadataKeys.get(nameAsNumber)!; + } + + const data = readDataBox(slice); + + this.metadataTags.raw ??= {}; + this.metadataTags.raw[metadataKey] ??= data; + + switch (metadataKey) { + case '©nam': + case 'titl': + case 'com.apple.quicktime.title': + case 'title': { + if (typeof data === 'string') { + this.metadataTags.title ??= data; + } + }; break; + + case '©des': + case 'desc': + case 'dscp': + case 'com.apple.quicktime.description': + case 'description': { + if (typeof data === 'string') { + this.metadataTags.description ??= data; + } + }; break; + + case '©ART': + case 'com.apple.quicktime.artist': + case 'artist': { + if (typeof data === 'string') { + this.metadataTags.artist ??= data; + } + }; break; + + case '©alb': + case 'albm': + case 'com.apple.quicktime.album': + case 'album': { + if (typeof data === 'string') { + this.metadataTags.album ??= data; + } + }; break; + + case 'aART': + case 'album_artist': { + if (typeof data === 'string') { + this.metadataTags.albumArtist ??= data; + } + }; break; + + case '©cmt': + case 'com.apple.quicktime.comment': + case 'comment': { + if (typeof data === 'string') { + this.metadataTags.comment ??= data; + } + }; break; + + case '©gen': + case 'gnre': + case 'com.apple.quicktime.genre': + case 'genre': { + if (typeof data === 'string') { + this.metadataTags.genre ??= data; + } + }; break; + + case '©lyr': + case 'lyrics': { + if (typeof data === 'string') { + this.metadataTags.lyrics ??= data; + } + }; break; + + case '©day': + case 'rldt': + case 'com.apple.quicktime.creationdate': + case 'date': { + if (typeof data === 'string') { + const date = new Date(data); + if (!Number.isNaN(date.getTime())) { + this.metadataTags.date ??= date; + } + } + }; break; + + case 'covr': + case 'com.apple.quicktime.artwork': { + if (data instanceof RichImageData) { + this.metadataTags.images ??= []; + this.metadataTags.images.push({ + data: data.data, + kind: 'coverFront', + mimeType: data.mimeType, + }); + } else if (data instanceof Uint8Array) { + this.metadataTags.images ??= []; + this.metadataTags.images.push({ + data, + kind: 'coverFront', + mimeType: 'image/*', + }); + } + }; break; + + case 'trkn': { + if (data instanceof Uint8Array) { + const view = toDataView(data); + + const trackNumber = view.getUint16(2, false); + const tracksTotal = view.getUint16(4, false); + + if (trackNumber > 0) { + this.metadataTags.trackNumber ??= trackNumber; + } + if (tracksTotal > 0) { + this.metadataTags.tracksTotal ??= tracksTotal; + } + } + }; break; + + case 'disc': + case 'disk': { + if (data instanceof Uint8Array) { + const view = toDataView(data); + + const discNumber = view.getUint16(2, false); + const discNumberMax = view.getUint16(4, false); + + if (discNumber > 0) { + this.metadataTags.discNumber ??= discNumber; + } + if (discNumberMax > 0) { + this.metadataTags.discsTotal ??= discNumberMax; + } + } + }; break; + } + } }; break; } diff --git a/src/isobmff/isobmff-muxer.ts b/src/isobmff/isobmff-muxer.ts index 1ffd0c8..cd14688 100644 --- a/src/isobmff/isobmff-muxer.ts +++ b/src/isobmff/isobmff-muxer.ts @@ -152,10 +152,10 @@ export class IsobmffMuxer extends Muxer { private mdat: Box | null = null; - private trackDatas: IsobmffTrackData[] = []; + trackDatas: IsobmffTrackData[] = []; private allTracksKnown = promiseWithResolvers(); - private creationTime = Math.floor(Date.now() / 1000) + TIMESTAMP_OFFSET; + creationTime = Math.floor(Date.now() / 1000) + TIMESTAMP_OFFSET; private finalizedChunks: Chunk[] = []; private nextFragmentNumber = 1; @@ -988,7 +988,7 @@ export class IsobmffMuxer extends Muxer { } // Write the moov box now that we have all decoder configs - const movieBox = moov(this.trackDatas, this.creationTime, true); + const movieBox = moov(this, true); this.boxWriter.writeBox(movieBox); if (this.format._options.onMoov) { @@ -1140,7 +1140,7 @@ export class IsobmffMuxer extends Muxer { // size of the moov box and can compute the proper chunk positions. for (let i = 0; i < 2; i++) { - const movieBox = moov(this.trackDatas, this.creationTime); + const movieBox = moov(this); const movieBoxSize = this.boxWriter.measureBox(movieBox); mdatSize = this.boxWriter.measureBox(this.mdat); let currentChunkPos = this.writer.getPos() + movieBoxSize + mdatSize; @@ -1162,7 +1162,7 @@ export class IsobmffMuxer extends Muxer { this.writer.startTrackingWrites(); } - const movieBox = moov(this.trackDatas, this.creationTime); + const movieBox = moov(this); this.boxWriter.writeBox(movieBox); if (this.format._options.onMoov) { @@ -1218,7 +1218,7 @@ export class IsobmffMuxer extends Muxer { this.writer.startTrackingWrites(); } - const movieBox = moov(this.trackDatas, this.creationTime); + const movieBox = moov(this); this.boxWriter.writeBox(movieBox); if (this.format._options.onMoov) { diff --git a/src/isobmff/isobmff-reader.ts b/src/isobmff/isobmff-reader.ts index f1d0605..584a0a8 100644 --- a/src/isobmff/isobmff-reader.ts +++ b/src/isobmff/isobmff-reader.ts @@ -6,7 +6,9 @@ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -import { FileSlice, readAscii, readI32Be, readU32Be, readU64Be, readU8 } from '../reader'; +import { RichImageData } from '../tags'; +import { textDecoder } from '../misc'; +import { FileSlice, readAscii, readBytes, readI32Be, readU16Be, readU32Be, readU64Be, readU8 } from '../reader'; export const MIN_BOX_HEADER_SIZE = 8; export const MAX_BOX_HEADER_SIZE = 16; @@ -53,3 +55,30 @@ export const readIsomVariableInteger = (slice: FileSlice) => { return result; }; + +export const readMetadataStringShort = (slice: FileSlice) => { + const stringLength = readU16Be(slice); + slice.skip(2); // Language + return textDecoder.decode(readBytes(slice, stringLength)); +}; + +export const readDataBox = (slice: FileSlice) => { + const header = readBoxHeader(slice); + if (!header || header.name !== 'data') { + return null; + } + + const typeIndicator = readU32Be(slice); + slice.skip(4); // Locale indicator + const data = readBytes(slice, header.contentSize - 8); + + switch (typeIndicator) { + case 1: return textDecoder.decode(data); // UTF-8 + case 2: return new TextDecoder('utf-16be').decode(data); // UTF-16-BE + case 13: return new RichImageData(data, 'image/jpeg'); + case 14: return new RichImageData(data, 'image/png'); + case 27: return new RichImageData(data, 'image/bmp'); + + default: return data; + } +}; diff --git a/src/matroska/ebml.ts b/src/matroska/ebml.ts index 9da08d4..1d89de4 100644 --- a/src/matroska/ebml.ts +++ b/src/matroska/ebml.ts @@ -129,8 +129,27 @@ export enum EBMLId { ProjectionType = 0x7671, ProjectionPoseRoll = 0x7675, Attachments = 0x1941a469, + AttachedFile = 0x61a7, + FileDescription = 0x467e, + FileName = 0x466e, + FileMediaType = 0x4660, + FileData = 0x465c, + FileUID = 0x46ae, Chapters = 0x1043a770, Tags = 0x1254c367, + Tag = 0x7373, + Targets = 0x63c0, + TargetTypeValue = 0x68ca, + TargetType = 0x63ca, + TagTrackUID = 0x63c5, + TagEditionUID = 0x63c9, + TagChapterUID = 0x63c4, + TagAttachmentUID = 0x63c6, + SimpleTag = 0x67c8, + TagName = 0x45a3, + TagLanguage = 0x447a, + TagString = 0x4487, + TagBinary = 0x4485, } export const LEVEL_0_EBML_IDS: EBMLId[] = [ diff --git a/src/matroska/matroska-demuxer.ts b/src/matroska/matroska-demuxer.ts index 21ba787..bda341e 100644 --- a/src/matroska/matroska-demuxer.ts +++ b/src/matroska/matroska-demuxer.ts @@ -18,6 +18,7 @@ import { extractAudioCodecString, extractVideoCodecString, MediaCodec, + OPUS_SAMPLE_RATE, VideoCodec, } from '../codec'; import { Demuxer } from '../demuxer'; @@ -30,6 +31,7 @@ import { InputVideoTrack, InputVideoTrackBacking, } from '../input-track'; +import { MetadataTags } from '../tags'; import { PacketRetrievalOptions } from '../media-sink'; import { assert, @@ -76,6 +78,8 @@ type Segment = { infoSeen: boolean; tracksSeen: boolean; cuesSeen: boolean; + attachmentsSeen: boolean; + tagsSeen: boolean; timestampScale: number; timestampFactor: number; @@ -90,6 +94,9 @@ type Segment = { clusters: Cluster[]; clusterLookupMutex: AsyncMutex; + + metadataTags: MetadataTags; + metadataTagsCollected: boolean; }; type SeekEntry = { @@ -198,6 +205,14 @@ export class MatroskaDemuxer extends Demuxer { currentCluster: Cluster | null = null; currentBlock: ClusterBlock | null = null; currentCueTime: number | null = null; + currentTagTargetIsMovie: boolean = true; + currentSimpleTagName: string | null = null; + currentAttachedFile: { + fileName: string | null; + fileMediaType: string | null; + fileData: Uint8Array | null; + fileDescription: string | null; + } | null = null; isWebM = false; @@ -232,6 +247,32 @@ export class MatroskaDemuxer extends Demuxer { }); } + async getMetadataTags() { + await this.readMetadata(); + + // Load metadata tags from each segment lazily (only once) + for (const segment of this.segments) { + if (!segment.metadataTagsCollected) { + if (this.reader.fileSize !== null) { + await this.loadSegmentMetadata(segment); + } else { + // The seeking would be too crazy, let's not + } + + segment.metadataTagsCollected = true; + } + } + + // This is kinda handwavy, and how we handle multiple segments isn't suuuuper well-defined anyway; so we just + // shallow-merge metadata tags from all (usually just one) segments. + let metadataTags: MetadataTags = {}; + for (const segment of this.segments) { + metadataTags = { ...metadataTags, ...segment.metadataTags }; + } + + return metadataTags; + } + readMetadata() { return this.readMetadataPromise ??= (async () => { let currentPos = 0; @@ -311,6 +352,8 @@ export class MatroskaDemuxer extends Demuxer { infoSeen: false, tracksSeen: false, cuesSeen: false, + tagsSeen: false, + attachmentsSeen: false, timestampScale: -1, timestampFactor: -1, @@ -327,6 +370,9 @@ export class MatroskaDemuxer extends Demuxer { clusters: [], clusterLookupMutex: new AsyncMutex(), + + metadataTags: {}, + metadataTagsCollected: false, }; this.segments.push(this.currentSegment); @@ -371,6 +417,22 @@ export class MatroskaDemuxer extends Demuxer { let slice = this.reader.requestSlice(dataStartPos, size); if (slice instanceof Promise) slice = await slice; + if (slice) { + this.readContiguousElements(slice); + } + } else if (id === EBMLId.Tags || id === EBMLId.Attachments) { + // Metadata found at the beginning of the segment, great, let's parse it + if (id === EBMLId.Tags) { + this.currentSegment.tagsSeen = true; + } else { + this.currentSegment.attachmentsSeen = true; + } + + assertDefinedSize(size); + + let slice = this.reader.requestSlice(dataStartPos, size); + if (slice instanceof Promise) slice = await slice; + if (slice) { this.readContiguousElements(slice); } @@ -386,10 +448,10 @@ export class MatroskaDemuxer extends Demuxer { } } - if (this.reader.fileSize !== null) { - // Sort the seek entries by file position so reading them exhibits a sequential pattern - this.currentSegment.seekEntries.sort((a, b) => a.segmentPosition - b.segmentPosition); + // Sort the seek entries by file position so reading them exhibits a sequential pattern + this.currentSegment.seekEntries.sort((a, b) => a.segmentPosition - b.segmentPosition); + if (this.reader.fileSize !== null) { // Use the seek head to read missing metadata elements for (const seekEntry of this.currentSegment.seekEntries) { const target = METADATA_ELEMENTS.find(x => x.id === seekEntry.id); @@ -745,6 +807,50 @@ export class MatroskaDemuxer extends Demuxer { } } + async loadSegmentMetadata(segment: Segment) { + for (const seekEntry of segment.seekEntries) { + if (seekEntry.id === EBMLId.Tags && !segment.tagsSeen) { + // We need to load the tags + } else if (seekEntry.id === EBMLId.Attachments && !segment.attachmentsSeen) { + // We need to load the attachments + } else { + continue; + } + + let slice = this.reader.requestSliceRange( + segment.dataStartPos + seekEntry.segmentPosition, + MIN_HEADER_SIZE, + MAX_HEADER_SIZE, + ); + if (slice instanceof Promise) slice = await slice; + if (!slice) continue; + + const header = readElementHeader(slice); + if (!header || header.id !== seekEntry.id) continue; + + const { size } = header; + assertDefinedSize(size); + + assert(!this.currentSegment); + this.currentSegment = segment; + + let dataSlice = this.reader.requestSlice(slice.filePos, size); + if (dataSlice instanceof Promise) dataSlice = await dataSlice; + if (dataSlice) { + this.readContiguousElements(dataSlice); + } + + this.currentSegment = null; + + // Mark as seen + if (seekEntry.id === EBMLId.Tags) { + segment.tagsSeen = true; + } else if (seekEntry.id === EBMLId.Attachments) { + segment.attachmentsSeen = true; + } + } + } + readContiguousElements(slice: FileSlice) { const startIndex = slice.filePos; @@ -883,6 +989,7 @@ export class MatroskaDemuxer extends Demuxer { } else if (codecIdWithoutSuffix === CODEC_STRING_MAP.opus) { this.currentTrack.info.codec = 'opus'; this.currentTrack.info.codecDescription = this.currentTrack.codecPrivate; + this.currentTrack.info.sampleRate = OPUS_SAMPLE_RATE; // Always the same } else if (codecIdWithoutSuffix === CODEC_STRING_MAP.vorbis) { this.currentTrack.info.codec = 'vorbis'; this.currentTrack.info.codecDescription = this.currentTrack.codecPrivate; @@ -1251,11 +1358,201 @@ export class MatroskaDemuxer extends Demuxer { // We'll offset this by the block's timestamp later this.currentBlock.referencedTimestamps.push(relativeTimestamp); }; break; + + case EBMLId.Tag: { + this.currentTagTargetIsMovie = true; + this.readContiguousElements(slice.slice(dataStartPos, size)); + }; break; + + case EBMLId.Targets: { + this.readContiguousElements(slice.slice(dataStartPos, size)); + }; break; + + case EBMLId.TargetTypeValue: { + const targetTypeValue = readUnsignedInt(slice, size); + if (targetTypeValue !== 50) { + this.currentTagTargetIsMovie = false; + } + }; break; + + case EBMLId.TagTrackUID: + case EBMLId.TagEditionUID: + case EBMLId.TagChapterUID: + case EBMLId.TagAttachmentUID: { + this.currentTagTargetIsMovie = false; + }; break; + + case EBMLId.SimpleTag: { + if (!this.currentTagTargetIsMovie) break; + + this.currentSimpleTagName = null; + this.readContiguousElements(slice.slice(dataStartPos, size)); + }; break; + + case EBMLId.TagName: { + this.currentSimpleTagName = readUnicodeString(slice, size); + }; break; + + case EBMLId.TagString: { + if (!this.currentSimpleTagName) break; + + const value = readUnicodeString(slice, size); + this.processTagValue(this.currentSimpleTagName, value); + }; break; + + case EBMLId.TagBinary: { + if (!this.currentSimpleTagName) break; + + const value = readBytes(slice, size); + this.processTagValue(this.currentSimpleTagName, value); + }; break; + + case EBMLId.AttachedFile: { + if (!this.currentSegment) break; + + this.currentAttachedFile = { + fileName: null, + fileMediaType: null, + fileData: null, + fileDescription: null, + }; + + this.readContiguousElements(slice.slice(dataStartPos, size)); + + // Only process image attachments + if (this.currentAttachedFile.fileMediaType?.startsWith('image/') && this.currentAttachedFile.fileData) { + const fileName = this.currentAttachedFile.fileName; + let kind: 'coverFront' | 'coverBack' | 'unknown' = 'unknown'; + + if (fileName) { + const lowerName = fileName.toLowerCase(); + if (lowerName.startsWith('cover.')) { + kind = 'coverFront'; + } else if (lowerName.startsWith('back.')) { + kind = 'coverBack'; + } + } + + this.currentSegment.metadataTags.images ??= []; + this.currentSegment.metadataTags.images.push({ + data: this.currentAttachedFile.fileData, + mimeType: this.currentAttachedFile.fileMediaType, + kind, + name: this.currentAttachedFile.fileName ?? undefined, + description: this.currentAttachedFile.fileDescription ?? undefined, + }); + } + + this.currentAttachedFile = null; + }; break; + + case EBMLId.FileName: { + if (!this.currentAttachedFile) break; + + this.currentAttachedFile.fileName = readUnicodeString(slice, size); + }; break; + + case EBMLId.FileMediaType: { + if (!this.currentAttachedFile) break; + + this.currentAttachedFile.fileMediaType = readAsciiString(slice, size); + }; break; + + case EBMLId.FileData: { + if (!this.currentAttachedFile) break; + + this.currentAttachedFile.fileData = readBytes(slice, size); + }; break; + + case EBMLId.FileDescription: { + if (!this.currentAttachedFile) break; + + this.currentAttachedFile.fileDescription = readUnicodeString(slice, size); + }; break; } slice.filePos = dataStartPos + size; return true; } + + processTagValue(name: string, value: string | Uint8Array) { + if (!this.currentSegment?.metadataTags) return; + + const metadataTags = this.currentSegment.metadataTags; + metadataTags.raw ??= {}; + metadataTags.raw[name] ??= value; + + if (typeof value === 'string') { + switch (name.toLowerCase()) { + case 'title': { + metadataTags.title ??= value; + }; break; + + case 'description': { + metadataTags.description ??= value; + }; break; + + case 'artist': { + metadataTags.artist ??= value; + }; break; + + case 'album': { + metadataTags.album ??= value; + }; break; + + case 'album_artist': { + metadataTags.albumArtist ??= value; + }; break; + + case 'genre': { + metadataTags.genre ??= value; + }; break; + + case 'comment': { + metadataTags.comment ??= value; + }; break; + + case 'lyrics': { + metadataTags.lyrics ??= value; + }; break; + + case 'date': { + const date = new Date(value); + if (!Number.isNaN(date.getTime())) { + metadataTags.date ??= date; + } + }; break; + + case 'track_number': + case 'part_number': { + const parts = value.split('/'); + const trackNum = Number.parseInt(parts[0]!, 10); + const tracksTotal = parts[1] && Number.parseInt(parts[1], 10); + + if (Number.isInteger(trackNum) && trackNum > 0) { + metadataTags.trackNumber ??= trackNum; + } + if (tracksTotal && Number.isInteger(tracksTotal) && tracksTotal > 0) { + metadataTags.tracksTotal ??= tracksTotal; + } + }; break; + + case 'disc_number': + case 'disc': { + const discParts = value.split('/'); + const discNum = Number.parseInt(discParts[0]!, 10); + const discsTotal = discParts[1] && Number.parseInt(discParts[1], 10); + + if (Number.isInteger(discNum) && discNum > 0) { + metadataTags.discNumber ??= discNum; + } + if (discsTotal && Number.isInteger(discsTotal) && discsTotal > 0) { + metadataTags.discsTotal ??= discsTotal; + } + }; break; + } + } + } } abstract class MatroskaTrackBacking implements InputTrackBacking { diff --git a/src/matroska/matroska-muxer.ts b/src/matroska/matroska-muxer.ts index 0e46954..93f052c 100644 --- a/src/matroska/matroska-muxer.ts +++ b/src/matroska/matroska-muxer.ts @@ -13,7 +13,10 @@ import { TRANSFER_CHARACTERISTICS_MAP, UNDETERMINED_LANGUAGE, assert, + assertNever, colorSpaceIsComplete, + imageMimeTypeToExtension, + keyValueIterator, normalizeRotation, promiseWithResolvers, roundToMultiple, @@ -44,7 +47,7 @@ import { parseSubtitleTimestamp, } from '../subtitles'; import { - OPUS_INTERNAL_SAMPLE_RATE, + OPUS_SAMPLE_RATE, PCM_AUDIO_CODECS, PcmAudioCodec, SubtitleCodec, @@ -62,7 +65,7 @@ import { parseOpusIdentificationHeader } from '../codec-data'; const MIN_CLUSTER_TIMESTAMP_MS = -(2 ** 15); const MAX_CLUSTER_TIMESTAMP_MS = 2 ** 15 - 1; -const APP_NAME = 'https://github.com/Vanilagy/mediabunny'; +const APP_NAME = 'Mediabunny'; const SEGMENT_SIZE_BYTES = 6; const CLUSTER_SIZE_BYTES = 5; @@ -74,22 +77,6 @@ type InternalMediaChunk = { additions: Uint8Array | null; }; -type SeekHead = { - id: number; - data: { - id: number; - data: ({ - id: number; - data: Uint8Array; - size?: undefined; - } | { - id: number; - size: number; - data: number; - })[]; - }[]; -}; - type MatroskaTrackData = { chunkQueue: InternalMediaChunk[]; lastWrittenMsTimestamp: number | null; @@ -137,8 +124,10 @@ export class MatroskaMuxer extends Muxer { private segment: EBMLElement | null = null; private segmentInfo: EBMLElement | null = null; - private seekHead: SeekHead | null = null; + private seekHead: EBMLElement | null = null; private tracksElement: EBMLElement | null = null; + private tagsElement: EBMLElement | null = null; + private attachmentsElement: EBMLElement | null = null; private segmentDuration: EBMLElement | null = null; private cues: EBMLElement | null = null; @@ -169,10 +158,6 @@ export class MatroskaMuxer extends Muxer { this.writeEBMLHeader(); - if (!this.format._options.appendOnly) { - this.createSeekHead(); - } - this.createSegmentInfo(); this.createCues(); @@ -207,24 +192,72 @@ export class MatroskaMuxer extends Muxer { * Creates a SeekHead element which is positioned near the start of the file and allows the media player to seek to * relevant sections more easily. Since we don't know the positions of those sections yet, we'll set them later. */ - private createSeekHead() { + private maybeCreateSeekHead(writeOffsets: boolean) { + if (this.format._options.appendOnly) { + return; + } + const kaxCues = new Uint8Array([0x1c, 0x53, 0xbb, 0x6b]); const kaxInfo = new Uint8Array([0x15, 0x49, 0xa9, 0x66]); const kaxTracks = new Uint8Array([0x16, 0x54, 0xae, 0x6b]); + const kaxAttachments = new Uint8Array([0x19, 0x41, 0xa4, 0x69]); + const kaxTags = new Uint8Array([0x12, 0x54, 0xc3, 0x67]); const seekHead = { id: EBMLId.SeekHead, data: [ { id: EBMLId.Seek, data: [ { id: EBMLId.SeekID, data: kaxCues }, - { id: EBMLId.SeekPosition, size: 5, data: 0 }, + { + id: EBMLId.SeekPosition, + size: 5, + data: writeOffsets + ? this.ebmlWriter.offsets.get(this.cues!)! - this.segmentDataOffset + : 0, + }, ] }, { id: EBMLId.Seek, data: [ { id: EBMLId.SeekID, data: kaxInfo }, - { id: EBMLId.SeekPosition, size: 5, data: 0 }, + { + id: EBMLId.SeekPosition, + size: 5, + data: writeOffsets + ? this.ebmlWriter.offsets.get(this.segmentInfo!)! - this.segmentDataOffset + : 0, + }, ] }, { id: EBMLId.Seek, data: [ { id: EBMLId.SeekID, data: kaxTracks }, - { id: EBMLId.SeekPosition, size: 5, data: 0 }, + { + id: EBMLId.SeekPosition, + size: 5, + data: writeOffsets + ? this.ebmlWriter.offsets.get(this.tracksElement!)! - this.segmentDataOffset + : 0, + }, ] }, + this.attachmentsElement + ? { id: EBMLId.Seek, data: [ + { id: EBMLId.SeekID, data: kaxAttachments }, + { + id: EBMLId.SeekPosition, + size: 5, + data: writeOffsets + ? this.ebmlWriter.offsets.get(this.attachmentsElement)! - this.segmentDataOffset + : 0, + }, + ] } + : null, + this.tagsElement + ? { id: EBMLId.Seek, data: [ + { id: EBMLId.SeekID, data: kaxTags }, + { + id: EBMLId.SeekPosition, + size: 5, + data: writeOffsets + ? this.ebmlWriter.offsets.get(this.tagsElement)! - this.segmentDataOffset + : 0, + }, + ] } + : null, ] }; this.seekHead = seekHead; } @@ -260,7 +293,7 @@ export class MatroskaMuxer extends Muxer { const header = parseOpusIdentificationHeader(bytes); // Use the preSkip value from the header - seekPreRollNs = Math.round(1e9 * (header.preSkip / OPUS_INTERNAL_SAMPLE_RATE)); + seekPreRollNs = Math.round(1e9 * (header.preSkip / OPUS_SAMPLE_RATE)); } } @@ -379,14 +412,187 @@ export class MatroskaMuxer extends Muxer { ]; } + private maybeCreateTags() { + const simpleTags: EBMLElement[] = []; + + const addSimpleTag = (key: string, value: string | Uint8Array) => { + simpleTags.push({ id: EBMLId.SimpleTag, data: [ + { id: EBMLId.TagName, data: new EBMLUnicodeString(key) }, + typeof value === 'string' + ? { id: EBMLId.TagString, data: new EBMLUnicodeString(value) } + : { id: EBMLId.TagBinary, data: value }, + ] }); + }; + + const metadataTags = this.output._metadataTags; + const writtenTags = new Set(); + + for (const { key, value } of keyValueIterator(metadataTags)) { + switch (key) { + case 'title': { + addSimpleTag('TITLE', value); + writtenTags.add('TITLE'); + }; break; + + case 'description': { + addSimpleTag('DESCRIPTION', value); + writtenTags.add('DESCRIPTION'); + }; break; + + case 'artist': { + addSimpleTag('ARTIST', value); + writtenTags.add('ARTIST'); + }; break; + + case 'album': { + addSimpleTag('ALBUM', value); + writtenTags.add('ALBUM'); + }; break; + + case 'albumArtist': { + addSimpleTag('ALBUM_ARTIST', value); + writtenTags.add('ALBUM_ARTIST'); + }; break; + + case 'genre': { + addSimpleTag('GENRE', value); + writtenTags.add('GENRE'); + }; break; + + case 'comment': { + addSimpleTag('COMMENT', value); + writtenTags.add('COMMENT'); + }; break; + + case 'lyrics': { + addSimpleTag('LYRICS', value); + writtenTags.add('LYRICS'); + }; break; + + case 'date': { + addSimpleTag('DATE', value.toISOString().slice(0, 10)); + writtenTags.add('DATE'); + }; break; + + case 'trackNumber': { + const string = metadataTags.tracksTotal !== undefined + ? `${value}/${metadataTags.tracksTotal}` + : value.toString(); + + addSimpleTag('PART_NUMBER', string); + writtenTags.add('PART_NUMBER'); + }; break; + + case 'discNumber': { + const string = metadataTags.discsTotal !== undefined + ? `${value}/${metadataTags.discsTotal}` + : value.toString(); + + addSimpleTag('DISC', string); + writtenTags.add('DISC'); + }; break; + + case 'tracksTotal': + case 'discsTotal': { + // Handled with trackNumber and discNumber respectively + }; break; + + case 'images': + case 'raw': { + // Handled elsewhere + }; break; + + default: assertNever(key); + } + } + + if (metadataTags.raw) { + for (const key in metadataTags.raw) { + const value = metadataTags.raw[key]!; + if (value == null || writtenTags.has(key)) { + continue; + } + + if (typeof value === 'string' || value instanceof Uint8Array) { + addSimpleTag(key, value); + } + } + } + + if (simpleTags.length === 0) { + return; + } + + this.tagsElement = { + id: EBMLId.Tags, + data: [{ id: EBMLId.Tag, data: [ + { id: EBMLId.Targets, data: [ + { id: EBMLId.TargetTypeValue, data: 50 }, + { id: EBMLId.TargetType, data: 'MOVIE' }, + ] }, + ...simpleTags, + ] }], + }; + } + + private maybeCreateAttachments() { + const metadataTags = this.output._metadataTags; + if (!metadataTags.images || metadataTags.images.length === 0) { + return; + } + + const existingFileUids = new Set(); + + this.attachmentsElement = { id: EBMLId.Attachments, data: metadataTags.images.map((image): EBMLElement => { + let imageName = image.name; + if (imageName === undefined) { + const baseName = image.kind === 'coverFront' ? 'cover' : image.kind === 'coverBack' ? 'back' : 'image'; + imageName = baseName + (imageMimeTypeToExtension(image.mimeType) ?? ''); + } + + let fileUid: number; + while (true) { + fileUid = Math.floor(Math.random() * Number.MAX_SAFE_INTEGER); + + if (fileUid !== 0 && !existingFileUids.has(fileUid)) { + break; + } + } + + existingFileUids.add(fileUid); + + return { + id: EBMLId.AttachedFile, + data: [ + image.description !== undefined + ? { id: EBMLId.FileDescription, data: new EBMLUnicodeString(image.description) } + : null, + { id: EBMLId.FileName, data: new EBMLUnicodeString(imageName) }, + { id: EBMLId.FileMediaType, data: image.mimeType }, + { id: EBMLId.FileData, data: image.data }, + { id: EBMLId.FileUID, data: fileUid }, + ], + }; + }) }; + } + private createSegment() { + this.createTracks(); + this.maybeCreateTags(); + this.maybeCreateAttachments(); + this.maybeCreateSeekHead(false); + const segment: EBML = { id: EBMLId.Segment, size: this.format._options.appendOnly ? -1 : SEGMENT_SIZE_BYTES, data: [ - !this.format._options.appendOnly ? this.seekHead as EBML : null, + this.seekHead, // null if append-only this.segmentInfo, this.tracksElement, + // Matroska spec says put this at the end of the file, but I think placing it before the first cluster + // makes more sense, and FFmpeg agrees (argumentum ad ffmpegum fallacy) + this.attachmentsElement, + this.tagsElement, ], }; this.segment = segment; @@ -754,7 +960,6 @@ export class MatroskaMuxer extends Muxer { private writeBlock(trackData: MatroskaTrackData, chunk: InternalMediaChunk) { // Due to the interlacing algorithm, this code will be run once we've seen one chunk from every media track. if (!this.segment) { - this.createTracks(); this.createSegment(); } @@ -956,7 +1161,6 @@ export class MatroskaMuxer extends Muxer { this.allTracksKnown.resolve(); if (!this.segment) { - this.createTracks(); this.createSegment(); } @@ -984,14 +1188,9 @@ export class MatroskaMuxer extends Muxer { this.ebmlWriter.writeEBML(this.segmentDuration); // Fill in SeekHead position data and write it again - this.seekHead!.data[0]!.data[1]!.data - = this.ebmlWriter.offsets.get(this.cues)! - this.segmentDataOffset; - this.seekHead!.data[1]!.data[1]!.data - = this.ebmlWriter.offsets.get(this.segmentInfo!)! - this.segmentDataOffset; - this.seekHead!.data[2]!.data[1]!.data - = this.ebmlWriter.offsets.get(this.tracksElement!)! - this.segmentDataOffset; - - this.writer.seek(this.ebmlWriter.offsets.get(this.seekHead!)!); + assert(this.seekHead); + this.writer.seek(this.ebmlWriter.offsets.get(this.seekHead)!); + this.maybeCreateSeekHead(true); this.ebmlWriter.writeEBML(this.seekHead); this.writer.seek(endPos); diff --git a/src/media-sink.ts b/src/media-sink.ts index 2fa5133..7bbabc1 100644 --- a/src/media-sink.ts +++ b/src/media-sink.ts @@ -7,6 +7,7 @@ */ import { parsePcmCodec, PCM_AUDIO_CODECS, PcmAudioCodec, VideoCodec, AudioCodec } from './codec'; +import { extractHevcNalUnits, extractNalUnitTypeForHevc, HevcNalUnitType } from './codec-data'; import { CustomVideoDecoder, customVideoDecoders, CustomAudioDecoder, customAudioDecoders } from './custom-coder'; import { InputAudioTrack, InputTrack, InputVideoTrack } from './input-track'; import { @@ -624,8 +625,8 @@ export abstract class BaseMediaSampleSink< const nextPacket = await packetSink.getNextPacket(currentPacket); assert(nextPacket); - currentPacket = nextPacket; decoder.decode(nextPacket); + currentPacket = nextPacket; } maxSequenceNumber = -1; @@ -757,12 +758,14 @@ class VideoDecoderWrapper extends DecoderWrapper { inputTimestamps: number[] = []; // Timestamps input into the decoder, sorted. sampleQueue: VideoSample[] = []; // Safari-specific thing, check usage. + currentPacketIndex = 0; + raslSkipped = false; // For HEVC stuff constructor( onSample: (sample: VideoSample) => unknown, onError: (error: DOMException) => unknown, - codec: VideoCodec, - decoderConfig: VideoDecoderConfig, + public codec: VideoCodec, + public decoderConfig: VideoDecoderConfig, public rotation: Rotation, public timeResolution: number, ) { @@ -848,6 +851,26 @@ class VideoDecoderWrapper extends DecoderWrapper { } decode(packet: EncodedPacket) { + if (this.codec === 'hevc' && this.currentPacketIndex > 0 && !this.raslSkipped) { + // If we're using HEVC, we need to make sure to skip any RASL slices that follow a non-IDR key frame such as + // CRA_NUT. This is because RASL slices cannot be decoded without data before the CRA_NUT. Browsers behave + // differently here: Chromium drops the packets, Safari throws a decoder error. Either way, it's not good + // and causes bugs upstream. So, let's take the dropping into our own hands. + const nalUnits = extractHevcNalUnits(packet.data, this.decoderConfig); + const hasRaslPicture = nalUnits.some((x) => { + const type = extractNalUnitTypeForHevc(x); + return type === HevcNalUnitType.RASL_N || type === HevcNalUnitType.RASL_R; + }); + + if (hasRaslPicture) { + return; // Drop + } + + this.raslSkipped = true; + } + + this.currentPacketIndex++; + if (this.customDecoder) { this.customDecoderQueueSize++; void this.customDecoderCallSerializer @@ -879,6 +902,9 @@ class VideoDecoderWrapper extends DecoderWrapper { this.sampleQueue.length = 0; } + + this.currentPacketIndex = 0; + this.raslSkipped = false; } close() { diff --git a/src/misc.ts b/src/misc.ts index 99aac7a..3126f03 100644 --- a/src/misc.ts +++ b/src/misc.ts @@ -177,6 +177,17 @@ export const toDataView = (source: AllowSharedBufferSource) => { export const textDecoder = new TextDecoder(); export const textEncoder = new TextEncoder(); +export const isIso88591Compatible = (text: string) => { + for (let i = 0; i < text.length; i++) { + const code = text.charCodeAt(i); + if (code > 255) { + return false; + } + } + + return true; +}; + const invertObject = (object: Record) => { return Object.fromEntries(Object.entries(object).map(([key, value]) => [value, key])) as Record; }; @@ -638,6 +649,77 @@ export const isSafari = () => { */ export type MaybePromise = T | Promise; +/** Acts like `??` except the condition is -1 and not null/undefined. */ +export const coalesceIndex = (a: number, b: number) => { + return a !== -1 ? a : b; +}; + export const closedIntervalsOverlap = (startA: number, endA: number, startB: number, endB: number) => { return startA <= endB && startB <= endA; }; + +type KeyValuePair> = { + [K in keyof T]-?: { + key: K; + value: T[K] extends infer R | undefined ? R : T[K]; + } +}[keyof T]; + +export const keyValueIterator = function* >(object: T) { + for (const key in object) { + const value = object[key]; + if (value === undefined) { + continue; + } + + yield { key, value } as KeyValuePair; + } +}; + +export const imageMimeTypeToExtension = (mimeType: string) => { + switch (mimeType.toLowerCase()) { + case 'image/jpeg': + case 'image/jpg': + return '.jpg'; + case 'image/png': + return '.png'; + case 'image/gif': + return '.gif'; + case 'image/webp': + return '.webp'; + case 'image/bmp': + return '.bmp'; + case 'image/svg+xml': + return '.svg'; + case 'image/tiff': + return '.tiff'; + case 'image/avif': + return '.avif'; + case 'image/x-icon': + case 'image/vnd.microsoft.icon': + return '.ico'; + default: + return null; + } +}; + +export const base64ToBytes = (base64: string) => { + const decoded = atob(base64); + const bytes = new Uint8Array(decoded.length); + + for (let i = 0; i < decoded.length; i++) { + bytes[i] = decoded.charCodeAt(i); + } + + return bytes; +}; + +export const bytesToBase64 = (bytes: Uint8Array) => { + let string = ''; + + for (let i = 0; i < bytes.length; i++) { + string += String.fromCharCode(bytes[i]!); + } + + return btoa(string); +}; diff --git a/src/mp3/mp3-demuxer.ts b/src/mp3/mp3-demuxer.ts index c31f463..dad16a1 100644 --- a/src/mp3/mp3-demuxer.ts +++ b/src/mp3/mp3-demuxer.ts @@ -10,12 +10,20 @@ import { AudioCodec } from '../codec'; import { Demuxer } from '../demuxer'; import { Input } from '../input'; import { InputAudioTrack, InputAudioTrackBacking } from '../input-track'; +import { MetadataTags } from '../tags'; import { PacketRetrievalOptions } from '../media-sink'; import { assert, AsyncMutex, binarySearchExact, binarySearchLessOrEqual, UNDETERMINED_LANGUAGE } from '../misc'; import { EncodedPacket, PLACEHOLDER_DATA } from '../packet'; import { FrameHeader, getXingOffset, INFO, XING } from '../../shared/mp3-misc'; -import { readId3, readNextFrameHeader } from './mp3-reader'; -import { readBytes, Reader, readU32Be } from '../reader'; +import { + ID3_V1_TAG_SIZE, + ID3_V2_HEADER_SIZE, + parseId3V1Tag, + parseId3V2Tag, + readId3V2Header, + readNextFrameHeader, +} from './mp3-reader'; +import { readAscii, readBytes, Reader, readU32Be } from '../reader'; type Sample = { timestamp: number; @@ -30,6 +38,7 @@ export class Mp3Demuxer extends Demuxer { metadataPromise: Promise | null = null; firstFrameHeader: FrameHeader | null = null; loadedSamples: Sample[] = []; // All samples from the start of the file to lastLoadedPos + metadataTags: MetadataTags | null = null; tracks: InputAudioTrack[] = []; @@ -51,8 +60,9 @@ export class Mp3Demuxer extends Demuxer { await this.advanceReader(); } - // There has to be a frame if this demuxer got selected - assert(this.firstFrameHeader); + if (!this.firstFrameHeader) { + throw new Error('No valid MP3 frame found.'); + } this.tracks = [new InputAudioTrack(new Mp3AudioTrackBacking(this))]; })(); @@ -60,18 +70,22 @@ export class Mp3Demuxer extends Demuxer { async advanceReader() { if (this.lastLoadedPos === 0) { - let slice = this.reader.requestSlice(0, 10); - if (slice instanceof Promise) slice = await slice; + // Let's skip all ID3v2 tags at the start of the file + while (true) { + let slice = this.reader.requestSlice(this.lastLoadedPos, ID3_V2_HEADER_SIZE); + if (slice instanceof Promise) slice = await slice; - if (!slice) { - this.lastSampleLoaded = true; - return; - } + if (!slice) { + this.lastSampleLoaded = true; + return; + } - // First time, let's see if there's an ID3 tag - const id3Tag = readId3(slice); - if (id3Tag) { - this.lastLoadedPos += 10 + id3Tag.size; + const id3V2Header = readId3V2Header(slice); + if (!id3V2Header) { + break; + } + + this.lastLoadedPos = slice.filePos + id3V2Header.size; } } @@ -134,6 +148,59 @@ export class Mp3Demuxer extends Demuxer { return track.computeDuration(); } + + async getMetadataTags() { + const release = await this.readingMutex.acquire(); + + try { + await this.readMetadata(); + + if (this.metadataTags) { + return this.metadataTags; + } + + this.metadataTags = {}; + let currentPos = 0; + let id3V2HeaderFound = false; + + while (true) { + let headerSlice = this.reader.requestSlice(currentPos, ID3_V2_HEADER_SIZE); + if (headerSlice instanceof Promise) headerSlice = await headerSlice; + if (!headerSlice) break; + + const id3V2Header = readId3V2Header(headerSlice); + if (!id3V2Header) { + break; + } + + id3V2HeaderFound = true; + + let contentSlice = this.reader.requestSlice(headerSlice.filePos, id3V2Header.size); + if (contentSlice instanceof Promise) contentSlice = await contentSlice; + if (!contentSlice) break; + + parseId3V2Tag(contentSlice, id3V2Header, this.metadataTags); + + currentPos = headerSlice.filePos + id3V2Header.size; + } + + if (!id3V2HeaderFound && this.reader.fileSize !== null && this.reader.fileSize >= ID3_V1_TAG_SIZE) { + // Try reading an ID3v1 tag at the end of the file + let slice = this.reader.requestSlice(this.reader.fileSize - ID3_V1_TAG_SIZE, ID3_V1_TAG_SIZE); + if (slice instanceof Promise) slice = await slice; + assert(slice); + + const tag = readAscii(slice, 3); + if (tag === 'TAG') { + parseId3V1Tag(slice, this.metadataTags); + } + } + + return this.metadataTags; + } finally { + release(); + } + } } class Mp3AudioTrackBacking implements InputAudioTrackBacking { diff --git a/src/mp3/mp3-muxer.ts b/src/mp3/mp3-muxer.ts index 509c005..a18335b 100644 --- a/src/mp3/mp3-muxer.ts +++ b/src/mp3/mp3-muxer.ts @@ -6,7 +6,8 @@ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -import { assert, toDataView } from '../misc'; +import { assert, assertNever, keyValueIterator, textEncoder, toDataView } from '../misc'; +import { metadataTagsAreEmpty, MetadataTags } from '../tags'; import { Muxer } from '../muxer'; import { Output, OutputAudioTrack } from '../output'; import { Mp3OutputFormat } from '../output-format'; @@ -14,6 +15,7 @@ import { EncodedPacket } from '../packet'; import { Writer } from '../writer'; import { getXingOffset, INFO, readFrameHeader, XING } from '../../shared/mp3-misc'; import { Mp3Writer, XingFrameData } from './mp3-writer'; +import { Id3V2TextEncoding } from './mp3-reader'; export class Mp3Muxer extends Muxer { private format: Mp3OutputFormat; @@ -22,6 +24,7 @@ export class Mp3Muxer extends Muxer { private xingFrameData: XingFrameData | null = null; private frameCount = 0; private framePositions: number[] = []; + private xingFramePos: number | null = null; constructor(output: Output, format: Mp3OutputFormat) { super(output); @@ -32,7 +35,9 @@ export class Mp3Muxer extends Muxer { } async start() { - // Nothing needed here + if (!metadataTagsAreEmpty(this.output._metadataTags)) { + this.writeId3v2Tag(this.output._metadataTags); + } } async getMimeType() { @@ -92,6 +97,7 @@ export class Mp3Muxer extends Muxer { // Write a Xing frame because this muxer doesn't make any bitrate constraints, meaning we don't know if // this will be a constant or variable bitrate file. Therefore, always write the Xing frame. + this.xingFramePos = this.writer.getPos(); this.mp3Writer.writeXingFrame(this.xingFrameData); this.frameCount++; @@ -116,8 +122,140 @@ export class Mp3Muxer extends Muxer { throw new Error('MP3 does not support subtitles.'); } + writeId3v2Tag(tags: MetadataTags) { + this.mp3Writer.writeAscii('ID3'); + this.mp3Writer.writeU8(0x04); // Version 2.4 + this.mp3Writer.writeU8(0x00); // Revision 0 + this.mp3Writer.writeU8(0x00); // Flags + this.mp3Writer.writeSynchsafeU32(0); // Size placeholder + + const startPos = this.writer.getPos(); + const writtenTags = new Set(); + + for (const { key, value } of keyValueIterator(tags)) { + switch (key) { + case 'title': { + this.mp3Writer.writeId3V2TextFrame('TIT2', value); + writtenTags.add('TIT2'); + }; break; + + case 'description': { + this.mp3Writer.writeId3V2TextFrame('TIT3', value); + writtenTags.add('TIT3'); + }; break; + + case 'artist': { + this.mp3Writer.writeId3V2TextFrame('TPE1', value); + writtenTags.add('TPE1'); + }; break; + + case 'album': { + this.mp3Writer.writeId3V2TextFrame('TALB', value); + writtenTags.add('TALB'); + }; break; + + case 'albumArtist': { + this.mp3Writer.writeId3V2TextFrame('TPE2', value); + writtenTags.add('TPE2'); + }; break; + + case 'trackNumber': { + const string = tags.tracksTotal !== undefined + ? `${value}/${tags.tracksTotal}` + : value.toString(); + + this.mp3Writer.writeId3V2TextFrame('TRCK', string); + writtenTags.add('TRCK'); + }; break; + + case 'discNumber': { + const string = tags.discsTotal !== undefined + ? `${value}/${tags.discsTotal}` + : value.toString(); + + this.mp3Writer.writeId3V2TextFrame('TPOS', string); + writtenTags.add('TPOS'); + }; break; + + case 'genre': { + this.mp3Writer.writeId3V2TextFrame('TCON', value); + writtenTags.add('TCON'); + }; break; + + case 'date': { + this.mp3Writer.writeId3V2TextFrame('TDRC', value.toISOString().slice(0, 10)); + writtenTags.add('TDRC'); + }; break; + + case 'lyrics': { + this.mp3Writer.writeId3V2LyricsFrame(value); + writtenTags.add('USLT'); + }; break; + + case 'comment': { + this.mp3Writer.writeId3V2CommentFrame(value); + writtenTags.add('COMM'); + }; break; + + case 'images': { + const pictureTypeMap = { coverFront: 0x03, coverBack: 0x04, unknown: 0x00 }; + for (const image of value) { + const pictureType = pictureTypeMap[image.kind]; + const description = image.description ?? ''; + this.mp3Writer.writeId3V2ApicFrame(image.mimeType, pictureType, description, image.data); + } + }; break; + + case 'tracksTotal': + case 'discsTotal': { + // Handled with trackNumber and discNumber respectively + }; break; + + case 'raw': { + // Handled later + }; break; + + default: assertNever(key); + } + } + + if (tags.raw) { + for (const key in tags.raw) { + const value = tags.raw[key]; + if (value == null || key.length !== 4 || writtenTags.has(key)) { + continue; + } + + let bytes: Uint8Array; + if (typeof value === 'string') { + const encoded = textEncoder.encode(value); + bytes = new Uint8Array(encoded.byteLength + 2); + bytes[0] = Id3V2TextEncoding.UTF_8; + bytes.set(encoded, 1); + // Last byte is the null terminator + } else if (value instanceof Uint8Array) { + bytes = value; + } else { + continue; + } + + this.mp3Writer.writeAscii(key); + this.mp3Writer.writeSynchsafeU32(bytes.byteLength); + this.mp3Writer.writeU16(0x0000); + this.writer.write(bytes); + } + } + + const endPos = this.writer.getPos(); + const framesSize = endPos - startPos; + + this.writer.seek(6); + this.mp3Writer.writeSynchsafeU32(framesSize); + this.writer.seek(endPos); + } + async finalize() { - if (!this.xingFrameData) { + if (!this.xingFrameData || this.xingFramePos === null) { return; } @@ -125,7 +263,7 @@ export class Mp3Muxer extends Muxer { const endPos = this.writer.getPos(); - this.writer.seek(0); + this.writer.seek(this.xingFramePos); const toc = new Uint8Array(100); for (let i = 0; i < 100; i++) { diff --git a/src/mp3/mp3-reader.ts b/src/mp3/mp3-reader.ts index d53708d..f857528 100644 --- a/src/mp3/mp3-reader.ts +++ b/src/mp3/mp3-reader.ts @@ -6,22 +6,65 @@ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -import { FRAME_HEADER_SIZE, FrameHeader, readFrameHeader } from '../../shared/mp3-misc'; -import { FileSlice, readAscii, Reader, readU32Be } from '../reader'; +import { decodeSynchsafe, FRAME_HEADER_SIZE, FrameHeader, readFrameHeader } from '../../shared/mp3-misc'; +import { MetadataTags } from '../tags'; +import { coalesceIndex, textDecoder } from '../misc'; +import { FileSlice, readAscii, readBytes, Reader, readU32Be, readU8 } from '../reader'; -export const readId3 = (slice: FileSlice) => { - const tag = readAscii(slice, 3); - if (tag !== 'ID3') { - slice.skip(-3); - return null; - } - - slice.skip(3); - - const size = decodeSynchsafe(readU32Be(slice)); - return { size }; +export type Id3V2Header = { + majorVersion: number; + revision: number; + flags: number; + size: number; }; +export enum Id3V2HeaderFlags { + Unsynchronisation = 1 << 7, + ExtendedHeader = 1 << 6, + ExperimentalIndicator = 1 << 5, + Footer = 1 << 4, +} + +export enum Id3V2TextEncoding { + ISO_8859_1, + UTF_16_WITH_BOM, + UTF_16_BE_NO_BOM, + UTF_8, +} + +export const ID3_V1_TAG_SIZE = 128; +export const ID3_V2_HEADER_SIZE = 10; + +export const ID3_V1_GENRES = [ + 'Blues', 'Classic rock', 'Country', 'Dance', 'Disco', 'Funk', 'Grunge', 'Hip-hop', 'Jazz', + 'Metal', 'New age', 'Oldies', 'Other', 'Pop', 'Rhythm and blues', 'Rap', 'Reggae', 'Rock', + 'Techno', 'Industrial', 'Alternative', 'Ska', 'Death metal', 'Pranks', 'Soundtrack', + 'Euro-techno', 'Ambient', 'Trip-hop', 'Vocal', 'Jazz & funk', 'Fusion', 'Trance', 'Classical', + 'Instrumental', 'Acid', 'House', 'Game', 'Sound clip', 'Gospel', 'Noise', 'Alternative rock', + 'Bass', 'Soul', 'Punk', 'Space', 'Meditative', 'Instrumental pop', 'Instrumental rock', + 'Ethnic', 'Gothic', 'Darkwave', 'Techno-industrial', 'Electronic', 'Pop-folk', 'Eurodance', + 'Dream', 'Southern rock', 'Comedy', 'Cult', 'Gangsta', 'Top 40', 'Christian rap', 'Pop/funk', + 'Jungle music', 'Native US', 'Cabaret', 'New wave', 'Psychedelic', 'Rave', 'Showtunes', + 'Trailer', 'Lo-fi', 'Tribal', 'Acid punk', 'Acid jazz', 'Polka', 'Retro', 'Musical', + 'Rock \'n\' roll', 'Hard rock', 'Folk', 'Folk rock', 'National folk', 'Swing', 'Fast fusion', + 'Bebop', 'Latin', 'Revival', 'Celtic', 'Bluegrass', 'Avantgarde', 'Gothic rock', + 'Progressive rock', 'Psychedelic rock', 'Symphonic rock', 'Slow rock', 'Big band', 'Chorus', + 'Easy listening', 'Acoustic', 'Humour', 'Speech', 'Chanson', 'Opera', 'Chamber music', + 'Sonata', 'Symphony', 'Booty bass', 'Primus', 'Porn groove', 'Satire', 'Slow jam', 'Club', + 'Tango', 'Samba', 'Folklore', 'Ballad', 'Power ballad', 'Rhythmic Soul', 'Freestyle', 'Duet', + 'Punk rock', 'Drum solo', 'A cappella', 'Euro-house', 'Dance hall', 'Goa music', 'Drum & bass', + 'Club-house', 'Hardcore techno', 'Terror', 'Indie', 'Britpop', 'Negerpunk', 'Polsk punk', + 'Beat', 'Christian gangsta rap', 'Heavy metal', 'Black metal', 'Crossover', + 'Contemporary Christian', 'Christian rock', 'Merengue', 'Salsa', 'Thrash metal', 'Anime', + 'Jpop', 'Synthpop', 'Christmas', 'Art rock', 'Baroque', 'Bhangra', 'Big beat', 'Breakbeat', + 'Chillout', 'Downtempo', 'Dub', 'EBM', 'Eclectic', 'Electro', 'Electroclash', 'Emo', + 'Experimental', 'Garage', 'Global', 'IDM', 'Illbient', 'Industro-Goth', 'Jam Band', + 'Krautrock', 'Leftfield', 'Lounge', 'Math rock', 'New romantic', 'Nu-breakz', 'Post-punk', + 'Post-rock', 'Psytrance', 'Shoegaze', 'Space rock', 'Trop rock', 'World music', 'Neoclassical', + 'Audiobook', 'Audio theatre', 'Neue Deutsche Welle', 'Podcast', 'Indie rock', 'G-Funk', + 'Dubstep', 'Garage rock', 'Psybient', +]; + export const readNextFrameHeader = async (reader: Reader, startPos: number, until: number | null): Promise<{ header: FrameHeader; startPos: number; @@ -46,15 +89,570 @@ export const readNextFrameHeader = async (reader: Reader, startPos: number, unti return null; }; -export const decodeSynchsafe = (synchsafed: number) => { - let mask = 0x7f000000; - let unsynchsafed = 0; +export const parseId3V1Tag = (slice: FileSlice, tags: MetadataTags) => { + const startPos = slice.filePos; + tags.raw ??= {}; + tags.raw['TAG'] ??= readBytes(slice, ID3_V1_TAG_SIZE - 3); // Dump the whole tag into the raw metadata + slice.filePos = startPos; - while (mask !== 0) { - unsynchsafed >>= 1; - unsynchsafed |= synchsafed & mask; - mask >>= 8; + const title = readId3V1String(slice, 30); + if (title) tags.title ??= title; + + const artist = readId3V1String(slice, 30); + if (artist) tags.artist ??= artist; + + const album = readId3V1String(slice, 30); + if (album) tags.album ??= album; + + const yearText = readId3V1String(slice, 4); + const year = Number.parseInt(yearText, 10); + if (Number.isInteger(year) && year > 0) { + tags.date ??= new Date(year, 0, 1); } - return unsynchsafed; + const commentBytes = readBytes(slice, 30); + let comment: string; + + // Check for the ID3v1.1 track number format: + // The 29th byte (index 28) is a null terminator, and the 30th byte is the track number. + if (commentBytes[28] === 0 && commentBytes[29] !== 0) { + const trackNum = commentBytes[29]!; + if (trackNum > 0) { + tags.trackNumber ??= trackNum; + } + + slice.skip(-30); + comment = readId3V1String(slice, 28); + slice.skip(2); + } else { + slice.skip(-30); + comment = readId3V1String(slice, 30); + } + + if (comment) tags.comment ??= comment; + + const genreIndex = readU8(slice); + if (genreIndex < ID3_V1_GENRES.length) { + tags.genre ??= ID3_V1_GENRES[genreIndex]; + } }; + +export const readId3V1String = (slice: FileSlice, length: number) => { + const bytes = readBytes(slice, length); + + const endIndex = coalesceIndex(bytes.indexOf(0), bytes.length); + const relevantBytes = bytes.subarray(0, endIndex); + + // Decode as ISO-8859-1 + let str = ''; + for (let i = 0; i < relevantBytes.length; i++) { + str += String.fromCharCode(relevantBytes[i]!); + } + + return str.trimEnd(); // String also may be padded with spaces +}; + +export const readId3V2Header = (slice: FileSlice): Id3V2Header | null => { + const startPos = slice.filePos; + + const tag = readAscii(slice, 3); + const majorVersion = readU8(slice); + const revision = readU8(slice); + const flags = readU8(slice); + const sizeRaw = readU32Be(slice); + + if (tag !== 'ID3' || majorVersion === 0xff || revision === 0xff || (sizeRaw & 0x80808080) !== 0) { + slice.filePos = startPos; + return null; + } + + const size = decodeSynchsafe(sizeRaw); + + return { majorVersion, revision, flags, size }; +}; + +export const parseId3V2Tag = (slice: FileSlice, header: Id3V2Header, tags: MetadataTags) => { + // https://id3.org/id3v2.3.0 + + if (![2, 3, 4].includes(header.majorVersion)) { + console.warn(`Unsupported ID3v2 major version: ${header.majorVersion}`); + return; + } + + const bytes = readBytes(slice, header.size); + const reader = new Id3V2Reader(header, bytes); + + if (header.flags & Id3V2HeaderFlags.Footer) { + reader.removeFooter(); + } + + if ((header.flags & Id3V2HeaderFlags.Unsynchronisation) && header.majorVersion === 3) { + reader.ununsynchronizeAll(); + } + + if (header.flags & Id3V2HeaderFlags.ExtendedHeader) { + const extendedHeaderSize = reader.readU32(); + + if (header.majorVersion === 3) { + reader.pos += extendedHeaderSize; // The extended header size excludes itself + } else { + reader.pos += extendedHeaderSize - 4; // The extended header size includes itself + } + } + + while (reader.pos <= reader.bytes.length - reader.frameHeaderSize()) { + const frame = reader.readId3V2Frame(); + if (!frame) { + break; + } + + const frameStartPos = reader.pos; + const frameEndPos = reader.pos + frame.size; + + let frameEncrypted = false; + let frameCompressed = false; + let frameUnsynchronized = false; + + if (header.majorVersion === 3) { + frameEncrypted = !!(frame.flags & (1 << 6)); + frameCompressed = !!(frame.flags & (1 << 7)); + } else if (header.majorVersion === 4) { + frameEncrypted = !!(frame.flags & (1 << 2)); + frameCompressed = !!(frame.flags & (1 << 3)); + frameUnsynchronized = !!(frame.flags & (1 << 1)) + || !!(header.flags & Id3V2HeaderFlags.Unsynchronisation); + } + + if (frameEncrypted) { + console.warn(`Skipping encrypted ID3v2 frame ${frame.id}`); + reader.pos = frameEndPos; + continue; + } + + if (frameCompressed) { + console.warn(`Skipping compressed ID3v2 frame ${frame.id}`); // Maybe someday? Idk + reader.pos = frameEndPos; + continue; + } + + if (frameUnsynchronized) { + reader.ununsynchronizeRegion(reader.pos, frameEndPos); + } + + tags.raw ??= {}; + if (frame.id[0] === 'T') { + // It's a text frame, let's decode as text + tags.raw[frame.id] ??= reader.readId3V2EncodingAndText(frameEndPos); + } else { + // For the others, let's just get the bytes + tags.raw[frame.id] ??= reader.readBytes(frame.size); + } + + reader.pos = frameStartPos; + + switch (frame.id) { + case 'TIT2': + case 'TT2': { + tags.title ??= reader.readId3V2EncodingAndText(frameEndPos); + }; break; + + case 'TIT3': + case 'TT3': { + tags.description ??= reader.readId3V2EncodingAndText(frameEndPos); + }; break; + + case 'TPE1': + case 'TP1': { + tags.artist ??= reader.readId3V2EncodingAndText(frameEndPos); + }; break; + + case 'TALB': + case 'TAL': { + tags.album ??= reader.readId3V2EncodingAndText(frameEndPos); + }; break; + + case 'TPE2': + case 'TP2': { + tags.albumArtist ??= reader.readId3V2EncodingAndText(frameEndPos); + }; break; + + case 'TRCK': + case 'TRK': { + const trackText = reader.readId3V2EncodingAndText(frameEndPos); + const parts = trackText.split('/'); + const trackNum = Number.parseInt(parts[0]!, 10); + const tracksTotal = parts[1] && Number.parseInt(parts[1], 10); + + if (Number.isInteger(trackNum) && trackNum > 0) { + tags.trackNumber ??= trackNum; + } + if (tracksTotal && Number.isInteger(tracksTotal) && tracksTotal > 0) { + tags.tracksTotal ??= tracksTotal; + } + }; break; + + case 'TPOS': + case 'TPA': { + const discText = reader.readId3V2EncodingAndText(frameEndPos); + const parts = discText.split('/'); + const discNum = Number.parseInt(parts[0]!, 10); + const discsTotal = parts[1] && Number.parseInt(parts[1], 10); + + if (Number.isInteger(discNum) && discNum > 0) { + tags.discNumber ??= discNum; + } + if (discsTotal && Number.isInteger(discsTotal) && discsTotal > 0) { + tags.discsTotal ??= discsTotal; + } + }; break; + + case 'TCON': + case 'TCO': { + const genreText = reader.readId3V2EncodingAndText(frameEndPos); + let match = /^\((\d+)\)/.exec(genreText); + if (match) { + const genreNumber = Number.parseInt(match[1]!); + if (ID3_V1_GENRES[genreNumber] !== undefined) { + tags.genre ??= ID3_V1_GENRES[genreNumber]; + break; + } + } + + match = /^\d+$/.exec(genreText); + if (match) { + const genreNumber = Number.parseInt(match[0]); + if (ID3_V1_GENRES[genreNumber] !== undefined) { + tags.genre ??= ID3_V1_GENRES[genreNumber]; + break; + } + } + + tags.genre ??= genreText; + }; break; + + case 'TDRC': + case 'TDAT': { + const dateText = reader.readId3V2EncodingAndText(frameEndPos); + const date = new Date(dateText); + + if (!Number.isNaN(date.getTime())) { + tags.date ??= date; + } + }; break; + + case 'TYER': + case 'TYE': { + const yearText = reader.readId3V2EncodingAndText(frameEndPos); + const year = Number.parseInt(yearText, 10); + + if (Number.isInteger(year)) { + tags.date ??= new Date(year, 0, 1); + } + }; break; + + case 'USLT': + case 'ULT': { + const encoding = reader.readU8(); + reader.pos += 3; // Skip language + reader.readId3V2Text(encoding, frameEndPos); // Short content description + tags.lyrics ??= reader.readId3V2Text(encoding, frameEndPos); + }; break; + + case 'COMM': + case 'COM': { + const encoding = reader.readU8(); + reader.pos += 3; // Skip language + reader.readId3V2Text(encoding, frameEndPos); // Short content description + tags.comment ??= reader.readId3V2Text(encoding, frameEndPos); + }; break; + + case 'APIC': + case 'PIC': { + const encoding = reader.readId3V2TextEncoding(); + + let mimeType: string; + if (header.majorVersion === 2) { + const imageFormat = reader.readAscii(3); + mimeType = imageFormat === 'PNG' + ? 'image/png' + : imageFormat === 'JPG' + ? 'image/jpeg' + : 'image/*'; + } else { + mimeType = reader.readId3V2Text(encoding, frameEndPos); + } + + const pictureType = reader.readU8(); + const description = reader.readId3V2Text(encoding, frameEndPos).trimEnd(); // Trim ending spaces + + const imageDataSize = frameEndPos - reader.pos; + if (imageDataSize >= 0) { + const imageData = reader.readBytes(imageDataSize); + + if (!tags.images) tags.images = []; + tags.images.push({ + data: imageData, + mimeType, + kind: pictureType === 3 + ? 'coverFront' + : pictureType === 4 + ? 'coverBack' + : 'unknown', + description, + }); + } + }; break; + + default: { + reader.pos += frame.size; + }; break; + } + + reader.pos = frameEndPos; + } +}; + +// https://id3.org/id3v2.3.0 +export class Id3V2Reader { + pos = 0; + view: DataView; + + constructor(public header: Id3V2Header, public bytes: Uint8Array) { + this.view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + } + + frameHeaderSize() { + return this.header.majorVersion === 2 ? 6 : 10; + } + + ununsynchronizeAll() { + const newBytes: number[] = []; + + for (let i = 0; i < this.bytes.length; i++) { + const value1 = this.bytes[i]!; + newBytes.push(value1); + + if (value1 === 0xff && i !== this.bytes.length - 1) { + const value2 = this.bytes[i]!; + if (value2 === 0x00) { + i++; + } + } + } + + this.bytes = new Uint8Array(newBytes); + this.view = new DataView(this.bytes.buffer); + } + + ununsynchronizeRegion(start: number, end: number) { + const newBytes: number[] = []; + + for (let i = start; i < end; i++) { + const value1 = this.bytes[i]!; + newBytes.push(value1); + + if (value1 === 0xff && i !== end - 1) { + const value2 = this.bytes[i + 1]!; + if (value2 === 0x00) { + i++; + } + } + } + + const before = this.bytes.subarray(0, start); + const after = this.bytes.subarray(end); + + this.bytes = new Uint8Array(before.length + newBytes.length + after.length); + this.bytes.set(before, 0); + this.bytes.set(newBytes, before.length); + this.bytes.set(after, before.length + newBytes.length); + + this.view = new DataView(this.bytes.buffer); + } + + removeFooter() { + this.bytes = this.bytes.subarray(0, this.bytes.length - ID3_V2_HEADER_SIZE); + this.view = new DataView(this.bytes.buffer); + } + + readBytes(length: number) { + const slice = this.bytes.subarray(this.pos, this.pos + length); + this.pos += length; + return slice; + } + + readU8() { + const value = this.view.getUint8(this.pos); + this.pos += 1; + return value; + } + + readU16() { + const value = this.view.getUint16(this.pos, false); + this.pos += 2; + return value; + } + + readU24() { + const high = this.view.getUint16(this.pos, false); + const low = this.view.getUint8(this.pos + 1); + this.pos += 3; + return high * 0x100 + low; + } + + readU32() { + const value = this.view.getUint32(this.pos, false); + this.pos += 4; + return value; + } + + readAscii(length: number) { + let str = ''; + for (let i = 0; i < length; i++) { + str += String.fromCharCode(this.view.getUint8(this.pos + i)); + } + this.pos += length; + return str; + } + + readId3V2Frame() { + if (this.header.majorVersion === 2) { + const id = this.readAscii(3); + if (id === '\x00\x00\x00') { + return null; + } + + const size = this.readU24(); + + return { id, size, flags: 0 }; + } else { + const id = this.readAscii(4); + if (id === '\x00\x00\x00\x00') { + // We've landed in the padding section + return null; + } + + const sizeRaw = this.readU32(); + let size = this.header.majorVersion === 4 + ? decodeSynchsafe(sizeRaw) + : sizeRaw; + const flags = this.readU16(); + const headerEndPos = this.pos; + + // Some files may have incorrectly synchsafed/unsynchsafed sizes. To validate which interpretation is valid, + // we validate a size by skipping ahead and seeing if we land at a valid frame header (or at the end of the + // tag. + + const isSizeValid = (size: number) => { + const nextPos = this.pos + size; + if (nextPos > this.bytes.length) { + return false; + } + + if (nextPos <= this.bytes.length - this.frameHeaderSize()) { + this.pos += size; + const nextId = this.readAscii(4); + if (nextId !== '\x00\x00\x00\x00' && !/[0-9A-Z]{4}/.test(nextId)) { + return false; + } + } + + return true; + }; + + if (!isSizeValid(size)) { + // Flip the synchsafing, and try if this one makes more sense + const otherSize = this.header.majorVersion === 4 + ? sizeRaw + : decodeSynchsafe(sizeRaw); + + if (isSizeValid(otherSize)) { + size = otherSize; + } + } + + this.pos = headerEndPos; + return { id, size, flags }; + } + } + + readId3V2TextEncoding(): Id3V2TextEncoding { + const number = this.readU8(); + if (number > 3) { + throw new Error(`Unsupported text encoding: ${number}`); + } + return number; + } + + readId3V2Text(encoding: Id3V2TextEncoding, until: number): string { + const startPos = this.pos; + const data = this.readBytes(until); + + switch (encoding) { + case Id3V2TextEncoding.ISO_8859_1: { + let str = ''; + + for (let i = 0; i < data.length; i++) { + const value = data[i]!; + if (value === 0) { + this.pos = startPos + i + 1; + break; + } + str += String.fromCharCode(value); + } + + return str; + } + + case Id3V2TextEncoding.UTF_16_WITH_BOM: { + if (data[0] === 0xff && data[1] === 0xfe) { + const decoder = new TextDecoder('utf-16le'); + const endIndex = coalesceIndex( + data.findIndex((x, i) => x === 0 && data[i + 1] === 0 && i % 2 === 0), + data.length, + ); + this.pos = startPos + Math.min(endIndex + 2, data.length); + return decoder.decode(data.subarray(2, endIndex)); + } else if (data[0] === 0xfe && data[1] === 0xff) { + const decoder = new TextDecoder('utf-16be'); + const endIndex = coalesceIndex( + data.findIndex((x, i) => x === 0 && data[i + 1] === 0 && i % 2 === 0), + data.length, + ); + this.pos = startPos + Math.min(endIndex + 2, data.length); + return decoder.decode(data.subarray(2, endIndex)); + } else { + // Treat it like UTF-8, some files do this + const endIndex = coalesceIndex(data.findIndex(x => x === 0), data.length); + this.pos = startPos + Math.min(endIndex + 1, data.length); + return textDecoder.decode(data.subarray(0, endIndex)); + } + } + + case Id3V2TextEncoding.UTF_16_BE_NO_BOM: { + const decoder = new TextDecoder('utf-16be'); + const endIndex = coalesceIndex( + data.findIndex((x, i) => x === 0 && data[i + 1] === 0 && i % 2 === 0), + data.length, + ); + this.pos = startPos + Math.min(endIndex + 2, data.length); + return decoder.decode(data.subarray(0, endIndex)); + } + + case Id3V2TextEncoding.UTF_8: { + const endIndex = coalesceIndex(data.findIndex(x => x === 0), data.length); + this.pos = startPos + Math.min(endIndex + 1, data.length); + return textDecoder.decode(data.subarray(0, endIndex)); + } + } + } + + readId3V2EncodingAndText(until: number) { + if (this.pos >= until) { + return ''; + } + + const encoding = this.readId3V2TextEncoding(); + return this.readId3V2Text(encoding, until); + } +} diff --git a/src/mp3/mp3-writer.ts b/src/mp3/mp3-writer.ts index 2f697f2..4bd726f 100644 --- a/src/mp3/mp3-writer.ts +++ b/src/mp3/mp3-writer.ts @@ -6,15 +6,18 @@ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ +import { isIso88591Compatible, textEncoder } from '../misc'; import { Writer } from '../writer'; import { computeMp3FrameSize, + encodeSynchsafe, getXingOffset, MPEG_V1_BITRATES, MPEG_V2_BITRATES, SAMPLING_RATES, XING, } from '../../shared/mp3-misc'; +import { Id3V2TextEncoding } from './mp3-reader'; export type XingFrameData = { mpegVersionId: number; @@ -37,11 +40,137 @@ export class Mp3Writer { constructor(private writer: Writer) {} + writeU8(value: number) { + this.helper[0] = value; + this.writer.write(this.helper.subarray(0, 1)); + } + + writeU16(value: number) { + this.helperView.setUint16(0, value, false); + this.writer.write(this.helper.subarray(0, 2)); + } + writeU32(value: number) { this.helperView.setUint32(0, value, false); this.writer.write(this.helper.subarray(0, 4)); } + writeAscii(text: string) { + for (let i = 0; i < text.length; i++) { + this.helper[i] = text.charCodeAt(i); + } + this.writer.write(this.helper.subarray(0, text.length)); + } + + writeSynchsafeU32(value: number) { + this.writeU32(encodeSynchsafe(value)); + } + + writeIsoString(text: string) { + const bytes = new Uint8Array(text.length + 1); + for (let i = 0; i < text.length; i++) { + bytes[i] = text.charCodeAt(i); + } + bytes[text.length] = 0x00; + this.writer.write(bytes); + } + + writeUtf8String(text: string) { + const utf8Data = textEncoder.encode(text); + this.writer.write(utf8Data); + this.writeU8(0x00); + } + + writeId3V2TextFrame(frameId: string, text: string) { + const useIso88591 = isIso88591Compatible(text); + const textDataLength = useIso88591 ? text.length : textEncoder.encode(text).byteLength; + const frameSize = 1 + textDataLength + 1; + + this.writeAscii(frameId); + this.writeSynchsafeU32(frameSize); + this.writeU16(0x0000); + + this.writeU8(useIso88591 ? Id3V2TextEncoding.ISO_8859_1 : Id3V2TextEncoding.UTF_8); + if (useIso88591) { + this.writeIsoString(text); + } else { + this.writeUtf8String(text); + } + } + + writeId3V2LyricsFrame(lyrics: string) { + const useIso88591 = isIso88591Compatible(lyrics); + const shortDescription = ''; + const frameSize = 1 + 3 + shortDescription.length + 1 + lyrics.length + 1; + + this.writeAscii('USLT'); + this.writeSynchsafeU32(frameSize); + this.writeU16(0x0000); + + this.writeU8(useIso88591 ? Id3V2TextEncoding.ISO_8859_1 : Id3V2TextEncoding.UTF_8); + this.writeAscii('und'); + + if (useIso88591) { + this.writeIsoString(shortDescription); + this.writeIsoString(lyrics); + } else { + this.writeUtf8String(shortDescription); + this.writeUtf8String(lyrics); + } + } + + writeId3V2CommentFrame(comment: string) { + const useIso88591 = isIso88591Compatible(comment); + const textDataLength = useIso88591 ? comment.length : textEncoder.encode(comment).byteLength; + const shortDescription = ''; + const frameSize = 1 + 3 + shortDescription.length + 1 + textDataLength + 1; + + this.writeAscii('COMM'); + this.writeSynchsafeU32(frameSize); + this.writeU16(0x0000); + + this.writeU8(useIso88591 ? Id3V2TextEncoding.ISO_8859_1 : Id3V2TextEncoding.UTF_8); + this.writeU8(0x75); // 'u' + this.writeU8(0x6E); // 'n' + this.writeU8(0x64); // 'd' + + if (useIso88591) { + this.writeIsoString(shortDescription); + this.writeIsoString(comment); + } else { + this.writeUtf8String(shortDescription); + this.writeUtf8String(comment); + } + } + + writeId3V2ApicFrame(mimeType: string, pictureType: number, description: string, imageData: Uint8Array) { + const useIso88591 = isIso88591Compatible(mimeType) && isIso88591Compatible(description); + const descriptionDataLength = useIso88591 ? description.length : textEncoder.encode(description).byteLength; + const frameSize = 1 + mimeType.length + 1 + 1 + descriptionDataLength + 1 + imageData.byteLength; + + this.writeAscii('APIC'); + this.writeSynchsafeU32(frameSize); + this.writeU16(0x0000); + + this.writeU8(useIso88591 ? Id3V2TextEncoding.ISO_8859_1 : Id3V2TextEncoding.UTF_8); + + if (useIso88591) { + this.writeIsoString(mimeType); + } else { + this.writeUtf8String(mimeType); + } + + this.writeU8(pictureType); + + if (useIso88591) { + this.writeIsoString(description); + } else { + this.writeUtf8String(description); + } + + this.writer.write(imageData); + } + writeXingFrame(data: XingFrameData) { const startPos = this.writer.getPos(); diff --git a/src/ogg/ogg-demuxer.ts b/src/ogg/ogg-demuxer.ts index 2632e49..264eabf 100644 --- a/src/ogg/ogg-demuxer.ts +++ b/src/ogg/ogg-demuxer.ts @@ -6,19 +6,22 @@ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -import { OPUS_INTERNAL_SAMPLE_RATE } from '../codec'; +import { OPUS_SAMPLE_RATE } from '../codec'; import { parseModesFromVorbisSetupPacket, parseOpusIdentificationHeader } from '../codec-data'; import { Demuxer } from '../demuxer'; import { Input } from '../input'; import { InputAudioTrack, InputAudioTrackBacking } from '../input-track'; import { PacketRetrievalOptions } from '../media-sink'; +import { MetadataTags } from '../tags'; import { assert, AsyncMutex, + base64ToBytes, binarySearchLessOrEqual, findLast, last, roundToPrecision, + textDecoder, toDataView, UNDETERMINED_LANGUAGE, } from '../misc'; @@ -57,6 +60,7 @@ export class OggDemuxer extends Demuxer { metadataPromise: Promise | null = null; bitstreams: LogicalBitstream[] = []; tracks: InputAudioTrack[] = []; + metadataTags: MetadataTags = {}; constructor(input: Input) { super(input); @@ -218,6 +222,8 @@ export class OggDemuxer extends Demuxer { ], modeBlockflags: parseModesFromVorbisSetupPacket(thirdPacket.data).modeBlockflags, }; + + this.readVorbisComments(secondPacket.data.subarray(7)); // Skip header type and 'vorbis' } async readOpusMetadata(firstPacket: Packet, bitstream: LogicalBitstream) { @@ -237,19 +243,175 @@ export class OggDemuxer extends Demuxer { return; } - // We don't make use of the comment header's data - bitstream.codecInfo.codec = 'opus'; bitstream.description = firstPacket.data; bitstream.lastMetadataPacket = secondPacket; const header = parseOpusIdentificationHeader(firstPacket.data); bitstream.numberOfChannels = header.outputChannelCount; - bitstream.sampleRate = header.inputSampleRate; + bitstream.sampleRate = OPUS_SAMPLE_RATE; // Always the same bitstream.codecInfo.opusInfo = { preSkip: header.preSkip, }; + + this.readVorbisComments(secondPacket.data.subarray(8)); // Skip 'OpusTags' + } + + readVorbisComments(bytes: Uint8Array) { + // https://datatracker.ietf.org/doc/html/rfc7845#section-5.2 + + const commentView = toDataView(bytes); + let commentPos = 0; + + const vendorStringLength = commentView.getUint32(commentPos, true); + commentPos += 4; + + const vendorString = textDecoder.decode( + bytes.subarray(commentPos, commentPos + vendorStringLength), + ); + commentPos += vendorStringLength; + + if (vendorStringLength > 0) { + // Expose the vendor string in the raw metadata + this.metadataTags.raw ??= {}; + this.metadataTags.raw['vendor'] ??= vendorString; + } + + const listLength = commentView.getUint32(commentPos, true); + commentPos += 4; + + // Loop over all metadata tags + for (let i = 0; i < listLength; i++) { + const stringLength = commentView.getUint32(commentPos, true); + commentPos += 4; + + const string = textDecoder.decode( + bytes.subarray(commentPos, commentPos + stringLength), + ); + commentPos += stringLength; + + const separatorIndex = string.indexOf('='); + if (separatorIndex === -1) { + continue; + } + + const key = string.slice(0, separatorIndex).toUpperCase(); + const value = string.slice(separatorIndex + 1); + + this.metadataTags.raw ??= {}; + this.metadataTags.raw[key] ??= value; + + switch (key) { + case 'TITLE': { + this.metadataTags.title ??= value; + }; break; + + case 'DESCRIPTION': { + this.metadataTags.description ??= value; + }; break; + + case 'ARTIST': { + this.metadataTags.artist ??= value; + }; break; + + case 'ALBUM': { + this.metadataTags.album ??= value; + }; break; + + case 'ALBUMARTIST': { + this.metadataTags.albumArtist ??= value; + }; break; + + case 'COMMENT': { + this.metadataTags.comment ??= value; + }; break; + + case 'LYRICS': { + this.metadataTags.lyrics ??= value; + }; break; + + case 'TRACKNUMBER': { + const parts = value.split('/'); + const trackNum = Number.parseInt(parts[0]!, 10); + const tracksTotal = parts[1] && Number.parseInt(parts[1], 10); + + if (Number.isInteger(trackNum) && trackNum > 0) { + this.metadataTags.trackNumber ??= trackNum; + } + if (tracksTotal && Number.isInteger(tracksTotal) && tracksTotal > 0) { + this.metadataTags.tracksTotal ??= tracksTotal; + } + }; break; + + case 'TRACKTOTAL': { + const tracksTotal = Number.parseInt(value, 10); + if (Number.isInteger(tracksTotal) && tracksTotal > 0) { + this.metadataTags.tracksTotal ??= tracksTotal; + } + }; break; + + case 'DISCNUMBER': { + const parts = value.split('/'); + const discNum = Number.parseInt(parts[0]!, 10); + const discsTotal = parts[1] && Number.parseInt(parts[1], 10); + + if (Number.isInteger(discNum) && discNum > 0) { + this.metadataTags.discNumber ??= discNum; + } + if (discsTotal && Number.isInteger(discsTotal) && discsTotal > 0) { + this.metadataTags.discsTotal ??= discsTotal; + } + }; break; + + case 'DISCTOTAL': { + const discsTotal = Number.parseInt(value, 10); + if (Number.isInteger(discsTotal) && discsTotal > 0) { + this.metadataTags.discsTotal ??= discsTotal; + } + }; break; + + case 'DATE': { + const date = new Date(value); + if (!Number.isNaN(date.getTime())) { + this.metadataTags.date ??= date; + } + }; break; + + case 'GENRE': { + this.metadataTags.genre ??= value; + }; break; + + case 'METADATA_BLOCK_PICTURE': { + // https://datatracker.ietf.org/doc/rfc9639/ Section 8.8 + const decoded = base64ToBytes(value); + + const view = toDataView(decoded); + const pictureType = view.getUint32(0, false); + const mediaTypeLength = view.getUint32(4, false); + const mediaType = String.fromCharCode(...decoded.subarray(8, 8 + mediaTypeLength)); // ASCII + const descriptionLength = view.getUint32(8 + mediaTypeLength, false); + const description = textDecoder.decode(decoded.subarray( + 12 + mediaTypeLength, + 12 + mediaTypeLength + descriptionLength, + )); + const dataLength = view.getUint32(mediaTypeLength + descriptionLength + 28); + const data = decoded.subarray( + mediaTypeLength + descriptionLength + 32, + mediaTypeLength + descriptionLength + 32 + dataLength, + ); + + this.metadataTags.images ??= []; + this.metadataTags.images.push({ + data, + mimeType: mediaType, + kind: pictureType === 3 ? 'coverFront' : pictureType === 4 ? 'coverBack' : 'unknown', + name: undefined, + description: description || undefined, + }); + }; break; + } + } } async readPacket(startPage: Page, startSegmentIndex: number): Promise { @@ -388,6 +550,11 @@ export class OggDemuxer extends Demuxer { const trackDurations = await Promise.all(tracks.map(x => x.computeDuration())); return Math.max(0, ...trackDurations); } + + async getMetadataTags() { + await this.readMetadata(); + return this.metadataTags; + } } type EncodedPacketMetadata = { @@ -407,7 +574,7 @@ class OggAudioTrackBacking implements InputAudioTrackBacking { constructor(public bitstream: LogicalBitstream, public demuxer: OggDemuxer) { // Opus always uses a fixed sample rate for its internal calculations, even if the actual rate is different this.internalSampleRate = bitstream.codecInfo.codec === 'opus' - ? OPUS_INTERNAL_SAMPLE_RATE + ? OPUS_SAMPLE_RATE : bitstream.sampleRate; } diff --git a/src/ogg/ogg-muxer.ts b/src/ogg/ogg-muxer.ts index 1b18333..8b7de06 100644 --- a/src/ogg/ogg-muxer.ts +++ b/src/ogg/ogg-muxer.ts @@ -6,11 +6,22 @@ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -import { OPUS_INTERNAL_SAMPLE_RATE, validateAudioChunkMetadata } from '../codec'; +import { OPUS_SAMPLE_RATE, validateAudioChunkMetadata } from '../codec'; import { parseModesFromVorbisSetupPacket, parseOpusIdentificationHeader } from '../codec-data'; -import { assert, promiseWithResolvers, setInt64, toDataView, toUint8Array } from '../misc'; +import { + assert, + assertNever, + bytesToBase64, + keyValueIterator, + promiseWithResolvers, + setInt64, + textEncoder, + toDataView, + toUint8Array, +} from '../misc'; import { Muxer } from '../muxer'; -import { Output, OutputAudioTrack } from '../output'; +import { Output, + OutputAudioTrack } from '../output'; import { OggOutputFormat } from '../output-format'; import { EncodedPacket } from '../packet'; import { Writer } from '../writer'; @@ -108,7 +119,7 @@ export class OggMuxer extends Muxer { track, serialNumber, internalSampleRate: track.source._codec === 'opus' - ? OPUS_INTERNAL_SAMPLE_RATE + ? OPUS_SAMPLE_RATE : meta.decoderConfig.sampleRate, codecInfo: { codec: track.source._codec, @@ -176,9 +187,20 @@ export class OggMuxer extends Muxer { } const identificationHeader = bytes.subarray(pos, pos += identificationHeaderLength); - const commentHeader = bytes.subarray(pos, pos += commentHeaderLength); + pos += commentHeaderLength; // Skip the comment header, we'll build our own const setupHeader = bytes.subarray(pos); + const commentHeaderHeader = new Uint8Array(7); + commentHeaderHeader[0] = 3; // Packet type + commentHeaderHeader[1] = 0x76; // 'v' + commentHeaderHeader[2] = 0x6f; // 'o' + commentHeaderHeader[3] = 0x72; // 'r' + commentHeaderHeader[4] = 0x62; // 'b' + commentHeaderHeader[5] = 0x69; // 'i' + commentHeaderHeader[6] = 0x73; // 's' + + const commentHeader = this.createVorbisComments(commentHeaderHeader); + trackData.packetQueue.push({ data: identificationHeader, endGranulePosition: 0, @@ -213,12 +235,11 @@ export class OggMuxer extends Muxer { const identificationHeader = toUint8Array(meta.decoderConfig.description); - const commentHeader = new Uint8Array(8 + 4 + 4); - const view = new DataView(commentHeader.buffer); - view.setUint32(0, 0x4f707573, false); // 'Opus' - view.setUint32(4, 0x54616773, false); // 'Tags' - view.setUint32(8, 0, true); // Vendor String Length - view.setUint32(12, 0, true); // User Comment List Length + const commentHeaderHeader = new Uint8Array(8); + const commentHeaderHeaderView = toDataView(commentHeaderHeader); + commentHeaderHeaderView.setUint32(0, 0x4f707573, false); // 'Opus' + commentHeaderHeaderView.setUint32(4, 0x54616773, false); // 'Tags' + const commentHeader = this.createVorbisComments(commentHeaderHeader); trackData.packetQueue.push({ data: identificationHeader, @@ -238,6 +259,177 @@ export class OggMuxer extends Muxer { } } + createVorbisComments(headerBytes: Uint8Array) { + // https://datatracker.ietf.org/doc/html/rfc7845#section-5.2 + + const tags = this.output._metadataTags; + const commentHeaderParts = [ + headerBytes, + ]; + + let vendorString = ''; + if (typeof tags.raw?.['vendor'] === 'string') { + vendorString = tags.raw?.['vendor']; + } + const encodedVendorString = textEncoder.encode(vendorString); + + let currentBuffer = new Uint8Array(4 + encodedVendorString.length); + let currentView = new DataView(currentBuffer.buffer); + currentView.setUint32(0, encodedVendorString.length, true); + currentBuffer.set(encodedVendorString, 4); + + commentHeaderParts.push(currentBuffer); + + const writtenTags = new Set(); + const addCommentTag = (key: string, value: string) => { + const joined = `${key}=${value}`; + const encoded = textEncoder.encode(joined); + + currentBuffer = new Uint8Array(4 + encoded.length); + currentView = new DataView(currentBuffer.buffer); + + currentView.setUint32(0, encoded.length, true); + currentBuffer.set(encoded, 4); + + commentHeaderParts.push(currentBuffer); + writtenTags.add(key); + }; + + for (const { key, value } of keyValueIterator(tags)) { + switch (key) { + case 'title': { + addCommentTag('TITLE', value); + }; break; + + case 'description': { + addCommentTag('DESCRIPTION', value); + }; break; + + case 'artist': { + addCommentTag('ARTIST', value); + }; break; + + case 'album': { + addCommentTag('ALBUM', value); + }; break; + + case 'albumArtist': { + addCommentTag('ALBUMARTIST', value); + }; break; + + case 'genre': { + addCommentTag('GENRE', value); + }; break; + + case 'date': { + addCommentTag('DATE', value.toISOString().slice(0, 10)); + }; break; + + case 'comment': { + addCommentTag('COMMENT', value); + }; break; + + case 'lyrics': { + addCommentTag('LYRICS', value); + }; break; + + case 'trackNumber': { + addCommentTag('TRACKNUMBER', value.toString()); + }; break; + + case 'tracksTotal': { + addCommentTag('TRACKTOTAL', value.toString()); + }; break; + + case 'discNumber': { + addCommentTag('DISCNUMBER', value.toString()); + }; break; + + case 'discsTotal': { + addCommentTag('DISCTOTAL', value.toString()); + }; break; + + case 'images': { + for (const image of value) { + // https://datatracker.ietf.org/doc/rfc9639/ Section 8.8 + const pictureType = image.kind === 'coverFront' ? 3 : image.kind === 'coverBack' ? 4 : 0; + const encodedMediaType = new Uint8Array(image.mimeType.length); + + for (let i = 0; i < image.mimeType.length; i++) { + encodedMediaType[i] = image.mimeType.charCodeAt(i); + } + + const encodedDescription = textEncoder.encode(image.description ?? ''); + + const buffer = new Uint8Array( + 4 // Picture type + + 4 // MIME type length + + encodedMediaType.length // MIME type + + 4 // Description length + + encodedDescription.length // Description + + 16 // Width, height, color depth, number of colors + + 4 // Picture data length + + image.data.length, // Picture data + ); + const view = toDataView(buffer); + + view.setUint32(0, pictureType, false); + view.setUint32(4, encodedMediaType.length, false); + buffer.set(encodedMediaType, 8); + view.setUint32(8 + encodedMediaType.length, encodedDescription.length, false); + buffer.set(encodedDescription, 12 + encodedMediaType.length); + // Skip a bunch of fields (width, height, color depth, number of colors) + view.setUint32( + 28 + encodedMediaType.length + encodedDescription.length, image.data.length, false, + ); + buffer.set( + image.data, + 32 + encodedMediaType.length + encodedDescription.length, + ); + + const encoded = bytesToBase64(buffer); + addCommentTag('METADATA_BLOCK_PICTURE', encoded); + } + }; break; + + case 'raw': { + // Handled later + }; break; + + default: assertNever(key); + } + } + + if (tags.raw) { + for (const key in tags.raw) { + const value = tags.raw[key]; + if (key === 'vendor' || value == null || writtenTags.has(key)) { + continue; + } + + if (typeof value === 'string') { + addCommentTag(key, value); + } + } + } + + const listLengthBuffer = new Uint8Array(4); + toDataView(listLengthBuffer).setUint32(0, writtenTags.size, true); + commentHeaderParts.splice(2, 0, listLengthBuffer); // Insert after the header and vendor section + + // Merge all comment header parts into a single buffer + const commentHeaderLength = commentHeaderParts.reduce((a, b) => a + b.length, 0); + const commentHeader = new Uint8Array(commentHeaderLength); + + let pos = 0; + for (const part of commentHeaderParts) { + commentHeader.set(part, pos); + pos += part.length; + } + + return commentHeader; + } + async addEncodedAudioPacket(track: OutputAudioTrack, packet: EncodedPacket, meta?: EncodedAudioChunkMetadata) { const release = await this.mutex.acquire(); diff --git a/src/output.ts b/src/output.ts index 24e71d7..5ce5e48 100644 --- a/src/output.ts +++ b/src/output.ts @@ -7,6 +7,7 @@ */ import { AsyncMutex, isIso639Dash2LanguageCode, Rotation } from './misc'; +import { MetadataTags, validateMetadataTags } from './tags'; import { Muxer } from './muxer'; import { OutputFormat } from './output-format'; import { AudioSource, MediaSource, SubtitleSource, VideoSource } from './media-source'; @@ -145,6 +146,8 @@ export class Output< _finalizePromise: Promise | null = null; /** @internal */ _mutex = new AsyncMutex(); + /** @internal */ + _metadataTags: MetadataTags = {}; /** * Creates a new instance of {@link Output} which can then be used to create a new media file according to the @@ -217,6 +220,22 @@ export class Output< this._addTrack('subtitle', source, metadata); } + /** + * Sets descriptive metadata tags about the media file, such as title, author, date, or cover art. When called + * multiple times, only the metadata from the last call will be used. + * + * Must be called before output is started. + */ + setMetadataTags(tags: MetadataTags) { + validateMetadataTags(tags); + + if (this.state !== 'pending') { + throw new Error('Cannot set metadata tags after output has been started or canceled.'); + } + + this._metadataTags = tags; + } + /** @internal */ private _addTrack(type: OutputTrack['type'], source: MediaSource, metadata: object) { if (this.state !== 'pending') { diff --git a/src/source.ts b/src/source.ts index ae24aaf..54aedae 100644 --- a/src/source.ts +++ b/src/source.ts @@ -36,6 +36,8 @@ export abstract class Source { abstract _retrieveSize(): MaybePromise; /** @internal */ abstract _read(start: number, end: number): MaybePromise; + /** @internal */ + abstract get _supportsRandomAccess(): boolean; /** @internal */ private _sizePromise: Promise | null = null; @@ -113,6 +115,11 @@ export class BufferSource extends Source { offset: 0, }; } + + /** @internal */ + get _supportsRandomAccess() { + return true; + } } /** @@ -210,6 +217,11 @@ export class BlobSource extends Source { worker.running = false; } + + /** @internal */ + get _supportsRandomAccess() { + return true; + } } const URL_SOURCE_MIN_LOAD_AMOUNT = 0.5 * 2 ** 20; // 0.5 MiB @@ -488,6 +500,11 @@ export class UrlSource extends Source { } } } + + /** @internal */ + get _supportsRandomAccess() { + return true; + } } /** @@ -559,6 +576,11 @@ export class FilePathSource extends Source { _retrieveSize(): MaybePromise { return this._streamSource._retrieveSize(); } + + /** @internal */ + get _supportsRandomAccess() { + return true; + } } /** @@ -728,6 +750,11 @@ export class StreamSource extends Source { worker.running = false; } + + /** @internal */ + get _supportsRandomAccess() { + return true; + } } type ReadableStreamSourcePendingSlice = { @@ -987,6 +1014,11 @@ export class ReadableStreamSource extends Source { this._pulling = false; } + + /** @internal */ + get _supportsRandomAccess() { + return false; + } } type PrefetchProfile = (start: number, end: number, workers: ReadWorker[]) => { diff --git a/src/tags.ts b/src/tags.ts new file mode 100644 index 0000000..825d18a --- /dev/null +++ b/src/tags.ts @@ -0,0 +1,215 @@ +/*! + * Copyright (c) 2025-present, Vanilagy and contributors + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +/** + * Represents descriptive (non-technical) metadata about a media file, such as title, author, date, or cover art. + * Common tags are normalized by Mediabunny into a uniform format, while the `raw` field can be used to directly read or + * write the underlying metadata tags (which differ by format). + * + * - For MP4/QuickTime files, the metadata refers to the data in `'moov'`-level `'udta'` and `'meta'` atoms. + * - For Matroska files, the metadata refers to the Tags and Attachments elements whose target is 50 (MOVIE). + * - For MP3 files, the metadata refers to the ID3v2 or ID3v1 tags. + * - For Ogg files, there is no global metadata so instead, the metadata refers to the combined metadata of all tracks, + * in Vorbis-style comment headers. + * - For WAVE files, the metadata refers to the chunks within the RIFF INFO chunk. + * - For ADTS files, there is no metadata. + * + * @group Metadata tags + * @public + */ +export type MetadataTags = { + /** Title of the media (e.g. Gangnam Style, Titanic, etc.) */ + title?: string; + /** Short description or subtitle of the media. */ + description?: string; + /** Primary artist(s) or creator(s) of the work. */ + artist?: string; + /** Album, collection, or compilation the media belongs to. */ + album?: string; + /** Main credited artist for the album/collection as a whole. */ + albumArtist?: string; + /** Position of this track within its album or collection (1-based). */ + trackNumber?: number; + /** Total number of tracks in the album or collection. */ + tracksTotal?: number; + /** Disc index if the release spans multiple discs (1-based). */ + discNumber?: number; + /** Total number of discs in the release. */ + discsTotal?: number; + /** Genre or category describing the media's style or content (e.g. Metal, Horror, etc.) */ + genre?: string; + /** Release, recording or creation date of the media. */ + date?: Date; + /** Full text lyrics or transcript associated with the media. */ + lyrics?: string; + /** Freeform notes, remarks or commentary about the media. */ + comment?: string; + /** Embedded images such as cover art, booklet scans, artwork or preview frames. */ + images?: AttachedImage[]; + /** + * The raw, underlying metadata tags. + * + * This field can be used for both reading and writing. When reading, it represents the original tags that were used + * to derive the normalized fields, and any additional metadata that Mediabunny doesn't understand. When writing, it + * can be used to set arbitrary metadata tags in the output file. + * + * The format of these tags differs per format: + * - MP4/QuickTime: By default, the keys refer to the names of the individual atoms in the `'ilst'` atom inside the + * `'meta'` atom, and the values are derived from the content of the `'data'` atom inside them. When a `'keys'` atom + * is also used, then the keys reflect the keys specified there (such as `'com.apple.quicktime.version'`). + * Additionally, any atoms within the `'udta'` atom are dumped into here, however with unknown internal format + * (`Uint8Array`). + * - Matroska: `SimpleTag` elements whose target is 50 (MOVIE), either containing string or `Uint8Array` values. + * - MP3: The ID3v2 tags, or a single `'TAG'` key with the contents of the ID3v1 tag. + * - Ogg: The key-value string pairs from the Vorbis-style comment header (see RFC 7845, Section 5.2). + * Additionally, the `'vendor'` key refers to the vendor string within this header. + * - WAVE: The individual metadata chunks within the RIFF INFO chunk. Values are always ISO 8859-1 strings. + */ + raw?: Record; +}; + +/** + * An embedded image such as cover art, booklet scan, artwork or preview frame. + * + * @group Metadata tags + * @public + */ +export type AttachedImage = { + /** The raw image data. */ + data: Uint8Array; + /** An RFC 6838 MIME type (e.g. image/jpeg, image/png, etc.) */ + mimeType: string; + /** The kind or purpose of the image. */ + kind: 'coverFront' | 'coverBack' | 'unknown'; + /** The name of the image file. */ + name?: string; + /** A short description of the image. */ + description?: string; +}; + +/** + * Image data with additional metadata. + * + * @group Metadata tags + * @public + */ +export class RichImageData { + /** Creates a new {@link RichImageData}. */ + constructor( + /** The raw image data. */ + public data: Uint8Array, + /** An RFC 6838 MIME type (e.g. image/jpeg, image/png, etc.) */ + public mimeType: string, + ) {} +} + +export const validateMetadataTags = (tags: MetadataTags) => { + if (!tags || typeof tags !== 'object') { + throw new TypeError('tags must be an object.'); + } + if (tags.title !== undefined && typeof tags.title !== 'string') { + throw new TypeError('tags.title, when provided, must be a string.'); + } + if (tags.description !== undefined && typeof tags.description !== 'string') { + throw new TypeError('tags.description, when provided, must be a string.'); + } + if (tags.artist !== undefined && typeof tags.artist !== 'string') { + throw new TypeError('tags.artist, when provided, must be a string.'); + } + if (tags.album !== undefined && typeof tags.album !== 'string') { + throw new TypeError('tags.album, when provided, must be a string.'); + } + if (tags.albumArtist !== undefined && typeof tags.albumArtist !== 'string') { + throw new TypeError('tags.albumArtist, when provided, must be a string.'); + } + if (tags.trackNumber !== undefined && (!Number.isInteger(tags.trackNumber) || tags.trackNumber <= 0)) { + throw new TypeError('tags.trackNumber, when provided, must be a positive integer.'); + } + if ( + tags.tracksTotal !== undefined + && (!Number.isInteger(tags.tracksTotal) || tags.tracksTotal <= 0) + ) { + throw new TypeError('tags.tracksTotal, when provided, must be a positive integer.'); + } + if (tags.discNumber !== undefined && (!Number.isInteger(tags.discNumber) || tags.discNumber <= 0)) { + throw new TypeError('tags.discNumber, when provided, must be a positive integer.'); + } + if ( + tags.discsTotal !== undefined + && (!Number.isInteger(tags.discsTotal) || tags.discsTotal <= 0) + ) { + throw new TypeError('tags.discsTotal, when provided, must be a positive integer.'); + } + if (tags.genre !== undefined && typeof tags.genre !== 'string') { + throw new TypeError('tags.genre, when provided, must be a string.'); + } + if (tags.date !== undefined && (!(tags.date instanceof Date) || Number.isNaN(tags.date.getTime()))) { + throw new TypeError('tags.date, when provided, must be a valid Date.'); + } + if (tags.lyrics !== undefined && typeof tags.lyrics !== 'string') { + throw new TypeError('tags.lyrics, when provided, must be a string.'); + } + if (tags.images !== undefined) { + if (!Array.isArray(tags.images)) { + throw new TypeError('tags.images, when provided, must be an array.'); + } + for (const image of tags.images) { + if (!image || typeof image !== 'object') { + throw new TypeError('Each image in tags.images must be an object.'); + } + if (!(image.data instanceof Uint8Array)) { + throw new TypeError('Each image.data must be a Uint8Array.'); + } + if (typeof image.mimeType !== 'string') { + throw new TypeError('Each image.mimeType must be a string.'); + } + if (!['coverFront', 'coverBack', 'unknown'].includes(image.kind)) { + throw new TypeError('Each image.kind must be \'coverFront\', \'coverBack\', or \'unknown\'.'); + } + } + } + if (tags.comment !== undefined && typeof tags.comment !== 'string') { + throw new TypeError('tags.comment, when provided, must be a string.'); + } + if (tags.raw !== undefined) { + if (!tags.raw || typeof tags.raw !== 'object') { + throw new TypeError('tags.raw, when provided, must be an object.'); + } + + for (const value of Object.values(tags.raw)) { + if ( + value !== null + && typeof value !== 'string' + && !(value instanceof Uint8Array) + && !(value instanceof RichImageData) + ) { + throw new TypeError( + 'Each value in tags.raw must be a string, Uint8Array, RichImageData, or null.', + ); + } + } + } +}; + +export const metadataTagsAreEmpty = (tags: MetadataTags) => { + return tags.title === undefined + && tags.description === undefined + && tags.artist === undefined + && tags.album === undefined + && tags.albumArtist === undefined + && tags.trackNumber === undefined + && tags.tracksTotal === undefined + && tags.discNumber === undefined + && tags.discsTotal === undefined + && tags.genre === undefined + && tags.date === undefined + && tags.lyrics === undefined + && (!tags.images || tags.images.length === 0) + && tags.comment === undefined + && (tags.raw === undefined || Object.keys(tags.raw).length === 0); +}; diff --git a/src/wave/wave-demuxer.ts b/src/wave/wave-demuxer.ts index 43795f5..974cf25 100644 --- a/src/wave/wave-demuxer.ts +++ b/src/wave/wave-demuxer.ts @@ -11,6 +11,7 @@ import { Demuxer } from '../demuxer'; import { Input } from '../input'; import { InputAudioTrack, InputAudioTrackBacking } from '../input-track'; import { PacketRetrievalOptions } from '../media-sink'; +import { MetadataTags } from '../tags'; import { assert, UNDETERMINED_LANGUAGE } from '../misc'; import { EncodedPacket, PLACEHOLDER_DATA } from '../packet'; import { readAscii, readBytes, Reader, readU16, readU32, readU64 } from '../reader'; @@ -39,6 +40,7 @@ export class WaveDemuxer extends Demuxer { tracks: InputAudioTrack[] = []; lastKnownPacketIndex = 0; + metadataTags: MetadataTags = {}; constructor(input: Input) { super(input); @@ -92,6 +94,10 @@ export class WaveDemuxer extends Demuxer { this.dataStart = slice.filePos; this.dataSize = Math.min(dataChunkSize, (totalFileSize ?? Infinity) - this.dataStart); + + if (this.reader.fileSize === null) { + break; // Stop once we hit the data chunk + } } else if (chunkId === 'ds64') { // File and data chunk sizes are defined in here instead @@ -99,6 +105,8 @@ export class WaveDemuxer extends Demuxer { dataChunkSize = readU64(slice, littleEndian); totalFileSize = Math.min(riffChunkSize + 8, this.reader.fileSize ?? Infinity); + } else if (chunkId === 'LIST') { + await this.parseListChunk(startPos, chunkSize, littleEndian); } currentPos = startPos + chunkSize + (chunkSize & 1); // Handle padding @@ -167,6 +175,102 @@ export class WaveDemuxer extends Demuxer { }; } + private async parseListChunk(startPos: number, size: number, littleEndian: boolean) { + let slice = this.reader.requestSlice(startPos, size); + if (slice instanceof Promise) slice = await slice; + if (!slice) return; // File too short + + const infoType = readAscii(slice, 4); + if (infoType !== 'INFO' && infoType !== 'INF0') { // exiftool.org claims INF0 can happen + return; // Not an INFO chunk + } + + let currentPos = slice.filePos; + while (currentPos <= startPos + size - 8) { + slice.filePos = currentPos; + + const chunkName = readAscii(slice, 4); + const chunkSize = readU32(slice, littleEndian); + const bytes = readBytes(slice, chunkSize); + + let stringLength = 0; + for (let i = 0; i < bytes.length; i++) { + if (bytes[i] === 0) { + break; + } + + stringLength++; + } + + const value = String.fromCharCode(...bytes.subarray(0, stringLength)); + + this.metadataTags.raw ??= {}; + this.metadataTags.raw[chunkName] = value; + + switch (chunkName) { + case 'INAM': + case 'TITL': { + this.metadataTags.title ??= value; + }; break; + + case 'TIT3': { + this.metadataTags.description ??= value; + }; break; + + case 'IART': { + this.metadataTags.artist ??= value; + }; break; + + case 'IPRD': { + this.metadataTags.album ??= value; + }; break; + + case 'IPRT': + case 'ITRK': + case 'TRCK': { + const parts = value.split('/'); + const trackNum = Number.parseInt(parts[0]!, 10); + const tracksTotal = parts[1] && Number.parseInt(parts[1], 10); + + if (Number.isInteger(trackNum) && trackNum > 0) { + this.metadataTags.trackNumber ??= trackNum; + } + if (tracksTotal && Number.isInteger(tracksTotal) && tracksTotal > 0) { + this.metadataTags.tracksTotal ??= tracksTotal; + } + }; break; + + case 'ICRD': + case 'IDIT': { + const date = new Date(value); + if (!Number.isNaN(date.getTime())) { + this.metadataTags.date ??= date; + } + }; break; + + case 'YEAR': { + const year = Number.parseInt(value, 10); + if (Number.isInteger(year) && year > 0) { + this.metadataTags.date ??= new Date(year, 0, 1); + } + }; break; + + case 'IGNR': + case 'GENR': { + this.metadataTags.genre ??= value; + }; break; + + case 'ICMT': + case 'CMNT': + case 'COMM': { + this.metadataTags.comment ??= value; + }; break; + } + + currentPos += 8 + chunkSize + (chunkSize & 1); // Handle padding + } + } + getCodec(): AudioCodec | null { assert(this.audioInfo); @@ -214,6 +318,11 @@ export class WaveDemuxer extends Demuxer { await this.readMetadata(); return this.tracks; } + + async getMetadataTags() { + await this.readMetadata(); + return this.metadataTags; + } } const PACKET_SIZE_IN_FRAMES = 2048; diff --git a/src/wave/wave-muxer.ts b/src/wave/wave-muxer.ts index 878939f..e1259f6 100644 --- a/src/wave/wave-muxer.ts +++ b/src/wave/wave-muxer.ts @@ -14,7 +14,8 @@ import { RiffWriter } from './riff-writer'; import { Writer } from '../writer'; import { EncodedPacket } from '../packet'; import { WavOutputFormat } from '../output-format'; -import { assert } from '../misc'; +import { assert, assertNever, isIso88591Compatible, keyValueIterator } from '../misc'; +import { MetadataTags, metadataTagsAreEmpty } from '../tags'; export class WaveMuxer extends Muxer { private format: WavOutputFormat; @@ -26,6 +27,12 @@ export class WaveMuxer extends Muxer { private sampleRate: number | null = null; private sampleCount = 0; + private riffSizePos: number | null = null; + private dataSizePos: number | null = null; + private ds64RiffSizePos: number | null = null; + private ds64DataSizePos: number | null = null; + private ds64SampleCountPos: number | null = null; + constructor(output: Output, format: WavOutputFormat) { super(output); @@ -119,6 +126,7 @@ export class WaveMuxer extends Muxer { if (this.isRf64) { this.riffWriter.writeU32(0xffffffff); // Not used in RF64 } else { + this.riffSizePos = this.writer.getPos(); this.riffWriter.writeU32(0); // File size placeholder } @@ -127,9 +135,16 @@ export class WaveMuxer extends Muxer { if (this.isRf64) { this.riffWriter.writeAscii('ds64'); this.riffWriter.writeU32(28); // Chunk size + + this.ds64RiffSizePos = this.writer.getPos(); this.riffWriter.writeU64(0); // RIFF size placeholder + + this.ds64DataSizePos = this.writer.getPos(); this.riffWriter.writeU64(0); // Data size placeholder + + this.ds64SampleCountPos = this.writer.getPos(); this.riffWriter.writeU64(0); // Sample count placeholder + this.riffWriter.writeU32(0); // Table length // Empty table } @@ -144,12 +159,18 @@ export class WaveMuxer extends Muxer { this.riffWriter.writeU16(blockSize); this.riffWriter.writeU16(8 * pcmInfo.sampleSize); + if (!metadataTagsAreEmpty(this.output._metadataTags)) { + // Metadata exists, let's write an INFO chunk + this.writeInfoChunk(this.output._metadataTags); + } + // data chunk this.riffWriter.writeAscii('data'); if (this.isRf64) { this.riffWriter.writeU32(0xffffffff); // Not used in RF64 } else { + this.dataSizePos = this.writer.getPos(); this.riffWriter.writeU32(0); // Data size placeholder } @@ -159,6 +180,126 @@ export class WaveMuxer extends Muxer { } } + private writeInfoChunk(metadata: MetadataTags) { + const startPos = this.writer.getPos(); + + this.riffWriter.writeAscii('LIST'); + this.riffWriter.writeU32(0); // Size placeholder + this.riffWriter.writeAscii('INFO'); + + const writtenTags = new Set(); + + const writeInfoTag = (tag: string, value: string) => { + if (!isIso88591Compatible(value)) { + // No Unicode supported here + console.warn(`Didn't write tag '${tag}' because '${value}' is not ISO 8859-1-compatible.`); + return; + } + + const size = value.length + 1; // +1 for null terminator + const bytes = new Uint8Array(size); + + for (let i = 0; i < value.length; i++) { + bytes[i] = value.charCodeAt(i); + } + + this.riffWriter.writeAscii(tag); + this.riffWriter.writeU32(size); + this.writer.write(bytes); + + // Add padding byte if size is odd + if (size & 1) { + this.writer.write(new Uint8Array(1)); + } + + writtenTags.add(tag); + }; + + for (const { key, value } of keyValueIterator(metadata)) { + switch (key) { + case 'title': { + writeInfoTag('INAM', value); + writtenTags.add('INAM'); + }; break; + + case 'artist': { + writeInfoTag('IART', value); + writtenTags.add('IART'); + }; break; + + case 'album': { + writeInfoTag('IPRD', value); + writtenTags.add('IPRD'); + }; break; + + case 'trackNumber': { + const string = metadata.tracksTotal !== undefined + ? `${value}/${metadata.tracksTotal}` + : value.toString(); + + writeInfoTag('ITRK', string); + writtenTags.add('ITRK'); + }; break; + + case 'genre': { + writeInfoTag('IGNR', value); + writtenTags.add('IGNR'); + }; break; + + case 'date': { + writeInfoTag('ICRD', value.toISOString().slice(0, 10)); + writtenTags.add('ICRD'); + }; break; + + case 'comment': { + writeInfoTag('ICMT', value); + writtenTags.add('ICMT'); + }; break; + + case 'albumArtist': + case 'discNumber': + case 'tracksTotal': + case 'discsTotal': + case 'description': + case 'lyrics': + case 'images': { + // Not supported in RIFF INFO + }; break; + + case 'raw': { + // Handled later + }; break; + + default: assertNever(key); + } + } + + if (metadata.raw) { + for (const key in metadata.raw) { + const value = metadata.raw[key]; + if (value == null || key.length !== 4 || writtenTags.has(key)) { + continue; + } + + if (typeof value === 'string') { + writeInfoTag(key, value); + } + } + } + + const endPos = this.writer.getPos(); + const chunkSize = endPos - startPos - 8; + + this.writer.seek(startPos + 4); + this.riffWriter.writeU32(chunkSize); + this.writer.seek(endPos); + + // Add padding byte if chunk size is odd + if (chunkSize & 1) { + this.writer.write(new Uint8Array(1)); + } + } + async finalize() { const release = await this.mutex.acquire(); @@ -166,23 +307,28 @@ export class WaveMuxer extends Muxer { if (this.isRf64) { // Write riff size - this.writer.seek(20); + assert(this.ds64RiffSizePos !== null); + this.writer.seek(this.ds64RiffSizePos); this.riffWriter.writeU64(endPos - 8); // Write data size - this.writer.seek(28); + assert(this.ds64DataSizePos !== null); + this.writer.seek(this.ds64DataSizePos); this.riffWriter.writeU64(this.dataSize); // Write sample count - this.writer.seek(36); + assert(this.ds64SampleCountPos !== null); + this.writer.seek(this.ds64SampleCountPos); this.riffWriter.writeU64(this.sampleCount); } else { // Write file size - this.writer.seek(4); + assert(this.riffSizePos !== null); + this.writer.seek(this.riffSizePos); this.riffWriter.writeU32(endPos - 8); // Write data chunk size - this.writer.seek(40); + assert(this.dataSizePos !== null); + this.writer.seek(this.dataSizePos); this.riffWriter.writeU32(this.dataSize); } diff --git a/test/node/metadata-tags.test.ts b/test/node/metadata-tags.test.ts new file mode 100644 index 0000000..dcf94a4 --- /dev/null +++ b/test/node/metadata-tags.test.ts @@ -0,0 +1,483 @@ +import { expect, test } from 'vitest'; +import { Output } from '../../src/output.js'; +import { + MkvOutputFormat, + MovOutputFormat, + Mp3OutputFormat, + Mp4OutputFormat, + OggOutputFormat, + WavOutputFormat, +} from '../../src/output-format.js'; +import { BufferTarget } from '../../src/target.js'; +import { EncodedAudioPacketSource } from '../../src/media-source.js'; +import { EncodedPacket } from '../../src/packet.js'; +import { Input } from '../../src/input.js'; +import { BufferSource, FilePathSource } from '../../src/source.js'; +import { ALL_FORMATS } from '../../src/input-format.js'; +import { MetadataTags } from '../../src/tags.js'; +import path from 'node:path'; +import { AudioCodec, buildAudioCodecString } from '../../src/codec.js'; +import { Conversion } from '../../src/conversion.js'; + +const __dirname = new URL('.', import.meta.url).pathname; + +const createDummyAudioTrack = (codec: AudioCodec, output: Output) => { + const source = new EncodedAudioPacketSource(codec); + output.addAudioTrack(source); + + return { + async addPacket() { + // Data to make it behave like an MP3 frame + const data = new Uint8Array(2000); + data[0] = 255; + data[1] = 251; + data[2] = 224; + data[3] = 100; + + // Opus description + const description = new Uint8Array([ + 79, 112, 117, 115, 72, 101, 97, 100, 1, 2, 56, 1, 68, 172, 0, 0, 0, 0, 0, + ]); + + await source.add( + new EncodedPacket(data, 'key', 0, 1), + { + decoderConfig: { + codec: buildAudioCodecString(codec, 2, 44100), + numberOfChannels: 2, + sampleRate: 44100, + description, + }, + }, + ); + }, + }; +}; + +const coverArt = new Uint8Array(1024); +coverArt[0] = 69; +const songMetadata: MetadataTags = { + title: 'Trying to Feel Alive', + description: 'A song', + artist: 'Porter Robinson & others', + album: 'Nurture', + albumArtist: 'Porter Robinson', + genre: 'Electronic', + comment: 'Some of this info is intentionally incorrect', + lyrics: 'Well, do you feel better now?\nI thought I\'d run until the sky came out', + trackNumber: 13, + tracksTotal: 14, + discNumber: 1, + discsTotal: 1, + date: new Date(2021, 3, 23), + images: [{ + data: coverArt, + kind: 'coverFront', + mimeType: 'image/jpeg', + description: 'This image shows a person laying in a field of grass', + }], +}; + +test('Read and write metadata, MP4', async () => { + const output = new Output({ + format: new Mp4OutputFormat(), + target: new BufferTarget(), + }); + + output.setMetadataTags({ + ...songMetadata, + raw: { + '©hog': 'pish', + '©nam': 'Cheerleader', // Test that it doesn't override the main title + }, + }); + + const dummyTrack = createDummyAudioTrack('aac', output); + + await output.start(); + await dummyTrack.addPacket(); + await output.finalize(); + + const input = new Input({ + source: new BufferSource(output.target.buffer!), + formats: ALL_FORMATS, + }); + + const readTags = await input.getMetadataTags(); + + expect(readTags.title).toBe(songMetadata.title); + expect(readTags.description).toBe(songMetadata.description); + expect(readTags.artist).toBe(songMetadata.artist); + expect(readTags.album).toBe(songMetadata.album); + expect(readTags.albumArtist).toBe(songMetadata.albumArtist); + expect(readTags.comment).toBe(songMetadata.comment); + expect(readTags.lyrics).toBe(songMetadata.lyrics); + expect(readTags.trackNumber).toBe(songMetadata.trackNumber); + expect(readTags.tracksTotal).toBe(songMetadata.tracksTotal); + expect(readTags.discNumber).toBe(songMetadata.discNumber); + expect(readTags.discsTotal).toBe(songMetadata.discsTotal); + expect(readTags.date).toEqual(readTags.date); + expect(readTags.images).toHaveLength(1); + expect(readTags.images![0]!.data).toEqual(coverArt); + expect(readTags.images![0]!.mimeType).toEqual('image/jpeg'); + expect(readTags.images![0]!.kind).toEqual('coverFront'); + expect(readTags.images![0]!.description).toBeUndefined(); // Lost in MP4 + + expect(readTags.raw!['©nam']).toBe(songMetadata.title); + expect(readTags.raw!['trkn']).instanceOf(Uint8Array); + expect(readTags.raw!['©hog']).toBe('pish'); +}); + +test('Read and write metadata, QuickTime', async () => { + const output = new Output({ + format: new MovOutputFormat(), + target: new BufferTarget(), + }); + + output.setMetadataTags({ + ...songMetadata, + raw: { + '©sic': 'ko mode', + }, + }); + + const dummyTrack = createDummyAudioTrack('aac', output); + + await output.start(); + await dummyTrack.addPacket(); + await output.finalize(); + + const input = new Input({ + source: new BufferSource(output.target.buffer!), + formats: ALL_FORMATS, + }); + + const readTags = await input.getMetadataTags(); + + expect(readTags.title).toBe(songMetadata.title); + expect(readTags.description).toBe(songMetadata.description); + expect(readTags.artist).toBe(songMetadata.artist); + expect(readTags.album).toBe(songMetadata.album); + expect(readTags.albumArtist).toBe(songMetadata.albumArtist); + expect(readTags.comment).toBe(songMetadata.comment); + expect(readTags.lyrics).toBe(songMetadata.lyrics); + expect(readTags.trackNumber).toBeUndefined(); // All of these don't work in MOV + expect(readTags.tracksTotal).toBeUndefined(); + expect(readTags.discNumber).toBeUndefined(); + expect(readTags.discsTotal).toBeUndefined(); + expect(readTags.date).toEqual(readTags.date); + expect(readTags.images).toBeUndefined(); + + expect(readTags.raw!['©nam']).toBe(songMetadata.title); + // We don't know what the data type is, so the demuxer just returns Uint8Array + expect(readTags.raw!['©sic']).toEqual(new Uint8Array([ + 0, 7, // String length 7 + 85, 196, // Language code for 'und' + 107, 111, 32, 109, 111, 100, 101, // 'ko mode' + ])); +}); + +test('Read MOV metadata tags, ilst with keys', async () => { + const input = new Input({ + source: new FilePathSource(path.join(__dirname, '../public/trunc-buck-bunny.mov')), + formats: ALL_FORMATS, + }); + + const tags = await input.getMetadataTags(); + expect(tags.raw).not.toBeUndefined(); + expect(tags.raw!['com.apple.quicktime.version']).toBe('7.4.1 (14) 0x7418000 (Mac OS X, 10.5.2, 9C31)'); + expect(tags.raw!['com.apple.quicktime.player.movie.audio.gain']).toEqual(new Uint8Array([63, 128, 0, 0])); +}); + +test('Read and write metadata, Matroska', async () => { + const output = new Output({ + format: new MkvOutputFormat(), + target: new BufferTarget(), + }); + + output.setMetadataTags({ + ...songMetadata, + raw: { + CUSTOM: 'Levels', + }, + }); + + const dummyTrack = createDummyAudioTrack('opus', output); + + await output.start(); + await dummyTrack.addPacket(); + await output.finalize(); + + const input = new Input({ + source: new BufferSource(output.target.buffer!), + formats: ALL_FORMATS, + }); + + const readTags = await input.getMetadataTags(); + + expect(readTags.title).toBe(songMetadata.title); + expect(readTags.description).toBe(songMetadata.description); + expect(readTags.artist).toBe(songMetadata.artist); + expect(readTags.album).toBe(songMetadata.album); + expect(readTags.albumArtist).toBe(songMetadata.albumArtist); + expect(readTags.comment).toBe(songMetadata.comment); + expect(readTags.lyrics).toBe(songMetadata.lyrics); + expect(readTags.trackNumber).toBe(songMetadata.trackNumber); + expect(readTags.tracksTotal).toBe(songMetadata.tracksTotal); + expect(readTags.discNumber).toBe(songMetadata.discNumber); + expect(readTags.discsTotal).toBe(songMetadata.discsTotal); + expect(readTags.date).toEqual(readTags.date); + expect(readTags.images).toHaveLength(1); + expect(readTags.images![0]!.data).toEqual(coverArt); + expect(readTags.images![0]!.mimeType).toEqual('image/jpeg'); + expect(readTags.images![0]!.kind).toEqual('coverFront'); + expect(readTags.images![0]!.description).toEqual(songMetadata.images![0]!.description); + expect(readTags.images![0]!.name).toEqual('cover.jpg'); // It constructed this name + + expect(readTags.raw!['TITLE']).toBe(songMetadata.title); + expect(readTags.raw!['PART_NUMBER']).toBe('13/14'); + expect(readTags.raw!['CUSTOM']).toBe('Levels'); +}); + +test('Read and write metadata, MP3', async () => { + const output = new Output({ + format: new Mp3OutputFormat(), + target: new BufferTarget(), + }); + + output.setMetadataTags({ + ...songMetadata, + raw: { + TXXY: 'ID3v2 goated', + }, + }); + + const dummyTrack = createDummyAudioTrack('mp3', output); + + await output.start(); + await dummyTrack.addPacket(); + await output.finalize(); + + const input = new Input({ + source: new BufferSource(output.target.buffer!), + formats: ALL_FORMATS, + }); + + const readTags = await input.getMetadataTags(); + + // ID3v2 is goated, so pretty much everything was copied: + expect(readTags.title).toBe(songMetadata.title); + expect(readTags.description).toBe(songMetadata.description); + expect(readTags.artist).toBe(songMetadata.artist); + expect(readTags.album).toBe(songMetadata.album); + expect(readTags.albumArtist).toBe(songMetadata.albumArtist); + expect(readTags.comment).toBe(songMetadata.comment); + expect(readTags.lyrics).toBe(songMetadata.lyrics); + expect(readTags.trackNumber).toBe(songMetadata.trackNumber); + expect(readTags.tracksTotal).toBe(songMetadata.tracksTotal); + expect(readTags.discNumber).toBe(songMetadata.discNumber); + expect(readTags.discsTotal).toBe(songMetadata.discsTotal); + expect(readTags.date).toEqual(readTags.date); + expect(readTags.images).toHaveLength(1); + expect(readTags.images![0]!.data).toEqual(coverArt); + expect(readTags.images![0]!.mimeType).toEqual('image/jpeg'); + expect(readTags.images![0]!.kind).toEqual('coverFront'); + expect(readTags.images![0]!.description).toEqual(songMetadata.images![0]!.description); + expect(readTags.images![0]!.name).toBeUndefined(); // Can't be contained in ID3v2 + + expect(readTags.raw!['TIT2']).toBe(songMetadata.title); + expect(readTags.raw!['APIC']).instanceOf(Uint8Array); + expect(readTags.raw!['TXXY']).toBe('ID3v2 goated'); +}); + +test('Read and write metadata, Ogg', async () => { + const output = new Output({ + format: new OggOutputFormat(), + target: new BufferTarget(), + }); + + output.setMetadataTags({ + ...songMetadata, + raw: { + vendor: 'mediabunny corp', + COMPOSER: 'Hans Zimmer', + }, + }); + + const dummyTrack = createDummyAudioTrack('opus', output); + + await output.start(); + await dummyTrack.addPacket(); + await output.finalize(); + + const input = new Input({ + source: new BufferSource(output.target.buffer!), + formats: ALL_FORMATS, + }); + + const readTags = await input.getMetadataTags(); + + expect(readTags.title).toBe(songMetadata.title); + expect(readTags.description).toBe(songMetadata.description); + expect(readTags.artist).toBe(songMetadata.artist); + expect(readTags.album).toBe(songMetadata.album); + expect(readTags.albumArtist).toBe(songMetadata.albumArtist); + expect(readTags.comment).toBe(songMetadata.comment); + expect(readTags.lyrics).toBe(songMetadata.lyrics); + expect(readTags.trackNumber).toBe(songMetadata.trackNumber); + expect(readTags.tracksTotal).toBe(songMetadata.tracksTotal); + expect(readTags.discNumber).toBe(songMetadata.discNumber); + expect(readTags.discsTotal).toBe(songMetadata.discsTotal); + expect(readTags.date).toEqual(readTags.date); + expect(readTags.images).toHaveLength(1); + expect(readTags.images![0]!.data).toEqual(coverArt); + expect(readTags.images![0]!.mimeType).toEqual('image/jpeg'); + expect(readTags.images![0]!.kind).toEqual('coverFront'); + expect(readTags.images![0]!.description).toEqual(songMetadata.images![0]!.description); + expect(readTags.images![0]!.name).toBeUndefined(); // Can't be contained in Vorbis-style metadata + + expect(readTags.raw!['vendor']).toBe('mediabunny corp'); + expect(readTags.raw!['COMPOSER']).toBe('Hans Zimmer'); +}); + +test('Read and write metadata, WAVE', async () => { + const output = new Output({ + format: new WavOutputFormat(), + target: new BufferTarget(), + }); + + output.setMetadataTags({ + ...songMetadata, + raw: { + IKEK: 'RIFF INFO lowkey mid', + }, + }); + + const dummyTrack = createDummyAudioTrack('pcm-s16', output); + + await output.start(); + await dummyTrack.addPacket(); + await output.finalize(); + + const input = new Input({ + source: new BufferSource(output.target.buffer!), + formats: ALL_FORMATS, + }); + + const readTags = await input.getMetadataTags(); + + expect(readTags.title).toBe(songMetadata.title); + expect(readTags.description).toBeUndefined(); + expect(readTags.artist).toBe(songMetadata.artist); + expect(readTags.album).toBe(songMetadata.album); + expect(readTags.albumArtist).toBeUndefined(); + expect(readTags.comment).toBe(songMetadata.comment); + expect(readTags.lyrics).toBeUndefined(); + expect(readTags.trackNumber).toBe(songMetadata.trackNumber); + expect(readTags.tracksTotal).toBe(songMetadata.tracksTotal); + expect(readTags.discNumber).toBeUndefined(); + expect(readTags.discsTotal).toBeUndefined(); + expect(readTags.date).toEqual(readTags.date); + expect(readTags.images).toBeUndefined(); + + expect(readTags.raw!['INAM']).toBe(songMetadata.title); + expect(readTags.raw!['IKEK']).toBe('RIFF INFO lowkey mid'); +}); + +test('Conversion metadata tags, default case', async () => { + const output = new Output({ + format: new Mp4OutputFormat(), + target: new BufferTarget(), + }); + + output.setMetadataTags(songMetadata); + + const dummyTrack = createDummyAudioTrack('opus', output); + + await output.start(); + await dummyTrack.addPacket(); + await output.finalize(); + + const input = new Input({ + source: new BufferSource(output.target.buffer!), + formats: ALL_FORMATS, + }); + + const output2 = new Output({ + format: new MkvOutputFormat(), + target: new BufferTarget(), + }); + + const conversion = await Conversion.init({ input, output: output2 }); + await conversion.execute(); + + const input2 = new Input({ + source: new BufferSource(output2.target.buffer!), + formats: ALL_FORMATS, + }); + + const readTags = await input2.getMetadataTags(); + + expect(readTags.title).toBe(songMetadata.title); + expect(readTags.description).toBe(songMetadata.description); + expect(readTags.artist).toBe(songMetadata.artist); + expect(readTags.album).toBe(songMetadata.album); + expect(readTags.albumArtist).toBe(songMetadata.albumArtist); + expect(readTags.comment).toBe(songMetadata.comment); + expect(readTags.lyrics).toBe(songMetadata.lyrics); + expect(readTags.trackNumber).toBe(songMetadata.trackNumber); + expect(readTags.tracksTotal).toBe(songMetadata.tracksTotal); + expect(readTags.discNumber).toBe(songMetadata.discNumber); + expect(readTags.discsTotal).toBe(songMetadata.discsTotal); + expect(readTags.date).toEqual(readTags.date); + expect(readTags.images).toHaveLength(1); + expect(readTags.images![0]!.data).toEqual(coverArt); +}); + +test('Conversion metadata tags, modified', async () => { + const output = new Output({ + format: new Mp4OutputFormat(), + target: new BufferTarget(), + }); + + output.setMetadataTags(songMetadata); + + const dummyTrack = createDummyAudioTrack('opus', output); + + await output.start(); + await dummyTrack.addPacket(); + await output.finalize(); + + const input = new Input({ + source: new BufferSource(output.target.buffer!), + formats: ALL_FORMATS, + }); + + const output2 = new Output({ + format: new MkvOutputFormat(), + target: new BufferTarget(), + }); + + const conversion = await Conversion.init({ + input, + output: output2, + tags: inputTags => ({ + title: 'Blossom', + artist: inputTags.artist, + raw: inputTags.raw, // This should NOT be copied + }), + }); + await conversion.execute(); + + const input2 = new Input({ + source: new BufferSource(output2.target.buffer!), + formats: ALL_FORMATS, + }); + + const readTags = await input2.getMetadataTags(); + + expect(Object.keys(readTags).length).toBe(3); + expect(readTags.title).toBe('Blossom'); + expect(readTags.artist).toBe(songMetadata.artist); + expect(Object.keys(readTags.raw!).length).toBe(2); +}); diff --git a/test/public/trunc-buck-bunny.mov b/test/public/trunc-buck-bunny.mov new file mode 100644 index 0000000..9192cd7 Binary files /dev/null and b/test/public/trunc-buck-bunny.mov differ