Add metadata read/write support for Matroska

This commit is contained in:
Vanilagy
2025-09-05 13:41:09 +02:00
parent efa933863d
commit 6d32de0fb1
6 changed files with 577 additions and 42 deletions
+19
View File
@@ -129,8 +129,27 @@ export enum EBMLId {
ProjectionType = 0x7671,
ProjectionPoseRoll = 0x7675,
Attachments = 0x1941a469,
AttachedFile = 0x61a7,
FileDescription = 0x467e,
FileName = 0x466e,
FileMediaType = 0x4660,
FileData = 0x465c,
FileUID = 0x46ae,
Chapters = 0x1043a770,
Tags = 0x1254c367,
Tag = 0x7373,
Targets = 0x63c0,
TargetTypeValue = 0x68ca,
TargetType = 0x63ca,
TagTrackUID = 0x63c5,
TagEditionUID = 0x63c9,
TagChapterUID = 0x63c4,
TagAttachmentUID = 0x63c6,
SimpleTag = 0x67c8,
TagName = 0x45a3,
TagLanguage = 0x447a,
TagString = 0x4487,
TagBinary = 0x4485,
}
export const LEVEL_0_EBML_IDS: EBMLId[] = [
+293 -3
View File
@@ -30,6 +30,7 @@ import {
InputVideoTrack,
InputVideoTrackBacking,
} from '../input-track';
import { MediaMetadata } from '../metadata';
import { PacketRetrievalOptions } from '../media-sink';
import {
assert,
@@ -76,6 +77,8 @@ type Segment = {
infoSeen: boolean;
tracksSeen: boolean;
cuesSeen: boolean;
attachmentsSeen: boolean;
tagsSeen: boolean;
timestampScale: number;
timestampFactor: number;
@@ -90,6 +93,9 @@ type Segment = {
clusters: Cluster[];
clusterLookupMutex: AsyncMutex;
metadata: MediaMetadata;
metadataCollected: boolean;
};
type SeekEntry = {
@@ -198,6 +204,14 @@ export class MatroskaDemuxer extends Demuxer {
currentCluster: Cluster | null = null;
currentBlock: ClusterBlock | null = null;
currentCueTime: number | null = null;
currentTagTargetIsMovie: boolean = true;
currentSimpleTagName: string | null = null;
currentAttachedFile: {
fileName: string | null;
fileMediaType: string | null;
fileData: Uint8Array | null;
fileDescription: string | null;
} | null = null;
isWebM = false;
@@ -232,6 +246,27 @@ export class MatroskaDemuxer extends Demuxer {
});
}
async getMetadata() {
await this.readMetadata();
// Load metadata from each segment lazily (only once)
for (const segment of this.segments) {
if (!segment.metadataCollected) {
await this.loadSegmentMetadata(segment);
segment.metadataCollected = true;
}
}
// This is kinda handwavy, and how we handle multiple segments isn't suuuuper well-defined anyway; so we just
// shallow-merge metadata from all (usually just one) segments.
let metadata: MediaMetadata = {};
for (const segment of this.segments) {
metadata = { ...metadata, ...segment.metadata };
}
return metadata;
}
readMetadata() {
return this.readMetadataPromise ??= (async () => {
let currentPos = 0;
@@ -311,6 +346,8 @@ export class MatroskaDemuxer extends Demuxer {
infoSeen: false,
tracksSeen: false,
cuesSeen: false,
tagsSeen: false,
attachmentsSeen: false,
timestampScale: -1,
timestampFactor: -1,
@@ -327,6 +364,9 @@ export class MatroskaDemuxer extends Demuxer {
clusters: [],
clusterLookupMutex: new AsyncMutex(),
metadata: {},
metadataCollected: false,
};
this.segments.push(this.currentSegment);
@@ -371,6 +411,22 @@ export class MatroskaDemuxer extends Demuxer {
let slice = this.reader.requestSlice(dataStartPos, size);
if (slice instanceof Promise) slice = await slice;
if (slice) {
this.readContiguousElements(slice);
}
} else if (id === EBMLId.Tags || id === EBMLId.Attachments) {
// Metadata found at the beginning of the segment, great, let's parse it
if (id === EBMLId.Tags) {
this.currentSegment.tagsSeen = true;
} else {
this.currentSegment.attachmentsSeen = true;
}
assertDefinedSize(size);
let slice = this.reader.requestSlice(dataStartPos, size);
if (slice instanceof Promise) slice = await slice;
if (slice) {
this.readContiguousElements(slice);
}
@@ -386,10 +442,10 @@ export class MatroskaDemuxer extends Demuxer {
}
}
if (this.reader.fileSize !== null) {
// Sort the seek entries by file position so reading them exhibits a sequential pattern
this.currentSegment.seekEntries.sort((a, b) => a.segmentPosition - b.segmentPosition);
// Sort the seek entries by file position so reading them exhibits a sequential pattern
this.currentSegment.seekEntries.sort((a, b) => a.segmentPosition - b.segmentPosition);
if (this.reader.fileSize !== null) {
// Use the seek head to read missing metadata elements
for (const seekEntry of this.currentSegment.seekEntries) {
const target = METADATA_ELEMENTS.find(x => x.id === seekEntry.id);
@@ -745,6 +801,50 @@ export class MatroskaDemuxer extends Demuxer {
}
}
async loadSegmentMetadata(segment: Segment) {
for (const seekEntry of segment.seekEntries) {
if (seekEntry.id === EBMLId.Tags && !segment.tagsSeen) {
// We need to load the tags
} else if (seekEntry.id === EBMLId.Attachments && !segment.attachmentsSeen) {
// We need to load the attachments
} else {
continue;
}
let slice = this.reader.requestSliceRange(
segment.dataStartPos + seekEntry.segmentPosition,
MIN_HEADER_SIZE,
MAX_HEADER_SIZE,
);
if (slice instanceof Promise) slice = await slice;
if (!slice) continue;
const header = readElementHeader(slice);
if (!header || header.id !== seekEntry.id) continue;
const { size } = header;
assertDefinedSize(size);
assert(!this.currentSegment);
this.currentSegment = segment;
let dataSlice = this.reader.requestSlice(slice.filePos, size);
if (dataSlice instanceof Promise) dataSlice = await dataSlice;
if (dataSlice) {
this.readContiguousElements(dataSlice);
}
this.currentSegment = null;
// Mark as seen
if (seekEntry.id === EBMLId.Tags) {
segment.tagsSeen = true;
} else if (seekEntry.id === EBMLId.Attachments) {
segment.attachmentsSeen = true;
}
}
}
readContiguousElements(slice: FileSlice) {
const startIndex = slice.filePos;
@@ -1251,11 +1351,201 @@ export class MatroskaDemuxer extends Demuxer {
// We'll offset this by the block's timestamp later
this.currentBlock.referencedTimestamps.push(relativeTimestamp);
}; break;
case EBMLId.Tag: {
this.currentTagTargetIsMovie = true;
this.readContiguousElements(slice.slice(dataStartPos, size));
}; break;
case EBMLId.Targets: {
this.readContiguousElements(slice.slice(dataStartPos, size));
}; break;
case EBMLId.TargetTypeValue: {
const targetTypeValue = readUnsignedInt(slice, size);
if (targetTypeValue !== 50) {
this.currentTagTargetIsMovie = false;
}
}; break;
case EBMLId.TagTrackUID:
case EBMLId.TagEditionUID:
case EBMLId.TagChapterUID:
case EBMLId.TagAttachmentUID: {
this.currentTagTargetIsMovie = false;
}; break;
case EBMLId.SimpleTag: {
if (!this.currentTagTargetIsMovie) break;
this.currentSimpleTagName = null;
this.readContiguousElements(slice.slice(dataStartPos, size));
}; break;
case EBMLId.TagName: {
this.currentSimpleTagName = readUnicodeString(slice, size);
}; break;
case EBMLId.TagString: {
if (!this.currentSimpleTagName) break;
const value = readUnicodeString(slice, size);
this.processTagValue(this.currentSimpleTagName, value);
}; break;
case EBMLId.TagBinary: {
if (!this.currentSimpleTagName) break;
const value = readBytes(slice, size);
this.processTagValue(this.currentSimpleTagName, value);
}; break;
case EBMLId.AttachedFile: {
if (!this.currentSegment) break;
this.currentAttachedFile = {
fileName: null,
fileMediaType: null,
fileData: null,
fileDescription: null,
};
this.readContiguousElements(slice.slice(dataStartPos, size));
// Only process image attachments
if (this.currentAttachedFile.fileMediaType?.startsWith('image/') && this.currentAttachedFile.fileData) {
const fileName = this.currentAttachedFile.fileName;
let kind: 'coverFront' | 'coverBack' | 'unknown' = 'unknown';
if (fileName) {
const lowerName = fileName.toLowerCase();
if (lowerName.startsWith('cover.')) {
kind = 'coverFront';
} else if (lowerName.startsWith('back.')) {
kind = 'coverBack';
}
}
this.currentSegment.metadata.images ??= [];
this.currentSegment.metadata.images.push({
data: this.currentAttachedFile.fileData,
mimeType: this.currentAttachedFile.fileMediaType,
kind,
name: this.currentAttachedFile.fileName ?? undefined,
description: this.currentAttachedFile.fileDescription ?? undefined,
});
}
this.currentAttachedFile = null;
}; break;
case EBMLId.FileName: {
if (!this.currentAttachedFile) break;
this.currentAttachedFile.fileName = readUnicodeString(slice, size);
}; break;
case EBMLId.FileMediaType: {
if (!this.currentAttachedFile) break;
this.currentAttachedFile.fileMediaType = readAsciiString(slice, size);
}; break;
case EBMLId.FileData: {
if (!this.currentAttachedFile) break;
this.currentAttachedFile.fileData = readBytes(slice, size);
}; break;
case EBMLId.FileDescription: {
if (!this.currentAttachedFile) break;
this.currentAttachedFile.fileDescription = readUnicodeString(slice, size);
}; break;
}
slice.filePos = dataStartPos + size;
return true;
}
processTagValue(name: string, value: string | Uint8Array) {
if (!this.currentSegment?.metadata) return;
const metadata = this.currentSegment.metadata;
metadata.raw ??= {};
metadata.raw[name] = value;
if (typeof value === 'string') {
switch (name.toLowerCase()) {
case 'title': {
metadata.title ??= value;
}; break;
case 'description': {
metadata.description ??= value;
}; break;
case 'artist': {
metadata.artist ??= value;
}; break;
case 'album': {
metadata.album ??= value;
}; break;
case 'album_artist': {
metadata.albumArtist ??= value;
}; break;
case 'genre': {
metadata.genre ??= value;
}; break;
case 'comment': {
metadata.comment ??= value;
}; break;
case 'lyrics': {
metadata.lyrics ??= value;
}; break;
case 'date': {
const date = new Date(value);
if (!Number.isNaN(date.getTime())) {
metadata.date ??= date;
}
}; break;
case 'track_number':
case 'part_number': {
const parts = value.split('/');
const trackNum = Number.parseInt(parts[0]!, 10);
const trackNumMax = parts[1] && Number.parseInt(parts[1], 10);
if (Number.isInteger(trackNum) && trackNum > 0) {
metadata.trackNumber ??= trackNum;
}
if (trackNumMax && Number.isInteger(trackNumMax) && trackNumMax > 0) {
metadata.trackNumberMax ??= trackNumMax;
}
}; break;
case 'disc_number':
case 'disc': {
const discParts = value.split('/');
const discNum = Number.parseInt(discParts[0]!, 10);
const discNumMax = discParts[1] && Number.parseInt(discParts[1], 10);
if (Number.isInteger(discNum) && discNum > 0) {
metadata.discNumber ??= discNum;
}
if (discNumMax && Number.isInteger(discNumMax) && discNumMax > 0) {
metadata.discNumberMax ??= discNumMax;
}
}; break;
}
}
}
}
abstract class MatroskaTrackBacking implements InputTrackBacking {
+235 -36
View File
@@ -13,7 +13,10 @@ import {
TRANSFER_CHARACTERISTICS_MAP,
UNDETERMINED_LANGUAGE,
assert,
assertNever,
colorSpaceIsComplete,
imageMimeTypeToExtension,
keyValueIterator,
normalizeRotation,
promiseWithResolvers,
roundToMultiple,
@@ -74,22 +77,6 @@ type InternalMediaChunk = {
additions: Uint8Array | null;
};
type SeekHead = {
id: number;
data: {
id: number;
data: ({
id: number;
data: Uint8Array;
size?: undefined;
} | {
id: number;
size: number;
data: number;
})[];
}[];
};
type MatroskaTrackData = {
chunkQueue: InternalMediaChunk[];
lastWrittenMsTimestamp: number | null;
@@ -137,8 +124,10 @@ export class MatroskaMuxer extends Muxer {
private segment: EBMLElement | null = null;
private segmentInfo: EBMLElement | null = null;
private seekHead: SeekHead | null = null;
private seekHead: EBMLElement | null = null;
private tracksElement: EBMLElement | null = null;
private tagsElement: EBMLElement | null = null;
private attachmentsElement: EBMLElement | null = null;
private segmentDuration: EBMLElement | null = null;
private cues: EBMLElement | null = null;
@@ -169,10 +158,6 @@ export class MatroskaMuxer extends Muxer {
this.writeEBMLHeader();
if (!this.format._options.appendOnly) {
this.createSeekHead();
}
this.createSegmentInfo();
this.createCues();
@@ -207,24 +192,72 @@ export class MatroskaMuxer extends Muxer {
* Creates a SeekHead element which is positioned near the start of the file and allows the media player to seek to
* relevant sections more easily. Since we don't know the positions of those sections yet, we'll set them later.
*/
private createSeekHead() {
private maybeCreateSeekHead(writeOffsets: boolean) {
if (this.format._options.appendOnly) {
return;
}
const kaxCues = new Uint8Array([0x1c, 0x53, 0xbb, 0x6b]);
const kaxInfo = new Uint8Array([0x15, 0x49, 0xa9, 0x66]);
const kaxTracks = new Uint8Array([0x16, 0x54, 0xae, 0x6b]);
const kaxAttachments = new Uint8Array([0x19, 0x41, 0xa4, 0x69]);
const kaxTags = new Uint8Array([0x12, 0x54, 0xc3, 0x67]);
const seekHead = { id: EBMLId.SeekHead, data: [
{ id: EBMLId.Seek, data: [
{ id: EBMLId.SeekID, data: kaxCues },
{ id: EBMLId.SeekPosition, size: 5, data: 0 },
{
id: EBMLId.SeekPosition,
size: 5,
data: writeOffsets
? this.ebmlWriter.offsets.get(this.cues!)! - this.segmentDataOffset
: 0,
},
] },
{ id: EBMLId.Seek, data: [
{ id: EBMLId.SeekID, data: kaxInfo },
{ id: EBMLId.SeekPosition, size: 5, data: 0 },
{
id: EBMLId.SeekPosition,
size: 5,
data: writeOffsets
? this.ebmlWriter.offsets.get(this.segmentInfo!)! - this.segmentDataOffset
: 0,
},
] },
{ id: EBMLId.Seek, data: [
{ id: EBMLId.SeekID, data: kaxTracks },
{ id: EBMLId.SeekPosition, size: 5, data: 0 },
{
id: EBMLId.SeekPosition,
size: 5,
data: writeOffsets
? this.ebmlWriter.offsets.get(this.tracksElement!)! - this.segmentDataOffset
: 0,
},
] },
this.attachmentsElement
? { id: EBMLId.Seek, data: [
{ id: EBMLId.SeekID, data: kaxAttachments },
{
id: EBMLId.SeekPosition,
size: 5,
data: writeOffsets
? this.ebmlWriter.offsets.get(this.attachmentsElement)! - this.segmentDataOffset
: 0,
},
] }
: null,
this.tagsElement
? { id: EBMLId.Seek, data: [
{ id: EBMLId.SeekID, data: kaxTags },
{
id: EBMLId.SeekPosition,
size: 5,
data: writeOffsets
? this.ebmlWriter.offsets.get(this.tagsElement)! - this.segmentDataOffset
: 0,
},
] }
: null,
] };
this.seekHead = seekHead;
}
@@ -379,14 +412,187 @@ export class MatroskaMuxer extends Muxer {
];
}
private maybeCreateTags() {
const simpleTags: EBMLElement[] = [];
const addSimpleTag = (key: string, value: string | Uint8Array) => {
simpleTags.push({ id: EBMLId.SimpleTag, data: [
{ id: EBMLId.TagName, data: new EBMLUnicodeString(key) },
typeof value === 'string'
? { id: EBMLId.TagString, data: new EBMLUnicodeString(value) }
: { id: EBMLId.TagBinary, data: value },
] });
};
const metadata = this.output._metadata;
const writtenTags = new Set<string>();
for (const { key, value } of keyValueIterator(metadata)) {
switch (key) {
case 'title': {
addSimpleTag('TITLE', value);
writtenTags.add('TITLE');
}; break;
case 'description': {
addSimpleTag('DESCRIPTION', value);
writtenTags.add('DESCRIPTION');
}; break;
case 'artist': {
addSimpleTag('ARTIST', value);
writtenTags.add('ARTIST');
}; break;
case 'album': {
addSimpleTag('ALBUM', value);
writtenTags.add('ALBUM');
}; break;
case 'albumArtist': {
addSimpleTag('ALBUM_ARTIST', value);
writtenTags.add('ALBUM_ARTIST');
}; break;
case 'genre': {
addSimpleTag('GENRE', value);
writtenTags.add('GENRE');
}; break;
case 'comment': {
addSimpleTag('COMMENT', value);
writtenTags.add('COMMENT');
}; break;
case 'lyrics': {
addSimpleTag('LYRICS', value);
writtenTags.add('LYRICS');
}; break;
case 'date': {
addSimpleTag('DATE', value.toISOString().split('T')[0]!);
writtenTags.add('DATE');
}; break;
case 'trackNumber': {
const string = metadata.trackNumberMax !== undefined
? `${value}/${metadata.trackNumberMax}`
: value.toString();
addSimpleTag('PART_NUMBER', string);
writtenTags.add('PART_NUMBER');
}; break;
case 'discNumber': {
const string = metadata.discNumberMax !== undefined
? `${value}/${metadata.discNumberMax}`
: value.toString();
addSimpleTag('DISC', string);
writtenTags.add('DISC');
}; break;
case 'trackNumberMax':
case 'discNumberMax': {
// Handled with trackNumber and discNumber respectively
}; break;
case 'images':
case 'raw': {
// Handled elsewhere
}; break;
default: assertNever(key);
}
}
if (metadata.raw) {
for (const key in metadata.raw) {
const value = metadata.raw[key]!;
if (value == null || writtenTags.has(key)) {
continue;
}
if (typeof value === 'string' || value instanceof Uint8Array) {
addSimpleTag(key, value);
}
}
}
if (simpleTags.length === 0) {
return;
}
this.tagsElement = {
id: EBMLId.Tags,
data: [{ id: EBMLId.Tag, data: [
{ id: EBMLId.Targets, data: [
{ id: EBMLId.TargetTypeValue, data: 50 },
{ id: EBMLId.TargetType, data: 'MOVIE' },
] },
...simpleTags,
] }],
};
}
private maybeCreateAttachments() {
const metadata = this.output._metadata;
if (!metadata.images || metadata.images.length === 0) {
return;
}
const existingFileUids = new Set<number>();
this.attachmentsElement = { id: EBMLId.Attachments, data: metadata.images.map((image): EBMLElement => {
let imageName = image.name;
if (imageName === undefined) {
const baseName = image.kind === 'coverFront' ? 'cover' : image.kind === 'coverBack' ? 'back' : 'image';
imageName = baseName + (imageMimeTypeToExtension(image.mimeType) ?? '');
}
let fileUid: number;
while (true) {
fileUid = Math.floor(Math.random() * Number.MAX_SAFE_INTEGER);
if (fileUid !== 0 && !existingFileUids.has(fileUid)) {
break;
}
}
existingFileUids.add(fileUid);
return {
id: EBMLId.AttachedFile,
data: [
image.description !== undefined
? { id: EBMLId.FileDescription, data: new EBMLUnicodeString(image.description) }
: null,
{ id: EBMLId.FileName, data: new EBMLUnicodeString(imageName) },
{ id: EBMLId.FileMediaType, data: image.mimeType },
{ id: EBMLId.FileData, data: image.data },
{ id: EBMLId.FileUID, data: fileUid },
],
};
}) };
}
private createSegment() {
this.createTracks();
this.maybeCreateTags();
this.maybeCreateAttachments();
this.maybeCreateSeekHead(false);
const segment: EBML = {
id: EBMLId.Segment,
size: this.format._options.appendOnly ? -1 : SEGMENT_SIZE_BYTES,
data: [
!this.format._options.appendOnly ? this.seekHead as EBML : null,
this.seekHead, // null if append-only
this.segmentInfo,
this.tracksElement,
// Matroska spec says put this at the end of the file, but I think placing it before the first cluster
// makes more sense, and FFmpeg agrees (argumentum ad ffmpegum fallacy)
this.attachmentsElement,
this.tagsElement,
],
};
this.segment = segment;
@@ -754,7 +960,6 @@ export class MatroskaMuxer extends Muxer {
private writeBlock(trackData: MatroskaTrackData, chunk: InternalMediaChunk) {
// Due to the interlacing algorithm, this code will be run once we've seen one chunk from every media track.
if (!this.segment) {
this.createTracks();
this.createSegment();
}
@@ -956,7 +1161,6 @@ export class MatroskaMuxer extends Muxer {
this.allTracksKnown.resolve();
if (!this.segment) {
this.createTracks();
this.createSegment();
}
@@ -984,14 +1188,9 @@ export class MatroskaMuxer extends Muxer {
this.ebmlWriter.writeEBML(this.segmentDuration);
// Fill in SeekHead position data and write it again
this.seekHead!.data[0]!.data[1]!.data
= this.ebmlWriter.offsets.get(this.cues)! - this.segmentDataOffset;
this.seekHead!.data[1]!.data[1]!.data
= this.ebmlWriter.offsets.get(this.segmentInfo!)! - this.segmentDataOffset;
this.seekHead!.data[2]!.data[1]!.data
= this.ebmlWriter.offsets.get(this.tracksElement!)! - this.segmentDataOffset;
this.writer.seek(this.ebmlWriter.offsets.get(this.seekHead!)!);
assert(this.seekHead);
this.writer.seek(this.ebmlWriter.offsets.get(this.seekHead)!);
this.maybeCreateSeekHead(true);
this.ebmlWriter.writeEBML(this.seekHead);
this.writer.seek(endPos);
+2 -2
View File
@@ -68,8 +68,8 @@ export const validateMediaMetadata = (metadata: MediaMetadata) => {
if (metadata.genre !== undefined && typeof metadata.genre !== 'string') {
throw new TypeError('metadata.genre, when provided, must be a string.');
}
if (metadata.date !== undefined && !(metadata.date instanceof Date)) {
throw new TypeError('metadata.date, when provided, must be a Date.');
if (metadata.date !== undefined && (!(metadata.date instanceof Date) || Number.isNaN(metadata.date.getTime()))) {
throw new TypeError('metadata.date, when provided, must be a valid Date.');
}
if (metadata.lyrics !== undefined && typeof metadata.lyrics !== 'string') {
throw new TypeError('metadata.lyrics, when provided, must be a string.');
+27
View File
@@ -675,3 +675,30 @@ export const keyValueIterator = function* <T extends Record<string, unknown>>(ob
yield { key, value } as KeyValuePair<T>;
}
};
export const imageMimeTypeToExtension = (mimeType: string) => {
switch (mimeType.toLowerCase()) {
case 'image/jpeg':
case 'image/jpg':
return '.jpg';
case 'image/png':
return '.png';
case 'image/gif':
return '.gif';
case 'image/webp':
return '.webp';
case 'image/bmp':
return '.bmp';
case 'image/svg+xml':
return '.svg';
case 'image/tiff':
return '.tiff';
case 'image/avif':
return '.avif';
case 'image/x-icon':
case 'image/vnd.microsoft.icon':
return '.ico';
default:
return null;
}
};