import { AacCodecInfo, AudioCodec, extractAudioCodecString, extractAv1CodecInfoFromFrame, extractVideoCodecString, extractVp9CodecInfoFromFrame, MediaCodec, VideoCodec, } from '../codec'; import { Demuxer } from '../demuxer'; import { Input } from '../input'; import { InputAudioTrack, InputAudioTrackBacking, InputTrack, InputTrackBacking, InputVideoTrack, InputVideoTrackBacking, } from '../input-track'; import { PacketRetrievalOptions } from '../media-sink'; import { assert, AsyncMutex, binarySearchExact, binarySearchLessOrEqual, COLOR_PRIMARIES_MAP_INVERSE, findLastIndex, isIso639Dash2LanguageCode, last, MATRIX_COEFFICIENTS_MAP_INVERSE, normalizeRotation, Rotation, roundToPrecision, TRANSFER_CHARACTERISTICS_MAP_INVERSE, UNDETERMINED_LANGUAGE, } from '../misc'; import { EncodedPacket, PLACEHOLDER_DATA } from '../packet'; import { Reader } from '../reader'; import { CODEC_STRING_MAP, EBMLId, EBMLReader, MAX_HEADER_SIZE, MIN_HEADER_SIZE } from './ebml'; type Segment = { seekHeadSeen: boolean; infoSeen: boolean; tracksSeen: boolean; cuesSeen: boolean; timestampScale: number; timestampFactor: number; duration: number; seekEntries: SeekEntry[]; tracks: InternalTrack[]; cuePoints: CuePoint[]; dataStartPos: number; elementEndPos: number; clusterSeekStartPos: number; clusters: Cluster[]; clusterLookupMutex: AsyncMutex; }; type SeekEntry = { id: number; segmentPosition: number; }; type Cluster = { elementStartPos: number; elementEndPos: number; dataStartPos: number; timestamp: number; trackData: Map; nextCluster: Cluster | null; isKnownToBeFirstCluster: boolean; }; type ClusterTrackData = { startTimestamp: number; endTimestamp: number; firstKeyFrameTimestamp: number | null; blocks: ClusterBlock[]; presentationTimestamps: { timestamp: number; blockIndex: number; }[]; }; type ClusterBlock = { timestamp: number; duration: number; isKeyFrame: boolean; referencedTimestamps: number[]; data: Uint8Array; }; type CuePoint = { time: number; trackId: number; clusterPosition: number; }; type InternalTrack = { id: number; demuxer: MatroskaDemuxer; segment: Segment; clusters: Cluster[]; clustersWithKeyFrame: Cluster[]; cuePoints: CuePoint[]; isDefault: boolean; inputTrack: InputTrack | null; codecId: string | null; codecPrivate: Uint8Array | null; languageCode: string; info: | null | { type: 'video'; width: number; height: number; rotation: Rotation; codec: VideoCodec | null; colorSpace: VideoColorSpaceInit | null; } | { type: 'audio'; numberOfChannels: number; sampleRate: number; bitDepth: number; codec: AudioCodec | null; aacCodecInfo: AacCodecInfo | null; }; }; type InternalVideoTrack = InternalTrack & { info: { type: 'video' } }; type InternalAudioTrack = InternalTrack & { info: { type: 'audio' } }; const METADATA_ELEMENTS = [ { id: EBMLId.SeekHead, flag: 'seekHeadSeen' }, { id: EBMLId.Info, flag: 'infoSeen' }, { id: EBMLId.Tracks, flag: 'tracksSeen' }, { id: EBMLId.Cues, flag: 'cuesSeen' }, ] as const; export class MatroskaDemuxer extends Demuxer { metadataReader: EBMLReader; clusterReader: EBMLReader; readMetadataPromise: Promise | null = null; segments: Segment[] = []; currentSegment: Segment | null = null; currentTrack: InternalTrack | null = null; currentCluster: Cluster | null = null; currentBlock: ClusterBlock | null = null; currentCueTime: number | null = null; isWebM = false; constructor(input: Input) { super(input); this.metadataReader = new EBMLReader(input._mainReader); // Max 64 MiB of stored clusters this.clusterReader = new EBMLReader(new Reader(input._source, 64 * 2 ** 20)); } override async computeDuration() { const tracks = await this.getTracks(); const trackDurations = await Promise.all(tracks.map(x => x.computeDuration())); return Math.max(0, ...trackDurations); } async getTracks() { await this.readMetadata(); return this.segments.flatMap(segment => segment.tracks.map(track => track.inputTrack!)); } override async getMimeType() { await this.readMetadata(); const base = this.segments.some(segment => segment.tracks.some(x => x.info?.type === 'video')) ? 'video/' : this.segments.some(segment => segment.tracks.some(x => x.info?.type === 'audio')) ? 'audio/' : 'application/'; let string = base + (this.isWebM ? 'webm' : 'x-matroska'); const tracks = await this.getTracks(); if (tracks.length > 0) { const codecMimeTypes = await Promise.all(tracks.map(x => x.getCodecParameterString())); const uniqueCodecMimeTypes = [...new Set(codecMimeTypes.filter(Boolean))]; string += `; codecs="${uniqueCodecMimeTypes.join(', ')}"`; } return string; } assertDefinedSize(size: number) { if (size === -1) { throw new Error('Undefined element size is used in a place where it is not supported.'); } } readMetadata() { return this.readMetadataPromise ??= (async () => { this.metadataReader.pos = 0; const fileSize = await this.input._source._getSize(); while (this.metadataReader.pos < fileSize - MIN_HEADER_SIZE) { await this.metadataReader.reader.loadRange( this.metadataReader.pos, this.metadataReader.pos + MAX_HEADER_SIZE, ); const { id, size } = this.metadataReader.readElementHeader(); const startPos = this.metadataReader.pos; if (id === EBMLId.EBML) { await this.metadataReader.reader.loadRange(this.metadataReader.pos, this.metadataReader.pos + size); this.readContiguousElements(this.metadataReader, size); } else if (id === EBMLId.Segment) { // Segment found! await this.readSegment(size); if (size === -1) { // Stop searching for other segments if this is the case break; } } this.assertDefinedSize(size); this.metadataReader.pos = startPos + size; } })(); } async readSegment(dataSize: number) { const segmentDataStart = this.metadataReader.pos; this.currentSegment = { seekHeadSeen: false, infoSeen: false, tracksSeen: false, cuesSeen: false, timestampScale: -1, timestampFactor: -1, duration: -1, seekEntries: [], tracks: [], cuePoints: [], dataStartPos: segmentDataStart, elementEndPos: dataSize === -1 ? (await this.input._source._getSize() - MIN_HEADER_SIZE) : segmentDataStart + dataSize, clusterSeekStartPos: segmentDataStart, clusters: [], clusterLookupMutex: new AsyncMutex(), }; this.segments.push(this.currentSegment); // Let's load a good amount of data, enough for all segment metadata to likely fit into (minus cues) await this.metadataReader.reader.loadRange( this.metadataReader.pos, this.metadataReader.pos + 2 ** 14, ); let clusterEncountered = false; while (this.metadataReader.pos < this.currentSegment.elementEndPos) { await this.metadataReader.reader.loadRange( this.metadataReader.pos, this.metadataReader.pos + MAX_HEADER_SIZE, ); const elementStartPos = this.metadataReader.pos; const { id, size } = this.metadataReader.readElementHeader(); const dataStartPos = this.metadataReader.pos; const metadataElementIndex = METADATA_ELEMENTS.findIndex(x => x.id === id); if (metadataElementIndex !== -1) { const field = METADATA_ELEMENTS[metadataElementIndex]!.flag; this.currentSegment[field] = true; await this.metadataReader.reader.loadRange(this.metadataReader.pos, this.metadataReader.pos + size); this.readContiguousElements(this.metadataReader, size); } else if (id === EBMLId.Cluster) { if (!clusterEncountered) { clusterEncountered = true; this.currentSegment.clusterSeekStartPos = elementStartPos; } } if (this.currentSegment.infoSeen && this.currentSegment.tracksSeen && this.currentSegment.cuesSeen) { // No need to search anymore, we have everything break; } if (this.currentSegment.seekHeadSeen) { let hasInfo = this.currentSegment.infoSeen; let hasTracks = this.currentSegment.tracksSeen; let hasCues = this.currentSegment.cuesSeen; for (const entry of this.currentSegment.seekEntries) { if (entry.id === EBMLId.Info) { hasInfo = true; } else if (entry.id === EBMLId.Tracks) { hasTracks = true; } else if (entry.id === EBMLId.Cues) { hasCues = true; } } if (hasInfo && hasTracks && hasCues) { // No need to search sequentially anymore, we can use the seek head break; } } this.assertDefinedSize(size); this.metadataReader.pos = dataStartPos + size; if (!clusterEncountered) { this.currentSegment.clusterSeekStartPos = this.metadataReader.pos; } } // Use the seek head to read missing metadata elements for (const target of METADATA_ELEMENTS) { if (this.currentSegment[target.flag]) continue; const seekEntry = this.currentSegment.seekEntries.find(entry => entry.id === target.id); if (!seekEntry) continue; this.metadataReader.pos = segmentDataStart + seekEntry.segmentPosition; await this.metadataReader.reader.loadRange( this.metadataReader.pos, this.metadataReader.pos + 2 ** 12, // Load a larger range, assuming the correct element will be there ); const { id, size } = this.metadataReader.readElementHeader(); if (id !== target.id) continue; this.currentSegment[target.flag] = true; await this.metadataReader.reader.loadRange(this.metadataReader.pos, this.metadataReader.pos + size); this.readContiguousElements(this.metadataReader, size); } // Put default tracks first this.currentSegment.tracks.sort((a, b) => Number(b.isDefault) - Number(a.isDefault)); // Sort cue points by cluster position (required for the next algorithm) this.currentSegment.cuePoints.sort((a, b) => a.clusterPosition - b.clusterPosition); // Now, let's distribute the cue points to each track. Ideally, each track has their own cue point, but some // Matroska files may only specify cue points for a single track. In this case, we still wanna use those cue // points for all tracks. const allTrackIds = this.currentSegment.tracks.map(x => x.id); const remainingTrackIds = new Set(); let lastClusterPosition: number | null = null; let lastCuePoint: CuePoint | null = null; for (const cuePoint of this.currentSegment.cuePoints) { if (cuePoint.clusterPosition !== lastClusterPosition) { for (const id of remainingTrackIds) { // These tracks didn't receive a cue point for the last cluster, so let's give them one assert(lastCuePoint); const track = this.currentSegment.tracks.find(x => x.id === id)!; track.cuePoints.push(lastCuePoint); } for (const id of allTrackIds) { remainingTrackIds.add(id); } } lastCuePoint = cuePoint; if (!remainingTrackIds.has(cuePoint.trackId)) { continue; } const track = this.currentSegment.tracks.find(x => x.id === cuePoint.trackId)!; track.cuePoints.push(cuePoint); remainingTrackIds.delete(cuePoint.trackId); lastClusterPosition = cuePoint.clusterPosition; } for (const id of remainingTrackIds) { assert(lastCuePoint); const track = this.currentSegment.tracks.find(x => x.id === id)!; track.cuePoints.push(lastCuePoint); } for (const track of this.currentSegment.tracks) { // Sort cue points by time track.cuePoints.sort((a, b) => a.time - b.time); } this.currentSegment = null; } async readCluster(segment: Segment) { await this.metadataReader.reader.loadRange(this.metadataReader.pos, this.metadataReader.pos + MAX_HEADER_SIZE); const elementStartPos = this.metadataReader.pos; const { id, size } = this.metadataReader.readElementHeader(); const dataStartPos = this.metadataReader.pos; assert(id === EBMLId.Cluster); // Load the entire cluster this.clusterReader.pos = dataStartPos; await this.clusterReader.reader.loadRange(this.clusterReader.pos, this.clusterReader.pos + size); const cluster: Cluster = { elementStartPos, elementEndPos: dataStartPos + size, dataStartPos, timestamp: -1, trackData: new Map(), nextCluster: null, isKnownToBeFirstCluster: false, }; this.currentCluster = cluster; this.readContiguousElements(this.clusterReader, size); for (const [trackId, trackData] of cluster.trackData) { let blockReferencesExist = false; // This must hold, as track datas only get created if a block for that track is encountered assert(trackData.blocks.length > 0); for (let i = 0; i < trackData.blocks.length; i++) { const block = trackData.blocks[i]!; block.timestamp += cluster.timestamp; blockReferencesExist ||= block.referencedTimestamps.length > 0; } if (blockReferencesExist) { trackData.blocks = sortBlocksByReferences(trackData.blocks); } trackData.presentationTimestamps = trackData.blocks .map((block, i) => ({ timestamp: block.timestamp, blockIndex: i })) .sort((a, b) => a.timestamp - b.timestamp); let hasKeyFrame = false; for (let i = 0; i < trackData.presentationTimestamps.length; i++) { const entry = trackData.presentationTimestamps[i]!; const block = trackData.blocks[entry.blockIndex]!; if (block.isKeyFrame) { hasKeyFrame = true; if (trackData.firstKeyFrameTimestamp === null && block.isKeyFrame) { trackData.firstKeyFrameTimestamp = block.timestamp; } } if (i < trackData.presentationTimestamps.length - 1) { // Update block durations based on presentation order const nextEntry = trackData.presentationTimestamps[i + 1]!; const nextBlock = trackData.blocks[nextEntry.blockIndex]!; block.duration = nextBlock.timestamp - block.timestamp; } } const firstBlock = trackData.blocks[trackData.presentationTimestamps[0]!.blockIndex]!; const lastBlock = trackData.blocks[last(trackData.presentationTimestamps)!.blockIndex]!; trackData.startTimestamp = firstBlock.timestamp; trackData.endTimestamp = lastBlock.timestamp + lastBlock.duration; const track = segment.tracks.find(x => x.id === trackId); if (track) { const insertionIndex = binarySearchLessOrEqual( track.clusters, cluster.timestamp, x => x.timestamp, ); track.clusters.splice(insertionIndex + 1, 0, cluster); if (hasKeyFrame) { const insertionIndex = binarySearchLessOrEqual( track.clustersWithKeyFrame, cluster.timestamp, x => x.timestamp, ); track.clustersWithKeyFrame.splice(insertionIndex + 1, 0, cluster); } } } const insertionIndex = binarySearchLessOrEqual( segment.clusters, elementStartPos, x => x.elementStartPos, ); segment.clusters.splice(insertionIndex + 1, 0, cluster); this.currentCluster = null; return cluster; } getTrackDataInCluster(cluster: Cluster, trackNumber: number) { let trackData = cluster.trackData.get(trackNumber); if (!trackData) { trackData = { startTimestamp: 0, endTimestamp: 0, firstKeyFrameTimestamp: null, blocks: [], presentationTimestamps: [], }; cluster.trackData.set(trackNumber, trackData); } return trackData; } readContiguousElements(reader: EBMLReader, totalSize: number) { const startIndex = reader.pos; while (reader.pos - startIndex < totalSize) { this.traverseElement(reader); } } traverseElement(reader: EBMLReader) { const { id, size } = reader.readElementHeader(); const dataStartPos = reader.pos; switch (id) { case EBMLId.DocType: { this.isWebM = reader.readString(size) === 'webm'; }; break; case EBMLId.Seek: { if (!this.currentSegment) break; const seekEntry: SeekEntry = { id: -1, segmentPosition: -1 }; this.currentSegment.seekEntries.push(seekEntry); this.readContiguousElements(reader, size); if (seekEntry.id === -1 || seekEntry.segmentPosition === -1) { this.currentSegment.seekEntries.pop(); } }; break; case EBMLId.SeekID: { const lastSeekEntry = this.currentSegment?.seekEntries[this.currentSegment.seekEntries.length - 1]; if (!lastSeekEntry) break; lastSeekEntry.id = reader.readUnsignedInt(size); }; break; case EBMLId.SeekPosition: { const lastSeekEntry = this.currentSegment?.seekEntries[this.currentSegment.seekEntries.length - 1]; if (!lastSeekEntry) break; lastSeekEntry.segmentPosition = reader.readUnsignedInt(size); }; break; case EBMLId.TimestampScale: { if (!this.currentSegment) break; this.currentSegment.timestampScale = reader.readUnsignedInt(size); this.currentSegment.timestampFactor = 1e9 / this.currentSegment.timestampScale; }; break; case EBMLId.Duration: { if (!this.currentSegment) break; this.currentSegment.duration = reader.readFloat(size); }; break; case EBMLId.TrackEntry: { if (!this.currentSegment) break; this.currentTrack = { id: -1, segment: this.currentSegment, demuxer: this, clusters: [], clustersWithKeyFrame: [], cuePoints: [], isDefault: false, inputTrack: null, codecId: null, codecPrivate: null, languageCode: UNDETERMINED_LANGUAGE, info: null, }; this.readContiguousElements(reader, size); if ( this.currentTrack && this.currentTrack.id !== -1 && this.currentTrack.codecId && this.currentTrack.info ) { const slashIndex = this.currentTrack.codecId.indexOf('/'); const codecIdWithoutSuffix = slashIndex === -1 ? this.currentTrack.codecId : this.currentTrack.codecId.slice(0, slashIndex); if ( this.currentTrack.info.type === 'video' && this.currentTrack.info.width !== -1 && this.currentTrack.info.height !== -1 ) { if (this.currentTrack.codecId === CODEC_STRING_MAP.avc) { this.currentTrack.info.codec = 'avc'; } else if (this.currentTrack.codecId === CODEC_STRING_MAP.hevc) { this.currentTrack.info.codec = 'hevc'; } else if (codecIdWithoutSuffix === CODEC_STRING_MAP.vp8) { this.currentTrack.info.codec = 'vp8'; } else if (codecIdWithoutSuffix === CODEC_STRING_MAP.vp9) { this.currentTrack.info.codec = 'vp9'; } else if (codecIdWithoutSuffix === CODEC_STRING_MAP.av1) { this.currentTrack.info.codec = 'av1'; } const videoTrack = this.currentTrack as InternalVideoTrack; const inputTrack = new InputVideoTrack(new MatroskaVideoTrackBacking(videoTrack)); this.currentTrack.inputTrack = inputTrack; this.currentSegment.tracks.push(this.currentTrack); } else if ( this.currentTrack.info.type === 'audio' && this.currentTrack.info.numberOfChannels !== -1 && this.currentTrack.info.sampleRate !== -1 ) { if (codecIdWithoutSuffix === CODEC_STRING_MAP.aac) { this.currentTrack.info.codec = 'aac'; this.currentTrack.info.aacCodecInfo = { isMpeg2: this.currentTrack.codecId.includes('MPEG2'), }; } else if (this.currentTrack.codecId === CODEC_STRING_MAP.mp3) { this.currentTrack.info.codec = 'mp3'; } else if (codecIdWithoutSuffix === CODEC_STRING_MAP.opus) { this.currentTrack.info.codec = 'opus'; } else if (codecIdWithoutSuffix === CODEC_STRING_MAP.vorbis) { this.currentTrack.info.codec = 'vorbis'; } else if (codecIdWithoutSuffix === CODEC_STRING_MAP.flac) { this.currentTrack.info.codec = 'flac'; } else if (this.currentTrack.codecId === 'A_PCM/INT/LIT') { if (this.currentTrack.info.bitDepth === 8) { this.currentTrack.info.codec = 'pcm-u8'; } else if (this.currentTrack.info.bitDepth === 16) { this.currentTrack.info.codec = 'pcm-s16'; } else if (this.currentTrack.info.bitDepth === 24) { this.currentTrack.info.codec = 'pcm-s24'; } else if (this.currentTrack.info.bitDepth === 32) { this.currentTrack.info.codec = 'pcm-s32'; } } else if (this.currentTrack.codecId === 'A_PCM/INT/BIG') { if (this.currentTrack.info.bitDepth === 8) { this.currentTrack.info.codec = 'pcm-u8'; } else if (this.currentTrack.info.bitDepth === 16) { this.currentTrack.info.codec = 'pcm-s16be'; } else if (this.currentTrack.info.bitDepth === 24) { this.currentTrack.info.codec = 'pcm-s24be'; } else if (this.currentTrack.info.bitDepth === 32) { this.currentTrack.info.codec = 'pcm-s32be'; } } else if (this.currentTrack.codecId === 'A_PCM/FLOAT/IEEE') { if (this.currentTrack.info.bitDepth === 32) { this.currentTrack.info.codec = 'pcm-f32'; } } const audioTrack = this.currentTrack as InternalAudioTrack; const inputTrack = new InputAudioTrack(new MatroskaAudioTrackBacking(audioTrack)); this.currentTrack.inputTrack = inputTrack; this.currentSegment.tracks.push(this.currentTrack); } } this.currentTrack = null; }; break; case EBMLId.TrackNumber: { if (!this.currentTrack) break; this.currentTrack.id = reader.readUnsignedInt(size); }; break; case EBMLId.TrackType: { if (!this.currentTrack) break; const type = reader.readUnsignedInt(size); if (type === 1) { this.currentTrack.info = { type: 'video', width: -1, height: -1, rotation: 0, codec: null, colorSpace: null, }; } else if (type === 2) { this.currentTrack.info = { type: 'audio', numberOfChannels: -1, sampleRate: -1, bitDepth: -1, codec: null, aacCodecInfo: null, }; } }; break; case EBMLId.FlagEnabled: { if (!this.currentTrack) break; const enabled = reader.readUnsignedInt(size); if (!enabled) { this.currentSegment!.tracks.pop(); this.currentTrack = null; } }; break; case EBMLId.FlagDefault: { if (!this.currentTrack) break; this.currentTrack.isDefault = !!reader.readUnsignedInt(size); }; break; case EBMLId.CodecID: { if (!this.currentTrack) break; this.currentTrack.codecId = reader.readString(size); }; break; case EBMLId.CodecPrivate: { if (!this.currentTrack) break; this.currentTrack.codecPrivate = reader.readBytes(size); }; break; case EBMLId.Language: { if (!this.currentTrack) break; this.currentTrack.languageCode = reader.readString(size); if (!isIso639Dash2LanguageCode(this.currentTrack.languageCode)) { this.currentTrack.languageCode = UNDETERMINED_LANGUAGE; } }; break; case EBMLId.Video: { if (this.currentTrack?.info?.type !== 'video') break; this.readContiguousElements(reader, size); }; break; case EBMLId.PixelWidth: { if (this.currentTrack?.info?.type !== 'video') break; this.currentTrack.info.width = reader.readUnsignedInt(size); }; break; case EBMLId.PixelHeight: { if (this.currentTrack?.info?.type !== 'video') break; this.currentTrack.info.height = reader.readUnsignedInt(size); }; break; case EBMLId.Colour: { if (this.currentTrack?.info?.type !== 'video') break; this.currentTrack.info.colorSpace = {}; this.readContiguousElements(reader, size); }; break; case EBMLId.MatrixCoefficients: { if (this.currentTrack?.info?.type !== 'video' || !this.currentTrack.info.colorSpace) break; const matrixCoefficients = reader.readUnsignedInt(size); const mapped = MATRIX_COEFFICIENTS_MAP_INVERSE[matrixCoefficients] ?? null; this.currentTrack.info.colorSpace.matrix = mapped as VideoColorSpaceInit['matrix']; }; break; case EBMLId.Range: { if (this.currentTrack?.info?.type !== 'video' || !this.currentTrack.info.colorSpace) break; this.currentTrack.info.colorSpace.fullRange = reader.readUnsignedInt(size) === 2; }; break; case EBMLId.TransferCharacteristics: { if (this.currentTrack?.info?.type !== 'video' || !this.currentTrack.info.colorSpace) break; const transferCharacteristics = reader.readUnsignedInt(size); const mapped = TRANSFER_CHARACTERISTICS_MAP_INVERSE[transferCharacteristics] ?? null; this.currentTrack.info.colorSpace.transfer = mapped as VideoColorSpaceInit['transfer']; }; break; case EBMLId.Primaries: { if (this.currentTrack?.info?.type !== 'video' || !this.currentTrack.info.colorSpace) break; const primaries = reader.readUnsignedInt(size); const mapped = COLOR_PRIMARIES_MAP_INVERSE[primaries] ?? null; this.currentTrack.info.colorSpace.primaries = mapped as VideoColorSpaceInit['primaries']; }; break; case EBMLId.Projection: { if (this.currentTrack?.info?.type !== 'video') break; this.readContiguousElements(reader, size); }; break; case EBMLId.ProjectionPoseRoll: { if (this.currentTrack?.info?.type !== 'video') break; const rotation = reader.readFloat(size); const flippedRotation = -rotation; // Convert counter-clockwise to clockwise try { this.currentTrack.info.rotation = normalizeRotation(flippedRotation); } catch { // It wasn't a valid rotation } }; break; case EBMLId.Audio: { if (this.currentTrack?.info?.type !== 'audio') break; this.readContiguousElements(reader, size); }; break; case EBMLId.SamplingFrequency: { if (this.currentTrack?.info?.type !== 'audio') break; this.currentTrack.info.sampleRate = reader.readFloat(size); }; break; case EBMLId.Channels: { if (this.currentTrack?.info?.type !== 'audio') break; this.currentTrack.info.numberOfChannels = reader.readUnsignedInt(size); }; break; case EBMLId.BitDepth: { if (this.currentTrack?.info?.type !== 'audio') break; this.currentTrack.info.bitDepth = reader.readUnsignedInt(size); }; break; case EBMLId.CuePoint: { if (!this.currentSegment) break; this.readContiguousElements(reader, size); this.currentCueTime = null; }; break; case EBMLId.CueTime: { this.currentCueTime = reader.readUnsignedInt(size); }; break; case EBMLId.CueTrackPositions: { if (this.currentCueTime === null) break; assert(this.currentSegment); const cuePoint: CuePoint = { time: this.currentCueTime, trackId: -1, clusterPosition: -1 }; this.currentSegment.cuePoints.push(cuePoint); this.readContiguousElements(reader, size); if (cuePoint.trackId === -1 || cuePoint.clusterPosition === -1) { this.currentSegment.cuePoints.pop(); } }; break; case EBMLId.CueTrack: { const lastCuePoint = this.currentSegment?.cuePoints[this.currentSegment.cuePoints.length - 1]; if (!lastCuePoint) break; lastCuePoint.trackId = reader.readUnsignedInt(size); }; break; case EBMLId.CueClusterPosition: { const lastCuePoint = this.currentSegment?.cuePoints[this.currentSegment.cuePoints.length - 1]; if (!lastCuePoint) break; assert(this.currentSegment); lastCuePoint.clusterPosition = this.currentSegment.dataStartPos + reader.readUnsignedInt(size); }; break; case EBMLId.Timestamp: { if (!this.currentCluster) break; this.currentCluster.timestamp = reader.readUnsignedInt(size); }; break; case EBMLId.SimpleBlock: { if (!this.currentCluster) break; const trackNumber = reader.readVarInt(); const relativeTimestamp = reader.readS16(); const flags = reader.readU8(); const isKeyFrame = !!(flags & 0x80); const trackData = this.getTrackDataInCluster(this.currentCluster, trackNumber); trackData.blocks.push({ timestamp: relativeTimestamp, // We'll add the cluster's timestamp to this later duration: 0, isKeyFrame, referencedTimestamps: [], data: reader.readBytes(size - (reader.pos - dataStartPos)), }); }; break; case EBMLId.BlockGroup: { if (!this.currentCluster) break; this.readContiguousElements(reader, size); if (this.currentBlock) { for (let i = 0; i < this.currentBlock.referencedTimestamps.length; i++) { this.currentBlock.referencedTimestamps[i]! += this.currentBlock.timestamp; } this.currentBlock = null; } }; break; case EBMLId.Block: { if (!this.currentCluster) break; const trackNumber = reader.readVarInt(); const relativeTimestamp = reader.readS16(); // eslint-disable-next-line @typescript-eslint/no-unused-vars const flags = reader.readU8(); const trackData = this.getTrackDataInCluster(this.currentCluster, trackNumber); this.currentBlock = { timestamp: relativeTimestamp, // We'll add the cluster's timestamp to this later duration: 0, isKeyFrame: true, referencedTimestamps: [], data: reader.readBytes(size - (reader.pos - dataStartPos)), }; trackData.blocks.push(this.currentBlock); }; break; case EBMLId.BlockDuration: { if (!this.currentBlock) break; this.currentBlock.duration = reader.readUnsignedInt(size); }; break; case EBMLId.ReferenceBlock: { if (!this.currentBlock) break; this.currentBlock.isKeyFrame = false; const relativeTimestamp = reader.readSignedInt(size); // We'll offset this by the block's timestamp later this.currentBlock.referencedTimestamps.push(relativeTimestamp); }; break; } this.assertDefinedSize(size); reader.pos = dataStartPos + size; } } abstract class MatroskaTrackBacking implements InputTrackBacking { packetToClusterLocation = new WeakMap(); constructor(public internalTrack: InternalTrack) {} getId() { return this.internalTrack.id; } getCodec(): MediaCodec | null { throw new Error('Not implemented on base class.'); } async computeDuration() { const lastPacket = await this.getPacket(Infinity, { metadataOnly: true }); return (lastPacket?.timestamp ?? 0) + (lastPacket?.duration ?? 0); } getLanguageCode() { return this.internalTrack.languageCode; } async getFirstTimestamp() { const firstPacket = await this.getFirstPacket({ metadataOnly: true }); return firstPacket?.timestamp ?? 0; } getTimeResolution() { return this.internalTrack.segment.timestampFactor; } async getFirstPacket(options: PacketRetrievalOptions) { return this.performClusterLookup( () => { const startCluster = this.internalTrack.segment.clusters[0] ?? null; if (startCluster?.isKnownToBeFirstCluster) { // Walk from the very first cluster in the file until we find one with our track in it let currentCluster: Cluster | null = startCluster; while (currentCluster) { const trackData = currentCluster.trackData.get(this.internalTrack.id); if (trackData) { return { clusterIndex: binarySearchExact( this.internalTrack.clusters, currentCluster.elementStartPos, x => x.elementStartPos, ), blockIndex: 0, correctBlockFound: true, }; } currentCluster = currentCluster.nextCluster; } } return { clusterIndex: -1, blockIndex: -1, correctBlockFound: false, }; }, -Infinity, // Use -Infinity as a search timestamp to avoid using the cues Infinity, options, ); } private intoTimescale(timestamp: number) { // Do a little rounding to catch cases where the result is very close to an integer. If it is, it's likely // that the number was originally an integer divided by the timescale. For stability, it's best // to return the integer in this case. return roundToPrecision(timestamp * this.internalTrack.segment.timestampFactor, 14); } async getPacket(timestamp: number, options: PacketRetrievalOptions) { const timestampInTimescale = this.intoTimescale(timestamp); return this.performClusterLookup( () => this.findBlockInClustersForTimestamp(timestampInTimescale), timestampInTimescale, timestampInTimescale, options, ); } async getNextPacket(packet: EncodedPacket, options: PacketRetrievalOptions) { const locationInCluster = this.packetToClusterLocation.get(packet); if (locationInCluster === undefined) { throw new Error('Packet was not created from this track.'); } const trackData = locationInCluster.cluster.trackData.get(this.internalTrack.id)!; const block = trackData.blocks[locationInCluster.blockIndex]!; const clusterIndex = binarySearchExact( this.internalTrack.clusters, locationInCluster.cluster.elementStartPos, x => x.elementStartPos, ); assert(clusterIndex !== -1); return this.performClusterLookup( () => { if (locationInCluster.blockIndex + 1 < trackData.blocks.length) { // We can simply take the next block in the cluster return { clusterIndex, blockIndex: locationInCluster.blockIndex + 1, correctBlockFound: true, }; } else { // Walk the list of clusters until we find the next cluster for this track let currentCluster = locationInCluster.cluster; while (currentCluster.nextCluster) { currentCluster = currentCluster.nextCluster; const trackData = currentCluster.trackData.get(this.internalTrack.id); if (trackData) { const clusterIndex = binarySearchExact( this.internalTrack.clusters, currentCluster.elementStartPos, x => x.elementStartPos, ); assert(clusterIndex !== -1); return { clusterIndex, blockIndex: 0, correctBlockFound: true, }; } } return { clusterIndex, blockIndex: -1, correctBlockFound: false, }; } }, block.timestamp, Infinity, options, ); } async getKeyPacket(timestamp: number, options: PacketRetrievalOptions) { const timestampInTimescale = this.intoTimescale(timestamp); return this.performClusterLookup( () => this.findKeyBlockInClustersForTimestamp(timestampInTimescale), timestampInTimescale, timestampInTimescale, options, ); } async getNextKeyPacket(packet: EncodedPacket, options: PacketRetrievalOptions) { const locationInCluster = this.packetToClusterLocation.get(packet); if (locationInCluster === undefined) { throw new Error('Packet was not created from this track.'); } const trackData = locationInCluster.cluster.trackData.get(this.internalTrack.id)!; const block = trackData.blocks[locationInCluster.blockIndex]!; const clusterIndex = binarySearchExact( this.internalTrack.clusters, locationInCluster.cluster.elementStartPos, x => x.elementStartPos, ); assert(clusterIndex !== -1); return this.performClusterLookup( () => { const nextKeyFrameIndex = trackData.blocks.findIndex( (x, i) => x.isKeyFrame && i > locationInCluster.blockIndex, ); if (nextKeyFrameIndex !== -1) { // We can simply take the next key frame in the cluster return { clusterIndex, blockIndex: nextKeyFrameIndex, correctBlockFound: true, }; } else { // Walk the list of clusters until we find the next cluster for this track let currentCluster = locationInCluster.cluster; while (currentCluster.nextCluster) { currentCluster = currentCluster.nextCluster; const trackData = currentCluster.trackData.get(this.internalTrack.id); if (trackData && trackData.firstKeyFrameTimestamp !== null) { const clusterIndex = binarySearchExact( this.internalTrack.clusters, currentCluster.elementStartPos, x => x.elementStartPos, ); assert(clusterIndex !== -1); const keyFrameIndex = trackData.blocks.findIndex(x => x.isKeyFrame); assert(keyFrameIndex !== -1); // There must be one return { clusterIndex, blockIndex: keyFrameIndex, correctBlockFound: true, }; } } return { clusterIndex, blockIndex: -1, correctBlockFound: false, }; } }, block.timestamp, Infinity, options, ); } private async fetchPacketInCluster(cluster: Cluster, blockIndex: number, options: PacketRetrievalOptions) { if (blockIndex === -1) { return null; } const trackData = cluster.trackData.get(this.internalTrack.id)!; const block = trackData.blocks[blockIndex]; assert(block); const data = options.metadataOnly ? PLACEHOLDER_DATA : block.data; const timestamp = block.timestamp / this.internalTrack.segment.timestampFactor; const duration = block.duration / this.internalTrack.segment.timestampFactor; const packet = new EncodedPacket( data, block.isKeyFrame ? 'key' : 'delta', timestamp, duration, cluster.dataStartPos + blockIndex, block.data.byteLength, ); this.packetToClusterLocation.set(packet, { cluster, blockIndex }); return packet; } private findBlockInClustersForTimestamp(timestampInTimescale: number) { const clusterIndex = binarySearchLessOrEqual( this.internalTrack.clusters, timestampInTimescale, x => x.trackData.get(this.internalTrack.id)!.startTimestamp, ); let blockIndex = -1; let correctBlockFound = false; if (clusterIndex !== -1) { const cluster = this.internalTrack.clusters[clusterIndex]!; const trackData = cluster.trackData.get(this.internalTrack.id)!; const index = binarySearchLessOrEqual( trackData.presentationTimestamps, timestampInTimescale, x => x.timestamp, ); assert(index !== -1); blockIndex = trackData.presentationTimestamps[index]!.blockIndex; correctBlockFound = timestampInTimescale < trackData.endTimestamp; } return { clusterIndex, blockIndex, correctBlockFound }; } private findKeyBlockInClustersForTimestamp(timestampInTimescale: number) { const indexInKeyFrameClusters = binarySearchLessOrEqual( this.internalTrack.clustersWithKeyFrame, timestampInTimescale, x => x.trackData.get(this.internalTrack.id)!.firstKeyFrameTimestamp!, ); let clusterIndex = -1; let blockIndex = -1; let correctBlockFound = false; if (indexInKeyFrameClusters !== -1) { const cluster = this.internalTrack.clustersWithKeyFrame[indexInKeyFrameClusters]!; // Now, let's find the actual index of the cluster in the list of ALL clusters, not just key frame ones clusterIndex = binarySearchExact( this.internalTrack.clusters, cluster.elementStartPos, x => x.elementStartPos, ); assert(clusterIndex !== -1); const trackData = cluster.trackData.get(this.internalTrack.id)!; const index = findLastIndex(trackData.presentationTimestamps, (x) => { const block = trackData.blocks[x.blockIndex]!; return block.isKeyFrame && x.timestamp <= timestampInTimescale; }); assert(index !== -1); // It's a key frame cluster, so there must be a key frame const entry = trackData.presentationTimestamps[index]!; blockIndex = entry.blockIndex; correctBlockFound = timestampInTimescale < trackData.endTimestamp; } return { clusterIndex, blockIndex, correctBlockFound }; } /** Looks for a packet in the clusters while trying to load as few clusters as possible to retrieve it. */ private async performClusterLookup( // This function returns the best-matching block that is currently loaded. Based on this information, we know // which clusters we need to load to find the actual match. getBestMatch: () => { clusterIndex: number; blockIndex: number; correctBlockFound: boolean }, // The timestamp with which we can search the lookup table searchTimestamp: number, // The timestamp for which we know the correct block will not come after it latestTimestamp: number, options: PacketRetrievalOptions, ): Promise { const { demuxer, segment } = this.internalTrack; const release = await segment.clusterLookupMutex.acquire(); // The algorithm requires exclusivity try { const { clusterIndex, blockIndex, correctBlockFound } = getBestMatch(); if (correctBlockFound) { // The correct block already exists, easy path. const cluster = this.internalTrack.clusters[clusterIndex]!; return this.fetchPacketInCluster(cluster, blockIndex, options); } // We use the metadata reader to find the cluster, but the cluster reader to load the cluster const metadataReader = demuxer.metadataReader; let prevCluster: Cluster | null = null; let bestClusterIndex = clusterIndex; let bestBlockIndex = blockIndex; // Search for a cue point; this way, we won't need to start searching from the start of the file // but can jump right into the correct cluster (or at least nearby). const cuePointIndex = binarySearchLessOrEqual( this.internalTrack.cuePoints, searchTimestamp, x => x.time, ); const cuePoint = cuePointIndex !== -1 ? this.internalTrack.cuePoints[cuePointIndex]! : null; let nextClusterIsFirstCluster = false; if (clusterIndex === -1) { metadataReader.pos = cuePoint?.clusterPosition ?? segment.clusterSeekStartPos; nextClusterIsFirstCluster = metadataReader.pos === segment.clusterSeekStartPos; } else { const cluster = this.internalTrack.clusters[clusterIndex]!; if (!cuePoint || cluster.elementStartPos >= cuePoint.clusterPosition) { metadataReader.pos = cluster.elementEndPos; prevCluster = cluster; } else { // Use the lookup entry metadataReader.pos = cuePoint.clusterPosition; } } while (metadataReader.pos < segment.elementEndPos) { if (prevCluster) { const trackData = prevCluster.trackData.get(this.internalTrack.id); if (trackData && trackData.startTimestamp > latestTimestamp) { // We're already past the upper bound, no need to keep searching break; } if (prevCluster.nextCluster) { // Skip ahead quickly without needing to read the file again metadataReader.pos = prevCluster.nextCluster.elementEndPos; prevCluster = prevCluster.nextCluster; continue; } } // Load the header await metadataReader.reader.loadRange(metadataReader.pos, metadataReader.pos + MAX_HEADER_SIZE); const elementStartPos = metadataReader.pos; const { id, size } = metadataReader.readElementHeader(); const dataStartPos = metadataReader.pos; if (id === EBMLId.Cluster) { const index = binarySearchExact(segment.clusters, elementStartPos, x => x.elementStartPos); let cluster: Cluster; if (index === -1) { // This is the first time we've seen this cluster metadataReader.pos = elementStartPos; cluster = await demuxer.readCluster(segment); } else { // We already know this cluster cluster = segment.clusters[index]!; } // Even if we already know the cluster, we might not yet know its predecessor, so always do this if (prevCluster) prevCluster.nextCluster = cluster; prevCluster = cluster; if (nextClusterIsFirstCluster) { cluster.isKnownToBeFirstCluster = true; nextClusterIsFirstCluster = false; } const { clusterIndex, blockIndex, correctBlockFound } = getBestMatch(); if (correctBlockFound) { const cluster = this.internalTrack.clusters[clusterIndex]!; return this.fetchPacketInCluster(cluster, blockIndex, options); } if (clusterIndex !== -1) { bestClusterIndex = clusterIndex; bestBlockIndex = blockIndex; } } metadataReader.pos = dataStartPos + size; } let result: EncodedPacket | null = null; const bestCluster = bestClusterIndex !== -1 ? this.internalTrack.clusters[bestClusterIndex]! : null; if (bestCluster) { // If we finished looping but didn't find a perfect match, still return the best match we found result = await this.fetchPacketInCluster(bestCluster, bestBlockIndex, options); } // Catch faulty cue points if (!result && cuePoint && (!bestCluster || bestCluster.elementStartPos < cuePoint.clusterPosition)) { // The cue point lied to us! We found a cue point but no cluster there that satisfied the match. In this // case, let's search again but using the cue point before that. const previousCuePoint = this.internalTrack.cuePoints[cuePointIndex - 1]; const newSearchTimestamp = previousCuePoint?.time ?? -Infinity; return this.performClusterLookup(getBestMatch, newSearchTimestamp, latestTimestamp, options); } return result; } finally { release(); } } } class MatroskaVideoTrackBacking extends MatroskaTrackBacking implements InputVideoTrackBacking { override internalTrack: InternalVideoTrack; decoderConfigPromise: Promise | null = null; constructor(internalTrack: InternalVideoTrack) { super(internalTrack); this.internalTrack = internalTrack; } override getCodec(): VideoCodec | null { return this.internalTrack.info.codec; } getCodedWidth() { return this.internalTrack.info.width; } getCodedHeight() { return this.internalTrack.info.height; } getRotation() { return this.internalTrack.info.rotation; } async getColorSpace(): Promise { return { primaries: this.internalTrack.info.colorSpace?.primaries, transfer: this.internalTrack.info.colorSpace?.transfer, matrix: this.internalTrack.info.colorSpace?.matrix, fullRange: this.internalTrack.info.colorSpace?.fullRange, }; } async getDecoderConfig(): Promise { if (!this.internalTrack.info.codec) { return null; } return this.decoderConfigPromise ??= (async (): Promise => { let firstPacket: EncodedPacket | null = null; const needsPacketForAdditionalInfo = this.internalTrack.info.codec === 'vp9' || this.internalTrack.info.codec === 'av1'; if (needsPacketForAdditionalInfo) { firstPacket = await this.getFirstPacket({}); } return { codec: extractVideoCodecString({ width: this.internalTrack.info.width, height: this.internalTrack.info.height, codec: this.internalTrack.info.codec, codecDescription: this.internalTrack.codecPrivate, colorSpace: this.internalTrack.info.colorSpace, vp9CodecInfo: this.internalTrack.info.codec === 'vp9' && firstPacket ? extractVp9CodecInfoFromFrame(firstPacket.data) : null, av1CodecInfo: this.internalTrack.info.codec === 'av1' && firstPacket ? extractAv1CodecInfoFromFrame(firstPacket.data) : null, }), codedWidth: this.internalTrack.info.width, codedHeight: this.internalTrack.info.height, description: this.internalTrack.codecPrivate ?? undefined, colorSpace: this.internalTrack.info.colorSpace ?? undefined, }; })(); } } class MatroskaAudioTrackBacking extends MatroskaTrackBacking implements InputAudioTrackBacking { override internalTrack: InternalAudioTrack; decoderConfig: AudioDecoderConfig | null = null; constructor(internalTrack: InternalAudioTrack) { super(internalTrack); this.internalTrack = internalTrack; } override getCodec(): AudioCodec | null { return this.internalTrack.info.codec; } getNumberOfChannels() { return this.internalTrack.info.numberOfChannels; } getSampleRate() { return this.internalTrack.info.sampleRate; } async getDecoderConfig(): Promise { if (!this.internalTrack.info.codec) { return null; } return this.decoderConfig ??= { codec: extractAudioCodecString({ codec: this.internalTrack.info.codec, codecDescription: this.internalTrack.codecPrivate, aacCodecInfo: this.internalTrack.info.aacCodecInfo, }), numberOfChannels: this.internalTrack.info.numberOfChannels, sampleRate: this.internalTrack.info.sampleRate, description: this.internalTrack.codecPrivate ?? undefined, }; } } /** Sorts blocks such that referenced blocks come before the blocks that reference them. */ const sortBlocksByReferences = (blocks: ClusterBlock[]) => { const timestampToBlock = new Map(); for (let i = 0; i < blocks.length; i++) { const block = blocks[i]!; timestampToBlock.set(block.timestamp, block); } const processedBlocks = new Set(); const result: ClusterBlock[] = []; const processBlock = (block: ClusterBlock) => { if (processedBlocks.has(block)) { return; } // Marking the block as processed here already; prevents this algorithm from dying on cycles processedBlocks.add(block); for (let j = 0; j < block.referencedTimestamps.length; j++) { const timestamp = block.referencedTimestamps[j]!; const otherBlock = timestampToBlock.get(timestamp); if (!otherBlock) { continue; } processBlock(otherBlock); } result.push(block); }; for (let i = 0; i < blocks.length; i++) { processBlock(blocks[i]!); } return result; };