From 1652b4f61462577d156002971667d0422a12d7db Mon Sep 17 00:00:00 2001 From: Vanilagy <1696106+Vanilagy@users.noreply.github.com> Date: Sat, 22 Mar 2025 14:50:35 +0100 Subject: [PATCH] More intentional codec description extraction for Matroska, & generate CodecPrivate for VP9 --- src/codec.ts | 18 ++++++++++++++++++ src/matroska/matroska-demuxer.ts | 18 ++++++++++++++---- src/matroska/matroska-muxer.ts | 12 +++++++++++- todo.txt | 3 +-- 4 files changed, 44 insertions(+), 7 deletions(-) diff --git a/src/codec.ts b/src/codec.ts index 6ffb865..30bba15 100644 --- a/src/codec.ts +++ b/src/codec.ts @@ -253,6 +253,24 @@ export const buildVideoCodecString = (codec: VideoCodec, width: number, height: throw new TypeError(`Unhandled codec '${codec}'.`); }; +export const generateVp9CodecConfigurationFromCodecString = (codecString: string) => { + // Reference: https://www.webmproject.org/docs/container/#vp9-codec-feature-metadata-codecprivate + + const parts = codecString.split('.'); // We can derive the required values from the codec string + + const profile = Number(parts[1]); + const level = Number(parts[2]); + const bitDepth = Number(parts[3]); + const chromaSubsampling = parts[4] ? Number(parts[4]) : 1; + + return [ + 1, 1, profile, + 2, 1, level, + 3, 1, bitDepth, + 4, 1, chromaSubsampling, + ]; +}; + export const generateAv1CodecConfigurationFromCodecString = (codecString: string) => { // Reference: https://aomediacodec.github.io/av1-isobmff/ diff --git a/src/matroska/matroska-demuxer.ts b/src/matroska/matroska-demuxer.ts index 9395cdc..c2afde4 100644 --- a/src/matroska/matroska-demuxer.ts +++ b/src/matroska/matroska-demuxer.ts @@ -121,6 +121,7 @@ type InternalTrack = { height: number; rotation: Rotation; codec: VideoCodec | null; + codecDescription: Uint8Array | null; colorSpace: VideoColorSpaceInit | null; } | { @@ -129,6 +130,7 @@ type InternalTrack = { sampleRate: number; bitDepth: number; codec: AudioCodec | null; + codecDescription: Uint8Array | null; aacCodecInfo: AacCodecInfo | null; }; }; @@ -620,8 +622,10 @@ export class MatroskaDemuxer extends Demuxer { ) { if (this.currentTrack.codecId === CODEC_STRING_MAP.avc) { this.currentTrack.info.codec = 'avc'; + this.currentTrack.info.codecDescription = this.currentTrack.codecPrivate; } else if (this.currentTrack.codecId === CODEC_STRING_MAP.hevc) { this.currentTrack.info.codec = 'hevc'; + this.currentTrack.info.codecDescription = this.currentTrack.codecPrivate; } else if (codecIdWithoutSuffix === CODEC_STRING_MAP.vp8) { this.currentTrack.info.codec = 'vp8'; } else if (codecIdWithoutSuffix === CODEC_STRING_MAP.vp9) { @@ -644,14 +648,18 @@ export class MatroskaDemuxer extends Demuxer { this.currentTrack.info.aacCodecInfo = { isMpeg2: this.currentTrack.codecId.includes('MPEG2'), }; + this.currentTrack.info.codecDescription = this.currentTrack.codecPrivate; } else if (this.currentTrack.codecId === CODEC_STRING_MAP.mp3) { this.currentTrack.info.codec = 'mp3'; } else if (codecIdWithoutSuffix === CODEC_STRING_MAP.opus) { this.currentTrack.info.codec = 'opus'; + this.currentTrack.info.codecDescription = this.currentTrack.codecPrivate; } else if (codecIdWithoutSuffix === CODEC_STRING_MAP.vorbis) { this.currentTrack.info.codec = 'vorbis'; + this.currentTrack.info.codecDescription = this.currentTrack.codecPrivate; } else if (codecIdWithoutSuffix === CODEC_STRING_MAP.flac) { this.currentTrack.info.codec = 'flac'; + this.currentTrack.info.codecDescription = this.currentTrack.codecPrivate; } else if (this.currentTrack.codecId === 'A_PCM/INT/LIT') { if (this.currentTrack.info.bitDepth === 8) { this.currentTrack.info.codec = 'pcm-u8'; @@ -705,6 +713,7 @@ export class MatroskaDemuxer extends Demuxer { height: -1, rotation: 0, codec: null, + codecDescription: null, colorSpace: null, }; } else if (type === 2) { @@ -714,6 +723,7 @@ export class MatroskaDemuxer extends Demuxer { sampleRate: -1, bitDepth: -1, codec: null, + codecDescription: null, aacCodecInfo: null, }; } @@ -1484,7 +1494,7 @@ class MatroskaVideoTrackBacking extends MatroskaTrackBacking implements InputVid width: this.internalTrack.info.width, height: this.internalTrack.info.height, codec: this.internalTrack.info.codec, - codecDescription: this.internalTrack.codecPrivate, + codecDescription: this.internalTrack.info.codecDescription, colorSpace: this.internalTrack.info.colorSpace, vp9CodecInfo: this.internalTrack.info.codec === 'vp9' && firstPacket ? extractVp9CodecInfoFromFrame(firstPacket.data) @@ -1495,7 +1505,7 @@ class MatroskaVideoTrackBacking extends MatroskaTrackBacking implements InputVid }), codedWidth: this.internalTrack.info.width, codedHeight: this.internalTrack.info.height, - description: this.internalTrack.codecPrivate ?? undefined, + description: this.internalTrack.info.codecDescription ?? undefined, colorSpace: this.internalTrack.info.colorSpace ?? undefined, }; })(); @@ -1531,12 +1541,12 @@ class MatroskaAudioTrackBacking extends MatroskaTrackBacking implements InputAud return this.decoderConfig ??= { codec: extractAudioCodecString({ codec: this.internalTrack.info.codec, - codecDescription: this.internalTrack.codecPrivate, + codecDescription: this.internalTrack.info.codecDescription, aacCodecInfo: this.internalTrack.info.aacCodecInfo, }), numberOfChannels: this.internalTrack.info.numberOfChannels, sampleRate: this.internalTrack.info.sampleRate, - description: this.internalTrack.codecPrivate ?? undefined, + description: this.internalTrack.info.codecDescription ?? undefined, }; } } diff --git a/src/matroska/matroska-muxer.ts b/src/matroska/matroska-muxer.ts index 14db5ca..3139e0f 100644 --- a/src/matroska/matroska-muxer.ts +++ b/src/matroska/matroska-muxer.ts @@ -36,6 +36,7 @@ import { PCM_AUDIO_CODECS, PcmAudioCodec, generateAv1CodecConfigurationFromCodecString, + generateVp9CodecConfigurationFromCodecString, parseOpusIdentificationHeader, parsePcmCodec, validateAudioChunkMetadata, @@ -408,7 +409,16 @@ export class MatroskaMuxer extends Muxer { lastWrittenMsTimestamp: null, }; - if (track.source._codec === 'av1') { + if (track.source._codec === 'vp9') { + // https://www.webmproject.org/docs/container specifies that VP9 "SHOULD" make use of the CodecPrivate + // field. Since WebCodecs makes no use of the description field for VP9, we need to derive it ourselves: + newTrackData.info.decoderConfig = { + ...newTrackData.info.decoderConfig, + description: new Uint8Array( + generateVp9CodecConfigurationFromCodecString(newTrackData.info.decoderConfig.codec), + ), + }; + } else if (track.source._codec === 'av1') { // Per https://github.com/ietf-wg-cellar/matroska-specification/blob/master/codec/av1.md, AV1 requires // CodecPrivate to be set, but WebCodecs makes no use of the description field for AV1. Thus, let's derive // it ourselves: diff --git a/todo.txt b/todo.txt index abc3ac6..07318e7 100644 --- a/todo.txt +++ b/todo.txt @@ -3,5 +3,4 @@ - is this fixed? https://github.com/Vanilagy/webm-muxer/issues/50 - cross-track offset for streaming sources - configurable fragmented mp4 fragment size, like the mp4-muxer PR -- textsubtitlesource, chunked piping -- for matroska demuxing, don't blindly pipe codecPrivate into description. This should be if-else'd on a per-codec basis, for the codecs that actually need it \ No newline at end of file +- textsubtitlesource, chunked piping \ No newline at end of file