From 2bec661985f8fa55a24a4f954a21a5dd5a13126e Mon Sep 17 00:00:00 2001 From: Vanilagy <1696106+Vanilagy@users.noreply.github.com> Date: Wed, 15 Jan 2025 20:00:29 +0100 Subject: [PATCH] Add setting track language code, add proper AV1 CodecPrivate for Matroska muxer --- dev/index.html | 6 ++-- src/codec.ts | 37 +++++++++++++++++++++- src/index.ts | 9 +++++- src/isobmff/isobmff-boxes.ts | 58 ++++++++++------------------------ src/matroska/ebml.ts | 3 ++ src/matroska/matroska-muxer.ts | 32 +++++++++++++++++++ src/output.ts | 34 ++++++++++++-------- 7 files changed, 120 insertions(+), 59 deletions(-) diff --git a/dev/index.html b/dev/index.html index ad720d0..efd9800 100644 --- a/dev/index.html +++ b/dev/index.html @@ -39,7 +39,7 @@ const context = canvas.getContext('2d'); let format = new Metamuxer.MkvOutputFormat({ streamable: false }); - //format = new Metamuxer.Mp4OutputFormat({ fastStart: 'fragmented' }); // new Metamuxer.MkvOutputFormat();// new Metamuxer.Mp4OutputFormat({ fastStart: false }); + format = new Metamuxer.Mp4OutputFormat({ fastStart: 'fragmented' }); // new Metamuxer.MkvOutputFormat();// new Metamuxer.Mp4OutputFormat({ fastStart: false }); let target = new Metamuxer.BufferTarget(); /* @@ -88,12 +88,12 @@ bitrate: 1e6 }); let audioSource = new Metamuxer.AudioBufferSource({ - codec: 'pcm-s16', + codec: 'opus', bitrate: 128e3, }); let subtitleSource = new Metamuxer.TextSubtitleSource('webvtt'); - output.addVideoTrack(videoSource); + output.addVideoTrack(videoSource, { languageCode: 'eng' }); output.addAudioTrack(audioSource); output.addSubtitleTrack(subtitleSource); diff --git a/src/codec.ts b/src/codec.ts index ffdd70b..522e22e 100644 --- a/src/codec.ts +++ b/src/codec.ts @@ -217,6 +217,42 @@ export const buildVideoCodecString = (codec: VideoCodec, width: number, height: throw new TypeError(`Unhandled codec '${codec}'.`); }; +export const generateAv1CodecConfigurationFromCodecString = (codecString: string) => { + // Reference: https://aomediacodec.github.io/av1-isobmff/ + + const parts = codecString.split('.'); // We can derive the required values from the codec string + + const marker = 1; + const version = 1; + const firstByte = (marker << 7) + version; + + const profile = Number(parts[1]); + const levelAndTier = parts[2]!; + const level = Number(levelAndTier.slice(0, -1)); + const secondByte = (profile << 5) + level; + + const tier = levelAndTier.slice(-1) === 'H' ? 1 : 0; + const bitDepth = Number(parts[3]); + const highBitDepth = bitDepth === 8 ? 0 : 1; + const twelveBit = 0; + const monochrome = parts[4] ? Number(parts[4]) : 0; + const chromaSubsamplingX = parts[5] ? Number(parts[5][0]) : 1; + const chromaSubsamplingY = parts[5] ? Number(parts[5][1]) : 1; + const chromaSamplePosition = parts[5] ? Number(parts[5][2]) : 0; // CSP_UNKNOWN + const thirdByte = (tier << 7) + + (highBitDepth << 6) + + (twelveBit << 5) + + (monochrome << 4) + + (chromaSubsamplingX << 3) + + (chromaSubsamplingY << 2) + + chromaSamplePosition; + + const initialPresentationDelayPresent = 0; // Should be fine + const fourthByte = initialPresentationDelayPresent; + + return [firstByte, secondByte, thirdByte, fourthByte]; +}; + export type Vp9CodecInfo = { profile: number; level: number; @@ -1364,7 +1400,6 @@ export const canEncodeAudio = async (codec: AudioCodec, { numberOfChannels = 2, }; /** @public */ - export const canEncodeSubtitles = async (codec: SubtitleCodec) => { if (!SUBTITLE_CODECS.includes(codec)) { return false; diff --git a/src/index.ts b/src/index.ts index 0813fe7..1e4c1c4 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,4 +1,11 @@ -export { Output, OutputOptions, VideoTrackMetadata, AudioTrackMetadata, SubtitleTrackMetadata } from './output'; +export { + Output, + OutputOptions, + BaseTrackMetadata, + VideoTrackMetadata, + AudioTrackMetadata, + SubtitleTrackMetadata, +} from './output'; export { OutputFormat, IsobmffOutputFormat, diff --git a/src/isobmff/isobmff-boxes.ts b/src/isobmff/isobmff-boxes.ts index 48ef072..b7f0449 100644 --- a/src/isobmff/isobmff-boxes.ts +++ b/src/isobmff/isobmff-boxes.ts @@ -12,7 +12,15 @@ import { IDENTITY_MATRIX, rotationMatrix, } from '../misc'; -import { AudioCodec, parsePcmCodec, PCM_CODECS, PcmAudioCodec, SubtitleCodec, VideoCodec } from '../codec'; +import { + AudioCodec, + generateAv1CodecConfigurationFromCodecString, + parsePcmCodec, + PCM_CODECS, + PcmAudioCodec, + SubtitleCodec, + VideoCodec, +} from '../codec'; import { formatSubtitleTimestamp } from '../subtitles'; import { Writer } from '../writer'; import { @@ -420,12 +428,18 @@ export const mdhd = ( const needsU64 = !isU32(creationTime) || !isU32(localDuration); const u32OrU64 = needsU64 ? u64 : u32; + let language = 0; + for (const character of (trackData.track.metadata.languageCode ?? 'und')) { + language <<= 5; + language += character.charCodeAt(0) - 0x60; + } + return fullBox('mdhd', +needsU64, 0, [ u32OrU64(creationTime), // Creation time u32OrU64(creationTime), // Modification time u32(trackData.timescale), // Timescale u32OrU64(localDuration), // Duration - u16(0b01010101_11000100), // Language ("und", undetermined) + u16(language), // Language u16(0), // Quality ]); }; @@ -653,45 +667,7 @@ export const vpcC = (trackData: IsobmffVideoTrackData) => { /** AV1 Configuration Box: Provides additional information to the decoder. */ export const av1C = (trackData: IsobmffVideoTrackData) => { - // Reference: https://aomediacodec.github.io/av1-isobmff/ - - const decoderConfig = trackData.info.decoderConfig; - const parts = decoderConfig.codec.split('.'); // We can derive the required values from the codec string - - const marker = 1; - const version = 1; - const firstByte = (marker << 7) + version; - - const profile = Number(parts[1]); - const levelAndTier = parts[2]!; - const level = Number(levelAndTier.slice(0, -1)); - const secondByte = (profile << 5) + level; - - const tier = levelAndTier.slice(-1) === 'H' ? 1 : 0; - const bitDepth = Number(parts[3]); - const highBitDepth = bitDepth === 8 ? 0 : 1; - const twelveBit = 0; - const monochrome = parts[4] ? Number(parts[4]) : 0; - const chromaSubsamplingX = parts[5] ? Number(parts[5][0]) : 1; - const chromaSubsamplingY = parts[5] ? Number(parts[5][1]) : 1; - const chromaSamplePosition = parts[5] ? Number(parts[5][2]) : 0; // CSP_UNKNOWN - const thirdByte = (tier << 7) - + (highBitDepth << 6) - + (twelveBit << 5) - + (monochrome << 4) - + (chromaSubsamplingX << 3) - + (chromaSubsamplingY << 2) - + chromaSamplePosition; - - const initialPresentationDelayPresent = 0; // Should be fine - const fourthByte = initialPresentationDelayPresent; - - return box('av1C', [ - firstByte, - secondByte, - thirdByte, - fourthByte, - ]); + return box('av1C', generateAv1CodecConfigurationFromCodecString(trackData.info.decoderConfig.codec)); }; /** Sound Sample Description Box: Contains information that defines how to interpret sound media data. */ diff --git a/src/matroska/ebml.ts b/src/matroska/ebml.ts index b590071..99c65c1 100644 --- a/src/matroska/ebml.ts +++ b/src/matroska/ebml.ts @@ -65,8 +65,11 @@ export enum EBMLId { FlagDefault = 0x88, FlagForced = 0x55aa, FlagLacing = 0x9c, + Language = 0x22b59c, CodecID = 0x86, CodecPrivate = 0x63a2, + CodecDelay = 0x56aa, + SeekPreRoll = 0x56bb, DefaultDuration = 0x23e383, Video = 0xe0, PixelWidth = 0xb0, diff --git a/src/matroska/matroska-muxer.ts b/src/matroska/matroska-muxer.ts index 96bc945..7c6f4a7 100644 --- a/src/matroska/matroska-muxer.ts +++ b/src/matroska/matroska-muxer.ts @@ -32,6 +32,7 @@ import { import { PCM_CODECS, PcmAudioCodec, + generateAv1CodecConfigurationFromCodecString, parsePcmCodec, validateAudioChunkMetadata, validateSubtitleMetadata, @@ -220,11 +221,30 @@ export class MatroskaMuxer extends Muxer { const codecId = CODEC_STRING_MAP[trackData.track.source._codec]; assert(codecId); + let seekPreRollNs = 0; + if (trackData.type === 'audio' && trackData.track.source._codec === 'opus') { + seekPreRollNs = 1e6 * 80; // In "Matroska ticks" (nanoseconds) + + const description = trackData.info.decoderConfig.description; + if (description) { + const view = ArrayBuffer.isView(description) + ? new DataView(description.buffer, description.byteOffset, description.byteLength) + : new DataView(description); + + // Read the value from the Opus Identification Header + seekPreRollNs = Math.round(1e9 * (view.getUint16(10, true) / 48000)); + } + } + tracksElement.data.push({ id: EBMLId.TrackEntry, data: [ { id: EBMLId.TrackNumber, data: trackData.track.id }, { id: EBMLId.TrackUID, data: trackData.track.id }, { id: EBMLId.TrackType, data: TRACK_TYPE_MAP[trackData.type] }, + { id: EBMLId.FlagLacing, data: 0 }, + { id: EBMLId.Language, data: trackData.track.metadata.languageCode ?? 'und' }, { id: EBMLId.CodecID, data: codecId }, + { id: EBMLId.CodecDelay, data: 0 }, + { id: EBMLId.SeekPreRoll, data: seekPreRollNs }, (trackData.type === 'video' ? this.videoSpecificTrackInfo(trackData) : null), (trackData.type === 'audio' ? this.audioSpecificTrackInfo(trackData) : null), (trackData.type === 'subtitle' ? this.subtitleSpecificTrackInfo(trackData) : null), @@ -381,6 +401,18 @@ export class MatroskaMuxer extends Muxer { lastWrittenMsTimestamp: null, }; + if (track.source._codec === 'av1') { + // Per https://github.com/ietf-wg-cellar/matroska-specification/blob/master/codec/av1.md, AV1 requires + // CodecPrivate to be set, but WebCodecs makes no use of the description field for AV1. Thus, let's derive + // it ourselves: + newTrackData.info.decoderConfig = { + ...newTrackData.info.decoderConfig, + description: new Uint8Array( + generateAv1CodecConfigurationFromCodecString(newTrackData.info.decoderConfig.codec), + ), + }; + } + this.trackDatas.push(newTrackData); this.trackDatas.sort((a, b) => a.track.id - b.track.id); diff --git a/src/output.ts b/src/output.ts index 2458332..dd648df 100644 --- a/src/output.ts +++ b/src/output.ts @@ -33,14 +33,28 @@ export type OutputAudioTrack = OutputTrack & { type: 'audio' }; export type OutputSubtitleTrack = OutputTrack & { type: 'subtitle' }; /** @public */ -export type VideoTrackMetadata = { - rotation?: 0 | 90 | 180 | 270 | TransformationMatrix; // TODO respect this field for Matroska +export type BaseTrackMetadata = { + languageCode?: string; +}; + +/** @public */ +export type VideoTrackMetadata = BaseTrackMetadata & { + rotation?: 0 | 90 | 180 | 270 | TransformationMatrix; frameRate?: number; }; /** @public */ -export type AudioTrackMetadata = {}; +export type AudioTrackMetadata = BaseTrackMetadata & {}; /** @public */ -export type SubtitleTrackMetadata = {}; +export type SubtitleTrackMetadata = BaseTrackMetadata & {}; + +const validateBaseTrackMetadata = (metadata: BaseTrackMetadata) => { + if (!metadata || typeof metadata !== 'object') { + throw new TypeError('metadata must be an object.'); + } + if (metadata.languageCode !== undefined && !/^[a-z]{3}$/.test(metadata.languageCode)) { + throw new TypeError('metadata.languageCode must be a three-letter, ISO 639-2 language code.'); + } +}; /** @public */ export class Output { @@ -90,9 +104,7 @@ export class Output { if (!(source instanceof VideoSource)) { throw new TypeError('source must be a VideoSource.'); } - if (!metadata || typeof metadata !== 'object') { - throw new TypeError('metadata must be an object.'); - } + validateBaseTrackMetadata(metadata); if (typeof metadata.rotation === 'number' && ![0, 90, 180, 270].includes(metadata.rotation)) { throw new TypeError(`Invalid video rotation: ${metadata.rotation}. Has to be 0, 90, 180 or 270.`); } else if ( @@ -117,9 +129,7 @@ export class Output { if (!(source instanceof AudioSource)) { throw new TypeError('source must be an AudioSource.'); } - if (!metadata || typeof metadata !== 'object') { - throw new TypeError('metadata must be an object.'); - } + validateBaseTrackMetadata(metadata); this._addTrack('audio', source, metadata); } @@ -128,9 +138,7 @@ export class Output { if (!(source instanceof SubtitleSource)) { throw new TypeError('source must be a SubtitleSource.'); } - if (!metadata || typeof metadata !== 'object') { - throw new TypeError('metadata must be an object.'); - } + validateBaseTrackMetadata(metadata); this._addTrack('subtitle', source, metadata); }