Add setting track language code, add proper AV1 CodecPrivate for Matroska muxer

dev/index.html

@@ -39,7 +39,7 @@
 	const context = canvas.getContext('2d');
 
 	let format = new Metamuxer.MkvOutputFormat({ streamable: false });
-	//format = new Metamuxer.Mp4OutputFormat({ fastStart: 'fragmented' }); // new Metamuxer.MkvOutputFormat();// new Metamuxer.Mp4OutputFormat({ fastStart: false });
+	format = new Metamuxer.Mp4OutputFormat({ fastStart: 'fragmented' }); // new Metamuxer.MkvOutputFormat();// new Metamuxer.Mp4OutputFormat({ fastStart: false });
 	let target = new Metamuxer.BufferTarget();
 
 	/*
@@ -88,12 +88,12 @@
 		bitrate: 1e6
 	});
 	let audioSource = new Metamuxer.AudioBufferSource({
-		codec: 'pcm-s16',
+		codec: 'opus',
 		bitrate: 128e3,
 	});
 	let subtitleSource = new Metamuxer.TextSubtitleSource('webvtt');
 
-	output.addVideoTrack(videoSource);
+	output.addVideoTrack(videoSource, { languageCode: 'eng' });
 	output.addAudioTrack(audioSource);
 	output.addSubtitleTrack(subtitleSource);
 

src/codec.ts

@@ -217,6 +217,42 @@ export const buildVideoCodecString = (codec: VideoCodec, width: number, height:
 	throw new TypeError(`Unhandled codec '${codec}'.`);
 };
 
+export const generateAv1CodecConfigurationFromCodecString = (codecString: string) => {
+	// Reference: https://aomediacodec.github.io/av1-isobmff/
+
+	const parts = codecString.split('.'); // We can derive the required values from the codec string
+
+	const marker = 1;
+	const version = 1;
+	const firstByte = (marker << 7) + version;
+
+	const profile = Number(parts[1]);
+	const levelAndTier = parts[2]!;
+	const level = Number(levelAndTier.slice(0, -1));
+	const secondByte = (profile << 5) + level;
+
+	const tier = levelAndTier.slice(-1) === 'H' ? 1 : 0;
+	const bitDepth = Number(parts[3]);
+	const highBitDepth = bitDepth === 8 ? 0 : 1;
+	const twelveBit = 0;
+	const monochrome = parts[4] ? Number(parts[4]) : 0;
+	const chromaSubsamplingX = parts[5] ? Number(parts[5][0]) : 1;
+	const chromaSubsamplingY = parts[5] ? Number(parts[5][1]) : 1;
+	const chromaSamplePosition = parts[5] ? Number(parts[5][2]) : 0; // CSP_UNKNOWN
+	const thirdByte = (tier << 7)
+		+ (highBitDepth << 6)
+		+ (twelveBit << 5)
+		+ (monochrome << 4)
+		+ (chromaSubsamplingX << 3)
+		+ (chromaSubsamplingY << 2)
+		+ chromaSamplePosition;
+
+	const initialPresentationDelayPresent = 0; // Should be fine
+	const fourthByte = initialPresentationDelayPresent;
+
+	return [firstByte, secondByte, thirdByte, fourthByte];
+};
+
 export type Vp9CodecInfo = {
 	profile: number;
 	level: number;
@@ -1364,7 +1400,6 @@ export const canEncodeAudio = async (codec: AudioCodec, { numberOfChannels = 2,
 };
 
 /** @public */
-
 export const canEncodeSubtitles = async (codec: SubtitleCodec) => {
 	if (!SUBTITLE_CODECS.includes(codec)) {
 		return false;

src/index.ts

@@ -1,4 +1,11 @@
-export { Output, OutputOptions, VideoTrackMetadata, AudioTrackMetadata, SubtitleTrackMetadata } from './output';
+export {
+	Output,
+	OutputOptions,
+	BaseTrackMetadata,
+	VideoTrackMetadata,
+	AudioTrackMetadata,
+	SubtitleTrackMetadata,
+} from './output';
 export {
 	OutputFormat,
 	IsobmffOutputFormat,

src/isobmff/isobmff-boxes.ts

@@ -12,7 +12,15 @@ import {
 	IDENTITY_MATRIX,
 	rotationMatrix,
 } from '../misc';
-import { AudioCodec, parsePcmCodec, PCM_CODECS, PcmAudioCodec, SubtitleCodec, VideoCodec } from '../codec';
+import {
+	AudioCodec,
+	generateAv1CodecConfigurationFromCodecString,
+	parsePcmCodec,
+	PCM_CODECS,
+	PcmAudioCodec,
+	SubtitleCodec,
+	VideoCodec,
+} from '../codec';
 import { formatSubtitleTimestamp } from '../subtitles';
 import { Writer } from '../writer';
 import {
@@ -420,12 +428,18 @@ export const mdhd = (
 	const needsU64 = !isU32(creationTime) || !isU32(localDuration);
 	const u32OrU64 = needsU64 ? u64 : u32;
 
+	let language = 0;
+	for (const character of (trackData.track.metadata.languageCode ?? 'und')) {
+		language <<= 5;
+		language += character.charCodeAt(0) - 0x60;
+	}
+
 	return fullBox('mdhd', +needsU64, 0, [
 		u32OrU64(creationTime), // Creation time
 		u32OrU64(creationTime), // Modification time
 		u32(trackData.timescale), // Timescale
 		u32OrU64(localDuration), // Duration
-		u16(0b01010101_11000100), // Language ("und", undetermined)
+		u16(language), // Language
 		u16(0), // Quality
 	]);
 };
@@ -653,45 +667,7 @@ export const vpcC = (trackData: IsobmffVideoTrackData) => {
 
 /** AV1 Configuration Box: Provides additional information to the decoder. */
 export const av1C = (trackData: IsobmffVideoTrackData) => {
-	// Reference: https://aomediacodec.github.io/av1-isobmff/
+	return box('av1C', generateAv1CodecConfigurationFromCodecString(trackData.info.decoderConfig.codec));
-
-	const decoderConfig = trackData.info.decoderConfig;
-	const parts = decoderConfig.codec.split('.'); // We can derive the required values from the codec string
-
-	const marker = 1;
-	const version = 1;
-	const firstByte = (marker << 7) + version;
-
-	const profile = Number(parts[1]);
-	const levelAndTier = parts[2]!;
-	const level = Number(levelAndTier.slice(0, -1));
-	const secondByte = (profile << 5) + level;
-
-	const tier = levelAndTier.slice(-1) === 'H' ? 1 : 0;
-	const bitDepth = Number(parts[3]);
-	const highBitDepth = bitDepth === 8 ? 0 : 1;
-	const twelveBit = 0;
-	const monochrome = parts[4] ? Number(parts[4]) : 0;
-	const chromaSubsamplingX = parts[5] ? Number(parts[5][0]) : 1;
-	const chromaSubsamplingY = parts[5] ? Number(parts[5][1]) : 1;
-	const chromaSamplePosition = parts[5] ? Number(parts[5][2]) : 0; // CSP_UNKNOWN
-	const thirdByte = (tier << 7)
-		+ (highBitDepth << 6)
-		+ (twelveBit << 5)
-		+ (monochrome << 4)
-		+ (chromaSubsamplingX << 3)
-		+ (chromaSubsamplingY << 2)
-		+ chromaSamplePosition;
-
-	const initialPresentationDelayPresent = 0; // Should be fine
-	const fourthByte = initialPresentationDelayPresent;
-
-	return box('av1C', [
-		firstByte,
-		secondByte,
-		thirdByte,
-		fourthByte,
-	]);
 };
 
 /** Sound Sample Description Box: Contains information that defines how to interpret sound media data. */

src/matroska/ebml.ts

@@ -65,8 +65,11 @@ export enum EBMLId {
 	FlagDefault = 0x88,
 	FlagForced = 0x55aa,
 	FlagLacing = 0x9c,
+	Language = 0x22b59c,
 	CodecID = 0x86,
 	CodecPrivate = 0x63a2,
+	CodecDelay = 0x56aa,
+	SeekPreRoll = 0x56bb,
 	DefaultDuration = 0x23e383,
 	Video = 0xe0,
 	PixelWidth = 0xb0,

src/matroska/matroska-muxer.ts

@@ -32,6 +32,7 @@ import {
 import {
 	PCM_CODECS,
 	PcmAudioCodec,
+	generateAv1CodecConfigurationFromCodecString,
 	parsePcmCodec,
 	validateAudioChunkMetadata,
 	validateSubtitleMetadata,
@@ -220,11 +221,30 @@ export class MatroskaMuxer extends Muxer {
 			const codecId = CODEC_STRING_MAP[trackData.track.source._codec];
 			assert(codecId);
 
+			let seekPreRollNs = 0;
+			if (trackData.type === 'audio' && trackData.track.source._codec === 'opus') {
+				seekPreRollNs = 1e6 * 80; // In "Matroska ticks" (nanoseconds)
+
+				const description = trackData.info.decoderConfig.description;
+				if (description) {
+					const view = ArrayBuffer.isView(description)
+						? new DataView(description.buffer, description.byteOffset, description.byteLength)
+						: new DataView(description);
+
+					// Read the value from the Opus Identification Header
+					seekPreRollNs = Math.round(1e9 * (view.getUint16(10, true) / 48000));
+				}
+			}
+
 			tracksElement.data.push({ id: EBMLId.TrackEntry, data: [
 				{ id: EBMLId.TrackNumber, data: trackData.track.id },
 				{ id: EBMLId.TrackUID, data: trackData.track.id },
 				{ id: EBMLId.TrackType, data: TRACK_TYPE_MAP[trackData.type] },
+				{ id: EBMLId.FlagLacing, data: 0 },
+				{ id: EBMLId.Language, data: trackData.track.metadata.languageCode ?? 'und' },
 				{ id: EBMLId.CodecID, data: codecId },
+				{ id: EBMLId.CodecDelay, data: 0 },
+				{ id: EBMLId.SeekPreRoll, data: seekPreRollNs },
 				(trackData.type === 'video' ? this.videoSpecificTrackInfo(trackData) : null),
 				(trackData.type === 'audio' ? this.audioSpecificTrackInfo(trackData) : null),
 				(trackData.type === 'subtitle' ? this.subtitleSpecificTrackInfo(trackData) : null),
@@ -381,6 +401,18 @@ export class MatroskaMuxer extends Muxer {
 			lastWrittenMsTimestamp: null,
 		};
 
+		if (track.source._codec === 'av1') {
+			// Per https://github.com/ietf-wg-cellar/matroska-specification/blob/master/codec/av1.md, AV1 requires
+			// CodecPrivate to be set, but WebCodecs makes no use of the description field for AV1. Thus, let's derive
+			// it ourselves:
+			newTrackData.info.decoderConfig = {
+				...newTrackData.info.decoderConfig,
+				description: new Uint8Array(
+					generateAv1CodecConfigurationFromCodecString(newTrackData.info.decoderConfig.codec),
+				),
+			};
+		}
+
 		this.trackDatas.push(newTrackData);
 		this.trackDatas.sort((a, b) => a.track.id - b.track.id);
 

src/output.ts

@@ -33,14 +33,28 @@ export type OutputAudioTrack = OutputTrack & { type: 'audio' };
 export type OutputSubtitleTrack = OutputTrack & { type: 'subtitle' };
 
 /** @public */
-export type VideoTrackMetadata = {
+export type BaseTrackMetadata = {
-	rotation?: 0 | 90 | 180 | 270 | TransformationMatrix; // TODO respect this field for Matroska
+	languageCode?: string;
+};
+
+/** @public */
+export type VideoTrackMetadata = BaseTrackMetadata & {
+	rotation?: 0 | 90 | 180 | 270 | TransformationMatrix;
 	frameRate?: number;
 };
 /** @public */
-export type AudioTrackMetadata = {};
+export type AudioTrackMetadata = BaseTrackMetadata & {};
 /** @public */
-export type SubtitleTrackMetadata = {};
+export type SubtitleTrackMetadata = BaseTrackMetadata & {};
+
+const validateBaseTrackMetadata = (metadata: BaseTrackMetadata) => {
+	if (!metadata || typeof metadata !== 'object') {
+		throw new TypeError('metadata must be an object.');
+	}
+	if (metadata.languageCode !== undefined && !/^[a-z]{3}$/.test(metadata.languageCode)) {
+		throw new TypeError('metadata.languageCode must be a three-letter, ISO 639-2 language code.');
+	}
+};
 
 /** @public */
 export class Output {
@@ -90,9 +104,7 @@ export class Output {
 		if (!(source instanceof VideoSource)) {
 			throw new TypeError('source must be a VideoSource.');
 		}
-		if (!metadata || typeof metadata !== 'object') {
+		validateBaseTrackMetadata(metadata);
-			throw new TypeError('metadata must be an object.');
-		}
 		if (typeof metadata.rotation === 'number' && ![0, 90, 180, 270].includes(metadata.rotation)) {
 			throw new TypeError(`Invalid video rotation: ${metadata.rotation}. Has to be 0, 90, 180 or 270.`);
 		} else if (
@@ -117,9 +129,7 @@ export class Output {
 		if (!(source instanceof AudioSource)) {
 			throw new TypeError('source must be an AudioSource.');
 		}
-		if (!metadata || typeof metadata !== 'object') {
+		validateBaseTrackMetadata(metadata);
-			throw new TypeError('metadata must be an object.');
-		}
 
 		this._addTrack('audio', source, metadata);
 	}
@@ -128,9 +138,7 @@ export class Output {
 		if (!(source instanceof SubtitleSource)) {
 			throw new TypeError('source must be a SubtitleSource.');
 		}
-		if (!metadata || typeof metadata !== 'object') {
+		validateBaseTrackMetadata(metadata);
-			throw new TypeError('metadata must be an object.');
-		}
 
 		this._addTrack('subtitle', source, metadata);
 	}