Add VP8, VP9 and AV1 support to MP4 demuxer

dev/demux.html

@@ -16,6 +16,40 @@
 		const videoTrack = await input.getPrimaryVideoTrack();
 		const drain = new Metamuxer.VideoFrameDrain(videoTrack);
 
+		const target = new Metamuxer.ArrayBufferTarget();
+		const format = new Metamuxer.Mp4OutputFormat();
+		const output = new Metamuxer.Output({ target, format });
+
+		const mediaSource = new Metamuxer.VideoFrameSource({
+			codec: 'av1',
+			bitrate: 1e6
+		});
+		output.addVideoTrack(mediaSource);
+
+		output.start();
+
+		for await (const frame of drain.frames(0, 10)) {
+			console.log(frame.timestamp, frame.duration)
+			await mediaSource.digest(frame);
+			frame.close();
+		}
+
+		await output.finalize();
+		function download(blob, filename) {
+			const url = URL.createObjectURL(blob);
+			const a = document.createElement('a');
+			a.href = url;
+			a.download = filename;
+			a.click();
+			URL.revokeObjectURL(url);
+		}
+
+		download(new Blob([target.buffer]), 'converted.mp4');
+
+		/*
+		const videoTrack = await input.getPrimaryVideoTrack();
+		const drain = new Metamuxer.VideoFrameDrain(videoTrack);
+
 		async function* timestamps() {
 			const fromTime = 0;
 			const toTime = await videoTrack.computeDuration();
@@ -50,6 +84,7 @@
 			console.log(clone)
 		}
 		console.log("done")
+		*/
 
 		/*
 		const videoTrack = await input.getPrimaryVideoTrack();

dev/player.html

@@ -203,6 +203,8 @@ async function runAudioIterator() {
 }
 
 function formatSeconds(seconds) {
+	seconds = Math.round(seconds * 1000) / 1000; // Round to milliseconds
+
 	const minutes = Math.floor(seconds / 60);
 	const remainingSeconds = Math.floor(seconds % 60);
 	return `${minutes}:${remainingSeconds.toString().padStart(2, '0')}.${Math.floor(1000 * seconds % 1000).toString().padStart(3, '0')}`;

src/codec.ts

@@ -166,7 +166,7 @@ export const buildVideoCodecString = (codec: VideoCodec, width: number, height:
 
 		const bitDepth = '08'; // 8-bit
 
-		return `vp09.${profile}.${levelInfo.level}.${bitDepth}`;
+		return `vp09.${profile}.${levelInfo.level.toString().padStart(2, '0')}.${bitDepth}`;
 	} else if (codec === 'av1') {
 		const profile = 0; // Main Profile
 
@@ -184,7 +184,27 @@ export const buildVideoCodecString = (codec: VideoCodec, width: number, height:
 	throw new TypeError(`Unhandled codec '${codec}'.`);
 };
 
-export const extractVideoCodecString = (codec: VideoCodec, description: Uint8Array | null) => {
+export type Vp9CodecInfo = {
+	profile: number;
+	level: number;
+	bitDepth: number;
+};
+
+export type Av1CodecInfo = {
+	seqProfile: number;
+	seqLevelIdx0: number;
+	seqTier0: number;
+	bitDepth: number;
+};
+
+export const extractVideoCodecString = (trackInfo: {
+	codec: VideoCodec | null;
+	codecDescription: Uint8Array | null;
+	vp9CodecInfo: Vp9CodecInfo | null;
+	av1CodecInfo: Av1CodecInfo | null;
+}) => {
+	const { codec, codecDescription: description, vp9CodecInfo, av1CodecInfo } = trackInfo;
+
 	if (codec === 'avc') {
 		if (!description || description.byteLength < 4) {
 			throw new TypeError('AVC description must be at least 4 bytes long.');
@@ -236,9 +256,77 @@ export const extractVideoCodecString = (codec: VideoCodec, description: Uint8Arr
 		codecString += constraintFlags.map(x => x.toString(16)).join('.');
 
 		return codecString;
-	}
+	} else if (codec === 'vp8') {
+		return 'vp8'; // Easy, this one
+	} else if (codec === 'vp9') {
+		if (!vp9CodecInfo) {
+			throw new Error('Missing VP9 codec info - unable to construct codec string.');
+		}
 
-	// TODO
+		const profile = vp9CodecInfo.profile.toString().padStart(2, '0');
+		const level = vp9CodecInfo.level.toString().padStart(2, '0');
+		const bitDepth = vp9CodecInfo.bitDepth.toString().padStart(2, '0');
+
+		return `vp09.${profile}.${level}.${bitDepth}`;
+	} else if (codec === 'av1') {
+		if (!av1CodecInfo) {
+			throw new Error('Missing AV1 codec info - unable to construct codec string.');
+		}
+
+		const profile = av1CodecInfo.seqProfile;
+		const level = av1CodecInfo.seqLevelIdx0.toString().padStart(2, '0');
+		const tier = av1CodecInfo.seqTier0 ? 'H' : 'M';
+		const bitDepth = av1CodecInfo.bitDepth.toString().padStart(2, '0');
+
+		return `av01.${profile}.${level}${tier}.${bitDepth}`;
+
+		/*
+		const chunk = await videoTrack._backing.getFirstChunk({});
+		if (!chunk) {
+			throw new Error('ahh');
+		}
+
+		const buffer = new ArrayBuffer(chunk.byteLength);
+		chunk.copyTo(buffer);
+
+		console.log(new Uint8Array(buffer));
+
+		const view = new DataView(buffer);
+		let pos = 0;
+		const byte = view.getUint8(pos++);
+
+		const obuType = (byte & 0b1111000) >> 3;
+		const obuExtensionFlag = (byte & 0b100) >> 2;
+		const obuHasSizeField = (byte & 0b10) >> 1;
+
+		if (obuExtensionFlag) {
+			pos++;
+		}
+
+		function leb128() {
+			let value = 0;
+
+			for (let i = 0; i < 8; i++) {
+				const leb128_byte = view.getUint8(pos);
+				value |= (leb128_byte & 0x7f) << (i * 7); // Extract the lower 7 bits and shift them accordingly
+				pos++;
+
+				// Check if the most significant bit (MSB) is 0, signaling the end of the LEB128 sequence
+				if ((leb128_byte & 0x80) === 0) {
+					break;
+				}
+			}
+
+			return value;
+		}
+
+		if (obuHasSizeField) {
+			console.log(leb128());
+		}
+
+		// Some extra stuff, todo
+		*/
+	}
 
 	throw new TypeError(`Unhandled codec '${codec}'.`);
 };

src/isobmff/isobmff-boxes.ts

@@ -630,6 +630,7 @@ export const av1C = () => {
 
 	// The box contents are not correct like this, but its length is. Getting the values for the last three bytes
 	// requires peeking into the bitstream of the coded chunks. Might come back later.
+	// TODO
 	return box('av1C', [
 		firstByte,
 		0,

src/isobmff/isobmff-demuxer.ts

@@ -1,10 +1,12 @@
 import {
 	AudioCodec,
+	Av1CodecInfo,
 	extractAudioCodecString,
 	extractVideoCodecString,
 	MediaCodec,
 	parseAacAudioSpecificConfig,
 	VideoCodec,
+	Vp9CodecInfo,
 } from '../codec';
 import { Demuxer } from '../demuxer';
 import { Input } from '../input';
@@ -54,7 +56,9 @@ type InternalTrack = {
 		height: number;
 		codec: VideoCodec | null;
 		codecDescription: Uint8Array | null;
-		colorSpace?: VideoColorSpaceInit | null;
+		colorSpace: VideoColorSpaceInit | null;
+		vp9CodecInfo: Vp9CodecInfo | null;
+		av1CodecInfo: Av1CodecInfo | null;
 	};
 } | {
 	info: {
@@ -543,6 +547,8 @@ export class IsobmffDemuxer extends Demuxer {
 						codec: null,
 						codecDescription: null,
 						colorSpace: null,
+						vp9CodecInfo: null,
+						av1CodecInfo: null,
 					};
 				} else if (handlerType === 'soun') {
 					track.info = {
@@ -585,9 +591,15 @@ export class IsobmffDemuxer extends Demuxer {
 							track.info.codec = 'avc';
 						} else if (sampleBoxInfo.name === 'hvc1' || sampleBoxInfo.name === 'hev1') {
 							track.info.codec = 'hevc';
+						} else if (sampleBoxInfo.name === 'vp08') {
+							track.info.codec = 'vp8';
+						} else if (sampleBoxInfo.name === 'vp09') {
+							track.info.codec = 'vp9';
+						} else if (sampleBoxInfo.name === 'av01') {
+							track.info.codec = 'av1';
 						} else {
-							// TODO a more user-friendly message
-							console.warn(`Unsupported video sample entry type ${sampleBoxInfo.name}.`);
+							const { name } = sampleBoxInfo;
+							console.warn(`Unsupported video codec (sample entry type '${name}') - discarding track.`);
 							break;
 						}
 
@@ -605,7 +617,8 @@ export class IsobmffDemuxer extends Demuxer {
 						} else if (sampleBoxInfo.name.toLowerCase() === 'opus') {
 							track.info.codec = 'opus';
 						} else {
-							console.warn(`Unsupported audio sample entry type ${sampleBoxInfo.name}.`);
+							const { name } = sampleBoxInfo;
+							console.warn(`Unsupported audio codec (sample entry type '${name}') - discarding track.`);
 							break;
 						}
 
@@ -706,6 +719,49 @@ export class IsobmffDemuxer extends Demuxer {
 				);
 			}; break;
 
+			case 'vpcC': {
+				const track = this.currentTrack;
+				assert(track && track.info?.type === 'video');
+
+				this.isobmffReader.pos += 4; // Version + flags
+
+				const profile = this.isobmffReader.readU8();
+				const level = this.isobmffReader.readU8();
+				const thirdByte = this.isobmffReader.readU8();
+
+				track.info.vp9CodecInfo = {
+					profile: profile,
+					level: level,
+					bitDepth: thirdByte >> 4,
+				};
+			}; break;
+
+			case 'av1C': {
+				const track = this.currentTrack;
+				assert(track && track.info?.type === 'video');
+
+				this.isobmffReader.pos += 1; // Marker + version
+
+				const secondByte = this.isobmffReader.readU8();
+				const seqProfile = secondByte >> 5;
+				const seqLevelIdx0 = secondByte & 0b11111;
+
+				const thirdByte = this.isobmffReader.readU8();
+				const seqTier0 = thirdByte >> 7;
+				const highBitDepth = (thirdByte >> 6) & 1;
+				const twelveBit = (thirdByte >> 5) & 1;
+
+				// Logic from https://aomediacodec.github.io/av1-spec/av1-spec.pdf
+				const bitDepth = seqProfile == 2 && highBitDepth ? (twelveBit ? 12 : 10) : (highBitDepth ? 10 : 8);
+
+				track.info.av1CodecInfo = {
+					seqProfile,
+					seqLevelIdx0,
+					seqTier0,
+					bitDepth,
+				};
+			}; break;
+
 			case 'colr': {
 				const track = this.currentTrack;
 				assert(track && track.info?.type === 'video');
@@ -1319,8 +1375,7 @@ abstract class IsobmffTrackBacking<Chunk extends EncodedVideoChunk | EncodedAudi
 
 	async computeDuration() {
 		const lastChunk = await this.getChunk(Infinity, { metadataOnly: true });
-		const timestamp = lastChunk?.timestamp;
-		return timestamp ? timestamp / 1e6 : 0;
+		return ((lastChunk?.timestamp ?? 0) + (lastChunk?.duration ?? 0)) / 1e6;
 	}
 
 	abstract createChunk(
@@ -1820,7 +1875,7 @@ class IsobmffVideoTrackBacking extends IsobmffTrackBacking<EncodedVideoChunk> im
 
 	async getDecoderConfig(): Promise<VideoDecoderConfig> {
 		return {
-			codec: extractVideoCodecString(this.internalTrack.info.codec!, this.internalTrack.info.codecDescription),
+			codec: extractVideoCodecString(this.internalTrack.info),
 			codedWidth: this.internalTrack.info.width,
 			codedHeight: this.internalTrack.info.height,
 			description: this.internalTrack.info.codecDescription ?? undefined,