diff --git a/dev/demux.html b/dev/demux.html index 39b1aed..48801c4 100644 --- a/dev/demux.html +++ b/dev/demux.html @@ -16,6 +16,40 @@ const videoTrack = await input.getPrimaryVideoTrack(); const drain = new Metamuxer.VideoFrameDrain(videoTrack); + const target = new Metamuxer.ArrayBufferTarget(); + const format = new Metamuxer.Mp4OutputFormat(); + const output = new Metamuxer.Output({ target, format }); + + const mediaSource = new Metamuxer.VideoFrameSource({ + codec: 'av1', + bitrate: 1e6 + }); + output.addVideoTrack(mediaSource); + + output.start(); + + for await (const frame of drain.frames(0, 10)) { + console.log(frame.timestamp, frame.duration) + await mediaSource.digest(frame); + frame.close(); + } + + await output.finalize(); + function download(blob, filename) { + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = filename; + a.click(); + URL.revokeObjectURL(url); + } + + download(new Blob([target.buffer]), 'converted.mp4'); + + /* + const videoTrack = await input.getPrimaryVideoTrack(); + const drain = new Metamuxer.VideoFrameDrain(videoTrack); + async function* timestamps() { const fromTime = 0; const toTime = await videoTrack.computeDuration(); @@ -50,6 +84,7 @@ console.log(clone) } console.log("done") + */ /* const videoTrack = await input.getPrimaryVideoTrack(); diff --git a/dev/player.html b/dev/player.html index 18f7d06..50dd21b 100644 --- a/dev/player.html +++ b/dev/player.html @@ -203,6 +203,8 @@ async function runAudioIterator() { } function formatSeconds(seconds) { + seconds = Math.round(seconds * 1000) / 1000; // Round to milliseconds + const minutes = Math.floor(seconds / 60); const remainingSeconds = Math.floor(seconds % 60); return `${minutes}:${remainingSeconds.toString().padStart(2, '0')}.${Math.floor(1000 * seconds % 1000).toString().padStart(3, '0')}`; diff --git a/src/codec.ts b/src/codec.ts index cb1528a..d8e4e97 100644 --- a/src/codec.ts +++ b/src/codec.ts @@ -166,7 +166,7 @@ export const buildVideoCodecString = (codec: VideoCodec, width: number, height: const bitDepth = '08'; // 8-bit - return `vp09.${profile}.${levelInfo.level}.${bitDepth}`; + return `vp09.${profile}.${levelInfo.level.toString().padStart(2, '0')}.${bitDepth}`; } else if (codec === 'av1') { const profile = 0; // Main Profile @@ -184,7 +184,27 @@ export const buildVideoCodecString = (codec: VideoCodec, width: number, height: throw new TypeError(`Unhandled codec '${codec}'.`); }; -export const extractVideoCodecString = (codec: VideoCodec, description: Uint8Array | null) => { +export type Vp9CodecInfo = { + profile: number; + level: number; + bitDepth: number; +}; + +export type Av1CodecInfo = { + seqProfile: number; + seqLevelIdx0: number; + seqTier0: number; + bitDepth: number; +}; + +export const extractVideoCodecString = (trackInfo: { + codec: VideoCodec | null; + codecDescription: Uint8Array | null; + vp9CodecInfo: Vp9CodecInfo | null; + av1CodecInfo: Av1CodecInfo | null; +}) => { + const { codec, codecDescription: description, vp9CodecInfo, av1CodecInfo } = trackInfo; + if (codec === 'avc') { if (!description || description.byteLength < 4) { throw new TypeError('AVC description must be at least 4 bytes long.'); @@ -236,9 +256,77 @@ export const extractVideoCodecString = (codec: VideoCodec, description: Uint8Arr codecString += constraintFlags.map(x => x.toString(16)).join('.'); return codecString; - } + } else if (codec === 'vp8') { + return 'vp8'; // Easy, this one + } else if (codec === 'vp9') { + if (!vp9CodecInfo) { + throw new Error('Missing VP9 codec info - unable to construct codec string.'); + } - // TODO + const profile = vp9CodecInfo.profile.toString().padStart(2, '0'); + const level = vp9CodecInfo.level.toString().padStart(2, '0'); + const bitDepth = vp9CodecInfo.bitDepth.toString().padStart(2, '0'); + + return `vp09.${profile}.${level}.${bitDepth}`; + } else if (codec === 'av1') { + if (!av1CodecInfo) { + throw new Error('Missing AV1 codec info - unable to construct codec string.'); + } + + const profile = av1CodecInfo.seqProfile; + const level = av1CodecInfo.seqLevelIdx0.toString().padStart(2, '0'); + const tier = av1CodecInfo.seqTier0 ? 'H' : 'M'; + const bitDepth = av1CodecInfo.bitDepth.toString().padStart(2, '0'); + + return `av01.${profile}.${level}${tier}.${bitDepth}`; + + /* + const chunk = await videoTrack._backing.getFirstChunk({}); + if (!chunk) { + throw new Error('ahh'); + } + + const buffer = new ArrayBuffer(chunk.byteLength); + chunk.copyTo(buffer); + + console.log(new Uint8Array(buffer)); + + const view = new DataView(buffer); + let pos = 0; + const byte = view.getUint8(pos++); + + const obuType = (byte & 0b1111000) >> 3; + const obuExtensionFlag = (byte & 0b100) >> 2; + const obuHasSizeField = (byte & 0b10) >> 1; + + if (obuExtensionFlag) { + pos++; + } + + function leb128() { + let value = 0; + + for (let i = 0; i < 8; i++) { + const leb128_byte = view.getUint8(pos); + value |= (leb128_byte & 0x7f) << (i * 7); // Extract the lower 7 bits and shift them accordingly + pos++; + + // Check if the most significant bit (MSB) is 0, signaling the end of the LEB128 sequence + if ((leb128_byte & 0x80) === 0) { + break; + } + } + + return value; + } + + if (obuHasSizeField) { + console.log(leb128()); + } + + // Some extra stuff, todo + */ + } throw new TypeError(`Unhandled codec '${codec}'.`); }; diff --git a/src/isobmff/isobmff-boxes.ts b/src/isobmff/isobmff-boxes.ts index 3d0d087..a18b513 100644 --- a/src/isobmff/isobmff-boxes.ts +++ b/src/isobmff/isobmff-boxes.ts @@ -630,6 +630,7 @@ export const av1C = () => { // The box contents are not correct like this, but its length is. Getting the values for the last three bytes // requires peeking into the bitstream of the coded chunks. Might come back later. + // TODO return box('av1C', [ firstByte, 0, diff --git a/src/isobmff/isobmff-demuxer.ts b/src/isobmff/isobmff-demuxer.ts index 32f59bb..6c009a7 100644 --- a/src/isobmff/isobmff-demuxer.ts +++ b/src/isobmff/isobmff-demuxer.ts @@ -1,10 +1,12 @@ import { AudioCodec, + Av1CodecInfo, extractAudioCodecString, extractVideoCodecString, MediaCodec, parseAacAudioSpecificConfig, VideoCodec, + Vp9CodecInfo, } from '../codec'; import { Demuxer } from '../demuxer'; import { Input } from '../input'; @@ -54,7 +56,9 @@ type InternalTrack = { height: number; codec: VideoCodec | null; codecDescription: Uint8Array | null; - colorSpace?: VideoColorSpaceInit | null; + colorSpace: VideoColorSpaceInit | null; + vp9CodecInfo: Vp9CodecInfo | null; + av1CodecInfo: Av1CodecInfo | null; }; } | { info: { @@ -543,6 +547,8 @@ export class IsobmffDemuxer extends Demuxer { codec: null, codecDescription: null, colorSpace: null, + vp9CodecInfo: null, + av1CodecInfo: null, }; } else if (handlerType === 'soun') { track.info = { @@ -585,9 +591,15 @@ export class IsobmffDemuxer extends Demuxer { track.info.codec = 'avc'; } else if (sampleBoxInfo.name === 'hvc1' || sampleBoxInfo.name === 'hev1') { track.info.codec = 'hevc'; + } else if (sampleBoxInfo.name === 'vp08') { + track.info.codec = 'vp8'; + } else if (sampleBoxInfo.name === 'vp09') { + track.info.codec = 'vp9'; + } else if (sampleBoxInfo.name === 'av01') { + track.info.codec = 'av1'; } else { - // TODO a more user-friendly message - console.warn(`Unsupported video sample entry type ${sampleBoxInfo.name}.`); + const { name } = sampleBoxInfo; + console.warn(`Unsupported video codec (sample entry type '${name}') - discarding track.`); break; } @@ -605,7 +617,8 @@ export class IsobmffDemuxer extends Demuxer { } else if (sampleBoxInfo.name.toLowerCase() === 'opus') { track.info.codec = 'opus'; } else { - console.warn(`Unsupported audio sample entry type ${sampleBoxInfo.name}.`); + const { name } = sampleBoxInfo; + console.warn(`Unsupported audio codec (sample entry type '${name}') - discarding track.`); break; } @@ -706,6 +719,49 @@ export class IsobmffDemuxer extends Demuxer { ); }; break; + case 'vpcC': { + const track = this.currentTrack; + assert(track && track.info?.type === 'video'); + + this.isobmffReader.pos += 4; // Version + flags + + const profile = this.isobmffReader.readU8(); + const level = this.isobmffReader.readU8(); + const thirdByte = this.isobmffReader.readU8(); + + track.info.vp9CodecInfo = { + profile: profile, + level: level, + bitDepth: thirdByte >> 4, + }; + }; break; + + case 'av1C': { + const track = this.currentTrack; + assert(track && track.info?.type === 'video'); + + this.isobmffReader.pos += 1; // Marker + version + + const secondByte = this.isobmffReader.readU8(); + const seqProfile = secondByte >> 5; + const seqLevelIdx0 = secondByte & 0b11111; + + const thirdByte = this.isobmffReader.readU8(); + const seqTier0 = thirdByte >> 7; + const highBitDepth = (thirdByte >> 6) & 1; + const twelveBit = (thirdByte >> 5) & 1; + + // Logic from https://aomediacodec.github.io/av1-spec/av1-spec.pdf + const bitDepth = seqProfile == 2 && highBitDepth ? (twelveBit ? 12 : 10) : (highBitDepth ? 10 : 8); + + track.info.av1CodecInfo = { + seqProfile, + seqLevelIdx0, + seqTier0, + bitDepth, + }; + }; break; + case 'colr': { const track = this.currentTrack; assert(track && track.info?.type === 'video'); @@ -1319,8 +1375,7 @@ abstract class IsobmffTrackBacking im async getDecoderConfig(): Promise { return { - codec: extractVideoCodecString(this.internalTrack.info.codec!, this.internalTrack.info.codecDescription), + codec: extractVideoCodecString(this.internalTrack.info), codedWidth: this.internalTrack.info.width, codedHeight: this.internalTrack.info.height, description: this.internalTrack.info.codecDescription ?? undefined,