diff --git a/dev/demux.html b/dev/demux.html index 6780e5c..eca56a0 100644 --- a/dev/demux.html +++ b/dev/demux.html @@ -11,23 +11,22 @@ document.body.append(fileInput); fileInput.addEventListener('change', async () => { - /* const file = fileInput.files[0]; const input = new Mediabunny.Input({ formats: Mediabunny.ALL_FORMATS, source: new Mediabunny.BlobSource(file), }); - const track = await input.getPrimaryAudioTrack(); - const sink3 = new Mediabunny.EncodedPacketSink(track); - console.log((await sink3.getFirstPacket()).data.join(', ')); - return; - console.log(await track.getDurationFromMetadata(), await track.computeDuration()); + const track = await input.getPrimaryVideoTrack(); + const sink = new Mediabunny.EncodedPacketSink(track); + const packet = await sink.getFirstKeyPacket(); - //console.log(await input.getDurationFromMetadata(), await input.computeDuration()); - return; - */ + console.log(packet.data.join(', ')); + const config = await track.getDecoderConfig(); + console.log(config.description.join(', ')); + + /* const input = new Mediabunny.Input({ source: new Mediabunny.UrlSource('https://storage.googleapis.com/shaka-demo-assets/angel-one-widevine-hls/hls.m3u8'), formats: Mediabunny.ALL_FORMATS, @@ -36,6 +35,7 @@ const videoTrack = await input.getPrimaryVideoTrack(); const sink = new Mediabunny.EncodedPacketSink(videoTrack); console.log(await sink.getFirstPacket()); + */ /* return diff --git a/src/codec-data.ts b/src/codec-data.ts index 235c080..8da5610 100644 --- a/src/codec-data.ts +++ b/src/codec-data.ts @@ -866,6 +866,21 @@ export type HevcSpsInfo = { minSpatialSegmentationIdc: number; }; +export const concatHevcNalUnits = (nalUnits: Uint8Array[], decoderConfig: VideoDecoderConfig) => { + if (decoderConfig.description) { + // Stream is length-prefixed. Let's extract the size of the length prefix from the decoder config + + const bytes = toUint8Array(decoderConfig.description); + const lengthSizeMinusOne = bytes[21]! & 0b11; + const lengthSize = (lengthSizeMinusOne + 1) as 1 | 2 | 3 | 4; + + return concatNalUnitsInLengthPrefixed(nalUnits, lengthSize); + } else { + // Stream is in Annex B format + return concatNalUnitsInAnnexB(nalUnits); + } +}; + export const iterateHevcNalUnits = (packetData: Uint8Array, decoderConfig: VideoDecoderConfig) => { if (decoderConfig.description) { const bytes = toUint8Array(decoderConfig.description); @@ -1602,6 +1617,103 @@ export const deserializeHevcDecoderConfigurationRecord = (data: Uint8Array): Hev } }; +enum HevcNaluOrderState { + audAllowed, + beforeFirstVcl, + afterFirstVcl, + eoBitstreamAllowed, + noMoreDataAllowed, +} + +// This function sanitzes the contents of an HEVC packet such that +// https://source.chromium.org/chromium/chromium/src/+/main:media/formats/mp4/hevc.cc's validation logic does not trip +// up on its contents. The validation is often too strict and rejects packets that Chromium could decode just fine. +// Chromium code retrieved on 2026-04-29. +// See https://issues.chromium.org/issues/507611247. +export const sanitizeHevcPacketForChromium = ( + packetData: Uint8Array, + decoderConfig: VideoDecoderConfig, +): Uint8Array | null => { + const removedNalUnits = new Set(); + let orderState: HevcNaluOrderState = HevcNaluOrderState.audAllowed; + + for (const loc of iterateHevcNalUnits(packetData, decoderConfig)) { + if (orderState === HevcNaluOrderState.noMoreDataAllowed) { + removedNalUnits.add(loc.offset); + continue; + } + + const type = extractNalUnitTypeForHevc(packetData[loc.offset]!); + + if (orderState === HevcNaluOrderState.eoBitstreamAllowed && type !== 37 /* EOB_NUT */) { + removedNalUnits.add(loc.offset); + continue; + } + + let remove = false; + + if (type === 35) { // AUD_NUT + if (orderState > HevcNaluOrderState.audAllowed) { + remove = true; + } else { + orderState = HevcNaluOrderState.beforeFirstVcl; + } + } else if (type <= 31) { // VCL (0-31) + if (orderState > HevcNaluOrderState.afterFirstVcl) { + remove = true; + } else { + orderState = HevcNaluOrderState.afterFirstVcl; + } + } else if (type === 36) { // EOS_NUT + if (orderState !== HevcNaluOrderState.afterFirstVcl) { + remove = true; + } else { + orderState = HevcNaluOrderState.eoBitstreamAllowed; + } + } else if (type === 37) { // EOB_NUT + if (orderState < HevcNaluOrderState.afterFirstVcl) { + remove = true; + } else { + orderState = HevcNaluOrderState.noMoreDataAllowed; + } + } else if ( + type === 32 || type === 33 || type === 34 || type === 39 + || (type >= 41 && type <= 44) || (type >= 48 && type <= 55) + ) { // VPS, SPS, PPS, PREFIX_SEI, RSV_NVCL41..44, UNSPEC48..55 + if (orderState > HevcNaluOrderState.beforeFirstVcl) { + remove = true; + } else { + orderState = HevcNaluOrderState.beforeFirstVcl; + } + } else if ( + type === 38 || type === 40 + || (type >= 45 && type <= 47) || (type >= 56 && type <= 63) + ) { // FD, SUFFIX_SEI, RSV_NVCL45..47, UNSPEC56..63 + if (orderState < HevcNaluOrderState.afterFirstVcl) { + remove = true; + } + } + + if (remove) { + removedNalUnits.add(loc.offset); + } + } + + // If nothing violated the rules, return null to signal that + if (removedNalUnits.size === 0) { + return null; + } + + const filteredNalUnits: Uint8Array[] = []; + for (const loc of iterateHevcNalUnits(packetData, decoderConfig)) { + if (!removedNalUnits.has(loc.offset)) { + filteredNalUnits.push(packetData.subarray(loc.offset, loc.offset + loc.length)); + } + } + + return concatHevcNalUnits(filteredNalUnits, decoderConfig); +}; + export type Vp9CodecInfo = { profile: number; level: number; diff --git a/src/media-sink.ts b/src/media-sink.ts index ac1c1c8..946ce19 100644 --- a/src/media-sink.ts +++ b/src/media-sink.ts @@ -17,6 +17,7 @@ import { iterateAvcNalUnits, iterateHevcNalUnits, parseAvcSps, + sanitizeHevcPacketForChromium, } from './codec-data'; import { CustomVideoDecoder, customVideoDecoders, CustomAudioDecoder, customAudioDecoders } from './custom-coder'; import { InputDisposedError } from './input'; @@ -980,20 +981,28 @@ class VideoDecoderWrapper extends DecoderWrapper { insertSorted(this.inputTimestamps, packet.timestamp, x => x); } - // Workaround for https://issues.chromium.org/issues/470109459 - if (isChromium() && this.currentPacketIndex === 0 && this.codec === 'avc') { - const filteredNalUnits: Uint8Array[] = []; + if (isChromium() && this.currentPacketIndex === 0) { + if (this.codec === 'avc') { + // Workaround for https://issues.chromium.org/issues/470109459 + const filteredNalUnits: Uint8Array[] = []; - for (const loc of iterateAvcNalUnits(packet.data, this.decoderConfig)) { - const type = extractNalUnitTypeForAvc(packet.data[loc.offset]!); - // These trip up Chromium's key frame detection, so let's strip them - if (!(type >= 20 && type <= 31)) { - filteredNalUnits.push(packet.data.subarray(loc.offset, loc.offset + loc.length)); + for (const loc of iterateAvcNalUnits(packet.data, this.decoderConfig)) { + const type = extractNalUnitTypeForAvc(packet.data[loc.offset]!); + // These trip up Chromium's key frame detection, so let's strip them + if (!(type >= 20 && type <= 31)) { + filteredNalUnits.push(packet.data.subarray(loc.offset, loc.offset + loc.length)); + } + } + + const newData = concatAvcNalUnits(filteredNalUnits, this.decoderConfig); + packet = new EncodedPacket(newData, packet.type, packet.timestamp, packet.duration); + } else if (this.codec === 'hevc') { + // Workaround for https://issues.chromium.org/issues/507611247 + const sanitizedData = sanitizeHevcPacketForChromium(packet.data, this.decoderConfig); + if (sanitizedData) { + packet = new EncodedPacket(sanitizedData, packet.type, packet.timestamp, packet.duration); } } - - const newData = concatAvcNalUnits(filteredNalUnits, this.decoderConfig); - packet = new EncodedPacket(newData, packet.type, packet.timestamp, packet.duration); } this.decoder.decode(packet.toEncodedVideoChunk());