From b9f7ab2fa2b9167784cbded044d466185308999f Mon Sep 17 00:00:00 2001 From: Vanilagy <1696106+Vanilagy@users.noreply.github.com> Date: Tue, 9 Sep 2025 13:09:41 +0200 Subject: [PATCH] Skip HEVC RASL slices following a decoder reset --- dev/demux.html | 10 +- .../thumbnail-generation.ts | 7 ++ package-lock.json | 12 +- package.json | 2 +- packages/mp3-encoder/package.json | 2 +- src/codec-data.ts | 117 ++++++++++-------- src/media-sink.ts | 32 ++++- 7 files changed, 120 insertions(+), 62 deletions(-) diff --git a/dev/demux.html b/dev/demux.html index 72543c6..d787459 100644 --- a/dev/demux.html +++ b/dev/demux.html @@ -17,7 +17,15 @@ const videoTrack = await input.getPrimaryVideoTrack(); const sink = new Mediabunny.VideoSampleSink(videoTrack); - console.log(await sink.getSample(await videoTrack.getFirstTimestamp())) + console.log(await sink.getSample(2.131875)); + + /* + const sink = new Mediabunny.EncodedPacketSink(videoTrack); + + for await (const packet of sink.packets()) { + console.log(packet) + } + */ /* for await (const sample of sink.samples(0.99)) { diff --git a/examples/thumbnail-generation/thumbnail-generation.ts b/examples/thumbnail-generation/thumbnail-generation.ts index b71b973..ea7716c 100644 --- a/examples/thumbnail-generation/thumbnail-generation.ts +++ b/examples/thumbnail-generation/thumbnail-generation.ts @@ -96,6 +96,13 @@ const generateThumbnails = async (resource: File | string) => { timestampElement.className = 'absolute bottom-0 right-0 bg-black/30 text-white px-1 py-0.5 text-[11px] rounded-tl-lg'; container.append(timestampElement); + } else { + // Add something to indicate that the thumbnail is missing + const p = document.createElement('p'); + p.textContent = '?'; + p.className = 'absolute inset-0 flex items-center justify-center text-3xl opacity-50'; + + container.append(p); } i++; diff --git a/package-lock.json b/package-lock.json index ab1d590..6d7ca5d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "mediabunny", - "version": "1.14.2", + "version": "1.14.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "mediabunny", - "version": "1.14.2", + "version": "1.14.3", "license": "MPL-2.0", "workspaces": [ "packages/*" @@ -7749,9 +7749,9 @@ } }, "node_modules/mediabunny": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.14.1.tgz", - "integrity": "sha512-TjLg8GQGGsnGePcA6i0NItGe9a5G8WaleCGaXpb5V5okwuK5KpNKfcmTZXItnjfPLo7FvfEZI0NFp1lIR8Os7Q==", + "version": "1.14.2", + "resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.14.2.tgz", + "integrity": "sha512-JCup3rpsJGIw3I7E7bZSmVEANCr/OwfwkH3g/Wq1TcHEJ7ymLLw+jC3+/Nnn/JLXzFnw8/I6fV3xECLnFedfHg==", "license": "MPL-2.0", "peer": true, "workspaces": [ @@ -12242,7 +12242,7 @@ }, "packages/mp3-encoder": { "name": "@mediabunny/mp3-encoder", - "version": "1.14.2", + "version": "1.14.3", "license": "MPL-2.0", "devDependencies": { "@types/emscripten": "^1.40.1" diff --git a/package.json b/package.json index b3d43b7..5ef01e4 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "mediabunny", "author": "Vanilagy", - "version": "1.14.2", + "version": "1.14.3", "description": "Pure TypeScript media toolkit for reading, writing, and converting media files, directly in the browser.", "type": "module", "workspaces": [ diff --git a/packages/mp3-encoder/package.json b/packages/mp3-encoder/package.json index 530fce7..14cb8af 100644 --- a/packages/mp3-encoder/package.json +++ b/packages/mp3-encoder/package.json @@ -1,7 +1,7 @@ { "name": "@mediabunny/mp3-encoder", "author": "Vanilagy", - "version": "1.14.2", + "version": "1.14.3", "description": "MP3 encoder extension for Mediabunny, based on LAME.", "main": "./dist/bundles/mediabunny-mp3-encoder.mjs", "module": "./dist/bundles/mediabunny-mp3-encoder.mjs", diff --git a/src/codec-data.ts b/src/codec-data.ts index d315199..895569f 100644 --- a/src/codec-data.ts +++ b/src/codec-data.ts @@ -26,8 +26,27 @@ import { EncodedPacket, PacketType } from './packet'; // Rec. ITU-T H.265 // https://stackoverflow.com/questions/24884827 +export enum AvcNalUnitType { + IDR = 5, + SPS = 7, + PPS = 8, + SPS_EXT = 13, +} + +export enum HevcNalUnitType { + RASL_N = 8, + RASL_R = 9, + BLA_W_LP = 16, + RSV_IRAP_VCL23 = 23, + VPS_NUT = 32, + SPS_NUT = 33, + PPS_NUT = 34, + PREFIX_SEI_NUT = 39, + SUFFIX_SEI_NUT = 40, +} + /** Finds all NAL units in an AVC packet in Annex B format. */ -const findNalUnitsInAnnexB = (packetData: Uint8Array) => { +export const findNalUnitsInAnnexB = (packetData: Uint8Array) => { const nalUnits: Uint8Array[] = []; let i = 0; @@ -184,6 +203,21 @@ export type AvcDecoderConfigurationRecord = { sequenceParameterSetExt: Uint8Array[] | null; }; +export const extractAvcNalUnits = (packetData: Uint8Array, decoderConfig: VideoDecoderConfig) => { + if (decoderConfig.description) { + // Stream is length-prefixed. Let's extract the size of the length prefix from the decoder config + + const bytes = toUint8Array(decoderConfig.description); + const lengthSizeMinusOne = bytes[4]! & 0b11; + const lengthSize = (lengthSizeMinusOne + 1) as 1 | 2 | 3 | 4; + + return findNalUnitsInLengthPrefixed(packetData, lengthSize); + } else { + // Stream is in Annex B format + return findNalUnitsInAnnexB(packetData); + } +}; + const extractNalUnitTypeForAvc = (data: Uint8Array) => { return data[0]! & 0x1F; }; @@ -193,9 +227,9 @@ export const extractAvcDecoderConfigurationRecord = (packetData: Uint8Array) => try { const nalUnits = findNalUnitsInAnnexB(packetData); - const spsUnits = nalUnits.filter(unit => extractNalUnitTypeForAvc(unit) === 7); - const ppsUnits = nalUnits.filter(unit => extractNalUnitTypeForAvc(unit) === 8); - const spsExtUnits = nalUnits.filter(unit => extractNalUnitTypeForAvc(unit) === 13); + const spsUnits = nalUnits.filter(unit => extractNalUnitTypeForAvc(unit) === AvcNalUnitType.SPS); + const ppsUnits = nalUnits.filter(unit => extractNalUnitTypeForAvc(unit) === AvcNalUnitType.PPS); + const spsExtUnits = nalUnits.filter(unit => extractNalUnitTypeForAvc(unit) === AvcNalUnitType.SPS_EXT); if (spsUnits.length === 0) { return null; @@ -337,12 +371,6 @@ export const serializeAvcDecoderConfigurationRecord = (record: AvcDecoderConfigu return new Uint8Array(bytes); }; -const NALU_TYPE_VPS = 32; -const NALU_TYPE_SPS = 33; -const NALU_TYPE_PPS = 34; -const NALU_TYPE_SEI_PREFIX = 39; -const NALU_TYPE_SEI_SUFFIX = 40; - // Data specified in ISO 14496-15 export type HevcDecoderConfigurationRecord = { configurationVersion: number; @@ -369,7 +397,22 @@ export type HevcDecoderConfigurationRecord = { }[]; }; -const extractNalUnitTypeForHevc = (data: Uint8Array) => { +export const extractHevcNalUnits = (packetData: Uint8Array, decoderConfig: VideoDecoderConfig) => { + if (decoderConfig.description) { + // Stream is length-prefixed. Let's extract the size of the length prefix from the decoder config + + const bytes = toUint8Array(decoderConfig.description); + const lengthSizeMinusOne = bytes[21]! & 0b11; + const lengthSize = (lengthSizeMinusOne + 1) as 1 | 2 | 3 | 4; + + return findNalUnitsInLengthPrefixed(packetData, lengthSize); + } else { + // Stream is in Annex B format + return findNalUnitsInAnnexB(packetData); + } +}; + +export const extractNalUnitTypeForHevc = (data: Uint8Array) => { return (data[0]! >> 1) & 0x3F; }; @@ -380,12 +423,12 @@ export const extractHevcDecoderConfigurationRecord = ( try { const nalUnits = findNalUnitsInAnnexB(packetData); - const vpsUnits = nalUnits.filter(unit => extractNalUnitTypeForHevc(unit) === NALU_TYPE_VPS); - const spsUnits = nalUnits.filter(unit => extractNalUnitTypeForHevc(unit) === NALU_TYPE_SPS); - const ppsUnits = nalUnits.filter(unit => extractNalUnitTypeForHevc(unit) === NALU_TYPE_PPS); + const vpsUnits = nalUnits.filter(unit => extractNalUnitTypeForHevc(unit) === HevcNalUnitType.VPS_NUT); + const spsUnits = nalUnits.filter(unit => extractNalUnitTypeForHevc(unit) === HevcNalUnitType.SPS_NUT); + const ppsUnits = nalUnits.filter(unit => extractNalUnitTypeForHevc(unit) === HevcNalUnitType.PPS_NUT); const seiUnits = nalUnits.filter( - unit => extractNalUnitTypeForHevc(unit) === NALU_TYPE_SEI_PREFIX - || extractNalUnitTypeForHevc(unit) === NALU_TYPE_SEI_SUFFIX, + unit => extractNalUnitTypeForHevc(unit) === HevcNalUnitType.PREFIX_SEI_NUT + || extractNalUnitTypeForHevc(unit) === HevcNalUnitType.SUFFIX_SEI_NUT, ); if (spsUnits.length === 0 || ppsUnits.length === 0) return null; @@ -521,7 +564,7 @@ export const extractHevcDecoderConfigurationRecord = ( ? [ { arrayCompleteness: 1, - nalUnitType: NALU_TYPE_VPS, + nalUnitType: HevcNalUnitType.VPS_NUT, nalUnits: vpsUnits, }, ] @@ -530,7 +573,7 @@ export const extractHevcDecoderConfigurationRecord = ( ? [ { arrayCompleteness: 1, - nalUnitType: NALU_TYPE_SPS, + nalUnitType: HevcNalUnitType.SPS_NUT, nalUnits: spsUnits, }, ] @@ -539,7 +582,7 @@ export const extractHevcDecoderConfigurationRecord = ( ? [ { arrayCompleteness: 1, - nalUnitType: NALU_TYPE_PPS, + nalUnitType: HevcNalUnitType.PPS_NUT, nalUnits: ppsUnits, }, ] @@ -1440,22 +1483,9 @@ export const determineVideoPacketType = async ( const decoderConfig = await videoTrack.getDecoderConfig(); assert(decoderConfig); - let nalUnits: Uint8Array[]; + const nalUnits = extractAvcNalUnits(packet.data, decoderConfig); + const isKeyframe = nalUnits.some(x => extractNalUnitTypeForAvc(x) === AvcNalUnitType.IDR); - if (decoderConfig.description) { - // Stream is length-prefixed. Let's extract the size of the length prefix from the decoder config - - const bytes = toUint8Array(decoderConfig.description); - const lengthSizeMinusOne = bytes[4]! & 0b11; - const lengthSize = (lengthSizeMinusOne + 1) as 1 | 2 | 3 | 4; - - nalUnits = findNalUnitsInLengthPrefixed(packet.data, lengthSize); - } else { - // Stream is in Annex B format - nalUnits = findNalUnitsInAnnexB(packet.data); - } - - const isKeyframe = nalUnits.some(x => extractNalUnitTypeForAvc(x) === 5); return isKeyframe ? 'key' : 'delta'; }; @@ -1463,25 +1493,12 @@ export const determineVideoPacketType = async ( const decoderConfig = await videoTrack.getDecoderConfig(); assert(decoderConfig); - let nalUnits: Uint8Array[]; - - if (decoderConfig.description) { - // Stream is length-prefixed. Let's extract the size of the length prefix from the decoder config - - const bytes = toUint8Array(decoderConfig.description); - const lengthSizeMinusOne = bytes[21]! & 0b11; - const lengthSize = (lengthSizeMinusOne + 1) as 1 | 2 | 3 | 4; - - nalUnits = findNalUnitsInLengthPrefixed(packet.data, lengthSize); - } else { - // Stream is in Annex B format - nalUnits = findNalUnitsInAnnexB(packet.data); - } - + const nalUnits = extractHevcNalUnits(packet.data, decoderConfig); const isKeyframe = nalUnits.some((x) => { const type = extractNalUnitTypeForHevc(x); - return 16 <= type && type <= 23; + return HevcNalUnitType.BLA_W_LP <= type && type <= HevcNalUnitType.RSV_IRAP_VCL23; }); + return isKeyframe ? 'key' : 'delta'; }; diff --git a/src/media-sink.ts b/src/media-sink.ts index 643f259..e11b4b3 100644 --- a/src/media-sink.ts +++ b/src/media-sink.ts @@ -7,6 +7,7 @@ */ import { parsePcmCodec, PCM_AUDIO_CODECS, PcmAudioCodec, VideoCodec, AudioCodec } from './codec'; +import { extractHevcNalUnits, extractNalUnitTypeForHevc, HevcNalUnitType } from './codec-data'; import { CustomVideoDecoder, customVideoDecoders, CustomAudioDecoder, customAudioDecoders } from './custom-coder'; import { InputAudioTrack, InputTrack, InputVideoTrack } from './input-track'; import { @@ -624,8 +625,8 @@ export abstract class BaseMediaSampleSink< const nextPacket = await packetSink.getNextPacket(currentPacket); assert(nextPacket); - currentPacket = nextPacket; decoder.decode(nextPacket); + currentPacket = nextPacket; } maxSequenceNumber = -1; @@ -757,12 +758,14 @@ class VideoDecoderWrapper extends DecoderWrapper { inputTimestamps: number[] = []; // Timestamps input into the decoder, sorted. sampleQueue: VideoSample[] = []; // Safari-specific thing, check usage. + currentPacketIndex = 0; + raslSkipped = false; // For HEVC stuff constructor( onSample: (sample: VideoSample) => unknown, onError: (error: DOMException) => unknown, - codec: VideoCodec, - decoderConfig: VideoDecoderConfig, + public codec: VideoCodec, + public decoderConfig: VideoDecoderConfig, public rotation: Rotation, public timeResolution: number, ) { @@ -848,6 +851,26 @@ class VideoDecoderWrapper extends DecoderWrapper { } decode(packet: EncodedPacket) { + if (this.codec === 'hevc' && this.currentPacketIndex > 0 && !this.raslSkipped) { + // If we're using HEVC, we need to make sure to skip any RASL slices that follow a non-IDR key frame such as + // CRA_NUT. This is because RASL slices cannot be decoded without data before the CRA_NUT. Browsers behave + // differently here: Chromium drops the packets, Safari throws a decoder error. Either way, it's not good + // and causes bugs upstream. So, let's take the dropping into our own hands. + const nalUnits = extractHevcNalUnits(packet.data, this.decoderConfig); + const hasRaslPicture = nalUnits.some((x) => { + const type = extractNalUnitTypeForHevc(x); + return type === HevcNalUnitType.RASL_N || type === HevcNalUnitType.RASL_R; + }); + + if (hasRaslPicture) { + return; // Drop + } + + this.raslSkipped = true; + } + + this.currentPacketIndex++; + if (this.customDecoder) { this.customDecoderQueueSize++; void this.customDecoderCallSerializer @@ -879,6 +902,9 @@ class VideoDecoderWrapper extends DecoderWrapper { this.sampleQueue.length = 0; } + + this.currentPacketIndex = 0; + this.raslSkipped = false; } close() {