diff --git a/dev/demux.html b/dev/demux.html
index 72543c6..d787459 100644
--- a/dev/demux.html
+++ b/dev/demux.html
@@ -17,7 +17,15 @@
const videoTrack = await input.getPrimaryVideoTrack();
const sink = new Mediabunny.VideoSampleSink(videoTrack);
- console.log(await sink.getSample(await videoTrack.getFirstTimestamp()))
+ console.log(await sink.getSample(2.131875));
+
+ /*
+ const sink = new Mediabunny.EncodedPacketSink(videoTrack);
+
+ for await (const packet of sink.packets()) {
+ console.log(packet)
+ }
+ */
/*
for await (const sample of sink.samples(0.99)) {
diff --git a/examples/thumbnail-generation/thumbnail-generation.ts b/examples/thumbnail-generation/thumbnail-generation.ts
index b71b973..ea7716c 100644
--- a/examples/thumbnail-generation/thumbnail-generation.ts
+++ b/examples/thumbnail-generation/thumbnail-generation.ts
@@ -96,6 +96,13 @@ const generateThumbnails = async (resource: File | string) => {
timestampElement.className
= 'absolute bottom-0 right-0 bg-black/30 text-white px-1 py-0.5 text-[11px] rounded-tl-lg';
container.append(timestampElement);
+ } else {
+ // Add something to indicate that the thumbnail is missing
+ const p = document.createElement('p');
+ p.textContent = '?';
+ p.className = 'absolute inset-0 flex items-center justify-center text-3xl opacity-50';
+
+ container.append(p);
}
i++;
diff --git a/package-lock.json b/package-lock.json
index ab1d590..6d7ca5d 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "mediabunny",
- "version": "1.14.2",
+ "version": "1.14.3",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "mediabunny",
- "version": "1.14.2",
+ "version": "1.14.3",
"license": "MPL-2.0",
"workspaces": [
"packages/*"
@@ -7749,9 +7749,9 @@
}
},
"node_modules/mediabunny": {
- "version": "1.14.1",
- "resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.14.1.tgz",
- "integrity": "sha512-TjLg8GQGGsnGePcA6i0NItGe9a5G8WaleCGaXpb5V5okwuK5KpNKfcmTZXItnjfPLo7FvfEZI0NFp1lIR8Os7Q==",
+ "version": "1.14.2",
+ "resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.14.2.tgz",
+ "integrity": "sha512-JCup3rpsJGIw3I7E7bZSmVEANCr/OwfwkH3g/Wq1TcHEJ7ymLLw+jC3+/Nnn/JLXzFnw8/I6fV3xECLnFedfHg==",
"license": "MPL-2.0",
"peer": true,
"workspaces": [
@@ -12242,7 +12242,7 @@
},
"packages/mp3-encoder": {
"name": "@mediabunny/mp3-encoder",
- "version": "1.14.2",
+ "version": "1.14.3",
"license": "MPL-2.0",
"devDependencies": {
"@types/emscripten": "^1.40.1"
diff --git a/package.json b/package.json
index b3d43b7..5ef01e4 100644
--- a/package.json
+++ b/package.json
@@ -1,7 +1,7 @@
{
"name": "mediabunny",
"author": "Vanilagy",
- "version": "1.14.2",
+ "version": "1.14.3",
"description": "Pure TypeScript media toolkit for reading, writing, and converting media files, directly in the browser.",
"type": "module",
"workspaces": [
diff --git a/packages/mp3-encoder/package.json b/packages/mp3-encoder/package.json
index 530fce7..14cb8af 100644
--- a/packages/mp3-encoder/package.json
+++ b/packages/mp3-encoder/package.json
@@ -1,7 +1,7 @@
{
"name": "@mediabunny/mp3-encoder",
"author": "Vanilagy",
- "version": "1.14.2",
+ "version": "1.14.3",
"description": "MP3 encoder extension for Mediabunny, based on LAME.",
"main": "./dist/bundles/mediabunny-mp3-encoder.mjs",
"module": "./dist/bundles/mediabunny-mp3-encoder.mjs",
diff --git a/src/codec-data.ts b/src/codec-data.ts
index d315199..895569f 100644
--- a/src/codec-data.ts
+++ b/src/codec-data.ts
@@ -26,8 +26,27 @@ import { EncodedPacket, PacketType } from './packet';
// Rec. ITU-T H.265
// https://stackoverflow.com/questions/24884827
+export enum AvcNalUnitType {
+ IDR = 5,
+ SPS = 7,
+ PPS = 8,
+ SPS_EXT = 13,
+}
+
+export enum HevcNalUnitType {
+ RASL_N = 8,
+ RASL_R = 9,
+ BLA_W_LP = 16,
+ RSV_IRAP_VCL23 = 23,
+ VPS_NUT = 32,
+ SPS_NUT = 33,
+ PPS_NUT = 34,
+ PREFIX_SEI_NUT = 39,
+ SUFFIX_SEI_NUT = 40,
+}
+
/** Finds all NAL units in an AVC packet in Annex B format. */
-const findNalUnitsInAnnexB = (packetData: Uint8Array) => {
+export const findNalUnitsInAnnexB = (packetData: Uint8Array) => {
const nalUnits: Uint8Array[] = [];
let i = 0;
@@ -184,6 +203,21 @@ export type AvcDecoderConfigurationRecord = {
sequenceParameterSetExt: Uint8Array[] | null;
};
+export const extractAvcNalUnits = (packetData: Uint8Array, decoderConfig: VideoDecoderConfig) => {
+ if (decoderConfig.description) {
+ // Stream is length-prefixed. Let's extract the size of the length prefix from the decoder config
+
+ const bytes = toUint8Array(decoderConfig.description);
+ const lengthSizeMinusOne = bytes[4]! & 0b11;
+ const lengthSize = (lengthSizeMinusOne + 1) as 1 | 2 | 3 | 4;
+
+ return findNalUnitsInLengthPrefixed(packetData, lengthSize);
+ } else {
+ // Stream is in Annex B format
+ return findNalUnitsInAnnexB(packetData);
+ }
+};
+
const extractNalUnitTypeForAvc = (data: Uint8Array) => {
return data[0]! & 0x1F;
};
@@ -193,9 +227,9 @@ export const extractAvcDecoderConfigurationRecord = (packetData: Uint8Array) =>
try {
const nalUnits = findNalUnitsInAnnexB(packetData);
- const spsUnits = nalUnits.filter(unit => extractNalUnitTypeForAvc(unit) === 7);
- const ppsUnits = nalUnits.filter(unit => extractNalUnitTypeForAvc(unit) === 8);
- const spsExtUnits = nalUnits.filter(unit => extractNalUnitTypeForAvc(unit) === 13);
+ const spsUnits = nalUnits.filter(unit => extractNalUnitTypeForAvc(unit) === AvcNalUnitType.SPS);
+ const ppsUnits = nalUnits.filter(unit => extractNalUnitTypeForAvc(unit) === AvcNalUnitType.PPS);
+ const spsExtUnits = nalUnits.filter(unit => extractNalUnitTypeForAvc(unit) === AvcNalUnitType.SPS_EXT);
if (spsUnits.length === 0) {
return null;
@@ -337,12 +371,6 @@ export const serializeAvcDecoderConfigurationRecord = (record: AvcDecoderConfigu
return new Uint8Array(bytes);
};
-const NALU_TYPE_VPS = 32;
-const NALU_TYPE_SPS = 33;
-const NALU_TYPE_PPS = 34;
-const NALU_TYPE_SEI_PREFIX = 39;
-const NALU_TYPE_SEI_SUFFIX = 40;
-
// Data specified in ISO 14496-15
export type HevcDecoderConfigurationRecord = {
configurationVersion: number;
@@ -369,7 +397,22 @@ export type HevcDecoderConfigurationRecord = {
}[];
};
-const extractNalUnitTypeForHevc = (data: Uint8Array) => {
+export const extractHevcNalUnits = (packetData: Uint8Array, decoderConfig: VideoDecoderConfig) => {
+ if (decoderConfig.description) {
+ // Stream is length-prefixed. Let's extract the size of the length prefix from the decoder config
+
+ const bytes = toUint8Array(decoderConfig.description);
+ const lengthSizeMinusOne = bytes[21]! & 0b11;
+ const lengthSize = (lengthSizeMinusOne + 1) as 1 | 2 | 3 | 4;
+
+ return findNalUnitsInLengthPrefixed(packetData, lengthSize);
+ } else {
+ // Stream is in Annex B format
+ return findNalUnitsInAnnexB(packetData);
+ }
+};
+
+export const extractNalUnitTypeForHevc = (data: Uint8Array) => {
return (data[0]! >> 1) & 0x3F;
};
@@ -380,12 +423,12 @@ export const extractHevcDecoderConfigurationRecord = (
try {
const nalUnits = findNalUnitsInAnnexB(packetData);
- const vpsUnits = nalUnits.filter(unit => extractNalUnitTypeForHevc(unit) === NALU_TYPE_VPS);
- const spsUnits = nalUnits.filter(unit => extractNalUnitTypeForHevc(unit) === NALU_TYPE_SPS);
- const ppsUnits = nalUnits.filter(unit => extractNalUnitTypeForHevc(unit) === NALU_TYPE_PPS);
+ const vpsUnits = nalUnits.filter(unit => extractNalUnitTypeForHevc(unit) === HevcNalUnitType.VPS_NUT);
+ const spsUnits = nalUnits.filter(unit => extractNalUnitTypeForHevc(unit) === HevcNalUnitType.SPS_NUT);
+ const ppsUnits = nalUnits.filter(unit => extractNalUnitTypeForHevc(unit) === HevcNalUnitType.PPS_NUT);
const seiUnits = nalUnits.filter(
- unit => extractNalUnitTypeForHevc(unit) === NALU_TYPE_SEI_PREFIX
- || extractNalUnitTypeForHevc(unit) === NALU_TYPE_SEI_SUFFIX,
+ unit => extractNalUnitTypeForHevc(unit) === HevcNalUnitType.PREFIX_SEI_NUT
+ || extractNalUnitTypeForHevc(unit) === HevcNalUnitType.SUFFIX_SEI_NUT,
);
if (spsUnits.length === 0 || ppsUnits.length === 0) return null;
@@ -521,7 +564,7 @@ export const extractHevcDecoderConfigurationRecord = (
? [
{
arrayCompleteness: 1,
- nalUnitType: NALU_TYPE_VPS,
+ nalUnitType: HevcNalUnitType.VPS_NUT,
nalUnits: vpsUnits,
},
]
@@ -530,7 +573,7 @@ export const extractHevcDecoderConfigurationRecord = (
? [
{
arrayCompleteness: 1,
- nalUnitType: NALU_TYPE_SPS,
+ nalUnitType: HevcNalUnitType.SPS_NUT,
nalUnits: spsUnits,
},
]
@@ -539,7 +582,7 @@ export const extractHevcDecoderConfigurationRecord = (
? [
{
arrayCompleteness: 1,
- nalUnitType: NALU_TYPE_PPS,
+ nalUnitType: HevcNalUnitType.PPS_NUT,
nalUnits: ppsUnits,
},
]
@@ -1440,22 +1483,9 @@ export const determineVideoPacketType = async (
const decoderConfig = await videoTrack.getDecoderConfig();
assert(decoderConfig);
- let nalUnits: Uint8Array[];
+ const nalUnits = extractAvcNalUnits(packet.data, decoderConfig);
+ const isKeyframe = nalUnits.some(x => extractNalUnitTypeForAvc(x) === AvcNalUnitType.IDR);
- if (decoderConfig.description) {
- // Stream is length-prefixed. Let's extract the size of the length prefix from the decoder config
-
- const bytes = toUint8Array(decoderConfig.description);
- const lengthSizeMinusOne = bytes[4]! & 0b11;
- const lengthSize = (lengthSizeMinusOne + 1) as 1 | 2 | 3 | 4;
-
- nalUnits = findNalUnitsInLengthPrefixed(packet.data, lengthSize);
- } else {
- // Stream is in Annex B format
- nalUnits = findNalUnitsInAnnexB(packet.data);
- }
-
- const isKeyframe = nalUnits.some(x => extractNalUnitTypeForAvc(x) === 5);
return isKeyframe ? 'key' : 'delta';
};
@@ -1463,25 +1493,12 @@ export const determineVideoPacketType = async (
const decoderConfig = await videoTrack.getDecoderConfig();
assert(decoderConfig);
- let nalUnits: Uint8Array[];
-
- if (decoderConfig.description) {
- // Stream is length-prefixed. Let's extract the size of the length prefix from the decoder config
-
- const bytes = toUint8Array(decoderConfig.description);
- const lengthSizeMinusOne = bytes[21]! & 0b11;
- const lengthSize = (lengthSizeMinusOne + 1) as 1 | 2 | 3 | 4;
-
- nalUnits = findNalUnitsInLengthPrefixed(packet.data, lengthSize);
- } else {
- // Stream is in Annex B format
- nalUnits = findNalUnitsInAnnexB(packet.data);
- }
-
+ const nalUnits = extractHevcNalUnits(packet.data, decoderConfig);
const isKeyframe = nalUnits.some((x) => {
const type = extractNalUnitTypeForHevc(x);
- return 16 <= type && type <= 23;
+ return HevcNalUnitType.BLA_W_LP <= type && type <= HevcNalUnitType.RSV_IRAP_VCL23;
});
+
return isKeyframe ? 'key' : 'delta';
};
diff --git a/src/media-sink.ts b/src/media-sink.ts
index 643f259..e11b4b3 100644
--- a/src/media-sink.ts
+++ b/src/media-sink.ts
@@ -7,6 +7,7 @@
*/
import { parsePcmCodec, PCM_AUDIO_CODECS, PcmAudioCodec, VideoCodec, AudioCodec } from './codec';
+import { extractHevcNalUnits, extractNalUnitTypeForHevc, HevcNalUnitType } from './codec-data';
import { CustomVideoDecoder, customVideoDecoders, CustomAudioDecoder, customAudioDecoders } from './custom-coder';
import { InputAudioTrack, InputTrack, InputVideoTrack } from './input-track';
import {
@@ -624,8 +625,8 @@ export abstract class BaseMediaSampleSink<
const nextPacket = await packetSink.getNextPacket(currentPacket);
assert(nextPacket);
- currentPacket = nextPacket;
decoder.decode(nextPacket);
+ currentPacket = nextPacket;
}
maxSequenceNumber = -1;
@@ -757,12 +758,14 @@ class VideoDecoderWrapper extends DecoderWrapper {
inputTimestamps: number[] = []; // Timestamps input into the decoder, sorted.
sampleQueue: VideoSample[] = []; // Safari-specific thing, check usage.
+ currentPacketIndex = 0;
+ raslSkipped = false; // For HEVC stuff
constructor(
onSample: (sample: VideoSample) => unknown,
onError: (error: DOMException) => unknown,
- codec: VideoCodec,
- decoderConfig: VideoDecoderConfig,
+ public codec: VideoCodec,
+ public decoderConfig: VideoDecoderConfig,
public rotation: Rotation,
public timeResolution: number,
) {
@@ -848,6 +851,26 @@ class VideoDecoderWrapper extends DecoderWrapper {
}
decode(packet: EncodedPacket) {
+ if (this.codec === 'hevc' && this.currentPacketIndex > 0 && !this.raslSkipped) {
+ // If we're using HEVC, we need to make sure to skip any RASL slices that follow a non-IDR key frame such as
+ // CRA_NUT. This is because RASL slices cannot be decoded without data before the CRA_NUT. Browsers behave
+ // differently here: Chromium drops the packets, Safari throws a decoder error. Either way, it's not good
+ // and causes bugs upstream. So, let's take the dropping into our own hands.
+ const nalUnits = extractHevcNalUnits(packet.data, this.decoderConfig);
+ const hasRaslPicture = nalUnits.some((x) => {
+ const type = extractNalUnitTypeForHevc(x);
+ return type === HevcNalUnitType.RASL_N || type === HevcNalUnitType.RASL_R;
+ });
+
+ if (hasRaslPicture) {
+ return; // Drop
+ }
+
+ this.raslSkipped = true;
+ }
+
+ this.currentPacketIndex++;
+
if (this.customDecoder) {
this.customDecoderQueueSize++;
void this.customDecoderCallSerializer
@@ -879,6 +902,9 @@ class VideoDecoderWrapper extends DecoderWrapper {
this.sampleQueue.length = 0;
}
+
+ this.currentPacketIndex = 0;
+ this.raslSkipped = false;
}
close() {