arcodange/mediabunny
1
0
Fork 0
mirror of https://github.com/arcodange-org/mediabunny.git synced 2026-10-02 21:33:52 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add packet type derivation logic, new verifyKeyPackets options, new determinePacketType method

Browse Source
This commit is contained in:
Vanilagy
2025-07-24 16:09:01 +02:00
parent 904ac2e366
commit 5e933d9362
12 changed files with 581 additions and 204 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dev/convert.html
+4 -2
View File
@@ -21,7 +21,7 @@
chunked: true, chunked: true,
chunkSize: 2**20 chunkSize: 2**20
}); });
const outputFormat = new Mediabunny.WavOutputFormat({ large: true }); const outputFormat = new Mediabunny.Mp4OutputFormat({});
const button = document.createElement('button'); const button = document.createElement('button');
button.textContent = 'Cancel'; button.textContent = 'Cancel';
@@ -70,6 +70,8 @@
}, },
*/ */
video: { video: {
forceTranscode: true,
codec: 'av1',
//discard: true, //discard: true,
//width: 1280, //width: 1280,
//discard: true, //discard: true,
@@ -87,7 +89,7 @@
}, },
trim: { trim: {
start: 0, start: 0,
end: 10 end: 20
}, },
}); });
console.log(conversion); console.log(conversion);
dev/demux.html
+57
View File
@@ -16,6 +16,62 @@
source source
}); });
const videoTrack = await input.getPrimaryVideoTrack();
const packetSink = new Mediabunny.EncodedPacketSink(videoTrack);
const sampleSink = new Mediabunny.VideoSampleSink(videoTrack);
for await (const packet of packetSink.packets(undefined, undefined, {verifyType: true})) {
const guess = packet.type;
const real = await videoTrack.determinePacketType(packet);
if (guess !== real) {
console.log(guess, real, packet);
}
}
console.log("don")
/*
console.time()
for await (const packet of packetSink.packets(undefined, undefined, { verifyType: true })) {
//console.log(packet)
}
console.timeEnd()
*/
//console.log(await packetSink.getPacket(6.666666666666667, { verifyType: true }))
/*
for await (const packet of packetSink.packets()) {
const guess = packet.type;
const real = await videoTrack.determinePacketType(packet);
if (guess !== real) {
console.log(guess, real, packet);
}
}
console.log("done")
*/
/*
const timestamp = 6.666666666666667;
const thePacket = await packetSink.getPacket(timestamp);
console.log(videoTrack.codec, thePacket, await videoTrack.determinePacketType(thePacket));
sampleSink.getSample(thePacket.timestamp);
*/
/*
let packet = await packetSink.getFirstPacket();
while (packet) {
console.log(packet)
const sample = await sampleSink.getSample(packet.timestamp);
packet = await packetSink.getNextKeyPacket(packet);
}
*/
/*
const canvas = document.createElement('canvas'); const canvas = document.createElement('canvas');
canvas.width = 1920; canvas.width = 1920;
canvas.height = 1080; canvas.height = 1080;
@@ -30,6 +86,7 @@
console.log(sample); console.log(sample);
sample.draw(ctx, 1500, 500, 50, 50, 0, 0); sample.draw(ctx, 1500, 500, 50, 50, 0, 0);
*/
/* /*
let timestamps = []; let timestamps = [];
docs/guide/media-sinks.md
+26
View File
@@ -144,6 +144,32 @@ for await (const packet of sink.packets(start, end)) {
The `packets` method is more performant than manual iteration as it will intelligently preload future packets before they are needed. The `packets` method is more performant than manual iteration as it will intelligently preload future packets before they are needed.
#### Verifying key packets
By default, packet types are determined using the metadata provided by the containing file. Some files can erroneously label some delta packets as key packets, leading to potential decoder errors. To be guaranteed that a key packet is actually a key packet, you can enable the `verifyKeyPackets` option:
```ts
// If the packet returned by this method has type: 'key', it's guaranteed
// to be a key packet.
await sink.getPacket(5, { verifyKeyPackets: true });
// Returned packets are guaranteed to be key packets
await sink.getKeyPacket(10, { verifyKeyPackets: true });
await sink.getNextKeyPacket(packet, { verifyKeyPackets: true });
// Also works for the iterator:
for await (const packet of sink.packets(
undefined,
undefined,
{ verifyKeyPackets: true },
)) {
// ...
}
```
::: info
`verifyKeyPackets` only works when `metadataOnly` is not also enabled.
:::
#### Metadata-only packet retrieval #### Metadata-only packet retrieval
Sometimes, you're only interested in a packet's metadata (timestamp, duration, type, ...) and not in its encoded media data. All methods on `EncodedPacketSink` accept a final `options` parameter which you can use to retrieve [metadata-only packets](./packets-and-samples#metadata-only-packets): Sometimes, you're only interested in a packet's metadata (timestamp, duration, type, ...) and not in its encoded media data. All methods on `EncodedPacketSink` accept a final `options` parameter which you can use to retrieve [metadata-only packets](./packets-and-samples#metadata-only-packets):
docs/guide/packets-and-samples.md
+10
View File
@@ -133,6 +133,16 @@ encodedPacket.type; // => PacketType ('key' | 'delta')
For example, in a video track, it is common to have a key frame about every few seconds. When seeking, if the user seeks to a position shortly after a key frame, the decoded data can be shown quickly; if they seek far away from a key frame, the decoder must first crunch through many delta frames before it can show anything. For example, in a video track, it is common to have a key frame about every few seconds. When seeking, if the user seeks to a position shortly after a key frame, the decoded data can be shown quickly; if they seek far away from a key frame, the decoder must first crunch through many delta frames before it can show anything.
#### Determining a packet's actual type
The `type` field is derived from metadata in the containing file, which can sometimes (in rare cases) be incorrect. To determine a packet's actual type with certainty, you can do this:
```ts
// `packet` must come from the InputTrack `track`
const type = await track.determinePacketType(packet); // => PacketType | null
```
This determines the packet's type by looking into its bitstream. `null` is returned when the type couldn't be determined.
--- ---
You can query the packet's timing information: You can query the packet's timing information:
package-lock.json
Generated
+2 -2
View File
@@ -1,12 +1,12 @@
{ {
"name": "mediabunny", "name": "mediabunny",
"version": "1.2.0", "version": "1.3.0",
"lockfileVersion": 3, "lockfileVersion": 3,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "mediabunny", "name": "mediabunny",
"version": "1.2.0", "version": "1.3.0",
"license": "MPL-2.0", "license": "MPL-2.0",
"dependencies": { "dependencies": {
"@types/dom-mediacapture-transform": "^0.1.11", "@types/dom-mediacapture-transform": "^0.1.11",
package.json
+2 -2
View File
@@ -1,7 +1,7 @@
{ {
"name": "mediabunny", "name": "mediabunny",
"author": "Vanilagy", "author": "Vanilagy",
"version": "1.2.0", "version": "1.3.0",
"description": "Pure TypeScript media toolkit for reading, writing, and converting media files, directly in the browser.", "description": "Pure TypeScript media toolkit for reading, writing, and converting media files, directly in the browser.",
"type": "module", "type": "module",
"main": "./dist/bundles/mediabunny.cjs", "main": "./dist/bundles/mediabunny.cjs",
@@ -31,7 +31,7 @@
"docs:preview": "vitepress preview docs", "docs:preview": "vitepress preview docs",
"dev": "vite", "dev": "vite",
"examples:build": "vite build", "examples:build": "vite build",
"fix-build-import-paths": "find dist -name \"*.js\" -type f -exec sed -i -r \"s/ from '([^']+)';/ from '\\1.js';/g\" {} \\;", "fix-build-import-paths": "find dist -name \"*.js\" -type f -exec sed -i '' \"s/ from '\\([^']*\\)';/ from '\\1.js';/g\" {} \\;",
"append-namespace": "echo 'export as namespace Mediabunny;' >> dist/mediabunny.d.ts" "append-namespace": "echo 'export as namespace Mediabunny;' >> dist/mediabunny.d.ts"
}, },
"license": "MPL-2.0", "license": "MPL-2.0",
src/codec-data.ts
+342 -180
View File
@@ -7,7 +7,18 @@
*/ */
import { VP9_LEVEL_TABLE } from './codec'; import { VP9_LEVEL_TABLE } from './codec';
import { assert, Bitstream, last, readExpGolomb, readSignedExpGolomb, toDataView } from './misc'; import { InputVideoTrack } from './input-track';
import {
assert,
assertNever,
Bitstream,
last,
readExpGolomb,
readSignedExpGolomb,
toDataView,
toUint8Array,
} from './misc';
import { EncodedPacket, PacketType } from './packet';
// References for AVC/HEVC code: // References for AVC/HEVC code:
// ISO 14496-15 // ISO 14496-15
@@ -72,6 +83,39 @@ const findNalUnitsInAnnexB = (packetData: Uint8Array) => {
return nalUnits; return nalUnits;
}; };
/** Finds all NAL units in an AVC packet in length-prefixed format. */
const findNalUnitsInLengthPrefixed = (packetData: Uint8Array, lengthSize: 1 | 2 | 3 | 4) => {
const nalUnits: Uint8Array[] = [];
let offset = 0;
const dataView = new DataView(packetData.buffer, packetData.byteOffset, packetData.byteLength);
while (offset + lengthSize <= packetData.length) {
let nalUnitLength: number;
if (lengthSize === 1) {
nalUnitLength = dataView.getUint8(offset);
} else if (lengthSize === 2) {
nalUnitLength = dataView.getUint16(offset, false);
} else if (lengthSize === 3) {
nalUnitLength = (dataView.getUint16(offset, false) << 8) + dataView.getUint8(offset + 2);
} else if (lengthSize === 4) {
nalUnitLength = dataView.getUint32(offset, false);
} else {
assertNever(lengthSize);
assert(false);
}
offset += lengthSize;
const nalUnit = packetData.subarray(offset, offset + nalUnitLength);
nalUnits.push(nalUnit);
offset += nalUnitLength;
}
return nalUnits;
};
const removeEmulationPreventionBytes = (data: Uint8Array) => { const removeEmulationPreventionBytes = (data: Uint8Array) => {
const result: number[] = []; const result: number[] = [];
const len = data.length; const len = data.length;
@@ -879,30 +923,6 @@ export const extractVp9CodecInfoFromPacket = (
// https://storage.googleapis.com/downloads.webmproject.org/docs/vp9/vp9-bitstream-specification-v0.7-20170222-draft.pdf // https://storage.googleapis.com/downloads.webmproject.org/docs/vp9/vp9-bitstream-specification-v0.7-20170222-draft.pdf
// http://downloads.webmproject.org/docs/vp9/vp9-bitstream_superframe-and-uncompressed-header_v1.0.pdf // http://downloads.webmproject.org/docs/vp9/vp9-bitstream_superframe-and-uncompressed-header_v1.0.pdf
// Handle superframe
const lastByte = packet[packet.length - 1];
if (lastByte && (lastByte & 0xe0) === 0xc0) { // Is superframe
const bytesPerFrameSize = ((lastByte & 0x18) >> 3) + 1;
const numFrames = (lastByte & 0x07) + 1;
const indexSize = 2 + numFrames * bytesPerFrameSize;
// Verify matching marker bytes
if (packet[packet.length - indexSize] !== lastByte) {
return null;
}
// Get first frame size
let frameSize = 0;
const offset = packet.length - indexSize + 1;
for (let i = 0; i < bytesPerFrameSize; i++) {
if (!packet[offset + i]) return null;
frameSize |= packet[offset + i]! << (8 * i);
}
packet = packet.subarray(0, frameSize);
}
const bitstream = new Bitstream(packet); const bitstream = new Bitstream(packet);
// Frame marker (0b10) // Frame marker (0b10)
@@ -1048,13 +1068,8 @@ export type Av1CodecInfo = {
chromaSamplePosition: number; chromaSamplePosition: number;
}; };
/** /** Iterates over all OBUs in an AV1 packet bistream. */
* When AV1 codec information is not provided by the container, we can still try to extract the information by digging export function* iterateAv1PacketObus(packet: Uint8Array) {
* into the AV1 bitstream.
*/
export const extractAv1CodecInfoFromPacket = (
packet: Uint8Array,
): Av1CodecInfo | null => {
// https://aomediacodec.github.io/av1-spec/av1-spec.pdf // https://aomediacodec.github.io/av1-spec/av1-spec.pdf
const bitstream = new Bitstream(packet); const bitstream = new Bitstream(packet);
@@ -1064,7 +1079,6 @@ export const extractAv1CodecInfoFromPacket = (
for (let i = 0; i < 8; i++) { for (let i = 0; i < 8; i++) {
const byte = bitstream.readAlignedByte(); const byte = bitstream.readAlignedByte();
if (byte === undefined) return 0;
value |= ((byte & 0x7f) << (i * 7)); value |= ((byte & 0x7f) << (i * 7));
@@ -1088,11 +1102,11 @@ export const extractAv1CodecInfoFromPacket = (
while (bitstream.getBitsLeft() >= 8) { while (bitstream.getBitsLeft() >= 8) {
// Parse OBU header // Parse OBU header
const obuHeader = bitstream.readBits(8); bitstream.skipBits(1);
const obuType = bitstream.readBits(4);
const obuType = (obuHeader >> 3) & 0xf; const obuExtension = bitstream.readBits(1);
const obuExtension = (obuHeader >> 2) & 0x1; const obuHasSizeField = bitstream.readBits(1);
const obuHasSizeField = (obuHeader >> 1) & 0x1; bitstream.skipBits(1);
// Skip extension header if present // Skip extension header if present
if (obuExtension) { if (obuExtension) {
@@ -1103,159 +1117,180 @@ export const extractAv1CodecInfoFromPacket = (
let obuSize: number; let obuSize: number;
if (obuHasSizeField) { if (obuHasSizeField) {
const obuSizeValue = readLeb128(); const obuSizeValue = readLeb128();
if (obuSizeValue === null) return null; // It was invalid if (obuSizeValue === null) return; // It was invalid
obuSize = obuSizeValue; obuSize = obuSizeValue;
} else { } else {
// Calculate remaining bits and convert to bytes, rounding down // Calculate remaining bits and convert to bytes, rounding down
obuSize = Math.floor(bitstream.getBitsLeft() / 8); obuSize = Math.floor(bitstream.getBitsLeft() / 8);
} }
// We're only interested in Sequence Header OBU (type 1) assert(bitstream.pos % 8 === 0);
if (obuType === 1) {
// Read sequence header fields
const seqProfile = bitstream.readBits(3);
// eslint-disable-next-line @typescript-eslint/no-unused-vars yield {
const stillPicture = bitstream.readBits(1); type: obuType,
data: packet.subarray(bitstream.pos / 8, bitstream.pos / 8 + obuSize),
const reducedStillPictureHeader = bitstream.readBits(1); };
let seqLevel = 0;
let seqTier = 0;
let bufferDelayLengthMinus1 = 0;
if (reducedStillPictureHeader) {
seqLevel = bitstream.readBits(5);
} else {
// Parse timing_info_present_flag
const timingInfoPresentFlag = bitstream.readBits(1);
if (timingInfoPresentFlag) {
// Skip timing info (num_units_in_display_tick, time_scale, equal_picture_interval)
bitstream.skipBits(32); // num_units_in_display_tick
bitstream.skipBits(32); // time_scale
const equalPictureInterval = bitstream.readBits(1);
if (equalPictureInterval) {
// Skip num_ticks_per_picture_minus_1 (uvlc)
// Since this is variable length, we'd need to implement uvlc reading
// For now, we'll return null as this is rare
return null;
}
}
// Parse decoder_model_info_present_flag
const decoderModelInfoPresentFlag = bitstream.readBits(1);
if (decoderModelInfoPresentFlag) {
// Store buffer_delay_length_minus_1 instead of just skipping
bufferDelayLengthMinus1 = bitstream.readBits(5);
bitstream.skipBits(32); // num_units_in_decoding_tick
bitstream.skipBits(5); // buffer_removal_time_length_minus_1
bitstream.skipBits(5); // frame_presentation_time_length_minus_1
}
// Parse operating_points_cnt_minus_1
const operatingPointsCntMinus1 = bitstream.readBits(5);
// For each operating point
for (let i = 0; i <= operatingPointsCntMinus1; i++) {
// operating_point_idc[i]
bitstream.skipBits(12);
// seq_level_idx[i]
const seqLevelIdx = bitstream.readBits(5);
if (i === 0) {
seqLevel = seqLevelIdx;
}
if (seqLevelIdx > 7) {
// seq_tier[i]
const seqTierTemp = bitstream.readBits(1);
if (i === 0) {
seqTier = seqTierTemp;
}
}
if (decoderModelInfoPresentFlag) {
// decoder_model_present_for_this_op[i]
const decoderModelPresentForThisOp = bitstream.readBits(1);
if (decoderModelPresentForThisOp) {
const n = bufferDelayLengthMinus1 + 1;
bitstream.skipBits(n); // decoder_buffer_delay[op]
bitstream.skipBits(n); // encoder_buffer_delay[op]
bitstream.skipBits(1); // low_delay_mode_flag[op]
}
}
// initial_display_delay_present_flag
const initialDisplayDelayPresentFlag = bitstream.readBits(1);
if (initialDisplayDelayPresentFlag) {
// initial_display_delay_minus_1[i]
bitstream.skipBits(4);
}
}
}
const highBitdepth = bitstream.readBits(1);
let bitDepth = 8;
if (seqProfile === 2 && highBitdepth) {
const twelveBit = bitstream.readBits(1);
bitDepth = twelveBit ? 12 : 10;
} else if (seqProfile <= 2) {
bitDepth = highBitdepth ? 10 : 8;
}
let monochrome = 0;
if (seqProfile !== 1) {
monochrome = bitstream.readBits(1);
}
let chromaSubsamplingX = 1;
let chromaSubsamplingY = 1;
let chromaSamplePosition = 0;
if (!monochrome) {
if (seqProfile === 0) {
chromaSubsamplingX = 1;
chromaSubsamplingY = 1;
} else if (seqProfile === 1) {
chromaSubsamplingX = 0;
chromaSubsamplingY = 0;
} else {
if (bitDepth === 12) {
chromaSubsamplingX = bitstream.readBits(1);
if (chromaSubsamplingX) {
chromaSubsamplingY = bitstream.readBits(1);
}
}
}
if (chromaSubsamplingX && chromaSubsamplingY) {
chromaSamplePosition = bitstream.readBits(2);
}
}
return {
profile: seqProfile,
level: seqLevel,
tier: seqTier,
bitDepth,
monochrome,
chromaSubsamplingX,
chromaSubsamplingY,
chromaSamplePosition,
};
}
// Move to next OBU // Move to next OBU
// The OBU size is in bytes, so skip that many bytes.
bitstream.skipBits(obuSize * 8); bitstream.skipBits(obuSize * 8);
} }
};
/**
* When AV1 codec information is not provided by the container, we can still try to extract the information by digging
* into the AV1 bitstream.
*/
export const extractAv1CodecInfoFromPacket = (
packet: Uint8Array,
): Av1CodecInfo | null => {
// https://aomediacodec.github.io/av1-spec/av1-spec.pdf
for (const { type, data } of iterateAv1PacketObus(packet)) {
if (type !== 1) {
continue; // 1 == OBU_SEQUENCE_HEADER
}
const bitstream = new Bitstream(data);
// Read sequence header fields
const seqProfile = bitstream.readBits(3);
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const stillPicture = bitstream.readBits(1);
const reducedStillPictureHeader = bitstream.readBits(1);
let seqLevel = 0;
let seqTier = 0;
let bufferDelayLengthMinus1 = 0;
if (reducedStillPictureHeader) {
seqLevel = bitstream.readBits(5);
} else {
// Parse timing_info_present_flag
const timingInfoPresentFlag = bitstream.readBits(1);
if (timingInfoPresentFlag) {
// Skip timing info (num_units_in_display_tick, time_scale, equal_picture_interval)
bitstream.skipBits(32); // num_units_in_display_tick
bitstream.skipBits(32); // time_scale
const equalPictureInterval = bitstream.readBits(1);
if (equalPictureInterval) {
// Skip num_ticks_per_picture_minus_1 (uvlc)
// Since this is variable length, we'd need to implement uvlc reading
// For now, we'll return null as this is rare
return null;
}
}
// Parse decoder_model_info_present_flag
const decoderModelInfoPresentFlag = bitstream.readBits(1);
if (decoderModelInfoPresentFlag) {
// Store buffer_delay_length_minus_1 instead of just skipping
bufferDelayLengthMinus1 = bitstream.readBits(5);
bitstream.skipBits(32); // num_units_in_decoding_tick
bitstream.skipBits(5); // buffer_removal_time_length_minus_1
bitstream.skipBits(5); // frame_presentation_time_length_minus_1
}
// Parse operating_points_cnt_minus_1
const operatingPointsCntMinus1 = bitstream.readBits(5);
// For each operating point
for (let i = 0; i <= operatingPointsCntMinus1; i++) {
// operating_point_idc[i]
bitstream.skipBits(12);
// seq_level_idx[i]
const seqLevelIdx = bitstream.readBits(5);
if (i === 0) {
seqLevel = seqLevelIdx;
}
if (seqLevelIdx > 7) {
// seq_tier[i]
const seqTierTemp = bitstream.readBits(1);
if (i === 0) {
seqTier = seqTierTemp;
}
}
if (decoderModelInfoPresentFlag) {
// decoder_model_present_for_this_op[i]
const decoderModelPresentForThisOp = bitstream.readBits(1);
if (decoderModelPresentForThisOp) {
const n = bufferDelayLengthMinus1 + 1;
bitstream.skipBits(n); // decoder_buffer_delay[op]
bitstream.skipBits(n); // encoder_buffer_delay[op]
bitstream.skipBits(1); // low_delay_mode_flag[op]
}
}
// initial_display_delay_present_flag
const initialDisplayDelayPresentFlag = bitstream.readBits(1);
if (initialDisplayDelayPresentFlag) {
// initial_display_delay_minus_1[i]
bitstream.skipBits(4);
}
}
}
const highBitdepth = bitstream.readBits(1);
let bitDepth = 8;
if (seqProfile === 2 && highBitdepth) {
const twelveBit = bitstream.readBits(1);
bitDepth = twelveBit ? 12 : 10;
} else if (seqProfile <= 2) {
bitDepth = highBitdepth ? 10 : 8;
}
let monochrome = 0;
if (seqProfile !== 1) {
monochrome = bitstream.readBits(1);
}
let chromaSubsamplingX = 1;
let chromaSubsamplingY = 1;
let chromaSamplePosition = 0;
if (!monochrome) {
if (seqProfile === 0) {
chromaSubsamplingX = 1;
chromaSubsamplingY = 1;
} else if (seqProfile === 1) {
chromaSubsamplingX = 0;
chromaSubsamplingY = 0;
} else {
if (bitDepth === 12) {
chromaSubsamplingX = bitstream.readBits(1);
if (chromaSubsamplingX) {
chromaSubsamplingY = bitstream.readBits(1);
}
}
}
if (chromaSubsamplingX && chromaSubsamplingY) {
chromaSamplePosition = bitstream.readBits(2);
}
}
return {
profile: seqProfile,
level: seqLevel,
tier: seqTier,
bitDepth,
monochrome,
chromaSubsamplingX,
chromaSubsamplingY,
chromaSamplePosition,
};
}
return null; return null;
}; };
@@ -1392,3 +1427,130 @@ export const parseModesFromVorbisSetupPacket = (setupHeader: Uint8Array) => {
return { modeBlockflags }; return { modeBlockflags };
}; };
/** Determines a packet's type (key or delta) by digging into the packet bitstream. */
export const determineVideoPacketType = async (
videoTrack: InputVideoTrack,
packet: EncodedPacket,
): Promise<PacketType | null> => {
assert(videoTrack.codec);
switch (videoTrack.codec) {
case 'avc': {
const decoderConfig = await videoTrack.getDecoderConfig();
assert(decoderConfig);
let nalUnits: Uint8Array[];
if (decoderConfig.description) {
// Stream is length-prefixed. Let's extract the size of the length prefix from the decoder config
const bytes = toUint8Array(decoderConfig.description);
const lengthSizeMinusOne = bytes[4]! & 0b11;
const lengthSize = (lengthSizeMinusOne + 1) as 1 | 2 | 3 | 4;
nalUnits = findNalUnitsInLengthPrefixed(packet.data, lengthSize);
} else {
// Stream is in Annex B format
nalUnits = findNalUnitsInAnnexB(packet.data);
}
const isKeyframe = nalUnits.some(x => extractNalUnitTypeForAvc(x) === 5);
return isKeyframe ? 'key' : 'delta';
};
case 'hevc': {
const decoderConfig = await videoTrack.getDecoderConfig();
assert(decoderConfig);
let nalUnits: Uint8Array[];
if (decoderConfig.description) {
// Stream is length-prefixed. Let's extract the size of the length prefix from the decoder config
const bytes = toUint8Array(decoderConfig.description);
const lengthSizeMinusOne = bytes[21]! & 0b11;
const lengthSize = (lengthSizeMinusOne + 1) as 1 | 2 | 3 | 4;
nalUnits = findNalUnitsInLengthPrefixed(packet.data, lengthSize);
} else {
// Stream is in Annex B format
nalUnits = findNalUnitsInAnnexB(packet.data);
}
const isKeyframe = nalUnits.some((x) => {
const type = extractNalUnitTypeForHevc(x);
return 16 <= type && type <= 23;
});
return isKeyframe ? 'key' : 'delta';
};
case 'vp8': {
// VP8, once again, by far the easiest to deal with.
const frameType = packet.data[0]! & 0b1;
return frameType === 0 ? 'key' : 'delta';
};
case 'vp9': {
const bitstream = new Bitstream(packet.data);
if (bitstream.readBits(2) !== 2) {
return null;
};
const profileLowBit = bitstream.readBits(1);
const profileHighBit = bitstream.readBits(1);
const profile = (profileHighBit << 1) + profileLowBit;
// Skip reserved bit for profile 3
if (profile === 3) {
bitstream.skipBits(1);
}
const showExistingFrame = bitstream.readBits(1);
if (showExistingFrame) {
return null;
}
const frameType = bitstream.readBits(1);
return frameType === 0 ? 'key' : 'delta';
};
case 'av1': {
let reducedStillPictureHeader = false;
for (const { type, data } of iterateAv1PacketObus(packet.data)) {
if (type === 1) { // OBU_SEQUENCE_HEADER
const bitstream = new Bitstream(data);
bitstream.skipBits(4);
reducedStillPictureHeader = !!bitstream.readBits(1);
} else if (
type === 3 // OBU_FRAME_HEADER
|| type === 6 // OBU_FRAME
|| type === 7 // OBU_REDUNDANT_FRAME_HEADER
) {
if (reducedStillPictureHeader) {
return 'key';
}
const bitstream = new Bitstream(data);
const showExistingFrame = bitstream.readBits(1);
if (showExistingFrame) {
return null;
}
const frameType = bitstream.readBits(2);
return frameType === 0 ? 'key' : 'delta';
}
}
return null;
};
default: {
assertNever(videoTrack.codec);
assert(false);
};
}
};
src/conversion.ts
+1 -1
View File
@@ -498,7 +498,7 @@ export class Conversion {
? await sink.getPacket(this._endTimestamp, { metadataOnly: true }) ?? undefined ? await sink.getPacket(this._endTimestamp, { metadataOnly: true }) ?? undefined
: undefined; : undefined;
for await (const packet of sink.packets(undefined, endPacket)) { for await (const packet of sink.packets(undefined, endPacket, { verifyKeyPackets: true })) {
if (this._synchronizer.shouldWait(track.id, packet.timestamp)) { if (this._synchronizer.shouldWait(track.id, packet.timestamp)) {
await this._synchronizer.wait(packet.timestamp); await this._synchronizer.wait(packet.timestamp);
} }
src/input-track.ts
+42 -3
View File
@@ -7,11 +7,12 @@
*/ */
import { AudioCodec, MediaCodec, VideoCodec } from './codec'; import { AudioCodec, MediaCodec, VideoCodec } from './codec';
import { determineVideoPacketType } from './codec-data';
import { customAudioDecoders, customVideoDecoders } from './custom-coder'; import { customAudioDecoders, customVideoDecoders } from './custom-coder';
import { EncodedPacketSink, PacketRetrievalOptions } from './media-sink'; import { EncodedPacketSink, PacketRetrievalOptions } from './media-sink';
import { assert, Rotation } from './misc'; import { assert, Rotation } from './misc';
import { TrackType } from './output'; import { TrackType } from './output';
import { EncodedPacket } from './packet'; import { EncodedPacket, PacketType } from './packet';
/** /**
* Contains aggregate statistics about the encoded packets of a track. * Contains aggregate statistics about the encoded packets of a track.
@@ -62,6 +63,11 @@ export abstract class InputTrack {
abstract getCodecParameterString(): Promise<string | null>; abstract getCodecParameterString(): Promise<string | null>;
/** Checks if this track's packets can be decoded by the browser. */ /** Checks if this track's packets can be decoded by the browser. */
abstract canDecode(): Promise<boolean>; abstract canDecode(): Promise<boolean>;
/**
* For a given packet of this track, this method determines the actual type of this packet (key/delta) by looking
* into its bitstream. Returns null if the type couldn't be determined.
*/
abstract determinePacketType(packet: EncodedPacket): Promise<PacketType | null>;
/** Returns true iff this track is a video track. */ /** Returns true iff this track is a video track. */
isVideoTrack(): this is InputVideoTrack { isVideoTrack(): this is InputVideoTrack {
@@ -222,7 +228,10 @@ export class InputVideoTrack extends InputTrack {
|| (colorSpace.matrix as string) === 'bt2020-ncl'; || (colorSpace.matrix as string) === 'bt2020-ncl';
} }
/** Returns the decoder configuration for decoding the track's packets using a VideoDecoder. */ /**
* Returns the decoder configuration for decoding the track's packets using a VideoDecoder. Returns null if the
* track's codec is unknown.
*/
getDecoderConfig() { getDecoderConfig() {
return this._backing.getDecoderConfig(); return this._backing.getDecoderConfig();
} }
@@ -257,6 +266,21 @@ export class InputVideoTrack extends InputTrack {
return false; return false;
} }
} }
async determinePacketType(packet: EncodedPacket): Promise<PacketType | null> {
if (!(packet instanceof EncodedPacket)) {
throw new TypeError('packet must be an EncodedPacket.');
}
if (packet.isMetadataOnly) {
throw new TypeError('packet must not be metadata-only to determine its type.');
}
if (this.codec === null) {
return null;
}
return determineVideoPacketType(this, packet);
}
} }
export interface InputAudioTrackBacking extends InputTrackBacking { export interface InputAudioTrackBacking extends InputTrackBacking {
@@ -299,7 +323,10 @@ export class InputAudioTrack extends InputTrack {
return this._backing.getSampleRate(); return this._backing.getSampleRate();
} }
/** Returns the decoder configuration for decoding the track's packets using an AudioDecoder. */ /**
* Returns the decoder configuration for decoding the track's packets using an AudioDecoder. Returns null if the
* track's codec is unknown.
*/
getDecoderConfig() { getDecoderConfig() {
return this._backing.getDecoderConfig(); return this._backing.getDecoderConfig();
} }
@@ -338,4 +365,16 @@ export class InputAudioTrack extends InputTrack {
return false; return false;
} }
} }
async determinePacketType(packet: EncodedPacket): Promise<PacketType | null> {
if (!(packet instanceof EncodedPacket)) {
throw new TypeError('packet must be an EncodedPacket.');
}
if (this.codec === null) {
return null;
}
return 'key'; // No audio codec with delta packets
}
} }
src/matroska/matroska-muxer.ts
+1 -2
View File
@@ -694,8 +694,7 @@ export class MatroskaMuxer extends Muxer {
const bitstream = new Bitstream(chunk.data); const bitstream = new Bitstream(chunk.data);
// Check if it's a "superframe" bitstream.skipBits(2);
if (bitstream.readBits(2) !== 0b10) return;
const profileLowBit = bitstream.readBits(1); const profileLowBit = bitstream.readBits(1);
const profileHighBit = bitstream.readBits(1); const profileHighBit = bitstream.readBits(1);
src/media-sink.ts
+88 -10
View File
@@ -39,6 +39,14 @@ export type PacketRetrievalOptions = {
* be loaded. * be loaded.
*/ */
metadataOnly?: boolean; metadataOnly?: boolean;
/**
* When set to true, key packets will be verified upon retrieval by looking into the packet's bitstream.
* If not enabled, the packet types will be determined solely by what's stored in the containing file and may be
* incorrect, potentially leading to decoder errors. Since determining a packet's actual type requires looking into
* its data, this option cannot be enabled together with `metadataOnly`.
*/
verifyKeyPackets?: boolean;
}; };
const validatePacketRetrievalOptions = (options: PacketRetrievalOptions) => { const validatePacketRetrievalOptions = (options: PacketRetrievalOptions) => {
@@ -48,6 +56,12 @@ const validatePacketRetrievalOptions = (options: PacketRetrievalOptions) => {
if (options.metadataOnly !== undefined && typeof options.metadataOnly !== 'boolean') { if (options.metadataOnly !== undefined && typeof options.metadataOnly !== 'boolean') {
throw new TypeError('options.metadataOnly, when defined, must be a boolean.'); throw new TypeError('options.metadataOnly, when defined, must be a boolean.');
} }
if (options.verifyKeyPackets !== undefined && typeof options.verifyKeyPackets !== 'boolean') {
throw new TypeError('options.verifyKeyPackets, when defined, must be a boolean.');
}
if (options.verifyKeyPackets && options.metadataOnly) {
throw new TypeError('options.verifyKeyPackets and options.metadataOnly cannot be enabled together.');
}
}; };
const validateTimestamp = (timestamp: number) => { const validateTimestamp = (timestamp: number) => {
@@ -56,6 +70,30 @@ const validateTimestamp = (timestamp: number) => {
} }
}; };
const maybeFixPacketType = (
track: InputTrack,
promise: Promise<EncodedPacket | null>,
options: PacketRetrievalOptions,
) => {
if (options.verifyKeyPackets) {
return promise.then(async (packet) => {
if (!packet || packet.type === 'delta') {
return packet;
}
const determinedType = await track.determinePacketType(packet);
if (determinedType) {
// @ts-expect-error Technically readonly
packet.type = determinedType;
}
return packet;
});
} else {
return promise;
}
};
/** /**
* Sink for retrieving encoded packets from an input track. * Sink for retrieving encoded packets from an input track.
* @public * @public
@@ -78,7 +116,8 @@ export class EncodedPacketSink {
*/ */
getFirstPacket(options: PacketRetrievalOptions = {}) { getFirstPacket(options: PacketRetrievalOptions = {}) {
validatePacketRetrievalOptions(options); validatePacketRetrievalOptions(options);
return this._track._backing.getFirstPacket(options);
return maybeFixPacketType(this._track, this._track._backing.getFirstPacket(options), options);
} }
/** /**
@@ -92,7 +131,8 @@ export class EncodedPacketSink {
getPacket(timestamp: number, options: PacketRetrievalOptions = {}) { getPacket(timestamp: number, options: PacketRetrievalOptions = {}) {
validateTimestamp(timestamp); validateTimestamp(timestamp);
validatePacketRetrievalOptions(options); validatePacketRetrievalOptions(options);
return this._track._backing.getPacket(timestamp, options);
return maybeFixPacketType(this._track, this._track._backing.getPacket(timestamp, options), options);
} }
/** /**
@@ -104,7 +144,8 @@ export class EncodedPacketSink {
throw new TypeError('packet must be an EncodedPacket.'); throw new TypeError('packet must be an EncodedPacket.');
} }
validatePacketRetrievalOptions(options); validatePacketRetrievalOptions(options);
return this._track._backing.getNextPacket(packet, options);
return maybeFixPacketType(this._track, this._track._backing.getNextPacket(packet, options), options);
} }
/** /**
@@ -114,24 +155,60 @@ export class EncodedPacketSink {
* last key packet using `getKeyPacket(Infinity)`. The method returns null if the timestamp is before the first * last key packet using `getKeyPacket(Infinity)`. The method returns null if the timestamp is before the first
* key packet in the track. * key packet in the track.
* *
* To ensure that the returned packet is guaranteed to be a real key frame, enable `options.verifyKeyPackets`.
*
* @param timestamp - The timestamp used for retrieval, in seconds. * @param timestamp - The timestamp used for retrieval, in seconds.
*/ */
getKeyPacket(timestamp: number, options: PacketRetrievalOptions = {}) { async getKeyPacket(timestamp: number, options: PacketRetrievalOptions = {}): Promise<EncodedPacket | null> {
validateTimestamp(timestamp); validateTimestamp(timestamp);
validatePacketRetrievalOptions(options); validatePacketRetrievalOptions(options);
return this._track._backing.getKeyPacket(timestamp, options);
if (!options.verifyKeyPackets) {
return this._track._backing.getKeyPacket(timestamp, options);
}
const packet = await this._track._backing.getKeyPacket(timestamp, options);
if (!packet || packet.type === 'delta') {
return packet;
}
const determinedType = await this._track.determinePacketType(packet);
if (determinedType === 'delta') {
// Try returning the previous key packet (in hopes that it's actually a key packet)
return this.getKeyPacket(packet.timestamp - 1 / this._track.timeResolution, options);
}
return packet;
} }
/** /**
* Retrieves the key packet following the given packet (in decode order), or null if the given packet is the last * Retrieves the key packet following the given packet (in decode order), or null if the given packet is the last
* key packet. * key packet.
*
* To ensure that the returned packet is guaranteed to be a real key frame, enable `options.verifyKeyPackets`.
*/ */
getNextKeyPacket(packet: EncodedPacket, options: PacketRetrievalOptions = {}) { async getNextKeyPacket(packet: EncodedPacket, options: PacketRetrievalOptions = {}): Promise<EncodedPacket | null> {
if (!(packet instanceof EncodedPacket)) { if (!(packet instanceof EncodedPacket)) {
throw new TypeError('packet must be an EncodedPacket.'); throw new TypeError('packet must be an EncodedPacket.');
} }
validatePacketRetrievalOptions(options); validatePacketRetrievalOptions(options);
return this._track._backing.getNextKeyPacket(packet, options);
if (!options.verifyKeyPackets) {
return this._track._backing.getNextKeyPacket(packet, options);
}
const nextPacket = await this._track._backing.getNextKeyPacket(packet, options);
if (!nextPacket || nextPacket.type === 'delta') {
return nextPacket;
}
const determinedType = await this._track.determinePacketType(nextPacket);
if (determinedType === 'delta') {
// Try returning the next key packet (in hopes that it's actually a key packet)
return this.getNextKeyPacket(nextPacket, options);
}
return nextPacket;
} }
/** /**
@@ -346,7 +423,8 @@ export abstract class BaseMediaSampleSink<
}); });
const packetSink = this._createPacketSink(); const packetSink = this._createPacketSink();
const keyPacket = await packetSink.getKeyPacket(startTimestamp) ?? await packetSink.getFirstPacket(); const keyPacket = await packetSink.getKeyPacket(startTimestamp, { verifyKeyPackets: true })
?? await packetSink.getFirstPacket();
if (!keyPacket) { if (!keyPacket) {
return; return;
} }
@@ -364,7 +442,7 @@ export abstract class BaseMediaSampleSink<
? null ? null
: packet.type === 'key' && packet.timestamp === endTimestamp : packet.type === 'key' && packet.timestamp === endTimestamp
? packet ? packet
: await packetSink.getNextKeyPacket(packet); : await packetSink.getNextKeyPacket(packet, { verifyKeyPackets: true });
if (keyPacket) { if (keyPacket) {
endPacket = keyPacket; endPacket = keyPacket;
@@ -567,7 +645,7 @@ export abstract class BaseMediaSampleSink<
} }
const targetPacket = await packetSink.getPacket(timestamp); const targetPacket = await packetSink.getPacket(timestamp);
const keyPacket = targetPacket && await packetSink.getKeyPacket(timestamp); const keyPacket = targetPacket && await packetSink.getKeyPacket(timestamp, { verifyKeyPackets: true });
if (!keyPacket) { if (!keyPacket) {
if (maxSequenceNumber !== -1) { if (maxSequenceNumber !== -1) {
src/misc.ts
+6 -2
View File
@@ -48,7 +48,7 @@ export class Bitstream {
this.pos = 8 * byteOffset; this.pos = 8 * byteOffset;
} }
readBit() { private readBit() {
const byteIndex = Math.floor(this.pos / 8); const byteIndex = Math.floor(this.pos / 8);
const byte = this.bytes[byteIndex] ?? 0; const byte = this.bytes[byteIndex] ?? 0;
const bitIndex = 0b111 - (this.pos & 0b111); const bitIndex = 0b111 - (this.pos & 0b111);
@@ -59,6 +59,10 @@ export class Bitstream {
} }
readBits(n: number) { readBits(n: number) {
if (n === 1) {
return this.readBit();
}
let result = 0; let result = 0;
for (let i = 0; i < n; i++) { for (let i = 0; i < n; i++) {
@@ -100,7 +104,7 @@ export class Bitstream {
/** Reads an exponential-Golomb universal code from a Bitstream. */ /** Reads an exponential-Golomb universal code from a Bitstream. */
export const readExpGolomb = (bitstream: Bitstream) => { export const readExpGolomb = (bitstream: Bitstream) => {
let leadingZeroBits = 0; let leadingZeroBits = 0;
while (bitstream.readBit() === 0 && leadingZeroBits < 32) { while (bitstream.readBits(1) === 0 && leadingZeroBits < 32) {
leadingZeroBits++; leadingZeroBits++;
} }