diff --git a/dev/demux.html b/dev/demux.html
index c6402f5..08ba369 100644
--- a/dev/demux.html
+++ b/dev/demux.html
@@ -14,10 +14,44 @@
source: new Mediabunny.BlobSource(file),
});
- const videoTrack = await input.getPrimaryVideoTrack();
- const sink = new Mediabunny.EncodedPacketSink(videoTrack);
+ const track = await input.getPrimaryVideoTrack();
+ const sink = new Mediabunny.EncodedPacketSink(track);
- console.log(await sink.getKeyPacket(Infinity));
+ const packet = await sink.getPacket(12.5);
+ console.log(packet, await sink.getNextPacket(packet))
+
+ /*
+
+ let currentPacket = await sink.getFirstPacket();
+ while (currentPacket) {
+ console.log(currentPacket);
+ currentPacket = await sink.getNextPacket(currentPacket);
+ }
+
+ //const secondPacket = await sink.getNextPacket(firstPacket);
+ //const thirdPacket = await sink.getNextPacket(secondPacket);
+ //const fourthPacket = await sink.getNextPacket(thirdPacket);
+ //const fifthPacket = await sink.getNextPacket(fourthPacket);
+ //const sixthPacket = await sink.getNextPacket(fifthPacket);
+ //const seventhPacket = await sink.getNextPacket(sixthPacket);
+ //const eighthPacket = await sink.getNextPacket(seventhPacket);
+ //const ninthPacket = await sink.getNextPacket(eighthPacket);
+ //const tenthPacket = await sink.getNextPacket(ninthPacket);
+
+ console.log(firstPacket, secondPacket, thirdPacket, fourthPacket, fifthPacket, sixthPacket, seventhPacket, eighthPacket, ninthPacket, tenthPacket);
+
+ const packets = [firstPacket, secondPacket, thirdPacket, fourthPacket, fifthPacket, sixthPacket, seventhPacket, eighthPacket, ninthPacket, tenthPacket];
+
+ const videoDecoder = new VideoDecoder({
+ output: (frame) => console.log(frame.timestamp, frame),
+ error: console.error,
+ });
+ videoDecoder.configure(await videoTrack.getDecoderConfig());
+
+ for (const packet of packets) {
+ //videoDecoder.decode(packet.toEncodedVideoChunk());
+ }
+ */
/*
diff --git a/src/codec-data.ts b/src/codec-data.ts
index ff4a2c7..83aa796 100644
--- a/src/codec-data.ts
+++ b/src/codec-data.ts
@@ -6,7 +6,7 @@
* file, You can obtain one at https://mozilla.org/MPL/2.0/.
*/
-import { VideoCodec, VP9_LEVEL_TABLE } from './codec';
+import { AVC_LEVEL_TABLE, VideoCodec, VP9_LEVEL_TABLE } from './codec';
import {
assert,
assertNever,
@@ -486,6 +486,8 @@ export type AvcSpsInfo = {
transferCharacteristics: number;
matrixCoefficients: number;
fullRangeFlag: number;
+ numReorderFrames: number;
+ maxDecFrameBuffering: number;
};
/** Parses an AVC SPS (Sequence Parameter Set) to extract basic information. */
@@ -573,8 +575,10 @@ export const parseAvcSps = (sps: Uint8Array): AvcSpsInfo | null => {
readExpGolomb(bitstream); // max_num_ref_frames
bitstream.skipBits(1); // gaps_in_frame_num_value_allowed_flag
- const codedWidth = 16 * (readExpGolomb(bitstream) + 1); // pic_width_in_mbs_minus1
- const codedHeight = 16 * (readExpGolomb(bitstream) + 1); // pic_height_in_map_units_minus1
+ const picWidthInMbsMinus1 = readExpGolomb(bitstream);
+ const picHeightInMapUnitsMinus1 = readExpGolomb(bitstream);
+ const codedWidth = 16 * (picWidthInMbsMinus1 + 1);
+ const codedHeight = 16 * (picHeightInMapUnitsMinus1 + 1);
let displayWidth = codedWidth;
let displayHeight = codedHeight;
@@ -619,6 +623,9 @@ export const parseAvcSps = (sps: Uint8Array): AvcSpsInfo | null => {
let matrixCoefficients = 2;
let fullRangeFlag = 0;
+ let numReorderFrames: number | null = null;
+ let maxDecFrameBuffering: number | null = null;
+
const vuiParametersPresentFlag = bitstream.readBits(1);
if (vuiParametersPresentFlag) {
const aspectRatioInfoPresentFlag = bitstream.readBits(1);
@@ -646,8 +653,85 @@ export const parseAvcSps = (sps: Uint8Array): AvcSpsInfo | null => {
matrixCoefficients = bitstream.readBits(8);
}
}
+
+ const chromaLocInfoPresentFlag = bitstream.readBits(1);
+ if (chromaLocInfoPresentFlag) {
+ readExpGolomb(bitstream); // chroma_sample_loc_type_top_field
+ readExpGolomb(bitstream); // chroma_sample_loc_type_bottom_field
+ }
+
+ const timingInfoPresentFlag = bitstream.readBits(1);
+ if (timingInfoPresentFlag) {
+ bitstream.skipBits(32); // num_units_in_tick
+ bitstream.skipBits(32); // time_scale
+ bitstream.skipBits(1); // fixed_frame_rate_flag
+ }
+
+ const nalHrdParametersPresentFlag = bitstream.readBits(1);
+ if (nalHrdParametersPresentFlag) {
+ skipAvcHrdParameters(bitstream);
+ }
+
+ const vclHrdParametersPresentFlag = bitstream.readBits(1);
+ if (vclHrdParametersPresentFlag) {
+ skipAvcHrdParameters(bitstream);
+ }
+
+ if (nalHrdParametersPresentFlag || vclHrdParametersPresentFlag) {
+ bitstream.skipBits(1); // low_delay_hrd_flag
+ }
+
+ bitstream.skipBits(1); // pic_struct_present_flag
+
+ const bitstreamRestrictionFlag = bitstream.readBits(1);
+ if (bitstreamRestrictionFlag) {
+ bitstream.skipBits(1); // motion_vectors_over_pic_boundaries_flag
+ readExpGolomb(bitstream); // max_bytes_per_pic_denom
+ readExpGolomb(bitstream); // max_bits_per_mb_denom
+ readExpGolomb(bitstream); // log2_max_mv_length_horizontal
+ readExpGolomb(bitstream); // log2_max_mv_length_vertical
+ numReorderFrames = readExpGolomb(bitstream);
+ maxDecFrameBuffering = readExpGolomb(bitstream);
+ }
}
+ if (numReorderFrames === null) {
+ assert(maxDecFrameBuffering === null);
+ const constraintSet3Flag = constraintFlags & 0b00010000;
+
+ if (
+ (profileIdc === 44 || profileIdc === 86 || profileIdc === 100
+ || profileIdc === 110 || profileIdc === 122 || profileIdc === 244
+ ) && constraintSet3Flag
+ ) {
+ // "If profile_idc is equal to 44, 86, 100, 110, 122, or 244 and constraint_set3_flag is equal to 1, the
+ // value of num_reorder_frames shall be inferred to be equal to 0."
+ numReorderFrames = 0;
+ maxDecFrameBuffering = 0;
+ } else {
+ const picWidthInMbs = picWidthInMbsMinus1 + 1;
+ const picHeightInMapUnits = picHeightInMapUnitsMinus1 + 1;
+ const frameHeightInMbs = (2 - frameMbsOnlyFlag) * picHeightInMapUnits;
+
+ const levelInfo = AVC_LEVEL_TABLE.find(
+ x => x.level >= levelIdc,
+ ) ?? last(AVC_LEVEL_TABLE)!;
+
+ // "MaxDpbFrames is equal to
+ // Min( MaxDpbMbs / ( picWidthInMbs * frameHeightInMbs ), 16 ) and MaxDpbMbs is given in Table A-1."
+ const maxDpbFrames = Math.min(
+ Math.floor(levelInfo.maxDpbMbs / (picWidthInMbs * frameHeightInMbs)),
+ 16,
+ );
+
+ // "Otherwise, [...] the value of num_reorder_frames shall be inferred to be equal to MaxDpbFrames."
+ numReorderFrames = maxDpbFrames;
+ maxDecFrameBuffering = maxDpbFrames;
+ }
+ }
+
+ assert(maxDecFrameBuffering !== null);
+
return {
profileIdc,
constraintFlags,
@@ -664,6 +748,8 @@ export const parseAvcSps = (sps: Uint8Array): AvcSpsInfo | null => {
matrixCoefficients,
transferCharacteristics,
fullRangeFlag,
+ numReorderFrames,
+ maxDecFrameBuffering,
};
} catch (error) {
console.error('Error parsing AVC SPS:', error);
@@ -671,6 +757,23 @@ export const parseAvcSps = (sps: Uint8Array): AvcSpsInfo | null => {
}
};
+const skipAvcHrdParameters = (bitstream: Bitstream) => {
+ const cpb_cnt_minus1 = readExpGolomb(bitstream);
+ bitstream.skipBits(4); // bit_rate_scale
+ bitstream.skipBits(4); // cpb_size_scale
+
+ for (let i = 0; i <= cpb_cnt_minus1; i++) {
+ readExpGolomb(bitstream); // bit_rate_value_minus1[i]
+ readExpGolomb(bitstream); // cpb_size_value_minus1[i]
+ bitstream.skipBits(1); // cbr_flag[i]
+ }
+
+ bitstream.skipBits(5); // initial_cpb_removal_delay_length_minus1
+ bitstream.skipBits(5); // cpb_removal_delay_length_minus1
+ bitstream.skipBits(5); // dpb_output_delay_length_minus1
+ bitstream.skipBits(5); // time_offset_length
+};
+
// Data specified in ISO 14496-15
export type HevcDecoderConfigurationRecord = {
configurationVersion: number;
@@ -1085,7 +1188,7 @@ const parseVuiForMinSpatialSegmentationIdc = (bitstream: Bitstream, sps_max_sub_
readExpGolomb(bitstream); // vui_num_ticks_poc_diff_one_minus1
}
if (bitstream.readBits(1)) {
- skipHrdParameters(bitstream, true, sps_max_sub_layers_minus1);
+ skipHevcHrdParameters(bitstream, true, sps_max_sub_layers_minus1);
}
}
if (bitstream.readBits(1)) { // bitstream_restriction_flag
@@ -1103,7 +1206,7 @@ const parseVuiForMinSpatialSegmentationIdc = (bitstream: Bitstream, sps_max_sub_
return 0;
};
-const skipHrdParameters = (
+const skipHevcHrdParameters = (
bitstream: Bitstream,
commonInfPresentFlag: boolean,
maxNumSubLayersMinus1: number,
diff --git a/src/codec.ts b/src/codec.ts
index 9429bd4..c43733e 100644
--- a/src/codec.ts
+++ b/src/codec.ts
@@ -116,26 +116,26 @@ export type SubtitleCodec = typeof SUBTITLE_CODECS[number];
export type MediaCodec = VideoCodec | AudioCodec | SubtitleCodec;
// https://en.wikipedia.org/wiki/Advanced_Video_Coding
-const AVC_LEVEL_TABLE = [
- { maxMacroblocks: 99, maxBitrate: 64000, level: 0x0A }, // Level 1
- { maxMacroblocks: 396, maxBitrate: 192000, level: 0x0B }, // Level 1.1
- { maxMacroblocks: 396, maxBitrate: 384000, level: 0x0C }, // Level 1.2
- { maxMacroblocks: 396, maxBitrate: 768000, level: 0x0D }, // Level 1.3
- { maxMacroblocks: 396, maxBitrate: 2000000, level: 0x14 }, // Level 2
- { maxMacroblocks: 792, maxBitrate: 4000000, level: 0x15 }, // Level 2.1
- { maxMacroblocks: 1620, maxBitrate: 4000000, level: 0x16 }, // Level 2.2
- { maxMacroblocks: 1620, maxBitrate: 10000000, level: 0x1E }, // Level 3
- { maxMacroblocks: 3600, maxBitrate: 14000000, level: 0x1F }, // Level 3.1
- { maxMacroblocks: 5120, maxBitrate: 20000000, level: 0x20 }, // Level 3.2
- { maxMacroblocks: 8192, maxBitrate: 20000000, level: 0x28 }, // Level 4
- { maxMacroblocks: 8192, maxBitrate: 50000000, level: 0x29 }, // Level 4.1
- { maxMacroblocks: 8704, maxBitrate: 50000000, level: 0x2A }, // Level 4.2
- { maxMacroblocks: 22080, maxBitrate: 135000000, level: 0x32 }, // Level 5
- { maxMacroblocks: 36864, maxBitrate: 240000000, level: 0x33 }, // Level 5.1
- { maxMacroblocks: 36864, maxBitrate: 240000000, level: 0x34 }, // Level 5.2
- { maxMacroblocks: 139264, maxBitrate: 240000000, level: 0x3C }, // Level 6
- { maxMacroblocks: 139264, maxBitrate: 480000000, level: 0x3D }, // Level 6.1
- { maxMacroblocks: 139264, maxBitrate: 800000000, level: 0x3E }, // Level 6.2
+export const AVC_LEVEL_TABLE = [
+ { maxMacroblocks: 99, maxBitrate: 64000, maxDpbMbs: 396, level: 0x0A }, // Level 1
+ { maxMacroblocks: 396, maxBitrate: 192000, maxDpbMbs: 900, level: 0x0B }, // Level 1.1
+ { maxMacroblocks: 396, maxBitrate: 384000, maxDpbMbs: 2376, level: 0x0C }, // Level 1.2
+ { maxMacroblocks: 396, maxBitrate: 768000, maxDpbMbs: 2376, level: 0x0D }, // Level 1.3
+ { maxMacroblocks: 396, maxBitrate: 2000000, maxDpbMbs: 2376, level: 0x14 }, // Level 2
+ { maxMacroblocks: 792, maxBitrate: 4000000, maxDpbMbs: 4752, level: 0x15 }, // Level 2.1
+ { maxMacroblocks: 1620, maxBitrate: 4000000, maxDpbMbs: 8100, level: 0x16 }, // Level 2.2
+ { maxMacroblocks: 1620, maxBitrate: 10000000, maxDpbMbs: 8100, level: 0x1E }, // Level 3
+ { maxMacroblocks: 3600, maxBitrate: 14000000, maxDpbMbs: 18000, level: 0x1F }, // Level 3.1
+ { maxMacroblocks: 5120, maxBitrate: 20000000, maxDpbMbs: 20480, level: 0x20 }, // Level 3.2
+ { maxMacroblocks: 8192, maxBitrate: 20000000, maxDpbMbs: 32768, level: 0x28 }, // Level 4
+ { maxMacroblocks: 8192, maxBitrate: 50000000, maxDpbMbs: 32768, level: 0x29 }, // Level 4.1
+ { maxMacroblocks: 8704, maxBitrate: 50000000, maxDpbMbs: 34816, level: 0x2A }, // Level 4.2
+ { maxMacroblocks: 22080, maxBitrate: 135000000, maxDpbMbs: 110400, level: 0x32 }, // Level 5
+ { maxMacroblocks: 36864, maxBitrate: 240000000, maxDpbMbs: 184320, level: 0x33 }, // Level 5.1
+ { maxMacroblocks: 36864, maxBitrate: 240000000, maxDpbMbs: 184320, level: 0x34 }, // Level 5.2
+ { maxMacroblocks: 139264, maxBitrate: 240000000, maxDpbMbs: 696320, level: 0x3C }, // Level 6
+ { maxMacroblocks: 139264, maxBitrate: 480000000, maxDpbMbs: 696320, level: 0x3D }, // Level 6.1
+ { maxMacroblocks: 139264, maxBitrate: 800000000, maxDpbMbs: 696320, level: 0x3E }, // Level 6.2
];
// https://en.wikipedia.org/wiki/High_Efficiency_Video_Coding
diff --git a/src/mpeg-ts/mpeg-ts-demuxer.ts b/src/mpeg-ts/mpeg-ts-demuxer.ts
index 4eaba45..f929ad8 100644
--- a/src/mpeg-ts/mpeg-ts-demuxer.ts
+++ b/src/mpeg-ts/mpeg-ts-demuxer.ts
@@ -37,6 +37,7 @@ import {
binarySearchLessOrEqual,
Bitstream,
COLOR_PRIMARIES_MAP_INVERSE,
+ findLastIndex,
last,
MATRIX_COEFFICIENTS_MAP_INVERSE,
Rotation,
@@ -65,6 +66,7 @@ type ElementaryStream = {
colorSpace: VideoColorSpaceInit;
width: number;
height: number;
+ reorderSize: number;
} | {
type: 'audio';
codec: AudioCodec;
@@ -234,6 +236,7 @@ export class MpegTsDemuxer extends Demuxer {
},
width: -1,
height: -1,
+ reorderSize: -1,
};
}; break;
}
@@ -290,6 +293,7 @@ export class MpegTsDemuxer extends Demuxer {
VideoMatrixCoefficients | undefined,
fullRange: !!spsInfo.fullRangeFlag,
};
+ elementaryStream.info.reorderSize = spsInfo.maxDecFrameBuffering;
elementaryStream.initialized = true;
}
@@ -638,7 +642,9 @@ export abstract class MpegTsTrackBacking implements InputTrackBacking {
*/
referencePesPackets: PesPacketHeader[] = [];
endReferencePesPacketAdded = false;
- readingContexts = new WeakMap();
+ packetBuffers = new WeakMap();
+ /** Used for recreating PacketBuffers if necessary. */
+ packetSectionStarts = new WeakMap();
mutex = new AsyncMutex();
constructor(public elementaryStream: ElementaryStream) {}
@@ -683,6 +689,22 @@ export abstract class MpegTsTrackBacking implements InputTrackBacking {
abstract getPacketType(packetData: Uint8Array): PacketType;
abstract markNextPacket(context: PacketReadingContext): Promise;
+ abstract getReorderSize(): number;
+
+ createEncodedPacket(
+ suppliedPacket: SuppliedPacket,
+ duration: number,
+ options: PacketRetrievalOptions,
+ ) {
+ return new EncodedPacket(
+ options.metadataOnly ? PLACEHOLDER_DATA : suppliedPacket.data,
+ this.getPacketType(suppliedPacket.data),
+ suppliedPacket.pts / TIMESCALE,
+ Math.max(duration / TIMESCALE, 0),
+ suppliedPacket.sequenceNumber,
+ suppliedPacket.data.byteLength,
+ );
+ }
maybeInsertReferencePacket(pesPacketHeader: PesPacketHeader, force: boolean, dropIfMutexLocked: boolean) {
if (dropIfMutexLocked && this.mutex.pending > 0) {
@@ -729,21 +751,72 @@ export abstract class MpegTsTrackBacking implements InputTrackBacking {
assert(pesPacket);
const context = new PacketReadingContext(this, pesPacket, true);
- await this.markNextPacket(context);
+ const buffer = new PacketBuffer(this, context);
- return context.createAndLinkEncodedPacket(context.suppliedPacket, options);
+ const result = await buffer.readNext();
+ if (!result) {
+ return null;
+ }
+
+ const packet = this.createEncodedPacket(result.packet, result.duration, options);
+ this.packetBuffers.set(packet, buffer);
+ this.packetSectionStarts.set(packet, result.packet.sectionStartPos);
+
+ return packet;
}
async getNextPacket(packet: EncodedPacket, options: PacketRetrievalOptions): Promise {
- const context = this.readingContexts.get(packet);
- if (!context) {
+ let buffer = this.packetBuffers.get(packet);
+
+ if (buffer) {
+ // Fast path
+ const result = await buffer.readNext();
+ if (!result) {
+ return null;
+ }
+
+ // Remove PacketBuffer access from the old packet, it belongs to the next packet now
+ this.packetBuffers.delete(packet);
+
+ const newPacket = this.createEncodedPacket(result.packet, result.duration, options);
+ this.packetBuffers.set(newPacket, buffer);
+ this.packetSectionStarts.set(newPacket, result.packet.sectionStartPos);
+
+ return newPacket;
+ }
+
+ // No buffer, we gotta do some rereading
+ const sectionStartPos = this.packetSectionStarts.get(packet);
+ if (sectionStartPos === undefined) {
throw new Error('Packet was not created from this track.');
}
- const clone = context.clone();
- await this.markNextPacket(clone);
+ const demuxer = this.elementaryStream.demuxer;
+ const section = await demuxer.readSection(sectionStartPos, true);
+ assert(section);
- return clone.createAndLinkEncodedPacket(clone.suppliedPacket, options);
+ const pesPacket = readPesPacket(section);
+ assert(pesPacket);
+
+ const context = new PacketReadingContext(this, pesPacket, true);
+ buffer = new PacketBuffer(this, context);
+
+ // Advance until we pass the current packet's sequence number
+ const targetSequenceNumber = packet.sequenceNumber;
+ while (true) {
+ const result = await buffer.readNext();
+ if (!result) {
+ return null;
+ }
+
+ if (result.packet.sequenceNumber > targetSequenceNumber) {
+ // We found the next packet!
+ const newPacket = this.createEncodedPacket(result.packet, result.duration, options);
+ this.packetBuffers.set(newPacket, buffer);
+ this.packetSectionStarts.set(newPacket, result.packet.sectionStartPos);
+ return newPacket;
+ }
+ }
}
async getNextKeyPacket(packet: EncodedPacket, options: PacketRetrievalOptions): Promise {
@@ -771,8 +844,6 @@ export abstract class MpegTsTrackBacking implements InputTrackBacking {
return this.doPacketLookup(timestamp, true, options);
}
- abstract getPacketLookaround(): number;
-
/**
* Searches for the packet with the largest timestamp not larger than `timestamp` in the file, using a combination
* of binary search and linear refinement.
@@ -914,9 +985,6 @@ export abstract class MpegTsTrackBacking implements InputTrackBacking {
release();
- /** Stores the best PES packet we've found so far (that meets all required criteria). */
- let bestPesPacketHeader: PesPacketHeader | null = null;
-
const pesPacketHasKeyframe = async (sectionStartPos: number) => {
const section = await demuxer.readSection(sectionStartPos, true);
assert(section);
@@ -925,6 +993,7 @@ export abstract class MpegTsTrackBacking implements InputTrackBacking {
const fullPesPacket = readPesPacket(section);
assert(fullPesPacket);
+ // Only mark the first packet
const context = new PacketReadingContext(this, fullPesPacket, false);
await this.markNextPacket(context);
@@ -932,16 +1001,9 @@ export abstract class MpegTsTrackBacking implements InputTrackBacking {
return false;
}
- return this.getPacketType(context.suppliedPacket.data);
+ return this.getPacketType(context.suppliedPacket.data) === 'key';
};
- if (!keyframesOnly || await pesPacketHasKeyframe(currentPesPacketHeader.sectionStartPos)) {
- bestPesPacketHeader = currentPesPacketHeader;
- }
-
- // "advanced" as in "moved past"
- const advancedPesPacketHeaders = [bestPesPacketHeader];
-
// Starting from the binary search guess, let's now find the moment where the packet timestamps cross the
// search timestamp. This point will then be used as the center around which we search.
outer:
@@ -976,17 +1038,7 @@ export abstract class MpegTsTrackBacking implements InputTrackBacking {
break;
}
- // Collect matching packets we find along the way
- if (
- (bestPesPacketHeader === null || bestPesPacketHeader.pts < nextPesPacketHeader.pts)
- && nextPesPacketHeader.pts <= searchPts
- && (!keyframesOnly || await pesPacketHasKeyframe(currentPos))
- ) {
- bestPesPacketHeader = nextPesPacketHeader;
- }
-
currentPesPacketHeader = nextPesPacketHeader;
- advancedPesPacketHeaders.push(nextPesPacketHeader);
if (reader.fileSize === null) {
// If the file size is undefined, that means that the binary search step is skipped, meaning no
@@ -995,191 +1047,132 @@ export abstract class MpegTsTrackBacking implements InputTrackBacking {
}
}
- // Lookaround is needed in the first place because packets don't need to appear in PTS order, they only appear
- // in decode order. When B-frames are present, finding the packet that's actually closest to the search
- // timestamp requires searching a small local window.
- const lookaround = this.getPacketLookaround();
+ const reorderSize = this.getReorderSize();
- // Depending on how long the previous scan went, we might not need to do the full lookbehind, or even none at
- // all if we're lucky
- const lookbehindNeeded = Math.max(lookaround - advancedPesPacketHeaders.length + 1, 0);
- let minPos = advancedPesPacketHeaders[0]!.sectionStartPos;
-
- /** Scans `n` contiguous PES packets in succession. */
- const doLinearScan = async (startPos: number, n: number) => {
- let currentPos = startPos;
-
- outer:
- for (let i = 0; i < n; i++) {
- while (true) {
- const packetHeader = await demuxer.readPacketHeader(currentPos);
- if (!packetHeader) {
- break outer; // End of file
- }
-
- if (
- packetHeader.pid === this.elementaryStream.pid
- && packetHeader.payloadUnitStartIndicator === 1
- ) {
- break;
- }
-
- currentPos += demuxer.packetStride;
- }
-
- const section = await demuxer.readSection(currentPos, false);
- assert(section);
- assert(section.pid === this.elementaryStream.pid);
-
- const pesPacketHeader = readPesPacketHeader(section);
- if (!pesPacketHeader) {
- throw new Error(MISSING_PES_PACKET_ERROR);
- }
-
- if (
- (bestPesPacketHeader === null || bestPesPacketHeader.pts < pesPacketHeader.pts)
- && pesPacketHeader.pts <= searchPts
- && (!keyframesOnly || await pesPacketHasKeyframe(currentPos))
- ) {
- bestPesPacketHeader = pesPacketHeader;
- }
-
- currentPos += demuxer.packetStride;
- }
- };
-
- // Lookbehind
- if (lookbehindNeeded > 0) {
- outer:
- for (let i = 0; i < lookbehindNeeded; i++) {
- let currentPos = minPos;
-
- while (true) {
- currentPos -= demuxer.packetStride;
-
- const packetHeader = await demuxer.readPacketHeader(currentPos);
- if (!packetHeader) {
- break outer;
- }
-
- if (
- packetHeader.pid === this.elementaryStream.pid
- && packetHeader.payloadUnitStartIndicator === 1
- ) {
- break;
- }
- }
-
- minPos = currentPos;
- }
-
- await doLinearScan(minPos, lookbehindNeeded);
- }
-
- // Lookahead
- await doLinearScan(currentPesPacketHeader.sectionStartPos + demuxer.packetStride, lookaround);
-
- // If we're looking specifically for a keyframe but haven't found one yet, that means we'll need to go left
- // until we find one.
- if (!bestPesPacketHeader && keyframesOnly) {
- let currentPos = minPos;
+ // Rewind by reorderSize PES packets (even for audio! To ensure proper durations)
+ for (let i = 0; i < reorderSize; i++) {
+ let pos = currentPesPacketHeader.sectionStartPos - demuxer.packetStride;
while (true) {
- currentPos -= demuxer.packetStride;
-
- const packetHeader = await demuxer.readPacketHeader(currentPos);
+ const packetHeader = await demuxer.readPacketHeader(pos);
if (!packetHeader) {
- break;
+ break; // Hit start of file
}
if (packetHeader.pid === this.elementaryStream.pid && packetHeader.payloadUnitStartIndicator === 1) {
- const section = await demuxer.readSection(currentPos, false);
- assert(section);
+ const headerSection = await demuxer.readSection(pos, false);
+ assert(headerSection);
- const pesPacketHeader = readPesPacketHeader(section);
- if (!pesPacketHeader) {
+ const header = readPesPacketHeader(headerSection);
+ if (!header) {
throw new Error(MISSING_PES_PACKET_ERROR);
}
- if (pesPacketHeader.pts <= searchPts && (await pesPacketHasKeyframe(currentPos))) {
- bestPesPacketHeader = pesPacketHeader;
- break;
- }
+ currentPesPacketHeader = header;
+ break;
}
+
+ pos -= demuxer.packetStride;
}
}
- if (!bestPesPacketHeader) {
- // Nothing was found
+ // Read the full section and create a PacketBuffer
+ const section = await demuxer.readSection(currentPesPacketHeader.sectionStartPos, true);
+ assert(section);
+
+ const pesPacket = readPesPacket(section);
+ assert(pesPacket);
+
+ const context = new PacketReadingContext(this, pesPacket, true);
+ const buffer = new PacketBuffer(this, context);
+
+ // Advance until the top-most presentation timestamp crosses or equals searchPts
+ while (true) {
+ const topPts = last(buffer.presentationOrderPackets)?.pts ?? -Infinity;
+ if (topPts >= searchPts) {
+ break;
+ }
+
+ const didRead = await buffer.readNextDecodeOrderPacket();
+ if (!didRead) {
+ break;
+ }
+ }
+
+ // Find the target packet: the one with largest PTS <= searchPts that is also a keyframe if required
+ const targetIndex = findLastIndex(
+ buffer.presentationOrderPackets,
+ p => p.pts <= searchPts && (!keyframesOnly || this.getPacketType(p.data) === 'key'),
+ );
+
+ if (targetIndex !== -1) {
+ const targetPacket = buffer.presentationOrderPackets[targetIndex]!;
+ const lastDuration = targetIndex === 0
+ ? 0
+ : targetPacket.pts - buffer.presentationOrderPackets[targetIndex - 1]!.pts;
+
+ // Pop packets in decode order until we hit the target packet
+ while (buffer.decodeOrderPackets[0] !== targetPacket) {
+ buffer.decodeOrderPackets.shift();
+ }
+ buffer.lastDuration = lastDuration;
+
+ // Now consume the target packet through readNext to get proper duration
+ const result = await buffer.readNext();
+ assert(result);
+
+ const packet = this.createEncodedPacket(result.packet, result.duration, options);
+ this.packetBuffers.set(packet, buffer);
+ this.packetSectionStarts.set(packet, result.packet.sectionStartPos);
+
+ return packet;
+ }
+
+ if (!keyframesOnly) {
+ // We didn't find a suitable packet
return null;
}
- const bestSection = await demuxer.readSection(bestPesPacketHeader.sectionStartPos, true); // Read it in full
- let bestPesPacket = readPesPacket(bestSection!);
- assert(bestPesPacket);
+ // Go backwards looking for a PES packet with a keyframe
+ let searchPos = currentPesPacketHeader.sectionStartPos;
- // Final stage: we found the best PES packet, but that PES packet might contain multiple individual encoded
- // packets. Or, it might not be the start of an encoded packet, and simply a continuation of a previous one.
- // So, we have one last search to do.
while (true) {
- const context = new PacketReadingContext(this, bestPesPacket, false); // Capped context
+ searchPos -= demuxer.packetStride;
- let bestPacket: SuppliedPacket | null = null;
- let bestContext: PacketReadingContext | null = null;
-
- while (true) {
- // Stupid aliasing trick to make TypeScript not infer stuff wrong
- const context2 = context;
- context2.suppliedPacket = null;
-
- await this.markNextPacket(context);
- if (!context.suppliedPacket) {
- break;
- }
-
- const eligible = context.suppliedPacket.pts <= searchPts
- && (!keyframesOnly || this.getPacketType(context.suppliedPacket.data) === 'key');
- if (!eligible) {
- continue;
- }
-
- if (!bestPacket || bestPacket.pts < context.suppliedPacket.pts) {
- bestPacket = context.suppliedPacket;
- bestContext = context.clone(); // Kiiiinda ugly
- }
+ const packetHeader = await demuxer.readPacketHeader(searchPos);
+ if (!packetHeader) {
+ return null; // Hit start of file
}
- if (bestPacket) {
- assert(bestContext);
- return bestContext.createAndLinkEncodedPacket(bestPacket, options);
+ if (packetHeader.pid !== this.elementaryStream.pid || packetHeader.payloadUnitStartIndicator !== 1) {
+ continue;
}
- // We didn't find an encoded packet! Let's go to the previous PES packet until we find one.
+ if (!(await pesPacketHasKeyframe(searchPos))) {
+ continue;
+ }
- let currentPos = bestPesPacket.sectionStartPos;
+ // Found a PES packet with a keyframe. Set up a PacketBuffer and pull until we get the keyframe.
+ const keySection = await demuxer.readSection(searchPos, true);
+ assert(keySection);
+ const keyPesPacket = readPesPacket(keySection);
+ assert(keyPesPacket);
+
+ const keyContext = new PacketReadingContext(this, keyPesPacket, true);
+ const keyBuffer = new PacketBuffer(this, keyContext);
+
+ // Pull until we get a keyframe
while (true) {
- currentPos -= demuxer.packetStride;
+ const result = await keyBuffer.readNext();
+ assert(result); // How else?
- const packetHeader = await demuxer.readPacketHeader(currentPos);
- if (!packetHeader) {
- // Past start of file
- return null;
- }
+ if (this.getPacketType(result.packet.data) === 'key') {
+ const packet = this.createEncodedPacket(result.packet, result.duration, options);
+ this.packetBuffers.set(packet, keyBuffer);
+ this.packetSectionStarts.set(packet, result.packet.sectionStartPos);
- if (packetHeader.pid === this.elementaryStream.pid && packetHeader.payloadUnitStartIndicator === 1) {
- const section = await demuxer.readSection(currentPos, true);
- assert(section);
-
- const pesPacket = readPesPacket(section);
- if (!pesPacket) {
- throw new Error(MISSING_PES_PACKET_ERROR);
- }
-
- if (pesPacket.pts <= searchPts) {
- bestPesPacket = pesPacket;
- break;
- }
+ return packet;
}
}
}
@@ -1245,14 +1238,13 @@ class MpegTsVideoTrackBacking extends MpegTsTrackBacking implements InputVideoTr
return determineVideoPacketType(this.elementaryStream.info.codec, this.decoderConfig, packetData) ?? 'key';
}
- override getPacketLookaround(): number {
- // Due to B-frames. A lookaround of +-5 packets will pretty much guarantee we find the correct packet for a
- // given timestamp, although of course, this could technically still fail.
- // todo, use max_num_reorder_frames here?
- return 5;
+ override getReorderSize(): number {
+ return this.elementaryStream.info.reorderSize;
}
override async markNextPacket(context: PacketReadingContext): Promise {
+ assert(!context.suppliedPacket);
+
const CHUNK_SIZE = 128;
let packetStartPos: number | null = null;
@@ -1388,11 +1380,13 @@ class MpegTsAudioTrackBacking extends MpegTsTrackBacking implements InputAudioTr
return 'key';
}
- override getPacketLookaround(): number {
- return 0;
+ override getReorderSize(): number {
+ return 1; // No reordering, since no B-frames because goated
}
override async markNextPacket(context: PacketReadingContext): Promise {
+ assert(!context.suppliedPacket);
+
const CHUNK_SIZE = 128;
while (true) {
@@ -1444,9 +1438,9 @@ class MpegTsAudioTrackBacking extends MpegTsTrackBacking implements InputAudioTr
type SuppliedPacket = {
pts: number;
- intrinsicDuration: number;
data: Uint8Array;
sequenceNumber: number;
+ sectionStartPos: number;
};
/** Stateful context used to extract exact encoded packets from the underlying data stream. */
@@ -1662,39 +1656,148 @@ class PacketReadingContext {
const pts = this.nextPts;
this.nextPts += intrinsicDuration;
+ const sectionStartPos = currentPesPacket.sectionStartPos;
// The sequence number is the starting position of the section the PES packet is in, PLUS the offset within the
// PES packet where the packet starts.
- const sequenceNumber = currentPesPacket.sectionStartPos + (this.currentPos - this.currentPesPacketPos);
+ const sequenceNumber = sectionStartPos + (this.currentPos - this.currentPesPacketPos);
const data = this.readBytes(packetLength);
this.suppliedPacket = {
pts,
- intrinsicDuration,
data,
sequenceNumber,
+ sectionStartPos,
};
this.pesPackets.splice(0, this.currentPesPacketIndex);
this.currentPesPacketIndex = 0;
}
+}
- createAndLinkEncodedPacket(suppliedPacket: SuppliedPacket | null, options: PacketRetrievalOptions) {
- if (!suppliedPacket) {
- return null;
+/**
+ * A buffer that simulates decoder frame reordering to compute packet durations. Packets arrive in decode order but
+ * durations are based on presentation order.
+ */
+class PacketBuffer {
+ backing: MpegTsTrackBacking;
+ context: PacketReadingContext;
+
+ decodeOrderPackets: SuppliedPacket[] = [];
+ reorderSize: number;
+ reorderBuffer: SuppliedPacket[] = [];
+ presentationOrderPackets: SuppliedPacket[] = [];
+ reachedEnd = false;
+ lastDuration = 0;
+
+ constructor(backing: MpegTsTrackBacking, context: PacketReadingContext) {
+ this.backing = backing;
+ this.context = context;
+ this.reorderSize = backing.getReorderSize();
+ assert(this.reorderSize >= 0);
+ }
+
+ async readNext(): Promise<{ packet: SuppliedPacket; duration: number } | null> {
+ if (this.decodeOrderPackets.length === 0) {
+ // We need the next packet
+ const didRead = await this.readNextDecodeOrderPacket();
+ if (!didRead) {
+ return null;
+ }
}
- const packet = new EncodedPacket(
- options.metadataOnly ? PLACEHOLDER_DATA : suppliedPacket.data,
- this.backing.getPacketType(suppliedPacket.data),
- suppliedPacket.pts / TIMESCALE,
- suppliedPacket.intrinsicDuration / TIMESCALE,
- suppliedPacket.sequenceNumber,
- suppliedPacket.data.byteLength,
- );
+ // Ensure we know the next packet in presentation order so we can compute the current packet's duration
+ await this.ensureCurrentPacketHasNext();
- // Link the context for next packet retrieval
- this.backing.readingContexts.set(packet, this);
+ const packet = this.decodeOrderPackets[0]!;
- return packet;
+ // Let's compute the duration
+ const presentationIndex = this.presentationOrderPackets.indexOf(packet);
+ assert(presentationIndex !== -1);
+
+ let duration: number;
+ if (presentationIndex === this.presentationOrderPackets.length - 1) {
+ duration = this.lastDuration; // Reasonable heuristic
+ } else {
+ const nextPacket = this.presentationOrderPackets[presentationIndex + 1]!;
+ duration = nextPacket.pts - packet.pts;
+ this.lastDuration = duration;
+ }
+
+ this.decodeOrderPackets.shift();
+
+ // Shrink the presentation array as much as possible
+ while (this.presentationOrderPackets.length > 0) {
+ const first = this.presentationOrderPackets[0]!;
+ if (this.decodeOrderPackets.includes(first)) {
+ break;
+ }
+
+ this.presentationOrderPackets.shift();
+ }
+
+ return { packet, duration };
+ }
+
+ async readNextDecodeOrderPacket() {
+ if (this.reachedEnd) {
+ return false;
+ }
+
+ this.context.suppliedPacket = null;
+ await this.backing.markNextPacket(this.context);
+
+ if (!this.context.suppliedPacket) {
+ this.reachedEnd = true;
+ this.flushReorderBuffer();
+
+ return false;
+ }
+
+ this.decodeOrderPackets.push(this.context.suppliedPacket);
+ this.processPacketThroughReorderBuffer(this.context.suppliedPacket);
+
+ return true;
+ }
+
+ async ensureCurrentPacketHasNext() {
+ const current = this.decodeOrderPackets[0];
+ assert(current);
+
+ while (true) {
+ const presentationIndex = this.presentationOrderPackets.indexOf(current);
+
+ // Check if current packet has a next packet
+ if (presentationIndex !== -1 && presentationIndex <= this.presentationOrderPackets.length - 2) {
+ break;
+ }
+
+ const didRead = await this.readNextDecodeOrderPacket();
+ if (!didRead) {
+ break;
+ }
+ }
+ }
+
+ processPacketThroughReorderBuffer(packet: SuppliedPacket) {
+ this.reorderBuffer.push(packet);
+
+ // If buffer is full, output the packet with smallest PTS
+ if (this.reorderBuffer.length >= this.reorderSize) {
+ let minIndex = 0;
+ for (let i = 1; i < this.reorderBuffer.length; i++) {
+ if (this.reorderBuffer[i]!.pts < this.reorderBuffer[minIndex]!.pts) {
+ minIndex = i;
+ }
+ }
+
+ const packet = this.reorderBuffer.splice(minIndex, 1)[0]!;
+ this.presentationOrderPackets.push(packet);
+ }
+ }
+
+ flushReorderBuffer() {
+ this.reorderBuffer.sort((a, b) => a.pts - b.pts);
+ this.presentationOrderPackets.push(...this.reorderBuffer);
+ this.reorderBuffer.length = 0;
}
}
diff --git a/test/node/mpeg-ts-demuxing.test.ts b/test/node/mpeg-ts-demuxing.test.ts
index aced224..09c0acb 100644
--- a/test/node/mpeg-ts-demuxing.test.ts
+++ b/test/node/mpeg-ts-demuxing.test.ts
@@ -89,7 +89,7 @@ test('MPEG-TS durations', async () => {
expect(videoFirstTimestamp).toBe(10.033333333333333);
const videoDuration = await videoTrack.computeDuration();
- expect(videoDuration).toBeCloseTo(14.983333333333333);
+ expect(videoDuration).toBeCloseTo(15);
const audioTrack = await input.getPrimaryAudioTrack();
assert(audioTrack);
@@ -119,7 +119,7 @@ test('MPEG-TS AVC video packets', async () => {
expect(firstPacket.data.byteLength).toBe(23813);
expect(firstPacket.type).toBe('key');
expect(firstPacket.timestamp).toBe(10.033333333333333);
- expect(firstPacket.duration).toBe(0);
+ expect(firstPacket.duration).toBe(0.016666666666666666);
expect(firstPacket.sequenceNumber).not.toBe(-1);
const firstPacketMetadataOnly = await sink.getFirstPacket({ metadataOnly: true });
@@ -134,15 +134,17 @@ test('MPEG-TS AVC video packets', async () => {
expect(secondPacket.data.byteLength).toBe(5700);
expect(secondPacket.type).toBe('delta');
expect(secondPacket.timestamp).toBe(10.1);
- expect(secondPacket.duration).toBe(0);
+ expect(secondPacket.duration).toBe(0.016666666666666666);
expect(secondPacket.sequenceNumber).toBeGreaterThan(firstPacket.sequenceNumber);
let currentPacket: EncodedPacket | null = firstPacket;
let count = 0;
while (currentPacket) {
- count++;
+ expect(currentPacket.duration).toBe(0.016666666666666666);
+
currentPacket = await sink.getNextPacket(currentPacket);
+ count++;
}
expect(count).toBe(298);
@@ -166,7 +168,7 @@ test('MPEG-TS AAC audio packets', async () => {
expect(firstPacket.data.byteLength).toBe(348);
expect(firstPacket.type).toBe('key');
expect(firstPacket.timestamp).toBe(10.012);
- expect(firstPacket.duration).toBeCloseTo(0.021333333333333333);
+ expect(firstPacket.duration).toBe(0.021333333333333333);
expect(firstPacket.sequenceNumber).not.toBe(-1);
const secondPacket = await sink.getNextPacket(firstPacket);
@@ -176,15 +178,17 @@ test('MPEG-TS AAC audio packets', async () => {
expect(secondPacket.data.byteLength).toBe(349);
expect(secondPacket.type).toBe('key');
expect(secondPacket.timestamp).toBeCloseTo(10.033333333333333);
- expect(secondPacket.duration).toBeCloseTo(0.021333333333333333);
+ expect(secondPacket.duration).toBe(0.021333333333333333);
expect(secondPacket.sequenceNumber).toBeGreaterThan(firstPacket.sequenceNumber);
let currentPacket: EncodedPacket | null = firstPacket;
let count = 0;
while (currentPacket) {
- count++;
+ expect(currentPacket.duration).toBe(0.021333333333333333);
+
currentPacket = await sink.getNextPacket(currentPacket);
+ count++;
}
expect(count).toBe(234);
@@ -205,12 +209,13 @@ test('MPEG-TS video seeking', async () => {
const firstPacket = await sink.getPacket(firstTimestamp);
assert(firstPacket);
expect(firstPacket.timestamp).toBe(firstTimestamp);
+ expect(firstPacket.duration).toBe(0.016666666666666666);
expect(firstPacket.sequenceNumber).toBe((await sink.getFirstPacket())?.sequenceNumber);
const lastPacket = await sink.getPacket(Infinity);
assert(lastPacket);
-
expect(lastPacket.timestamp).toBeCloseTo(14.983333333333333);
+ expect(lastPacket.duration).toBe(0.016666666666666666);
const beforeFirst = await sink.getPacket(-10);
expect(beforeFirst).toBeNull();
@@ -218,6 +223,7 @@ test('MPEG-TS video seeking', async () => {
const middlePacket = await sink.getPacket(12.5);
assert(middlePacket);
expect(middlePacket.timestamp).toBeCloseTo(12.5);
+ expect(middlePacket.duration).toBe(0.016666666666666666);
const allPackets: EncodedPacket[] = [];
let currentPacket: EncodedPacket | null = firstPacket;
@@ -233,6 +239,7 @@ test('MPEG-TS video seeking', async () => {
const seekedPacked = await sink.getPacket(packet.timestamp);
assert(seekedPacked);
expect(seekedPacked.timestamp).toBe(packet.timestamp); // The correct timestamp was retrieved for this packet
+ expect(seekedPacked.duration).toBe(packet.duration); // The correct duration was retrieved for this packet
expect(seekedPacked.sequenceNumber).toBe(packet.sequenceNumber);
}
});
@@ -252,12 +259,13 @@ test('MPEG-TS audio seeking', async () => {
const firstPacket = await sink.getPacket(firstTimestamp);
assert(firstPacket);
expect(firstPacket.timestamp).toBe(firstTimestamp);
+ expect(firstPacket.duration).toBe(0.021333333333333333);
expect(firstPacket.sequenceNumber).toBe((await sink.getFirstPacket())?.sequenceNumber);
const lastPacket = await sink.getPacket(Infinity);
assert(lastPacket);
-
expect(lastPacket.timestamp).toBeCloseTo(14.982666666666667);
+ expect(lastPacket.duration).toBe(0.021333333333333333);
const beforeFirst = await sink.getPacket(-10);
expect(beforeFirst).toBeNull();
@@ -265,6 +273,7 @@ test('MPEG-TS audio seeking', async () => {
const middlePacket = await sink.getPacket(12.5);
assert(middlePacket);
expect(middlePacket.timestamp).toBeCloseTo(12.486666666666666);
+ expect(middlePacket.duration).toBe(0.021333333333333333);
const allPackets: EncodedPacket[] = [];
let currentPacket: EncodedPacket | null = firstPacket;
@@ -280,6 +289,7 @@ test('MPEG-TS audio seeking', async () => {
const seekedPacket = await sink.getPacket(packet.timestamp);
assert(seekedPacket);
expect(seekedPacket.timestamp).toBe(packet.timestamp); // The correct timestamp was retrieved for this packet
+ expect(seekedPacket.duration).toBe(packet.duration); // The correct duration was retrieved for this packet
expect(seekedPacket.sequenceNumber).toBe(packet.sequenceNumber);
}
});
@@ -312,6 +322,7 @@ test('MPEG-TS seeking race condition test', async () => {
const seekedPacket = seekedPackets[i]!;
assert(seekedPacket);
expect(seekedPacket.timestamp).toBe(originalPacket.timestamp);
+ expect(seekedPacket.duration).toBe(originalPacket.duration);
expect(seekedPacket.sequenceNumber).toBe(originalPacket.sequenceNumber);
}
});
@@ -340,7 +351,7 @@ test('MPEG-TS video key packets', async () => {
expect(nextKeyPacket.type).toBe('key');
expect(nextKeyPacket.sequenceNumber).toBeGreaterThan(secondPacket.sequenceNumber);
- const firstKeyPacket = await sink.getKeyPacket(firstPacket.timestamp + 1);
+ const firstKeyPacket = await sink.getKeyPacket(firstPacket.timestamp + 1.0);
assert(firstKeyPacket);
expect(firstKeyPacket.type).toBe('key');
expect(firstKeyPacket.sequenceNumber).toBe(firstPacket.sequenceNumber);
@@ -367,6 +378,7 @@ test('MPEG-TS video key packets', async () => {
const keyPacket = await sink.getKeyPacket(packet.timestamp);
assert(keyPacket);
expect(keyPacket.timestamp).toBe(packet.timestamp); // The correct timestamp was retrieved for this packet
+ expect(keyPacket.duration).toBe(packet.duration); // The correct duration was retrieved for this packet
expect(keyPacket.sequenceNumber).toBe(packet.sequenceNumber);
}
});
@@ -415,6 +427,7 @@ test('MPEG-TS audio key packets', async () => {
const keyPacket = await sink.getKeyPacket(packet.timestamp);
assert(keyPacket);
expect(keyPacket.timestamp).toBe(packet.timestamp); // The correct timestamp was retrieved for this packet
+ expect(keyPacket.duration).toBe(packet.duration); // The correct duration was retrieved for this packet
expect(keyPacket.sequenceNumber).toBe(packet.sequenceNumber);
}
});
@@ -444,7 +457,7 @@ test('MPEG-TS with unknown file size (ReadableStreamSource)', async () => {
expect(middlePacket.timestamp).toBeCloseTo(12.5);
const duration = await videoTrack.computeDuration();
- expect(duration).toBeCloseTo(14.983333333333333);
+ expect(duration).toBeCloseTo(15);
// Ensure that reference points have still been added
expect((videoTrack._backing as unknown as MpegTsTrackBacking).referencePesPackets.length)