diff --git a/dev/demux.html b/dev/demux.html index c6402f5..08ba369 100644 --- a/dev/demux.html +++ b/dev/demux.html @@ -14,10 +14,44 @@ source: new Mediabunny.BlobSource(file), }); - const videoTrack = await input.getPrimaryVideoTrack(); - const sink = new Mediabunny.EncodedPacketSink(videoTrack); + const track = await input.getPrimaryVideoTrack(); + const sink = new Mediabunny.EncodedPacketSink(track); - console.log(await sink.getKeyPacket(Infinity)); + const packet = await sink.getPacket(12.5); + console.log(packet, await sink.getNextPacket(packet)) + + /* + + let currentPacket = await sink.getFirstPacket(); + while (currentPacket) { + console.log(currentPacket); + currentPacket = await sink.getNextPacket(currentPacket); + } + + //const secondPacket = await sink.getNextPacket(firstPacket); + //const thirdPacket = await sink.getNextPacket(secondPacket); + //const fourthPacket = await sink.getNextPacket(thirdPacket); + //const fifthPacket = await sink.getNextPacket(fourthPacket); + //const sixthPacket = await sink.getNextPacket(fifthPacket); + //const seventhPacket = await sink.getNextPacket(sixthPacket); + //const eighthPacket = await sink.getNextPacket(seventhPacket); + //const ninthPacket = await sink.getNextPacket(eighthPacket); + //const tenthPacket = await sink.getNextPacket(ninthPacket); + + console.log(firstPacket, secondPacket, thirdPacket, fourthPacket, fifthPacket, sixthPacket, seventhPacket, eighthPacket, ninthPacket, tenthPacket); + + const packets = [firstPacket, secondPacket, thirdPacket, fourthPacket, fifthPacket, sixthPacket, seventhPacket, eighthPacket, ninthPacket, tenthPacket]; + + const videoDecoder = new VideoDecoder({ + output: (frame) => console.log(frame.timestamp, frame), + error: console.error, + }); + videoDecoder.configure(await videoTrack.getDecoderConfig()); + + for (const packet of packets) { + //videoDecoder.decode(packet.toEncodedVideoChunk()); + } + */ /* diff --git a/src/codec-data.ts b/src/codec-data.ts index ff4a2c7..83aa796 100644 --- a/src/codec-data.ts +++ b/src/codec-data.ts @@ -6,7 +6,7 @@ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -import { VideoCodec, VP9_LEVEL_TABLE } from './codec'; +import { AVC_LEVEL_TABLE, VideoCodec, VP9_LEVEL_TABLE } from './codec'; import { assert, assertNever, @@ -486,6 +486,8 @@ export type AvcSpsInfo = { transferCharacteristics: number; matrixCoefficients: number; fullRangeFlag: number; + numReorderFrames: number; + maxDecFrameBuffering: number; }; /** Parses an AVC SPS (Sequence Parameter Set) to extract basic information. */ @@ -573,8 +575,10 @@ export const parseAvcSps = (sps: Uint8Array): AvcSpsInfo | null => { readExpGolomb(bitstream); // max_num_ref_frames bitstream.skipBits(1); // gaps_in_frame_num_value_allowed_flag - const codedWidth = 16 * (readExpGolomb(bitstream) + 1); // pic_width_in_mbs_minus1 - const codedHeight = 16 * (readExpGolomb(bitstream) + 1); // pic_height_in_map_units_minus1 + const picWidthInMbsMinus1 = readExpGolomb(bitstream); + const picHeightInMapUnitsMinus1 = readExpGolomb(bitstream); + const codedWidth = 16 * (picWidthInMbsMinus1 + 1); + const codedHeight = 16 * (picHeightInMapUnitsMinus1 + 1); let displayWidth = codedWidth; let displayHeight = codedHeight; @@ -619,6 +623,9 @@ export const parseAvcSps = (sps: Uint8Array): AvcSpsInfo | null => { let matrixCoefficients = 2; let fullRangeFlag = 0; + let numReorderFrames: number | null = null; + let maxDecFrameBuffering: number | null = null; + const vuiParametersPresentFlag = bitstream.readBits(1); if (vuiParametersPresentFlag) { const aspectRatioInfoPresentFlag = bitstream.readBits(1); @@ -646,8 +653,85 @@ export const parseAvcSps = (sps: Uint8Array): AvcSpsInfo | null => { matrixCoefficients = bitstream.readBits(8); } } + + const chromaLocInfoPresentFlag = bitstream.readBits(1); + if (chromaLocInfoPresentFlag) { + readExpGolomb(bitstream); // chroma_sample_loc_type_top_field + readExpGolomb(bitstream); // chroma_sample_loc_type_bottom_field + } + + const timingInfoPresentFlag = bitstream.readBits(1); + if (timingInfoPresentFlag) { + bitstream.skipBits(32); // num_units_in_tick + bitstream.skipBits(32); // time_scale + bitstream.skipBits(1); // fixed_frame_rate_flag + } + + const nalHrdParametersPresentFlag = bitstream.readBits(1); + if (nalHrdParametersPresentFlag) { + skipAvcHrdParameters(bitstream); + } + + const vclHrdParametersPresentFlag = bitstream.readBits(1); + if (vclHrdParametersPresentFlag) { + skipAvcHrdParameters(bitstream); + } + + if (nalHrdParametersPresentFlag || vclHrdParametersPresentFlag) { + bitstream.skipBits(1); // low_delay_hrd_flag + } + + bitstream.skipBits(1); // pic_struct_present_flag + + const bitstreamRestrictionFlag = bitstream.readBits(1); + if (bitstreamRestrictionFlag) { + bitstream.skipBits(1); // motion_vectors_over_pic_boundaries_flag + readExpGolomb(bitstream); // max_bytes_per_pic_denom + readExpGolomb(bitstream); // max_bits_per_mb_denom + readExpGolomb(bitstream); // log2_max_mv_length_horizontal + readExpGolomb(bitstream); // log2_max_mv_length_vertical + numReorderFrames = readExpGolomb(bitstream); + maxDecFrameBuffering = readExpGolomb(bitstream); + } } + if (numReorderFrames === null) { + assert(maxDecFrameBuffering === null); + const constraintSet3Flag = constraintFlags & 0b00010000; + + if ( + (profileIdc === 44 || profileIdc === 86 || profileIdc === 100 + || profileIdc === 110 || profileIdc === 122 || profileIdc === 244 + ) && constraintSet3Flag + ) { + // "If profile_idc is equal to 44, 86, 100, 110, 122, or 244 and constraint_set3_flag is equal to 1, the + // value of num_reorder_frames shall be inferred to be equal to 0." + numReorderFrames = 0; + maxDecFrameBuffering = 0; + } else { + const picWidthInMbs = picWidthInMbsMinus1 + 1; + const picHeightInMapUnits = picHeightInMapUnitsMinus1 + 1; + const frameHeightInMbs = (2 - frameMbsOnlyFlag) * picHeightInMapUnits; + + const levelInfo = AVC_LEVEL_TABLE.find( + x => x.level >= levelIdc, + ) ?? last(AVC_LEVEL_TABLE)!; + + // "MaxDpbFrames is equal to + // Min( MaxDpbMbs / ( picWidthInMbs * frameHeightInMbs ), 16 ) and MaxDpbMbs is given in Table A-1." + const maxDpbFrames = Math.min( + Math.floor(levelInfo.maxDpbMbs / (picWidthInMbs * frameHeightInMbs)), + 16, + ); + + // "Otherwise, [...] the value of num_reorder_frames shall be inferred to be equal to MaxDpbFrames." + numReorderFrames = maxDpbFrames; + maxDecFrameBuffering = maxDpbFrames; + } + } + + assert(maxDecFrameBuffering !== null); + return { profileIdc, constraintFlags, @@ -664,6 +748,8 @@ export const parseAvcSps = (sps: Uint8Array): AvcSpsInfo | null => { matrixCoefficients, transferCharacteristics, fullRangeFlag, + numReorderFrames, + maxDecFrameBuffering, }; } catch (error) { console.error('Error parsing AVC SPS:', error); @@ -671,6 +757,23 @@ export const parseAvcSps = (sps: Uint8Array): AvcSpsInfo | null => { } }; +const skipAvcHrdParameters = (bitstream: Bitstream) => { + const cpb_cnt_minus1 = readExpGolomb(bitstream); + bitstream.skipBits(4); // bit_rate_scale + bitstream.skipBits(4); // cpb_size_scale + + for (let i = 0; i <= cpb_cnt_minus1; i++) { + readExpGolomb(bitstream); // bit_rate_value_minus1[i] + readExpGolomb(bitstream); // cpb_size_value_minus1[i] + bitstream.skipBits(1); // cbr_flag[i] + } + + bitstream.skipBits(5); // initial_cpb_removal_delay_length_minus1 + bitstream.skipBits(5); // cpb_removal_delay_length_minus1 + bitstream.skipBits(5); // dpb_output_delay_length_minus1 + bitstream.skipBits(5); // time_offset_length +}; + // Data specified in ISO 14496-15 export type HevcDecoderConfigurationRecord = { configurationVersion: number; @@ -1085,7 +1188,7 @@ const parseVuiForMinSpatialSegmentationIdc = (bitstream: Bitstream, sps_max_sub_ readExpGolomb(bitstream); // vui_num_ticks_poc_diff_one_minus1 } if (bitstream.readBits(1)) { - skipHrdParameters(bitstream, true, sps_max_sub_layers_minus1); + skipHevcHrdParameters(bitstream, true, sps_max_sub_layers_minus1); } } if (bitstream.readBits(1)) { // bitstream_restriction_flag @@ -1103,7 +1206,7 @@ const parseVuiForMinSpatialSegmentationIdc = (bitstream: Bitstream, sps_max_sub_ return 0; }; -const skipHrdParameters = ( +const skipHevcHrdParameters = ( bitstream: Bitstream, commonInfPresentFlag: boolean, maxNumSubLayersMinus1: number, diff --git a/src/codec.ts b/src/codec.ts index 9429bd4..c43733e 100644 --- a/src/codec.ts +++ b/src/codec.ts @@ -116,26 +116,26 @@ export type SubtitleCodec = typeof SUBTITLE_CODECS[number]; export type MediaCodec = VideoCodec | AudioCodec | SubtitleCodec; // https://en.wikipedia.org/wiki/Advanced_Video_Coding -const AVC_LEVEL_TABLE = [ - { maxMacroblocks: 99, maxBitrate: 64000, level: 0x0A }, // Level 1 - { maxMacroblocks: 396, maxBitrate: 192000, level: 0x0B }, // Level 1.1 - { maxMacroblocks: 396, maxBitrate: 384000, level: 0x0C }, // Level 1.2 - { maxMacroblocks: 396, maxBitrate: 768000, level: 0x0D }, // Level 1.3 - { maxMacroblocks: 396, maxBitrate: 2000000, level: 0x14 }, // Level 2 - { maxMacroblocks: 792, maxBitrate: 4000000, level: 0x15 }, // Level 2.1 - { maxMacroblocks: 1620, maxBitrate: 4000000, level: 0x16 }, // Level 2.2 - { maxMacroblocks: 1620, maxBitrate: 10000000, level: 0x1E }, // Level 3 - { maxMacroblocks: 3600, maxBitrate: 14000000, level: 0x1F }, // Level 3.1 - { maxMacroblocks: 5120, maxBitrate: 20000000, level: 0x20 }, // Level 3.2 - { maxMacroblocks: 8192, maxBitrate: 20000000, level: 0x28 }, // Level 4 - { maxMacroblocks: 8192, maxBitrate: 50000000, level: 0x29 }, // Level 4.1 - { maxMacroblocks: 8704, maxBitrate: 50000000, level: 0x2A }, // Level 4.2 - { maxMacroblocks: 22080, maxBitrate: 135000000, level: 0x32 }, // Level 5 - { maxMacroblocks: 36864, maxBitrate: 240000000, level: 0x33 }, // Level 5.1 - { maxMacroblocks: 36864, maxBitrate: 240000000, level: 0x34 }, // Level 5.2 - { maxMacroblocks: 139264, maxBitrate: 240000000, level: 0x3C }, // Level 6 - { maxMacroblocks: 139264, maxBitrate: 480000000, level: 0x3D }, // Level 6.1 - { maxMacroblocks: 139264, maxBitrate: 800000000, level: 0x3E }, // Level 6.2 +export const AVC_LEVEL_TABLE = [ + { maxMacroblocks: 99, maxBitrate: 64000, maxDpbMbs: 396, level: 0x0A }, // Level 1 + { maxMacroblocks: 396, maxBitrate: 192000, maxDpbMbs: 900, level: 0x0B }, // Level 1.1 + { maxMacroblocks: 396, maxBitrate: 384000, maxDpbMbs: 2376, level: 0x0C }, // Level 1.2 + { maxMacroblocks: 396, maxBitrate: 768000, maxDpbMbs: 2376, level: 0x0D }, // Level 1.3 + { maxMacroblocks: 396, maxBitrate: 2000000, maxDpbMbs: 2376, level: 0x14 }, // Level 2 + { maxMacroblocks: 792, maxBitrate: 4000000, maxDpbMbs: 4752, level: 0x15 }, // Level 2.1 + { maxMacroblocks: 1620, maxBitrate: 4000000, maxDpbMbs: 8100, level: 0x16 }, // Level 2.2 + { maxMacroblocks: 1620, maxBitrate: 10000000, maxDpbMbs: 8100, level: 0x1E }, // Level 3 + { maxMacroblocks: 3600, maxBitrate: 14000000, maxDpbMbs: 18000, level: 0x1F }, // Level 3.1 + { maxMacroblocks: 5120, maxBitrate: 20000000, maxDpbMbs: 20480, level: 0x20 }, // Level 3.2 + { maxMacroblocks: 8192, maxBitrate: 20000000, maxDpbMbs: 32768, level: 0x28 }, // Level 4 + { maxMacroblocks: 8192, maxBitrate: 50000000, maxDpbMbs: 32768, level: 0x29 }, // Level 4.1 + { maxMacroblocks: 8704, maxBitrate: 50000000, maxDpbMbs: 34816, level: 0x2A }, // Level 4.2 + { maxMacroblocks: 22080, maxBitrate: 135000000, maxDpbMbs: 110400, level: 0x32 }, // Level 5 + { maxMacroblocks: 36864, maxBitrate: 240000000, maxDpbMbs: 184320, level: 0x33 }, // Level 5.1 + { maxMacroblocks: 36864, maxBitrate: 240000000, maxDpbMbs: 184320, level: 0x34 }, // Level 5.2 + { maxMacroblocks: 139264, maxBitrate: 240000000, maxDpbMbs: 696320, level: 0x3C }, // Level 6 + { maxMacroblocks: 139264, maxBitrate: 480000000, maxDpbMbs: 696320, level: 0x3D }, // Level 6.1 + { maxMacroblocks: 139264, maxBitrate: 800000000, maxDpbMbs: 696320, level: 0x3E }, // Level 6.2 ]; // https://en.wikipedia.org/wiki/High_Efficiency_Video_Coding diff --git a/src/mpeg-ts/mpeg-ts-demuxer.ts b/src/mpeg-ts/mpeg-ts-demuxer.ts index 4eaba45..f929ad8 100644 --- a/src/mpeg-ts/mpeg-ts-demuxer.ts +++ b/src/mpeg-ts/mpeg-ts-demuxer.ts @@ -37,6 +37,7 @@ import { binarySearchLessOrEqual, Bitstream, COLOR_PRIMARIES_MAP_INVERSE, + findLastIndex, last, MATRIX_COEFFICIENTS_MAP_INVERSE, Rotation, @@ -65,6 +66,7 @@ type ElementaryStream = { colorSpace: VideoColorSpaceInit; width: number; height: number; + reorderSize: number; } | { type: 'audio'; codec: AudioCodec; @@ -234,6 +236,7 @@ export class MpegTsDemuxer extends Demuxer { }, width: -1, height: -1, + reorderSize: -1, }; }; break; } @@ -290,6 +293,7 @@ export class MpegTsDemuxer extends Demuxer { VideoMatrixCoefficients | undefined, fullRange: !!spsInfo.fullRangeFlag, }; + elementaryStream.info.reorderSize = spsInfo.maxDecFrameBuffering; elementaryStream.initialized = true; } @@ -638,7 +642,9 @@ export abstract class MpegTsTrackBacking implements InputTrackBacking { */ referencePesPackets: PesPacketHeader[] = []; endReferencePesPacketAdded = false; - readingContexts = new WeakMap(); + packetBuffers = new WeakMap(); + /** Used for recreating PacketBuffers if necessary. */ + packetSectionStarts = new WeakMap(); mutex = new AsyncMutex(); constructor(public elementaryStream: ElementaryStream) {} @@ -683,6 +689,22 @@ export abstract class MpegTsTrackBacking implements InputTrackBacking { abstract getPacketType(packetData: Uint8Array): PacketType; abstract markNextPacket(context: PacketReadingContext): Promise; + abstract getReorderSize(): number; + + createEncodedPacket( + suppliedPacket: SuppliedPacket, + duration: number, + options: PacketRetrievalOptions, + ) { + return new EncodedPacket( + options.metadataOnly ? PLACEHOLDER_DATA : suppliedPacket.data, + this.getPacketType(suppliedPacket.data), + suppliedPacket.pts / TIMESCALE, + Math.max(duration / TIMESCALE, 0), + suppliedPacket.sequenceNumber, + suppliedPacket.data.byteLength, + ); + } maybeInsertReferencePacket(pesPacketHeader: PesPacketHeader, force: boolean, dropIfMutexLocked: boolean) { if (dropIfMutexLocked && this.mutex.pending > 0) { @@ -729,21 +751,72 @@ export abstract class MpegTsTrackBacking implements InputTrackBacking { assert(pesPacket); const context = new PacketReadingContext(this, pesPacket, true); - await this.markNextPacket(context); + const buffer = new PacketBuffer(this, context); - return context.createAndLinkEncodedPacket(context.suppliedPacket, options); + const result = await buffer.readNext(); + if (!result) { + return null; + } + + const packet = this.createEncodedPacket(result.packet, result.duration, options); + this.packetBuffers.set(packet, buffer); + this.packetSectionStarts.set(packet, result.packet.sectionStartPos); + + return packet; } async getNextPacket(packet: EncodedPacket, options: PacketRetrievalOptions): Promise { - const context = this.readingContexts.get(packet); - if (!context) { + let buffer = this.packetBuffers.get(packet); + + if (buffer) { + // Fast path + const result = await buffer.readNext(); + if (!result) { + return null; + } + + // Remove PacketBuffer access from the old packet, it belongs to the next packet now + this.packetBuffers.delete(packet); + + const newPacket = this.createEncodedPacket(result.packet, result.duration, options); + this.packetBuffers.set(newPacket, buffer); + this.packetSectionStarts.set(newPacket, result.packet.sectionStartPos); + + return newPacket; + } + + // No buffer, we gotta do some rereading + const sectionStartPos = this.packetSectionStarts.get(packet); + if (sectionStartPos === undefined) { throw new Error('Packet was not created from this track.'); } - const clone = context.clone(); - await this.markNextPacket(clone); + const demuxer = this.elementaryStream.demuxer; + const section = await demuxer.readSection(sectionStartPos, true); + assert(section); - return clone.createAndLinkEncodedPacket(clone.suppliedPacket, options); + const pesPacket = readPesPacket(section); + assert(pesPacket); + + const context = new PacketReadingContext(this, pesPacket, true); + buffer = new PacketBuffer(this, context); + + // Advance until we pass the current packet's sequence number + const targetSequenceNumber = packet.sequenceNumber; + while (true) { + const result = await buffer.readNext(); + if (!result) { + return null; + } + + if (result.packet.sequenceNumber > targetSequenceNumber) { + // We found the next packet! + const newPacket = this.createEncodedPacket(result.packet, result.duration, options); + this.packetBuffers.set(newPacket, buffer); + this.packetSectionStarts.set(newPacket, result.packet.sectionStartPos); + return newPacket; + } + } } async getNextKeyPacket(packet: EncodedPacket, options: PacketRetrievalOptions): Promise { @@ -771,8 +844,6 @@ export abstract class MpegTsTrackBacking implements InputTrackBacking { return this.doPacketLookup(timestamp, true, options); } - abstract getPacketLookaround(): number; - /** * Searches for the packet with the largest timestamp not larger than `timestamp` in the file, using a combination * of binary search and linear refinement. @@ -914,9 +985,6 @@ export abstract class MpegTsTrackBacking implements InputTrackBacking { release(); - /** Stores the best PES packet we've found so far (that meets all required criteria). */ - let bestPesPacketHeader: PesPacketHeader | null = null; - const pesPacketHasKeyframe = async (sectionStartPos: number) => { const section = await demuxer.readSection(sectionStartPos, true); assert(section); @@ -925,6 +993,7 @@ export abstract class MpegTsTrackBacking implements InputTrackBacking { const fullPesPacket = readPesPacket(section); assert(fullPesPacket); + // Only mark the first packet const context = new PacketReadingContext(this, fullPesPacket, false); await this.markNextPacket(context); @@ -932,16 +1001,9 @@ export abstract class MpegTsTrackBacking implements InputTrackBacking { return false; } - return this.getPacketType(context.suppliedPacket.data); + return this.getPacketType(context.suppliedPacket.data) === 'key'; }; - if (!keyframesOnly || await pesPacketHasKeyframe(currentPesPacketHeader.sectionStartPos)) { - bestPesPacketHeader = currentPesPacketHeader; - } - - // "advanced" as in "moved past" - const advancedPesPacketHeaders = [bestPesPacketHeader]; - // Starting from the binary search guess, let's now find the moment where the packet timestamps cross the // search timestamp. This point will then be used as the center around which we search. outer: @@ -976,17 +1038,7 @@ export abstract class MpegTsTrackBacking implements InputTrackBacking { break; } - // Collect matching packets we find along the way - if ( - (bestPesPacketHeader === null || bestPesPacketHeader.pts < nextPesPacketHeader.pts) - && nextPesPacketHeader.pts <= searchPts - && (!keyframesOnly || await pesPacketHasKeyframe(currentPos)) - ) { - bestPesPacketHeader = nextPesPacketHeader; - } - currentPesPacketHeader = nextPesPacketHeader; - advancedPesPacketHeaders.push(nextPesPacketHeader); if (reader.fileSize === null) { // If the file size is undefined, that means that the binary search step is skipped, meaning no @@ -995,191 +1047,132 @@ export abstract class MpegTsTrackBacking implements InputTrackBacking { } } - // Lookaround is needed in the first place because packets don't need to appear in PTS order, they only appear - // in decode order. When B-frames are present, finding the packet that's actually closest to the search - // timestamp requires searching a small local window. - const lookaround = this.getPacketLookaround(); + const reorderSize = this.getReorderSize(); - // Depending on how long the previous scan went, we might not need to do the full lookbehind, or even none at - // all if we're lucky - const lookbehindNeeded = Math.max(lookaround - advancedPesPacketHeaders.length + 1, 0); - let minPos = advancedPesPacketHeaders[0]!.sectionStartPos; - - /** Scans `n` contiguous PES packets in succession. */ - const doLinearScan = async (startPos: number, n: number) => { - let currentPos = startPos; - - outer: - for (let i = 0; i < n; i++) { - while (true) { - const packetHeader = await demuxer.readPacketHeader(currentPos); - if (!packetHeader) { - break outer; // End of file - } - - if ( - packetHeader.pid === this.elementaryStream.pid - && packetHeader.payloadUnitStartIndicator === 1 - ) { - break; - } - - currentPos += demuxer.packetStride; - } - - const section = await demuxer.readSection(currentPos, false); - assert(section); - assert(section.pid === this.elementaryStream.pid); - - const pesPacketHeader = readPesPacketHeader(section); - if (!pesPacketHeader) { - throw new Error(MISSING_PES_PACKET_ERROR); - } - - if ( - (bestPesPacketHeader === null || bestPesPacketHeader.pts < pesPacketHeader.pts) - && pesPacketHeader.pts <= searchPts - && (!keyframesOnly || await pesPacketHasKeyframe(currentPos)) - ) { - bestPesPacketHeader = pesPacketHeader; - } - - currentPos += demuxer.packetStride; - } - }; - - // Lookbehind - if (lookbehindNeeded > 0) { - outer: - for (let i = 0; i < lookbehindNeeded; i++) { - let currentPos = minPos; - - while (true) { - currentPos -= demuxer.packetStride; - - const packetHeader = await demuxer.readPacketHeader(currentPos); - if (!packetHeader) { - break outer; - } - - if ( - packetHeader.pid === this.elementaryStream.pid - && packetHeader.payloadUnitStartIndicator === 1 - ) { - break; - } - } - - minPos = currentPos; - } - - await doLinearScan(minPos, lookbehindNeeded); - } - - // Lookahead - await doLinearScan(currentPesPacketHeader.sectionStartPos + demuxer.packetStride, lookaround); - - // If we're looking specifically for a keyframe but haven't found one yet, that means we'll need to go left - // until we find one. - if (!bestPesPacketHeader && keyframesOnly) { - let currentPos = minPos; + // Rewind by reorderSize PES packets (even for audio! To ensure proper durations) + for (let i = 0; i < reorderSize; i++) { + let pos = currentPesPacketHeader.sectionStartPos - demuxer.packetStride; while (true) { - currentPos -= demuxer.packetStride; - - const packetHeader = await demuxer.readPacketHeader(currentPos); + const packetHeader = await demuxer.readPacketHeader(pos); if (!packetHeader) { - break; + break; // Hit start of file } if (packetHeader.pid === this.elementaryStream.pid && packetHeader.payloadUnitStartIndicator === 1) { - const section = await demuxer.readSection(currentPos, false); - assert(section); + const headerSection = await demuxer.readSection(pos, false); + assert(headerSection); - const pesPacketHeader = readPesPacketHeader(section); - if (!pesPacketHeader) { + const header = readPesPacketHeader(headerSection); + if (!header) { throw new Error(MISSING_PES_PACKET_ERROR); } - if (pesPacketHeader.pts <= searchPts && (await pesPacketHasKeyframe(currentPos))) { - bestPesPacketHeader = pesPacketHeader; - break; - } + currentPesPacketHeader = header; + break; } + + pos -= demuxer.packetStride; } } - if (!bestPesPacketHeader) { - // Nothing was found + // Read the full section and create a PacketBuffer + const section = await demuxer.readSection(currentPesPacketHeader.sectionStartPos, true); + assert(section); + + const pesPacket = readPesPacket(section); + assert(pesPacket); + + const context = new PacketReadingContext(this, pesPacket, true); + const buffer = new PacketBuffer(this, context); + + // Advance until the top-most presentation timestamp crosses or equals searchPts + while (true) { + const topPts = last(buffer.presentationOrderPackets)?.pts ?? -Infinity; + if (topPts >= searchPts) { + break; + } + + const didRead = await buffer.readNextDecodeOrderPacket(); + if (!didRead) { + break; + } + } + + // Find the target packet: the one with largest PTS <= searchPts that is also a keyframe if required + const targetIndex = findLastIndex( + buffer.presentationOrderPackets, + p => p.pts <= searchPts && (!keyframesOnly || this.getPacketType(p.data) === 'key'), + ); + + if (targetIndex !== -1) { + const targetPacket = buffer.presentationOrderPackets[targetIndex]!; + const lastDuration = targetIndex === 0 + ? 0 + : targetPacket.pts - buffer.presentationOrderPackets[targetIndex - 1]!.pts; + + // Pop packets in decode order until we hit the target packet + while (buffer.decodeOrderPackets[0] !== targetPacket) { + buffer.decodeOrderPackets.shift(); + } + buffer.lastDuration = lastDuration; + + // Now consume the target packet through readNext to get proper duration + const result = await buffer.readNext(); + assert(result); + + const packet = this.createEncodedPacket(result.packet, result.duration, options); + this.packetBuffers.set(packet, buffer); + this.packetSectionStarts.set(packet, result.packet.sectionStartPos); + + return packet; + } + + if (!keyframesOnly) { + // We didn't find a suitable packet return null; } - const bestSection = await demuxer.readSection(bestPesPacketHeader.sectionStartPos, true); // Read it in full - let bestPesPacket = readPesPacket(bestSection!); - assert(bestPesPacket); + // Go backwards looking for a PES packet with a keyframe + let searchPos = currentPesPacketHeader.sectionStartPos; - // Final stage: we found the best PES packet, but that PES packet might contain multiple individual encoded - // packets. Or, it might not be the start of an encoded packet, and simply a continuation of a previous one. - // So, we have one last search to do. while (true) { - const context = new PacketReadingContext(this, bestPesPacket, false); // Capped context + searchPos -= demuxer.packetStride; - let bestPacket: SuppliedPacket | null = null; - let bestContext: PacketReadingContext | null = null; - - while (true) { - // Stupid aliasing trick to make TypeScript not infer stuff wrong - const context2 = context; - context2.suppliedPacket = null; - - await this.markNextPacket(context); - if (!context.suppliedPacket) { - break; - } - - const eligible = context.suppliedPacket.pts <= searchPts - && (!keyframesOnly || this.getPacketType(context.suppliedPacket.data) === 'key'); - if (!eligible) { - continue; - } - - if (!bestPacket || bestPacket.pts < context.suppliedPacket.pts) { - bestPacket = context.suppliedPacket; - bestContext = context.clone(); // Kiiiinda ugly - } + const packetHeader = await demuxer.readPacketHeader(searchPos); + if (!packetHeader) { + return null; // Hit start of file } - if (bestPacket) { - assert(bestContext); - return bestContext.createAndLinkEncodedPacket(bestPacket, options); + if (packetHeader.pid !== this.elementaryStream.pid || packetHeader.payloadUnitStartIndicator !== 1) { + continue; } - // We didn't find an encoded packet! Let's go to the previous PES packet until we find one. + if (!(await pesPacketHasKeyframe(searchPos))) { + continue; + } - let currentPos = bestPesPacket.sectionStartPos; + // Found a PES packet with a keyframe. Set up a PacketBuffer and pull until we get the keyframe. + const keySection = await demuxer.readSection(searchPos, true); + assert(keySection); + const keyPesPacket = readPesPacket(keySection); + assert(keyPesPacket); + + const keyContext = new PacketReadingContext(this, keyPesPacket, true); + const keyBuffer = new PacketBuffer(this, keyContext); + + // Pull until we get a keyframe while (true) { - currentPos -= demuxer.packetStride; + const result = await keyBuffer.readNext(); + assert(result); // How else? - const packetHeader = await demuxer.readPacketHeader(currentPos); - if (!packetHeader) { - // Past start of file - return null; - } + if (this.getPacketType(result.packet.data) === 'key') { + const packet = this.createEncodedPacket(result.packet, result.duration, options); + this.packetBuffers.set(packet, keyBuffer); + this.packetSectionStarts.set(packet, result.packet.sectionStartPos); - if (packetHeader.pid === this.elementaryStream.pid && packetHeader.payloadUnitStartIndicator === 1) { - const section = await demuxer.readSection(currentPos, true); - assert(section); - - const pesPacket = readPesPacket(section); - if (!pesPacket) { - throw new Error(MISSING_PES_PACKET_ERROR); - } - - if (pesPacket.pts <= searchPts) { - bestPesPacket = pesPacket; - break; - } + return packet; } } } @@ -1245,14 +1238,13 @@ class MpegTsVideoTrackBacking extends MpegTsTrackBacking implements InputVideoTr return determineVideoPacketType(this.elementaryStream.info.codec, this.decoderConfig, packetData) ?? 'key'; } - override getPacketLookaround(): number { - // Due to B-frames. A lookaround of +-5 packets will pretty much guarantee we find the correct packet for a - // given timestamp, although of course, this could technically still fail. - // todo, use max_num_reorder_frames here? - return 5; + override getReorderSize(): number { + return this.elementaryStream.info.reorderSize; } override async markNextPacket(context: PacketReadingContext): Promise { + assert(!context.suppliedPacket); + const CHUNK_SIZE = 128; let packetStartPos: number | null = null; @@ -1388,11 +1380,13 @@ class MpegTsAudioTrackBacking extends MpegTsTrackBacking implements InputAudioTr return 'key'; } - override getPacketLookaround(): number { - return 0; + override getReorderSize(): number { + return 1; // No reordering, since no B-frames because goated } override async markNextPacket(context: PacketReadingContext): Promise { + assert(!context.suppliedPacket); + const CHUNK_SIZE = 128; while (true) { @@ -1444,9 +1438,9 @@ class MpegTsAudioTrackBacking extends MpegTsTrackBacking implements InputAudioTr type SuppliedPacket = { pts: number; - intrinsicDuration: number; data: Uint8Array; sequenceNumber: number; + sectionStartPos: number; }; /** Stateful context used to extract exact encoded packets from the underlying data stream. */ @@ -1662,39 +1656,148 @@ class PacketReadingContext { const pts = this.nextPts; this.nextPts += intrinsicDuration; + const sectionStartPos = currentPesPacket.sectionStartPos; // The sequence number is the starting position of the section the PES packet is in, PLUS the offset within the // PES packet where the packet starts. - const sequenceNumber = currentPesPacket.sectionStartPos + (this.currentPos - this.currentPesPacketPos); + const sequenceNumber = sectionStartPos + (this.currentPos - this.currentPesPacketPos); const data = this.readBytes(packetLength); this.suppliedPacket = { pts, - intrinsicDuration, data, sequenceNumber, + sectionStartPos, }; this.pesPackets.splice(0, this.currentPesPacketIndex); this.currentPesPacketIndex = 0; } +} - createAndLinkEncodedPacket(suppliedPacket: SuppliedPacket | null, options: PacketRetrievalOptions) { - if (!suppliedPacket) { - return null; +/** + * A buffer that simulates decoder frame reordering to compute packet durations. Packets arrive in decode order but + * durations are based on presentation order. + */ +class PacketBuffer { + backing: MpegTsTrackBacking; + context: PacketReadingContext; + + decodeOrderPackets: SuppliedPacket[] = []; + reorderSize: number; + reorderBuffer: SuppliedPacket[] = []; + presentationOrderPackets: SuppliedPacket[] = []; + reachedEnd = false; + lastDuration = 0; + + constructor(backing: MpegTsTrackBacking, context: PacketReadingContext) { + this.backing = backing; + this.context = context; + this.reorderSize = backing.getReorderSize(); + assert(this.reorderSize >= 0); + } + + async readNext(): Promise<{ packet: SuppliedPacket; duration: number } | null> { + if (this.decodeOrderPackets.length === 0) { + // We need the next packet + const didRead = await this.readNextDecodeOrderPacket(); + if (!didRead) { + return null; + } } - const packet = new EncodedPacket( - options.metadataOnly ? PLACEHOLDER_DATA : suppliedPacket.data, - this.backing.getPacketType(suppliedPacket.data), - suppliedPacket.pts / TIMESCALE, - suppliedPacket.intrinsicDuration / TIMESCALE, - suppliedPacket.sequenceNumber, - suppliedPacket.data.byteLength, - ); + // Ensure we know the next packet in presentation order so we can compute the current packet's duration + await this.ensureCurrentPacketHasNext(); - // Link the context for next packet retrieval - this.backing.readingContexts.set(packet, this); + const packet = this.decodeOrderPackets[0]!; - return packet; + // Let's compute the duration + const presentationIndex = this.presentationOrderPackets.indexOf(packet); + assert(presentationIndex !== -1); + + let duration: number; + if (presentationIndex === this.presentationOrderPackets.length - 1) { + duration = this.lastDuration; // Reasonable heuristic + } else { + const nextPacket = this.presentationOrderPackets[presentationIndex + 1]!; + duration = nextPacket.pts - packet.pts; + this.lastDuration = duration; + } + + this.decodeOrderPackets.shift(); + + // Shrink the presentation array as much as possible + while (this.presentationOrderPackets.length > 0) { + const first = this.presentationOrderPackets[0]!; + if (this.decodeOrderPackets.includes(first)) { + break; + } + + this.presentationOrderPackets.shift(); + } + + return { packet, duration }; + } + + async readNextDecodeOrderPacket() { + if (this.reachedEnd) { + return false; + } + + this.context.suppliedPacket = null; + await this.backing.markNextPacket(this.context); + + if (!this.context.suppliedPacket) { + this.reachedEnd = true; + this.flushReorderBuffer(); + + return false; + } + + this.decodeOrderPackets.push(this.context.suppliedPacket); + this.processPacketThroughReorderBuffer(this.context.suppliedPacket); + + return true; + } + + async ensureCurrentPacketHasNext() { + const current = this.decodeOrderPackets[0]; + assert(current); + + while (true) { + const presentationIndex = this.presentationOrderPackets.indexOf(current); + + // Check if current packet has a next packet + if (presentationIndex !== -1 && presentationIndex <= this.presentationOrderPackets.length - 2) { + break; + } + + const didRead = await this.readNextDecodeOrderPacket(); + if (!didRead) { + break; + } + } + } + + processPacketThroughReorderBuffer(packet: SuppliedPacket) { + this.reorderBuffer.push(packet); + + // If buffer is full, output the packet with smallest PTS + if (this.reorderBuffer.length >= this.reorderSize) { + let minIndex = 0; + for (let i = 1; i < this.reorderBuffer.length; i++) { + if (this.reorderBuffer[i]!.pts < this.reorderBuffer[minIndex]!.pts) { + minIndex = i; + } + } + + const packet = this.reorderBuffer.splice(minIndex, 1)[0]!; + this.presentationOrderPackets.push(packet); + } + } + + flushReorderBuffer() { + this.reorderBuffer.sort((a, b) => a.pts - b.pts); + this.presentationOrderPackets.push(...this.reorderBuffer); + this.reorderBuffer.length = 0; } } diff --git a/test/node/mpeg-ts-demuxing.test.ts b/test/node/mpeg-ts-demuxing.test.ts index aced224..09c0acb 100644 --- a/test/node/mpeg-ts-demuxing.test.ts +++ b/test/node/mpeg-ts-demuxing.test.ts @@ -89,7 +89,7 @@ test('MPEG-TS durations', async () => { expect(videoFirstTimestamp).toBe(10.033333333333333); const videoDuration = await videoTrack.computeDuration(); - expect(videoDuration).toBeCloseTo(14.983333333333333); + expect(videoDuration).toBeCloseTo(15); const audioTrack = await input.getPrimaryAudioTrack(); assert(audioTrack); @@ -119,7 +119,7 @@ test('MPEG-TS AVC video packets', async () => { expect(firstPacket.data.byteLength).toBe(23813); expect(firstPacket.type).toBe('key'); expect(firstPacket.timestamp).toBe(10.033333333333333); - expect(firstPacket.duration).toBe(0); + expect(firstPacket.duration).toBe(0.016666666666666666); expect(firstPacket.sequenceNumber).not.toBe(-1); const firstPacketMetadataOnly = await sink.getFirstPacket({ metadataOnly: true }); @@ -134,15 +134,17 @@ test('MPEG-TS AVC video packets', async () => { expect(secondPacket.data.byteLength).toBe(5700); expect(secondPacket.type).toBe('delta'); expect(secondPacket.timestamp).toBe(10.1); - expect(secondPacket.duration).toBe(0); + expect(secondPacket.duration).toBe(0.016666666666666666); expect(secondPacket.sequenceNumber).toBeGreaterThan(firstPacket.sequenceNumber); let currentPacket: EncodedPacket | null = firstPacket; let count = 0; while (currentPacket) { - count++; + expect(currentPacket.duration).toBe(0.016666666666666666); + currentPacket = await sink.getNextPacket(currentPacket); + count++; } expect(count).toBe(298); @@ -166,7 +168,7 @@ test('MPEG-TS AAC audio packets', async () => { expect(firstPacket.data.byteLength).toBe(348); expect(firstPacket.type).toBe('key'); expect(firstPacket.timestamp).toBe(10.012); - expect(firstPacket.duration).toBeCloseTo(0.021333333333333333); + expect(firstPacket.duration).toBe(0.021333333333333333); expect(firstPacket.sequenceNumber).not.toBe(-1); const secondPacket = await sink.getNextPacket(firstPacket); @@ -176,15 +178,17 @@ test('MPEG-TS AAC audio packets', async () => { expect(secondPacket.data.byteLength).toBe(349); expect(secondPacket.type).toBe('key'); expect(secondPacket.timestamp).toBeCloseTo(10.033333333333333); - expect(secondPacket.duration).toBeCloseTo(0.021333333333333333); + expect(secondPacket.duration).toBe(0.021333333333333333); expect(secondPacket.sequenceNumber).toBeGreaterThan(firstPacket.sequenceNumber); let currentPacket: EncodedPacket | null = firstPacket; let count = 0; while (currentPacket) { - count++; + expect(currentPacket.duration).toBe(0.021333333333333333); + currentPacket = await sink.getNextPacket(currentPacket); + count++; } expect(count).toBe(234); @@ -205,12 +209,13 @@ test('MPEG-TS video seeking', async () => { const firstPacket = await sink.getPacket(firstTimestamp); assert(firstPacket); expect(firstPacket.timestamp).toBe(firstTimestamp); + expect(firstPacket.duration).toBe(0.016666666666666666); expect(firstPacket.sequenceNumber).toBe((await sink.getFirstPacket())?.sequenceNumber); const lastPacket = await sink.getPacket(Infinity); assert(lastPacket); - expect(lastPacket.timestamp).toBeCloseTo(14.983333333333333); + expect(lastPacket.duration).toBe(0.016666666666666666); const beforeFirst = await sink.getPacket(-10); expect(beforeFirst).toBeNull(); @@ -218,6 +223,7 @@ test('MPEG-TS video seeking', async () => { const middlePacket = await sink.getPacket(12.5); assert(middlePacket); expect(middlePacket.timestamp).toBeCloseTo(12.5); + expect(middlePacket.duration).toBe(0.016666666666666666); const allPackets: EncodedPacket[] = []; let currentPacket: EncodedPacket | null = firstPacket; @@ -233,6 +239,7 @@ test('MPEG-TS video seeking', async () => { const seekedPacked = await sink.getPacket(packet.timestamp); assert(seekedPacked); expect(seekedPacked.timestamp).toBe(packet.timestamp); // The correct timestamp was retrieved for this packet + expect(seekedPacked.duration).toBe(packet.duration); // The correct duration was retrieved for this packet expect(seekedPacked.sequenceNumber).toBe(packet.sequenceNumber); } }); @@ -252,12 +259,13 @@ test('MPEG-TS audio seeking', async () => { const firstPacket = await sink.getPacket(firstTimestamp); assert(firstPacket); expect(firstPacket.timestamp).toBe(firstTimestamp); + expect(firstPacket.duration).toBe(0.021333333333333333); expect(firstPacket.sequenceNumber).toBe((await sink.getFirstPacket())?.sequenceNumber); const lastPacket = await sink.getPacket(Infinity); assert(lastPacket); - expect(lastPacket.timestamp).toBeCloseTo(14.982666666666667); + expect(lastPacket.duration).toBe(0.021333333333333333); const beforeFirst = await sink.getPacket(-10); expect(beforeFirst).toBeNull(); @@ -265,6 +273,7 @@ test('MPEG-TS audio seeking', async () => { const middlePacket = await sink.getPacket(12.5); assert(middlePacket); expect(middlePacket.timestamp).toBeCloseTo(12.486666666666666); + expect(middlePacket.duration).toBe(0.021333333333333333); const allPackets: EncodedPacket[] = []; let currentPacket: EncodedPacket | null = firstPacket; @@ -280,6 +289,7 @@ test('MPEG-TS audio seeking', async () => { const seekedPacket = await sink.getPacket(packet.timestamp); assert(seekedPacket); expect(seekedPacket.timestamp).toBe(packet.timestamp); // The correct timestamp was retrieved for this packet + expect(seekedPacket.duration).toBe(packet.duration); // The correct duration was retrieved for this packet expect(seekedPacket.sequenceNumber).toBe(packet.sequenceNumber); } }); @@ -312,6 +322,7 @@ test('MPEG-TS seeking race condition test', async () => { const seekedPacket = seekedPackets[i]!; assert(seekedPacket); expect(seekedPacket.timestamp).toBe(originalPacket.timestamp); + expect(seekedPacket.duration).toBe(originalPacket.duration); expect(seekedPacket.sequenceNumber).toBe(originalPacket.sequenceNumber); } }); @@ -340,7 +351,7 @@ test('MPEG-TS video key packets', async () => { expect(nextKeyPacket.type).toBe('key'); expect(nextKeyPacket.sequenceNumber).toBeGreaterThan(secondPacket.sequenceNumber); - const firstKeyPacket = await sink.getKeyPacket(firstPacket.timestamp + 1); + const firstKeyPacket = await sink.getKeyPacket(firstPacket.timestamp + 1.0); assert(firstKeyPacket); expect(firstKeyPacket.type).toBe('key'); expect(firstKeyPacket.sequenceNumber).toBe(firstPacket.sequenceNumber); @@ -367,6 +378,7 @@ test('MPEG-TS video key packets', async () => { const keyPacket = await sink.getKeyPacket(packet.timestamp); assert(keyPacket); expect(keyPacket.timestamp).toBe(packet.timestamp); // The correct timestamp was retrieved for this packet + expect(keyPacket.duration).toBe(packet.duration); // The correct duration was retrieved for this packet expect(keyPacket.sequenceNumber).toBe(packet.sequenceNumber); } }); @@ -415,6 +427,7 @@ test('MPEG-TS audio key packets', async () => { const keyPacket = await sink.getKeyPacket(packet.timestamp); assert(keyPacket); expect(keyPacket.timestamp).toBe(packet.timestamp); // The correct timestamp was retrieved for this packet + expect(keyPacket.duration).toBe(packet.duration); // The correct duration was retrieved for this packet expect(keyPacket.sequenceNumber).toBe(packet.sequenceNumber); } }); @@ -444,7 +457,7 @@ test('MPEG-TS with unknown file size (ReadableStreamSource)', async () => { expect(middlePacket.timestamp).toBeCloseTo(12.5); const duration = await videoTrack.computeDuration(); - expect(duration).toBeCloseTo(14.983333333333333); + expect(duration).toBeCloseTo(15); // Ensure that reference points have still been added expect((videoTrack._backing as unknown as MpegTsTrackBacking).referencePesPackets.length)