From 94678dd356944e284bc79a5ac14b65f5ea8df038 Mon Sep 17 00:00:00 2001 From: Vanilagy <1696106+Vanilagy@users.noreply.github.com> Date: Fri, 27 Mar 2026 18:48:27 +0100 Subject: [PATCH] Add support for writing I-frame only tracks, fix track closing race conditions in other muxers --- src/hls/hls-muxer.ts | 80 ++++++++++++++++++++++++++-------- src/isobmff/isobmff-muxer.ts | 12 ++++- src/matroska/matroska-muxer.ts | 19 ++++++-- src/media-source.ts | 12 +++++ src/mpeg-ts/mpeg-ts-muxer.ts | 15 ++++--- src/ogg/ogg-muxer.ts | 18 ++++++-- src/output.ts | 5 +++ test/node/hls-output.test.ts | 69 +++++++++++++++++++++++++++++ 8 files changed, 200 insertions(+), 30 deletions(-) diff --git a/src/hls/hls-muxer.ts b/src/hls/hls-muxer.ts index 2c4a68c..58f70d9 100644 --- a/src/hls/hls-muxer.ts +++ b/src/hls/hls-muxer.ts @@ -118,6 +118,7 @@ export class HlsMuxer extends Muxer { let hasVideo = false; let illegalPairingDetected = false; + let keyPacketsOnlyPairingWarned = false; // First, let's build the "sibling" groups induced by track pairability for (const track of this.output._tracks) { @@ -148,6 +149,22 @@ export class HlsMuxer extends Muxer { continue; } + // Key-packets-only tracks can neither pair with nor be paired with other tracks + if ( + (track.isVideoTrack() && track.metadata.hasOnlyKeyPackets) + || (otherTrack.isVideoTrack() && otherTrack.metadata.hasOnlyKeyPackets) + ) { + if (!keyPacketsOnlyPairingWarned) { + console.warn( + `A key-packets-only video track is pairable with another track, which is not` + + ` possible in HLS; treating them as unpaired.`, + ); + keyPacketsOnlyPairingWarned = true; + } + + continue; + } + let groupTracks = pairableGroups.get(otherTrack.source._codec); if (!groupTracks) { pairableGroups.set(otherTrack.source._codec, groupTracks = []); @@ -462,12 +479,13 @@ export class HlsMuxer extends Muxer { try { const trackData = this.trackDatas.find(x => x.track === track); - if (!trackData) { - return; + if (trackData) { + trackData.closed = true; } - trackData.closed = true; - await this.advancePlaylist(trackData.playlist); + const playlist = this.playlists.find(x => x.tracks.includes(track)); + assert(playlist); // If there isn't one then the assignment algo failed innit + await this.advancePlaylist(playlist); } finally { release(); } @@ -956,15 +974,26 @@ export class HlsMuxer extends Muxer { } let targetDuration = this.targetSegmentDuration; + let hasByteOffsets = false; for (const segment of playlist.writtenSegments) { targetDuration = Math.max(targetDuration, segment.duration); + hasByteOffsets ||= segment.byteOffset !== null; + } + + const isKeyPacketsOnly = playlist.tracks[0]!.isVideoTrack() + && playlist.tracks[0].metadata.hasOnlyKeyPackets; + + let version = 3; + if (isKeyPacketsOnly || hasByteOffsets) { + version = 4; } const playlistPath = joinPaths(this.output._rootPath!, playlist.path); const playlistText = '#EXTM3U\n' - + '#EXT-X-VERSION:3\n' + + `#EXT-X-VERSION:${version}\n` + '#EXT-X-PLAYLIST-TYPE:VOD\n' + `#EXT-X-TARGETDURATION:${Math.ceil(targetDuration)}\n` // Must be a "decimal-integer" + + (isKeyPacketsOnly ? '#EXT-X-I-FRAMES-ONLY\n' : '') + '\n' + (playlist.writtenSegments .map(segment => ( @@ -999,6 +1028,9 @@ export class HlsMuxer extends Muxer { for (const decl of this.playlistDeclarations) { if (decl.groupId === null) { + const isKeyPacketsOnly = decl.playlist.tracks[0]!.isVideoTrack() + && decl.playlist.tracks[0].metadata.hasOnlyKeyPackets; + const codecs: string[] = []; for (const track of decl.playlist.tracks) { const trackData = this.trackDatas.find(x => x.track === track); @@ -1036,7 +1068,12 @@ export class HlsMuxer extends Muxer { firstVariantWritten = true; } - masterPlaylistText += `#EXT-X-STREAM-INF:`; + if (isKeyPacketsOnly) { + masterPlaylistText += `#EXT-X-I-FRAME-STREAM-INF:`; + } else { + masterPlaylistText += `#EXT-X-STREAM-INF:`; + } + masterPlaylistText += `BANDWIDTH=${Math.ceil(totalPeakBitrate)}`; if (totalAverageBitrate > 0) { @@ -1066,7 +1103,8 @@ export class HlsMuxer extends Muxer { } } - if (videoTrack.metadata.frameRate !== undefined) { + // FRAME-RATE is not defined for EXT-X-I-FRAME-STREAM-INF + if (!isKeyPacketsOnly && videoTrack.metadata.frameRate !== undefined) { masterPlaylistText += `,FRAME-RATE=${+videoTrack.metadata.frameRate.toFixed(16)}`; } } @@ -1083,19 +1121,27 @@ export class HlsMuxer extends Muxer { } } - const groupIdForType = new Map(); - for (const ref of decl.references) { - assert(ref.groupId !== null); - const type = ref.playlist.tracks[0]!.type; - groupIdForType.set(type, ref.groupId); + if (!isKeyPacketsOnly) { + const groupIdForType = new Map(); + for (const ref of decl.references) { + assert(ref.groupId !== null); + const type = ref.playlist.tracks[0]!.type; + groupIdForType.set(type, ref.groupId); + } + + for (const [type, id] of groupIdForType) { + masterPlaylistText += `,${type.toUpperCase()}="${id}"`; + } } - for (const [type, id] of groupIdForType) { - masterPlaylistText += `,${type.toUpperCase()}="${id}"`; + if (isKeyPacketsOnly) { + // EXT-X-I-FRAME-STREAM-INF is standalone with a URI attribute + masterPlaylistText += `,URI="${decl.playlist.path}"`; + masterPlaylistText += '\n'; + } else { + masterPlaylistText += '\n'; + masterPlaylistText += `${decl.playlist.path}\n`; } - - masterPlaylistText += '\n'; - masterPlaylistText += `${decl.playlist.path}\n`; } else { assert(decl.playlist.tracks.length === 1); diff --git a/src/isobmff/isobmff-muxer.ts b/src/isobmff/isobmff-muxer.ts index b6431ae..760e638 100644 --- a/src/isobmff/isobmff-muxer.ts +++ b/src/isobmff/isobmff-muxer.ts @@ -78,6 +78,7 @@ export type IsobmffTrackData = { firstChunk: number; samplesPerChunk: number; }[]; + closed: boolean; } & ({ track: OutputVideoTrack; type: 'video'; @@ -372,6 +373,7 @@ export class IsobmffMuxer extends Muxer { finalizedChunks: [], currentChunk: null, compactlyCodedChunkTable: [], + closed: false, }; this.trackDatas.push(newTrackData); @@ -451,6 +453,7 @@ export class IsobmffMuxer extends Muxer { finalizedChunks: [], currentChunk: null, compactlyCodedChunkTable: [], + closed: false, }; this.trackDatas.push(newTrackData); @@ -492,6 +495,7 @@ export class IsobmffMuxer extends Muxer { finalizedChunks: [], currentChunk: null, compactlyCodedChunkTable: [], + closed: false, lastCueEndTimestamp: 0, cueQueue: [], @@ -965,7 +969,7 @@ export class IsobmffMuxer extends Muxer { return firstQueuedSample.type === 'key'; } - return otherTrackData.track.source._closed; + return otherTrackData.closed; }); if ( @@ -1055,7 +1059,7 @@ export class IsobmffMuxer extends Muxer { let minTimestamp = Infinity; for (const trackData of this.trackDatas) { - if (!isFinalCall && trackData.sampleQueue.length === 0 && !trackData.track.source._closed) { + if (!isFinalCall && trackData.sampleQueue.length === 0 && !trackData.closed) { break outer; } @@ -1246,6 +1250,8 @@ export class IsobmffMuxer extends Muxer { const trackData = this.trackDatas.find(x => x.track === track); if (trackData) { + trackData.closed = true; + if (trackData.type === 'subtitle' && track.source._codec === 'webvtt') { await this.processWebVTTCues(trackData, Infinity); } @@ -1272,6 +1278,8 @@ export class IsobmffMuxer extends Muxer { this.allTracksKnown.resolve(); for (const trackData of this.trackDatas) { + trackData.closed = true; + if (trackData.type === 'subtitle' && trackData.track.source._codec === 'webvtt') { await this.processWebVTTCues(trackData, Infinity); } diff --git a/src/matroska/matroska-muxer.ts b/src/matroska/matroska-muxer.ts index df9a97f..587aabd 100644 --- a/src/matroska/matroska-muxer.ts +++ b/src/matroska/matroska-muxer.ts @@ -87,6 +87,7 @@ type InternalMediaChunk = { type MatroskaTrackData = { chunkQueue: InternalMediaChunk[]; lastWrittenMsTimestamp: number | null; + closed: boolean; } & ({ track: OutputVideoTrack; type: 'video'; @@ -771,6 +772,7 @@ export class MatroskaMuxer extends Muxer { }, chunkQueue: [], lastWrittenMsTimestamp: null, + closed: false, }; if (track.source._codec === 'vp9') { @@ -857,6 +859,7 @@ export class MatroskaMuxer extends Muxer { }, chunkQueue: [], lastWrittenMsTimestamp: null, + closed: false, }; this.trackDatas.push(newTrackData); @@ -888,6 +891,7 @@ export class MatroskaMuxer extends Muxer { }, chunkQueue: [], lastWrittenMsTimestamp: null, + closed: false, }; this.trackDatas.push(newTrackData); @@ -1009,7 +1013,7 @@ export class MatroskaMuxer extends Muxer { let minTimestamp = Infinity; for (const trackData of this.trackDatas) { - if (!isFinalCall && trackData.chunkQueue.length === 0 && !trackData.track.source._closed) { + if (!isFinalCall && trackData.chunkQueue.length === 0 && !trackData.closed) { break outer; } @@ -1120,7 +1124,7 @@ export class MatroskaMuxer extends Muxer { return firstQueuedSample.type === 'key'; } - return otherTrackData.track.source._closed; + return otherTrackData.closed; }); let shouldCreateNewCluster = false; @@ -1285,9 +1289,14 @@ export class MatroskaMuxer extends Muxer { } // eslint-disable-next-line @typescript-eslint/no-misused-promises - override async onTrackClose() { + override async onTrackClose(track: OutputTrack) { const release = await this.mutex.acquire(); + const trackData = this.trackDatas.find(x => x.track === track); + if (trackData) { + trackData.closed = true; + } + if (this.allTracksAreKnown()) { this.allTracksKnown.resolve(); } @@ -1304,6 +1313,10 @@ export class MatroskaMuxer extends Muxer { this.allTracksKnown.resolve(); + for (const trackData of this.trackDatas) { + trackData.closed = true; + } + if (!this.segment) { this.createSegment(); } diff --git a/src/media-source.ts b/src/media-source.ts index 233b56f..3392765 100644 --- a/src/media-source.ts +++ b/src/media-source.ts @@ -171,6 +171,12 @@ export abstract class VideoSource extends MediaSource { } } +const maybeEnsureIsKeyPacket = (track: OutputVideoTrack, packet: EncodedPacket) => { + if (track.metadata.hasOnlyKeyPackets && packet.type !== 'key') { + throw new Error('Cannot add non-key packets to a hasOnlyKeyPackets video track.'); + } +}; + /** * The most basic video source; can be used to directly pipe encoded packets into the output file. * @group Media sources @@ -204,6 +210,8 @@ export class EncodedVideoPacketSource extends VideoSource { } this._ensureValidAdd(); + + maybeEnsureIsKeyPacket(this._connectedTrack!, packet); return this._connectedTrack!.output._muxer.addEncodedVideoPacket(this._connectedTrack!, packet, meta); } } @@ -493,6 +501,8 @@ class VideoEncoderWrapper { throw new TypeError('The second argument passed to onPacket must be an object or undefined.'); } + maybeEnsureIsKeyPacket(this.source._connectedTrack!, packet); + this.encodingConfig.onEncodedPacket?.(packet, meta); void this.muxer!.addEncodedVideoPacket(this.source._connectedTrack!, packet, meta) .catch((error) => { @@ -582,6 +592,8 @@ class VideoEncoderWrapper { }); } + maybeEnsureIsKeyPacket(this.source._connectedTrack!, packet); + this.encodingConfig.onEncodedPacket?.(packet, meta); void this.muxer!.addEncodedVideoPacket(this.source._connectedTrack!, packet, meta) .catch((error) => { diff --git a/src/mpeg-ts/mpeg-ts-muxer.ts b/src/mpeg-ts/mpeg-ts-muxer.ts index a9fa0d2..3cc7fc9 100644 --- a/src/mpeg-ts/mpeg-ts-muxer.ts +++ b/src/mpeg-ts/mpeg-ts-muxer.ts @@ -60,6 +60,7 @@ type MpegTsTrackData = { adtsHeader: Uint8Array | null; adtsHeaderBitstream: Bitstream | null; firstPacketWritten: boolean; + closed: boolean; }; type QueuedPacket = { @@ -139,6 +140,7 @@ export class MpegTsMuxer extends Muxer { adtsHeader: null, adtsHeaderBitstream: null, firstPacketWritten: false, + closed: false, }; this.trackDatas.push(newTrackData); @@ -204,6 +206,7 @@ export class MpegTsMuxer extends Muxer { adtsHeader: null, adtsHeaderBitstream: null, firstPacketWritten: false, + closed: false, }; this.trackDatas.push(newTrackData); @@ -498,7 +501,7 @@ export class MpegTsMuxer extends Muxer { if ( !isFinalCall && trackData.packetQueue.length === 0 - && !trackData.track.source._closed + && !trackData.closed ) { break outer; } @@ -714,15 +717,16 @@ export class MpegTsMuxer extends Muxer { override async onTrackClose(track: OutputTrack) { const release = await this.mutex.acquire(); - if (this.allTracksAreKnown()) { - this.allTracksKnown.resolve(); - } - const trackData = this.trackDatas.find(x => x.track === track); if (trackData) { + trackData.closed = true; await this.flushTimestampQueue(trackData, false); } + if (this.allTracksAreKnown()) { + this.allTracksKnown.resolve(); + } + await this.interleavePackets(); release(); @@ -734,6 +738,7 @@ export class MpegTsMuxer extends Muxer { this.allTracksKnown.resolve(); for (const trackData of this.trackDatas) { + trackData.closed = true; await this.flushTimestampQueue(trackData, false); } diff --git a/src/ogg/ogg-muxer.ts b/src/ogg/ogg-muxer.ts index 17b6c9a..931f087 100644 --- a/src/ogg/ogg-muxer.ts +++ b/src/ogg/ogg-muxer.ts @@ -17,7 +17,8 @@ import { } from '../misc'; import { Muxer } from '../muxer'; import { Output, - OutputAudioTrack } from '../output'; + OutputAudioTrack, + OutputTrack } from '../output'; import { OggOutputFormat } from '../output-format'; import { EncodedPacket } from '../packet'; import { Writer } from '../writer'; @@ -48,6 +49,7 @@ type OggTrackData = { currentPageSize: number; currentPageStartsWithFreshPacket: boolean; currentPageStartTimestampInSamples: number; + closed: boolean; }; type Packet = { @@ -136,6 +138,7 @@ export class OggMuxer extends Muxer { currentPageSize: 27, currentPageStartsWithFreshPacket: true, currentPageStartTimestampInSamples: 0, + closed: false, }; this.queueHeaderPackets(newTrackData, meta); @@ -335,7 +338,7 @@ export class OggMuxer extends Muxer { if ( !isFinalCall && trackData.packetQueue.length <= 1 // Limit is 1, not 0, for correct EOS flag logic - && !trackData.track.source._closed + && !trackData.closed ) { break outer; } @@ -485,9 +488,14 @@ export class OggMuxer extends Muxer { } // eslint-disable-next-line @typescript-eslint/no-misused-promises - override async onTrackClose() { + override async onTrackClose(track: OutputTrack) { const release = await this.mutex.acquire(); + const trackData = this.trackDatas.find(x => x.track === track); + if (trackData) { + trackData.closed = true; + } + if (this.allTracksAreKnown()) { this.allTracksKnown.resolve(); } @@ -503,6 +511,10 @@ export class OggMuxer extends Muxer { this.allTracksKnown.resolve(); + for (const trackData of this.trackDatas) { + trackData.closed = true; + } + await this.interleavePages(true); for (const trackData of this.trackDatas) { diff --git a/src/output.ts b/src/output.ts index 6b7bb7a..c80d0d0 100644 --- a/src/output.ts +++ b/src/output.ts @@ -232,6 +232,11 @@ export type VideoTrackMetadata = BaseTrackMetadata & { * with the same timestamp. */ frameRate?: number; + /** + * When true, this track is marked as being made only out of key frames (I-frames). It is an error to add a non-key + * frame to this track. + */ + hasOnlyKeyPackets?: boolean; }; /** * Additional metadata for audio tracks. diff --git a/test/node/hls-output.test.ts b/test/node/hls-output.test.ts index 8ed6cda..85e7d80 100644 --- a/test/node/hls-output.test.ts +++ b/test/node/hls-output.test.ts @@ -1924,6 +1924,7 @@ test('Single-file mode', async () => { expect(playlistText).not.toBeNull(); expect(playlistText!.match(/#EXT-X-BYTERANGE/g)).toHaveLength(2); + expect(playlistText).toContain('#EXT-X-VERSION:4'); }); test('StreamTarget, write is called for each target', async () => { @@ -1968,3 +1969,71 @@ test('StreamTarget, write is called for each target', async () => { expect(count, `Expected writes for ${path}`).toBeGreaterThanOrEqual(1); } }); + +test('I-frame stream', async () => { + let masterText = ''; + let playlistText = ''; + + const output = new Output({ + format: new HlsOutputFormat({ + segmentFormats: [new MpegTsOutputFormat()], + onMaster: (text) => { masterText = text; }, + onPlaylist: (text) => { playlistText = text; }, + }), + target: () => new BufferTarget(), + rootPath: '', + }); + + const source = videoSource(); + output.addVideoTrack(source, { hasOnlyKeyPackets: true }); + + await output.start(); + + const muxer = output._muxer as HlsMuxer; + expect(muxer.playlistDeclarations).toHaveLength(1); + expect(muxer.playlistDeclarations[0]!.playlist.tracks).toHaveLength(1); + expect(muxer.playlistDeclarations[0]!.groupId).toBeNull(); + + await source.add(new EncodedPacket(avcPacketData, 'key', 0, 1), avcMetadata); + await source.add(new EncodedPacket(avcPacketData, 'key', 1, 1), avcMetadata); + await source.add(new EncodedPacket(avcPacketData, 'key', 2, 1), avcMetadata); + + await output.finalize(); + + expect(playlistText).toContain('#EXT-X-I-FRAMES-ONLY'); + expect(playlistText).toContain('#EXT-X-VERSION:4'); + + expect(masterText).toContain('#EXT-X-I-FRAME-STREAM-INF:'); + expect(masterText).toMatch(/#EXT-X-I-FRAME-STREAM-INF:[^\n]*URI="/); + expect(masterText).not.toContain('#EXT-X-STREAM-INF:'); +}); + +test('I-frame stream, pairing warning', async () => { + const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}); + + const output = new Output({ + format: new HlsOutputFormat({ + segmentFormats: [new MpegTsOutputFormat()], + }), + target: () => new NullTarget(), + rootPath: '', + }); + + const group = new OutputTrackGroup(); + output.addVideoTrack(videoSource(), { hasOnlyKeyPackets: true, group }); + output.addAudioTrack(audioSource(), { group }); + + await output.start(); + + const muxer = output._muxer as HlsMuxer; + // Despite being pairable, they must end up as separate unpaired declarations + expect(muxer.playlistDeclarations).toHaveLength(2); + expect(muxer.playlistDeclarations[0]!.playlist.tracks).toHaveLength(1); + expect(muxer.playlistDeclarations[1]!.playlist.tracks).toHaveLength(1); + expect(muxer.playlistDeclarations[0]!.groupId).toBeNull(); + expect(muxer.playlistDeclarations[1]!.groupId).toBeNull(); + + expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('key-packets-only')); + + warnSpy.mockRestore(); +});