Add support for writing I-frame only tracks, fix track closing race conditions in other muxers

src/hls/hls-muxer.ts

@@ -118,6 +118,7 @@ export class HlsMuxer extends Muxer {
 
 		let hasVideo = false;
 		let illegalPairingDetected = false;
+		let keyPacketsOnlyPairingWarned = false;
 
 		// First, let's build the "sibling" groups induced by track pairability
 		for (const track of this.output._tracks) {
@@ -148,6 +149,22 @@ export class HlsMuxer extends Muxer {
 					continue;
 				}
 
+				// Key-packets-only tracks can neither pair with nor be paired with other tracks
+				if (
+					(track.isVideoTrack() && track.metadata.hasOnlyKeyPackets)
+					|| (otherTrack.isVideoTrack() && otherTrack.metadata.hasOnlyKeyPackets)
+				) {
+					if (!keyPacketsOnlyPairingWarned) {
+						console.warn(
+							`A key-packets-only video track is pairable with another track, which is not`
+							+ ` possible in HLS; treating them as unpaired.`,
+						);
+						keyPacketsOnlyPairingWarned = true;
+					}
+
+					continue;
+				}
+
 				let groupTracks = pairableGroups.get(otherTrack.source._codec);
 				if (!groupTracks) {
 					pairableGroups.set(otherTrack.source._codec, groupTracks = []);
@@ -462,12 +479,13 @@ export class HlsMuxer extends Muxer {
 
 		try {
 			const trackData = this.trackDatas.find(x => x.track === track);
-			if (!trackData) {
-				return;
+			if (trackData) {
+				trackData.closed = true;
 			}
 
-			trackData.closed = true;
-			await this.advancePlaylist(trackData.playlist);
+			const playlist = this.playlists.find(x => x.tracks.includes(track));
+			assert(playlist); // If there isn't one then the assignment algo failed innit
+			await this.advancePlaylist(playlist);
 		} finally {
 			release();
 		}
@@ -956,15 +974,26 @@ export class HlsMuxer extends Muxer {
 			}
 
 			let targetDuration = this.targetSegmentDuration;
+			let hasByteOffsets = false;
 			for (const segment of playlist.writtenSegments) {
 				targetDuration = Math.max(targetDuration, segment.duration);
+				hasByteOffsets ||= segment.byteOffset !== null;
+			}
+
+			const isKeyPacketsOnly = playlist.tracks[0]!.isVideoTrack()
+				&& playlist.tracks[0].metadata.hasOnlyKeyPackets;
+
+			let version = 3;
+			if (isKeyPacketsOnly || hasByteOffsets) {
+				version = 4;
 			}
 
 			const playlistPath = joinPaths(this.output._rootPath!, playlist.path);
 			const playlistText = '#EXTM3U\n'
-				+ '#EXT-X-VERSION:3\n'
+				+ `#EXT-X-VERSION:${version}\n`
 				+ '#EXT-X-PLAYLIST-TYPE:VOD\n'
 				+ `#EXT-X-TARGETDURATION:${Math.ceil(targetDuration)}\n` // Must be a "decimal-integer"
+				+ (isKeyPacketsOnly ? '#EXT-X-I-FRAMES-ONLY\n' : '')
 				+ '\n'
 				+ (playlist.writtenSegments
 					.map(segment => (
@@ -999,6 +1028,9 @@ export class HlsMuxer extends Muxer {
 
 			for (const decl of this.playlistDeclarations) {
 				if (decl.groupId === null) {
+					const isKeyPacketsOnly = decl.playlist.tracks[0]!.isVideoTrack()
+						&& decl.playlist.tracks[0].metadata.hasOnlyKeyPackets;
+
 					const codecs: string[] = [];
 					for (const track of decl.playlist.tracks) {
 						const trackData = this.trackDatas.find(x => x.track === track);
@@ -1036,7 +1068,12 @@ export class HlsMuxer extends Muxer {
 						firstVariantWritten = true;
 					}
 
-					masterPlaylistText += `#EXT-X-STREAM-INF:`;
+					if (isKeyPacketsOnly) {
+						masterPlaylistText += `#EXT-X-I-FRAME-STREAM-INF:`;
+					} else {
+						masterPlaylistText += `#EXT-X-STREAM-INF:`;
+					}
+
 					masterPlaylistText += `BANDWIDTH=${Math.ceil(totalPeakBitrate)}`;
 
 					if (totalAverageBitrate > 0) {
@@ -1066,7 +1103,8 @@ export class HlsMuxer extends Muxer {
 							}
 						}
 
-						if (videoTrack.metadata.frameRate !== undefined) {
+						// FRAME-RATE is not defined for EXT-X-I-FRAME-STREAM-INF
+						if (!isKeyPacketsOnly && videoTrack.metadata.frameRate !== undefined) {
 							masterPlaylistText += `,FRAME-RATE=${+videoTrack.metadata.frameRate.toFixed(16)}`;
 						}
 					}
@@ -1083,19 +1121,27 @@ export class HlsMuxer extends Muxer {
 						}
 					}
 
-					const groupIdForType = new Map<string, string>();
-					for (const ref of decl.references) {
-						assert(ref.groupId !== null);
-						const type = ref.playlist.tracks[0]!.type;
-						groupIdForType.set(type, ref.groupId);
+					if (!isKeyPacketsOnly) {
+						const groupIdForType = new Map<string, string>();
+						for (const ref of decl.references) {
+							assert(ref.groupId !== null);
+							const type = ref.playlist.tracks[0]!.type;
+							groupIdForType.set(type, ref.groupId);
+						}
+
+						for (const [type, id] of groupIdForType) {
+							masterPlaylistText += `,${type.toUpperCase()}="${id}"`;
+						}
 					}
 
-					for (const [type, id] of groupIdForType) {
-						masterPlaylistText += `,${type.toUpperCase()}="${id}"`;
+					if (isKeyPacketsOnly) {
+						// EXT-X-I-FRAME-STREAM-INF is standalone with a URI attribute
+						masterPlaylistText += `,URI="${decl.playlist.path}"`;
+						masterPlaylistText += '\n';
+					} else {
+						masterPlaylistText += '\n';
+						masterPlaylistText += `${decl.playlist.path}\n`;
 					}
-
-					masterPlaylistText += '\n';
-					masterPlaylistText += `${decl.playlist.path}\n`;
 				} else {
 					assert(decl.playlist.tracks.length === 1);
 

src/isobmff/isobmff-muxer.ts

@@ -78,6 +78,7 @@ export type IsobmffTrackData = {
 		firstChunk: number;
 		samplesPerChunk: number;
 	}[];
+	closed: boolean;
 } & ({
 	track: OutputVideoTrack;
 	type: 'video';
@@ -372,6 +373,7 @@ export class IsobmffMuxer extends Muxer {
 			finalizedChunks: [],
 			currentChunk: null,
 			compactlyCodedChunkTable: [],
+			closed: false,
 		};
 
 		this.trackDatas.push(newTrackData);
@@ -451,6 +453,7 @@ export class IsobmffMuxer extends Muxer {
 			finalizedChunks: [],
 			currentChunk: null,
 			compactlyCodedChunkTable: [],
+			closed: false,
 		};
 
 		this.trackDatas.push(newTrackData);
@@ -492,6 +495,7 @@ export class IsobmffMuxer extends Muxer {
 			finalizedChunks: [],
 			currentChunk: null,
 			compactlyCodedChunkTable: [],
+			closed: false,
 
 			lastCueEndTimestamp: 0,
 			cueQueue: [],
@@ -965,7 +969,7 @@ export class IsobmffMuxer extends Muxer {
 						return firstQueuedSample.type === 'key';
 					}
 
-					return otherTrackData.track.source._closed;
+					return otherTrackData.closed;
 				});
 
 				if (
@@ -1055,7 +1059,7 @@ export class IsobmffMuxer extends Muxer {
 			let minTimestamp = Infinity;
 
 			for (const trackData of this.trackDatas) {
-				if (!isFinalCall && trackData.sampleQueue.length === 0 && !trackData.track.source._closed) {
+				if (!isFinalCall && trackData.sampleQueue.length === 0 && !trackData.closed) {
 					break outer;
 				}
 
@@ -1246,6 +1250,8 @@ export class IsobmffMuxer extends Muxer {
 
 		const trackData = this.trackDatas.find(x => x.track === track);
 		if (trackData) {
+			trackData.closed = true;
+
 			if (trackData.type === 'subtitle' && track.source._codec === 'webvtt') {
 				await this.processWebVTTCues(trackData, Infinity);
 			}
@@ -1272,6 +1278,8 @@ export class IsobmffMuxer extends Muxer {
 		this.allTracksKnown.resolve();
 
 		for (const trackData of this.trackDatas) {
+			trackData.closed = true;
+
 			if (trackData.type === 'subtitle' && trackData.track.source._codec === 'webvtt') {
 				await this.processWebVTTCues(trackData, Infinity);
 			}

src/matroska/matroska-muxer.ts

@@ -87,6 +87,7 @@ type InternalMediaChunk = {
 type MatroskaTrackData = {
 	chunkQueue: InternalMediaChunk[];
 	lastWrittenMsTimestamp: number | null;
+	closed: boolean;
 } & ({
 	track: OutputVideoTrack;
 	type: 'video';
@@ -771,6 +772,7 @@ export class MatroskaMuxer extends Muxer {
 			},
 			chunkQueue: [],
 			lastWrittenMsTimestamp: null,
+			closed: false,
 		};
 
 		if (track.source._codec === 'vp9') {
@@ -857,6 +859,7 @@ export class MatroskaMuxer extends Muxer {
 			},
 			chunkQueue: [],
 			lastWrittenMsTimestamp: null,
+			closed: false,
 		};
 
 		this.trackDatas.push(newTrackData);
@@ -888,6 +891,7 @@ export class MatroskaMuxer extends Muxer {
 			},
 			chunkQueue: [],
 			lastWrittenMsTimestamp: null,
+			closed: false,
 		};
 
 		this.trackDatas.push(newTrackData);
@@ -1009,7 +1013,7 @@ export class MatroskaMuxer extends Muxer {
 			let minTimestamp = Infinity;
 
 			for (const trackData of this.trackDatas) {
-				if (!isFinalCall && trackData.chunkQueue.length === 0 && !trackData.track.source._closed) {
+				if (!isFinalCall && trackData.chunkQueue.length === 0 && !trackData.closed) {
 					break outer;
 				}
 
@@ -1120,7 +1124,7 @@ export class MatroskaMuxer extends Muxer {
 				return firstQueuedSample.type === 'key';
 			}
 
-			return otherTrackData.track.source._closed;
+			return otherTrackData.closed;
 		});
 
 		let shouldCreateNewCluster = false;
@@ -1285,9 +1289,14 @@ export class MatroskaMuxer extends Muxer {
 	}
 
 	// eslint-disable-next-line @typescript-eslint/no-misused-promises
-	override async onTrackClose() {
+	override async onTrackClose(track: OutputTrack) {
 		const release = await this.mutex.acquire();
 
+		const trackData = this.trackDatas.find(x => x.track === track);
+		if (trackData) {
+			trackData.closed = true;
+		}
+
 		if (this.allTracksAreKnown()) {
 			this.allTracksKnown.resolve();
 		}
@@ -1304,6 +1313,10 @@ export class MatroskaMuxer extends Muxer {
 
 		this.allTracksKnown.resolve();
 
+		for (const trackData of this.trackDatas) {
+			trackData.closed = true;
+		}
+
 		if (!this.segment) {
 			this.createSegment();
 		}

src/media-source.ts

@@ -171,6 +171,12 @@ export abstract class VideoSource extends MediaSource {
 	}
 }
 
+const maybeEnsureIsKeyPacket = (track: OutputVideoTrack, packet: EncodedPacket) => {
+	if (track.metadata.hasOnlyKeyPackets && packet.type !== 'key') {
+		throw new Error('Cannot add non-key packets to a hasOnlyKeyPackets video track.');
+	}
+};
+
 /**
  * The most basic video source; can be used to directly pipe encoded packets into the output file.
  * @group Media sources
@@ -204,6 +210,8 @@ export class EncodedVideoPacketSource extends VideoSource {
 		}
 
 		this._ensureValidAdd();
+
+		maybeEnsureIsKeyPacket(this._connectedTrack!, packet);
 		return this._connectedTrack!.output._muxer.addEncodedVideoPacket(this._connectedTrack!, packet, meta);
 	}
 }
@@ -493,6 +501,8 @@ class VideoEncoderWrapper {
 						throw new TypeError('The second argument passed to onPacket must be an object or undefined.');
 					}
 
+					maybeEnsureIsKeyPacket(this.source._connectedTrack!, packet);
+
 					this.encodingConfig.onEncodedPacket?.(packet, meta);
 					void this.muxer!.addEncodedVideoPacket(this.source._connectedTrack!, packet, meta)
 						.catch((error) => {
@@ -582,6 +592,8 @@ class VideoEncoderWrapper {
 						});
 					}
 
+					maybeEnsureIsKeyPacket(this.source._connectedTrack!, packet);
+
 					this.encodingConfig.onEncodedPacket?.(packet, meta);
 					void this.muxer!.addEncodedVideoPacket(this.source._connectedTrack!, packet, meta)
 						.catch((error) => {

src/mpeg-ts/mpeg-ts-muxer.ts

@@ -60,6 +60,7 @@ type MpegTsTrackData = {
 	adtsHeader: Uint8Array | null;
 	adtsHeaderBitstream: Bitstream | null;
 	firstPacketWritten: boolean;
+	closed: boolean;
 };
 
 type QueuedPacket = {
@@ -139,6 +140,7 @@ export class MpegTsMuxer extends Muxer {
 			adtsHeader: null,
 			adtsHeaderBitstream: null,
 			firstPacketWritten: false,
+			closed: false,
 		};
 
 		this.trackDatas.push(newTrackData);
@@ -204,6 +206,7 @@ export class MpegTsMuxer extends Muxer {
 			adtsHeader: null,
 			adtsHeaderBitstream: null,
 			firstPacketWritten: false,
+			closed: false,
 		};
 
 		this.trackDatas.push(newTrackData);
@@ -498,7 +501,7 @@ export class MpegTsMuxer extends Muxer {
 				if (
 					!isFinalCall
 					&& trackData.packetQueue.length === 0
-					&& !trackData.track.source._closed
+					&& !trackData.closed
 				) {
 					break outer;
 				}
@@ -714,15 +717,16 @@ export class MpegTsMuxer extends Muxer {
 	override async onTrackClose(track: OutputTrack) {
 		const release = await this.mutex.acquire();
 
-		if (this.allTracksAreKnown()) {
-			this.allTracksKnown.resolve();
-		}
-
 		const trackData = this.trackDatas.find(x => x.track === track);
 		if (trackData) {
+			trackData.closed = true;
 			await this.flushTimestampQueue(trackData, false);
 		}
 
+		if (this.allTracksAreKnown()) {
+			this.allTracksKnown.resolve();
+		}
+
 		await this.interleavePackets();
 
 		release();
@@ -734,6 +738,7 @@ export class MpegTsMuxer extends Muxer {
 		this.allTracksKnown.resolve();
 
 		for (const trackData of this.trackDatas) {
+			trackData.closed = true;
 			await this.flushTimestampQueue(trackData, false);
 		}
 

src/ogg/ogg-muxer.ts

@@ -17,7 +17,8 @@ import {
 } from '../misc';
 import { Muxer } from '../muxer';
 import { Output,
-	OutputAudioTrack } from '../output';
+	OutputAudioTrack,
+	OutputTrack } from '../output';
 import { OggOutputFormat } from '../output-format';
 import { EncodedPacket } from '../packet';
 import { Writer } from '../writer';
@@ -48,6 +49,7 @@ type OggTrackData = {
 	currentPageSize: number;
 	currentPageStartsWithFreshPacket: boolean;
 	currentPageStartTimestampInSamples: number;
+	closed: boolean;
 };
 
 type Packet = {
@@ -136,6 +138,7 @@ export class OggMuxer extends Muxer {
 			currentPageSize: 27,
 			currentPageStartsWithFreshPacket: true,
 			currentPageStartTimestampInSamples: 0,
+			closed: false,
 		};
 
 		this.queueHeaderPackets(newTrackData, meta);
@@ -335,7 +338,7 @@ export class OggMuxer extends Muxer {
 				if (
 					!isFinalCall
 					&& trackData.packetQueue.length <= 1 // Limit is 1, not 0, for correct EOS flag logic
-					&& !trackData.track.source._closed
+					&& !trackData.closed
 				) {
 					break outer;
 				}
@@ -485,9 +488,14 @@ export class OggMuxer extends Muxer {
 	}
 
 	// eslint-disable-next-line @typescript-eslint/no-misused-promises
-	override async onTrackClose() {
+	override async onTrackClose(track: OutputTrack) {
 		const release = await this.mutex.acquire();
 
+		const trackData = this.trackDatas.find(x => x.track === track);
+		if (trackData) {
+			trackData.closed = true;
+		}
+
 		if (this.allTracksAreKnown()) {
 			this.allTracksKnown.resolve();
 		}
@@ -503,6 +511,10 @@ export class OggMuxer extends Muxer {
 
 		this.allTracksKnown.resolve();
 
+		for (const trackData of this.trackDatas) {
+			trackData.closed = true;
+		}
+
 		await this.interleavePages(true);
 
 		for (const trackData of this.trackDatas) {

src/output.ts

@@ -232,6 +232,11 @@ export type VideoTrackMetadata = BaseTrackMetadata & {
 	 * with the same timestamp.
 	 */
 	frameRate?: number;
+	/**
+	 * When true, this track is marked as being made only out of key frames (I-frames). It is an error to add a non-key
+	 * frame to this track.
+	 */
+	hasOnlyKeyPackets?: boolean;
 };
 /**
  * Additional metadata for audio tracks.

test/node/hls-output.test.ts

@@ -1924,6 +1924,7 @@ test('Single-file mode', async () => {
 
 	expect(playlistText).not.toBeNull();
 	expect(playlistText!.match(/#EXT-X-BYTERANGE/g)).toHaveLength(2);
+	expect(playlistText).toContain('#EXT-X-VERSION:4');
 });
 
 test('StreamTarget, write is called for each target', async () => {
@@ -1968,3 +1969,71 @@ test('StreamTarget, write is called for each target', async () => {
 		expect(count, `Expected writes for ${path}`).toBeGreaterThanOrEqual(1);
 	}
 });
+
+test('I-frame stream', async () => {
+	let masterText = '';
+	let playlistText = '';
+
+	const output = new Output({
+		format: new HlsOutputFormat({
+			segmentFormats: [new MpegTsOutputFormat()],
+			onMaster: (text) => { masterText = text; },
+			onPlaylist: (text) => { playlistText = text; },
+		}),
+		target: () => new BufferTarget(),
+		rootPath: '',
+	});
+
+	const source = videoSource();
+	output.addVideoTrack(source, { hasOnlyKeyPackets: true });
+
+	await output.start();
+
+	const muxer = output._muxer as HlsMuxer;
+	expect(muxer.playlistDeclarations).toHaveLength(1);
+	expect(muxer.playlistDeclarations[0]!.playlist.tracks).toHaveLength(1);
+	expect(muxer.playlistDeclarations[0]!.groupId).toBeNull();
+
+	await source.add(new EncodedPacket(avcPacketData, 'key', 0, 1), avcMetadata);
+	await source.add(new EncodedPacket(avcPacketData, 'key', 1, 1), avcMetadata);
+	await source.add(new EncodedPacket(avcPacketData, 'key', 2, 1), avcMetadata);
+
+	await output.finalize();
+
+	expect(playlistText).toContain('#EXT-X-I-FRAMES-ONLY');
+	expect(playlistText).toContain('#EXT-X-VERSION:4');
+
+	expect(masterText).toContain('#EXT-X-I-FRAME-STREAM-INF:');
+	expect(masterText).toMatch(/#EXT-X-I-FRAME-STREAM-INF:[^\n]*URI="/);
+	expect(masterText).not.toContain('#EXT-X-STREAM-INF:');
+});
+
+test('I-frame stream, pairing warning', async () => {
+	const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
+
+	const output = new Output({
+		format: new HlsOutputFormat({
+			segmentFormats: [new MpegTsOutputFormat()],
+		}),
+		target: () => new NullTarget(),
+		rootPath: '',
+	});
+
+	const group = new OutputTrackGroup();
+	output.addVideoTrack(videoSource(), { hasOnlyKeyPackets: true, group });
+	output.addAudioTrack(audioSource(), { group });
+
+	await output.start();
+
+	const muxer = output._muxer as HlsMuxer;
+	// Despite being pairable, they must end up as separate unpaired declarations
+	expect(muxer.playlistDeclarations).toHaveLength(2);
+	expect(muxer.playlistDeclarations[0]!.playlist.tracks).toHaveLength(1);
+	expect(muxer.playlistDeclarations[1]!.playlist.tracks).toHaveLength(1);
+	expect(muxer.playlistDeclarations[0]!.groupId).toBeNull();
+	expect(muxer.playlistDeclarations[1]!.groupId).toBeNull();
+
+	expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('key-packets-only'));
+
+	warnSpy.mockRestore();
+});