diff --git a/src/adts/adts-muxer.ts b/src/adts/adts-muxer.ts index d0e49a0..1e33ee7 100644 --- a/src/adts/adts-muxer.ts +++ b/src/adts/adts-muxer.ts @@ -64,7 +64,7 @@ export class AdtsMuxer extends Muxer { // First packet - determine input format from metadata if (this.inputIsAdts === null) { - validateAudioChunkMetadata(meta); + validateAudioChunkMetadata(meta, track.source._codec); const description = meta?.decoderConfig?.description; @@ -119,6 +119,11 @@ export class AdtsMuxer extends Muxer { async finalize() { const release = await this.mutex.acquire(); // Required so that finalize() can't resolve before other calls + + if (this.inputIsAdts === null) { + throw new Error('Cannot finalize an empty ADTS file: not a single packet was added.'); + } + release(); } } diff --git a/src/codec.ts b/src/codec.ts index 97e2bd5..08f1b52 100644 --- a/src/codec.ts +++ b/src/codec.ts @@ -814,7 +814,10 @@ const HEVC_CODEC_STRING_REGEX = /^(hev1|hvc1)\.(?:[ABC]?\d+)\.[0-9a-fA-F]{1,8}\. const VP9_CODEC_STRING_REGEX = /^vp09(?:\.\d{2}){3}(?:(?:\.\d{2}){5})?$/; const AV1_CODEC_STRING_REGEX = /^av01\.\d\.\d{2}[MH]\.\d{2}(?:\.\d\.\d{3}\.\d{2}\.\d{2}\.\d{2}\.\d)?$/; -export const validateVideoChunkMetadata = (metadata: EncodedVideoChunkMetadata | undefined) => { +export const validateVideoChunkMetadata = ( + metadata: EncodedVideoChunkMetadata | undefined, + trackCodec: VideoCodec | null, +) => { if (!metadata) { throw new TypeError('Video chunk metadata must be provided.'); } @@ -985,13 +988,23 @@ export const validateVideoChunkMetadata = (metadata: EncodedVideoChunkMetadata | ); } } + + if (trackCodec !== null && inferCodecFromCodecString(metadata.decoderConfig.codec) !== trackCodec) { + throw new TypeError( + `Video chunk metadata decoder configuration codec string '${metadata.decoderConfig.codec}' does not fit to` + + ` the track codec '${trackCodec}'.`, + ); + } }; const VALID_AUDIO_CODEC_STRING_PREFIXES = [ 'mp4a', 'mp3', 'opus', 'vorbis', 'flac', 'ulaw', 'alaw', 'pcm', 'ac-3', 'ec-3', ]; -export const validateAudioChunkMetadata = (metadata: EncodedAudioChunkMetadata | undefined) => { +export const validateAudioChunkMetadata = ( + metadata: EncodedAudioChunkMetadata | undefined, + trackCodec: AudioCodec | null, +) => { if (!metadata) { throw new TypeError('Audio chunk metadata must be provided.'); } @@ -1133,6 +1146,13 @@ export const validateAudioChunkMetadata = (metadata: EncodedAudioChunkMetadata | ); } } + + if (trackCodec !== null && inferCodecFromCodecString(metadata.decoderConfig.codec) !== trackCodec) { + throw new TypeError( + `Audio chunk metadata decoder configuration codec string '${metadata.decoderConfig.codec}' does not fit to` + + ` the track codec '${trackCodec}'.`, + ); + } }; export const validateSubtitleMetadata = (metadata: SubtitleMetadata | undefined) => { diff --git a/src/conversion.ts b/src/conversion.ts index f2f1ca6..b8690bc 100644 --- a/src/conversion.ts +++ b/src/conversion.ts @@ -1018,9 +1018,9 @@ export class Conversion { // Let's check if the conversion can actually be executed if (!this._composable) { - this.isValid = this.output.hasEnoughTracks(); + this.isValid = this.output.hasEnoughTracks() && this.output.tracks.length > 0; } else { - // Checking Output start validity is not up to us. We consider even zero-track conversions to be valid + // Checking Output start validity is not up to us. We even consider zero-track conversions to be valid this.isValid = true; } diff --git a/src/flac/flac-muxer.ts b/src/flac/flac-muxer.ts index 02e2067..411240c 100644 --- a/src/flac/flac-muxer.ts +++ b/src/flac/flac-muxer.ts @@ -57,9 +57,57 @@ export class FlacMuxer extends Muxer { this.writer = await this.output._getRootWriter(!!this.format._options.appendOnly); this.writer.write(FLAC_HEADER); + // If the track already tells us what the stream looks like, we can pin the stream info down right now + const track = this.output.tracks[0]; + assert(track?.isAudioTrack()); + + if (track.metadata.decoderConfig) { + validateAudioChunkMetadata({ decoderConfig: track.metadata.decoderConfig }, track.source._codec); + this.applyDecoderConfig(track.metadata.decoderConfig); + } + release(); } + applyDecoderConfig(decoderConfig: AudioDecoderConfig) { + assert(decoderConfig.description); + + this.sampleRate = decoderConfig.sampleRate; + this.channels = decoderConfig.numberOfChannels; + + const descriptionBitstream = new Bitstream( + toUint8Array(decoderConfig.description), + ); + // skip 'fLaC' + block size + frame size + sample rate + number of channels + // See demuxer for the exact structure + descriptionBitstream.skipBits(103 + 64); + this.bitsPerSample = descriptionBitstream.readBits(5) + 1; + + if (this.format._options.appendOnly) { + // Write STREAMINFO immediately since we can't seek back later. + this.writeHeader({ + // https://www.rfc-editor.org/rfc/rfc9639.html#name-streaminfo + // Per RFC 9639, min/max block sizes can be looser than + // actual values, so we use the full valid range (16–65535). + // "The actual max block size MAY be smaller than what's + // listed, and the actual min (excluding last block) MAY be + // larger. This is because the encoder has to write these + // fields before receiving any input audio data and cannot + // know beforehand what block sizes it will use." + minimumBlockSize: 16, + maximumBlockSize: 65535, + // https://www.rfc-editor.org/rfc/rfc9639.html#name-streaminfo + // "A value of 0 signifies that the value is not known." + minimumFrameSize: 0, + maximumFrameSize: 0, + sampleRate: this.sampleRate, + channels: this.channels, + bitsPerSample: this.bitsPerSample, + totalSamples: 0, + }); + } + } + writeHeader({ bitsPerSample, minimumBlockSize, @@ -222,47 +270,12 @@ export class FlacMuxer extends Muxer { if (this.sampleRate === null) { // It's the first packet - validateAudioChunkMetadata(meta); + validateAudioChunkMetadata(meta, track.source._codec); assert(meta); assert(meta.decoderConfig); - assert(meta.decoderConfig.description); - this.sampleRate = meta.decoderConfig.sampleRate; - this.channels = meta.decoderConfig.numberOfChannels; - - const descriptionBitstream = new Bitstream( - toUint8Array(meta.decoderConfig.description), - ); - // skip 'fLaC' + block size + frame size + sample rate + number of channels - // See demuxer for the exact structure - descriptionBitstream.skipBits(103 + 64); - const bitsPerSample = descriptionBitstream.readBits(5) + 1; - this.bitsPerSample = bitsPerSample; - - if (this.format._options.appendOnly) { - // Write STREAMINFO immediately since we can't seek back later. - this.writeHeader({ - // https://www.rfc-editor.org/rfc/rfc9639.html#name-streaminfo - // Per RFC 9639, min/max block sizes can be looser than - // actual values, so we use the full valid range (16–65535). - // "The actual max block size MAY be smaller than what's - // listed, and the actual min (excluding last block) MAY be - // larger. This is because the encoder has to write these - // fields before receiving any input audio data and cannot - // know beforehand what block sizes it will use." - minimumBlockSize: 16, - maximumBlockSize: 65535, - // https://www.rfc-editor.org/rfc/rfc9639.html#name-streaminfo - // "A value of 0 signifies that the value is not known." - minimumFrameSize: 0, - maximumFrameSize: 0, - sampleRate: this.sampleRate, - channels: this.channels, - bitsPerSample: this.bitsPerSample, - totalSamples: 0, - }); - } + this.applyDecoderConfig(meta.decoderConfig); } if (!this.metadataWritten) { @@ -306,6 +319,18 @@ export class FlacMuxer extends Muxer { async finalize(): Promise { const release = await this.mutex.acquire(); + if (this.sampleRate === null) { + throw new Error( + 'Cannot finalize an empty FLAC file: no packets were added and the track specified no decoderConfig in' + + ' its metadata, so there\'s no telling what the file should look like.', + ); + } + + if (!this.metadataWritten) { + // Not a single packet came in, so this never happened yet + this.writeVorbisCommentAndPictureBlock(); + } + if (!this.format._options.appendOnly) { let minimumBlockSize = Infinity; let maximumBlockSize = 0; @@ -328,7 +353,15 @@ export class FlacMuxer extends Muxer { minimumBlockSize = Math.min(minimumBlockSize, this.blockSizes[i]!); } - assert(this.sampleRate !== null); + if (this.blockSizes.length === 0) { + // There are no frames to derive these from, so let's use the full valid range like we do for + // append-only output + minimumBlockSize = 16; + maximumBlockSize = 65535; + minimumFrameSize = 0; + maximumFrameSize = 0; + } + assert(this.channels !== null); assert(this.bitsPerSample !== null); diff --git a/src/hls/hls-muxer.ts b/src/hls/hls-muxer.ts index d58a315..f001794 100644 --- a/src/hls/hls-muxer.ts +++ b/src/hls/hls-muxer.ts @@ -51,9 +51,11 @@ type HlsTrackData = { info: { type: 'video'; decoderConfig: VideoDecoderConfig; + primingPacket: EncodedPacket | null; } | { type: 'audio'; decoderConfig: AudioDecoderConfig; + primingPacket: EncodedPacket | null; }; }; type HlsVideoTrackData = HlsTrackData & { info: { type: 'video' } }; @@ -524,6 +526,22 @@ export class HlsMuxer extends Muxer { }); } + for (const track of this.output.tracks) { + if (track.isVideoTrack() && track.metadata.decoderConfig) { + this.getVideoTrackData( + track, + track.metadata.primingPacket ?? null, + { decoderConfig: track.metadata.decoderConfig }, + ); + } else if (track.isAudioTrack() && track.metadata.decoderConfig) { + this.getAudioTrackData( + track, + track.metadata.primingPacket ?? null, + { decoderConfig: track.metadata.decoderConfig }, + ); + } + } + release(); } @@ -560,13 +578,13 @@ export class HlsMuxer extends Muxer { } } - getVideoTrackData(track: OutputVideoTrack, meta?: EncodedVideoChunkMetadata) { + getVideoTrackData(track: OutputVideoTrack, packet: EncodedPacket | null, meta?: EncodedVideoChunkMetadata) { let trackData = this.trackDatas.find(x => x.track === track) as HlsVideoTrackData; if (trackData) { return trackData; } - validateVideoChunkMetadata(meta); + validateVideoChunkMetadata(meta, track.source._codec); assert(meta); assert(meta?.decoderConfig); @@ -582,6 +600,7 @@ export class HlsMuxer extends Muxer { info: { type: 'video', decoderConfig: meta.decoderConfig, + primingPacket: packet, }, }; this.trackDatas.push(trackData); @@ -589,13 +608,13 @@ export class HlsMuxer extends Muxer { return trackData; } - getAudioTrackData(track: OutputAudioTrack, meta?: EncodedAudioChunkMetadata) { + getAudioTrackData(track: OutputAudioTrack, packet: EncodedPacket | null, meta?: EncodedAudioChunkMetadata) { let trackData = this.trackDatas.find(x => x.track === track) as HlsAudioTrackData; if (trackData) { return trackData; } - validateAudioChunkMetadata(meta); + validateAudioChunkMetadata(meta, track.source._codec); assert(meta); assert(meta?.decoderConfig); @@ -611,6 +630,7 @@ export class HlsMuxer extends Muxer { info: { type: 'audio', decoderConfig: meta.decoderConfig, + primingPacket: packet, }, }; this.trackDatas.push(trackData); @@ -623,7 +643,7 @@ export class HlsMuxer extends Muxer { packet: EncodedPacket, meta?: EncodedVideoChunkMetadata, ) { - const trackData = this.getVideoTrackData(track, meta); + const trackData = this.getVideoTrackData(track, packet, meta); const playlist = trackData.playlist; const release = await playlist.mutex.acquire(); @@ -652,7 +672,7 @@ export class HlsMuxer extends Muxer { packet: EncodedPacket, meta?: EncodedAudioChunkMetadata, ) { - const trackData = this.getAudioTrackData(track, meta); + const trackData = this.getAudioTrackData(track, packet, meta); const playlist = trackData.playlist; const release = await playlist.mutex.acquire(); @@ -694,14 +714,18 @@ export class HlsMuxer extends Muxer { return; } + const trackDatas = this.trackDatas.filter(x => playlist.tracks.includes(x.track)); + if (playlist.currentSegmentStartTimestamp === null) { - // All tracks are known but we never received any data - all tracks must be closed already - await this.onPlaylistDone(playlist); + // All tracks are known but we never received any data. Tracks that declared themselves up front are known + // before their first packet, so we can only call it a day once they're actually closed. + if (trackDatas.every(x => x.closed)) { + await this.onPlaylistDone(playlist); + } return; } - const trackDatas = this.trackDatas.filter(x => playlist.tracks.includes(x.track)); const videoTrack = trackDatas.find(x => x.info.type === 'video') as HlsVideoTrackData | undefined; const audioTrack = trackDatas.find(x => x.info.type === 'audio') as HlsAudioTrackData | undefined; @@ -917,7 +941,11 @@ export class HlsMuxer extends Muxer { ); fragmentedIsobmffOutput.output.addVideoTrack( fragmentedIsobmffOutput.videoSource, - videoTrack.track.metadata, + { + ...videoTrack.track.metadata, + decoderConfig: videoTrack.info.decoderConfig, + primingPacket: videoTrack.info.primingPacket ?? undefined, + }, ); } @@ -928,7 +956,11 @@ export class HlsMuxer extends Muxer { ); fragmentedIsobmffOutput.output.addAudioTrack( fragmentedIsobmffOutput.audioSource, - audioTrack.track.metadata, + { + ...audioTrack.track.metadata, + decoderConfig: audioTrack.info.decoderConfig, + primingPacket: audioTrack.info.primingPacket ?? undefined, + }, ); } @@ -1074,7 +1106,11 @@ export class HlsMuxer extends Muxer { videoSource = new EncodedVideoPacketSource( (videoTrack.track as OutputVideoTrack).source._codec, ); - output.addVideoTrack(videoSource, videoTrack.track.metadata); + output.addVideoTrack(videoSource, { + ...videoTrack.track.metadata, + decoderConfig: videoTrack.info.decoderConfig, + primingPacket: videoTrack.info.primingPacket ?? undefined, + }); } if (audioTrack) { @@ -1082,7 +1118,11 @@ export class HlsMuxer extends Muxer { audioSource = new EncodedAudioPacketSource( (audioTrack.track as OutputAudioTrack).source._codec, ); - output.addAudioTrack(audioSource, audioTrack.track.metadata); + output.addAudioTrack(audioSource, { + ...audioTrack.track.metadata, + decoderConfig: audioTrack.info.decoderConfig, + primingPacket: audioTrack.info.primingPacket ?? undefined, + }); } await output.start(); diff --git a/src/input.ts b/src/input.ts index 211478c..a58fd76 100644 --- a/src/input.ts +++ b/src/input.ts @@ -336,8 +336,11 @@ export class Input extends EventEmitter return 0; } - const firstTimestamps = await Promise.all(filtered.map(x => x.getFirstTimestamp())); - return Math.min(...firstTimestamps); + // Only count the timestamps of tracks that have at least one packet + const firstPackets = await Promise.all(filtered.map(x => x._backing.getFirstPacket({ metadataOnly: true }))); + const result = Math.min(...firstPackets.map(x => x?.timestamp ?? Infinity)); + + return result === Infinity ? 0 : result; } /** diff --git a/src/isobmff/isobmff-boxes.ts b/src/isobmff/isobmff-boxes.ts index 57fea8d..3867fc4 100644 --- a/src/isobmff/isobmff-boxes.ts +++ b/src/isobmff/isobmff-boxes.ts @@ -1048,7 +1048,9 @@ const pcmC = (trackData: IsobmffAudioTrackData) => { /** AC3SpecificBox */ const dac3 = (trackData: IsobmffAudioTrackData) => { - const frameInfo = parseAc3SyncFrame(trackData.info.firstPacket.data); + assert(trackData.info.primingPacket); + + const frameInfo = parseAc3SyncFrame(trackData.info.primingPacket.data); if (!frameInfo) { throw new Error( 'Couldn\'t extract AC-3 frame info from the audio packet. ' @@ -1072,7 +1074,9 @@ const dac3 = (trackData: IsobmffAudioTrackData) => { /** EC3SpecificBox */ const dec3 = (trackData: IsobmffAudioTrackData) => { - const frameInfo = parseEac3SyncFrame(trackData.info.firstPacket.data); + assert(trackData.info.primingPacket); + + const frameInfo = parseEac3SyncFrame(trackData.info.primingPacket.data); if (!frameInfo) { throw new Error( 'Couldn\'t extract E-AC-3 frame info from the audio packet. ' @@ -1437,7 +1441,7 @@ export const mfra = (trackDatas: IsobmffTrackData[]) => { }; /** Track Fragment Random Access Box: Provides pointers to sync samples within the file for random access. */ -export const tfra = (trackData: IsobmffTrackData, trackIndex: number) => { +export const tfra = (trackData: IsobmffTrackData) => { const version = 1; // Using this version allows us to use 64-bit time and offset values return fullBox('tfra', version, 0, [ @@ -1447,7 +1451,7 @@ export const tfra = (trackData: IsobmffTrackData, trackIndex: number) => { trackData.finalizedChunks.map(chunk => [ u64(intoTimescale(chunk.samples[0]!.timestamp, trackData.timescale)), // Time (in presentation time) u64(chunk.moofOffset!), // moof offset - u32(trackIndex + 1), // traf number + u32(chunk.trafIndex! + 1), // traf number u32(1), // trun number u32(1), // Sample number ]), diff --git a/src/isobmff/isobmff-demuxer.ts b/src/isobmff/isobmff-demuxer.ts index 2d96f9d..98ab435 100644 --- a/src/isobmff/isobmff-demuxer.ts +++ b/src/isobmff/isobmff-demuxer.ts @@ -352,6 +352,7 @@ export class IsobmffDemuxer extends Demuxer { return this.metadataPromise ??= (async () => { let currentPos = 0; let lookForMfraBox = false; + let foundMovieBoxes = false; while (true) { let slice = this.reader.requestSliceRange(currentPos, MIN_BOX_HEADER_SIZE, MAX_BOX_HEADER_SIZE); @@ -388,6 +389,7 @@ export class IsobmffDemuxer extends Demuxer { lookForMfraBox = this.isFragmented && this.reader.fileSize !== null && this.reader.fileSize > startPos + boxInfo.totalSize; // There's more after the moov box + foundMovieBoxes = true; break; } else if (boxInfo.name === 'moof') { @@ -399,65 +401,10 @@ export class IsobmffDemuxer extends Demuxer { ); } - const initDemuxer = (await this.input._initInput._getDemuxer()) as IsobmffDemuxer; - if (initDemuxer.constructor !== IsobmffDemuxer) { - throw new Error('Init input must match the input\'s format.'); - } - - await initDemuxer.readMetadata(); - - this.movieTimescale = initDemuxer.movieTimescale; - this.movieDurationInTimescale = initDemuxer.movieDurationInTimescale; - this.metadataTags = initDemuxer.metadataTags; - this.isFragmented = true; - this.fragmentTrackDefaults = initDemuxer.fragmentTrackDefaults; - this.psshBoxes = initDemuxer.psshBoxes; - - // Create tracks from the init input's tracks - for (const foreignTrack of initDemuxer.tracks) { - const track: InternalTrack = { - id: foreignTrack.id, - demuxer: this, - trackBacking: null, - disposition: foreignTrack.disposition, - timescale: foreignTrack.timescale, - durationInMediaTimescale: foreignTrack.durationInMediaTimescale, - durationInMovieTimescale: foreignTrack.durationInMovieTimescale, - rotation: foreignTrack.rotation, - internalCodecId: foreignTrack.internalCodecId, - name: foreignTrack.name, - languageCode: foreignTrack.languageCode, - sampleTableByteOffset: null, - sampleTable: null, - fragmentLookupTable: [], - currentFragmentState: null, - fragmentPositionCache: [], - editListPreviousSegmentDurations: foreignTrack.editListPreviousSegmentDurations, - editListOffset: foreignTrack.editListOffset, - encryptionInfo: foreignTrack.encryptionInfo, - encryptionAuxInfo: null, - frmaCodecString: null, - info: foreignTrack.info, - }; - - if (foreignTrack.trackBacking) { - assert(track.info); - - if (track.info.type === 'video' && track.info.width !== -1) { - const videoTrack = track as InternalVideoTrack; - track.trackBacking = new IsobmffVideoTrackBacking(videoTrack); - this.tracks.push(track); - } else if (track.info.type === 'audio' && track.info.numberOfChannels !== -1) { - const audioTrack = track as InternalAudioTrack; - track.trackBacking = new IsobmffAudioTrackBacking(audioTrack); - this.tracks.push(track); - } - } else { - // The track didn't have enough info to warrant a backing - } - } + await this.copyMetadataFromInitInput(this.input._initInput); lookForMfraBox = false; // No point in doing it for segment files + foundMovieBoxes = true; break; } @@ -465,6 +412,12 @@ export class IsobmffDemuxer extends Demuxer { currentPos = startPos + boxInfo.totalSize; } + if (!foundMovieBoxes && this.input._initInput) { + // A segment file is allowed to hold zero fragments, in which case there's no moof box to key off of. + // It's still a perfectly valid segment, so let's take the tracks from the init input. + await this.copyMetadataFromInitInput(this.input._initInput); + } + if (lookForMfraBox) { assert(this.reader.fileSize !== null); @@ -502,6 +455,66 @@ export class IsobmffDemuxer extends Demuxer { })(); } + private async copyMetadataFromInitInput(initInput: Input) { + const initDemuxer = (await initInput._getDemuxer()) as IsobmffDemuxer; + if (initDemuxer.constructor !== IsobmffDemuxer) { + throw new Error('Init input must match the input\'s format.'); + } + + await initDemuxer.readMetadata(); + + this.movieTimescale = initDemuxer.movieTimescale; + this.movieDurationInTimescale = initDemuxer.movieDurationInTimescale; + this.metadataTags = initDemuxer.metadataTags; + this.isFragmented = true; + this.fragmentTrackDefaults = initDemuxer.fragmentTrackDefaults; + this.psshBoxes = initDemuxer.psshBoxes; + + // Create tracks from the init input's tracks + for (const foreignTrack of initDemuxer.tracks) { + const track: InternalTrack = { + id: foreignTrack.id, + demuxer: this, + trackBacking: null, + disposition: foreignTrack.disposition, + timescale: foreignTrack.timescale, + durationInMediaTimescale: foreignTrack.durationInMediaTimescale, + durationInMovieTimescale: foreignTrack.durationInMovieTimescale, + rotation: foreignTrack.rotation, + internalCodecId: foreignTrack.internalCodecId, + name: foreignTrack.name, + languageCode: foreignTrack.languageCode, + sampleTableByteOffset: null, + sampleTable: null, + fragmentLookupTable: [], + currentFragmentState: null, + fragmentPositionCache: [], + editListPreviousSegmentDurations: foreignTrack.editListPreviousSegmentDurations, + editListOffset: foreignTrack.editListOffset, + encryptionInfo: foreignTrack.encryptionInfo, + encryptionAuxInfo: null, + frmaCodecString: null, + info: foreignTrack.info, + }; + + if (foreignTrack.trackBacking) { + assert(track.info); + + if (track.info.type === 'video' && track.info.width !== -1) { + const videoTrack = track as InternalVideoTrack; + track.trackBacking = new IsobmffVideoTrackBacking(videoTrack); + this.tracks.push(track); + } else if (track.info.type === 'audio' && track.info.numberOfChannels !== -1) { + const audioTrack = track as InternalAudioTrack; + track.trackBacking = new IsobmffAudioTrackBacking(audioTrack); + this.tracks.push(track); + } + } else { + // The track didn't have enough info to warrant a backing + } + } + } + getSampleTableForTrack(internalTrack: InternalTrack) { if (internalTrack.sampleTable) { return internalTrack.sampleTable; diff --git a/src/isobmff/isobmff-muxer.ts b/src/isobmff/isobmff-muxer.ts index dc033e3..06b5f15 100644 --- a/src/isobmff/isobmff-muxer.ts +++ b/src/isobmff/isobmff-muxer.ts @@ -72,6 +72,9 @@ type Chunk = { offset: number | null; // In the case of a fragmented file, this indicates the position of the moof box pointing to the data in this chunk moofOffset: number | null; + // In the case of a fragmented file, this indicates the index of the traf box in the moof box pointing to the data + // in this chunk + trafIndex: number | null; }; export type IsobmffTrackData = { @@ -128,7 +131,7 @@ export type IsobmffTrackData = { * ADTS-wrapped data. */ requiresAdtsStripping: boolean; - firstPacket: EncodedPacket; + primingPacket: EncodedPacket | null; }; } | { track: OutputSubtitleTrack; @@ -312,6 +315,22 @@ export class IsobmffMuxer extends Muxer { await this.writer?.flush(); + for (const track of this.output.tracks) { + if (track.isVideoTrack() && track.metadata.decoderConfig) { + this.getVideoTrackData( + track, + track.metadata.primingPacket ?? null, + { decoderConfig: track.metadata.decoderConfig }, + ); + } else if (track.isAudioTrack() && track.metadata.decoderConfig) { + this.getAudioTrackData( + track, + track.metadata.primingPacket ?? null, + { decoderConfig: track.metadata.decoderConfig }, + ); + } + } + release(); } @@ -349,13 +368,13 @@ export class IsobmffMuxer extends Muxer { }); } - private getVideoTrackData(track: OutputVideoTrack, packet: EncodedPacket, meta?: EncodedVideoChunkMetadata) { + private getVideoTrackData(track: OutputVideoTrack, packet: EncodedPacket | null, meta?: EncodedVideoChunkMetadata) { const existingTrackData = this.trackDatas.find(x => x.track === track); if (existingTrackData) { return existingTrackData as IsobmffVideoTrackData; } - validateVideoChunkMetadata(meta); + validateVideoChunkMetadata(meta, track.source._codec); assert(meta); assert(meta.decoderConfig); @@ -370,6 +389,10 @@ export class IsobmffMuxer extends Muxer { // ISOBMFF can only hold AVC in the AVCC format, not in Annex B, but the missing description indicates // Annex B. This means we'll need to do some converterino. + if (!packet) { + throw new Error('No AVC description provided; you must therefore provide a priming packet.'); + } + const decoderConfigurationRecord = extractAvcDecoderConfigurationRecord(packet.data); if (!decoderConfigurationRecord) { throw new Error( @@ -386,6 +409,10 @@ export class IsobmffMuxer extends Muxer { // ISOBMFF can only hold HEVC in the HEVC format, not in Annex B, but the missing description indicates // Annex B. This means we'll need to do some converterino. + if (!packet) { + throw new Error('No HEVC description provided; you must therefore provide a priming packet.'); + } + const decoderConfigurationRecord = extractHevcDecoderConfigurationRecord(packet.data); if (!decoderConfigurationRecord) { throw new Error( @@ -457,13 +484,13 @@ export class IsobmffMuxer extends Muxer { return newTrackData; } - private getAudioTrackData(track: OutputAudioTrack, packet: EncodedPacket, meta?: EncodedAudioChunkMetadata) { + private getAudioTrackData(track: OutputAudioTrack, packet: EncodedPacket | null, meta?: EncodedAudioChunkMetadata) { const existingTrackData = this.trackDatas.find(x => x.track === track); if (existingTrackData) { return existingTrackData as IsobmffAudioTrackData; } - validateAudioChunkMetadata(meta); + validateAudioChunkMetadata(meta, track.source._codec); assert(meta); assert(meta.decoderConfig); @@ -474,6 +501,11 @@ export class IsobmffMuxer extends Muxer { if (track.source._codec === 'aac' && !decoderConfig.description) { // ISOBMFF can only hold AAC in raw format, not ADTS, but the missing description indicates ADTS. // Parse the first packet to extract the AudioSpecificConfig. + + if (!packet) { + throw new Error('No AAC description provided; you must therefore provide a priming packet.'); + } + const adtsFrame = readAdtsFrameHeader(FileSlice.tempFromBytes(packet.data)); if (!adtsFrame) { throw new Error( @@ -499,6 +531,12 @@ export class IsobmffMuxer extends Muxer { requiresAdtsStripping = true; } + if (track.source._codec === 'ac3' || track.source._codec === 'eac3') { + if (!packet) { + throw new Error('AC-3/E-AC-3 require a priming packet.'); + } + } + const newTrackData: IsobmffAudioTrackData = { muxer: this, track, @@ -512,7 +550,7 @@ export class IsobmffMuxer extends Muxer { && (PCM_AUDIO_CODECS as readonly string[]).includes(track.source._codec), expectedNextPcmPacketTimestamp: null, requiresAdtsStripping, - firstPacket: packet, + primingPacket: packet, }, timescale: decoderConfig.sampleRate, samples: [], @@ -1094,6 +1132,7 @@ export class IsobmffMuxer extends Muxer { samples: [], offset: null, moofOffset: null, + trafIndex: null, }; } @@ -1247,9 +1286,12 @@ export class IsobmffMuxer extends Muxer { let currentPos = mdatStartPos + MIN_BOX_HEADER_SIZE; let fragmentStartTimestamp = Infinity; - for (const trackData of tracksInFragment) { + for (let i = 0; i < tracksInFragment.length; i++) { + const trackData = tracksInFragment[i]!; + trackData.currentChunk!.offset = currentPos; trackData.currentChunk!.moofOffset = moofOffset; + trackData.currentChunk!.trafIndex = i; for (const sample of trackData.currentChunk!.samples) { currentPos += sample.size; @@ -1318,38 +1360,9 @@ export class IsobmffMuxer extends Muxer { } private async registerSampleFastStartReserve(trackData: IsobmffTrackData, sample: Sample) { - assert(this.writer); - assert(this.boxWriter); - if (this.allTracksAreKnown()) { if (!this.mdat) { - this.ensureOneEnabledTrack(); - - // We finally know all tracks, let's reserve space for the moov box - const moovBox = moov(this); - const moovSize = this.boxWriter.measureBox(moovBox); - - const reservedSize = moovSize - + this.computeSampleTableSizeUpperBound() - + 4096; // Just a little extra headroom - - assert(this.ftypSize !== null); - this.writer.seek(this.ftypSize + reservedSize); - - if (this.formatOptions.onMdat) { - this.writer.startTrackingWrites(); - } - - this.mdat = mdat(true); - this.boxWriter.writeBox(this.mdat); - - // Now write everything that was queued - for (const trackData of this.trackDatas) { - for (const sample of trackData.sampleQueue) { - await this.addSampleToTrack(trackData, sample); - } - trackData.sampleQueue.length = 0; - } + await this.createFastStartReserveMdat(); } await this.addSampleToTrack(trackData, sample); @@ -1359,6 +1372,39 @@ export class IsobmffMuxer extends Muxer { } } + private async createFastStartReserveMdat() { + assert(this.writer); + assert(this.boxWriter); + + this.ensureOneEnabledTrack(); + + // We finally know all tracks, let's reserve space for the moov box + const moovBox = moov(this); + const moovSize = this.boxWriter.measureBox(moovBox); + + const reservedSize = moovSize + + this.computeSampleTableSizeUpperBound() + + 4096; // Just a little extra headroom + + assert(this.ftypSize !== null); + this.writer.seek(this.ftypSize + reservedSize); + + if (this.formatOptions.onMdat) { + this.writer.startTrackingWrites(); + } + + this.mdat = mdat(true); + this.boxWriter.writeBox(this.mdat); + + // Now write everything that was queued + for (const trackData of this.trackDatas) { + for (const sample of trackData.sampleQueue) { + await this.addSampleToTrack(trackData, sample); + } + trackData.sampleQueue.length = 0; + } + } + private computeSampleTableSizeUpperBound() { assert(this.fastStart === 'reserve'); @@ -1466,6 +1512,10 @@ export class IsobmffMuxer extends Muxer { this.allTracksKnown.resolve(); this.ensureOneEnabledTrack(); + if (!this.mdat && this.fastStart === 'reserve') { + await this.createFastStartReserveMdat(); + } + for (const trackData of this.trackDatas) { trackData.closed = true; @@ -1483,15 +1533,14 @@ export class IsobmffMuxer extends Muxer { for (const trackData of this.trackDatas) { await this.finalizeCurrentChunk(trackData); - // Must hold because we will have processed at least one sample - assert(trackData.startTimestampOffset !== null); - - // Shift all of the samples by the start offset. We'll then write out an edit list that will shift them - // back to their proper spot in the composition. - for (let i = 0; i < trackData.samples.length; i++) { - const sample = trackData.samples[i]!; - sample.timestamp -= trackData.startTimestampOffset; - sample.decodeTimestamp -= trackData.startTimestampOffset; + if (trackData.startTimestampOffset !== null) { + // Shift all of the samples by the start offset. We'll then write out an edit list that will shift + // them back to their proper spot in the composition. + for (let i = 0; i < trackData.samples.length; i++) { + const sample = trackData.samples[i]!; + sample.timestamp -= trackData.startTimestampOffset; + sample.decodeTimestamp -= trackData.startTimestampOffset; + } } } } diff --git a/src/matroska/matroska-muxer.ts b/src/matroska/matroska-muxer.ts index 468a356..41b0525 100644 --- a/src/matroska/matroska-muxer.ts +++ b/src/matroska/matroska-muxer.ts @@ -97,7 +97,8 @@ type MatroskaTrackData = { height: number; aspectRatio: Rational | null; decoderConfig: VideoDecoderConfig; - alphaMode: boolean; + /** Null until the first packet comes in, which is what determines if this track has alpha or not. */ + alphaMode: boolean | null; }; } | { track: OutputAudioTrack; @@ -176,6 +177,22 @@ export class MatroskaMuxer extends Muxer { await this.writer.flush(); + for (const track of this.output.tracks) { + if (track.isVideoTrack() && track.metadata.decoderConfig) { + this.getVideoTrackData( + track, + track.metadata.primingPacket ?? null, + { decoderConfig: track.metadata.decoderConfig }, + ); + } else if (track.isAudioTrack() && track.metadata.decoderConfig) { + this.getAudioTrackData( + track, + track.metadata.primingPacket ?? null, + { decoderConfig: track.metadata.decoderConfig }, + ); + } + } + release(); } @@ -724,13 +741,13 @@ export class MatroskaMuxer extends Muxer { }); } - private getVideoTrackData(track: OutputVideoTrack, packet: EncodedPacket, meta?: EncodedVideoChunkMetadata) { + private getVideoTrackData(track: OutputVideoTrack, packet: EncodedPacket | null, meta?: EncodedVideoChunkMetadata) { const existingTrackData = this.trackDatas.find(x => x.track === track); if (existingTrackData) { return existingTrackData as MatroskaVideoTrackData; } - validateVideoChunkMetadata(meta); + validateVideoChunkMetadata(meta, track.source._codec); assert(meta); assert(meta.decoderConfig); @@ -755,7 +772,7 @@ export class MatroskaMuxer extends Muxer { height: meta.decoderConfig.codedHeight, aspectRatio, decoderConfig: meta.decoderConfig, - alphaMode: !!packet.sideData.alpha, // The first packet determines if this track has alpha or not + alphaMode: packet ? !!packet.sideData.alpha : null, }, chunkQueue: [], lastWrittenMsTimestamp: null, @@ -791,13 +808,13 @@ export class MatroskaMuxer extends Muxer { return newTrackData; } - private getAudioTrackData(track: OutputAudioTrack, packet: EncodedPacket, meta?: EncodedAudioChunkMetadata) { + private getAudioTrackData(track: OutputAudioTrack, packet: EncodedPacket | null, meta?: EncodedAudioChunkMetadata) { const existingTrackData = this.trackDatas.find(x => x.track === track); if (existingTrackData) { return existingTrackData as MatroskaAudioTrackData; } - validateAudioChunkMetadata(meta); + validateAudioChunkMetadata(meta, track.source._codec); assert(meta); assert(meta.decoderConfig); @@ -808,6 +825,11 @@ export class MatroskaMuxer extends Muxer { if (track.source._codec === 'aac' && !decoderConfig.description) { // Matroska stores raw AAC with AudioSpecificConfig in CodecPrivate, not ADTS-wrapped data. // Parse the first packet to extract the AudioSpecificConfig. + + if (!packet) { + throw new Error('No AAC description provided; you must therefore provide a priming packet.'); + } + const adtsFrame = readAdtsFrameHeader(FileSlice.tempFromBytes(packet.data)); if (!adtsFrame) { throw new Error( @@ -896,6 +918,7 @@ export class MatroskaMuxer extends Muxer { try { const trackData = this.getVideoTrackData(track, packet, meta); + trackData.info.alphaMode ??= !!packet.sideData.alpha; let packetData = packet.data; if (track.source._codec === 'prores') { diff --git a/src/mp3/mp3-demuxer.ts b/src/mp3/mp3-demuxer.ts index ebe5426..48b3dd0 100644 --- a/src/mp3/mp3-demuxer.ts +++ b/src/mp3/mp3-demuxer.ts @@ -53,6 +53,8 @@ export class Mp3Demuxer extends Demuxer { metadataPromise: Promise | null = null; firstFrameHeader: Mp3FrameHeader | null = null; firstFrameHeaderPos: number | null = null; + xingFrameHeader: Mp3FrameHeader | null = null; + xingFrameHeaderPos: number | null = null; loadedSamples: Sample[] = []; // All samples from the start of the file to lastLoadedPos metadataTags: MetadataTags | null = null; xingData: { @@ -80,6 +82,13 @@ export class Mp3Demuxer extends Demuxer { await this.advanceReader(); } + if (!this.firstFrameHeader && this.xingFrameHeader) { + // The file consists of nothing but a Xing frame, so it holds no audio data - but that frame still + // tells us everything about the track + this.firstFrameHeader = this.xingFrameHeader; + this.firstFrameHeaderPos = this.xingFrameHeaderPos; + } + if (!this.firstFrameHeader) { throw new Error('No valid MP3 frame found.'); } @@ -135,6 +144,11 @@ export class Mp3Demuxer extends Demuxer { if (isXing) { // There's no actual audio data in this frame, so let's skip it + if (!this.xingFrameHeader) { + this.xingFrameHeader = header; + this.xingFrameHeaderPos = result.startPos; + } + if (!this.xingData) { let xingDataSlice = this.reader.requestSlice(result.startPos + xingOffset + 4, 12); if (xingDataSlice instanceof Promise) xingDataSlice = await xingDataSlice; diff --git a/src/mp3/mp3-muxer.ts b/src/mp3/mp3-muxer.ts index f1a1413..1eea0c4 100644 --- a/src/mp3/mp3-muxer.ts +++ b/src/mp3/mp3-muxer.ts @@ -6,14 +6,14 @@ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -import { toDataView } from '../misc'; +import { assert, toDataView } from '../misc'; import { metadataTagsAreEmpty } from '../metadata'; import { Muxer } from '../muxer'; import { Output, OutputAudioTrack } from '../output'; import { Mp3OutputFormat } from '../output-format'; import { EncodedPacket } from '../packet'; import { Writer } from '../writer'; -import { getXingOffset, INFO, readMp3FrameHeader, XING } from '../../shared/mp3-misc'; +import { getXingOffset, INFO, readMp3FrameHeader, SAMPLING_RATES, XING } from '../../shared/mp3-misc'; import { Mp3Writer, XingFrameData } from './mp3-writer'; import { Id3V2Writer } from '../id3'; @@ -130,28 +130,122 @@ export class Mp3Muxer extends Muxer { } async finalize() { - if (!this.xingFrameData || this.xingFramePos === null) { - return; + const release = await this.mutex.acquire(); + + if (!this.xingFrameData && this.format._options.xingHeader === false) { + // MP3 has no container-level header, so the Xing frame is the only thing we could have synthesized + throw new Error( + 'Cannot finalize an empty MP3 file: not a single packet was added and the Xing header is disabled, so' + + ' there\'s no frame we could write.', + ); } - const release = await this.mutex.acquire(); + if (!this.xingFrameData) { + // Not a single packet came in, so let's write a lone Xing frame; that way, the file is still a valid + // (if empty) MP3. We derive its header from whatever the track told us up front. + const track = this.output.tracks[0]; + assert(track?.isAudioTrack()); + + const primingPacket = track.metadata.primingPacket; + if (primingPacket) { + // The best case: an actual frame tells us exactly what the header should look like + const view = toDataView(primingPacket.data); + if (view.byteLength < 4) { + throw new Error('Invalid MP3 header in priming packet.'); + } + + const word = view.getUint32(0, false); + const header = readMp3FrameHeader(word, null).header; + if (!header) { + throw new Error('Invalid MP3 header in priming packet.'); + } + + this.xingFrameData = { + mpegVersionId: header.mpegVersionId, + layer: header.layer, + frequencyIndex: header.frequencyIndex, + sampleRate: header.sampleRate, + channel: header.channel, + modeExtension: header.modeExtension, + copyright: header.copyright, + original: header.original, + emphasis: header.emphasis, + + frameCount: null, + fileSize: null, + toc: null, + }; + } else if (track.metadata.decoderConfig) { + // All we know is the sample rate and channel count, so let's derive the rest + const { sampleRate, numberOfChannels } = track.metadata.decoderConfig; + + // MPEG Version 1 uses the sampling rates directly, Version 2 halves them, and Version 2.5 quarters them + const mpegVersionIds = [3, 2, 0]; + let mpegVersionId: number | null = null; + let frequencyIndex = -1; + + for (let i = 0; i < mpegVersionIds.length; i++) { + frequencyIndex = SAMPLING_RATES.indexOf(sampleRate << i); + if (frequencyIndex !== -1) { + mpegVersionId = mpegVersionIds[i]!; + break; + } + } + + if (mpegVersionId === null) { + throw new Error(`${sampleRate} Hz is not a valid MP3 sample rate.`); + } + + this.xingFrameData = { + mpegVersionId, + layer: 1, // Layer III + frequencyIndex, + sampleRate, + channel: numberOfChannels === 1 ? 3 : 0, // 3 = single channel, 0 = stereo + modeExtension: 0, + copyright: 0, + original: 0, + emphasis: 0, + + frameCount: null, + fileSize: null, + toc: null, + }; + } else { + throw new Error( + 'Cannot finalize an empty MP3 file: no packets were added and the track specified neither a' + + ' decoderConfig nor a primingPacket in its metadata, so there\'s no telling what the file' + + ' should look like.', + ); + } + + this.xingFramePos = this.writer.getPos(); + this.mp3Writer.writeXingFrame(this.xingFrameData); + + this.frameCount++; + } + + assert(this.xingFramePos !== null); const endPos = this.writer.getPos(); const audioDataEndPos = endPos - this.xingFramePos; this.writer.seek(this.xingFramePos); - const toc = new Uint8Array(100); - for (let i = 0; i < 100; i++) { - const index = Math.floor(this.framePositions.length * (i / 100)); + if (this.framePositions.length > 0) { + const toc = new Uint8Array(100); + for (let i = 0; i < 100; i++) { + const index = Math.floor(this.framePositions.length * (i / 100)); - const byteOffset = this.framePositions[index]! - this.xingFramePos; - toc[i] = 256 * (byteOffset / audioDataEndPos); + const byteOffset = this.framePositions[index]! - this.xingFramePos; + toc[i] = 256 * (byteOffset / audioDataEndPos); + } + + this.xingFrameData.toc = toc; } this.xingFrameData.frameCount = this.frameCount; this.xingFrameData.fileSize = audioDataEndPos; - this.xingFrameData.toc = toc; if (this.format._options.onXingFrame) { this.writer.startTrackingWrites(); diff --git a/src/mp3/mp3-writer.ts b/src/mp3/mp3-writer.ts index 3ebd833..bdc9eed 100644 --- a/src/mp3/mp3-writer.ts +++ b/src/mp3/mp3-writer.ts @@ -116,6 +116,8 @@ export class Mp3Writer { const frameSize = computeMp3FrameSize( lowSamplingFrequency, data.layer, 1000 * kilobitRate, data.sampleRate, padding, ); - this.writer.seek(startPos + frameSize); + + // Pad the frame out to its full size + this.writer.write(new Uint8Array(startPos + frameSize - this.writer.getPos())); } } diff --git a/src/mpeg-ts/mpeg-ts-demuxer.ts b/src/mpeg-ts/mpeg-ts-demuxer.ts index f00e2a8..f82a09c 100644 --- a/src/mpeg-ts/mpeg-ts-demuxer.ts +++ b/src/mpeg-ts/mpeg-ts-demuxer.ts @@ -713,6 +713,11 @@ export class MpegTsDemuxer extends Demuxer { } for (const stream of this.elementaryStreams) { + if (!stream.initialized) { + // Stream was declared but no initialization data for it was found + continue; + } + if (stream.info.type === 'video') { this.trackBackingEntries.push( new MpegTsVideoTrackBacking(stream as ElementaryVideoStream), diff --git a/src/mpeg-ts/mpeg-ts-muxer.ts b/src/mpeg-ts/mpeg-ts-muxer.ts index 5f4aa11..d48f99e 100644 --- a/src/mpeg-ts/mpeg-ts-muxer.ts +++ b/src/mpeg-ts/mpeg-ts-muxer.ts @@ -112,7 +112,7 @@ export class MpegTsMuxer extends Muxer { return existingTrackData; } - validateVideoChunkMetadata(meta); + validateVideoChunkMetadata(meta, track.source._codec); assert(meta?.decoderConfig); const codec = track.source._codec; @@ -157,7 +157,7 @@ export class MpegTsMuxer extends Muxer { return existingTrackData; } - validateAudioChunkMetadata(meta); + validateAudioChunkMetadata(meta, track.source._codec); assert(meta?.decoderConfig); const codec = track.source._codec; diff --git a/src/ogg/ogg-muxer.ts b/src/ogg/ogg-muxer.ts index c7243dd..e8f8215 100644 --- a/src/ogg/ogg-muxer.ts +++ b/src/ogg/ogg-muxer.ts @@ -81,6 +81,13 @@ export class OggMuxer extends Muxer { this.writer = await this.output._getRootWriter(true); // Ogg is always monotonically written! + for (const track of this.output.tracks) { + assert(track.isAudioTrack()); + if (track.metadata.decoderConfig) { + this.getTrackData(track, { decoderConfig: track.metadata.decoderConfig }); + } + } + release(); } @@ -110,7 +117,7 @@ export class OggMuxer extends Muxer { assert(track.source._codec === 'vorbis' || track.source._codec === 'opus'); - validateAudioChunkMetadata(meta); + validateAudioChunkMetadata(meta, track.source._codec); assert(meta); assert(meta.decoderConfig); diff --git a/src/output-format.ts b/src/output-format.ts index 0c5707c..9098282 100644 --- a/src/output-format.ts +++ b/src/output-format.ts @@ -266,7 +266,7 @@ export abstract class IsobmffOutputFormat extends OutputFormat { video: { min: 0, max }, audio: { min: 0, max }, subtitle: { min: 0, max }, - total: { min: 1, max }, + total: { min: 0, max }, }; } @@ -550,7 +550,7 @@ export class MkvOutputFormat extends OutputFormat { video: { min: 0, max }, audio: { min: 0, max }, subtitle: { min: 0, max }, - total: { min: 1, max }, + total: { min: 0, max }, }; } @@ -891,7 +891,7 @@ export class OggOutputFormat extends OutputFormat { video: { min: 0, max: 0 }, audio: { min: 0, max }, subtitle: { min: 0, max: 0 }, - total: { min: 1, max }, + total: { min: 0, max }, }; } @@ -1138,7 +1138,7 @@ export class MpegTsOutputFormat extends OutputFormat { video: { min: 0, max: maxVideo }, audio: { min: 0, max: maxAudio }, subtitle: { min: 0, max: 0 }, - total: { min: 1, max: maxTotal }, + total: { min: 0, max: maxTotal }, }; } @@ -1415,7 +1415,7 @@ export class HlsOutputFormat extends OutputFormat { video: { min: 0, max: supportsVideo ? Infinity : 0 }, audio: { min: 0, max: supportsAudio ? Infinity : 0 }, subtitle: { min: 0, max: 0 }, // Currently disabled - total: { min: 1, max: Infinity }, + total: { min: 0, max: Infinity }, }; } diff --git a/src/output.ts b/src/output.ts index 44f1a49..2c8329b 100644 --- a/src/output.ts +++ b/src/output.ts @@ -14,6 +14,8 @@ import { AudioSource, MediaSource, SubtitleSource, VideoSource } from './media-s import { PathedTarget, Target, TargetRequest } from './target'; import { Writer } from './writer'; import { Logging } from './logging'; +import { EncodedPacket } from './packet'; +import { validateAudioChunkMetadata, validateVideoChunkMetadata } from './codec'; /** * List of all track types. @@ -260,13 +262,36 @@ export type VideoTrackMetadata = BaseTrackMetadata & { * frame to this track. */ hasOnlyKeyPackets?: boolean; + /** + * The decoder config for this video track, provided ahead of time. This is provided automatically when media data + * added to the track, but by specifying it here, you give the muxer additional information that it can make use of. + * Zero-packet tracks become possible to write when this field is set. + */ + decoderConfig?: VideoDecoderConfig; + /** + * Can be provided in addition to {@link VideoTrackMetadata.decoderConfig} to provide additional track information + * not included in the decoder config. This packet will not be added to the media data. + */ + primingPacket?: EncodedPacket; }; /** * Additional metadata for audio tracks. * @group Output files * @public */ -export type AudioTrackMetadata = BaseTrackMetadata & {}; +export type AudioTrackMetadata = BaseTrackMetadata & { + /** + * The decoder config for this audio track, provided ahead of time. This is provided automatically when media data + * added to the track, but by specifying it here, you give the muxer additional information that it can make use of. + * Zero-packet tracks become possible to write when this field is set. + */ + decoderConfig?: AudioDecoderConfig; + /** + * Can be provided in addition to {@link AudioTrackMetadata.decoderConfig} to provide additional track information + * not included in the decoder config. This packet will not be added to the media data. + */ + primingPacket?: EncodedPacket; +}; /** * Additional metadata for subtitle tracks. * @group Output files @@ -617,6 +642,17 @@ export class Output< `Invalid video frame rate: ${metadata.frameRate}. Must be a positive number.`, ); } + if (metadata.decoderConfig !== undefined) { + validateVideoChunkMetadata({ decoderConfig: metadata.decoderConfig }, source._codec); + } + if (metadata.primingPacket !== undefined) { + if (!(metadata.primingPacket instanceof EncodedPacket)) { + throw new TypeError('metadata.primingPacket, when provided, must be an EncodedPacket.'); + } + if (metadata.decoderConfig === undefined) { + throw new TypeError('metadata.primingPacket can only be provided alongside metadata.decoderConfig.'); + } + } const metadataCopy = { ...metadata }; metadataCopy.group ??= this.defaultTrackGroup; @@ -632,6 +668,17 @@ export class Output< throw new TypeError('source must be an AudioSource.'); } validateBaseTrackMetadata(metadata); + if (metadata.decoderConfig !== undefined) { + validateAudioChunkMetadata({ decoderConfig: metadata.decoderConfig }, source._codec); + } + if (metadata.primingPacket !== undefined) { + if (!(metadata.primingPacket instanceof EncodedPacket)) { + throw new TypeError('metadata.primingPacket, when provided, must be an EncodedPacket.'); + } + if (metadata.decoderConfig === undefined) { + throw new TypeError('metadata.primingPacket can only be provided alongside metadata.decoderConfig.'); + } + } const metadataCopy = { ...metadata }; metadataCopy.group ??= this.defaultTrackGroup; diff --git a/src/segmented-input.ts b/src/segmented-input.ts index 5f104df..49ed79a 100644 --- a/src/segmented-input.ts +++ b/src/segmented-input.ts @@ -240,6 +240,7 @@ class SegmentedInputInputTrackBacking implements InputTrackBacking { hydrationPromise: Promise | null = null; firstInputTrack: InputTrack | null = null; + firstSegment: Segment | null = null; constructor(segmentedInput: SegmentedInput, decl: SegmentedInputTrackDeclaration, number: number) { this.segmentedInput = segmentedInput; @@ -254,15 +255,29 @@ class SegmentedInputInputTrackBacking implements InputTrackBacking { throw new Error('Missing first segment, can\'t retrieve track.'); } - const input = this.segmentedInput.getInputForSegment(this.segmentedInput.firstSegment); - const inputTracks = await input.getTracks(); + let currentSegment: Segment | null = this.segmentedInput.firstSegment; + let track: InputTrack | null = null; + + // For playlists with sparse tracks (rare af!!), not every segment has every track, so we need to loop to + // find the first segment that actually contains the track we want. + while (currentSegment) { + const input = this.segmentedInput.getInputForSegment(currentSegment); + const inputTracks = await input.getTracks(); + track = inputTracks.find(x => x.type === this.decl.type && x.number === this.number) ?? null; + + if (track) { + break; + } + + currentSegment = await this.segmentedInput.getNextSegment(currentSegment, {}); + } - const track = inputTracks.find(x => x.type === this.decl.type && x.number === this.number); if (!track) { throw new Error('No matching track found in underlying media data.'); } this.firstInputTrack = track; + this.firstSegment = currentSegment; })(); } @@ -380,15 +395,36 @@ class SegmentedInputInputTrackBacking implements InputTrackBacking { async getFirstPacket(options: PacketRetrievalOptions): Promise { await this.hydrate(); - assert(this.segmentedInput.firstSegment); assert(this.firstInputTrack); + assert(this.firstSegment); - const packet = await this.firstInputTrack._backing.getFirstPacket(options); - if (!packet) { - return null; + let currentTrack: InputTrack | null = this.firstInputTrack; + let currentSegment: Segment | null = this.firstSegment; + + // Loop until we found a segment with a packet (segments may contain zero packets in rare cases) + while (true) { + if (currentTrack) { + const packet = await currentTrack._backing.getFirstPacket(options); + if (packet) { + return this.createAdjustedPacket(packet, currentSegment, currentTrack); + } + } + + currentSegment = await this.segmentedInput.getNextSegment(currentSegment, { + skipLiveWait: options.skipLiveWait, + }); + if (!currentSegment) { + break; + } + + const nextInput = this.segmentedInput.getInputForSegment(currentSegment); + const nextTracks = await nextInput.getTracks(); + currentTrack = nextTracks.find(t => ( + t.type === this.firstInputTrack!.type && t.number === this.firstInputTrack!.number + )) ?? null; } - return this.createAdjustedPacket(packet, this.segmentedInput.firstSegment, this.firstInputTrack); + return null; } getNextPacket(packet: EncodedPacket, options: PacketRetrievalOptions): Promise { diff --git a/src/wave/wave-muxer.ts b/src/wave/wave-muxer.ts index f094646..df5da26 100644 --- a/src/wave/wave-muxer.ts +++ b/src/wave/wave-muxer.ts @@ -48,7 +48,18 @@ export class WaveMuxer extends Muxer { this.writer = await this.output._getRootWriter(false); this.riffWriter = new RiffWriter(this.writer); - // No writing needed here - we'll write the header with the first sample + // If the track already tells us everything we need, we can write the header right now. Otherwise, we'll write + // it with the first sample. + const track = this.output.tracks[0]; + assert(track?.isAudioTrack()); + + if (track.metadata.decoderConfig) { + validateAudioChunkMetadata({ decoderConfig: track.metadata.decoderConfig }, track.source._codec); + + this.writeHeader(track, track.metadata.decoderConfig); + this.sampleRate = track.metadata.decoderConfig.sampleRate; + this.headerWritten = true; + } release(); } @@ -70,7 +81,7 @@ export class WaveMuxer extends Muxer { try { if (!this.headerWritten) { - validateAudioChunkMetadata(meta); + validateAudioChunkMetadata(meta, track.source._codec); assert(meta); assert(meta.decoderConfig); @@ -341,6 +352,13 @@ export class WaveMuxer extends Muxer { async finalize() { const release = await this.mutex.acquire(); + if (!this.headerWritten) { + throw new Error( + 'Cannot finalize an empty WAVE file: no packets were added and the track specified no decoderConfig in' + + ' its metadata, so there\'s no telling what the file should look like.', + ); + } + const endPos = this.writer.getPos(); if (this.isRf64) { diff --git a/test/browser/mpeg-ts-muxing.test.ts b/test/browser/mpeg-ts-muxing.test.ts index 9e0d2c7..ac841f7 100644 --- a/test/browser/mpeg-ts-muxing.test.ts +++ b/test/browser/mpeg-ts-muxing.test.ts @@ -21,7 +21,7 @@ test('MPEG-TS output format', async () => { video: { min: 0, max: 16 }, audio: { min: 0, max: 32 }, subtitle: { min: 0, max: 0 }, - total: { min: 1, max: 48 }, + total: { min: 0, max: 48 }, }); }); diff --git a/test/node/empty-media.test.ts b/test/node/empty-media.test.ts new file mode 100644 index 0000000..eb90f2c --- /dev/null +++ b/test/node/empty-media.test.ts @@ -0,0 +1,711 @@ +import { expect, test } from 'vitest'; +import { Input } from '../../src/input.js'; +import { ALL_FORMATS } from '../../src/input-format.js'; +import { EncodedAudioPacketSource, EncodedVideoPacketSource } from '../../src/media-source.js'; +import { AudioTrackMetadata, Output } from '../../src/output.js'; +import { + AdtsOutputFormat, + CmafOutputFormat, + FlacOutputFormat, + HlsOutputFormat, + MkvOutputFormat, + Mp3OutputFormat, + Mp4OutputFormat, + MpegTsOutputFormat, + OggOutputFormat, + WavOutputFormat, +} from '../../src/output-format.js'; +import { EncodedPacket } from '../../src/packet.js'; +import { Bitstream } from '../../shared/bitstream.js'; +import { BufferSource } from '../../src/source.js'; +import { BufferTarget, PathedTarget } from '../../src/target.js'; +import { InputAudioTrack, InputVideoTrack } from '../../src/input-track.js'; + +type EmptyMediaVariant = + | { type: 'mp4'; fastStart: false | 'in-memory' | 'reserve' | 'fragmented' } + | { type: 'cmaf' } + | { type: 'matroska' }; + +test('No tracks, MP4, fastStart: false', async () => { + await testNoTracks({ type: 'mp4', fastStart: false }); +}); + +test('No tracks, MP4, fastStart: in-memory', async () => { + await testNoTracks({ type: 'mp4', fastStart: 'in-memory' }); +}); + +test('No tracks, MP4, fastStart: reserve', async () => { + await testNoTracks({ type: 'mp4', fastStart: 'reserve' }); +}); + +test('No tracks, MP4, fastStart: fragmented', async () => { + await testNoTracks({ type: 'mp4', fastStart: 'fragmented' }); +}); + +test('No tracks, CMAF', async () => { + await testNoTracks({ type: 'cmaf' }); +}); + +test('No tracks, Matroska', async () => { + await testNoTracks({ type: 'matroska' }); +}); + +// These formats are inherently multi-track, so holding zero tracks is perfectly fine +const testNoTracks = async (variant: EmptyMediaVariant) => { + const initTarget = new BufferTarget(); + + const output = new Output({ + format: createFormat(variant), + target: new BufferTarget(), + initTarget: variant.type === 'cmaf' ? initTarget : undefined, + }); + + await output.start(); + await output.finalize(); + + using initInput = variant.type === 'cmaf' + ? new Input({ source: new BufferSource(initTarget.buffer!), formats: ALL_FORMATS }) + : undefined; + + using input = new Input({ + source: new BufferSource(output.target.buffer!), + formats: ALL_FORMATS, + initInput, + }); + + expect(await input.getTracks()).toHaveLength(0); +}; + +test('No tracks, MPEG-TS', async () => { + const output = new Output({ + format: new MpegTsOutputFormat(), + target: new BufferTarget(), + }); + + await output.start(); + await output.finalize(); + + using input = new Input({ + source: new BufferSource(output.target.buffer!), + formats: ALL_FORMATS, + }); + + expect(await input.getTracks()).toHaveLength(0); +}); + +test('No tracks, Ogg', async () => { + const output = new Output({ + format: new OggOutputFormat(), + target: new BufferTarget(), + }); + + await output.start(); + await output.finalize(); + + // An Ogg file is nothing but its logical bitstreams, so zero tracks means zero bytes + expect(output.target.buffer!.byteLength).toBe(0); +}); + +test('No tracks, HLS', async () => { + const files = new Map(); + + const output = new Output({ + format: new HlsOutputFormat({ segmentFormat: new MpegTsOutputFormat() }), + target: new PathedTarget('', (request) => { + const target = new BufferTarget(); + target.on('finalized', () => files.set(request.path, new Uint8Array(target.buffer!))); + return target; + }), + }); + + await output.start(); + await output.finalize(); + + // Without tracks there are no playlists either, so all we get is the master playlist (the root target) + expect([...files.keys()]).toEqual(['']); +}); + +test('No tracks, MP3', async () => { + const output = new Output({ + format: new Mp3OutputFormat(), + target: new BufferTarget(), + }); + + await expect(output.start()).rejects.toThrow('MP3 requires exactly 1 audio track'); +}); + +test('No tracks, WAVE', async () => { + const output = new Output({ + format: new WavOutputFormat(), + target: new BufferTarget(), + }); + + await expect(output.start()).rejects.toThrow('WAVE requires exactly 1 audio track'); +}); + +test('No tracks, ADTS', async () => { + const output = new Output({ + format: new AdtsOutputFormat(), + target: new BufferTarget(), + }); + + await expect(output.start()).rejects.toThrow('ADTS requires exactly 1 audio track'); +}); + +test('No tracks, FLAC', async () => { + const output = new Output({ + format: new FlacOutputFormat(), + target: new BufferTarget(), + }); + + await expect(output.start()).rejects.toThrow('FLAC requires exactly 1 audio track'); +}); + +test('Empty MP4, fastStart: false', async () => { + await testEmptyMedia({ type: 'mp4', fastStart: false }); +}); + +test('Empty MP4, fastStart: in-memory', async () => { + await testEmptyMedia({ type: 'mp4', fastStart: 'in-memory' }); +}); + +test('Empty MP4, fastStart: reserve', async () => { + await testEmptyMedia({ type: 'mp4', fastStart: 'reserve' }); +}); + +test('Empty MP4, fastStart: fragmented', async () => { + await testEmptyMedia({ type: 'mp4', fastStart: 'fragmented' }); +}); + +test('Empty CMAF', async () => { + await testEmptyMedia({ type: 'cmaf' }); +}); + +test('Empty Matroska', async () => { + await testEmptyMedia({ type: 'matroska' }); +}); + +const testEmptyMedia = async (variant: EmptyMediaVariant) => { + const initTarget = new BufferTarget(); + + const output = new Output({ + format: createFormat(variant), + target: new BufferTarget(), + initTarget: variant.type === 'cmaf' ? initTarget : undefined, + }); + + output.addVideoTrack(new EncodedVideoPacketSource('avc'), { maximumPacketCount: 100 }); + output.addAudioTrack(new EncodedAudioPacketSource('opus'), { maximumPacketCount: 100 }); + + await output.start(); + await output.finalize(); + + using initInput = variant.type === 'cmaf' + ? new Input({ source: new BufferSource(initTarget.buffer!), formats: ALL_FORMATS }) + : undefined; + + using input = new Input({ + source: new BufferSource(output.target.buffer!), + formats: ALL_FORMATS, + initInput, + }); + + expect(await input.getTracks()).toHaveLength(0); +}; + +test('Empty MP4 with declared decoder config, fastStart: false', async () => { + await testEmptyMediaWithDecoderConfig({ type: 'mp4', fastStart: false }); +}); + +test('Empty MP4 with declared decoder config, fastStart: in-memory', async () => { + await testEmptyMediaWithDecoderConfig({ type: 'mp4', fastStart: 'in-memory' }); +}); + +test('Empty MP4 with declared decoder config, fastStart: reserve', async () => { + await testEmptyMediaWithDecoderConfig({ type: 'mp4', fastStart: 'reserve' }); +}); + +test('Empty MP4 with declared decoder config, fastStart: fragmented', async () => { + await testEmptyMediaWithDecoderConfig({ type: 'mp4', fastStart: 'fragmented' }); +}); + +test('Empty CMAF with declared decoder config', async () => { + await testEmptyMediaWithDecoderConfig({ type: 'cmaf' }); +}); + +test('Empty Matroska with declared decoder config', async () => { + await testEmptyMediaWithDecoderConfig({ type: 'matroska' }); +}); + +// VP9 and Opus need no description, so the decoder config alone is enough to fully define the tracks +const testEmptyMediaWithDecoderConfig = async (variant: EmptyMediaVariant) => { + const initTarget = new BufferTarget(); + + const output = new Output({ + format: createFormat(variant), + target: new BufferTarget(), + initTarget: variant.type === 'cmaf' ? initTarget : undefined, + }); + + output.addVideoTrack(new EncodedVideoPacketSource('vp9'), { + maximumPacketCount: 100, + decoderConfig: { + codec: 'vp09.00.10.08', + codedWidth: 1280, + codedHeight: 720, + }, + }); + output.addAudioTrack(new EncodedAudioPacketSource('opus'), { + maximumPacketCount: 100, + decoderConfig: { + codec: 'opus', + sampleRate: 48000, + numberOfChannels: 2, + }, + }); + + await output.start(); + await output.finalize(); + + using initInput = variant.type === 'cmaf' + ? new Input({ source: new BufferSource(initTarget.buffer!), formats: ALL_FORMATS }) + : undefined; + + using input = new Input({ + source: new BufferSource(output.target.buffer!), + formats: ALL_FORMATS, + initInput, + }); + + const tracks = await input.getTracks(); + expect(tracks).toHaveLength(2); + + const videoTrack = tracks[0] as InputVideoTrack; + expect(videoTrack.isVideoTrack()).toBe(true); + expect(await videoTrack.getCodec()).toBe('vp9'); + expect(await videoTrack.getCodedWidth()).toBe(1280); + expect(await videoTrack.getCodedHeight()).toBe(720); + expect((await videoTrack.computePacketStats()).packetCount).toBe(0); + + const audioTrack = tracks[1] as InputAudioTrack; + expect(audioTrack.isAudioTrack()).toBe(true); + expect(await audioTrack.getCodec()).toBe('opus'); + expect(await audioTrack.getSampleRate()).toBe(48000); + expect(await audioTrack.getNumberOfChannels()).toBe(2); + expect((await audioTrack.computePacketStats()).packetCount).toBe(0); +}; + +test('Empty Ogg', async () => { + const output = new Output({ + format: new OggOutputFormat(), + target: new BufferTarget(), + }); + + // Ogg is audio-only, so let's go for two audio tracks here + output.addAudioTrack(new EncodedAudioPacketSource('opus'), { maximumPacketCount: 100 }); + output.addAudioTrack(new EncodedAudioPacketSource('opus'), { maximumPacketCount: 100 }); + + await output.start(); + await output.finalize(); + + // Ogg has no container-level header, so without any packets, nothing at all gets written + expect(output.target.buffer!.byteLength).toBe(0); +}); + +test('Empty Ogg with declared decoder config', async () => { + const output = new Output({ + format: new OggOutputFormat(), + target: new BufferTarget(), + }); + + // An OpusHead packet as specified in RFC 7845 + const description = new Uint8Array(19); + const view = new DataView(description.buffer); + description.set([0x4f, 0x70, 0x75, 0x73, 0x48, 0x65, 0x61, 0x64]); // 'OpusHead' + view.setUint8(8, 1); // Version + view.setUint8(9, 2); // Channel count + view.setUint16(10, 312, true); // Pre-skip + view.setUint32(12, 48000, true); // Sample rate + view.setInt16(16, 0, true); // Output gain + view.setUint8(18, 0); // Channel mapping family + + const metadata: AudioTrackMetadata = { + maximumPacketCount: 100, + decoderConfig: { + codec: 'opus', + sampleRate: 48000, + numberOfChannels: 2, + description, + }, + }; + + output.addAudioTrack(new EncodedAudioPacketSource('opus'), metadata); + output.addAudioTrack(new EncodedAudioPacketSource('opus'), metadata); + + await output.start(); + await output.finalize(); + + using input = new Input({ + source: new BufferSource(output.target.buffer!), + formats: ALL_FORMATS, + }); + + const tracks = await input.getTracks(); + expect(tracks).toHaveLength(2); + + for (const track of tracks) { + const audioTrack = track as InputAudioTrack; + expect(audioTrack.isAudioTrack()).toBe(true); + expect(await audioTrack.getCodec()).toBe('opus'); + expect(await audioTrack.getSampleRate()).toBe(48000); + expect(await audioTrack.getNumberOfChannels()).toBe(2); + expect((await audioTrack.computePacketStats()).packetCount).toBe(0); + } +}); + +test('Empty WAVE', async () => { + const output = new Output({ + format: new WavOutputFormat(), + target: new BufferTarget(), + }); + + // WAVE holds a single audio track + output.addAudioTrack(new EncodedAudioPacketSource('pcm-s16'), { maximumPacketCount: 100 }); + + await output.start(); + + // There's no information to go on, so the muxer can't make anything up + await expect(output.finalize()).rejects.toThrow('Cannot finalize an empty WAVE file'); +}); + +test('Empty WAVE with declared decoder config', async () => { + const output = new Output({ + format: new WavOutputFormat(), + target: new BufferTarget(), + }); + + // Deliberately not the fallback values, so we can tell the declared config was actually used + output.addAudioTrack(new EncodedAudioPacketSource('pcm-s16'), { + maximumPacketCount: 100, + decoderConfig: { + codec: 'pcm-s16', + sampleRate: 44100, + numberOfChannels: 1, + }, + }); + + await output.start(); + await output.finalize(); + + using input = new Input({ + source: new BufferSource(output.target.buffer!), + formats: ALL_FORMATS, + }); + + const tracks = await input.getTracks(); + expect(tracks).toHaveLength(1); + + const audioTrack = tracks[0] as InputAudioTrack; + expect(audioTrack.isAudioTrack()).toBe(true); + expect(await audioTrack.getCodec()).toBe('pcm-s16'); + expect(await audioTrack.getSampleRate()).toBe(44100); + expect(await audioTrack.getNumberOfChannels()).toBe(1); + expect((await audioTrack.computePacketStats()).packetCount).toBe(0); +}); + +test('Empty MP3', async () => { + const output = new Output({ + format: new Mp3OutputFormat(), + target: new BufferTarget(), + }); + + output.addAudioTrack(new EncodedAudioPacketSource('mp3'), { maximumPacketCount: 100 }); + + await output.start(); + + // There's no information to go on, so the muxer can't make anything up + await expect(output.finalize()).rejects.toThrow('Cannot finalize an empty MP3 file'); +}); + +test('Empty MP3 with declared decoder config', async () => { + const output = new Output({ + format: new Mp3OutputFormat(), + target: new BufferTarget(), + }); + + // Deliberately not the fallback values, so we can tell the declared config was actually used + output.addAudioTrack(new EncodedAudioPacketSource('mp3'), { + maximumPacketCount: 100, + decoderConfig: { + codec: 'mp3', + sampleRate: 44100, + numberOfChannels: 1, + }, + }); + + await output.start(); + await output.finalize(); + + using input = new Input({ + source: new BufferSource(output.target.buffer!), + formats: ALL_FORMATS, + }); + + const tracks = await input.getTracks(); + expect(tracks).toHaveLength(1); + + const audioTrack = tracks[0] as InputAudioTrack; + expect(audioTrack.isAudioTrack()).toBe(true); + expect(await audioTrack.getCodec()).toBe('mp3'); + expect(await audioTrack.getSampleRate()).toBe(44100); + expect(await audioTrack.getNumberOfChannels()).toBe(1); + expect((await audioTrack.computePacketStats()).packetCount).toBe(0); +}); + +test('Empty MP3 with priming packet', async () => { + const output = new Output({ + format: new Mp3OutputFormat(), + target: new BufferTarget(), + }); + + // An MPEG Version 1 Layer III frame header, 128 kbps, 32 kHz, single channel + const frameHeader = new Uint8Array([0xff, 0xfb, 0x98, 0xc0]); + + // The priming packet takes precedence over the declared config, so we can tell which one was used + output.addAudioTrack(new EncodedAudioPacketSource('mp3'), { + maximumPacketCount: 100, + decoderConfig: { + codec: 'mp3', + sampleRate: 44100, + numberOfChannels: 2, + }, + primingPacket: new EncodedPacket(frameHeader, 'key', 0, 0), + }); + + await output.start(); + await output.finalize(); + + using input = new Input({ + source: new BufferSource(output.target.buffer!), + formats: ALL_FORMATS, + }); + + const tracks = await input.getTracks(); + expect(tracks).toHaveLength(1); + + const audioTrack = tracks[0] as InputAudioTrack; + expect(audioTrack.isAudioTrack()).toBe(true); + expect(await audioTrack.getCodec()).toBe('mp3'); + expect(await audioTrack.getSampleRate()).toBe(32000); + expect(await audioTrack.getNumberOfChannels()).toBe(1); + expect((await audioTrack.computePacketStats()).packetCount).toBe(0); +}); + +test('Empty MP3 without a Xing header', async () => { + const output = new Output({ + format: new Mp3OutputFormat({ xingHeader: false }), + target: new BufferTarget(), + }); + + output.addAudioTrack(new EncodedAudioPacketSource('mp3'), { + maximumPacketCount: 100, + decoderConfig: { + codec: 'mp3', + sampleRate: 44100, + numberOfChannels: 1, + }, + }); + + await output.start(); + + // The Xing frame is the only frame we could have synthesized, so there'd be nothing to write at all + await expect(output.finalize()).rejects.toThrow('Cannot finalize an empty MP3 file'); +}); + +test('Empty ADTS', async () => { + const output = new Output({ + format: new AdtsOutputFormat(), + target: new BufferTarget(), + }); + + output.addAudioTrack(new EncodedAudioPacketSource('aac'), { maximumPacketCount: 100 }); + + await output.start(); + + // ADTS is a bare sequence of frames, each carrying its own header, so there'd be nothing to write + await expect(output.finalize()).rejects.toThrow('Cannot finalize an empty ADTS file'); +}); + +test('Empty ADTS with declared decoder config', async () => { + const output = new Output({ + format: new AdtsOutputFormat(), + target: new BufferTarget(), + }); + + output.addAudioTrack(new EncodedAudioPacketSource('aac'), { + maximumPacketCount: 100, + decoderConfig: { + codec: 'mp4a.40.2', + sampleRate: 44100, + numberOfChannels: 2, + description: new Uint8Array([0x12, 0x10]), // AudioSpecificConfig: AAC-LC, 44100 Hz, stereo + }, + }); + + await output.start(); + + // The declared config doesn't help; ADTS has no place to put it + await expect(output.finalize()).rejects.toThrow('Cannot finalize an empty ADTS file'); +}); + +test('Empty FLAC', async () => { + const output = new Output({ + format: new FlacOutputFormat(), + target: new BufferTarget(), + }); + + output.addAudioTrack(new EncodedAudioPacketSource('flac'), { maximumPacketCount: 100 }); + + await output.start(); + + // There's no information to go on, so the muxer can't make anything up + await expect(output.finalize()).rejects.toThrow('Cannot finalize an empty FLAC file'); +}); + +test('Empty FLAC with declared decoder config', async () => { + const output = new Output({ + format: new FlacOutputFormat(), + target: new BufferTarget(), + }); + + // A STREAMINFO metadata block for a 44100 Hz mono, 16-bit stream + const description = new Uint8Array(4 + 4 + 34); + description.set([0x66, 0x4c, 0x61, 0x43]); // 'fLaC' + description[4] = 0x80; // Last metadata block, type STREAMINFO + description[7] = 34; // Block size + const streamInfo = new Bitstream(description.subarray(8)); + streamInfo.writeBits(16, 4096); // Minimum block size + streamInfo.writeBits(16, 4096); // Maximum block size + streamInfo.writeBits(24, 0); // Minimum frame size + streamInfo.writeBits(24, 0); // Maximum frame size + streamInfo.writeBits(20, 44100); // Sample rate + streamInfo.writeBits(3, 0); // Channels - 1 + streamInfo.writeBits(5, 15); // Bits per sample - 1 + + output.addAudioTrack(new EncodedAudioPacketSource('flac'), { + maximumPacketCount: 100, + decoderConfig: { + codec: 'flac', + sampleRate: 44100, + numberOfChannels: 1, + description, + }, + }); + + await output.start(); + await output.finalize(); + + using input = new Input({ + source: new BufferSource(output.target.buffer!), + formats: ALL_FORMATS, + }); + + const tracks = await input.getTracks(); + expect(tracks).toHaveLength(1); + + const audioTrack = tracks[0] as InputAudioTrack; + expect(audioTrack.isAudioTrack()).toBe(true); + expect(await audioTrack.getCodec()).toBe('flac'); + expect(await audioTrack.getSampleRate()).toBe(44100); + expect(await audioTrack.getNumberOfChannels()).toBe(1); + expect((await audioTrack.computePacketStats()).packetCount).toBe(0); +}); + +test('Empty MPEG-TS', async () => { + const output = new Output({ + format: new MpegTsOutputFormat(), + target: new BufferTarget(), + }); + + output.addVideoTrack(new EncodedVideoPacketSource('avc'), { maximumPacketCount: 100 }); + output.addAudioTrack(new EncodedAudioPacketSource('aac'), { maximumPacketCount: 100 }); + + await output.start(); + await output.finalize(); + + using input = new Input({ + source: new BufferSource(output.target.buffer!), + formats: ALL_FORMATS, + }); + + expect(await input.getTracks()).toHaveLength(0); +}); + +test('Empty MPEG-TS with declared decoder config', async () => { + const output = new Output({ + format: new MpegTsOutputFormat(), + target: new BufferTarget(), + }); + + // An MPEG-TS stream cannot be described without a packet - it's the first packet that defines the stream - so + // declaring the config up front changes nothing + output.addVideoTrack(new EncodedVideoPacketSource('avc'), { + maximumPacketCount: 100, + decoderConfig: { + codec: 'avc1.42001f', + codedWidth: 1280, + codedHeight: 720, + description: new Uint8Array([0x01, 0x42, 0x00, 0x1f, 0xff, 0xe1, 0x00, 0x00, 0x01, 0x00, 0x00]), + }, + }); + + await output.start(); + await output.finalize(); + + using input = new Input({ + source: new BufferSource(output.target.buffer!), + formats: ALL_FORMATS, + }); + + expect(await input.getTracks()).toHaveLength(0); +}); + +test('Empty HLS', async () => { + const files = new Map(); + + const output = new Output({ + format: new HlsOutputFormat({ segmentFormat: new MpegTsOutputFormat() }), + target: new PathedTarget('', (request) => { + const target = new BufferTarget(); + target.on('finalized', () => files.set(request.path, new Uint8Array(target.buffer!))); + return target; + }), + }); + + output.addVideoTrack(new EncodedVideoPacketSource('avc'), { + maximumPacketCount: 100, + decoderConfig: { + codec: 'avc1.42001f', + codedWidth: 1280, + codedHeight: 720, + description: new Uint8Array([0x01, 0x42, 0x00, 0x1f, 0xff, 0xe1, 0x00, 0x00, 0x01, 0x00, 0x00]), + }, + }); + + await output.start(); + await output.finalize(); + + // HLS only ever writes segments for packets it has actually seen, so all we get is the master playlist (which is + // the root target) and one empty media playlist + expect([...files.keys()].sort()).toEqual(['', 'playlist-1.m3u8']); +}); + +const createFormat = (variant: EmptyMediaVariant) => { + if (variant.type === 'mp4') { + return new Mp4OutputFormat({ fastStart: variant.fastStart }); + } else if (variant.type === 'cmaf') { + return new CmafOutputFormat(); + } else { + return new MkvOutputFormat(); + } +}; diff --git a/test/node/hls-output.test.ts b/test/node/hls-output.test.ts index b4dd8ea..e683d44 100644 --- a/test/node/hls-output.test.ts +++ b/test/node/hls-output.test.ts @@ -3,6 +3,7 @@ import { Output, OutputTrackGroup } from '../../src/output.js'; import { CmafOutputFormat, HlsOutputFormat, + HlsOutputFormatOptions, HlsOutputSegmentInfo, Mp4OutputFormat, MpegTsOutputFormat, @@ -21,7 +22,7 @@ import { AudioCodec, VideoCodec } from '../../src/codec.js'; import { EncodedPacket, PacketType } from '../../src/packet.js'; import { assert, promiseWithResolvers } from '../../src/misc.js'; import { Input } from '../../src/input.js'; -import { BufferSource } from '../../src/source.js'; +import { BufferSource, CustomPathedSource } from '../../src/source.js'; import { ALL_FORMATS } from '../../src/input-format.js'; import { InputAudioTrack, InputVideoTrack } from '../../src/input-track.js'; import { EncodedPacketSink } from '../../src/media-sink.js'; @@ -2331,6 +2332,110 @@ test('CMAF segmentation, single file per playlist', async () => { expect(playlistText).toContain('#EXT-X-MAP:URI='); }); +test('Sparse tracks in segments, MPEG-TS', async () => { + await runSparseTracksInSegments({ + segmentFormat: new MpegTsOutputFormat(), + }); +}); + +test('Sparse tracks in segments, CMAF', async () => { + await runSparseTracksInSegments({ + segmentFormat: new CmafOutputFormat(), + }); +}); + +test('Sparse tracks in segments, standard MP4', async () => { + await runSparseTracksInSegments({ + segmentFormat: new Mp4OutputFormat(), + }); +}); + +test('Sparse tracks in segments, fragmented MP4', async () => { + await runSparseTracksInSegments({ + segmentFormat: new Mp4OutputFormat({ fastStart: 'fragmented' }), + }); +}); + +test('Sparse tracks in segments, fragmented MP4 + single file', async () => { + await runSparseTracksInSegments({ + segmentFormat: new Mp4OutputFormat({ fastStart: 'fragmented' }), + singleFilePerPlaylist: true, + }); +}); + +const runSparseTracksInSegments = async (hlsOptions: HlsOutputFormatOptions) => { + const targets = new Map(); + + const output = new Output({ + format: new HlsOutputFormat(hlsOptions), + target: new PathedTarget('master.m3u8', (request) => { + const target = new BufferTarget(); + targets.set(request.path, target); + + return target; + }), + }); + + const video = videoSource(); + const audio = audioSource(); + output.addVideoTrack(video); + output.addAudioTrack(audio); + + await output.start(); + + // We expect three segments: First one with just video, second with video and audio, last one with just audio. + + await video.add(new EncodedPacket(avcPacketData, 'key', 0, 0), avcMetadata); + await video.add(new EncodedPacket(avcPacketData, 'delta', 0.5, 0), avcMetadata); + await video.add(new EncodedPacket(avcPacketData, 'delta', 1, 0), avcMetadata); + await video.add(new EncodedPacket(avcPacketData, 'delta', 1.5, 0), avcMetadata); + await video.add(new EncodedPacket(avcPacketData, 'key', 2, 0), avcMetadata); + await video.add(new EncodedPacket(avcPacketData, 'delta', 2.5, 0), avcMetadata); + await video.add(new EncodedPacket(avcPacketData, 'delta', 3, 0), avcMetadata); + await video.add(new EncodedPacket(avcPacketData, 'delta', 3.5, 0), avcMetadata); + + await audio.add(new EncodedPacket(aacPacketData, 'key', 2, 0), aacMetadata); + await audio.add(new EncodedPacket(aacPacketData, 'key', 2.5, 0), aacMetadata); + await audio.add(new EncodedPacket(aacPacketData, 'key', 3, 0), aacMetadata); + await audio.add(new EncodedPacket(aacPacketData, 'key', 3.5, 0), aacMetadata); + await audio.add(new EncodedPacket(aacPacketData, 'key', 4, 0), aacMetadata); + await audio.add(new EncodedPacket(aacPacketData, 'key', 4.5, 0), aacMetadata); + await audio.add(new EncodedPacket(aacPacketData, 'key', 5, 0), aacMetadata); + await audio.add(new EncodedPacket(aacPacketData, 'key', 5.5, 0), aacMetadata); + + await output.finalize(); + + // Read the entire output back using the HLS input + using input = new Input({ + source: new CustomPathedSource('master.m3u8', ({ path }) => { + const target = targets.get(path); + assert(target); + + return new BufferSource(target.buffer!); + }), + formats: ALL_FORMATS, + }); + + const videoTrack = await input.getPrimaryVideoTrack() as InputVideoTrack; + expect(videoTrack).toBeTruthy(); + const audioTrack = await input.getPrimaryAudioTrack() as InputAudioTrack; + expect(audioTrack).toBeTruthy(); + + const videoSink = new EncodedPacketSink(videoTrack); + const videoTimestamps: number[] = []; + for await (const packet of videoSink.packets()) { + videoTimestamps.push(packet.timestamp); + } + expect(videoTimestamps).toEqual([0, 0.5, 1, 1.5, 2, 2.5, 3, 3.5]); + + const audioSink = new EncodedPacketSink(audioTrack); + const audioTimestamps: number[] = []; + for await (const packet of audioSink.packets()) { + audioTimestamps.push(packet.timestamp); + } + expect(audioTimestamps).toEqual([2, 2.5, 3, 3.5, 4, 4.5, 5, 5.5]); +}; + test('Live mode', async () => { const writtenTexts = new Map(); const writeCounts = new Map();