diff --git a/dev/demux.html b/dev/demux.html index 50a2bf5..f22e924 100644 --- a/dev/demux.html +++ b/dev/demux.html @@ -17,7 +17,9 @@ const track = await input.getPrimaryAudioTrack(); const sink = new Mediabunny.EncodedPacketSink(track); - console.log("Done") + for await (const packet of sink.packets()) { + console.log(packet.timestamp, packet.duration, packet.type); + } /* diff --git a/src/adts/adts-misc.ts b/src/adts/adts-misc.ts new file mode 100644 index 0000000..ab611c3 --- /dev/null +++ b/src/adts/adts-misc.ts @@ -0,0 +1,47 @@ +/*! + * Copyright (c) 2025-present, Vanilagy and contributors + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +import { AacAudioSpecificConfig } from '../codec'; +import { Bitstream } from '../misc'; + +export type AdtsHeaderTemplate = { + header: Uint8Array; + bitstream: Bitstream; +}; + +export const buildAdtsHeaderTemplate = (config: AacAudioSpecificConfig): AdtsHeaderTemplate => { + const header = new Uint8Array(7); + const bitstream = new Bitstream(header); + + const { objectType, frequencyIndex, channelConfiguration } = config; + const profile = objectType - 1; + + bitstream.writeBits(12, 0b1111_11111111); // Syncword + bitstream.writeBits(1, 0); // MPEG Version + bitstream.writeBits(2, 0); // Layer + bitstream.writeBits(1, 1); // Protection absence + bitstream.writeBits(2, profile); // Profile + bitstream.writeBits(4, frequencyIndex); // MPEG-4 Sampling Frequency Index + bitstream.writeBits(1, 0); // Private bit + bitstream.writeBits(3, channelConfiguration); // MPEG-4 Channel Configuration + bitstream.writeBits(1, 0); // Originality + bitstream.writeBits(1, 0); // Home + bitstream.writeBits(1, 0); // Copyright ID bit + bitstream.writeBits(1, 0); // Copyright ID start + bitstream.skipBits(13); // Frame length (to be filled per packet) + bitstream.writeBits(11, 0x7ff); // Buffer fullness + bitstream.writeBits(2, 0); // Number of AAC frames minus 1 + // Omit CRC check + + return { header, bitstream }; +}; + +export const writeAdtsFrameLength = (bitstream: Bitstream, frameLength: number) => { + bitstream.pos = 30; + bitstream.writeBits(13, frameLength); +}; diff --git a/src/adts/adts-muxer.ts b/src/adts/adts-muxer.ts index 0d1cf3c..458011b 100644 --- a/src/adts/adts-muxer.ts +++ b/src/adts/adts-muxer.ts @@ -6,20 +6,20 @@ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -import { AacAudioSpecificConfig, parseAacAudioSpecificConfig, validateAudioChunkMetadata } from '../codec'; -import { Bitstream, toUint8Array } from '../misc'; +import { parseAacAudioSpecificConfig, validateAudioChunkMetadata } from '../codec'; +import { assert, Bitstream, toUint8Array } from '../misc'; import { Muxer } from '../muxer'; import { Output, OutputAudioTrack } from '../output'; import { AdtsOutputFormat } from '../output-format'; import { EncodedPacket } from '../packet'; import { Writer } from '../writer'; +import { buildAdtsHeaderTemplate, writeAdtsFrameLength } from './adts-misc'; export class AdtsMuxer extends Muxer { private format: AdtsOutputFormat; private writer: Writer; - private header = new Uint8Array(7); - private headerBitstream = new Bitstream(this.header); - private audioSpecificConfig: AacAudioSpecificConfig | null = null; + private header: Uint8Array | null = null; + private headerBitstream: Bitstream | null = null; private inputIsAdts: boolean | null = null; constructor(output: Output, format: AdtsOutputFormat) { @@ -46,13 +46,12 @@ export class AdtsMuxer extends Muxer { packet: EncodedPacket, meta?: EncodedAudioChunkMetadata, ) { - // https://wiki.multimedia.cx/index.php/ADTS (last visited: 2025/08/17) - const release = await this.mutex.acquire(); try { this.validateAndNormalizeTimestamp(track, packet.timestamp, packet.type === 'key'); + // First packet - determine input format from metadata if (this.inputIsAdts === null) { validateAudioChunkMetadata(meta); @@ -65,27 +64,10 @@ export class AdtsMuxer extends Muxer { this.inputIsAdts = !description; if (!this.inputIsAdts) { - this.audioSpecificConfig = parseAacAudioSpecificConfig(toUint8Array(description!)); - - const { objectType, frequencyIndex, channelConfiguration } = this.audioSpecificConfig; - const profile = objectType - 1; - - this.headerBitstream.writeBits(12, 0b1111_11111111); // Syncword - this.headerBitstream.writeBits(1, 0); // MPEG Version - this.headerBitstream.writeBits(2, 0); // Layer - this.headerBitstream.writeBits(1, 1); // Protection absence - this.headerBitstream.writeBits(2, profile); // Profile - this.headerBitstream.writeBits(4, frequencyIndex); // MPEG-4 Sampling Frequency Index - this.headerBitstream.writeBits(1, 0); // Private bit - this.headerBitstream.writeBits(3, channelConfiguration); // MPEG-4 Channel Configuration - this.headerBitstream.writeBits(1, 0); // Originality - this.headerBitstream.writeBits(1, 0); // Home - this.headerBitstream.writeBits(1, 0); // Copyright ID bit - this.headerBitstream.writeBits(1, 0); // Copyright ID start - this.headerBitstream.skipBits(13); // Frame length - this.headerBitstream.writeBits(11, 0x7ff); // Buffer fullness - this.headerBitstream.writeBits(2, 0); // Number of AAC frames minus 1 - // Omit CRC check + const config = parseAacAudioSpecificConfig(toUint8Array(description!)); + const template = buildAdtsHeaderTemplate(config); + this.header = template.header; + this.headerBitstream = template.bitstream; } } @@ -98,10 +80,11 @@ export class AdtsMuxer extends Muxer { this.format._options.onFrame(packet.data, startPos); } } else { - // Packets are raw AAC, prepend ADTS header + assert(this.header); + + // Packets are raw AAC, we gotta turn it into ADTS const frameLength = packet.data.byteLength + this.header.byteLength; - this.headerBitstream.pos = 30; - this.headerBitstream.writeBits(13, frameLength); + writeAdtsFrameLength(this.headerBitstream!, frameLength); const startPos = this.writer.getPos(); this.writer.write(this.header); diff --git a/src/codec-data.ts b/src/codec-data.ts index 109b195..369a8ba 100644 --- a/src/codec-data.ts +++ b/src/codec-data.ts @@ -118,7 +118,7 @@ export const iterateNalUnitsInAnnexB = function* (packetData: Uint8Array): Gener } }; -const iterateNalUnitsInLengthPrefixed = function* ( +export const iterateNalUnitsInLengthPrefixed = function* ( packetData: Uint8Array, lengthSize: 1 | 2 | 3 | 4, ): Generator { @@ -1472,6 +1472,99 @@ export const serializeHevcDecoderConfigurationRecord = (record: HevcDecoderConfi return new Uint8Array(bytes); }; +/** Deserializes an HevcDecoderConfigurationRecord from the format specified in Section 8.3.3.1 of ISO 14496-15. */ +export const deserializeHevcDecoderConfigurationRecord = (data: Uint8Array): HevcDecoderConfigurationRecord | null => { + try { + const view = toDataView(data); + let offset = 0; + + const configurationVersion = view.getUint8(offset++); + + const byte1 = view.getUint8(offset++); + const generalProfileSpace = (byte1 >> 6) & 0x3; + const generalTierFlag = (byte1 >> 5) & 0x1; + const generalProfileIdc = byte1 & 0x1F; + + const generalProfileCompatibilityFlags = view.getUint32(offset, false); + offset += 4; + + const generalConstraintIndicatorFlags = data.subarray(offset, offset + 6); + offset += 6; + + const generalLevelIdc = view.getUint8(offset++); + + const minSpatialSegmentationIdc = ((view.getUint8(offset++) & 0x0F) << 8) | view.getUint8(offset++); + + const parallelismType = view.getUint8(offset++) & 0x03; + + const chromaFormatIdc = view.getUint8(offset++) & 0x03; + + const bitDepthLumaMinus8 = view.getUint8(offset++) & 0x07; + + const bitDepthChromaMinus8 = view.getUint8(offset++) & 0x07; + + const avgFrameRate = view.getUint16(offset, false); + offset += 2; + + const byte21 = view.getUint8(offset++); + const constantFrameRate = (byte21 >> 6) & 0x03; + const numTemporalLayers = (byte21 >> 3) & 0x07; + const temporalIdNested = (byte21 >> 2) & 0x01; + const lengthSizeMinusOne = byte21 & 0x03; + + const numOfArrays = view.getUint8(offset++); + + const arrays: HevcDecoderConfigurationRecord['arrays'] = []; + for (let i = 0; i < numOfArrays; i++) { + const arrByte = view.getUint8(offset++); + const arrayCompleteness = (arrByte >> 7) & 0x01; + const nalUnitType = arrByte & 0x3F; + + const numNalus = view.getUint16(offset, false); + offset += 2; + + const nalUnits: Uint8Array[] = []; + for (let j = 0; j < numNalus; j++) { + const nalUnitLength = view.getUint16(offset, false); + offset += 2; + + nalUnits.push(data.subarray(offset, offset + nalUnitLength)); + offset += nalUnitLength; + } + + arrays.push({ + arrayCompleteness, + nalUnitType, + nalUnits, + }); + } + + return { + configurationVersion, + generalProfileSpace, + generalTierFlag, + generalProfileIdc, + generalProfileCompatibilityFlags, + generalConstraintIndicatorFlags, + generalLevelIdc, + minSpatialSegmentationIdc, + parallelismType, + chromaFormatIdc, + bitDepthLumaMinus8, + bitDepthChromaMinus8, + avgFrameRate, + constantFrameRate, + numTemporalLayers, + temporalIdNested, + lengthSizeMinusOne, + arrays, + }; + } catch (error) { + console.error('Error deserializing HEVC Decoder Configuration Record:', error); + return null; + } +}; + export type Vp9CodecInfo = { profile: number; level: number; diff --git a/src/index.ts b/src/index.ts index 6cd723f..5c8af58 100644 --- a/src/index.ts +++ b/src/index.ts @@ -31,6 +31,8 @@ export { Mp3OutputFormat, Mp3OutputFormatOptions, Mp4OutputFormat, + MpegTsOutputFormat, + MpegTsOutputFormatOptions, OggOutputFormat, OggOutputFormatOptions, WavOutputFormat, @@ -126,25 +128,27 @@ export { export { InputFormat, AdtsInputFormat, + FlacInputFormat, IsobmffInputFormat, MatroskaInputFormat, Mp3InputFormat, Mp4InputFormat, + MpegTsInputFormat, OggInputFormat, QuickTimeInputFormat, WaveInputFormat, WebMInputFormat, - FlacInputFormat, ALL_FORMATS, ADTS, + FLAC, MATROSKA, MP3, MP4, + MPEG_TS, OGG, QTFF, WAVE, WEBM, - FLAC, } from './input-format'; export { Input, diff --git a/src/mpeg-ts/mpeg-ts-demuxer.ts b/src/mpeg-ts/mpeg-ts-demuxer.ts index fc75fd4..e1d6318 100644 --- a/src/mpeg-ts/mpeg-ts-demuxer.ts +++ b/src/mpeg-ts/mpeg-ts-demuxer.ts @@ -53,19 +53,10 @@ import { import { FRAME_HEADER_SIZE as MP3_FRAME_HEADER_SIZE, readMp3FrameHeader } from '../../shared/mp3-misc'; import { EncodedPacket, PacketType, PLACEHOLDER_DATA } from '../packet'; import { FileSlice, readBytes, Reader, readU16Be, readU32Be, readU8 } from '../reader'; +import { buildMpegTsMimeType, MpegTsStreamType, TIMESCALE, TS_PACKET_SIZE } from './mpeg-ts-misc'; -const TIMESCALE = 90_000; // MPEG-TS timestamps run on a 90 kHz clock -const TS_PACKET_SIZE = 188; const MISSING_PES_PACKET_ERROR = 'No PES packet found where one was expected.'; -const enum MpegTsStreamType { - MP3_MPEG1 = 0x03, - MP3_MPEG2 = 0x04, - AAC = 0x0f, - AVC = 0x1b, - HEVC = 0x24, -} - type ElementaryStream = { demuxer: MpegTsDemuxer; pid: number; @@ -446,14 +437,7 @@ export class MpegTsDemuxer extends Demuxer { const tracks = await this.getTracks(); const codecStrings = await Promise.all(tracks.map(x => x.getCodecParameterString())); - let string = 'video/MP2T'; - - const uniqueCodecStrings = [...new Set(codecStrings.filter(Boolean))]; - if (uniqueCodecStrings.length > 0) { - string += `; codecs="${uniqueCodecStrings.join(', ')}"`; - } - - return string; + return buildMpegTsMimeType(codecStrings); } async readSection(startPos: number, full: boolean): Promise
{ @@ -1229,6 +1213,7 @@ export abstract class MpegTsTrackBacking implements InputTrackBacking { continue; } + context.uncapped = true; // Upgrade it to an uncapped context const buffer = new PacketBuffer(this, context); const result = await buffer.readNext(); diff --git a/src/mpeg-ts/mpeg-ts-misc.ts b/src/mpeg-ts/mpeg-ts-misc.ts new file mode 100644 index 0000000..0f5dd43 --- /dev/null +++ b/src/mpeg-ts/mpeg-ts-misc.ts @@ -0,0 +1,29 @@ +/*! + * Copyright (c) 2025-present, Vanilagy and contributors + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +export const TIMESCALE = 90_000; // MPEG-TS timestamps run on a 90 kHz clock +export const TS_PACKET_SIZE = 188; + +export const enum MpegTsStreamType { + MP3_MPEG1 = 0x03, + MP3_MPEG2 = 0x04, + AAC = 0x0f, + AVC = 0x1b, + HEVC = 0x24, +} + +export const buildMpegTsMimeType = (codecStrings: (string | null)[]) => { + let string = 'video/MP2T'; + + const uniqueCodecStrings = [...new Set(codecStrings.filter(Boolean))]; + if (uniqueCodecStrings.length > 0) { + string += `; codecs="${uniqueCodecStrings.join(', ')}"`; + } + + return string; +}; diff --git a/src/mpeg-ts/mpeg-ts-muxer.ts b/src/mpeg-ts/mpeg-ts-muxer.ts new file mode 100644 index 0000000..8f4ee62 --- /dev/null +++ b/src/mpeg-ts/mpeg-ts-muxer.ts @@ -0,0 +1,711 @@ +/*! + * Copyright (c) 2025-present, Vanilagy and contributors + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +import { parseAacAudioSpecificConfig, validateAudioChunkMetadata, validateVideoChunkMetadata } from '../codec'; +import { buildAdtsHeaderTemplate, writeAdtsFrameLength } from '../adts/adts-misc'; +import { + AvcDecoderConfigurationRecord, + AvcNalUnitType, + concatNalUnitsInAnnexB, + deserializeAvcDecoderConfigurationRecord, + deserializeHevcDecoderConfigurationRecord, + extractNalUnitTypeForAvc, + extractNalUnitTypeForHevc, + HevcDecoderConfigurationRecord, + HevcNalUnitType, + iterateNalUnitsInAnnexB, + iterateNalUnitsInLengthPrefixed, +} from '../codec-data'; +import { assert, Bitstream, promiseWithResolvers, setUint24, toDataView, toUint8Array } from '../misc'; +import { Muxer } from '../muxer'; +import { Output, OutputAudioTrack, OutputVideoTrack } from '../output'; +import { MpegTsOutputFormat } from '../output-format'; +import { EncodedPacket } from '../packet'; +import { Writer } from '../writer'; +import { buildMpegTsMimeType, MpegTsStreamType, TIMESCALE, TS_PACKET_SIZE } from './mpeg-ts-misc'; + +const PAT_PID = 0x0000; +const PMT_PID = 0x1000; +const FIRST_TRACK_PID = 0x0100; + +const VIDEO_STREAM_ID_BASE = 0xE0; +const AUDIO_STREAM_ID_BASE = 0xC0; + +const AVC_AUD_NAL = new Uint8Array([0x09, 0xF0]); +const HEVC_AUD_NAL = new Uint8Array([0x46, 0x01]); + +type MpegTsTrackData = { + track: OutputVideoTrack | OutputAudioTrack; + pid: number; + streamType: MpegTsStreamType; + streamId: number; + codecString: string; + packetQueue: QueuedPacket[]; + inputIsAnnexB: boolean | null; + inputIsAdts: boolean | null; + avcDecoderConfig: AvcDecoderConfigurationRecord | null; + hevcDecoderConfig: HevcDecoderConfigurationRecord | null; + adtsHeader: Uint8Array | null; + adtsHeaderBitstream: Bitstream | null; +}; + +type QueuedPacket = { + data: Uint8Array; + timestamp: number; + isKeyframe: boolean; +}; + +export class MpegTsMuxer extends Muxer { + private format: MpegTsOutputFormat; + private writer: Writer; + + private trackDatas: MpegTsTrackData[] = []; + private tablesWritten = false; + private continuityCounters = new Map(); + private packetBuffer = new Uint8Array(TS_PACKET_SIZE); + private packetView = toDataView(this.packetBuffer); + private allTracksKnown = promiseWithResolvers(); + + private videoTrackIndex = 0; + private audioTrackIndex = 0; + + private pesHeaderBuffer = new Uint8Array(14); + private pesHeaderView = toDataView(this.pesHeaderBuffer); + private ptsBitstream = new Bitstream(this.pesHeaderBuffer.subarray(9, 14)); + private adaptationFieldBuffer = new Uint8Array(184); + private payloadBuffer = new Uint8Array(184); + + constructor(output: Output, format: MpegTsOutputFormat) { + super(output); + + this.format = format; + this.writer = output._writer; + this.writer.ensureMonotonicity = true; + } + + async start() { + // Nothing to do here + } + + async getMimeType() { + await this.allTracksKnown.promise; + return buildMpegTsMimeType(this.trackDatas.map(x => x.codecString)); + } + + private getVideoTrackData(track: OutputVideoTrack, meta?: EncodedVideoChunkMetadata) { + const existingTrackData = this.trackDatas.find(x => x.track === track); + if (existingTrackData) { + return existingTrackData; + } + + validateVideoChunkMetadata(meta); + assert(meta?.decoderConfig); + + const codec = track.source._codec; + assert(codec === 'avc' || codec === 'hevc'); + + const streamType = codec === 'avc' + ? MpegTsStreamType.AVC + : MpegTsStreamType.HEVC; + const pid = FIRST_TRACK_PID + this.trackDatas.length; + const streamId = VIDEO_STREAM_ID_BASE + this.videoTrackIndex++; + + const newTrackData: MpegTsTrackData = { + track, + pid, + streamType, + streamId, + codecString: meta.decoderConfig.codec, + packetQueue: [], + inputIsAnnexB: null, + inputIsAdts: null, + avcDecoderConfig: null, + hevcDecoderConfig: null, + adtsHeader: null, + adtsHeaderBitstream: null, + }; + + this.trackDatas.push(newTrackData); + + if (this.allTracksAreKnown()) { + this.allTracksKnown.resolve(); + } + + return newTrackData; + } + + private getAudioTrackData(track: OutputAudioTrack, meta?: EncodedAudioChunkMetadata) { + const existingTrackData = this.trackDatas.find(x => x.track === track); + if (existingTrackData) { + return existingTrackData; + } + + validateAudioChunkMetadata(meta); + assert(meta?.decoderConfig); + + const codec = track.source._codec; + assert(codec === 'aac' || codec === 'mp3'); + + const streamType = codec === 'aac' ? MpegTsStreamType.AAC : MpegTsStreamType.MP3_MPEG1; + const pid = FIRST_TRACK_PID + this.trackDatas.length; + const streamId = AUDIO_STREAM_ID_BASE + this.audioTrackIndex++; + + const newTrackData: MpegTsTrackData = { + track, + pid, + streamType, + streamId, + codecString: meta.decoderConfig.codec, + packetQueue: [], + inputIsAnnexB: null, + inputIsAdts: null, + avcDecoderConfig: null, + hevcDecoderConfig: null, + adtsHeader: null, + adtsHeaderBitstream: null, + }; + + this.trackDatas.push(newTrackData); + + if (this.allTracksAreKnown()) { + this.allTracksKnown.resolve(); + } + + return newTrackData; + } + + async addEncodedVideoPacket( + track: OutputVideoTrack, + packet: EncodedPacket, + meta?: EncodedVideoChunkMetadata, + ) { + const release = await this.mutex.acquire(); + + try { + const trackData = this.getVideoTrackData(track, meta); + + this.validateAndNormalizeTimestamp(trackData.track, packet.timestamp, packet.type === 'key'); + + const preparedData = this.prepareVideoPacket(trackData, packet, meta); + + trackData.packetQueue.push({ + data: preparedData, + timestamp: packet.timestamp, + isKeyframe: packet.type === 'key', + }); + + await this.interleavePackets(); + } finally { + release(); + } + } + + async addEncodedAudioPacket( + track: OutputAudioTrack, + packet: EncodedPacket, + meta?: EncodedAudioChunkMetadata, + ) { + const release = await this.mutex.acquire(); + + try { + const trackData = this.getAudioTrackData(track, meta); + + this.validateAndNormalizeTimestamp(trackData.track, packet.timestamp, packet.type === 'key'); + + const preparedData = this.prepareAudioPacket(trackData, packet, meta); + + trackData.packetQueue.push({ + data: preparedData, + timestamp: packet.timestamp, + isKeyframe: packet.type === 'key', + }); + + await this.interleavePackets(); + } finally { + release(); + } + } + + async addSubtitleCue(): Promise { + throw new Error('MPEG-TS does not support subtitles.'); + } + + private prepareVideoPacket( + trackData: MpegTsTrackData, + packet: EncodedPacket, + meta?: EncodedVideoChunkMetadata, + ): Uint8Array { + const codec = (trackData.track as OutputVideoTrack).source._codec; + + if (trackData.inputIsAnnexB === null) { + // This is the first packet + const description = meta?.decoderConfig?.description; + trackData.inputIsAnnexB = !description; + + if (!trackData.inputIsAnnexB) { + const bytes = toUint8Array(description!); + if (codec === 'avc') { + trackData.avcDecoderConfig = deserializeAvcDecoderConfigurationRecord(bytes); + } else { + trackData.hevcDecoderConfig = deserializeHevcDecoderConfigurationRecord(bytes); + } + } + } + + if (trackData.inputIsAnnexB) { + return this.prepareAnnexBVideoPacket(packet.data, codec as 'avc' | 'hevc'); + } else { + return this.prepareLengthPrefixedVideoPacket(trackData, packet, codec as 'avc' | 'hevc'); + } + } + + private prepareAnnexBVideoPacket(data: Uint8Array, codec: 'avc' | 'hevc'): Uint8Array { + const nalUnits: Uint8Array[] = []; + + for (const loc of iterateNalUnitsInAnnexB(data)) { + const nalUnit = data.subarray(loc.offset, loc.offset + loc.length); + const isAud = codec === 'avc' + ? extractNalUnitTypeForAvc(nalUnit[0]!) === AvcNalUnitType.AUD + : extractNalUnitTypeForHevc(nalUnit[0]!) === HevcNalUnitType.AUD_NUT; + + if (!isAud) { + nalUnits.push(nalUnit); + } + } + + // Pretend the AUD + const aud = codec === 'avc' + ? AVC_AUD_NAL + : HEVC_AUD_NAL; + nalUnits.unshift(aud); + + return concatNalUnitsInAnnexB(nalUnits); + } + + private prepareLengthPrefixedVideoPacket( + trackData: MpegTsTrackData, + packet: EncodedPacket, + codec: 'avc' | 'hevc', + ): Uint8Array { + const data = packet.data; + const lengthSize = codec === 'avc' + ? (trackData.avcDecoderConfig!.lengthSizeMinusOne + 1) as 1 | 2 | 3 | 4 + : (trackData.hevcDecoderConfig!.lengthSizeMinusOne + 1) as 1 | 2 | 3 | 4; + + const nalUnits: Uint8Array[] = []; + + for (const loc of iterateNalUnitsInLengthPrefixed(data, lengthSize)) { + const nalUnit = data.subarray(loc.offset, loc.offset + loc.length); + const isAud = codec === 'avc' + ? extractNalUnitTypeForAvc(nalUnit[0]!) === AvcNalUnitType.AUD + : extractNalUnitTypeForHevc(nalUnit[0]!) === HevcNalUnitType.AUD_NUT; + + if (!isAud) { + nalUnits.push(nalUnit); + } + } + + if (packet.type === 'key') { + // Add whichever NALUs are missing + if (codec === 'avc') { + const config = trackData.avcDecoderConfig!; + for (const pps of config.pictureParameterSets) { + nalUnits.unshift(pps); + } + for (const sps of config.sequenceParameterSets) { + nalUnits.unshift(sps); + } + } else { + const config = trackData.hevcDecoderConfig!; + for (const arr of config.arrays) { + if (arr.nalUnitType === HevcNalUnitType.PPS_NUT) { + for (const nal of arr.nalUnits) { + nalUnits.unshift(nal); + } + } + } + for (const arr of config.arrays) { + if (arr.nalUnitType === HevcNalUnitType.SPS_NUT) { + for (const nal of arr.nalUnits) { + nalUnits.unshift(nal); + } + } + } + for (const arr of config.arrays) { + if (arr.nalUnitType === HevcNalUnitType.VPS_NUT) { + for (const nal of arr.nalUnits) { + nalUnits.unshift(nal); + } + } + } + } + } + + // Prepend the AUD + const aud = codec === 'avc' + ? AVC_AUD_NAL + : HEVC_AUD_NAL; + nalUnits.unshift(aud); + + return concatNalUnitsInAnnexB(nalUnits); + } + + private prepareAudioPacket( + trackData: MpegTsTrackData, + packet: EncodedPacket, + meta?: EncodedAudioChunkMetadata, + ): Uint8Array { + const codec = (trackData.track as OutputAudioTrack).source._codec; + + if (codec === 'mp3') { + // We're good + return packet.data; + } + + if (trackData.inputIsAdts === null) { + // It's the first packet + const description = meta?.decoderConfig?.description; + trackData.inputIsAdts = !description; + + if (!trackData.inputIsAdts) { + const config = parseAacAudioSpecificConfig(toUint8Array(description!)); + const template = buildAdtsHeaderTemplate(config); + trackData.adtsHeader = template.header; + trackData.adtsHeaderBitstream = template.bitstream; + } + } + + if (trackData.inputIsAdts) { + return packet.data; + } + + assert(trackData.adtsHeader); + assert(trackData.adtsHeaderBitstream); + + const header = trackData.adtsHeader; + const frameLength = packet.data.byteLength + header.byteLength; + writeAdtsFrameLength(trackData.adtsHeaderBitstream, frameLength); + + const result = new Uint8Array(frameLength); + result.set(header, 0); + result.set(packet.data, header.byteLength); + + return result; + } + + private allTracksAreKnown() { + for (const track of this.output._tracks) { + if (!track.source._closed && !this.trackDatas.some(x => x.track === track)) { + return false; + } + } + + return true; + } + + private async interleavePackets(isFinalCall = false) { + if (!this.tablesWritten) { + if (!this.allTracksAreKnown() && !isFinalCall) { + return; + } + + this.writeTables(); + } + + outer: + while (true) { + let trackWithMinTimestamp: MpegTsTrackData | null = null; + let minTimestamp = Infinity; + + for (const trackData of this.trackDatas) { + if ( + !isFinalCall + && trackData.packetQueue.length === 0 + && !trackData.track.source._closed + ) { + break outer; + } + + if ( + trackData.packetQueue.length > 0 + && trackData.packetQueue[0]!.timestamp < minTimestamp + ) { + trackWithMinTimestamp = trackData; + minTimestamp = trackData.packetQueue[0]!.timestamp; + } + } + + if (!trackWithMinTimestamp) { + break; + } + + const queuedPacket = trackWithMinTimestamp.packetQueue.shift()!; + this.writePesPacket(trackWithMinTimestamp, queuedPacket); + } + + if (!isFinalCall) { + await this.writer.flush(); + } + } + + private writeTables() { + assert(!this.tablesWritten); + + this.writePsiSection(PAT_PID, PAT_SECTION); + this.writePsiSection(PMT_PID, buildPmt(this.trackDatas)); + + this.tablesWritten = true; + } + + private writePsiSection(pid: number, section: Uint8Array) { + let offset = 0; + let isFirst = true; + + // Long PSI sections might span more than one TS packet + while (offset < section.length) { + const pointerFieldSize = isFirst ? 1 : 0; + const availablePayload = 184 - pointerFieldSize; + const remainingData = section.length - offset; + const chunkSize = Math.min(availablePayload, remainingData); + + let payload: Uint8Array; + if (isFirst) { + payload = this.payloadBuffer.subarray(0, 1 + chunkSize); + payload[0] = 0x00; // pointer_field + payload.set(section.subarray(offset, offset + chunkSize), 1); + } else { + payload = section.subarray(offset, offset + chunkSize); + } + + this.writeTsPacket(pid, isFirst, null, payload); + + offset += chunkSize; + isFirst = false; + } + } + + private writePesPacket(trackData: MpegTsTrackData, queuedPacket: QueuedPacket) { + const pesView = this.pesHeaderView; + + setUint24(pesView, 0, 0x000001, false); // packet_start_code_prefix + this.pesHeaderBuffer[3] = trackData.streamId; // stream_id + + const pesPacketLength = trackData.track.type === 'video' + ? 0 // Unbounded + : Math.min(8 + queuedPacket.data.length, 0xFFFF); // Required for audio for some reason + pesView.setUint16(4, pesPacketLength, false); + + // '10' marker, PES_scrambling_control=0, PES_priority=0, + // data_alignment_indicator=1, copyright=0, original_or_copy=0 + pesView.setUint8(6, 0x84); + pesView.setUint8(7, 0x80); // PTS_DTS_flags=10 (PTS only), other flags=0 + pesView.setUint8(8, 5); // PES_header_data_length (5 bytes for PTS) + + const pts = Math.round(queuedPacket.timestamp * TIMESCALE); + this.ptsBitstream.pos = 0; + this.ptsBitstream.writeBits(4, 0b0010); // marker + this.ptsBitstream.writeBits(3, (pts >>> 30) & 0x7); // PTS[32:30] + this.ptsBitstream.writeBits(1, 1); // marker_bit + this.ptsBitstream.writeBits(15, (pts >>> 15) & 0x7FFF); // PTS[29:15] + this.ptsBitstream.writeBits(1, 1); // marker_bit + this.ptsBitstream.writeBits(15, pts & 0x7FFF); // PTS[14:0] + this.ptsBitstream.writeBits(1, 1); // marker_bit + + const totalLength = this.pesHeaderBuffer.length + queuedPacket.data.length; + let offset = 0; + let isFirst = true; + + while (offset < totalLength) { + const pusi = isFirst; + const remainingData = totalLength - offset; + + const needsRandomAccessIndicator = isFirst && queuedPacket.isKeyframe; + const basePaddingNeeded = Math.max(0, 184 - remainingData); + + let adaptationFieldSize: number; + if (needsRandomAccessIndicator) { + // Random access indicator requires at least 2 bytes + adaptationFieldSize = Math.max(2, basePaddingNeeded); + } else { + adaptationFieldSize = basePaddingNeeded; + } + + let adaptationField: Uint8Array | null = null; + if (adaptationFieldSize > 0) { + const buf = this.adaptationFieldBuffer; + + if (adaptationFieldSize === 1) { + buf[0] = 0; // adaptation_field_length + } else { + buf[0] = adaptationFieldSize - 1; // adaptation_field_length + buf[1] = Number(needsRandomAccessIndicator) << 6; // flags (random_access_indicator in bit 6) + buf.fill(0xFF, 2, adaptationFieldSize); // stuffing_bytes + } + + adaptationField = buf.subarray(0, adaptationFieldSize); + } + + const payloadSize = Math.min(184 - adaptationFieldSize, remainingData); + const payload = this.payloadBuffer.subarray(0, payloadSize); + + let payloadOffset = 0; + if (offset < this.pesHeaderBuffer.length) { + const headerBytes = Math.min(this.pesHeaderBuffer.length - offset, payloadSize); + payload.set(this.pesHeaderBuffer.subarray(offset, offset + headerBytes), 0); + payloadOffset = headerBytes; + } + + const dataStart = Math.max(0, offset - this.pesHeaderBuffer.length); + const dataEnd = dataStart + (payloadSize - payloadOffset); + if (payloadOffset < payloadSize) { + payload.set(queuedPacket.data.subarray(dataStart, dataEnd), payloadOffset); + } + + this.writeTsPacket(trackData.pid, pusi, adaptationField, payload); + + offset += payloadSize; + isFirst = false; + } + } + + private writeTsPacket( + pid: number, + pusi: boolean, + adaptationField: Uint8Array | null, + payload: Uint8Array, + ) { + const cc = this.continuityCounters.get(pid) ?? 0; + const hasPayload = payload.length > 0; + const adaptCtrl = adaptationField + ? (hasPayload ? 0b11 : 0b10) + : (hasPayload ? 0b01 : 0b00); + + this.packetBuffer[0] = 0x47; // sync_byte + this.packetView.setUint16(1, (pusi ? 0x4000 : 0) | (pid & 0x1FFF), false); // TEI=0, PUSI, priority=0, PID + // scrambling=0, adaptation_field_control, continuity_counter + this.packetBuffer[3] = (adaptCtrl << 4) | (cc & 0x0F); + + if (hasPayload) { + this.continuityCounters.set(pid, (cc + 1) & 0x0F); + } + + let offset = 4; + + if (adaptationField) { + this.packetBuffer.set(adaptationField, offset); + offset += adaptationField.length; + } + + this.packetBuffer.set(payload, offset); + offset += payload.length; + + if (offset < TS_PACKET_SIZE) { + this.packetBuffer.fill(0xFF, offset); // stuffing_bytes + } + + const startPos = this.writer.getPos(); + this.writer.write(this.packetBuffer); + + if (this.format._options.onPacket) { + this.format._options.onPacket(this.packetBuffer.slice(), startPos); + } + } + + // eslint-disable-next-line @typescript-eslint/no-misused-promises + override async onTrackClose() { + const release = await this.mutex.acquire(); + + if (this.allTracksAreKnown()) { + this.allTracksKnown.resolve(); + } + + await this.interleavePackets(); + + release(); + } + + async finalize() { + const release = await this.mutex.acquire(); + + this.allTracksKnown.resolve(); + + await this.interleavePackets(true); + + release(); + } +} + +// CRC-32 for MPEG-TS (polynomial 0x04C11DB7, initial value 0xFFFFFFFF) +const MPEG_TS_CRC_POLYNOMIAL = 0x04c11db7; +const MPEG_TS_CRC_TABLE = new Uint32Array(256); +for (let n = 0; n < 256; n++) { + let crc = n << 24; + + for (let k = 0; k < 8; k++) { + crc = (crc & 0x80000000) + ? ((crc << 1) ^ MPEG_TS_CRC_POLYNOMIAL) + : (crc << 1); + } + + MPEG_TS_CRC_TABLE[n] = (crc >>> 0) & 0xffffffff; +} + +const computeMpegTsCrc32 = (data: Uint8Array) => { + let crc = 0xFFFFFFFF; + for (let i = 0; i < data.length; i++) { + const byte = data[i]!; + crc = ((crc << 8) ^ MPEG_TS_CRC_TABLE[(crc >>> 24) ^ byte]!) >>> 0; + } + return crc; +}; + +const PAT_SECTION = new Uint8Array(16); +{ + const view = toDataView(PAT_SECTION); + PAT_SECTION[0] = 0x00; // table_id + view.setUint16(1, 0xB00D, false); // section_syntax_indicator=1, '0', reserved=11, section_length=13 + view.setUint16(3, 0x0001, false); // transport_stream_id + PAT_SECTION[5] = 0xC1; // reserved=11, version_number=0, current_next_indicator=1 + PAT_SECTION[6] = 0x00; // section_number + PAT_SECTION[7] = 0x00; // last_section_number + view.setUint16(8, 0x0001, false); // program_number + view.setUint16(10, 0xE000 | (PMT_PID & 0x1FFF), false); // reserved=111, program_map_PID + view.setUint32(12, computeMpegTsCrc32(PAT_SECTION.subarray(0, 12)), false); // CRC_32 +} + +const buildPmt = (trackDatas: MpegTsTrackData[]) => { + const sectionLength = 9 + trackDatas.length * 5 + 4; + const section = new Uint8Array(3 + sectionLength - 4); + const view = toDataView(section); + + section[0] = 0x02; // table_id + // section_syntax_indicator=1, '0', reserved=11, section_length + view.setUint16(1, 0xB000 | (sectionLength & 0x0FFF), false); + view.setUint16(3, 0x0001, false); // program_number + section[5] = 0xC1; // reserved=11, version_number=0, current_next_indicator=1 + section[6] = 0x00; // section_number + section[7] = 0x00; // last_section_number + view.setUint16(8, 0xE000 | 0x1FFF, false); // reserved=111, PCR_PID=0x1FFF (none) + view.setUint16(10, 0xF000, false); // reserved=1111, program_info_length=0 + + let offset = 12; + for (const trackData of trackDatas) { + section[offset++] = trackData.streamType; // stream_type + view.setUint16(offset, 0xE000 | (trackData.pid & 0x1FFF), false); // reserved=111, elementary_PID + offset += 2; + view.setUint16(offset, 0xF000, false); // reserved=1111, ES_info_length=0 + offset += 2; + } + + const crc = computeMpegTsCrc32(section); + const result = new Uint8Array(section.length + 4); + result.set(section, 0); + toDataView(result).setUint32(section.length, crc, false); // CRC_32 + + return result; +}; diff --git a/src/ogg/ogg-muxer.ts b/src/ogg/ogg-muxer.ts index 91ee535..5ab6caa 100644 --- a/src/ogg/ogg-muxer.ts +++ b/src/ogg/ogg-muxer.ts @@ -302,7 +302,7 @@ export class OggMuxer extends Muxer { async interleavePages(isFinalCall = false) { if (!this.bosPagesWritten) { - if (!this.allTracksAreKnown()) { + if (!this.allTracksAreKnown() && !isFinalCall) { return; // We can't interleave yet as we don't yet know how many tracks we'll truly have } diff --git a/src/output-format.ts b/src/output-format.ts index de50fe3..6a3143b 100644 --- a/src/output-format.ts +++ b/src/output-format.ts @@ -26,6 +26,7 @@ import { Mp3Muxer } from './mp3/mp3-muxer'; import { Muxer } from './muxer'; import { OggMuxer } from './ogg/ogg-muxer'; import { Output, TrackType } from './output'; +import { MpegTsMuxer } from './mpeg-ts/mpeg-ts-muxer'; import { WaveMuxer } from './wave/wave-muxer'; /** @@ -243,11 +244,13 @@ export abstract class IsobmffOutputFormat extends OutputFormat { } getSupportedTrackCounts(): TrackCountLimits { + const max = 2 ** 32 - 1; // Have fun reaching this one + return { - video: { min: 0, max: Infinity }, - audio: { min: 0, max: Infinity }, - subtitle: { min: 0, max: Infinity }, - total: { min: 1, max: 2 ** 32 - 1 }, // Have fun reaching this one + video: { min: 0, max }, + audio: { min: 0, max }, + subtitle: { min: 0, max }, + total: { min: 1, max }, }; } @@ -454,11 +457,13 @@ export class MkvOutputFormat extends OutputFormat { } getSupportedTrackCounts(): TrackCountLimits { + const max = 127; + return { - video: { min: 0, max: Infinity }, - audio: { min: 0, max: Infinity }, - subtitle: { min: 0, max: Infinity }, - total: { min: 1, max: 127 }, + video: { min: 0, max }, + audio: { min: 0, max }, + subtitle: { min: 0, max }, + total: { min: 1, max }, }; } @@ -769,11 +774,13 @@ export class OggOutputFormat extends OutputFormat { } getSupportedTrackCounts(): TrackCountLimits { + const max = 2 ** 32; // Have fun reaching this one + return { video: { min: 0, max: 0 }, - audio: { min: 0, max: Infinity }, + audio: { min: 0, max }, subtitle: { min: 0, max: 0 }, - total: { min: 1, max: 2 ** 32 }, + total: { min: 1, max }, }; } @@ -940,3 +947,84 @@ export class FlacOutputFormat extends OutputFormat { return false; } } + +/** + * MPEG-TS-specific output options. + * @group Output formats + * @public + */ +export type MpegTsOutputFormatOptions = { + /** + * Will be called for each 188-byte Transport Stream packet that is written. + * + * @param data - The raw bytes. + * @param position - The byte offset of the data in the file. + */ + onPacket?: (data: Uint8Array, position: number) => unknown; +}; + +/** + * MPEG Transport Stream file format. + * @group Output formats + * @public + */ +export class MpegTsOutputFormat extends OutputFormat { + /** @internal */ + _options: MpegTsOutputFormatOptions; + + /** Creates a new {@link MpegTsOutputFormat} configured with the specified `options`. */ + constructor(options: MpegTsOutputFormatOptions = {}) { + if (!options || typeof options !== 'object') { + throw new TypeError('options must be an object.'); + } + if (options.onPacket !== undefined && typeof options.onPacket !== 'function') { + throw new TypeError('options.onPacket, when provided, must be a function.'); + } + + super(); + + this._options = options; + } + + /** @internal */ + _createMuxer(output: Output) { + return new MpegTsMuxer(output, this); + } + + /** @internal */ + get _name() { + return 'MPEG-TS'; + } + + getSupportedTrackCounts(): TrackCountLimits { + const maxVideo = 16; // Stream IDs 0xE0-0xEF + const maxAudio = 32; + const maxTotal = maxVideo + maxAudio; + + return { + video: { min: 0, max: maxVideo }, + audio: { min: 0, max: maxAudio }, + subtitle: { min: 0, max: 0 }, + total: { min: 1, max: maxTotal }, + }; + } + + get fileExtension() { + return '.ts'; + } + + get mimeType() { + return 'video/MP2T'; + } + + getSupportedCodecs(): MediaCodec[] { + return [ + ...VIDEO_CODECS.filter(codec => ['avc', 'hevc'].includes(codec)), + ...AUDIO_CODECS.filter(codec => ['aac', 'mp3'].includes(codec)), + ]; + } + + get supportsVideoRotationMetadata() { + return false; + } +} diff --git a/test/browser/mpeg-ts-muxing.test.ts b/test/browser/mpeg-ts-muxing.test.ts new file mode 100644 index 0000000..e8d3f54 --- /dev/null +++ b/test/browser/mpeg-ts-muxing.test.ts @@ -0,0 +1,681 @@ +import { expect, test } from 'vitest'; +import { Input } from '../../src/input.js'; +import { BufferSource, UrlSource } from '../../src/source.js'; +import { ALL_FORMATS, MPEG_TS } from '../../src/input-format.js'; +import { Output } from '../../src/output.js'; +import { MpegTsOutputFormat } from '../../src/output-format.js'; +import { BufferTarget, StreamTarget, StreamTargetChunk } from '../../src/target.js'; +import { AudioBufferSource, CanvasSource, EncodedAudioPacketSource } from '../../src/media-source.js'; +import { QUALITY_HIGH } from '../../src/encode.js'; +import { EncodedPacketSink } from '../../src/media-sink.js'; +import { assert } from '../../src/misc.js'; +import { Conversion } from '../../src/conversion.js'; + +test('MPEG-TS output format', async () => { + const format = new MpegTsOutputFormat(); + expect(format.mimeType).toBe('video/MP2T'); + expect(format.fileExtension).toBe('.ts'); + expect(format.supportsVideoRotationMetadata).toBe(false); + expect(format.getSupportedCodecs()).toEqual(['avc', 'hevc', 'aac', 'mp3']); + expect(format.getSupportedTrackCounts()).toEqual({ + video: { min: 0, max: 16 }, + audio: { min: 0, max: 32 }, + subtitle: { min: 0, max: 0 }, + total: { min: 1, max: 48 }, + }); +}); + +test('MPEG-TS muxing with AVC and AAC', async () => { + let tsPacketCount = 0; + let started = false; + let finalized = false; + let mimeTypeResolved = false; + + const output = new Output({ + format: new MpegTsOutputFormat({ + onPacket: (data) => { + expect(data[0]).toBe(0x47); + tsPacketCount++; + }, + }), + target: new BufferTarget(), + }); + + // Test getMimeType - it resolves once all tracks are known (first packet arrives) + void output.getMimeType().then((mimeType) => { + expect(started).toBe(true); + expect(finalized).toBe(false); + expect(mimeType).toMatch(/^video\/MP2T; codecs="/); + expect(mimeType).toMatch(/avc1\./); + expect(mimeType).toMatch(/mp4a\.40\./); + + mimeTypeResolved = true; + }); + + const canvas = new OffscreenCanvas(640, 480); + const context = canvas.getContext('2d')!; + context.fillStyle = '#000000'; + context.fillRect(0, 0, canvas.width, canvas.height); + + const videoSource = new CanvasSource(canvas, { + codec: 'avc', + bitrate: QUALITY_HIGH, + }); + output.addVideoTrack(videoSource); + + const audioSource = new AudioBufferSource({ + codec: 'aac', + bitrate: QUALITY_HIGH, + }); + output.addAudioTrack(audioSource); + + await output.start(); + started = true; + + const fps = 30; + const duration = 5; + const frameCount = fps * duration; + const frameDuration = 1 / fps; + + for (let i = 0; i < frameCount; i++) { + await videoSource.add(i * frameDuration, frameDuration); + } + + const audioBuffer = new AudioBuffer({ + length: 48000 * duration, + numberOfChannels: 2, + sampleRate: 48000, + }); + await audioSource.add(audioBuffer); + + await output.finalize(); + finalized = true; + + expect(mimeTypeResolved).toBe(true); + expect(tsPacketCount).toBeGreaterThan(100); + + // Now let's read it back using the demuxer + using input = new Input({ + source: new BufferSource(output.target.buffer!), + formats: ALL_FORMATS, + }); + + expect(await input.getFormat()).toBe(MPEG_TS); + + // Verify video track + const videoTrack = await input.getPrimaryVideoTrack(); + assert(videoTrack); + + expect(videoTrack.id).toBe(0x100); + expect(videoTrack.codec).toBe('avc'); + expect(videoTrack.displayWidth).toBe(640); + expect(videoTrack.displayHeight).toBe(480); + + const videoDecoderConfig = await videoTrack.getDecoderConfig(); + assert(videoDecoderConfig); + expect(videoDecoderConfig.codec).toMatch(/^avc1\./); + expect(videoDecoderConfig.codedWidth).toBe(640); + expect(videoDecoderConfig.codedHeight).toBe(480); + expect(videoDecoderConfig.description).toBeUndefined(); // Annex B, no description + + // Verify audio track + const audioTrack = await input.getPrimaryAudioTrack(); + assert(audioTrack); + + expect(audioTrack.id).toBe(0x101); + expect(audioTrack.codec).toBe('aac'); + expect(audioTrack.numberOfChannels).toBe(2); + expect(audioTrack.sampleRate).toBe(48000); + + const audioDecoderConfig = await audioTrack.getDecoderConfig(); + assert(audioDecoderConfig); + expect(audioDecoderConfig.codec).toMatch(/^mp4a\.40\./); + expect(audioDecoderConfig.numberOfChannels).toBe(2); + expect(audioDecoderConfig.sampleRate).toBe(48000); + expect(audioDecoderConfig.description).toBeUndefined(); // ADTS, no description + + // Verify video packets are Annex B + const videoSink = new EncodedPacketSink(videoTrack); + let videoPacketCount = 0; + + const firstVideoPacket = await videoSink.getFirstPacket(); + assert(firstVideoPacket); + expect(firstVideoPacket.type).toBe('key'); + + const secondVideoPacket = await videoSink.getNextPacket(firstVideoPacket); + assert(secondVideoPacket); + expect(secondVideoPacket.type).toBe('delta'); + + for await (const packet of videoSink.packets()) { + expect(packet.data.slice(0, 4)).toEqual(new Uint8Array([0, 0, 0, 1])); // Annex B start code + videoPacketCount++; + } + + // Check that seeking works + const middlePacket = await videoSink.getPacket(2.5); + assert(middlePacket); + expect(middlePacket.timestamp).toBeCloseTo(2.5); + + expect(videoPacketCount).toBe(frameCount); + + // Verify audio packets are ADTS + const audioSink = new EncodedPacketSink(audioTrack); + let audioPacketCount = 0; + + const firstAudioPacket = await audioSink.getFirstPacket(); + assert(firstAudioPacket); + expect(firstAudioPacket.type).toBe('key'); + + const secondAudioPacket = await audioSink.getNextPacket(firstAudioPacket); + assert(secondAudioPacket); + expect(secondAudioPacket.type).toBe('key'); + + for await (const packet of audioSink.packets()) { + expect(packet.data[0]).toBe(0xff); // ADTS sync word + expect(packet.data[1]! & 0xf0).toBe(0xf0); // ADTS sync word continued + audioPacketCount++; + } + + // Check that seeking works + const audioMiddlePacket = await audioSink.getPacket(2.5); + assert(audioMiddlePacket); + expect(audioMiddlePacket.timestamp).toBeCloseTo(2.5); + + expect(audioPacketCount).toBeGreaterThan(0); + + // Verify duration is approximately 5 seconds + const videoDuration = await videoTrack.computeDuration(); + const audioDuration = await audioTrack.computeDuration(); + + expect(videoDuration).toBeCloseTo(5, 1); + expect(audioDuration).toBeCloseTo(5.077333333333334, 1); +}); + +test('MPEG-TS muxing with HEVC and MP3', async () => { + let tsPacketCount = 0; + const output = new Output({ + format: new MpegTsOutputFormat({ + onPacket: (data) => { + expect(data[0]).toBe(0x47); + tsPacketCount++; + }, + }), + target: new BufferTarget(), + }); + + const canvas = new OffscreenCanvas(1280, 720); + const context = canvas.getContext('2d')!; + context.fillStyle = '#000000'; + context.fillRect(0, 0, canvas.width, canvas.height); + + const videoSource = new CanvasSource(canvas, { + codec: 'hevc', + bitrate: QUALITY_HIGH, + }); + output.addVideoTrack(videoSource); + + const audioSource = new EncodedAudioPacketSource('mp3'); + output.addAudioTrack(audioSource); + + await output.start(); + + const fps = 30; + const duration = 5; + const frameCount = fps * duration; + const frameDuration = 1 / fps; + + for (let i = 0; i < frameCount; i++) { + await videoSource.add(i * frameDuration, frameDuration); + } + + // Extract MP3 packets from existing file + using mp3Input = new Input({ + source: new UrlSource('/Toothsome-Meme.VBRv2.mp3'), + formats: ALL_FORMATS, + }); + + const mp3Track = await mp3Input.getPrimaryAudioTrack(); + assert(mp3Track); + + const mp3Sink = new EncodedPacketSink(mp3Track); + + let isFirst = true; + for await (const packet of mp3Sink.packets()) { + if (packet.timestamp >= duration) break; + + await audioSource.add(packet, { + decoderConfig: isFirst + ? (await mp3Track.getDecoderConfig())! + : undefined, + }); + isFirst = false; + } + + await output.finalize(); + + expect(tsPacketCount).toBeGreaterThan(100); + + // Now let's read it back using the demuxer + using input = new Input({ + source: new BufferSource(output.target.buffer!), + formats: ALL_FORMATS, + }); + + expect(await input.getFormat()).toBe(MPEG_TS); + + // Verify video track + const videoTrack = await input.getPrimaryVideoTrack(); + assert(videoTrack); + + expect(videoTrack.id).toBe(0x100); + expect(videoTrack.codec).toBe('hevc'); + expect(videoTrack.displayWidth).toBe(1280); + expect(videoTrack.displayHeight).toBe(720); + + const videoDecoderConfig = await videoTrack.getDecoderConfig(); + assert(videoDecoderConfig); + expect(videoDecoderConfig.codec).toMatch(/^hev1\./); + expect(videoDecoderConfig.codedWidth).toBe(1280); + expect(videoDecoderConfig.codedHeight).toBe(720); + expect(videoDecoderConfig.description).toBeUndefined(); // Annex B, no description + + // Verify audio track + const audioTrack = await input.getPrimaryAudioTrack(); + assert(audioTrack); + + expect(audioTrack.id).toBe(0x101); + expect(audioTrack.codec).toBe('mp3'); + expect(audioTrack.numberOfChannels).toBe(2); + expect(audioTrack.sampleRate).toBe(24000); + + const audioDecoderConfig = await audioTrack.getDecoderConfig(); + assert(audioDecoderConfig); + expect(audioDecoderConfig.codec).toBe('mp3'); + expect(audioDecoderConfig.numberOfChannels).toBe(2); + expect(audioDecoderConfig.sampleRate).toBe(24000); + expect(audioDecoderConfig.description).toBeUndefined(); // MP3 has no description + + // Verify video packets are Annex B + const videoSink = new EncodedPacketSink(videoTrack); + let videoPacketCount = 0; + + for await (const packet of videoSink.packets()) { + expect(packet.data.slice(0, 4)).toEqual(new Uint8Array([0, 0, 0, 1])); // Annex B start code + videoPacketCount++; + } + + expect(videoPacketCount).toBe(frameCount); + + // Verify audio packets are MP3 frames + const audioSink = new EncodedPacketSink(audioTrack); + let audioPacketCount = 0; + + for await (const packet of audioSink.packets()) { + expect(packet.data[0]).toBe(0xff); // MP3 sync word + audioPacketCount++; + } + + expect(audioPacketCount).toBeGreaterThan(0); + + // Verify duration is approximately 5 seconds + const videoDuration = await videoTrack.computeDuration(); + const audioDuration = await audioTrack.computeDuration(); + + expect(videoDuration).toBeCloseTo(5, 1); + expect(audioDuration).toBeCloseTo(5, 1); +}); + +test('MPEG-TS muxing with no data', async () => { + const output = new Output({ + format: new MpegTsOutputFormat(), + target: new BufferTarget(), + }); + + const canvas = new OffscreenCanvas(640, 480); + const videoSource = new CanvasSource(canvas, { + codec: 'avc', + bitrate: QUALITY_HIGH, + }); + output.addVideoTrack(videoSource); + + await output.start(); + await output.finalize(); + + // Read it back - should have zero tracks since no packets were written + using input = new Input({ + source: new BufferSource(output.target.buffer!), + formats: ALL_FORMATS, + }); + + const tracks = await input.getTracks(); + expect(tracks.length).toBe(0); +}); + +test('MPEG-TS muxing with video only', async () => { + const output = new Output({ + format: new MpegTsOutputFormat(), + target: new BufferTarget(), + }); + + const canvas = new OffscreenCanvas(640, 480); + const context = canvas.getContext('2d')!; + context.fillStyle = '#ff0000'; + context.fillRect(0, 0, canvas.width, canvas.height); + + const videoSource = new CanvasSource(canvas, { + codec: 'avc', + bitrate: QUALITY_HIGH, + }); + output.addVideoTrack(videoSource); + + await output.start(); + + const fps = 30; + const duration = 1; + const frameCount = fps * duration; + const frameDuration = 1 / fps; + + for (let i = 0; i < frameCount; i++) { + await videoSource.add(i * frameDuration, frameDuration); + } + + await output.finalize(); + + using input = new Input({ + source: new BufferSource(output.target.buffer!), + formats: ALL_FORMATS, + }); + + expect(await input.getFormat()).toBe(MPEG_TS); + + const videoTrack = await input.getPrimaryVideoTrack(); + assert(videoTrack); + expect(videoTrack.codec).toBe('avc'); + + const audioTrack = await input.getPrimaryAudioTrack(); + expect(audioTrack).toBeNull(); + + const videoSink = new EncodedPacketSink(videoTrack); + let videoPacketCount = 0; + // eslint-disable-next-line @typescript-eslint/no-unused-vars + for await (const packet of videoSink.packets()) { + videoPacketCount++; + } + expect(videoPacketCount).toBe(frameCount); +}); + +test('MPEG-TS muxing with audio only', async () => { + const output = new Output({ + format: new MpegTsOutputFormat(), + target: new BufferTarget(), + }); + + const audioSource = new AudioBufferSource({ + codec: 'aac', + bitrate: QUALITY_HIGH, + }); + output.addAudioTrack(audioSource); + + await output.start(); + + const duration = 1; + const audioBuffer = new AudioBuffer({ + length: 48000 * duration, + numberOfChannels: 2, + sampleRate: 48000, + }); + await audioSource.add(audioBuffer); + + await output.finalize(); + + using input = new Input({ + source: new BufferSource(output.target.buffer!), + formats: ALL_FORMATS, + }); + + expect(await input.getFormat()).toBe(MPEG_TS); + + const videoTrack = await input.getPrimaryVideoTrack(); + expect(videoTrack).toBeNull(); + + const audioTrack = await input.getPrimaryAudioTrack(); + assert(audioTrack); + expect(audioTrack.codec).toBe('aac'); + + const audioSink = new EncodedPacketSink(audioTrack); + let audioPacketCount = 0; + // eslint-disable-next-line @typescript-eslint/no-unused-vars + for await (const packet of audioSink.packets()) { + audioPacketCount++; + } + expect(audioPacketCount).toBeGreaterThan(0); +}); + +test('MPEG-TS muxing with two video tracks', async () => { + const output = new Output({ + format: new MpegTsOutputFormat(), + target: new BufferTarget(), + }); + + const canvas1 = new OffscreenCanvas(640, 480); + const ctx1 = canvas1.getContext('2d')!; + ctx1.fillStyle = '#ff0000'; + ctx1.fillRect(0, 0, canvas1.width, canvas1.height); + + const canvas2 = new OffscreenCanvas(320, 240); + const ctx2 = canvas2.getContext('2d')!; + ctx2.fillStyle = '#00ff00'; + ctx2.fillRect(0, 0, canvas2.width, canvas2.height); + + const videoSource1 = new CanvasSource(canvas1, { codec: 'hevc', bitrate: QUALITY_HIGH }); + const videoSource2 = new CanvasSource(canvas2, { codec: 'hevc', bitrate: QUALITY_HIGH }); + + output.addVideoTrack(videoSource1); + output.addVideoTrack(videoSource2); + + await output.start(); + + const fps = 30; + const duration = 1; + const frameCount = fps * duration; + const frameDuration = 1 / fps; + + for (let i = 0; i < frameCount; i++) { + await videoSource1.add(i * frameDuration, frameDuration); + await videoSource2.add(i * frameDuration, frameDuration); + } + + await output.finalize(); + + using input = new Input({ + source: new BufferSource(output.target.buffer!), + formats: ALL_FORMATS, + }); + + expect(await input.getFormat()).toBe(MPEG_TS); + + const tracks = await input.getTracks(); + const videoTracks = tracks.filter(t => t.type === 'video'); + expect(videoTracks.length).toBe(2); + + expect(videoTracks[0]!.codec).toBe('hevc'); + expect(videoTracks[1]!.codec).toBe('hevc'); + expect(videoTracks[0]!.id).toBe(0x100); + expect(videoTracks[1]!.id).toBe(0x101); +}); + +test('MPEG-TS muxing with two audio tracks', async () => { + const output = new Output({ + format: new MpegTsOutputFormat(), + target: new BufferTarget(), + }); + + const audioSource1 = new AudioBufferSource({ codec: 'aac', bitrate: QUALITY_HIGH }); + const audioSource2 = new AudioBufferSource({ codec: 'aac', bitrate: QUALITY_HIGH }); + + output.addAudioTrack(audioSource1); + output.addAudioTrack(audioSource2); + + await output.start(); + + const duration = 1; + const audioBuffer1 = new AudioBuffer({ + length: 48000 * duration, + numberOfChannels: 2, + sampleRate: 48000, + }); + const audioBuffer2 = new AudioBuffer({ + length: 44100 * duration, + numberOfChannels: 1, + sampleRate: 44100, + }); + + await audioSource1.add(audioBuffer1); + await audioSource2.add(audioBuffer2); + + await output.finalize(); + + using input = new Input({ + source: new BufferSource(output.target.buffer!), + formats: ALL_FORMATS, + }); + + expect(await input.getFormat()).toBe(MPEG_TS); + + const tracks = await input.getTracks(); + const audioTracks = tracks.filter(t => t.type === 'audio'); + expect(audioTracks.length).toBe(2); + + expect(audioTracks[0]!.codec).toBe('aac'); + expect(audioTracks[1]!.codec).toBe('aac'); + expect(audioTracks[0]!.id).toBe(0x100); + expect(audioTracks[1]!.id).toBe(0x101); +}); + +test('MPEG-TS transmux (Annex B and ADTS passthrough)', async () => { + using input = new Input({ + source: new UrlSource('/0.ts'), + formats: ALL_FORMATS, + }); + + expect(await input.getFormat()).toBe(MPEG_TS); + + const output = new Output({ + format: new MpegTsOutputFormat(), + target: new BufferTarget(), + }); + + const conversion = await Conversion.init({ input, output }); + expect(conversion.isValid).toBe(true); + + await conversion.execute(); + + // Read the output back + using outputInput = new Input({ + source: new BufferSource(output.target.buffer!), + formats: ALL_FORMATS, + }); + + expect(await outputInput.getFormat()).toBe(MPEG_TS); + + const inputVideoTrack = await input.getPrimaryVideoTrack(); + const inputAudioTrack = await input.getPrimaryAudioTrack(); + const outputVideoTrack = await outputInput.getPrimaryVideoTrack(); + const outputAudioTrack = await outputInput.getPrimaryAudioTrack(); + + assert(inputVideoTrack); + assert(inputAudioTrack); + assert(outputVideoTrack); + assert(outputAudioTrack); + + // Codecs should match + expect(outputVideoTrack.codec).toBe(inputVideoTrack.codec); + expect(outputAudioTrack.codec).toBe(inputAudioTrack.codec); + + // Verify video packets are Annex B + const videoSink = new EncodedPacketSink(outputVideoTrack); + const firstVideoPacket = await videoSink.getFirstPacket(); + assert(firstVideoPacket); + expect(firstVideoPacket.data.slice(0, 4)).toEqual(new Uint8Array([0, 0, 0, 1])); + + // Verify audio packets are ADTS + const audioSink = new EncodedPacketSink(outputAudioTrack); + const firstAudioPacket = await audioSink.getFirstPacket(); + assert(firstAudioPacket); + expect(firstAudioPacket.data[0]).toBe(0xff); + expect(firstAudioPacket.data[1]! & 0xf0).toBe(0xf0); + + expect(await outputInput.getFirstTimestamp()).toBe(10.012); + expect(await outputInput.computeDuration()).toBe(15.004); +}); + +test('MPEG-TS muxing with StreamTarget', async () => { + let nextPos = 0; + const chunks: Uint8Array[] = []; + + const writable = new WritableStream({ + write(chunk) { + chunks.push(chunk.data); + expect(chunk.position).toBe(nextPos); + nextPos += chunk.data.byteLength; + }, + }); + + const output = new Output({ + format: new MpegTsOutputFormat(), + target: new StreamTarget(writable), + }); + + const canvas = new OffscreenCanvas(640, 480); + const context = canvas.getContext('2d')!; + context.fillStyle = '#0000ff'; + context.fillRect(0, 0, canvas.width, canvas.height); + + const videoSource = new CanvasSource(canvas, { + codec: 'avc', + bitrate: QUALITY_HIGH, + }); + output.addVideoTrack(videoSource); + + await output.start(); + + const fps = 30; + const duration = 1; + const frameCount = fps * duration; + const frameDuration = 1 / fps; + + for (let i = 0; i < frameCount; i++) { + await videoSource.add(i * frameDuration, frameDuration); + } + + await output.finalize(); + + expect(chunks.length).toBe(frameCount); + + const buffer = new Uint8Array(nextPos); + nextPos = 0; + for (const chunk of chunks) { + buffer.set(chunk, nextPos); + nextPos += chunk.byteLength; + } + + // Verify the concatenated output + using input = new Input({ + source: new BufferSource(buffer), + formats: ALL_FORMATS, + }); + + expect(await input.getFormat()).toBe(MPEG_TS); + + const videoTrack = await input.getPrimaryVideoTrack(); + assert(videoTrack); + expect(videoTrack.codec).toBe('avc'); + + const videoSink = new EncodedPacketSink(videoTrack); + let videoPacketCount = 0; + // eslint-disable-next-line @typescript-eslint/no-unused-vars + for await (const packet of videoSink.packets()) { + videoPacketCount++; + } + expect(videoPacketCount).toBe(frameCount); +}); diff --git a/test/node/mpeg-ts-demuxing.test.ts b/test/node/mpeg-ts-demuxing.test.ts index ee9678f..a46844d 100644 --- a/test/node/mpeg-ts-demuxing.test.ts +++ b/test/node/mpeg-ts-demuxing.test.ts @@ -12,6 +12,11 @@ import { MpegTsTrackBacking } from '../../src/mpeg-ts/mpeg-ts-demuxer.js'; const __dirname = new URL('.', import.meta.url).pathname; +test('MPEG-TS input format', async () => { + expect(MPEG_TS.mimeType).toBe('video/MP2T'); + expect(MPEG_TS.name).toBe('MPEG Transport Stream'); +}); + test('MPEG-TS metadata reading', async () => { using input = new Input({ source: new FilePathSource(path.join(__dirname, '../public/0.ts')), @@ -356,6 +361,11 @@ test('MPEG-TS video key packets', async () => { expect(firstKeyPacket.type).toBe('key'); expect(firstKeyPacket.sequenceNumber).toBe(firstPacket.sequenceNumber); + const afterKeyPacket = await sink.getNextPacket(firstKeyPacket); + expect(afterKeyPacket).not.toBe(null); + expect(afterKeyPacket!.type).toBe('delta'); + expect(afterKeyPacket!.sequenceNumber).toBe(secondPacket.sequenceNumber); + const secondKeyPacket = await sink.getKeyPacket(15); assert(secondKeyPacket); expect(secondKeyPacket.type).toBe('key'); @@ -407,6 +417,14 @@ test('MPEG-TS audio key packets', async () => { expect(nextKeyPacket.type).toBe('key'); expect(nextKeyPacket.sequenceNumber).toBe(secondPacket.sequenceNumber); // All audio packets are key packets + const middleKeyPacket = await sink.getKeyPacket(12.5); + assert(middleKeyPacket); + expect(middleKeyPacket.type).toBe('key'); + + const afterKeyPacket = await sink.getNextPacket(middleKeyPacket); + expect(afterKeyPacket).not.toBe(null); + expect(afterKeyPacket!.type).toBe('key'); + const lastPacket = await sink.getPacket(Infinity); assert(lastPacket); diff --git a/test/public/Toothsome-Meme.VBRv2.mp3 b/test/public/Toothsome-Meme.VBRv2.mp3 new file mode 100644 index 0000000..4a70e3a Binary files /dev/null and b/test/public/Toothsome-Meme.VBRv2.mp3 differ