diff --git a/src/adts/adts-demuxer.ts b/src/adts/adts-demuxer.ts index bb68f0b..8d4985a 100644 --- a/src/adts/adts-demuxer.ts +++ b/src/adts/adts-demuxer.ts @@ -21,7 +21,12 @@ import { import { EncodedPacket, PLACEHOLDER_DATA } from '../packet'; import { readBytes, Reader } from '../reader'; import { DEFAULT_TRACK_DISPOSITION } from '../metadata'; -import { AdtsFrameHeader, MIN_FRAME_HEADER_SIZE, MAX_FRAME_HEADER_SIZE, readAdtsFrameHeader } from './adts-reader'; +import { + AdtsFrameHeader, + MIN_ADTS_FRAME_HEADER_SIZE, + MAX_ADTS_FRAME_HEADER_SIZE, + readAdtsFrameHeader, +} from './adts-reader'; export const SAMPLES_PER_AAC_FRAME = 1024; @@ -68,7 +73,11 @@ export class AdtsDemuxer extends Demuxer { } async advanceReader() { - let slice = this.reader.requestSliceRange(this.lastLoadedPos, MIN_FRAME_HEADER_SIZE, MAX_FRAME_HEADER_SIZE); + let slice = this.reader.requestSliceRange( + this.lastLoadedPos, + MIN_ADTS_FRAME_HEADER_SIZE, + MAX_ADTS_FRAME_HEADER_SIZE, + ); if (slice instanceof Promise) slice = await slice; if (!slice) { this.lastSampleLoaded = true; diff --git a/src/adts/adts-reader.ts b/src/adts/adts-reader.ts index ff390de..c6803e9 100644 --- a/src/adts/adts-reader.ts +++ b/src/adts/adts-reader.ts @@ -9,8 +9,8 @@ import { Bitstream } from '../misc'; import { FileSlice, readBytes } from '../reader'; -export const MIN_FRAME_HEADER_SIZE = 7; -export const MAX_FRAME_HEADER_SIZE = 9; +export const MIN_ADTS_FRAME_HEADER_SIZE = 7; +export const MAX_ADTS_FRAME_HEADER_SIZE = 9; export type AdtsFrameHeader = { objectType: number; diff --git a/src/input-format.ts b/src/input-format.ts index 8004b08..5bf09fe 100644 --- a/src/input-format.ts +++ b/src/input-format.ts @@ -26,7 +26,7 @@ import { ID3_V2_HEADER_SIZE, readId3V2Header } from './id3'; import { readNextMp3FrameHeader } from './mp3/mp3-reader'; import { OggDemuxer } from './ogg/ogg-demuxer'; import { WaveDemuxer } from './wave/wave-demuxer'; -import { MAX_FRAME_HEADER_SIZE, MIN_FRAME_HEADER_SIZE, readAdtsFrameHeader } from './adts/adts-reader'; +import { MAX_ADTS_FRAME_HEADER_SIZE, MIN_ADTS_FRAME_HEADER_SIZE, readAdtsFrameHeader } from './adts/adts-reader'; import { AdtsDemuxer } from './adts/adts-demuxer'; import { readAscii, readBytes } from './reader'; import { FlacDemuxer } from './flac/flac-demuxer'; @@ -440,7 +440,11 @@ export class FlacInputFormat extends InputFormat { export class AdtsInputFormat extends InputFormat { /** @internal */ async _canReadInput(input: Input) { - let slice = input._reader.requestSliceRange(0, MIN_FRAME_HEADER_SIZE, MAX_FRAME_HEADER_SIZE); + let slice = input._reader.requestSliceRange( + 0, + MIN_ADTS_FRAME_HEADER_SIZE, + MAX_ADTS_FRAME_HEADER_SIZE, + ); if (slice instanceof Promise) slice = await slice; if (!slice) return false; @@ -449,7 +453,11 @@ export class AdtsInputFormat extends InputFormat { return false; } - slice = input._reader.requestSliceRange(firstHeader.frameLength, MIN_FRAME_HEADER_SIZE, MAX_FRAME_HEADER_SIZE); + slice = input._reader.requestSliceRange( + firstHeader.frameLength, + MIN_ADTS_FRAME_HEADER_SIZE, + MAX_ADTS_FRAME_HEADER_SIZE, + ); if (slice instanceof Promise) slice = await slice; if (!slice) return false; diff --git a/src/isobmff/isobmff-muxer.ts b/src/isobmff/isobmff-muxer.ts index 0a5dace..d78b8fe 100644 --- a/src/isobmff/isobmff-muxer.ts +++ b/src/isobmff/isobmff-muxer.ts @@ -14,6 +14,9 @@ import { assert, computeRationalApproximation, last, promiseWithResolvers } from import { IsobmffOutputFormatOptions, IsobmffOutputFormat, MovOutputFormat } from '../output-format'; import { inlineTimestampRegex, SubtitleConfig, SubtitleCue, SubtitleMetadata } from '../subtitles'; import { + aacChannelMap, + aacFrequencyTable, + buildAacAudioSpecificConfig, parsePcmCodec, PCM_AUDIO_CODECS, PcmAudioCodec, @@ -22,6 +25,8 @@ import { validateSubtitleMetadata, validateVideoChunkMetadata, } from '../codec'; +import { MAX_ADTS_FRAME_HEADER_SIZE, MIN_ADTS_FRAME_HEADER_SIZE, readAdtsFrameHeader } from '../adts/adts-reader'; +import { FileSlice } from '../reader'; import { BufferTarget } from '../target'; import { EncodedPacket, PacketType } from '../packet'; import { @@ -101,6 +106,11 @@ export type IsobmffTrackData = { * Some players expect this for PCM audio. */ requiresPcmTransformation: boolean; + /** + * The "ADTS stripping" involves removing the ADTS header from each AAC packet. SOBMFF stores raw AAC data, not + * ADTS-wrapped data. + */ + requiresAdtsStripping: boolean; }; } | { track: OutputSubtitleTrack; @@ -365,7 +375,7 @@ export class IsobmffMuxer extends Muxer { return newTrackData; } - private getAudioTrackData(track: OutputAudioTrack, meta?: EncodedAudioChunkMetadata) { + private getAudioTrackData(track: OutputAudioTrack, packet: EncodedPacket, meta?: EncodedAudioChunkMetadata) { const existingTrackData = this.trackDatas.find(x => x.track === track); if (existingTrackData) { return existingTrackData as IsobmffAudioTrackData; @@ -376,6 +386,37 @@ export class IsobmffMuxer extends Muxer { assert(meta); assert(meta.decoderConfig); + const decoderConfig = { ...meta.decoderConfig }; + let requiresAdtsStripping = false; + + if (track.source._codec === 'aac' && !decoderConfig.description) { + // ISOBMFF can only hold AAC in raw format, not ADTS, but the missing description indicates ADTS. + // Parse the first packet to extract the AudioSpecificConfig. + const adtsFrame = readAdtsFrameHeader(FileSlice.tempFromBytes(packet.data)); + if (!adtsFrame) { + throw new Error( + 'Couldn\'t parse ADTS header from the AAC packet. Make sure the packets are in ADTS format' + + ' (as specified in ISO 13818-7) when not providing a description, or provide a description' + + ' (must be an AudioSpecificConfig as specified in ISO 14496-3) and ensure the packets' + + ' are raw AAC data.', + ); + } + + const sampleRate = aacFrequencyTable[adtsFrame.samplingFrequencyIndex]; + const numberOfChannels = aacChannelMap[adtsFrame.channelConfiguration]; + + if (sampleRate === undefined || numberOfChannels === undefined) { + throw new Error('Invalid ADTS frame header.'); + } + + decoderConfig.description = buildAacAudioSpecificConfig({ + objectType: adtsFrame.objectType, + sampleRate, + numberOfChannels, + }); + requiresAdtsStripping = true; + } + const newTrackData: IsobmffAudioTrackData = { muxer: this, track, @@ -383,10 +424,11 @@ export class IsobmffMuxer extends Muxer { info: { numberOfChannels: meta.decoderConfig.numberOfChannels, sampleRate: meta.decoderConfig.sampleRate, - decoderConfig: meta.decoderConfig, + decoderConfig, requiresPcmTransformation: !this.isFragmented && (PCM_AUDIO_CODECS as readonly string[]).includes(track.source._codec), + requiresAdtsStripping, }, timescale: meta.decoderConfig.sampleRate, samples: [], @@ -503,7 +545,20 @@ export class IsobmffMuxer extends Muxer { const release = await this.mutex.acquire(); try { - const trackData = this.getAudioTrackData(track, meta); + const trackData = this.getAudioTrackData(track, packet, meta); + + let packetData = packet.data; + if (trackData.info.requiresAdtsStripping) { + const adtsFrame = readAdtsFrameHeader(FileSlice.tempFromBytes(packetData)); + if (!adtsFrame) { + throw new Error('Expected ADTS frame, didn\'t get one.'); + } + + const headerLength = adtsFrame.crcCheck === null + ? MIN_ADTS_FRAME_HEADER_SIZE + : MAX_ADTS_FRAME_HEADER_SIZE; + packetData = packetData.subarray(headerLength); + } const timestamp = this.validateAndNormalizeTimestamp( trackData.track, @@ -512,7 +567,7 @@ export class IsobmffMuxer extends Muxer { ); const internalSample = this.createSampleForTrack( trackData, - packet.data, + packetData, timestamp, packet.duration, packet.type, diff --git a/src/matroska/matroska-muxer.ts b/src/matroska/matroska-muxer.ts index 5bb3646..d4a73af 100644 --- a/src/matroska/matroska-muxer.ts +++ b/src/matroska/matroska-muxer.ts @@ -48,6 +48,9 @@ import { parseSubtitleTimestamp, } from '../subtitles'; import { + aacChannelMap, + aacFrequencyTable, + buildAacAudioSpecificConfig, OPUS_SAMPLE_RATE, PCM_AUDIO_CODECS, PcmAudioCodec, @@ -59,6 +62,8 @@ import { validateSubtitleMetadata, validateVideoChunkMetadata, } from '../codec'; +import { MAX_ADTS_FRAME_HEADER_SIZE, MIN_ADTS_FRAME_HEADER_SIZE, readAdtsFrameHeader } from '../adts/adts-reader'; +import { FileSlice } from '../reader'; import { Muxer } from '../muxer'; import { Writer } from '../writer'; import { EncodedPacket } from '../packet'; @@ -98,6 +103,11 @@ type MatroskaTrackData = { numberOfChannels: number; sampleRate: number; decoderConfig: AudioDecoderConfig; + /** + * The "ADTS stripping" involves removing the ADTS header from each AAC packet. SOBMFF stores raw AAC data, not + * ADTS-wrapped data. + */ + requiresAdtsStripping: boolean; }; } | { track: OutputSubtitleTrack; @@ -772,7 +782,7 @@ export class MatroskaMuxer extends Muxer { return newTrackData; } - private getAudioTrackData(track: OutputAudioTrack, meta?: EncodedAudioChunkMetadata) { + private getAudioTrackData(track: OutputAudioTrack, packet: EncodedPacket, meta?: EncodedAudioChunkMetadata) { const existingTrackData = this.trackDatas.find(x => x.track === track); if (existingTrackData) { return existingTrackData as MatroskaAudioTrackData; @@ -783,13 +793,45 @@ export class MatroskaMuxer extends Muxer { assert(meta); assert(meta.decoderConfig); + const decoderConfig = { ...meta.decoderConfig }; + let requiresAdtsStripping = false; + + if (track.source._codec === 'aac' && !decoderConfig.description) { + // Matroska stores raw AAC with AudioSpecificConfig in CodecPrivate, not ADTS-wrapped data. + // Parse the first packet to extract the AudioSpecificConfig. + const adtsFrame = readAdtsFrameHeader(FileSlice.tempFromBytes(packet.data)); + if (!adtsFrame) { + throw new Error( + 'Couldn\'t parse ADTS header from the AAC packet. Make sure the packets are in ADTS format' + + ' (as specified in ISO 13818-7) when not providing a description, or provide a description' + + ' (must be an AudioSpecificConfig as specified in ISO 14496-3) and ensure the packets' + + ' are raw AAC data.', + ); + } + + const sampleRate = aacFrequencyTable[adtsFrame.samplingFrequencyIndex]; + const numberOfChannels = aacChannelMap[adtsFrame.channelConfiguration]; + + if (sampleRate === undefined || numberOfChannels === undefined) { + throw new Error('Invalid ADTS frame header.'); + } + + decoderConfig.description = buildAacAudioSpecificConfig({ + objectType: adtsFrame.objectType, + sampleRate, + numberOfChannels, + }); + requiresAdtsStripping = true; + } + const newTrackData: MatroskaAudioTrackData = { track, type: 'audio', info: { numberOfChannels: meta.decoderConfig.numberOfChannels, sampleRate: meta.decoderConfig.sampleRate, - decoderConfig: meta.decoderConfig, + decoderConfig, + requiresAdtsStripping, }, chunkQueue: [], lastWrittenMsTimestamp: null, @@ -870,11 +912,24 @@ export class MatroskaMuxer extends Muxer { const release = await this.mutex.acquire(); try { - const trackData = this.getAudioTrackData(track, meta); + const trackData = this.getAudioTrackData(track, packet, meta); + + let packetData = packet.data; + if (trackData.info.requiresAdtsStripping) { + const adtsFrame = readAdtsFrameHeader(FileSlice.tempFromBytes(packetData)); + if (!adtsFrame) { + throw new Error('Expected ADTS frame, didn\'t get one.'); + } + + const headerLength = adtsFrame.crcCheck === null + ? MIN_ADTS_FRAME_HEADER_SIZE + : MAX_ADTS_FRAME_HEADER_SIZE; + packetData = packetData.subarray(headerLength); + } const isKeyFrame = packet.type === 'key'; const timestamp = this.validateAndNormalizeTimestamp(trackData.track, packet.timestamp, isKeyFrame); - const audioChunk = this.createInternalChunk(packet.data, timestamp, packet.duration, packet.type); + const audioChunk = this.createInternalChunk(packetData, timestamp, packet.duration, packet.type); trackData.chunkQueue.push(audioChunk); await this.interleaveChunks(); diff --git a/src/mpeg-ts/mpeg-ts-demuxer.ts b/src/mpeg-ts/mpeg-ts-demuxer.ts index 56a4964..4c4de76 100644 --- a/src/mpeg-ts/mpeg-ts-demuxer.ts +++ b/src/mpeg-ts/mpeg-ts-demuxer.ts @@ -7,7 +7,7 @@ */ import { SAMPLES_PER_AAC_FRAME } from '../adts/adts-demuxer'; -import { MAX_FRAME_HEADER_SIZE, readAdtsFrameHeader } from '../adts/adts-reader'; +import { MAX_ADTS_FRAME_HEADER_SIZE, readAdtsFrameHeader } from '../adts/adts-reader'; import { aacChannelMap, AacCodecInfo, @@ -1468,14 +1468,14 @@ class MpegTsAudioTrackBacking extends MpegTsTrackBacking implements InputAudioTr context.skip(-1); const possibleHeaderStartPos = context.currentPos; - let remaining = context.ensureBuffered(MAX_FRAME_HEADER_SIZE); + let remaining = context.ensureBuffered(MAX_ADTS_FRAME_HEADER_SIZE); if (remaining instanceof Promise) remaining = await remaining; - if (remaining < MAX_FRAME_HEADER_SIZE) { + if (remaining < MAX_ADTS_FRAME_HEADER_SIZE) { return; } - const headerBytes = context.readBytes(MAX_FRAME_HEADER_SIZE); + const headerBytes = context.readBytes(MAX_ADTS_FRAME_HEADER_SIZE); const header = readAdtsFrameHeader(FileSlice.tempFromBytes(headerBytes)); if (header) { diff --git a/test/node/isobmff-muxer.test.ts b/test/node/isobmff-muxer.test.ts new file mode 100644 index 0000000..55cee50 --- /dev/null +++ b/test/node/isobmff-muxer.test.ts @@ -0,0 +1,63 @@ +import { expect, test } from 'vitest'; +import path from 'node:path'; +import { Input } from '../../src/input.js'; +import { BufferSource, FilePathSource } from '../../src/source.js'; +import { ADTS, ALL_FORMATS } from '../../src/input-format.js'; +import { EncodedPacketSink } from '../../src/media-sink.js'; +import { Output } from '../../src/output.js'; +import { BufferTarget } from '../../src/target.js'; +import { Mp4OutputFormat } from '../../src/output-format.js'; +import { Conversion } from '../../src/conversion.js'; +import { assert } from '../../src/misc.js'; + +const __dirname = new URL('.', import.meta.url).pathname; + +test('ISOBMFF muxer internally converts ADTS to AAC', async () => { + using input = new Input({ + source: new FilePathSource(path.join(__dirname, '../public/sample3.aac')), + formats: ALL_FORMATS, + }); + + expect(await input.getFormat()).toBe(ADTS); + + const inputTrack = await input.getPrimaryAudioTrack(); + assert(inputTrack); + + const inputDecoderConfig = await inputTrack.getDecoderConfig(); + expect(inputDecoderConfig!.description).toBeUndefined(); // ADTS input has no description + + const output = new Output({ + format: new Mp4OutputFormat(), + target: new BufferTarget(), + }); + + const conversion = await Conversion.init({ input, output, showWarnings: false }); + await conversion.execute(); + + using outputAsInput = new Input({ + source: new BufferSource(output.target.buffer!), + formats: ALL_FORMATS, + }); + + const outputTrack = await outputAsInput.getPrimaryAudioTrack(); + assert(outputTrack); + + expect(outputTrack.codec).toBe('aac'); + expect(outputTrack.sampleRate).toBe(inputTrack.sampleRate); + expect(outputTrack.numberOfChannels).toBe(inputTrack.numberOfChannels); + + const outputDecoderConfig = await outputTrack.getDecoderConfig(); + expect(outputDecoderConfig!.description).toBeDefined(); + + const outputSink = new EncodedPacketSink(outputTrack); + + let count = 0; + for await (const packet of outputSink.packets()) { + // Packets should NOT be ADTS frames (should not start with 0xFFF sync word) + const isAdts = packet.data[0] === 0xff && (packet.data[1]! & 0xf0) === 0xf0; + expect(isAdts).toBe(false); + count++; + } + + expect(count).toBe(4557); +}); diff --git a/test/node/matroska-muxer.test.ts b/test/node/matroska-muxer.test.ts new file mode 100644 index 0000000..f42f3c3 --- /dev/null +++ b/test/node/matroska-muxer.test.ts @@ -0,0 +1,63 @@ +import { expect, test } from 'vitest'; +import path from 'node:path'; +import { Input } from '../../src/input.js'; +import { BufferSource, FilePathSource } from '../../src/source.js'; +import { ADTS, ALL_FORMATS } from '../../src/input-format.js'; +import { EncodedPacketSink } from '../../src/media-sink.js'; +import { Output } from '../../src/output.js'; +import { BufferTarget } from '../../src/target.js'; +import { MkvOutputFormat } from '../../src/output-format.js'; +import { Conversion } from '../../src/conversion.js'; +import { assert } from '../../src/misc.js'; + +const __dirname = new URL('.', import.meta.url).pathname; + +test('Matroska muxer internally converts ADTS to AAC', async () => { + using input = new Input({ + source: new FilePathSource(path.join(__dirname, '../public/sample3.aac')), + formats: ALL_FORMATS, + }); + + expect(await input.getFormat()).toBe(ADTS); + + const inputTrack = await input.getPrimaryAudioTrack(); + assert(inputTrack); + + const inputDecoderConfig = await inputTrack.getDecoderConfig(); + expect(inputDecoderConfig!.description).toBeUndefined(); // ADTS input has no description + + const output = new Output({ + format: new MkvOutputFormat(), + target: new BufferTarget(), + }); + + const conversion = await Conversion.init({ input, output, showWarnings: false }); + await conversion.execute(); + + using outputAsInput = new Input({ + source: new BufferSource(output.target.buffer!), + formats: ALL_FORMATS, + }); + + const outputTrack = await outputAsInput.getPrimaryAudioTrack(); + assert(outputTrack); + + expect(outputTrack.codec).toBe('aac'); + expect(outputTrack.sampleRate).toBe(inputTrack.sampleRate); + expect(outputTrack.numberOfChannels).toBe(inputTrack.numberOfChannels); + + const outputDecoderConfig = await outputTrack.getDecoderConfig(); + expect(outputDecoderConfig!.description).toBeDefined(); + + const outputSink = new EncodedPacketSink(outputTrack); + + let count = 0; + for await (const packet of outputSink.packets()) { + // Packets should NOT be ADTS frames (should not start with 0xFFF sync word) + const isAdts = packet.data[0] === 0xff && (packet.data[1]! & 0xf0) === 0xf0; + expect(isAdts).toBe(false); + count++; + } + + expect(count).toBe(4557); +});