Convert ADTS to AAC in muxers that accept it

src/adts/adts-demuxer.ts

@@ -21,7 +21,12 @@ import {
 import { EncodedPacket, PLACEHOLDER_DATA } from '../packet';
 import { readBytes, Reader } from '../reader';
 import { DEFAULT_TRACK_DISPOSITION } from '../metadata';
-import { AdtsFrameHeader, MIN_FRAME_HEADER_SIZE, MAX_FRAME_HEADER_SIZE, readAdtsFrameHeader } from './adts-reader';
+import {
+	AdtsFrameHeader,
+	MIN_ADTS_FRAME_HEADER_SIZE,
+	MAX_ADTS_FRAME_HEADER_SIZE,
+	readAdtsFrameHeader,
+} from './adts-reader';
 
 export const SAMPLES_PER_AAC_FRAME = 1024;
 
@@ -68,7 +73,11 @@ export class AdtsDemuxer extends Demuxer {
 	}
 
 	async advanceReader() {
-		let slice = this.reader.requestSliceRange(this.lastLoadedPos, MIN_FRAME_HEADER_SIZE, MAX_FRAME_HEADER_SIZE);
+		let slice = this.reader.requestSliceRange(
+			this.lastLoadedPos,
+			MIN_ADTS_FRAME_HEADER_SIZE,
+			MAX_ADTS_FRAME_HEADER_SIZE,
+		);
 		if (slice instanceof Promise) slice = await slice;
 		if (!slice) {
 			this.lastSampleLoaded = true;

src/adts/adts-reader.ts

@@ -9,8 +9,8 @@
 import { Bitstream } from '../misc';
 import { FileSlice, readBytes } from '../reader';
 
-export const MIN_FRAME_HEADER_SIZE = 7;
+export const MIN_ADTS_FRAME_HEADER_SIZE = 7;
-export const MAX_FRAME_HEADER_SIZE = 9;
+export const MAX_ADTS_FRAME_HEADER_SIZE = 9;
 
 export type AdtsFrameHeader = {
 	objectType: number;

src/input-format.ts

@@ -26,7 +26,7 @@ import { ID3_V2_HEADER_SIZE, readId3V2Header } from './id3';
 import { readNextMp3FrameHeader } from './mp3/mp3-reader';
 import { OggDemuxer } from './ogg/ogg-demuxer';
 import { WaveDemuxer } from './wave/wave-demuxer';
-import { MAX_FRAME_HEADER_SIZE, MIN_FRAME_HEADER_SIZE, readAdtsFrameHeader } from './adts/adts-reader';
+import { MAX_ADTS_FRAME_HEADER_SIZE, MIN_ADTS_FRAME_HEADER_SIZE, readAdtsFrameHeader } from './adts/adts-reader';
 import { AdtsDemuxer } from './adts/adts-demuxer';
 import { readAscii, readBytes } from './reader';
 import { FlacDemuxer } from './flac/flac-demuxer';
@@ -440,7 +440,11 @@ export class FlacInputFormat extends InputFormat {
 export class AdtsInputFormat extends InputFormat {
 	/** @internal */
 	async _canReadInput(input: Input) {
-		let slice = input._reader.requestSliceRange(0, MIN_FRAME_HEADER_SIZE, MAX_FRAME_HEADER_SIZE);
+		let slice = input._reader.requestSliceRange(
+			0,
+			MIN_ADTS_FRAME_HEADER_SIZE,
+			MAX_ADTS_FRAME_HEADER_SIZE,
+		);
 		if (slice instanceof Promise) slice = await slice;
 		if (!slice) return false;
 
@@ -449,7 +453,11 @@ export class AdtsInputFormat extends InputFormat {
 			return false;
 		}
 
-		slice = input._reader.requestSliceRange(firstHeader.frameLength, MIN_FRAME_HEADER_SIZE, MAX_FRAME_HEADER_SIZE);
+		slice = input._reader.requestSliceRange(
+			firstHeader.frameLength,
+			MIN_ADTS_FRAME_HEADER_SIZE,
+			MAX_ADTS_FRAME_HEADER_SIZE,
+		);
 		if (slice instanceof Promise) slice = await slice;
 		if (!slice) return false;
 

src/isobmff/isobmff-muxer.ts

@@ -14,6 +14,9 @@ import { assert, computeRationalApproximation, last, promiseWithResolvers } from
 import { IsobmffOutputFormatOptions, IsobmffOutputFormat, MovOutputFormat } from '../output-format';
 import { inlineTimestampRegex, SubtitleConfig, SubtitleCue, SubtitleMetadata } from '../subtitles';
 import {
+	aacChannelMap,
+	aacFrequencyTable,
+	buildAacAudioSpecificConfig,
 	parsePcmCodec,
 	PCM_AUDIO_CODECS,
 	PcmAudioCodec,
@@ -22,6 +25,8 @@ import {
 	validateSubtitleMetadata,
 	validateVideoChunkMetadata,
 } from '../codec';
+import { MAX_ADTS_FRAME_HEADER_SIZE, MIN_ADTS_FRAME_HEADER_SIZE, readAdtsFrameHeader } from '../adts/adts-reader';
+import { FileSlice } from '../reader';
 import { BufferTarget } from '../target';
 import { EncodedPacket, PacketType } from '../packet';
 import {
@@ -101,6 +106,11 @@ export type IsobmffTrackData = {
 		 * Some players expect this for PCM audio.
 		 */
 		requiresPcmTransformation: boolean;
+		/**
+		 * The "ADTS stripping" involves removing the ADTS header from each AAC packet. SOBMFF stores raw AAC data, not
+		 * ADTS-wrapped data.
+		 */
+		requiresAdtsStripping: boolean;
 	};
 } | {
 	track: OutputSubtitleTrack;
@@ -365,7 +375,7 @@ export class IsobmffMuxer extends Muxer {
 		return newTrackData;
 	}
 
-	private getAudioTrackData(track: OutputAudioTrack, meta?: EncodedAudioChunkMetadata) {
+	private getAudioTrackData(track: OutputAudioTrack, packet: EncodedPacket, meta?: EncodedAudioChunkMetadata) {
 		const existingTrackData = this.trackDatas.find(x => x.track === track);
 		if (existingTrackData) {
 			return existingTrackData as IsobmffAudioTrackData;
@@ -376,6 +386,37 @@ export class IsobmffMuxer extends Muxer {
 		assert(meta);
 		assert(meta.decoderConfig);
 
+		const decoderConfig = { ...meta.decoderConfig };
+		let requiresAdtsStripping = false;
+
+		if (track.source._codec === 'aac' && !decoderConfig.description) {
+			// ISOBMFF can only hold AAC in raw format, not ADTS, but the missing description indicates ADTS.
+			// Parse the first packet to extract the AudioSpecificConfig.
+			const adtsFrame = readAdtsFrameHeader(FileSlice.tempFromBytes(packet.data));
+			if (!adtsFrame) {
+				throw new Error(
+					'Couldn\'t parse ADTS header from the AAC packet. Make sure the packets are in ADTS format'
+					+ ' (as specified in ISO 13818-7) when not providing a description, or provide a description'
+					+ ' (must be an AudioSpecificConfig as specified in ISO 14496-3) and ensure the packets'
+					+ ' are raw AAC data.',
+				);
+			}
+
+			const sampleRate = aacFrequencyTable[adtsFrame.samplingFrequencyIndex];
+			const numberOfChannels = aacChannelMap[adtsFrame.channelConfiguration];
+
+			if (sampleRate === undefined || numberOfChannels === undefined) {
+				throw new Error('Invalid ADTS frame header.');
+			}
+
+			decoderConfig.description = buildAacAudioSpecificConfig({
+				objectType: adtsFrame.objectType,
+				sampleRate,
+				numberOfChannels,
+			});
+			requiresAdtsStripping = true;
+		}
+
 		const newTrackData: IsobmffAudioTrackData = {
 			muxer: this,
 			track,
@@ -383,10 +424,11 @@ export class IsobmffMuxer extends Muxer {
 			info: {
 				numberOfChannels: meta.decoderConfig.numberOfChannels,
 				sampleRate: meta.decoderConfig.sampleRate,
-				decoderConfig: meta.decoderConfig,
+				decoderConfig,
 				requiresPcmTransformation:
 					!this.isFragmented
 					&& (PCM_AUDIO_CODECS as readonly string[]).includes(track.source._codec),
+				requiresAdtsStripping,
 			},
 			timescale: meta.decoderConfig.sampleRate,
 			samples: [],
@@ -503,7 +545,20 @@ export class IsobmffMuxer extends Muxer {
 		const release = await this.mutex.acquire();
 
 		try {
-			const trackData = this.getAudioTrackData(track, meta);
+			const trackData = this.getAudioTrackData(track, packet, meta);
+
+			let packetData = packet.data;
+			if (trackData.info.requiresAdtsStripping) {
+				const adtsFrame = readAdtsFrameHeader(FileSlice.tempFromBytes(packetData));
+				if (!adtsFrame) {
+					throw new Error('Expected ADTS frame, didn\'t get one.');
+				}
+
+				const headerLength = adtsFrame.crcCheck === null
+					? MIN_ADTS_FRAME_HEADER_SIZE
+					: MAX_ADTS_FRAME_HEADER_SIZE;
+				packetData = packetData.subarray(headerLength);
+			}
 
 			const timestamp = this.validateAndNormalizeTimestamp(
 				trackData.track,
@@ -512,7 +567,7 @@ export class IsobmffMuxer extends Muxer {
 			);
 			const internalSample = this.createSampleForTrack(
 				trackData,
-				packet.data,
+				packetData,
 				timestamp,
 				packet.duration,
 				packet.type,

src/matroska/matroska-muxer.ts

@@ -48,6 +48,9 @@ import {
 	parseSubtitleTimestamp,
 } from '../subtitles';
 import {
+	aacChannelMap,
+	aacFrequencyTable,
+	buildAacAudioSpecificConfig,
 	OPUS_SAMPLE_RATE,
 	PCM_AUDIO_CODECS,
 	PcmAudioCodec,
@@ -59,6 +62,8 @@ import {
 	validateSubtitleMetadata,
 	validateVideoChunkMetadata,
 } from '../codec';
+import { MAX_ADTS_FRAME_HEADER_SIZE, MIN_ADTS_FRAME_HEADER_SIZE, readAdtsFrameHeader } from '../adts/adts-reader';
+import { FileSlice } from '../reader';
 import { Muxer } from '../muxer';
 import { Writer } from '../writer';
 import { EncodedPacket } from '../packet';
@@ -98,6 +103,11 @@ type MatroskaTrackData = {
 		numberOfChannels: number;
 		sampleRate: number;
 		decoderConfig: AudioDecoderConfig;
+		/**
+		 * The "ADTS stripping" involves removing the ADTS header from each AAC packet. SOBMFF stores raw AAC data, not
+		 * ADTS-wrapped data.
+		 */
+		requiresAdtsStripping: boolean;
 	};
 } | {
 	track: OutputSubtitleTrack;
@@ -772,7 +782,7 @@ export class MatroskaMuxer extends Muxer {
 		return newTrackData;
 	}
 
-	private getAudioTrackData(track: OutputAudioTrack, meta?: EncodedAudioChunkMetadata) {
+	private getAudioTrackData(track: OutputAudioTrack, packet: EncodedPacket, meta?: EncodedAudioChunkMetadata) {
 		const existingTrackData = this.trackDatas.find(x => x.track === track);
 		if (existingTrackData) {
 			return existingTrackData as MatroskaAudioTrackData;
@@ -783,13 +793,45 @@ export class MatroskaMuxer extends Muxer {
 		assert(meta);
 		assert(meta.decoderConfig);
 
+		const decoderConfig = { ...meta.decoderConfig };
+		let requiresAdtsStripping = false;
+
+		if (track.source._codec === 'aac' && !decoderConfig.description) {
+			// Matroska stores raw AAC with AudioSpecificConfig in CodecPrivate, not ADTS-wrapped data.
+			// Parse the first packet to extract the AudioSpecificConfig.
+			const adtsFrame = readAdtsFrameHeader(FileSlice.tempFromBytes(packet.data));
+			if (!adtsFrame) {
+				throw new Error(
+					'Couldn\'t parse ADTS header from the AAC packet. Make sure the packets are in ADTS format'
+					+ ' (as specified in ISO 13818-7) when not providing a description, or provide a description'
+					+ ' (must be an AudioSpecificConfig as specified in ISO 14496-3) and ensure the packets'
+					+ ' are raw AAC data.',
+				);
+			}
+
+			const sampleRate = aacFrequencyTable[adtsFrame.samplingFrequencyIndex];
+			const numberOfChannels = aacChannelMap[adtsFrame.channelConfiguration];
+
+			if (sampleRate === undefined || numberOfChannels === undefined) {
+				throw new Error('Invalid ADTS frame header.');
+			}
+
+			decoderConfig.description = buildAacAudioSpecificConfig({
+				objectType: adtsFrame.objectType,
+				sampleRate,
+				numberOfChannels,
+			});
+			requiresAdtsStripping = true;
+		}
+
 		const newTrackData: MatroskaAudioTrackData = {
 			track,
 			type: 'audio',
 			info: {
 				numberOfChannels: meta.decoderConfig.numberOfChannels,
 				sampleRate: meta.decoderConfig.sampleRate,
-				decoderConfig: meta.decoderConfig,
+				decoderConfig,
+				requiresAdtsStripping,
 			},
 			chunkQueue: [],
 			lastWrittenMsTimestamp: null,
@@ -870,11 +912,24 @@ export class MatroskaMuxer extends Muxer {
 		const release = await this.mutex.acquire();
 
 		try {
-			const trackData = this.getAudioTrackData(track, meta);
+			const trackData = this.getAudioTrackData(track, packet, meta);
+
+			let packetData = packet.data;
+			if (trackData.info.requiresAdtsStripping) {
+				const adtsFrame = readAdtsFrameHeader(FileSlice.tempFromBytes(packetData));
+				if (!adtsFrame) {
+					throw new Error('Expected ADTS frame, didn\'t get one.');
+				}
+
+				const headerLength = adtsFrame.crcCheck === null
+					? MIN_ADTS_FRAME_HEADER_SIZE
+					: MAX_ADTS_FRAME_HEADER_SIZE;
+				packetData = packetData.subarray(headerLength);
+			}
 
 			const isKeyFrame = packet.type === 'key';
 			const timestamp = this.validateAndNormalizeTimestamp(trackData.track, packet.timestamp, isKeyFrame);
-			const audioChunk = this.createInternalChunk(packet.data, timestamp, packet.duration, packet.type);
+			const audioChunk = this.createInternalChunk(packetData, timestamp, packet.duration, packet.type);
 
 			trackData.chunkQueue.push(audioChunk);
 			await this.interleaveChunks();

src/mpeg-ts/mpeg-ts-demuxer.ts

@@ -7,7 +7,7 @@
  */
 
 import { SAMPLES_PER_AAC_FRAME } from '../adts/adts-demuxer';
-import { MAX_FRAME_HEADER_SIZE, readAdtsFrameHeader } from '../adts/adts-reader';
+import { MAX_ADTS_FRAME_HEADER_SIZE, readAdtsFrameHeader } from '../adts/adts-reader';
 import {
 	aacChannelMap,
 	AacCodecInfo,
@@ -1468,14 +1468,14 @@ class MpegTsAudioTrackBacking extends MpegTsTrackBacking implements InputAudioTr
 					context.skip(-1);
 					const possibleHeaderStartPos = context.currentPos;
 
-					let remaining = context.ensureBuffered(MAX_FRAME_HEADER_SIZE);
+					let remaining = context.ensureBuffered(MAX_ADTS_FRAME_HEADER_SIZE);
 					if (remaining instanceof Promise) remaining = await remaining;
 
-					if (remaining < MAX_FRAME_HEADER_SIZE) {
+					if (remaining < MAX_ADTS_FRAME_HEADER_SIZE) {
 						return;
 					}
 
-					const headerBytes = context.readBytes(MAX_FRAME_HEADER_SIZE);
+					const headerBytes = context.readBytes(MAX_ADTS_FRAME_HEADER_SIZE);
 					const header = readAdtsFrameHeader(FileSlice.tempFromBytes(headerBytes));
 
 					if (header) {

test/node/isobmff-muxer.test.ts

@@ -0,0 +1,63 @@
+import { expect, test } from 'vitest';
+import path from 'node:path';
+import { Input } from '../../src/input.js';
+import { BufferSource, FilePathSource } from '../../src/source.js';
+import { ADTS, ALL_FORMATS } from '../../src/input-format.js';
+import { EncodedPacketSink } from '../../src/media-sink.js';
+import { Output } from '../../src/output.js';
+import { BufferTarget } from '../../src/target.js';
+import { Mp4OutputFormat } from '../../src/output-format.js';
+import { Conversion } from '../../src/conversion.js';
+import { assert } from '../../src/misc.js';
+
+const __dirname = new URL('.', import.meta.url).pathname;
+
+test('ISOBMFF muxer internally converts ADTS to AAC', async () => {
+	using input = new Input({
+		source: new FilePathSource(path.join(__dirname, '../public/sample3.aac')),
+		formats: ALL_FORMATS,
+	});
+
+	expect(await input.getFormat()).toBe(ADTS);
+
+	const inputTrack = await input.getPrimaryAudioTrack();
+	assert(inputTrack);
+
+	const inputDecoderConfig = await inputTrack.getDecoderConfig();
+	expect(inputDecoderConfig!.description).toBeUndefined(); // ADTS input has no description
+
+	const output = new Output({
+		format: new Mp4OutputFormat(),
+		target: new BufferTarget(),
+	});
+
+	const conversion = await Conversion.init({ input, output, showWarnings: false });
+	await conversion.execute();
+
+	using outputAsInput = new Input({
+		source: new BufferSource(output.target.buffer!),
+		formats: ALL_FORMATS,
+	});
+
+	const outputTrack = await outputAsInput.getPrimaryAudioTrack();
+	assert(outputTrack);
+
+	expect(outputTrack.codec).toBe('aac');
+	expect(outputTrack.sampleRate).toBe(inputTrack.sampleRate);
+	expect(outputTrack.numberOfChannels).toBe(inputTrack.numberOfChannels);
+
+	const outputDecoderConfig = await outputTrack.getDecoderConfig();
+	expect(outputDecoderConfig!.description).toBeDefined();
+
+	const outputSink = new EncodedPacketSink(outputTrack);
+
+	let count = 0;
+	for await (const packet of outputSink.packets()) {
+		// Packets should NOT be ADTS frames (should not start with 0xFFF sync word)
+		const isAdts = packet.data[0] === 0xff && (packet.data[1]! & 0xf0) === 0xf0;
+		expect(isAdts).toBe(false);
+		count++;
+	}
+
+	expect(count).toBe(4557);
+});

test/node/matroska-muxer.test.ts

@@ -0,0 +1,63 @@
+import { expect, test } from 'vitest';
+import path from 'node:path';
+import { Input } from '../../src/input.js';
+import { BufferSource, FilePathSource } from '../../src/source.js';
+import { ADTS, ALL_FORMATS } from '../../src/input-format.js';
+import { EncodedPacketSink } from '../../src/media-sink.js';
+import { Output } from '../../src/output.js';
+import { BufferTarget } from '../../src/target.js';
+import { MkvOutputFormat } from '../../src/output-format.js';
+import { Conversion } from '../../src/conversion.js';
+import { assert } from '../../src/misc.js';
+
+const __dirname = new URL('.', import.meta.url).pathname;
+
+test('Matroska muxer internally converts ADTS to AAC', async () => {
+	using input = new Input({
+		source: new FilePathSource(path.join(__dirname, '../public/sample3.aac')),
+		formats: ALL_FORMATS,
+	});
+
+	expect(await input.getFormat()).toBe(ADTS);
+
+	const inputTrack = await input.getPrimaryAudioTrack();
+	assert(inputTrack);
+
+	const inputDecoderConfig = await inputTrack.getDecoderConfig();
+	expect(inputDecoderConfig!.description).toBeUndefined(); // ADTS input has no description
+
+	const output = new Output({
+		format: new MkvOutputFormat(),
+		target: new BufferTarget(),
+	});
+
+	const conversion = await Conversion.init({ input, output, showWarnings: false });
+	await conversion.execute();
+
+	using outputAsInput = new Input({
+		source: new BufferSource(output.target.buffer!),
+		formats: ALL_FORMATS,
+	});
+
+	const outputTrack = await outputAsInput.getPrimaryAudioTrack();
+	assert(outputTrack);
+
+	expect(outputTrack.codec).toBe('aac');
+	expect(outputTrack.sampleRate).toBe(inputTrack.sampleRate);
+	expect(outputTrack.numberOfChannels).toBe(inputTrack.numberOfChannels);
+
+	const outputDecoderConfig = await outputTrack.getDecoderConfig();
+	expect(outputDecoderConfig!.description).toBeDefined();
+
+	const outputSink = new EncodedPacketSink(outputTrack);
+
+	let count = 0;
+	for await (const packet of outputSink.packets()) {
+		// Packets should NOT be ADTS frames (should not start with 0xFFF sync word)
+		const isAdts = packet.data[0] === 0xff && (packet.data[1]! & 0xf0) === 0xf0;
+		expect(isAdts).toBe(false);
+		count++;
+	}
+
+	expect(count).toBe(4557);
+});