Make ADTS demuxer spit out ADTS packets instead of AAC, make ADTS muxer accept both AAC and ADTS input

src/adts/adts-demuxer.ts

@@ -16,13 +16,12 @@ import {
 	AsyncMutex,
 	binarySearchExact,
 	binarySearchLessOrEqual,
-	Bitstream,
 	UNDETERMINED_LANGUAGE,
 } from '../misc';
 import { EncodedPacket, PLACEHOLDER_DATA } from '../packet';
 import { readBytes, Reader } from '../reader';
 import { DEFAULT_TRACK_DISPOSITION } from '../metadata';
-import { AdtsFrameHeader, MAX_FRAME_HEADER_SIZE, MIN_FRAME_HEADER_SIZE, readAdtsFrameHeader } from './adts-reader';
+import { AdtsFrameHeader, MIN_FRAME_HEADER_SIZE, MAX_FRAME_HEADER_SIZE, readAdtsFrameHeader } from './adts-reader';
 
 export const SAMPLES_PER_AAC_FRAME = 1024;
 
@@ -95,13 +94,12 @@ export class AdtsDemuxer extends Demuxer {
 		const sampleRate = aacFrequencyTable[header.samplingFrequencyIndex];
 		assert(sampleRate !== undefined);
 		const sampleDuration = SAMPLES_PER_AAC_FRAME / sampleRate;
-		const headerSize = header.crcCheck ? MAX_FRAME_HEADER_SIZE : MIN_FRAME_HEADER_SIZE;
 
 		const sample: Sample = {
 			timestamp: this.nextTimestampInSamples / sampleRate,
 			duration: sampleDuration,
-			dataStart: header.startPos + headerSize,
-			dataSize: header.frameLength - headerSize,
+			dataStart: header.startPos,
+			dataSize: header.frameLength,
 		};
 
 		this.loadedSamples.push(sample);
@@ -198,27 +196,10 @@ class AdtsAudioTrackBacking implements InputAudioTrackBacking {
 	async getDecoderConfig(): Promise<AudioDecoderConfig> {
 		assert(this.demuxer.firstFrameHeader);
 
-		const bytes = new Uint8Array(3); // 19 bits max
-		const bitstream = new Bitstream(bytes);
-
-		const { objectType, samplingFrequencyIndex, channelConfiguration } = this.demuxer.firstFrameHeader;
-
-		if (objectType > 31) {
-			bitstream.writeBits(5, 31);
-			bitstream.writeBits(6, objectType - 32);
-		} else {
-			bitstream.writeBits(5, objectType);
-		}
-
-		bitstream.writeBits(4, samplingFrequencyIndex); // samplingFrequencyIndex === 15 is forbidden
-
-		bitstream.writeBits(4, channelConfiguration);
-
 		return {
 			codec: `mp4a.40.${this.demuxer.firstFrameHeader.objectType}`,
 			numberOfChannels: this.getNumberOfChannels(),
 			sampleRate: this.getSampleRate(),
-			description: bytes.subarray(0, Math.ceil((bitstream.pos - 1) / 8)),
 		};
 	}
 

src/adts/adts-muxer.ts

@@ -7,7 +7,7 @@
  */
 
 import { AacAudioSpecificConfig, parseAacAudioSpecificConfig, validateAudioChunkMetadata } from '../codec';
-import { assert, Bitstream, toUint8Array } from '../misc';
+import { Bitstream, toUint8Array } from '../misc';
 import { Muxer } from '../muxer';
 import { Output, OutputAudioTrack } from '../output';
 import { AdtsOutputFormat } from '../output-format';
@@ -20,6 +20,7 @@ export class AdtsMuxer extends Muxer {
 	private header = new Uint8Array(7);
 	private headerBitstream = new Bitstream(this.header);
 	private audioSpecificConfig: AacAudioSpecificConfig | null = null;
+	private inputIsAdts: boolean | null = null;
 
 	constructor(output: Output, format: AdtsOutputFormat) {
 		super(output);
@@ -52,49 +53,67 @@ export class AdtsMuxer extends Muxer {
 		try {
 			this.validateAndNormalizeTimestamp(track, packet.timestamp, packet.type === 'key');
 
-			if (!this.audioSpecificConfig) {
+			if (this.inputIsAdts === null) {
 				validateAudioChunkMetadata(meta);
 
 				const description = meta?.decoderConfig?.description;
-				assert(description);
 
-				this.audioSpecificConfig = parseAacAudioSpecificConfig(toUint8Array(description));
+				// From the WebCodecs Codec Registry:
+				// "If description is present, it is assumed to a AudioSpecificConfig as defined in [iso14496-3] section
+				// 1.6.2.1, Table 1.15, and the bitstream is assumed to be in aac.
+				// If the description is not present, the bitstream is assumed to be in adts format."
+				this.inputIsAdts = !description;
 
-				const { objectType, frequencyIndex, channelConfiguration } = this.audioSpecificConfig;
-				const profile = objectType - 1;
+				if (!this.inputIsAdts) {
+					this.audioSpecificConfig = parseAacAudioSpecificConfig(toUint8Array(description!));
 
-				this.headerBitstream.writeBits(12, 0b1111_11111111); // Syncword
-				this.headerBitstream.writeBits(1, 0); // MPEG Version
-				this.headerBitstream.writeBits(2, 0); // Layer
-				this.headerBitstream.writeBits(1, 1); // Protection absence
-				this.headerBitstream.writeBits(2, profile); // Profile
-				this.headerBitstream.writeBits(4, frequencyIndex); // MPEG-4 Sampling Frequency Index
-				this.headerBitstream.writeBits(1, 0); // Private bit
-				this.headerBitstream.writeBits(3, channelConfiguration); // MPEG-4 Channel Configuration
-				this.headerBitstream.writeBits(1, 0); // Originality
-				this.headerBitstream.writeBits(1, 0); // Home
-				this.headerBitstream.writeBits(1, 0); // Copyright ID bit
-				this.headerBitstream.writeBits(1, 0); // Copyright ID start
-				this.headerBitstream.skipBits(13); // Frame length
-				this.headerBitstream.writeBits(11, 0x7ff); // Buffer fullness
-				this.headerBitstream.writeBits(2, 0); // Number of AAC frames minus 1
-				// Omit CRC check
+					const { objectType, frequencyIndex, channelConfiguration } = this.audioSpecificConfig;
+					const profile = objectType - 1;
+
+					this.headerBitstream.writeBits(12, 0b1111_11111111); // Syncword
+					this.headerBitstream.writeBits(1, 0); // MPEG Version
+					this.headerBitstream.writeBits(2, 0); // Layer
+					this.headerBitstream.writeBits(1, 1); // Protection absence
+					this.headerBitstream.writeBits(2, profile); // Profile
+					this.headerBitstream.writeBits(4, frequencyIndex); // MPEG-4 Sampling Frequency Index
+					this.headerBitstream.writeBits(1, 0); // Private bit
+					this.headerBitstream.writeBits(3, channelConfiguration); // MPEG-4 Channel Configuration
+					this.headerBitstream.writeBits(1, 0); // Originality
+					this.headerBitstream.writeBits(1, 0); // Home
+					this.headerBitstream.writeBits(1, 0); // Copyright ID bit
+					this.headerBitstream.writeBits(1, 0); // Copyright ID start
+					this.headerBitstream.skipBits(13); // Frame length
+					this.headerBitstream.writeBits(11, 0x7ff); // Buffer fullness
+					this.headerBitstream.writeBits(2, 0); // Number of AAC frames minus 1
+					// Omit CRC check
+				}
 			}
 
-			const frameLength = packet.data.byteLength + this.header.byteLength;
-			this.headerBitstream.pos = 30;
-			this.headerBitstream.writeBits(13, frameLength);
+			if (this.inputIsAdts) {
+				// Packets are already ADTS frames, write them directly
+				const startPos = this.writer.getPos();
+				this.writer.write(packet.data);
 
-			const startPos = this.writer.getPos();
-			this.writer.write(this.header);
-			this.writer.write(packet.data);
+				if (this.format._options.onFrame) {
+					this.format._options.onFrame(packet.data, startPos);
+				}
+			} else {
+				// Packets are raw AAC, prepend ADTS header
+				const frameLength = packet.data.byteLength + this.header.byteLength;
+				this.headerBitstream.pos = 30;
+				this.headerBitstream.writeBits(13, frameLength);
 
-			if (this.format._options.onFrame) {
-				const frameBytes = new Uint8Array(frameLength);
-				frameBytes.set(this.header, 0);
-				frameBytes.set(packet.data, this.header.byteLength);
+				const startPos = this.writer.getPos();
+				this.writer.write(this.header);
+				this.writer.write(packet.data);
 
-				this.format._options.onFrame(frameBytes, startPos);
+				if (this.format._options.onFrame) {
+					const frameBytes = new Uint8Array(frameLength);
+					frameBytes.set(this.header, 0);
+					frameBytes.set(packet.data, this.header.byteLength);
+
+					this.format._options.onFrame(frameBytes, startPos);
+				}
 			}
 
 			await this.writer.flush();

src/codec.ts

@@ -993,12 +993,9 @@ export const validateAudioChunkMetadata = (metadata: EncodedAudioChunkMetadata |
 			);
 		}
 
-		if (!metadata.decoderConfig.description) {
-			throw new TypeError(
-				'Audio chunk metadata decoder configuration for AAC must include a description, which is expected to be'
-				+ ' an AudioSpecificConfig as specified in ISO 14496-3.',
-			);
-		}
+		// `description` may or may not be set, depending on if the format is AAC or ADTS, so don't perform any
+		// validation for it.
+		// https://www.w3.org/TR/webcodecs-aac-codec-registration
 	} else if (metadata.decoderConfig.codec.startsWith('mp3') || metadata.decoderConfig.codec.startsWith('mp4a')) {
 		// MP3-specific validation
 

test/browser/adts-demuxing.test.ts

@@ -0,0 +1,68 @@
+import { expect, test } from 'vitest';
+import { Input } from '../../src/input.js';
+import { UrlSource } from '../../src/source.js';
+import { ADTS, ALL_FORMATS } from '../../src/input-format.js';
+import { AudioSampleSink, EncodedPacketSink } from '../../src/media-sink.js';
+import { assert } from '../../src/misc.js';
+
+test('ADTS demuxing', async () => {
+	using input = new Input({
+		source: new UrlSource('/sample3.aac'),
+		formats: ALL_FORMATS,
+	});
+
+	expect(await input.getFormat()).toBe(ADTS);
+
+	const audioTrack = await input.getPrimaryAudioTrack();
+	assert(audioTrack);
+
+	expect(audioTrack.codec).toBe('aac');
+	expect(audioTrack.sampleRate).toBe(44100);
+	expect(audioTrack.numberOfChannels).toBe(2);
+
+	const decoderConfig = await audioTrack.getDecoderConfig();
+	assert(decoderConfig);
+
+	expect(decoderConfig).toEqual({
+		codec: 'mp4a.40.2',
+		sampleRate: 44100,
+		numberOfChannels: 2,
+		// No description
+	});
+
+	const sink = new EncodedPacketSink(audioTrack);
+
+	const firstPacket = await sink.getFirstPacket();
+	assert(firstPacket);
+
+	expect(firstPacket.data[0]).toBe(0xff);
+	expect((firstPacket.data[1]! & 0xf0)).toBe(0xf0); // Second nibble is also all 1s
+	expect(firstPacket.type).toBe('key');
+
+	const secondPacket = await sink.getNextPacket(firstPacket);
+	assert(secondPacket);
+
+	expect(secondPacket.data[0]).toBe(0xff);
+	expect((secondPacket.data[1]! & 0xf0)).toBe(0xf0);
+	expect(secondPacket.type).toBe('key');
+});
+
+test('ADTS packet decodability', async () => {
+	using input = new Input({
+		source: new UrlSource('/sample3.aac'),
+		formats: ALL_FORMATS,
+	});
+
+	const audioTrack = await input.getPrimaryAudioTrack();
+	assert(audioTrack);
+
+	const sink = new AudioSampleSink(audioTrack);
+
+	let count = 0;
+	// eslint-disable-next-line @typescript-eslint/no-unused-vars
+	for await (using sample of sink.samples()) {
+		count++;
+	}
+
+	expect(count).toBeGreaterThan(0);
+});

test/node/adts-muxer.test.ts

@@ -0,0 +1,116 @@
+import { expect, test } from 'vitest';
+import path from 'node:path';
+import { Input } from '../../src/input.js';
+import { BufferSource, FilePathSource } from '../../src/source.js';
+import { ADTS, ALL_FORMATS } from '../../src/input-format.js';
+import { EncodedPacketSink } from '../../src/media-sink.js';
+import { Output } from '../../src/output.js';
+import { BufferTarget } from '../../src/target.js';
+import { AdtsOutputFormat } from '../../src/output-format.js';
+import { Conversion } from '../../src/conversion.js';
+import { assert } from '../../src/misc.js';
+
+const __dirname = new URL('.', import.meta.url).pathname;
+
+test('ADTS muxer with raw AAC input', async () => {
+	using input = new Input({
+		source: new FilePathSource(path.join(__dirname, '../public/video.mp4')),
+		formats: ALL_FORMATS,
+	});
+
+	const audioTrack = await input.getPrimaryAudioTrack();
+	assert(audioTrack);
+
+	const inputDecoderConfig = await audioTrack.getDecoderConfig();
+	assert(inputDecoderConfig!.description); // MP4 has description
+
+	const output = new Output({
+		format: new AdtsOutputFormat(),
+		target: new BufferTarget(),
+	});
+
+	const conversion = await Conversion.init({ input, output, showWarnings: false });
+	await conversion.execute();
+
+	using outputAsInput = new Input({
+		source: new BufferSource(output.target.buffer!),
+		formats: ALL_FORMATS,
+	});
+
+	expect(await outputAsInput.getFormat()).toBe(ADTS);
+
+	const outputTrack = await outputAsInput.getPrimaryAudioTrack();
+	assert(outputTrack);
+
+	expect(outputTrack.codec).toBe('aac');
+	expect(outputTrack.sampleRate).toBe(audioTrack.sampleRate);
+	expect(outputTrack.numberOfChannels).toBe(audioTrack.numberOfChannels);
+
+	const outputDecoderConfig = await outputTrack.getDecoderConfig();
+	expect(outputDecoderConfig!.description).toBeUndefined(); // ADTS has no description
+
+	const outputSink = new EncodedPacketSink(outputTrack);
+
+	let count = 0;
+	for await (const packet of outputSink.packets()) {
+		// All packets should be ADTS frames now (start with 0xfff sync word)
+		expect(packet.data[0]).toBe(0xff);
+		expect((packet.data[1]! & 0xf0)).toBe(0xf0);
+		count++;
+	}
+
+	expect(count).toBe(237);
+});
+
+test('ADTS muxer with ADTS input (passthrough)', async () => {
+	using input = new Input({
+		source: new FilePathSource(path.join(__dirname, '../public/sample3.aac')),
+		formats: ALL_FORMATS,
+	});
+
+	expect(await input.getFormat()).toBe(ADTS);
+
+	const inputTrack = await input.getPrimaryAudioTrack();
+	assert(inputTrack);
+
+	const inputDecoderConfig = await inputTrack.getDecoderConfig();
+	expect(inputDecoderConfig!.description).toBeUndefined(); // ADTS input has no description
+
+	const output = new Output({
+		format: new AdtsOutputFormat(),
+		target: new BufferTarget(),
+	});
+
+	const conversion = await Conversion.init({ input, output, showWarnings: false });
+	await conversion.execute();
+
+	using outputAsInput = new Input({
+		source: new BufferSource(output.target.buffer!),
+		formats: ALL_FORMATS,
+	});
+
+	const outputTrack = await outputAsInput.getPrimaryAudioTrack();
+	assert(outputTrack);
+
+	const inputSink = new EncodedPacketSink(inputTrack);
+	const outputSink = new EncodedPacketSink(outputTrack);
+
+	let inputPacket = await inputSink.getFirstPacket();
+	let outputPacket = await outputSink.getFirstPacket();
+	let count = 0;
+
+	while (inputPacket && outputPacket) {
+		// Verify that the packets are identical
+		expect(outputPacket.data).toEqual(inputPacket.data);
+		expect(outputPacket.timestamp).toBe(inputPacket.timestamp);
+		expect(outputPacket.duration).toBe(inputPacket.duration);
+
+		inputPacket = await inputSink.getNextPacket(inputPacket);
+		outputPacket = await outputSink.getNextPacket(outputPacket);
+		count++;
+	}
+
+	expect(inputPacket).toBeNull();
+	expect(outputPacket).toBeNull();
+	expect(count).toBe(4557);
+});