Implement ADTS demuxer, Fix MP3 demuxer race conditions

This commit is contained in:
Ally
2025-08-18 01:29:52 +02:00
parent 1ce5e108a7
commit 8da619a624
10 changed files with 560 additions and 85 deletions
@@ -117,7 +117,7 @@ const compressFile = async (file: File) => {
selectMediaButton.addEventListener('click', () => {
const fileInput = document.createElement('input');
fileInput.type = 'file';
fileInput.accept = 'video/*,video/x-matroska,audio/*';
fileInput.accept = 'video/*,video/x-matroska,audio/*,audio/aac';
fileInput.addEventListener('change', () => {
const file = fileInput.files?.[0];
if (!file) {
+1 -1
View File
@@ -581,7 +581,7 @@ const formatSeconds = (seconds: number) => {
selectMediaButton.addEventListener('click', () => {
const fileInput = document.createElement('input');
fileInput.type = 'file';
fileInput.accept = 'video/*,video/x-matroska,audio/*';
fileInput.accept = 'video/*,video/x-matroska,audio/*,audio/aac';
fileInput.addEventListener('change', () => {
const file = fileInput.files?.[0];
if (!file) {
@@ -157,7 +157,7 @@ const shortDelay = () => {
selectMediaButton.addEventListener('click', () => {
const fileInput = document.createElement('input');
fileInput.type = 'file';
fileInput.accept = 'video/*,video/x-matroska,audio/*';
fileInput.accept = 'video/*,video/x-matroska,audio/*,audio/aac';
fileInput.addEventListener('change', () => {
const file = fileInput.files?.[0];
if (!file) {
@@ -109,7 +109,7 @@ const generateThumbnails = async (file: File) => {
selectMediaButton.addEventListener('click', () => {
const fileInput = document.createElement('input');
fileInput.type = 'file';
fileInput.accept = 'video/*,video/x-matroska,audio/*';
fileInput.accept = 'video/*,video/x-matroska,audio/*,audio/aac';
fileInput.addEventListener('change', () => {
const file = fileInput.files?.[0];
if (!file) {
+312
View File
@@ -0,0 +1,312 @@
/*!
* Copyright (c) 2025-present, Vanilagy and contributors
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/.
*/
import { aacChannelMap, aacFrequencyTable, AudioCodec } from '../codec';
import { Demuxer } from '../demuxer';
import { Input } from '../input';
import { InputAudioTrack, InputAudioTrackBacking } from '../input-track';
import { PacketRetrievalOptions } from '../media-sink';
import {
assert,
AsyncMutex,
binarySearchExact,
binarySearchLessOrEqual,
Bitstream,
UNDETERMINED_LANGUAGE,
} from '../misc';
import { EncodedPacket, PLACEHOLDER_DATA } from '../packet';
import { AdtsReader, FrameHeader, MAX_FRAME_HEADER_SIZE } from './adts-reader';
const SAMPLES_PER_AAC_FRAME = 1024;
type Sample = {
timestamp: number;
duration: number;
dataStart: number;
dataSize: number;
};
export class AdtsDemuxer extends Demuxer {
reader: AdtsReader;
metadataPromise: Promise<void> | null = null;
firstFrameHeader: FrameHeader | null = null;
loadedSamples: Sample[] = []; // All samples from the start of the file to lastLoadedPos
tracks: InputAudioTrack[] = [];
readingMutex = new AsyncMutex();
lastLoadedPos = 0;
fileSize = 0;
nextTimestampInSamples = 0;
constructor(input: Input) {
super(input);
this.reader = new AdtsReader(input._mainReader);
}
async readMetadata() {
return this.metadataPromise ??= (async () => {
this.fileSize = await this.input.source.getSize();
await this.loadNextChunk();
// There has to be a frame if this demuxer got selected
assert(this.firstFrameHeader);
// Create the single audio track
this.tracks = [new InputAudioTrack(new AdtsAudioTrackBacking(this))];
})();
}
async loadNextChunk() {
assert(this.lastLoadedPos < this.fileSize);
const chunkSize = 0.5 * 1024 * 1024; // 0.5 MiB
const endPos = Math.min(this.lastLoadedPos + chunkSize, this.fileSize);
await this.reader.reader.loadRange(this.lastLoadedPos, endPos);
this.lastLoadedPos = endPos;
assert(this.lastLoadedPos <= this.fileSize);
this.parseFramesFromLoadedData();
}
private parseFramesFromLoadedData() {
while (this.reader.pos <= this.fileSize - MAX_FRAME_HEADER_SIZE) {
const startPos = this.reader.pos;
const header = this.reader.readFrameHeader();
if (!header) {
break;
}
// Check if the entire frame fits in the loaded data
if (startPos + header.frameLength > this.lastLoadedPos) {
// Frame doesn't fit, reset positions and stop
this.reader.pos = startPos;
this.lastLoadedPos = startPos;
break;
}
if (!this.firstFrameHeader) {
this.firstFrameHeader = header;
}
const sampleRate = aacFrequencyTable[header.samplingFrequencyIndex];
assert(sampleRate !== undefined);
const sampleDuration = SAMPLES_PER_AAC_FRAME / sampleRate;
const headerSize = header.crcCheck ? MAX_FRAME_HEADER_SIZE : MAX_FRAME_HEADER_SIZE - 2;
const sample: Sample = {
timestamp: this.nextTimestampInSamples / sampleRate,
duration: sampleDuration,
dataStart: startPos + headerSize,
dataSize: header.frameLength - headerSize,
};
this.loadedSamples.push(sample);
this.nextTimestampInSamples += SAMPLES_PER_AAC_FRAME;
this.reader.pos = startPos + header.frameLength;
}
}
async getMimeType() {
return 'audio/aac';
}
async getTracks() {
await this.readMetadata();
return this.tracks;
}
async computeDuration() {
await this.readMetadata();
const track = this.tracks[0];
assert(track);
return track.computeDuration();
}
}
class AdtsAudioTrackBacking implements InputAudioTrackBacking {
constructor(public demuxer: AdtsDemuxer) {}
getId() {
return 1;
}
async getFirstTimestamp() {
return 0;
}
getTimeResolution() {
const sampleRate = this.getSampleRate();
return sampleRate / SAMPLES_PER_AAC_FRAME;
}
async computeDuration() {
const lastPacket = await this.getPacket(Infinity, { metadataOnly: true });
return (lastPacket?.timestamp ?? 0) + (lastPacket?.duration ?? 0);
}
getLanguageCode() {
return UNDETERMINED_LANGUAGE;
}
getCodec(): AudioCodec {
return 'aac';
}
getNumberOfChannels() {
assert(this.demuxer.firstFrameHeader);
const numberOfChannels = aacChannelMap[this.demuxer.firstFrameHeader.channelConfiguration];
assert(numberOfChannels !== undefined);
return numberOfChannels;
}
getSampleRate() {
assert(this.demuxer.firstFrameHeader);
const sampleRate = aacFrequencyTable[this.demuxer.firstFrameHeader.samplingFrequencyIndex];
assert(sampleRate !== undefined);
return sampleRate;
}
async getDecoderConfig(): Promise<AudioDecoderConfig> {
assert(this.demuxer.firstFrameHeader);
const bytes = new Uint8Array(3); // 19 bits max
const bitstream = new Bitstream(bytes);
const { objectType, samplingFrequencyIndex, channelConfiguration } = this.demuxer.firstFrameHeader;
if (objectType > 31) {
bitstream.writeBits(5, 31);
bitstream.writeBits(6, objectType - 32);
} else {
bitstream.writeBits(5, objectType);
}
bitstream.writeBits(4, samplingFrequencyIndex); // samplingFrequencyIndex === 15 is forbidden
bitstream.writeBits(4, channelConfiguration);
return {
codec: `mp4a.40.${this.demuxer.firstFrameHeader.objectType}`,
numberOfChannels: this.getNumberOfChannels(),
sampleRate: this.getSampleRate(),
description: bytes.subarray(0, Math.ceil((bitstream.pos - 1) / 8)),
};
}
getPacketAtIndex(sampleIndex: number, options: PacketRetrievalOptions) {
if (sampleIndex === -1) {
return null;
}
const rawSample = this.demuxer.loadedSamples[sampleIndex];
if (!rawSample) {
return null;
}
let data: Uint8Array;
if (options.metadataOnly) {
data = PLACEHOLDER_DATA;
} else {
this.demuxer.reader.pos = rawSample.dataStart;
data = this.demuxer.reader.readBytes(rawSample.dataSize);
}
return new EncodedPacket(
data,
'key',
rawSample.timestamp,
rawSample.duration,
sampleIndex,
rawSample.dataSize,
);
}
async getFirstPacket(options: PacketRetrievalOptions) {
return this.getPacketAtIndex(0, options);
}
async getNextPacket(packet: EncodedPacket, options: PacketRetrievalOptions) {
const release = await this.demuxer.readingMutex.acquire();
try {
const sampleIndex = binarySearchExact(
this.demuxer.loadedSamples,
packet.timestamp,
x => x.timestamp,
);
if (sampleIndex === -1) {
throw new Error('Packet was not created from this track.');
}
const nextIndex = sampleIndex + 1;
// Ensure the next sample exists
while (
nextIndex >= this.demuxer.loadedSamples.length
&& this.demuxer.lastLoadedPos < this.demuxer.fileSize
) {
await this.demuxer.loadNextChunk();
}
return this.getPacketAtIndex(nextIndex, options);
} finally {
release();
}
}
async getPacket(timestamp: number, options: PacketRetrievalOptions) {
const release = await this.demuxer.readingMutex.acquire();
try {
while (true) {
const index = binarySearchLessOrEqual(
this.demuxer.loadedSamples,
timestamp,
x => x.timestamp,
);
if (index === -1 && this.demuxer.loadedSamples.length > 0) {
// We're before the first sample
return null;
}
if (this.demuxer.lastLoadedPos === this.demuxer.fileSize) {
// All data is loaded, return what we found
return this.getPacketAtIndex(index, options);
}
if (index >= 0 && index + 1 < this.demuxer.loadedSamples.length) {
// The next packet also exists, we're done
return this.getPacketAtIndex(index, options);
}
// Otherwise, keep loading data
await this.demuxer.loadNextChunk();
}
} finally {
release();
}
}
getKeyPacket(timestamp: number, options: PacketRetrievalOptions) {
return this.getPacket(timestamp, options);
}
getNextKeyPacket(packet: EncodedPacket, options: PacketRetrievalOptions) {
return this.getNextPacket(packet, options);
}
}
+96
View File
@@ -0,0 +1,96 @@
/*!
* Copyright (c) 2025-present, Vanilagy and contributors
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/.
*/
import { Bitstream } from '../misc';
import { Reader } from '../reader';
export const MAX_FRAME_HEADER_SIZE = 9;
export type FrameHeader = {
objectType: number;
samplingFrequencyIndex: number;
channelConfiguration: number;
frameLength: number;
numberOfAacFrames: number;
crcCheck: number | null;
startPos: number;
};
export class AdtsReader {
pos = 0;
constructor(public reader: Reader) {}
readBytes(length: number) {
const { view, offset } = this.reader.getViewAndOffset(this.pos, this.pos + length);
this.pos += length;
return new Uint8Array(view.buffer, offset, length);
}
readFrameHeader(): FrameHeader | null {
// https://wiki.multimedia.cx/index.php/ADTS (last visited: 2025/08/17)
const startPos = this.pos;
const bytes = this.readBytes(9); // 9 with CRC, 7 without CRC
const bitstream = new Bitstream(bytes);
const syncword = bitstream.readBits(12);
if (syncword !== 0b1111_11111111) {
return null;
}
bitstream.skipBits(1); // MPEG version
const layer = bitstream.readBits(2);
if (layer !== 0) {
return null;
}
const protectionAbsence = bitstream.readBits(1);
const objectType = bitstream.readBits(2) + 1;
const samplingFrequencyIndex = bitstream.readBits(4);
if (samplingFrequencyIndex === 15) {
return null;
}
bitstream.skipBits(1); // Private bit
const channelConfiguration = bitstream.readBits(3);
if (channelConfiguration === 0) {
throw new Error('ADTS frames with channel configuration 0 are not supported.');
}
bitstream.skipBits(1); // Originality
bitstream.skipBits(1); // Home
bitstream.skipBits(1); // Copyright ID bit
bitstream.skipBits(1); // Copyright ID start
const frameLength = bitstream.readBits(13);
bitstream.skipBits(11); // Buffer fullness
const numberOfAacFrames = bitstream.readBits(2) + 1;
if (numberOfAacFrames !== 1) {
throw new Error('ADTS frames with more than one AAC frame are not supported.');
}
let crcCheck: number | null = null;
if (protectionAbsence === 1) { // No CRC
this.pos -= 2;
} else { // CRC
crcCheck = bitstream.readBits(16);
}
return {
objectType,
samplingFrequencyIndex,
channelConfiguration,
frameLength,
numberOfAacFrames,
crcCheck,
startPos,
};
}
}
+10 -16
View File
@@ -569,6 +569,13 @@ export type AacAudioSpecificConfig = {
numberOfChannels: number | null;
};
export const aacFrequencyTable = [
96000, 88200, 64000, 48000, 44100, 32000,
24000, 22050, 16000, 12000, 11025, 8000, 7350,
];
export const aacChannelMap = [-1, 1, 2, 3, 4, 5, 6, 8];
export const parseAacAudioSpecificConfig = (bytes: Uint8Array | null): AacAudioSpecificConfig => {
if (!bytes || bytes.byteLength < 2) {
throw new TypeError('AAC description must be at least 2 bytes long.');
@@ -586,28 +593,15 @@ export const parseAacAudioSpecificConfig = (bytes: Uint8Array | null): AacAudioS
if (frequencyIndex === 15) {
sampleRate = bitstream.readBits(24);
} else {
const freqTable = [
96000, 88200, 64000, 48000, 44100, 32000, 24000, 22050,
16000, 12000, 11025, 8000, 7350,
];
if (frequencyIndex < freqTable.length) {
sampleRate = freqTable[frequencyIndex]!;
if (frequencyIndex < aacFrequencyTable.length) {
sampleRate = aacFrequencyTable[frequencyIndex]!;
}
}
const channelConfiguration = bitstream.readBits(4);
let numberOfChannels: number | null = null;
if (channelConfiguration >= 1 && channelConfiguration <= 7) {
const channelMap = {
1: 1,
2: 2,
3: 3,
4: 4,
5: 5,
6: 6,
7: 8,
};
numberOfChannels = channelMap[channelConfiguration as keyof typeof channelMap];
numberOfChannels = aacChannelMap[channelConfiguration]!;
}
return {
+56 -1
View File
@@ -19,6 +19,8 @@ import { OggDemuxer } from './ogg/ogg-demuxer';
import { OggReader } from './ogg/ogg-reader';
import { RiffReader } from './wave/riff-reader';
import { WaveDemuxer } from './wave/wave-demuxer';
import { AdtsReader, MAX_FRAME_HEADER_SIZE } from './adts/adts-reader';
import { AdtsDemuxer } from './adts/adts-demuxer';
/**
* Base class representing an input media file format.
@@ -357,6 +359,54 @@ export class OggInputFormat extends InputFormat {
}
}
/**
* ADTS file format.
* @public
*/
export class AdtsInputFormat extends InputFormat {
/** @internal */
async _canReadInput(input: Input) {
const sourceSize = await input._mainReader.source.getSize();
if (sourceSize < MAX_FRAME_HEADER_SIZE) {
return false;
}
const adtsReader = new AdtsReader(input._mainReader);
const firstHeader = adtsReader.readFrameHeader();
if (!firstHeader) {
return false;
}
if (sourceSize < firstHeader.frameLength + MAX_FRAME_HEADER_SIZE) {
return false;
}
adtsReader.pos = firstHeader.frameLength;
await adtsReader.reader.loadRange(adtsReader.pos, adtsReader.pos + MAX_FRAME_HEADER_SIZE);
const secondHeader = adtsReader.readFrameHeader();
if (!secondHeader) {
return false;
}
return firstHeader.objectType === secondHeader.objectType
&& firstHeader.samplingFrequencyIndex === secondHeader.samplingFrequencyIndex
&& firstHeader.channelConfiguration === secondHeader.channelConfiguration;
}
/** @internal */
_createDemuxer(input: Input) {
return new AdtsDemuxer(input);
}
get name() {
return 'ADTS';
}
get mimeType() {
return 'audio/aac';
}
}
/**
* MP4 input format singleton.
* @public
@@ -392,10 +442,15 @@ export const WAVE = new WaveInputFormat();
* @public
*/
export const OGG = new OggInputFormat();
/**
* ADTS input format singleton.
* @public
*/
export const ADTS = new AdtsInputFormat();
/**
* List of all input format singletons. If you don't need to support all input formats, you should specify the
* formats individually for better tree shaking.
* @public
*/
export const ALL_FORMATS: InputFormat[] = [MP4, QTFF, MATROSKA, WEBM, WAVE, OGG, MP3];
export const ALL_FORMATS: InputFormat[] = [MP4, QTFF, MATROSKA, WEBM, WAVE, OGG, MP3, ADTS];
+16
View File
@@ -73,6 +73,22 @@ export class Bitstream {
return result;
}
writeBits(n: number, value: number) {
const end = this.pos + n;
for (let i = this.pos; i < end; i++) {
const byteIndex = Math.floor(i / 8);
let byte = this.bytes[byteIndex]!;
const bitIndex = 0b111 - (i & 0b111);
byte &= ~(1 << bitIndex);
byte |= ((value & (1 << (end - i - 1))) >> (end - i - 1)) << bitIndex;
this.bytes[byteIndex] = byte;
}
this.pos = end;
};
readAlignedByte() {
// Ensure we're byte-aligned
if (this.pos % 8 !== 0) {
+66 -64
View File
@@ -32,7 +32,7 @@ export class Mp3Demuxer extends Demuxer {
tracks: InputAudioTrack[] = [];
loadingMutex = new AsyncMutex();
readingMutex = new AsyncMutex();
lastLoadedPos = 0;
fileSize = 0;
nextTimestampInSamples = 0;
@@ -53,9 +53,8 @@ export class Mp3Demuxer extends Demuxer {
await this.loadNextChunk();
}
if (!this.firstFrameHeader) {
throw new Error('No MP3 frames found.');
}
// There has to be a frame if this demuxer got selected
assert(this.firstFrameHeader);
this.tracks = [new InputAudioTrack(new Mp3AudioTrackBacking(this))];
})();
@@ -63,30 +62,24 @@ export class Mp3Demuxer extends Demuxer {
/** Loads the next 0.5 MiB of frames. */
async loadNextChunk() {
const release = await this.loadingMutex.acquire();
assert(this.lastLoadedPos < this.fileSize);
try {
assert(this.lastLoadedPos < this.fileSize);
const chunkSize = 0.5 * 1024 * 1024; // 0.5 MiB
const endPos = Math.min(this.lastLoadedPos + chunkSize, this.fileSize);
await this.reader.reader.loadRange(this.lastLoadedPos, endPos);
const chunkSize = 0.5 * 1024 * 1024; // 0.5 MiB
const endPos = Math.min(this.lastLoadedPos + chunkSize, this.fileSize);
await this.reader.reader.loadRange(this.lastLoadedPos, endPos);
this.lastLoadedPos = endPos;
assert(this.lastLoadedPos <= this.fileSize);
this.lastLoadedPos = endPos;
assert(this.lastLoadedPos <= this.fileSize);
if (this.reader.pos === 0) {
// First time, let's see if there's an ID3 tag
const id3Tag = this.reader.readId3();
if (id3Tag) {
this.reader.pos += id3Tag.size;
}
if (this.reader.pos === 0) {
// First time, let's see if there's an ID3 tag
const id3Tag = this.reader.readId3();
if (id3Tag) {
this.reader.pos += id3Tag.size;
}
this.parseFramesFromLoadedData();
} finally {
release();
}
this.parseFramesFromLoadedData();
}
private parseFramesFromLoadedData() {
@@ -232,58 +225,67 @@ class Mp3AudioTrackBacking implements InputAudioTrackBacking {
}
async getFirstPacket(options: PacketRetrievalOptions) {
// Ensure we have at least one frame loaded
while (this.demuxer.loadedSamples.length === 0 && this.demuxer.lastLoadedPos < this.demuxer.fileSize) {
await this.demuxer.loadNextChunk();
}
return this.getPacketAtIndex(0, options);
}
async getNextPacket(packet: EncodedPacket, options: PacketRetrievalOptions) {
const sampleIndex = binarySearchExact(
this.demuxer.loadedSamples,
packet.timestamp,
x => x.timestamp,
);
if (sampleIndex === -1) {
throw new Error('Packet was not created from this track.');
}
const release = await this.demuxer.readingMutex.acquire();
const nextIndex = sampleIndex + 1;
// Ensure the next sample exists
while (nextIndex >= this.demuxer.loadedSamples.length && this.demuxer.lastLoadedPos < this.demuxer.fileSize) {
await this.demuxer.loadNextChunk();
}
try {
const sampleIndex = binarySearchExact(
this.demuxer.loadedSamples,
packet.timestamp,
x => x.timestamp,
);
if (sampleIndex === -1) {
throw new Error('Packet was not created from this track.');
}
return this.getPacketAtIndex(nextIndex, options);
const nextIndex = sampleIndex + 1;
// Ensure the next sample exists
while (
nextIndex >= this.demuxer.loadedSamples.length
&& this.demuxer.lastLoadedPos < this.demuxer.fileSize
) {
await this.demuxer.loadNextChunk();
}
return this.getPacketAtIndex(nextIndex, options);
} finally {
release();
}
}
async getPacket(timestamp: number, options: PacketRetrievalOptions) {
while (true) {
const index = binarySearchLessOrEqual(
this.demuxer.loadedSamples,
timestamp,
x => x.timestamp,
);
const release = await this.demuxer.readingMutex.acquire();
try {
while (true) {
const index = binarySearchLessOrEqual(
this.demuxer.loadedSamples,
timestamp,
x => x.timestamp,
);
if (index === -1 && this.demuxer.loadedSamples.length > 0) {
// We're before the first sample
return null;
if (index === -1 && this.demuxer.loadedSamples.length > 0) {
// We're before the first sample
return null;
}
if (this.demuxer.lastLoadedPos === this.demuxer.fileSize) {
// All data is loaded, return what we found
return this.getPacketAtIndex(index, options);
}
if (index >= 0 && index + 1 < this.demuxer.loadedSamples.length) {
// The next packet also exists, we're done
return this.getPacketAtIndex(index, options);
}
// Otherwise, keep loading data
await this.demuxer.loadNextChunk();
}
if (this.demuxer.lastLoadedPos === this.demuxer.fileSize) {
// All data is loaded, return what we found
return this.getPacketAtIndex(index, options);
}
if (index >= 0 && index + 1 < this.demuxer.loadedSamples.length) {
// The next packet also exists, we're done
return this.getPacketAtIndex(index, options);
}
// Otherwise, keep loading data
await this.demuxer.loadNextChunk();
} finally {
release();
}
}