Implement the bulk of the MPEG-TS demuxer, add a bunch of tests, and a few other things

src/adts/adts-demuxer.ts

@@ -22,9 +22,9 @@ import {
 import { EncodedPacket, PLACEHOLDER_DATA } from '../packet';
 import { readBytes, Reader } from '../reader';
 import { DEFAULT_TRACK_DISPOSITION } from '../metadata';
-import { FrameHeader, MAX_FRAME_HEADER_SIZE, MIN_FRAME_HEADER_SIZE, readFrameHeader } from './adts-reader';
+import { AdtsFrameHeader, MAX_FRAME_HEADER_SIZE, MIN_FRAME_HEADER_SIZE, readAdtsFrameHeader } from './adts-reader';
 
-const SAMPLES_PER_AAC_FRAME = 1024;
+export const SAMPLES_PER_AAC_FRAME = 1024;
 
 type Sample = {
 	timestamp: number;
@@ -37,7 +37,7 @@ export class AdtsDemuxer extends Demuxer {
 	reader: Reader;
 
 	metadataPromise: Promise<void> | null = null;
-	firstFrameHeader: FrameHeader | null = null;
+	firstFrameHeader: AdtsFrameHeader | null = null;
 	loadedSamples: Sample[] = [];
 
 	tracks: InputAudioTrack[] = [];
@@ -76,7 +76,7 @@ export class AdtsDemuxer extends Demuxer {
 			return;
 		}
 
-		const header = readFrameHeader(slice);
+		const header = readAdtsFrameHeader(slice);
 		if (!header) {
 			this.lastSampleLoaded = true;
 			return;

src/adts/adts-reader.ts

@@ -12,7 +12,7 @@ import { FileSlice, readBytes } from '../reader';
 export const MIN_FRAME_HEADER_SIZE = 7;
 export const MAX_FRAME_HEADER_SIZE = 9;
 
-export type FrameHeader = {
+export type AdtsFrameHeader = {
 	objectType: number;
 	samplingFrequencyIndex: number;
 	channelConfiguration: number;
@@ -22,7 +22,7 @@ export type FrameHeader = {
 	startPos: number;
 };
 
-export const readFrameHeader = (slice: FileSlice): FrameHeader | null => {
+export const readAdtsFrameHeader = (slice: FileSlice): AdtsFrameHeader | null => {
 	// https://wiki.multimedia.cx/index.php/ADTS (last visited: 2025/08/17)
 
 	const startPos = slice.filePos;

src/codec-data.ts

@@ -40,6 +40,7 @@ export enum AvcNalUnitType {
 	SEI = 6,
 	SPS = 7,
 	PPS = 8,
+	AUD = 9,
 	SPS_EXT = 13,
 }
 
@@ -474,9 +475,17 @@ export type AvcSpsInfo = {
 	constraintFlags: number;
 	levelIdc: number;
 	frameMbsOnlyFlag: number;
-	chromaFormatIdc: number | null;
-	bitDepthLumaMinus8: number | null;
-	bitDepthChromaMinus8: number | null;
+	chromaFormatIdc: number;
+	bitDepthLumaMinus8: number;
+	bitDepthChromaMinus8: number;
+	codedWidth: number;
+	codedHeight: number;
+	displayWidth: number;
+	displayHeight: number;
+	colourPrimaries: number;
+	transferCharacteristics: number;
+	matrixCoefficients: number;
+	fullRangeFlag: number;
 };
 
 /** Parses an AVC SPS (Sequence Parameter Set) to extract basic information. */
@@ -498,9 +507,14 @@ export const parseAvcSps = (sps: Uint8Array): AvcSpsInfo | null => {
 
 		readExpGolomb(bitstream); // seq_parameter_set_id
 
-		let chromaFormatIdc: number | null = null;
-		let bitDepthLumaMinus8: number | null = null;
-		let bitDepthChromaMinus8: number | null = null;
+		// "When chroma_format_idc is not present, it shall be inferred to be equal to 1 (4:2:0 chroma format)."
+		let chromaFormatIdc = 1;
+		// "When bit_depth_luma_minus8 is not present, it shall be inferred to be equal to 0.""
+		let bitDepthLumaMinus8 = 0;
+		// "When bit_depth_chroma_minus8 is not present, it shall be inferred to be equal to 0."
+		let bitDepthChromaMinus8 = 0;
+		// "When separate_colour_plane_flag is not present, it shall be inferred to be equal to 0."
+		let separateColourPlaneFlag = 0;
 
 		// Handle high profile chroma_format_idc
 		if (
@@ -516,7 +530,7 @@ export const parseAvcSps = (sps: Uint8Array): AvcSpsInfo | null => {
 		) {
 			chromaFormatIdc = readExpGolomb(bitstream);
 			if (chromaFormatIdc === 3) {
-				bitstream.skipBits(1); // separate_colour_plane_flag
+				separateColourPlaneFlag = bitstream.readBits(1);
 			}
 			bitDepthLumaMinus8 = readExpGolomb(bitstream);
 			bitDepthChromaMinus8 = readExpGolomb(bitstream);
@@ -559,10 +573,80 @@ export const parseAvcSps = (sps: Uint8Array): AvcSpsInfo | null => {
 		readExpGolomb(bitstream); // max_num_ref_frames
 		bitstream.skipBits(1); // gaps_in_frame_num_value_allowed_flag
 
-		readExpGolomb(bitstream); // pic_width_in_mbs_minus1
-		readExpGolomb(bitstream); // pic_height_in_map_units_minus1
+		const codedWidth = 16 * (readExpGolomb(bitstream) + 1); // pic_width_in_mbs_minus1
+		const codedHeight = 16 * (readExpGolomb(bitstream) + 1); // pic_height_in_map_units_minus1
+		let displayWidth = codedWidth;
+		let displayHeight = codedHeight;
 
 		const frameMbsOnlyFlag = bitstream.readBits(1);
+		if (!frameMbsOnlyFlag) {
+			bitstream.skipBits(1); // mb_adaptive_frame_field_flag
+		}
+
+		bitstream.skipBits(1); // direct_8x8_inference_flag
+		const frameCroppingFlag = bitstream.readBits(1);
+
+		if (frameCroppingFlag) {
+			const frameCropLeftOffset = readExpGolomb(bitstream);
+			const frameCropRightOffset = readExpGolomb(bitstream);
+			const frameCropTopOffset = readExpGolomb(bitstream);
+			const frameCropBottomOffset = readExpGolomb(bitstream);
+
+			let cropUnitX: number;
+			let cropUnitY: number;
+
+			const chromaArrayType = separateColourPlaneFlag === 0 ? chromaFormatIdc : 0;
+			if (chromaArrayType === 0) {
+				// "If ChromaArrayType is equal to 0, CropUnitX and CropUnitY are derived as:"
+				cropUnitX = 1;
+				cropUnitY = 2 - frameMbsOnlyFlag;
+			} else {
+				// "Otherwise (ChromaArrayType is equal to 1, 2, or 3), CropUnitX and CropUnitY are derived as:"
+				const subWidthC = chromaFormatIdc === 3 ? 1 : 2;
+				const subHeightC = chromaFormatIdc === 1 ? 2 : 1;
+
+				cropUnitX = subWidthC;
+				cropUnitY = subHeightC * (2 - frameMbsOnlyFlag);
+			}
+
+			displayWidth -= (cropUnitX * (frameCropLeftOffset + frameCropRightOffset));
+			displayHeight -= (cropUnitY * (frameCropTopOffset + frameCropBottomOffset));
+		}
+
+		// 2 = unspecified
+		let colourPrimaries = 2;
+		let transferCharacteristics = 2;
+		let matrixCoefficients = 2;
+		let fullRangeFlag = 0;
+
+		const vuiParametersPresentFlag = bitstream.readBits(1);
+		if (vuiParametersPresentFlag) {
+			const aspectRatioInfoPresentFlag = bitstream.readBits(1);
+			if (aspectRatioInfoPresentFlag) {
+				const aspectRatioIdc = bitstream.readBits(8);
+				if (aspectRatioIdc === 255) { // Extended_SAR
+					bitstream.skipBits(16); // sar_width
+					bitstream.skipBits(16); // sar_height
+				}
+			}
+
+			const overscanInfoPresentFlag = bitstream.readBits(1);
+			if (overscanInfoPresentFlag) {
+				bitstream.skipBits(1); // overscan_appropriate_flag
+			}
+
+			const videoSignalTypePresentFlag = bitstream.readBits(1);
+			if (videoSignalTypePresentFlag) {
+				bitstream.skipBits(3); // video_format
+				fullRangeFlag = bitstream.readBits(1);
+				const colourDescriptionPresentFlag = bitstream.readBits(1);
+				if (colourDescriptionPresentFlag) {
+					colourPrimaries = bitstream.readBits(8);
+					transferCharacteristics = bitstream.readBits(8);
+					matrixCoefficients = bitstream.readBits(8);
+				}
+			}
+		}
 
 		return {
 			profileIdc,
@@ -572,6 +656,14 @@ export const parseAvcSps = (sps: Uint8Array): AvcSpsInfo | null => {
 			chromaFormatIdc,
 			bitDepthLumaMinus8,
 			bitDepthChromaMinus8,
+			codedWidth,
+			codedHeight,
+			displayWidth,
+			displayHeight,
+			colourPrimaries,
+			matrixCoefficients,
+			transferCharacteristics,
+			fullRangeFlag,
 		};
 	} catch (error) {
 		console.error('Error parsing AVC SPS:', error);

src/codec.ts

@@ -537,6 +537,7 @@ export const buildAudioCodecString = (codec: AudioCodec, numberOfChannels: numbe
 
 export type AacCodecInfo = {
 	isMpeg2: boolean;
+	objectType: number | null;
 };
 
 export const extractAudioCodecString = (trackInfo: {
@@ -554,8 +555,15 @@ export const extractAudioCodecString = (trackInfo: {
 		if (aacCodecInfo.isMpeg2) {
 			return 'mp4a.67';
 		} else {
-			const audioSpecificConfig = parseAacAudioSpecificConfig(codecDescription);
-			return `mp4a.40.${audioSpecificConfig.objectType}`;
+			let objectType: number;
+			if (aacCodecInfo.objectType !== null) {
+				objectType = aacCodecInfo.objectType;
+			} else {
+				const audioSpecificConfig = parseAacAudioSpecificConfig(codecDescription);
+				objectType = audioSpecificConfig.objectType;
+			}
+
+			return `mp4a.40.${objectType}`;
 		}
 	} else if (codec === 'mp3') {
 		return 'mp3';

src/input-format.ts

@@ -23,10 +23,10 @@ import { MatroskaDemuxer } from './matroska/matroska-demuxer';
 import { Mp3Demuxer } from './mp3/mp3-demuxer';
 import { FRAME_HEADER_SIZE } from '../shared/mp3-misc';
 import { ID3_V2_HEADER_SIZE, readId3V2Header } from './id3';
-import { readNextFrameHeader } from './mp3/mp3-reader';
+import { readNextMp3FrameHeader } from './mp3/mp3-reader';
 import { OggDemuxer } from './ogg/ogg-demuxer';
 import { WaveDemuxer } from './wave/wave-demuxer';
-import { MAX_FRAME_HEADER_SIZE, MIN_FRAME_HEADER_SIZE, readFrameHeader } from './adts/adts-reader';
+import { MAX_FRAME_HEADER_SIZE, MIN_FRAME_HEADER_SIZE, readAdtsFrameHeader } from './adts/adts-reader';
 import { AdtsDemuxer } from './adts/adts-demuxer';
 import { readAscii } from './reader';
 import { FlacDemuxer } from './flac/flac-demuxer';
@@ -281,7 +281,7 @@ export class Mp3InputFormat extends InputFormat {
 			currentPos = slice.filePos + id3V2Header.size;
 		}
 
-		const firstResult = await readNextFrameHeader(input._reader, currentPos, currentPos + 4096);
+		const firstResult = await readNextMp3FrameHeader(input._reader, currentPos, currentPos + 4096);
 		if (!firstResult) {
 			return false;
 		}
@@ -295,7 +295,7 @@ export class Mp3InputFormat extends InputFormat {
 
 		// Fine, we found one frame header, but we're still not entirely sure this is MP3. Let's check if we can find
 		// another header right after it:
-		const secondResult = await readNextFrameHeader(input._reader, currentPos, currentPos + FRAME_HEADER_SIZE);
+		const secondResult = await readNextMp3FrameHeader(input._reader, currentPos, currentPos + FRAME_HEADER_SIZE);
 		if (!secondResult) {
 			return false;
 		}
@@ -444,7 +444,7 @@ export class AdtsInputFormat extends InputFormat {
 		if (slice instanceof Promise) slice = await slice;
 		if (!slice) return false;
 
-		const firstHeader = readFrameHeader(slice);
+		const firstHeader = readAdtsFrameHeader(slice);
 		if (!firstHeader) {
 			return false;
 		}
@@ -453,7 +453,7 @@ export class AdtsInputFormat extends InputFormat {
 		if (slice instanceof Promise) slice = await slice;
 		if (!slice) return false;
 
-		const secondHeader = readFrameHeader(slice);
+		const secondHeader = readAdtsFrameHeader(slice);
 		if (!secondHeader) {
 			return false;
 		}

src/input.ts

@@ -118,6 +118,20 @@ export class Input<S extends Source = Source> implements Disposable {
 		return demuxer.computeDuration();
 	}
 
+	/**
+	 * Returns the timestamp at which the input file starts. More precisely, returns the smallest starting timestamp
+	 * among all tracks.
+	 */
+	async getFirstTimestamp() {
+		const tracks = await this.getTracks();
+		if (tracks.length === 0) {
+			return 0;
+		}
+
+		const firstTimestamps = await Promise.all(tracks.map(x => x.getFirstTimestamp()));
+		return Math.min(...firstTimestamps);
+	}
+
 	/** Returns the list of all tracks of this input file. */
 	async getTracks() {
 		const demuxer = await this._getDemuxer();

src/isobmff/isobmff-demuxer.ts

@@ -1235,7 +1235,10 @@ export class IsobmffDemuxer extends Demuxer {
 				const objectTypeIndication = readU8(slice);
 				if (objectTypeIndication === 0x40 || objectTypeIndication === 0x67) {
 					track.info.codec = 'aac';
-					track.info.aacCodecInfo = { isMpeg2: objectTypeIndication === 0x67 };
+					track.info.aacCodecInfo = {
+						isMpeg2: objectTypeIndication === 0x67,
+						objectType: null,
+					};
 				} else if (objectTypeIndication === 0x69 || objectTypeIndication === 0x6b) {
 					track.info.codec = 'mp3';
 				} else if (objectTypeIndication === 0xdd) {

src/matroska/matroska-demuxer.ts

@@ -1057,6 +1057,7 @@ export class MatroskaDemuxer extends Demuxer {
 							this.currentTrack.info.codec = 'aac';
 							this.currentTrack.info.aacCodecInfo = {
 								isMpeg2: this.currentTrack.codecId.includes('MPEG2'),
+								objectType: null,
 							};
 							this.currentTrack.info.codecDescription = this.currentTrack.codecPrivate;
 						} else if (this.currentTrack.codecId === CODEC_STRING_MAP.mp3) {

src/mp3/mp3-demuxer.ts

@@ -14,7 +14,7 @@ import { DEFAULT_TRACK_DISPOSITION, MetadataTags } from '../metadata';
 import { PacketRetrievalOptions } from '../media-sink';
 import { assert, AsyncMutex, binarySearchExact, binarySearchLessOrEqual, UNDETERMINED_LANGUAGE } from '../misc';
 import { EncodedPacket, PLACEHOLDER_DATA } from '../packet';
-import { FrameHeader, getXingOffset, INFO, XING } from '../../shared/mp3-misc';
+import { Mp3FrameHeader, getXingOffset, INFO, XING } from '../../shared/mp3-misc';
 import {
 	ID3_V1_TAG_SIZE,
 	ID3_V2_HEADER_SIZE,
@@ -22,7 +22,7 @@ import {
 	parseId3V2Tag,
 	readId3V2Header,
 } from '../id3';
-import { readNextFrameHeader } from './mp3-reader';
+import { readNextMp3FrameHeader } from './mp3-reader';
 import { readAscii, readBytes, Reader, readU32Be } from '../reader';
 
 type Sample = {
@@ -36,7 +36,7 @@ export class Mp3Demuxer extends Demuxer {
 	reader: Reader;
 
 	metadataPromise: Promise<void> | null = null;
-	firstFrameHeader: FrameHeader | null = null;
+	firstFrameHeader: Mp3FrameHeader | null = null;
 	loadedSamples: Sample[] = []; // All samples from the start of the file to lastLoadedPos
 	metadataTags: MetadataTags | null = null;
 
@@ -89,7 +89,7 @@ export class Mp3Demuxer extends Demuxer {
 			}
 		}
 
-		const result = await readNextFrameHeader(this.reader, this.lastLoadedPos, this.reader.fileSize);
+		const result = await readNextMp3FrameHeader(this.reader, this.lastLoadedPos, this.reader.fileSize);
 		if (!result) {
 			this.lastSampleLoaded = true;
 			return;

src/mp3/mp3-muxer.ts

@@ -13,7 +13,7 @@ import { Output, OutputAudioTrack } from '../output';
 import { Mp3OutputFormat } from '../output-format';
 import { EncodedPacket } from '../packet';
 import { Writer } from '../writer';
-import { getXingOffset, INFO, readFrameHeader, XING } from '../../shared/mp3-misc';
+import { getXingOffset, INFO, readMp3FrameHeader, XING } from '../../shared/mp3-misc';
 import { Mp3Writer, XingFrameData } from './mp3-writer';
 import { Id3V2Writer } from '../id3';
 
@@ -65,7 +65,7 @@ export class Mp3Muxer extends Muxer {
 				}
 
 				const word = view.getUint32(0, false);
-				const header = readFrameHeader(word, null).header;
+				const header = readMp3FrameHeader(word, null).header;
 				if (!header) {
 					throw new Error('Invalid MP3 header in sample.');
 				}

src/mp3/mp3-reader.ts

@@ -6,11 +6,11 @@
  * file, You can obtain one at https://mozilla.org/MPL/2.0/.
  */
 
-import { FRAME_HEADER_SIZE, FrameHeader, readFrameHeader } from '../../shared/mp3-misc';
+import { FRAME_HEADER_SIZE, Mp3FrameHeader, readMp3FrameHeader } from '../../shared/mp3-misc';
 import { Reader, readU32Be } from '../reader';
 
-export const readNextFrameHeader = async (reader: Reader, startPos: number, until: number | null): Promise<{
-	header: FrameHeader;
+export const readNextMp3FrameHeader = async (reader: Reader, startPos: number, until: number | null): Promise<{
+	header: Mp3FrameHeader;
 	startPos: number;
 } | null> => {
 	let currentPos = startPos;
@@ -22,7 +22,7 @@ export const readNextFrameHeader = async (reader: Reader, startPos: number, unti
 
 		const word = readU32Be(slice);
 
-		const result = readFrameHeader(word, reader.fileSize !== null ? reader.fileSize - currentPos : null);
+		const result = readMp3FrameHeader(word, reader.fileSize !== null ? reader.fileSize - currentPos : null);
 		if (result.header) {
 			return { header: result.header, startPos: currentPos };
 		}

src/reader.ts

@@ -20,6 +20,10 @@ export class Reader {
 			throw new InputDisposedError();
 		}
 
+		if (start < 0) {
+			return null;
+		}
+
 		if (this.fileSize !== null && start + length > this.fileSize) {
 			return null;
 		}
@@ -49,6 +53,10 @@ export class Reader {
 			throw new InputDisposedError();
 		}
 
+		if (start < 0) {
+			return null;
+		}
+
 		if (this.fileSize !== null) {
 			return this.requestSlice(
 				start,