diff --git a/docs/guide/reading-media-files.md b/docs/guide/reading-media-files.md index dab7aec..36b44e2 100644 --- a/docs/guide/reading-media-files.md +++ b/docs/guide/reading-media-files.md @@ -60,6 +60,11 @@ await input.computeDuration(); // => 1905.4615 ``` More specifically, the duration is defined as the maximum end timestamp across all tracks. +Since not all media files begin at time zero, you can also retrieve the *starting timestamp* of the media file in seconds: +```ts +await input.getFirstTimestamp(); // => 0.0 +``` + Mediabunny also lets you read descriptive metadata tags from media files, such as title, artist, or cover art: ```ts await input.getMetadataTags(); // => MetadataTags diff --git a/examples/metadata-extraction/metadata-extraction.ts b/examples/metadata-extraction/metadata-extraction.ts index d2bc280..1e932d9 100644 --- a/examples/metadata-extraction/metadata-extraction.ts +++ b/examples/metadata-extraction/metadata-extraction.ts @@ -43,12 +43,14 @@ const extractMetadata = (resource: File | string) => { const object = { 'Format': input.getFormat().then(format => format.name), 'Full MIME type': input.getMimeType(), - 'Duration': input.computeDuration().then(duration => `${duration} seconds`), + 'Starts at': input.getFirstTimestamp().then(start => `${start} seconds`), + 'Ends at': input.computeDuration().then(duration => `${duration} seconds`), 'Tracks': input.getTracks().then(tracks => tracks.map(track => ({ 'Type': track.type, 'Codec': track.codec, 'Full codec string': track.getCodecParameterString(), - 'Duration': track.computeDuration().then(duration => `${duration} seconds`), + 'Starts at': track.getFirstTimestamp().then(start => `${start} seconds`), + 'Ends at': track.computeDuration().then(duration => `${duration} seconds`), 'Language code': track.languageCode, ...(track.isVideoTrack() ? { diff --git a/packages/mp3-encoder/src/index.ts b/packages/mp3-encoder/src/index.ts index 5c99658..12f0486 100644 --- a/packages/mp3-encoder/src/index.ts +++ b/packages/mp3-encoder/src/index.ts @@ -7,7 +7,7 @@ */ import { CustomAudioEncoder, AudioCodec, AudioSample, EncodedPacket, registerEncoder } from 'mediabunny'; -import { FRAME_HEADER_SIZE, readFrameHeader, SAMPLING_RATES } from '../../../shared/mp3-misc'; +import { FRAME_HEADER_SIZE, readMp3FrameHeader, SAMPLING_RATES } from '../../../shared/mp3-misc'; import type { WorkerCommand, WorkerResponse, WorkerResponseData } from './shared'; // @ts-expect-error An esbuild plugin handles this, TypeScript doesn't need to understand import createWorker from './encode.worker'; @@ -145,7 +145,7 @@ class Mp3Encoder extends CustomAudioEncoder { let pos = 0; while (pos <= this.currentBufferOffset - FRAME_HEADER_SIZE) { const word = new DataView(this.buffer.buffer).getUint32(pos, false); - const header = readFrameHeader(word, null).header; + const header = readMp3FrameHeader(word, null).header; if (!header) { break; } diff --git a/shared/mp3-misc.ts b/shared/mp3-misc.ts index 19fd947..1909274 100644 --- a/shared/mp3-misc.ts +++ b/shared/mp3-misc.ts @@ -27,7 +27,7 @@ export const XING = 0x58696e67; /** 'Info' */ export const INFO = 0x496e666f; -export type FrameHeader = { +export type Mp3FrameHeader = { totalSize: number; mpegVersionId: number; layer: number; @@ -66,8 +66,8 @@ export const getXingOffset = (mpegVersionId: number, channel: number) => { : (channel === 3 ? 13 : 21); }; -export const readFrameHeader = (word: number, remainingBytes: number | null): { - header: FrameHeader | null; +export const readMp3FrameHeader = (word: number, remainingBytes: number | null): { + header: Mp3FrameHeader | null; bytesAdvanced: number; } => { const firstByte = word >>> 24; diff --git a/src/adts/adts-demuxer.ts b/src/adts/adts-demuxer.ts index 3295a9a..8852f1d 100644 --- a/src/adts/adts-demuxer.ts +++ b/src/adts/adts-demuxer.ts @@ -22,9 +22,9 @@ import { import { EncodedPacket, PLACEHOLDER_DATA } from '../packet'; import { readBytes, Reader } from '../reader'; import { DEFAULT_TRACK_DISPOSITION } from '../metadata'; -import { FrameHeader, MAX_FRAME_HEADER_SIZE, MIN_FRAME_HEADER_SIZE, readFrameHeader } from './adts-reader'; +import { AdtsFrameHeader, MAX_FRAME_HEADER_SIZE, MIN_FRAME_HEADER_SIZE, readAdtsFrameHeader } from './adts-reader'; -const SAMPLES_PER_AAC_FRAME = 1024; +export const SAMPLES_PER_AAC_FRAME = 1024; type Sample = { timestamp: number; @@ -37,7 +37,7 @@ export class AdtsDemuxer extends Demuxer { reader: Reader; metadataPromise: Promise | null = null; - firstFrameHeader: FrameHeader | null = null; + firstFrameHeader: AdtsFrameHeader | null = null; loadedSamples: Sample[] = []; tracks: InputAudioTrack[] = []; @@ -76,7 +76,7 @@ export class AdtsDemuxer extends Demuxer { return; } - const header = readFrameHeader(slice); + const header = readAdtsFrameHeader(slice); if (!header) { this.lastSampleLoaded = true; return; diff --git a/src/adts/adts-reader.ts b/src/adts/adts-reader.ts index bf21f6a..f1d6bb4 100644 --- a/src/adts/adts-reader.ts +++ b/src/adts/adts-reader.ts @@ -12,7 +12,7 @@ import { FileSlice, readBytes } from '../reader'; export const MIN_FRAME_HEADER_SIZE = 7; export const MAX_FRAME_HEADER_SIZE = 9; -export type FrameHeader = { +export type AdtsFrameHeader = { objectType: number; samplingFrequencyIndex: number; channelConfiguration: number; @@ -22,7 +22,7 @@ export type FrameHeader = { startPos: number; }; -export const readFrameHeader = (slice: FileSlice): FrameHeader | null => { +export const readAdtsFrameHeader = (slice: FileSlice): AdtsFrameHeader | null => { // https://wiki.multimedia.cx/index.php/ADTS (last visited: 2025/08/17) const startPos = slice.filePos; diff --git a/src/codec-data.ts b/src/codec-data.ts index 4882752..ff4a2c7 100644 --- a/src/codec-data.ts +++ b/src/codec-data.ts @@ -40,6 +40,7 @@ export enum AvcNalUnitType { SEI = 6, SPS = 7, PPS = 8, + AUD = 9, SPS_EXT = 13, } @@ -474,9 +475,17 @@ export type AvcSpsInfo = { constraintFlags: number; levelIdc: number; frameMbsOnlyFlag: number; - chromaFormatIdc: number | null; - bitDepthLumaMinus8: number | null; - bitDepthChromaMinus8: number | null; + chromaFormatIdc: number; + bitDepthLumaMinus8: number; + bitDepthChromaMinus8: number; + codedWidth: number; + codedHeight: number; + displayWidth: number; + displayHeight: number; + colourPrimaries: number; + transferCharacteristics: number; + matrixCoefficients: number; + fullRangeFlag: number; }; /** Parses an AVC SPS (Sequence Parameter Set) to extract basic information. */ @@ -498,9 +507,14 @@ export const parseAvcSps = (sps: Uint8Array): AvcSpsInfo | null => { readExpGolomb(bitstream); // seq_parameter_set_id - let chromaFormatIdc: number | null = null; - let bitDepthLumaMinus8: number | null = null; - let bitDepthChromaMinus8: number | null = null; + // "When chroma_format_idc is not present, it shall be inferred to be equal to 1 (4:2:0 chroma format)." + let chromaFormatIdc = 1; + // "When bit_depth_luma_minus8 is not present, it shall be inferred to be equal to 0."" + let bitDepthLumaMinus8 = 0; + // "When bit_depth_chroma_minus8 is not present, it shall be inferred to be equal to 0." + let bitDepthChromaMinus8 = 0; + // "When separate_colour_plane_flag is not present, it shall be inferred to be equal to 0." + let separateColourPlaneFlag = 0; // Handle high profile chroma_format_idc if ( @@ -516,7 +530,7 @@ export const parseAvcSps = (sps: Uint8Array): AvcSpsInfo | null => { ) { chromaFormatIdc = readExpGolomb(bitstream); if (chromaFormatIdc === 3) { - bitstream.skipBits(1); // separate_colour_plane_flag + separateColourPlaneFlag = bitstream.readBits(1); } bitDepthLumaMinus8 = readExpGolomb(bitstream); bitDepthChromaMinus8 = readExpGolomb(bitstream); @@ -559,10 +573,80 @@ export const parseAvcSps = (sps: Uint8Array): AvcSpsInfo | null => { readExpGolomb(bitstream); // max_num_ref_frames bitstream.skipBits(1); // gaps_in_frame_num_value_allowed_flag - readExpGolomb(bitstream); // pic_width_in_mbs_minus1 - readExpGolomb(bitstream); // pic_height_in_map_units_minus1 + const codedWidth = 16 * (readExpGolomb(bitstream) + 1); // pic_width_in_mbs_minus1 + const codedHeight = 16 * (readExpGolomb(bitstream) + 1); // pic_height_in_map_units_minus1 + let displayWidth = codedWidth; + let displayHeight = codedHeight; const frameMbsOnlyFlag = bitstream.readBits(1); + if (!frameMbsOnlyFlag) { + bitstream.skipBits(1); // mb_adaptive_frame_field_flag + } + + bitstream.skipBits(1); // direct_8x8_inference_flag + const frameCroppingFlag = bitstream.readBits(1); + + if (frameCroppingFlag) { + const frameCropLeftOffset = readExpGolomb(bitstream); + const frameCropRightOffset = readExpGolomb(bitstream); + const frameCropTopOffset = readExpGolomb(bitstream); + const frameCropBottomOffset = readExpGolomb(bitstream); + + let cropUnitX: number; + let cropUnitY: number; + + const chromaArrayType = separateColourPlaneFlag === 0 ? chromaFormatIdc : 0; + if (chromaArrayType === 0) { + // "If ChromaArrayType is equal to 0, CropUnitX and CropUnitY are derived as:" + cropUnitX = 1; + cropUnitY = 2 - frameMbsOnlyFlag; + } else { + // "Otherwise (ChromaArrayType is equal to 1, 2, or 3), CropUnitX and CropUnitY are derived as:" + const subWidthC = chromaFormatIdc === 3 ? 1 : 2; + const subHeightC = chromaFormatIdc === 1 ? 2 : 1; + + cropUnitX = subWidthC; + cropUnitY = subHeightC * (2 - frameMbsOnlyFlag); + } + + displayWidth -= (cropUnitX * (frameCropLeftOffset + frameCropRightOffset)); + displayHeight -= (cropUnitY * (frameCropTopOffset + frameCropBottomOffset)); + } + + // 2 = unspecified + let colourPrimaries = 2; + let transferCharacteristics = 2; + let matrixCoefficients = 2; + let fullRangeFlag = 0; + + const vuiParametersPresentFlag = bitstream.readBits(1); + if (vuiParametersPresentFlag) { + const aspectRatioInfoPresentFlag = bitstream.readBits(1); + if (aspectRatioInfoPresentFlag) { + const aspectRatioIdc = bitstream.readBits(8); + if (aspectRatioIdc === 255) { // Extended_SAR + bitstream.skipBits(16); // sar_width + bitstream.skipBits(16); // sar_height + } + } + + const overscanInfoPresentFlag = bitstream.readBits(1); + if (overscanInfoPresentFlag) { + bitstream.skipBits(1); // overscan_appropriate_flag + } + + const videoSignalTypePresentFlag = bitstream.readBits(1); + if (videoSignalTypePresentFlag) { + bitstream.skipBits(3); // video_format + fullRangeFlag = bitstream.readBits(1); + const colourDescriptionPresentFlag = bitstream.readBits(1); + if (colourDescriptionPresentFlag) { + colourPrimaries = bitstream.readBits(8); + transferCharacteristics = bitstream.readBits(8); + matrixCoefficients = bitstream.readBits(8); + } + } + } return { profileIdc, @@ -572,6 +656,14 @@ export const parseAvcSps = (sps: Uint8Array): AvcSpsInfo | null => { chromaFormatIdc, bitDepthLumaMinus8, bitDepthChromaMinus8, + codedWidth, + codedHeight, + displayWidth, + displayHeight, + colourPrimaries, + matrixCoefficients, + transferCharacteristics, + fullRangeFlag, }; } catch (error) { console.error('Error parsing AVC SPS:', error); diff --git a/src/codec.ts b/src/codec.ts index b9414fd..07940d9 100644 --- a/src/codec.ts +++ b/src/codec.ts @@ -537,6 +537,7 @@ export const buildAudioCodecString = (codec: AudioCodec, numberOfChannels: numbe export type AacCodecInfo = { isMpeg2: boolean; + objectType: number | null; }; export const extractAudioCodecString = (trackInfo: { @@ -554,8 +555,15 @@ export const extractAudioCodecString = (trackInfo: { if (aacCodecInfo.isMpeg2) { return 'mp4a.67'; } else { - const audioSpecificConfig = parseAacAudioSpecificConfig(codecDescription); - return `mp4a.40.${audioSpecificConfig.objectType}`; + let objectType: number; + if (aacCodecInfo.objectType !== null) { + objectType = aacCodecInfo.objectType; + } else { + const audioSpecificConfig = parseAacAudioSpecificConfig(codecDescription); + objectType = audioSpecificConfig.objectType; + } + + return `mp4a.40.${objectType}`; } } else if (codec === 'mp3') { return 'mp3'; diff --git a/src/input-format.ts b/src/input-format.ts index 0182f4a..160c26a 100644 --- a/src/input-format.ts +++ b/src/input-format.ts @@ -23,10 +23,10 @@ import { MatroskaDemuxer } from './matroska/matroska-demuxer'; import { Mp3Demuxer } from './mp3/mp3-demuxer'; import { FRAME_HEADER_SIZE } from '../shared/mp3-misc'; import { ID3_V2_HEADER_SIZE, readId3V2Header } from './id3'; -import { readNextFrameHeader } from './mp3/mp3-reader'; +import { readNextMp3FrameHeader } from './mp3/mp3-reader'; import { OggDemuxer } from './ogg/ogg-demuxer'; import { WaveDemuxer } from './wave/wave-demuxer'; -import { MAX_FRAME_HEADER_SIZE, MIN_FRAME_HEADER_SIZE, readFrameHeader } from './adts/adts-reader'; +import { MAX_FRAME_HEADER_SIZE, MIN_FRAME_HEADER_SIZE, readAdtsFrameHeader } from './adts/adts-reader'; import { AdtsDemuxer } from './adts/adts-demuxer'; import { readAscii } from './reader'; import { FlacDemuxer } from './flac/flac-demuxer'; @@ -281,7 +281,7 @@ export class Mp3InputFormat extends InputFormat { currentPos = slice.filePos + id3V2Header.size; } - const firstResult = await readNextFrameHeader(input._reader, currentPos, currentPos + 4096); + const firstResult = await readNextMp3FrameHeader(input._reader, currentPos, currentPos + 4096); if (!firstResult) { return false; } @@ -295,7 +295,7 @@ export class Mp3InputFormat extends InputFormat { // Fine, we found one frame header, but we're still not entirely sure this is MP3. Let's check if we can find // another header right after it: - const secondResult = await readNextFrameHeader(input._reader, currentPos, currentPos + FRAME_HEADER_SIZE); + const secondResult = await readNextMp3FrameHeader(input._reader, currentPos, currentPos + FRAME_HEADER_SIZE); if (!secondResult) { return false; } @@ -444,7 +444,7 @@ export class AdtsInputFormat extends InputFormat { if (slice instanceof Promise) slice = await slice; if (!slice) return false; - const firstHeader = readFrameHeader(slice); + const firstHeader = readAdtsFrameHeader(slice); if (!firstHeader) { return false; } @@ -453,7 +453,7 @@ export class AdtsInputFormat extends InputFormat { if (slice instanceof Promise) slice = await slice; if (!slice) return false; - const secondHeader = readFrameHeader(slice); + const secondHeader = readAdtsFrameHeader(slice); if (!secondHeader) { return false; } diff --git a/src/input.ts b/src/input.ts index 5e78243..c164f44 100644 --- a/src/input.ts +++ b/src/input.ts @@ -118,6 +118,20 @@ export class Input implements Disposable { return demuxer.computeDuration(); } + /** + * Returns the timestamp at which the input file starts. More precisely, returns the smallest starting timestamp + * among all tracks. + */ + async getFirstTimestamp() { + const tracks = await this.getTracks(); + if (tracks.length === 0) { + return 0; + } + + const firstTimestamps = await Promise.all(tracks.map(x => x.getFirstTimestamp())); + return Math.min(...firstTimestamps); + } + /** Returns the list of all tracks of this input file. */ async getTracks() { const demuxer = await this._getDemuxer(); diff --git a/src/isobmff/isobmff-demuxer.ts b/src/isobmff/isobmff-demuxer.ts index b524bb2..4396aaa 100644 --- a/src/isobmff/isobmff-demuxer.ts +++ b/src/isobmff/isobmff-demuxer.ts @@ -1235,7 +1235,10 @@ export class IsobmffDemuxer extends Demuxer { const objectTypeIndication = readU8(slice); if (objectTypeIndication === 0x40 || objectTypeIndication === 0x67) { track.info.codec = 'aac'; - track.info.aacCodecInfo = { isMpeg2: objectTypeIndication === 0x67 }; + track.info.aacCodecInfo = { + isMpeg2: objectTypeIndication === 0x67, + objectType: null, + }; } else if (objectTypeIndication === 0x69 || objectTypeIndication === 0x6b) { track.info.codec = 'mp3'; } else if (objectTypeIndication === 0xdd) { diff --git a/src/matroska/matroska-demuxer.ts b/src/matroska/matroska-demuxer.ts index db421b6..22e2557 100644 --- a/src/matroska/matroska-demuxer.ts +++ b/src/matroska/matroska-demuxer.ts @@ -1057,6 +1057,7 @@ export class MatroskaDemuxer extends Demuxer { this.currentTrack.info.codec = 'aac'; this.currentTrack.info.aacCodecInfo = { isMpeg2: this.currentTrack.codecId.includes('MPEG2'), + objectType: null, }; this.currentTrack.info.codecDescription = this.currentTrack.codecPrivate; } else if (this.currentTrack.codecId === CODEC_STRING_MAP.mp3) { diff --git a/src/mp3/mp3-demuxer.ts b/src/mp3/mp3-demuxer.ts index fe654d8..2278a79 100644 --- a/src/mp3/mp3-demuxer.ts +++ b/src/mp3/mp3-demuxer.ts @@ -14,7 +14,7 @@ import { DEFAULT_TRACK_DISPOSITION, MetadataTags } from '../metadata'; import { PacketRetrievalOptions } from '../media-sink'; import { assert, AsyncMutex, binarySearchExact, binarySearchLessOrEqual, UNDETERMINED_LANGUAGE } from '../misc'; import { EncodedPacket, PLACEHOLDER_DATA } from '../packet'; -import { FrameHeader, getXingOffset, INFO, XING } from '../../shared/mp3-misc'; +import { Mp3FrameHeader, getXingOffset, INFO, XING } from '../../shared/mp3-misc'; import { ID3_V1_TAG_SIZE, ID3_V2_HEADER_SIZE, @@ -22,7 +22,7 @@ import { parseId3V2Tag, readId3V2Header, } from '../id3'; -import { readNextFrameHeader } from './mp3-reader'; +import { readNextMp3FrameHeader } from './mp3-reader'; import { readAscii, readBytes, Reader, readU32Be } from '../reader'; type Sample = { @@ -36,7 +36,7 @@ export class Mp3Demuxer extends Demuxer { reader: Reader; metadataPromise: Promise | null = null; - firstFrameHeader: FrameHeader | null = null; + firstFrameHeader: Mp3FrameHeader | null = null; loadedSamples: Sample[] = []; // All samples from the start of the file to lastLoadedPos metadataTags: MetadataTags | null = null; @@ -89,7 +89,7 @@ export class Mp3Demuxer extends Demuxer { } } - const result = await readNextFrameHeader(this.reader, this.lastLoadedPos, this.reader.fileSize); + const result = await readNextMp3FrameHeader(this.reader, this.lastLoadedPos, this.reader.fileSize); if (!result) { this.lastSampleLoaded = true; return; diff --git a/src/mp3/mp3-muxer.ts b/src/mp3/mp3-muxer.ts index f2c86e8..f4a67cb 100644 --- a/src/mp3/mp3-muxer.ts +++ b/src/mp3/mp3-muxer.ts @@ -13,7 +13,7 @@ import { Output, OutputAudioTrack } from '../output'; import { Mp3OutputFormat } from '../output-format'; import { EncodedPacket } from '../packet'; import { Writer } from '../writer'; -import { getXingOffset, INFO, readFrameHeader, XING } from '../../shared/mp3-misc'; +import { getXingOffset, INFO, readMp3FrameHeader, XING } from '../../shared/mp3-misc'; import { Mp3Writer, XingFrameData } from './mp3-writer'; import { Id3V2Writer } from '../id3'; @@ -65,7 +65,7 @@ export class Mp3Muxer extends Muxer { } const word = view.getUint32(0, false); - const header = readFrameHeader(word, null).header; + const header = readMp3FrameHeader(word, null).header; if (!header) { throw new Error('Invalid MP3 header in sample.'); } diff --git a/src/mp3/mp3-reader.ts b/src/mp3/mp3-reader.ts index f489ca9..abc5625 100644 --- a/src/mp3/mp3-reader.ts +++ b/src/mp3/mp3-reader.ts @@ -6,11 +6,11 @@ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -import { FRAME_HEADER_SIZE, FrameHeader, readFrameHeader } from '../../shared/mp3-misc'; +import { FRAME_HEADER_SIZE, Mp3FrameHeader, readMp3FrameHeader } from '../../shared/mp3-misc'; import { Reader, readU32Be } from '../reader'; -export const readNextFrameHeader = async (reader: Reader, startPos: number, until: number | null): Promise<{ - header: FrameHeader; +export const readNextMp3FrameHeader = async (reader: Reader, startPos: number, until: number | null): Promise<{ + header: Mp3FrameHeader; startPos: number; } | null> => { let currentPos = startPos; @@ -22,7 +22,7 @@ export const readNextFrameHeader = async (reader: Reader, startPos: number, unti const word = readU32Be(slice); - const result = readFrameHeader(word, reader.fileSize !== null ? reader.fileSize - currentPos : null); + const result = readMp3FrameHeader(word, reader.fileSize !== null ? reader.fileSize - currentPos : null); if (result.header) { return { header: result.header, startPos: currentPos }; } diff --git a/src/mpeg-ts/mpeg-ts-demuxer.ts b/src/mpeg-ts/mpeg-ts-demuxer.ts index 61c6baf..c9860bf 100644 --- a/src/mpeg-ts/mpeg-ts-demuxer.ts +++ b/src/mpeg-ts/mpeg-ts-demuxer.ts @@ -1,13 +1,71 @@ -import { Demuxer } from "../demuxer"; -import { Input } from "../input"; -import { InputTrack } from "../input-track"; -import { Bitstream } from "../misc"; -import { readBytes, Reader, readU24Be, readU8 } from "../reader"; +import { SAMPLES_PER_AAC_FRAME } from '../adts/adts-demuxer'; +import { MAX_FRAME_HEADER_SIZE, MIN_FRAME_HEADER_SIZE, readAdtsFrameHeader } from '../adts/adts-reader'; +import { aacChannelMap, AacCodecInfo, aacFrequencyTable, AudioCodec, extractAudioCodecString, extractVideoCodecString, MediaCodec, VideoCodec } from '../codec'; +import { + AvcDecoderConfigurationRecord, + AvcNalUnitType, + determineVideoPacketType, + extractAvcDecoderConfigurationRecord, + extractNalUnitTypeForAvc, + findNalUnitsInAnnexB, + parseAvcSps, +} from '../codec-data'; +import { Demuxer } from '../demuxer'; +import { Input } from '../input'; +import { InputAudioTrack, InputAudioTrackBacking, InputTrack, InputTrackBacking, InputVideoTrack, InputVideoTrackBacking } from '../input-track'; +import { PacketRetrievalOptions } from '../media-sink'; +import { DEFAULT_TRACK_DISPOSITION, MetadataTags } from '../metadata'; +import { assert, binarySearchLessOrEqual, Bitstream, COLOR_PRIMARIES_MAP_INVERSE, last, MATRIX_COEFFICIENTS_MAP_INVERSE, removeItem, Rotation, roundIfAlmostInteger, roundToMultiple, TRANSFER_CHARACTERISTICS_MAP_INVERSE, UNDETERMINED_LANGUAGE } from '../misc'; +import { EncodedPacket, PacketType, PLACEHOLDER_DATA } from '../packet'; +import { FileSlice, readBytes, Reader, readU16Be, readU8 } from '../reader'; + +const TIMESCALE = 90_000; // MPEG-TS timestamps run on a 90 kHz clock + +type ElementaryStream = { + demuxer: MpegTsDemuxer; + pid: number; + streamType: number; + initialized: boolean; + firstSection: Section | null; + info: { + type: 'video'; + codec: VideoCodec; + avcCodecInfo: AvcDecoderConfigurationRecord | null; + colorSpace: VideoColorSpaceInit; + width: number; + height: number; + } | { + type: 'audio'; + codec: AudioCodec; + aacCodecInfo: AacCodecInfo | null; + numberOfChannels: number; + sampleRate: number; + }; +}; + +type ElementaryVideoStream = ElementaryStream & { info: { type: 'video' } }; +type ElementaryAudioStream = ElementaryStream & { info: { type: 'audio' } }; + +type PacketHeader = { + payloadUnitStartIndicator: number; + pid: number; + adaptationFieldControl: number; + body: Uint8Array; +}; + +type Section = { + startPos: number; + endPos: number; + pid: number; + payload: Uint8Array; +}; export class MpegTsDemuxer extends Demuxer { reader: Reader; metadataPromise: Promise | null = null; + elementaryStreams: ElementaryStream[] = []; + tracks: InputTrack[] = []; constructor(input: Input) { super(input); @@ -17,18 +75,300 @@ export class MpegTsDemuxer extends Demuxer { async readMetadata() { return this.metadataPromise ??= (async () => { - for (let i = 0; i < 2; i++) { - this.readPacket(188 * i); + let currentPos = 0; + + let programMapPid: number | null = null; + let hasProgramMap = false; + + while (true) { + const section = await this.readSection(currentPos); + if (!section) { + break; + } + + const BYTES_BEFORE_SECTION_LENGTH = 3; + const BITS_IN_CRC_32 = 32; + + if (section.pid === 0) { + const bitstream = new Bitstream(section.payload); + const pointerField = bitstream.readAlignedByte(); + + bitstream.skipBits(8 * pointerField); + + bitstream.skipBits(14); + const sectionLength = bitstream.readBits(10); + + bitstream.skipBits(40); + + while (8 * (sectionLength + BYTES_BEFORE_SECTION_LENGTH) - bitstream.pos > BITS_IN_CRC_32) { + const programNumber = bitstream.readBits(16); + bitstream.skipBits(3); // Reserved + + if (programNumber !== 0) { + if (programMapPid !== null) { + throw new Error('Only files with a single program are supported.'); + } else { + programMapPid = bitstream.readBits(13); + } + } + } + + if (programMapPid === null) { + throw new Error('Program Association Table must link to a Program Map Table.'); + } + } else if (section.pid === programMapPid) { + const bitstream = new Bitstream(section.payload); + const pointerField = bitstream.readAlignedByte(); + + bitstream.skipBits(8 * pointerField); + + bitstream.skipBits(12); + const sectionLength = bitstream.readBits(12); + + bitstream.skipBits(43); + const pcrPid = bitstream.readBits(13); + + bitstream.skipBits(6); + + // "The remaining 10 bits specify the number of bytes of the descriptors immediately following the + // program_info_length field" + const programInfoLength = bitstream.readBits(10); + bitstream.skipBits(8 * programInfoLength); + + while (8 * (sectionLength + BYTES_BEFORE_SECTION_LENGTH) - bitstream.pos > BITS_IN_CRC_32) { + const streamType = bitstream.readBits(8); + bitstream.skipBits(3); + const elementaryPid = bitstream.readBits(13); + + bitstream.skipBits(6); + const esInfoLength = bitstream.readBits(10); + + bitstream.skipBits(8 * esInfoLength); + + let info: ElementaryStream['info'] | null = null; + + switch (streamType) { + case 0xf: { + info = { + type: 'audio', + codec: 'aac', + aacCodecInfo: null, + numberOfChannels: -1, + sampleRate: -1, + }; + }; break; + + case 0x1b: { + info = { + type: 'video', + codec: 'avc', + avcCodecInfo: null, + colorSpace: { + primaries: null, + transfer: null, + matrix: null, + fullRange: null, + }, + width: -1, + height: -1, + }; + }; break; + } + + if (info) { + this.elementaryStreams.push({ + demuxer: this, + pid: elementaryPid, + streamType, + initialized: false, + firstSection: null, + info, + }); + } + } + + hasProgramMap = true; + } else { + const elementaryStream = this.elementaryStreams.find(x => x.pid === section.pid); + if (elementaryStream && !elementaryStream.initialized) { + const pesPacket = readPesPacket(section); + if (!pesPacket) { + throw new Error( + `Couldn't read first PES packet for Elementary Stream with PID ${elementaryStream.pid}`, + ); + } + + elementaryStream.firstSection = section; + + if (elementaryStream.info.type === 'video') { + if (elementaryStream.info.codec === 'avc') { + elementaryStream.info.avcCodecInfo + = extractAvcDecoderConfigurationRecord(pesPacket.data); + + if (!elementaryStream.info.avcCodecInfo) { + throw new Error('TODO message'); + } + + const nalUnits = findNalUnitsInAnnexB(pesPacket.data); + const spsUnit = nalUnits.find(x => extractNalUnitTypeForAvc(x) === AvcNalUnitType.SPS)!; + const spsInfo = parseAvcSps(spsUnit)!; + + elementaryStream.info.width = spsInfo.displayWidth; + elementaryStream.info.height = spsInfo.displayHeight; + elementaryStream.info.colorSpace = { + primaries: COLOR_PRIMARIES_MAP_INVERSE[spsInfo.colourPrimaries] as + VideoColorPrimaries | undefined, + transfer: TRANSFER_CHARACTERISTICS_MAP_INVERSE[spsInfo.transferCharacteristics] as + VideoTransferCharacteristics | undefined, + matrix: MATRIX_COEFFICIENTS_MAP_INVERSE[spsInfo.matrixCoefficients] as + VideoMatrixCoefficients | undefined, + fullRange: !!spsInfo.fullRangeFlag, + }; + + elementaryStream.initialized = true; + } + } else { + if (elementaryStream.info.codec === 'aac') { + const slice = FileSlice.tempFromBytes(pesPacket.data); + const header = readAdtsFrameHeader(slice); + if (!header) { + throw new Error('TODO message'); + } + + elementaryStream.info.aacCodecInfo = { + isMpeg2: false, + objectType: header.objectType, + }; + elementaryStream.info.numberOfChannels + = aacChannelMap[header.channelConfiguration]!; + elementaryStream.info.sampleRate + = aacFrequencyTable[header.samplingFrequencyIndex]!; + + elementaryStream.initialized = true; + } + } + } + } + + const isDone = hasProgramMap && this.elementaryStreams.every(x => x.initialized); + if (isDone) { + break; + } + + currentPos = section.endPos; + } + + for (const stream of this.elementaryStreams) { + if (stream.info.type === 'video') { + this.tracks.push(new InputVideoTrack( + this.input, + new MpegTsVideoTrackBacking(stream as ElementaryVideoStream)), + ); + } else { + this.tracks.push(new InputAudioTrack( + this.input, + new MpegTsAudioTrackBacking(stream as ElementaryAudioStream)), + ); + } } })(); } async getTracks(): Promise { await this.readMetadata(); - return []; + return this.tracks; } - async readPacket(pos: number) { + async getMetadataTags(): Promise { + return {}; // TODO + } + + async computeDuration() { + const tracks = await this.getTracks(); + const trackDurations = await Promise.all(tracks.map(x => x.computeDuration())); + return Math.max(0, ...trackDurations); + } + + async getMimeType(): Promise { + return 'video/MP2T'; // TODO TODO + } + + async readSection(startPos: number): Promise
{ + let endPos = startPos; + let currentPos = startPos; + const chunks: Uint8Array[] = []; + + let firstPacket: PacketHeader | null = null; + + while (true) { + const packet = await this.readPacket(currentPos); + currentPos += 188; + + if (!packet) { + break; + } + + if (!firstPacket) { + if (packet.payloadUnitStartIndicator === 0) { + break; + } + + firstPacket = packet; + } else { + if (packet.pid !== firstPacket.pid) { + continue; // Ignore this packet + } + + if (packet.payloadUnitStartIndicator === 1) { + break; + } + } + + const hasAdaptationField = !!(packet.adaptationFieldControl & 0b10); + const hasPayload = !!(packet.adaptationFieldControl & 0b01); + + let adaptationFieldLength = 0; + if (hasAdaptationField) { + adaptationFieldLength = 1 + packet.body[0]!; + } + + if (hasPayload) { + if (adaptationFieldLength === 0) { + chunks.push(packet.body); + } else { + chunks.push(packet.body.subarray(adaptationFieldLength)); + } + } + + endPos = currentPos; + } + + if (!firstPacket) { + return null; + } + + let merged: Uint8Array; + if (chunks.length === 1) { + merged = chunks[0]!; + } else { + const totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0); + merged = new Uint8Array(totalLength); + let offset = 0; + for (const chunk of chunks) { + merged.set(chunk, offset); + offset += chunk.length; + } + } + + return { + startPos, + endPos, + pid: firstPacket.pid, + payload: merged, + }; + } + + async readPacket(pos: number): Promise { let slice = this.reader.requestSlice(pos, 188); if (slice instanceof Promise) slice = await slice; @@ -36,16 +376,1109 @@ export class MpegTsDemuxer extends Demuxer { return null; } - const bitstream = new Bitstream(readBytes(slice, 3)); - const syncByte = bitstream.readBits(8); - const transportErrorIndicator = bitstream.readBits(1); - const payloadUnitStartIndicator = bitstream.readBits(1); - const transportPriority = bitstream.readBits(1); - const pid = bitstream.readBits(13); - const transportScramblingControl = bitstream.readBits(2); - const adaptationFieldControl = bitstream.readBits(2); - const continuityCounter = bitstream.readBits(4); + const syncByte = readU8(slice); + if (syncByte !== 0x47) { + throw new Error('Invalid sync byte.'); + } - console.log(syncByte, transportErrorIndicator, payloadUnitStartIndicator, transportPriority, pid, transportScramblingControl, adaptationFieldControl, continuityCounter); + const nextTwoBytes = readU16Be(slice); + const transportErrorIndicator = nextTwoBytes >> 15; + const payloadUnitStartIndicator = (nextTwoBytes >> 14) & 0x1; + const transportPriority = (nextTwoBytes >> 13) & 0x1; + const pid = nextTwoBytes & 0x1FFF; + + const nextByte = readU8(slice); + const transportScramblingControl = nextByte >> 6; + const adaptationFieldControl = (nextByte >> 4) & 0x3; + const continuityCounter = nextByte & 0xF; + + return { + payloadUnitStartIndicator, + pid, + adaptationFieldControl, + body: readBytes(slice, 184), + }; } -} \ No newline at end of file +} + +type PesPacket = { + sectionStartPos: number; + sectionEndPos: number; + pts: number; + data: Uint8Array; +}; + +const readPesPacket = (section: Section): PesPacket | null => { + const bitstream = new Bitstream(section.payload); + + const startCodePrefix = bitstream.readBits(24); + if (startCodePrefix !== 0x000001) { + return null; + } + + const streamId = bitstream.readBits(8); + + const pesPacketLength = bitstream.readBits(16); + + if ( + streamId === 0b10111100 // program_stream_map + || streamId === 0b10111110 // padding_stream + || streamId === 0b10111111 // private_stream_2 + || streamId === 0b11110000 // ECM + || streamId === 0b11110001 // EMM + || streamId === 0b11111111 // program_stream_directory + || streamId === 0b11110010 // DSMCC_stream + || streamId === 0b11111000 // ITU-T Rec. H.222.1 type E stream + ) { + return null; + } + + bitstream.skipBits(8); + + const ptsDtsFlags = bitstream.readBits(2); + + bitstream.skipBits(6); + const pesHeaderDataLength = bitstream.readBits(8); + const pesHeaderEndPos = bitstream.pos + 8 * pesHeaderDataLength; + + if (ptsDtsFlags !== 0b10 && ptsDtsFlags !== 0b11) { + return null; // Support only timestamped packets + } + + let pts = 0; + if (ptsDtsFlags === 0b10 || ptsDtsFlags === 0b11) { + bitstream.skipBits(4); + pts += bitstream.readBits(3) * (1 << 30); + bitstream.skipBits(1); + pts += bitstream.readBits(15) * (1 << 15); + bitstream.skipBits(1); + pts += bitstream.readBits(15); + } else { + return null; // Support only timestamped packets + } + + bitstream.pos = pesHeaderEndPos; + + const bytePos = pesHeaderEndPos / 8; + assert(Number.isInteger(bytePos)); + + const data = section.payload.subarray(bytePos); + + return { + sectionStartPos: section.startPos, + sectionEndPos: section.endPos, + pts, + data, + }; +}; + +abstract class MpegTsTrackBacking implements InputTrackBacking { + /** + * Reference PES packets, spread throughout the file, to be used to speed up random access and perform + * binary search for packets. + */ + referencePesPackets: PesPacket[] = []; + endReferencePesPacketAdded = false; + readingContexts = new WeakMap(); + + constructor(public elementaryStream: ElementaryStream) {} + + getId() { + return this.elementaryStream.pid; + } + + getCodec(): MediaCodec | null { + throw new Error('Not implemented on base class.'); + } + + getInternalCodecId() { + return this.elementaryStream.streamType; + } + + getName() { + return null; + } + + getLanguageCode() { + return UNDETERMINED_LANGUAGE; + } + + getDisposition() { + return DEFAULT_TRACK_DISPOSITION; + } + + getTimeResolution() { + return TIMESCALE; + } + + async computeDuration(): Promise { + const lastPacket = await this.getPacket(Infinity, { metadataOnly: true }); + return (lastPacket?.timestamp ?? 0) + (lastPacket?.duration ?? 0); + } + + async getFirstTimestamp(): Promise { + const firstPacket = await this.getFirstPacket({ metadataOnly: true }); + return firstPacket?.timestamp ?? 0; + } + + abstract getPacketType(packetData: Uint8Array): PacketType; + abstract markNextPacket(context: PacketReadingContext): Promise; + + maybeInsertReferencePacket(pesPacket: PesPacket, force: boolean) { + const index = binarySearchLessOrEqual(this.referencePesPackets, pesPacket.pts, x => x.pts); + if (index >= 0) { + // Since pts and file position don't necessarily have a monotonic relationship (since pts can go crazy), + // let's see if inserting at the given index would violate the file position order. If so, return. + const entry = this.referencePesPackets[index]!; + if (pesPacket.sectionStartPos <= entry.sectionStartPos) { + return false; + } + + // Too close temporally + if (!force && pesPacket.pts - entry.pts < TIMESCALE / 2) { + return false; + } + + if (index < this.referencePesPackets.length - 1) { + const nextEntry = this.referencePesPackets[index + 1]!; + if (nextEntry.sectionStartPos < pesPacket.sectionStartPos) { + return false; + } + + // Too close temporally + if (!force && nextEntry.pts - pesPacket.pts < TIMESCALE / 2) { + return false; + } + } + } + + this.referencePesPackets.splice(index + 1, 0, pesPacket); + return true; + } + + async getFirstPacket(options: PacketRetrievalOptions): Promise { + const section = this.elementaryStream.firstSection; + assert(section); + + const pesPacket = readPesPacket(section); + if (!pesPacket) { + throw new Error('TODO message'); + } + + const context = new PacketReadingContext(this, pesPacket, true); + await this.markNextPacket(context); + + return context.toEncodedPacket(options); + } + + async getNextPacket(packet: EncodedPacket, options: PacketRetrievalOptions): Promise { + const context = this.readingContexts.get(packet); + if (!context) { + throw new Error('Packet was not created from this track.'); + } + + const clone = context.clone(); + await this.markNextPacket(clone); + + return clone.toEncodedPacket(options); + } + + async getNextKeyPacket(packet: EncodedPacket, options: PacketRetrievalOptions): Promise { + let currentPacket: EncodedPacket | null = packet; + + // Just loop until we hit one + while (true) { + currentPacket = await this.getNextPacket(currentPacket, options); + + if (!currentPacket) { + return null; + } + + if (currentPacket.type === 'key') { + return currentPacket; + } + } + } + + getPacket(timestamp: number, options: PacketRetrievalOptions): Promise { + return this.doPacketLookup(timestamp, false, options); + } + + getKeyPacket(timestamp: number, options: PacketRetrievalOptions): Promise { + return this.doPacketLookup(timestamp, true, options); + } + + abstract getPacketLookaround(): number; + + /** + * Searches for the packet with the largest timestamp not larger than `timestamp` in the file, using a combination + * of binary search and linear refinement. + */ + async doPacketLookup( + timestamp: number, + keyframesOnly: boolean, + options: PacketRetrievalOptions, + ): Promise { + const searchPts = roundIfAlmostInteger(timestamp * TIMESCALE); + + const demuxer = this.elementaryStream.demuxer; + const reader = demuxer.reader; + + if (reader.fileSize === null) { + throw new Error('TODO TODO TODO'); + } + + if (this.referencePesPackets.length === 0) { + // We've never read a packet, let's read the first one + const firstPacket = await this.getFirstPacket({}); + if (!firstPacket) { + return null; + } + + // @ts-expect-error Faulty inference + assert(this.referencePesPackets.length === 1); + } + + let currentIndex = binarySearchLessOrEqual(this.referencePesPackets, searchPts, x => x.pts); + if (currentIndex === -1) { + return null; // We're before the first packet + } + + // If we're at the end of the reference array, we must make sure we also know about the last packet of the + // track. Without it, we can't perform binary search. + const needsToLookForLastPacket + = currentIndex === this.referencePesPackets.length - 1 && !this.endReferencePesPacketAdded; + if (needsToLookForLastPacket) { + // Todo, better sync logic here? + let currentPos = reader.fileSize - 188; + let packet = await demuxer.readPacket(currentPos); + if (!packet) { + return null; + } + + while (packet.pid !== this.elementaryStream.pid || packet.payloadUnitStartIndicator === 0) { + currentPos -= 188; + const previousPacket = await demuxer.readPacket(currentPos); + if (!previousPacket) { + return null; + } + + packet = previousPacket; + } + + const section = await demuxer.readSection(currentPos); + assert(section); + + const pesPacket = readPesPacket(section); + if (!pesPacket) { + throw new Error('TODO message'); + } + + this.maybeInsertReferencePacket(pesPacket, true); + this.endReferencePesPacketAdded = true; + } + + // Find the reference point closest to the search timestamp + currentIndex = binarySearchLessOrEqual(this.referencePesPackets, searchPts, x => x.pts); + assert(currentIndex !== -1); + + while (true) { + const currentEntry = this.referencePesPackets[currentIndex]!; + const nextEntry = this.referencePesPackets[currentIndex + 1]; + + if (searchPts - currentEntry.pts < TIMESCALE || !nextEntry) { + // We're at the end or close enough to the entry to the left, stop + break; + } + + // Jump in between the two entries, and then find a fitting packet there + const midpoint = roundToMultiple((currentEntry.sectionStartPos + nextEntry.sectionStartPos) / 2, 188); + let currentPos = midpoint; + let packet = await demuxer.readPacket(currentPos); + assert(packet); + + while ( + currentPos < nextEntry.sectionStartPos + && (packet.pid !== this.elementaryStream.pid || packet.payloadUnitStartIndicator === 0) + ) { + currentPos += 188; + const previousPacket = await demuxer.readPacket(currentPos); + if (!previousPacket) { + return null; + } + + packet = previousPacket; + } + + if (currentPos >= nextEntry.sectionStartPos) { + // We couldn't find a packet in the middle + break; + } + + const section = await demuxer.readSection(currentPos); + assert(section); + + const pesPacket = readPesPacket(section); + if (!pesPacket) { + throw new Error('TODO message'); + } + + const addedPoint = this.maybeInsertReferencePacket(pesPacket, false); + if (!addedPoint) { + break; // Should rarely kick + } + + if (pesPacket.pts <= searchPts) { + // The midpoint packet is to the left of our search timestamp, so continue with the right half now + currentIndex++; + } + } + + let currentPesPacket = this.referencePesPackets[currentIndex]!; + assert(currentPesPacket.pts <= searchPts); + + /** Stores the best PES packet we've found so far (that meets all required criteria). */ + let bestPesPacket: PesPacket | null = null; + + const pesPacketHasKeyframe = async (pesPacket: PesPacket) => { + const context = new PacketReadingContext(this, pesPacket, false); + await this.markNextPacket(context); + + if (!context.suppliedPacket) { + return false; + } + + return this.getPacketType(context.suppliedPacket.data); + }; + + if (!keyframesOnly || await pesPacketHasKeyframe(currentPesPacket)) { + bestPesPacket = currentPesPacket; + } + + const advancedPesPackets = [bestPesPacket]; + + // Starting from the binary search guess, let's now find the moment where the packet timestamps cross the + // search timestamp. This point will then be used as the center around which we search. + outer: + while (true) { + let currentPos = currentPesPacket.sectionEndPos; + + while (true) { + const packet = await demuxer.readPacket(currentPos); + if (!packet) { + break outer; // End of file + } + + if (packet.pid === this.elementaryStream.pid) { + break; + } + + currentPos += 188; + } + + const nextSection = await demuxer.readSection(currentPos); + if (!nextSection) { + break; + } + + const nextPesPacket = readPesPacket(nextSection); + if (!nextPesPacket) { + throw new Error('TODO message'); + } + + if (nextPesPacket.pts > searchPts) { + // The timestamps cross the search timestamp, stop + break; + } + + // Collect matching packets we find along the way + if ( + (bestPesPacket === null || bestPesPacket.pts < nextPesPacket.pts) + && nextPesPacket.pts <= searchPts + && (!keyframesOnly || await pesPacketHasKeyframe(nextPesPacket)) + ) { + bestPesPacket = nextPesPacket; + } + + currentPesPacket = nextPesPacket; + advancedPesPackets.push(nextPesPacket); + } + + // Lookaround is needed in the first place because packets don't need to appear in PTS order, they only appear + // in decode order. When B-frames are present, finding the packet that's actually closest to the search + // timestamp requires searching a small local window. + const lookaround = this.getPacketLookaround(); + + // Depending on how long the previous scan went, we might not need to do the full lookbehind, or even none at + // all if we're lucky + const lookbehindNeeded = Math.max(lookaround - advancedPesPackets.length + 1, 0); + let minPos = advancedPesPackets[0]!.sectionStartPos; + + /** Scans `n` contiguous PES packets in succession. */ + const doLinearScan = async (startPos: number, n: number) => { + let currentPos = startPos; + + outer: + for (let i = 0; i < n; i++) { + while (true) { + const packet = await demuxer.readPacket(currentPos); + if (!packet) { + break outer; // End of file + } + + if (packet.pid === this.elementaryStream.pid) { + break; + } + + currentPos += 188; + } + + const section = await demuxer.readSection(currentPos); + assert(section); + assert(section.pid === this.elementaryStream.pid); + + const pesPacket = readPesPacket(section); + if (!pesPacket) { + throw new Error('TODO message'); + } + + if ( + (bestPesPacket === null || bestPesPacket.pts < pesPacket.pts) + && pesPacket.pts <= searchPts + && (!keyframesOnly || await pesPacketHasKeyframe(pesPacket)) + ) { + bestPesPacket = pesPacket; + } + + currentPos = section.endPos; + } + }; + + // Lookbehind + if (lookbehindNeeded > 0) { + outer: + for (let i = 0; i < lookbehindNeeded; i++) { + let currentPos = minPos; + + while (true) { + currentPos -= 188; + + const packet = await demuxer.readPacket(currentPos); + if (!packet) { + break outer; + } + + if (packet.pid === this.elementaryStream.pid && packet.payloadUnitStartIndicator === 1) { + break; + } + } + + minPos = currentPos; + } + + await doLinearScan(minPos, lookbehindNeeded); + } + + // Lookahead + await doLinearScan(currentPesPacket.sectionEndPos, lookaround); + + // If we're looking specifically for a keyframe but haven't found one yet, that means we'll need to go left + // until we find one. + if (!bestPesPacket && keyframesOnly) { + let currentPos = minPos; + + while (true) { + currentPos -= 188; + + const packet = await demuxer.readPacket(currentPos); + if (!packet) { + break; + } + + if (packet.pid === this.elementaryStream.pid && packet.payloadUnitStartIndicator === 1) { + const section = await demuxer.readSection(currentPos); + assert(section); + + const pesPacket = readPesPacket(section); + if (!pesPacket) { + throw new Error('TODO message'); + } + + if (pesPacket.pts <= searchPts && (await pesPacketHasKeyframe(pesPacket))) { + bestPesPacket = pesPacket; + break; + } + } + } + } + + if (!bestPesPacket) { + // Nothing was found + return null; + } + + // Final stage: we found the best PES packet, but that PES packet might contain multiple individual encoded + // packets. Or, it might not be the start of an encoded packet, and simply a continuation of a previous one. + // So, we have one last search to do. + const searchTimestamp = searchPts / TIMESCALE; // We use this instead of 'timestamp' due to the rounding + while (true) { + const context = new PacketReadingContext(this, bestPesPacket, false); // Capped context + + let bestPacket: EncodedPacket | null = null; + + while (true) { + context.suppliedPacket = null; + await this.markNextPacket(context); + const packet = context.toEncodedPacket(options); + if (!packet) { + break; + } + + const eligible = packet.timestamp <= searchTimestamp && (!keyframesOnly || packet.type === 'key'); + if (!eligible) { + continue; + } + + if (!bestPacket || bestPacket.timestamp < packet.timestamp) { + bestPacket = packet; + } + } + + if (bestPacket) { + return bestPacket; + } + + // We didn't find an encoded packet! Let's go to the previous PES packet until we find one. + + let currentPos = bestPesPacket.sectionStartPos; + + while (true) { + currentPos -= 188; + + const packet = await demuxer.readPacket(currentPos); + if (!packet) { + // Past start of file + return null; + } + + if (packet.pid === this.elementaryStream.pid && packet.payloadUnitStartIndicator === 1) { + const section = await demuxer.readSection(currentPos); + assert(section); + + const pesPacket = readPesPacket(section); + if (!pesPacket) { + throw new Error('TODO message'); + } + + if (pesPacket.pts <= searchPts) { + bestPesPacket = pesPacket; + break; + } + } + } + } + } +} + +class MpegTsVideoTrackBacking extends MpegTsTrackBacking implements InputVideoTrackBacking { + override elementaryStream: ElementaryVideoStream; + decoderConfig: VideoDecoderConfig; + + constructor(elementaryStream: ElementaryVideoStream) { + super(elementaryStream); + this.elementaryStream = elementaryStream; + + this.decoderConfig = { + codec: extractVideoCodecString({ + width: this.elementaryStream.info.width, + height: this.elementaryStream.info.height, + codec: this.elementaryStream.info.codec, + codecDescription: null, + colorSpace: this.elementaryStream.info.colorSpace, + avcType: 1, + avcCodecInfo: this.elementaryStream.info.avcCodecInfo, + hevcCodecInfo: null, + vp9CodecInfo: null, + av1CodecInfo: null, + }), + codedWidth: this.elementaryStream.info.width, + codedHeight: this.elementaryStream.info.height, + colorSpace: this.elementaryStream.info.colorSpace, + }; + } + + override getCodec(): VideoCodec { + return this.elementaryStream.info.codec; + } + + getCodedWidth() { + return this.elementaryStream.info.width; + } + + getCodedHeight() { + return this.elementaryStream.info.height; + } + + getRotation(): Rotation { + return 0; + } + + async getColorSpace(): Promise { + return this.elementaryStream.info.colorSpace; + } + + async canBeTransparent() { + return false; + } + + async getDecoderConfig(): Promise { + return this.decoderConfig; + } + + override getPacketType(packetData: Uint8Array): PacketType { + return determineVideoPacketType(this.elementaryStream.info.codec, this.decoderConfig, packetData) ?? 'key'; + } + + override getPacketLookaround(): number { + // Due to B-frames. A lookaround of +-5 packets will pretty much guarantee we find the correct packet for a + // given timestamp, although of course, this could technically still fail. + // todo, use max_num_reorder_frames here? + return 5; + } + + override async markNextPacket(context: PacketReadingContext): Promise { + const CHUNK_SIZE = 128; + + let packetStartPos: number | null = null; + + while (true) { + let remaining = context.ensureBuffered(CHUNK_SIZE); + if (remaining instanceof Promise) remaining = await remaining; + + // Search for start codes in the current chunk + for (let i = 0; i < remaining; i++) { + const byte = context.readU8(); + + // Look for 0x00 as potential start of a start code + if (byte !== 0x00) { + continue; + } + + // Check if we have enough bytes to identify a start code + const posBeforeZero = context.currentPos - 1; + + let remaining = context.ensureBuffered(4); + if (remaining instanceof Promise) remaining = await remaining; + + if (remaining < 4) { + // Not enough data left + if (packetStartPos !== null) { + // Return what we have + const packetLength = context.endPos - packetStartPos; + context.seekTo(packetStartPos); + return context.supplyPacket(packetLength, 0); + } + return; + } + + // Read potential start code bytes + const b1 = context.readU8(); + const b2 = context.readU8(); + const b3 = context.readU8(); + + let startCodeLength = 0; + let nalUnitTypeByte: number | null = null; + + // Check for 4-byte start code (0x00000001) + if (b1 === 0x00 && b2 === 0x00 && b3 === 0x01) { + startCodeLength = 4; + nalUnitTypeByte = context.readU8(); + // context.skip(-1); + } else if (b1 === 0x00 && b2 === 0x01) { + // 3-byte start code (0x000001) + startCodeLength = 3; + nalUnitTypeByte = b3; + // context.skip(-1); // Back up since b3 is the NAL unit type byte + } + + if (startCodeLength === 0) { + // Not a start code, rewind and continue + context.seekTo(posBeforeZero + 1); + continue; + } + + const startCodePos = posBeforeZero; + + if (packetStartPos === null) { + // This is our first start code, mark packet start + packetStartPos = startCodePos; + continue; + } + + // We have a second start code. Check if it's an AUD. + if (nalUnitTypeByte !== null) { + const nalUnitType = extractNalUnitTypeForAvc(new Uint8Array([nalUnitTypeByte])); + + if (nalUnitType === AvcNalUnitType.AUD) { + // End the packet at this start code (before the AUD) + const packetLength = startCodePos - packetStartPos; + context.seekTo(packetStartPos); + return context.supplyPacket(packetLength, 0); + } + } + + // Not an AUD, continue searching + } + + if (remaining < CHUNK_SIZE) { + // End of stream + break; + } + } + + // End of stream - return remaining data if we have a packet start + if (packetStartPos !== null) { + const packetLength = context.endPos - packetStartPos; + context.seekTo(packetStartPos); + return context.supplyPacket(packetLength, 0); + } + } +} + +class MpegTsAudioTrackBacking extends MpegTsTrackBacking implements InputAudioTrackBacking { + override elementaryStream: ElementaryAudioStream; + + constructor(elementaryStream: ElementaryAudioStream) { + super(elementaryStream); + this.elementaryStream = elementaryStream; + } + + override getCodec(): AudioCodec { + return this.elementaryStream.info.codec; + } + + getNumberOfChannels() { + return this.elementaryStream.info.numberOfChannels; + } + + getSampleRate() { + return this.elementaryStream.info.sampleRate; + } + + async getDecoderConfig(): Promise { + return { + codec: extractAudioCodecString({ + codec: this.elementaryStream.info.codec, + codecDescription: null, + aacCodecInfo: this.elementaryStream.info.aacCodecInfo, + }), + numberOfChannels: this.elementaryStream.info.numberOfChannels, + sampleRate: this.elementaryStream.info.sampleRate, + }; + } + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + override getPacketType(packetData: Uint8Array): PacketType { + return 'key'; + } + + override getPacketLookaround(): number { + return 0; + } + + override async markNextPacket(context: PacketReadingContext): Promise { + const CHUNK_SIZE = 128; + + while (true) { + let remaining = context.ensureBuffered(CHUNK_SIZE); + if (remaining instanceof Promise) remaining = await remaining; + + for (let i = 0; i < remaining; i++) { + const byte = context.readU8(); + if (byte !== 0xff) { + continue; + } + + context.skip(-1); + const startPos = context.currentPos; + + let remaining = context.ensureBuffered(MAX_FRAME_HEADER_SIZE); + if (remaining instanceof Promise) remaining = await remaining; + + if (remaining < MAX_FRAME_HEADER_SIZE) { + return; + } + + const headerBytes = context.readBytes(MAX_FRAME_HEADER_SIZE); + const header = readAdtsFrameHeader(FileSlice.tempFromBytes(headerBytes)); + + if (header) { + context.seekTo(startPos); + + let remaining = context.ensureBuffered(header.frameLength); + if (remaining instanceof Promise) remaining = await remaining; + + return context.supplyPacket( + remaining, + Math.round(SAMPLES_PER_AAC_FRAME * TIMESCALE / this.elementaryStream.info.sampleRate), + ); + } + } + + if (remaining < CHUNK_SIZE) { + break; + } + } + } +} + +/** Stateful context used to extract exact encoded packets from the underlying data stream. */ +class PacketReadingContext { + backing: MpegTsTrackBacking; + pid: number; + demuxer: MpegTsDemuxer; + startingPesPacket: PesPacket; + uncapped: boolean; + + currentPos = 0; // Relative to the data in startingPesPacket + pesPackets: PesPacket[] = []; + currentPesPacketIndex = 0; + currentPesPacketPos = 0; + endPos = 0; + nextPts = 0; + + suppliedPacket: { + pts: number; + intrinsicDuration: number; + data: Uint8Array; + sequenceNumber: number; + } | null = null; + + constructor(backing: MpegTsTrackBacking, startingPesPacket: PesPacket, uncapped: boolean) { + this.backing = backing; + this.pid = backing.elementaryStream.pid; + this.demuxer = backing.elementaryStream.demuxer; + this.startingPesPacket = startingPesPacket; + this.uncapped = uncapped; + } + + clone() { + const clone = new PacketReadingContext(this.backing, this.startingPesPacket, this.uncapped); + clone.currentPos = this.currentPos; + clone.pesPackets = [...this.pesPackets]; + clone.currentPesPacketIndex = this.currentPesPacketIndex; + clone.currentPesPacketPos = this.currentPesPacketPos; + clone.endPos = this.endPos; + clone.nextPts = this.nextPts; + + return clone; + } + + ensureBuffered(length: number) { + const remaining = this.endPos - this.currentPos; + if (remaining >= length) { + return length; + } + + return this.bufferData(length - remaining) + .then(() => Math.min(this.endPos - this.currentPos, length)); + } + + getCurrentPesPacket() { + const packet = this.pesPackets[this.currentPesPacketIndex]; + assert(packet); + + return packet; + } + + async bufferData(length: number): Promise { + const targetEndPos = this.endPos + length; + + while (this.endPos < targetEndPos) { + let pesPacket: PesPacket; + if (this.pesPackets.length === 0) { + pesPacket = this.startingPesPacket; + } else { + // Find the next PES packet + let currentPos = last(this.pesPackets)!.sectionEndPos; + + while (true) { + const packet = await this.demuxer.readPacket(currentPos); + if (!packet) { + return; + } + + if (packet.pid === this.pid) { + break; + } + + currentPos += 188; + } + + const nextSection = await this.demuxer.readSection(currentPos); + if (!nextSection) { + return; + } + + const maybePesPacket = readPesPacket(nextSection); + if (!maybePesPacket) { + throw new Error('TODO message'); + } + + pesPacket = maybePesPacket; + } + + this.pesPackets.push(pesPacket); + this.endPos += pesPacket.data.byteLength; + + if (this.pesPackets.length === 1) { + // It's the first PES packet, set the PTS + this.nextPts = pesPacket.pts; + } + } + } + + readBytes(length: number) { + const currentPesPacket = this.getCurrentPesPacket(); + + const relativeStartOffset = this.currentPos - this.currentPesPacketPos; + const relativeEndOffset = relativeStartOffset + length; + + this.currentPos += length; + + if (relativeEndOffset <= currentPesPacket.data.byteLength) { + // Request can be satisfied with one PES packet + return currentPesPacket.data.subarray(relativeStartOffset, relativeEndOffset); + } + + // Data spans multiple PES packets, we must do some merging + const result = new Uint8Array(length); + result.set(currentPesPacket.data.subarray(relativeStartOffset)); + let offset = currentPesPacket.data.byteLength - relativeStartOffset; + + while (true) { + this.advanceCurrentPacket(); + const currentPesPacket = this.getCurrentPesPacket(); + const relativeStartOffset = 0; + const relativeEndOffset = length - offset; + + if (relativeEndOffset <= currentPesPacket.data.byteLength) { + result.set(currentPesPacket.data.subarray(relativeStartOffset, relativeEndOffset), offset); + break; + } + + result.set(currentPesPacket.data.subarray(relativeStartOffset), offset); + offset += currentPesPacket.data.byteLength; + } + + return result; + } + + readU8() { + let currentPesPacket = this.getCurrentPesPacket(); + + const relativeOffset = this.currentPos - this.currentPesPacketPos; + this.currentPos++; + + if (relativeOffset < currentPesPacket.data.byteLength) { + return currentPesPacket.data[relativeOffset]!; + } + + this.advanceCurrentPacket(); + + currentPesPacket = this.getCurrentPesPacket(); + return currentPesPacket.data[0]!; + } + + seekTo(pos: number) { + if (pos === this.currentPos) { + return; + } + + if (pos < this.currentPos) { + while (pos < this.currentPesPacketPos) { + // Move to the previous PES packet + this.currentPesPacketIndex--; + const currentPacket = this.getCurrentPesPacket(); + this.currentPesPacketPos -= currentPacket.data.byteLength; + this.nextPts = currentPacket.pts; + } + } else { + while (true) { + // Move to the next PES packet + const currentPesPacket = this.getCurrentPesPacket(); + const currentEndPos = this.currentPesPacketPos + currentPesPacket.data.byteLength; + + if (pos < currentEndPos) { + break; + } + + this.currentPesPacketPos += currentPesPacket.data.byteLength; + this.currentPesPacketIndex++; + + this.nextPts = this.getCurrentPesPacket().pts; + } + } + + this.currentPos = pos; + } + + skip(n: number) { + this.seekTo(this.currentPos + n); + } + + advanceCurrentPacket() { + this.currentPesPacketPos += this.getCurrentPesPacket().data.byteLength; + this.currentPesPacketIndex++; + + this.nextPts = this.getCurrentPesPacket().pts; + } + + /** Supplies the context with a new encoded packet, beginning at the current position. */ + supplyPacket(packetLength: number, intrinsicDuration: number) { + const currentPesPacket = this.getCurrentPesPacket(); + if (!this.uncapped && currentPesPacket !== this.startingPesPacket) { + // The packet is "outside" of the valid region, the valid region is any packet starting in the starting + // section + this.suppliedPacket = null; + return; + } + + this.backing.maybeInsertReferencePacket(currentPesPacket, false); + + const pts = this.nextPts; + this.nextPts += intrinsicDuration; + + // The sequence number is the starting position of the section the PES packet is in, PLUS the offset within the + // PES packet where the packet starts. + const sequenceNumber = currentPesPacket.sectionStartPos + (this.currentPos - this.currentPesPacketPos); + + this.suppliedPacket = { + pts, + intrinsicDuration, + data: this.readBytes(packetLength), + sequenceNumber, + }; + + this.pesPackets.splice(0, this.currentPesPacketIndex); + this.currentPesPacketIndex = 0; + } + + toEncodedPacket(options: PacketRetrievalOptions) { + if (!this.suppliedPacket) { + return null; + } + + const packet = new EncodedPacket( + options.metadataOnly ? PLACEHOLDER_DATA : this.suppliedPacket.data, + this.backing.getPacketType(this.suppliedPacket.data), + this.suppliedPacket.pts / TIMESCALE, + this.suppliedPacket.intrinsicDuration / TIMESCALE, + this.suppliedPacket.sequenceNumber, + this.suppliedPacket.data.byteLength, + ); + this.backing.readingContexts.set(packet, this); + + return packet; + } +} diff --git a/src/reader.ts b/src/reader.ts index ebcda44..3133094 100644 --- a/src/reader.ts +++ b/src/reader.ts @@ -20,6 +20,10 @@ export class Reader { throw new InputDisposedError(); } + if (start < 0) { + return null; + } + if (this.fileSize !== null && start + length > this.fileSize) { return null; } @@ -49,6 +53,10 @@ export class Reader { throw new InputDisposedError(); } + if (start < 0) { + return null; + } + if (this.fileSize !== null) { return this.requestSlice( start, diff --git a/test/browser/mpeg-ts-decoding.test.ts b/test/browser/mpeg-ts-decoding.test.ts new file mode 100644 index 0000000..0462bf5 --- /dev/null +++ b/test/browser/mpeg-ts-decoding.test.ts @@ -0,0 +1,46 @@ +import { test } from 'vitest'; +import { Input } from '../../src/input.js'; +import { UrlSource } from '../../src/source.js'; +import { ALL_FORMATS } from '../../src/input-format.js'; +import { VideoSampleSink, AudioSampleSink } from '../../src/media-sink.js'; +import { assert } from '../../src/misc.js'; + +test('MPEG-TS video samples are decodable', async () => { + using input = new Input({ + source: new UrlSource('/0.ts'), + formats: ALL_FORMATS, + }); + + const videoTrack = await input.getPrimaryVideoTrack(); + assert(videoTrack); + + const sink = new VideoSampleSink(videoTrack); + + let count = 0; + // eslint-disable-next-line @typescript-eslint/no-unused-vars + for await (using sample of sink.samples()) { + count++; + } + + assert(count > 0); +}); + +test('MPEG-TS audio samples are decodable', async () => { + using input = new Input({ + source: new UrlSource('/0.ts'), + formats: ALL_FORMATS, + }); + + const audioTrack = await input.getPrimaryAudioTrack(); + assert(audioTrack); + + const sink = new AudioSampleSink(audioTrack); + + let count = 0; + // eslint-disable-next-line @typescript-eslint/no-unused-vars + for await (using sample of sink.samples()) { + count++; + } + + assert(count > 0); +}); diff --git a/test/node/mpeg-ts-demuxing.test.ts b/test/node/mpeg-ts-demuxing.test.ts new file mode 100644 index 0000000..1af894e --- /dev/null +++ b/test/node/mpeg-ts-demuxing.test.ts @@ -0,0 +1,349 @@ +import { expect, test } from 'vitest'; +import { Input } from '../../src/input.js'; +import { FilePathSource } from '../../src/source.js'; +import path from 'node:path'; +import { ALL_FORMATS, MPEG_TS } from '../../src/input-format.js'; +import { assert } from '../../src/misc.js'; +import { EncodedPacketSink } from '../../src/media-sink.js'; +import { EncodedPacket } from '../../src/packet.js'; + +const __dirname = new URL('.', import.meta.url).pathname; + +test('MPEG-TS metadata reading', async () => { + using input = new Input({ + source: new FilePathSource(path.join(__dirname, '../public/0.ts')), + formats: ALL_FORMATS, + }); + + expect(await input.getFormat()).toBe(MPEG_TS); + expect(await input.getMimeType()).toBe('video/MP2T'); + + const tracks = await input.getTracks(); + expect(tracks).toHaveLength(2); + + const videoTrack = await input.getPrimaryVideoTrack(); + assert(videoTrack); + + expect(videoTrack.id).toBe(0x100); + expect(videoTrack.codec).toBe('avc'); + expect(videoTrack.internalCodecId).toBe(0x1b); + expect(videoTrack.displayWidth).toEqual(720); + expect(videoTrack.displayHeight).toEqual(720); + expect(videoTrack.timeResolution).toBe(90_000); + + const videoDecoderConfig = await videoTrack.getDecoderConfig(); + expect(videoDecoderConfig).toEqual({ + codec: 'avc1.640020', + codedWidth: 720, + codedHeight: 720, + colorSpace: { + primaries: 'bt2020', + transfer: 'hlg', + matrix: 'bt2020-ncl', + fullRange: false, + }, + // No description, it's Annex B + }); + + expect(await videoTrack.getFirstTimestamp()).toBe(10.033333333333333); + + const audioTrack = await input.getPrimaryAudioTrack(); + assert(audioTrack); + + expect(audioTrack.id).toBe(0x101); + expect(audioTrack.codec).toBe('aac'); + expect(audioTrack.numberOfChannels).toBe(2); + expect(audioTrack.sampleRate).toBe(48000); + + const audioDecoderConfig = await audioTrack.getDecoderConfig(); + expect(audioDecoderConfig).toEqual({ + codec: 'mp4a.40.2', + numberOfChannels: 2, + sampleRate: 48000, + // No description, it's ADTS audio + }); + + expect(await audioTrack.getFirstTimestamp()).toBe(10.012); +}); + +test('MPEG-TS AVC video packets', async () => { + using input = new Input({ + source: new FilePathSource(path.join(__dirname, '../public/0.ts')), + formats: ALL_FORMATS, + }); + + const videoTrack = await input.getPrimaryVideoTrack(); + assert(videoTrack); + + const sink = new EncodedPacketSink(videoTrack); + + const firstPacket = await sink.getFirstPacket(); + assert(firstPacket); + + expect([...firstPacket.data.slice(0, 4)]).toEqual([0, 0, 0, 1]); + expect(firstPacket.data.byteLength).toBe(23813); + expect(firstPacket.type).toBe('key'); + expect(firstPacket.timestamp).toBe(10.033333333333333); + expect(firstPacket.duration).toBe(0); + expect(firstPacket.sequenceNumber).not.toBe(-1); + + const firstPacketMetadataOnly = await sink.getFirstPacket({ metadataOnly: true }); + assert(firstPacketMetadataOnly); + expect(firstPacketMetadataOnly.data).toHaveLength(0); + expect(firstPacketMetadataOnly.byteLength).toBe(23813); + + const secondPacket = await sink.getNextPacket(firstPacket); + assert(secondPacket); + + expect([...secondPacket.data.slice(0, 4)]).toEqual([0, 0, 0, 1]); + expect(secondPacket.data.byteLength).toBe(5700); + expect(secondPacket.type).toBe('delta'); + expect(secondPacket.timestamp).toBe(10.1); + expect(secondPacket.duration).toBe(0); + expect(secondPacket.sequenceNumber).toBeGreaterThan(firstPacket.sequenceNumber); + + let currentPacket: EncodedPacket | null = firstPacket; + let count = 0; + + while (currentPacket) { + count++; + currentPacket = await sink.getNextPacket(currentPacket); + } + + expect(count).toBe(298); +}); + +test('MPEG-TS AAC audio packets', async () => { + using input = new Input({ + source: new FilePathSource(path.join(__dirname, '../public/0.ts')), + formats: ALL_FORMATS, + }); + + const audioTrack = await input.getPrimaryAudioTrack(); + assert(audioTrack); + + const sink = new EncodedPacketSink(audioTrack); + + const firstPacket = await sink.getFirstPacket(); + assert(firstPacket); + + expect(firstPacket.data[0]).toEqual(0xff); + expect(firstPacket.data.byteLength).toBe(348); + expect(firstPacket.type).toBe('key'); + expect(firstPacket.timestamp).toBe(10.012); + expect(firstPacket.duration).toBeCloseTo(0.021333333333333333); + expect(firstPacket.sequenceNumber).not.toBe(-1); + + const secondPacket = await sink.getNextPacket(firstPacket); + assert(secondPacket); + + expect(secondPacket.data[0]).toEqual(0xff); + expect(secondPacket.data.byteLength).toBe(349); + expect(secondPacket.type).toBe('key'); + expect(secondPacket.timestamp).toBeCloseTo(10.033333333333333); + expect(secondPacket.duration).toBeCloseTo(0.021333333333333333); + expect(secondPacket.sequenceNumber).toBeGreaterThan(firstPacket.sequenceNumber); + + let currentPacket: EncodedPacket | null = firstPacket; + let count = 0; + + while (currentPacket) { + count++; + currentPacket = await sink.getNextPacket(currentPacket); + } + + expect(count).toBe(234); +}); + +test('MPEG-TS video seeking', async () => { + using input = new Input({ + source: new FilePathSource(path.join(__dirname, '../public/0.ts')), + formats: ALL_FORMATS, + }); + + const videoTrack = await input.getPrimaryVideoTrack(); + assert(videoTrack); + + const sink = new EncodedPacketSink(videoTrack); + + const firstTimestamp = await videoTrack.getFirstTimestamp(); + const firstPacket = await sink.getPacket(firstTimestamp); + assert(firstPacket); + + expect(firstPacket.timestamp).toBe(firstTimestamp); + + const lastPacket = await sink.getPacket(Infinity); + assert(lastPacket); + + expect(lastPacket.timestamp).toBeCloseTo(14.983333333333333); + + const beforeFirst = await sink.getPacket(-10); + expect(beforeFirst).toBeNull(); + + const middlePacket = await sink.getPacket(12.5); + assert(middlePacket); + expect(middlePacket.timestamp).toBeCloseTo(12.5); + + const allPackets: EncodedPacket[] = []; + let currentPacket: EncodedPacket | null = firstPacket; + + while (currentPacket) { + allPackets.push(currentPacket); + currentPacket = await sink.getNextPacket(currentPacket); + } + + for (const packet of allPackets) { + const seekedPacked = await sink.getPacket(packet.timestamp); + assert(seekedPacked); + expect(seekedPacked.timestamp).toBe(packet.timestamp); // The correct timestamp was retrieved for this packet + expect(seekedPacked.sequenceNumber).toBe(packet.sequenceNumber); + } +}); + +test('MPEG-TS audio seeking', async () => { + using input = new Input({ + source: new FilePathSource(path.join(__dirname, '../public/0.ts')), + formats: ALL_FORMATS, + }); + + const audioTrack = await input.getPrimaryAudioTrack(); + assert(audioTrack); + + const sink = new EncodedPacketSink(audioTrack); + + const firstTimestamp = await audioTrack.getFirstTimestamp(); + const firstPacket = await sink.getPacket(firstTimestamp); + assert(firstPacket); + + expect(firstPacket.timestamp).toBe(firstTimestamp); + + const lastPacket = await sink.getPacket(Infinity); + assert(lastPacket); + + expect(lastPacket.timestamp).toBeCloseTo(14.982666666666667); + + const beforeFirst = await sink.getPacket(-10); + expect(beforeFirst).toBeNull(); + + const middlePacket = await sink.getPacket(12.5); + assert(middlePacket); + expect(middlePacket.timestamp).toBeCloseTo(12.486666666666666); + + const allPackets: EncodedPacket[] = []; + let currentPacket: EncodedPacket | null = firstPacket; + + while (currentPacket) { + allPackets.push(currentPacket); + currentPacket = await sink.getNextPacket(currentPacket); + } + + for (const packet of allPackets) { + const seekedPacket = await sink.getPacket(packet.timestamp); + assert(seekedPacket); + expect(seekedPacket.timestamp).toBe(packet.timestamp); // The correct timestamp was retrieved for this packet + expect(seekedPacket.sequenceNumber).toBe(packet.sequenceNumber); + } +}); + +test('MPEG-TS video key packets', async () => { + using input = new Input({ + source: new FilePathSource(path.join(__dirname, '../public/193039199_mp4_h264_aac_fhd_7.ts')), + formats: ALL_FORMATS, + }); + + const videoTrack = await input.getPrimaryVideoTrack(); + assert(videoTrack); + + const sink = new EncodedPacketSink(videoTrack); + + const firstPacket = await sink.getFirstPacket(); + assert(firstPacket); + expect(firstPacket.type).toBe('key'); + + const secondPacket = await sink.getNextPacket(firstPacket); + assert(secondPacket); + expect(secondPacket.type).toBe('delta'); + + const nextKeyPacket = await sink.getNextKeyPacket(firstPacket); + assert(nextKeyPacket); + expect(nextKeyPacket.type).toBe('key'); + expect(nextKeyPacket.sequenceNumber).toBeGreaterThan(secondPacket.sequenceNumber); + + const firstKeyPacket = await sink.getKeyPacket(firstPacket.timestamp + 1); + assert(firstKeyPacket); + expect(firstKeyPacket.type).toBe('key'); + expect(firstKeyPacket.sequenceNumber).toBe(firstPacket.sequenceNumber); + + const secondKeyPacket = await sink.getKeyPacket(15); + assert(secondKeyPacket); + expect(secondKeyPacket.type).toBe('key'); + expect(secondKeyPacket.sequenceNumber).toBe(nextKeyPacket.sequenceNumber); + + const lastKeyPacket = await sink.getKeyPacket(Infinity); + assert(lastKeyPacket); + expect(lastKeyPacket.type).toBe('key'); + expect(lastKeyPacket.sequenceNumber).toBe(secondKeyPacket.sequenceNumber); // It's actually the last key packet + + const allKeyPackets: EncodedPacket[] = []; + let currentKeyPacket: EncodedPacket | null = firstPacket; + + while (currentKeyPacket) { + allKeyPackets.push(currentKeyPacket); + currentKeyPacket = await sink.getNextKeyPacket(currentKeyPacket); + } + + for (const packet of allKeyPackets) { + const keyPacket = await sink.getKeyPacket(packet.timestamp); + assert(keyPacket); + expect(keyPacket.timestamp).toBe(packet.timestamp); // The correct timestamp was retrieved for this packet + expect(keyPacket.sequenceNumber).toBe(packet.sequenceNumber); + } +}); + +test('MPEG-TS audio key packets', async () => { + using input = new Input({ + source: new FilePathSource(path.join(__dirname, '../public/193039199_mp4_h264_aac_fhd_7.ts')), + formats: ALL_FORMATS, + }); + + const audioTrack = await input.getPrimaryAudioTrack(); + assert(audioTrack); + + const sink = new EncodedPacketSink(audioTrack); + + const firstPacket = await sink.getFirstPacket(); + assert(firstPacket); + expect(firstPacket.type).toBe('key'); + + const secondPacket = await sink.getNextPacket(firstPacket); + assert(secondPacket); + expect(secondPacket.type).toBe('key'); + + const nextKeyPacket = await sink.getNextKeyPacket(firstPacket); + assert(nextKeyPacket); + expect(nextKeyPacket.type).toBe('key'); + expect(nextKeyPacket.sequenceNumber).toBe(secondPacket.sequenceNumber); // All audio packets are key packets + + const lastPacket = await sink.getPacket(Infinity); + assert(lastPacket); + + const lastKeyPacket = await sink.getKeyPacket(Infinity); + assert(lastKeyPacket); + expect(lastKeyPacket.type).toBe('key'); + expect(lastKeyPacket.sequenceNumber).toBe(lastPacket.sequenceNumber); // It's actually the last packet + + const allKeyPackets: EncodedPacket[] = []; + let currentKeyPacket: EncodedPacket | null = firstPacket; + + while (currentKeyPacket) { + allKeyPackets.push(currentKeyPacket); + currentKeyPacket = await sink.getNextKeyPacket(currentKeyPacket); + } + + for (const packet of allKeyPackets) { + const keyPacket = await sink.getKeyPacket(packet.timestamp); + assert(keyPacket); + expect(keyPacket.timestamp).toBe(packet.timestamp); // The correct timestamp was retrieved for this packet + expect(keyPacket.sequenceNumber).toBe(packet.sequenceNumber); + } +}); diff --git a/test/public/0.ts b/test/public/0.ts new file mode 100644 index 0000000..0c02006 Binary files /dev/null and b/test/public/0.ts differ diff --git a/test/public/193039199_mp4_h264_aac_fhd_7.ts b/test/public/193039199_mp4_h264_aac_fhd_7.ts new file mode 100644 index 0000000..9cc0f16 Binary files /dev/null and b/test/public/193039199_mp4_h264_aac_fhd_7.ts differ diff --git a/tsconfig.vitest.json b/tsconfig.vitest.json index f9a4276..7a24606 100644 --- a/tsconfig.vitest.json +++ b/tsconfig.vitest.json @@ -7,5 +7,6 @@ "noEmit": false }, "include": ["vitest.config.ts", "./test/**/*"], + "exclude": ["./test/public/**/*"], "references": [{ "path": "./src" }] }