From c2b20ccda5d3feea41e11bda3d95851964be9be2 Mon Sep 17 00:00:00 2001 From: David Payr <1696106+Vanilagy@users.noreply.github.com> Date: Sat, 22 Feb 2025 19:54:59 +0100 Subject: [PATCH] EncodedVideoSample/EncodedAudioSample -> EncodedPacket --- dev/player.html | 38 ++++ eslint.config.mjs | 1 + src/conversion.ts | 49 +++-- src/custom-coder.ts | 10 +- src/index.ts | 18 +- src/input-track.ts | 95 ++++----- src/isobmff/isobmff-boxes.ts | 18 +- src/isobmff/isobmff-demuxer.ts | 128 +++++------- src/isobmff/isobmff-muxer.ts | 30 +-- src/matroska/matroska-demuxer.ts | 110 ++++------ src/matroska/matroska-muxer.ts | 18 +- src/media-sink.ts | 334 +++++++++++++------------------ src/media-source.ts | 68 +++---- src/misc.ts | 3 + src/mp3/mp3-demuxer.ts | 32 +-- src/mp3/mp3-muxer.ts | 14 +- src/muxer.ts | 10 +- src/ogg/ogg-demuxer.ts | 100 ++++----- src/ogg/ogg-misc.ts | 3 +- src/ogg/ogg-muxer.ts | 12 +- src/packet.ts | 116 +++++++++++ src/wave/wave-demuxer.ts | 54 ++--- src/wave/wave-muxer.ts | 14 +- todo.txt | 3 +- 24 files changed, 650 insertions(+), 628 deletions(-) create mode 100644 src/packet.ts diff --git a/dev/player.html b/dev/player.html index 4fbbe07..6f366e2 100644 --- a/dev/player.html +++ b/dev/player.html @@ -53,6 +53,44 @@ if (!(await audioTrack?.canDecode())) { audioTrack = null; } + +//const startTime2 = audioContext.currentTime + 0.2; +//const sink = new Metamuxer.AudioDataSink(audioTrack); +//for await (const { data } of sink.data(0)) { +// const sample = new Metamuxer.AudioSample(data) +// +// console.log(sample.toAudioData()) +// +// const audioBuffer = sample.toAudioBuffer() +// +// //console.log(sample, sample.allocationSize({ planeIndex: 7, format: 'u8-planar' })) +// +// //break; +// +// /* +// const audioBuffer = new AudioBuffer({ +// numberOfChannels: sample.numberOfChannels, +// length: sample.numberOfFrames, +// sampleRate: sample.sampleRate, +// }); +// +// // All user agents are required to support conversion to f32-planar +// const dataBytes = new Uint8Array(sample.allocationSize({ planeIndex: 0, format: 'u8-planar' })); +// +// for (let i = 0; i < sample.numberOfChannels; i++) { +// sample.copyTo(dataBytes, { planeIndex: i, format: 'u8-planar' }); +// audioBuffer.copyToChannel(new Float32Array(dataBytes).map(x => (x - 128)/128), i); +// } +// */ +// +// const node = audioContext.createBufferSource(); +// node.buffer = audioBuffer; +// node.connect(audioContext.destination); +// node.start(startTime2 + sample.timestamp); +//} +// +//await new Promise(() => {}) + const canvas = document.querySelector('canvas'); const context = canvas.getContext('2d'); let videoRotation = 0; diff --git a/eslint.config.mjs b/eslint.config.mjs index 8b1b830..8f8d66d 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -24,6 +24,7 @@ export default tseslint.config( '@stylistic/max-len': ['error', { code: 120, }], + 'curly': ['error', 'multi-line'], '@typescript-eslint/no-empty-object-type': 'off', '@typescript-eslint/require-await': 'off', '@stylistic/yield-star-spacing': ['error', { before: false, after: true }], diff --git a/src/conversion.ts b/src/conversion.ts index 0583141..ed72dec 100644 --- a/src/conversion.ts +++ b/src/conversion.ts @@ -15,8 +15,7 @@ import { AudioBufferSink, AudioDataSink, CanvasSink, - EncodedAudioSampleSink, - EncodedVideoSampleSink, + EncodedPacketSink, VideoFrameSink, } from './media-sink'; import { @@ -25,8 +24,8 @@ import { AudioEncodingConfig, AudioSource, CanvasSource, - EncodedAudioSampleSource, - EncodedVideoSampleSource, + EncodedVideoPacketSource, + EncodedAudioPacketSource, VideoEncodingConfig, VideoFrameSource, VideoSource, @@ -363,29 +362,29 @@ class Conversion { && videoCodecs.includes(sourceCodec) && (!this.options.video?.codec || this.options.video?.codec === sourceCodec) ) { - // Fast path, we can simply copy over the encoded samples + // Fast path, we can simply copy over the encoded packets - const source = new EncodedVideoSampleSource(sourceCodec); + const source = new EncodedVideoPacketSource(sourceCodec); videoSource = source; this.trackPromises.push((async () => { await this.started; - const sink = new EncodedVideoSampleSink(track); + const sink = new EncodedPacketSink(track); const decoderConfig = await track.getDecoderConfig(); const meta: EncodedVideoChunkMetadata = { decoderConfig: decoderConfig ?? undefined }; - for await (const sample of sink.samples(undefined, this.endTimestamp)) { - if (this.synchronizer.shouldWait(track.id, sample.timestamp)) { - await this.synchronizer.wait(sample.timestamp); + for await (const packet of sink.packets(undefined, this.endTimestamp)) { + if (this.synchronizer.shouldWait(track.id, packet.timestamp)) { + await this.synchronizer.wait(packet.timestamp); } if (this.options.abortSignal?.aborted) { return; } - await source.add(sample, meta); - this.reportProgress(track.id, sample.timestamp + sample.duration); + await source.add(packet, meta); + this.reportProgress(track.id, packet.timestamp + packet.duration); } await source.close(); @@ -422,7 +421,7 @@ class Conversion { const encodingConfig: VideoEncodingConfig = { codec: encodableCodecs[0]!, bitrate: this.options.video?.bitrate ?? QUALITY_HIGH, - onEncodedSample: sample => this.reportProgress(track.id, sample.timestamp + sample.duration), + onEncodedPacket: sample => this.reportProgress(track.id, sample.timestamp + sample.duration), }; if (needsResize) { @@ -553,29 +552,29 @@ class Conversion { && audioCodecs.includes(sourceCodec) && (!this.options.audio?.codec || this.options.audio.codec === sourceCodec) ) { - // Fast path, we can simply copy over the encoded samples + // Fast path, we can simply copy over the encoded packets - const source = new EncodedAudioSampleSource(sourceCodec); + const source = new EncodedAudioPacketSource(sourceCodec); audioSource = source; this.trackPromises.push((async () => { await this.started; - const sink = new EncodedAudioSampleSink(track); + const sink = new EncodedPacketSink(track); const decoderConfig = await track.getDecoderConfig(); const meta: EncodedAudioChunkMetadata = { decoderConfig: decoderConfig ?? undefined }; - for await (const sample of sink.samples(undefined, this.endTimestamp)) { - if (this.synchronizer.shouldWait(track.id, sample.timestamp)) { - await this.synchronizer.wait(sample.timestamp); + for await (const packet of sink.packets(undefined, this.endTimestamp)) { + if (this.synchronizer.shouldWait(track.id, packet.timestamp)) { + await this.synchronizer.wait(packet.timestamp); } if (this.options.abortSignal?.aborted) { return; } - await source.add(sample, meta); - this.reportProgress(track.id, sample.timestamp + sample.duration); + await source.add(packet, meta); + this.reportProgress(track.id, packet.timestamp + packet.duration); } await source.close(); @@ -646,7 +645,7 @@ class Conversion { const source = new AudioDataSource({ codec: codecOfChoice, bitrate: this.options.audio?.bitrate ?? QUALITY_HIGH, - onEncodedSample: sample => this.reportProgress(track.id, sample.timestamp + sample.duration), + onEncodedPacket: packet => this.reportProgress(track.id, packet.timestamp + packet.duration), }); audioSource = source; @@ -695,7 +694,7 @@ class Conversion { const source = new AudioBufferSource({ codec, bitrate: this.options.audio?.bitrate ?? QUALITY_HIGH, - onEncodedSample: sample => this.reportProgress(track.id, sample.timestamp + sample.duration), + onEncodedPacket: packet => this.reportProgress(track.id, packet.timestamp + packet.duration), }); this.trackPromises.push((async () => { @@ -813,8 +812,8 @@ class Conversion { const MAX_TIMESTAMP_GAP = 5; /** - * Utility class for synchronizing multiple track sample consumers with one another. We don't want one consumer to get - * too out-of-sync with the others, as that may lead to a large number of samples that need to be internally buffered + * Utility class for synchronizing multiple track packet consumers with one another. We don't want one consumer to get + * too out-of-sync with the others, as that may lead to a large number of packets that need to be internally buffered * before they can be written. Therefore, we use this class to slow down a consumer if it is too far ahead of the * slowest consumer. */ diff --git a/src/custom-coder.ts b/src/custom-coder.ts index f2794f2..9a6c3c8 100644 --- a/src/custom-coder.ts +++ b/src/custom-coder.ts @@ -1,5 +1,5 @@ import { AudioCodec, VideoCodec } from './codec'; -import { EncodedAudioSample, EncodedVideoSample } from './sample'; +import { EncodedPacket } from './packet'; /** @public */ export abstract class CustomVideoDecoder { @@ -13,7 +13,7 @@ export abstract class CustomVideoDecoder { } abstract init(): void; - abstract decode(sample: EncodedVideoSample): Promise | void; + abstract decode(packet: EncodedPacket): Promise | void; abstract flush(): Promise | void; abstract close(): Promise | void; } @@ -30,7 +30,7 @@ export abstract class CustomAudioDecoder { } abstract init(): void; - abstract decode(sample: EncodedAudioSample): Promise | void; + abstract decode(packet: EncodedPacket): Promise | void; abstract flush(): Promise | void; abstract close(): Promise | void; } @@ -39,7 +39,7 @@ export abstract class CustomAudioDecoder { export abstract class CustomVideoEncoder { codec!: VideoCodec; config!: VideoEncoderConfig; - onSample!: (sample: EncodedVideoSample, meta?: EncodedVideoChunkMetadata) => unknown; + onPacket!: (packet: EncodedPacket, meta?: EncodedVideoChunkMetadata) => unknown; // eslint-disable-next-line @typescript-eslint/no-unused-vars static supports(codec: VideoCodec, config: VideoEncoderConfig): boolean { @@ -56,7 +56,7 @@ export abstract class CustomVideoEncoder { export abstract class CustomAudioEncoder { codec!: AudioCodec; config!: AudioEncoderConfig; - onSample!: (sample: EncodedAudioSample, meta?: EncodedAudioChunkMetadata) => unknown; + onPacket!: (packet: EncodedPacket, meta?: EncodedAudioChunkMetadata) => unknown; // eslint-disable-next-line @typescript-eslint/no-unused-vars static supports(codec: AudioCodec, config: AudioEncoderConfig): boolean { diff --git a/src/index.ts b/src/index.ts index d30c020..21998b9 100644 --- a/src/index.ts +++ b/src/index.ts @@ -29,12 +29,12 @@ export { AudioEncodingConfig, MediaSource, VideoSource, - EncodedVideoSampleSource, + EncodedVideoPacketSource, VideoFrameSource, CanvasSource, MediaStreamVideoTrackSource, AudioSource, - EncodedAudioSampleSource, + EncodedAudioPacketSource, AudioDataSource, AudioBufferSource, MediaStreamAudioTrackSource, @@ -89,20 +89,18 @@ export { OGG, } from './input-format'; export { Input, InputOptions } from './input'; -export { InputTrack, InputVideoTrack, InputAudioTrack, SampleStats } from './input-track'; +export { InputTrack, InputVideoTrack, InputAudioTrack, PacketStats } from './input-track'; +export { EncodedPacket, PacketType } from './packet'; export { - EncodedVideoSample, - EncodedAudioSample, - SampleType, + AudioSample, + AudioSampleInit, } from './sample'; export { - SampleRetrievalOptions, - BaseSampleSink, + PacketRetrievalOptions, + EncodedPacketSink, BaseMediaFrameSink, - EncodedVideoSampleSink, VideoFrameSink, WrappedVideoFrame, - EncodedAudioSampleSink, CanvasSink, WrappedCanvas, AudioDataSink, diff --git a/src/input-track.ts b/src/input-track.ts index f7188fe..4b1ac25 100644 --- a/src/input-track.ts +++ b/src/input-track.ts @@ -1,9 +1,16 @@ import { AudioCodec, MediaCodec, VideoCodec } from './codec'; import { customAudioDecoders, customVideoDecoders } from './custom-coder'; -import { EncodedAudioSampleSink, EncodedVideoSampleSink, SampleRetrievalOptions } from './media-sink'; +import { EncodedPacketSink, PacketRetrievalOptions } from './media-sink'; import { assert, Rotation } from './misc'; import { TrackType } from './output'; -import { EncodedAudioSample, EncodedVideoSample } from './sample'; +import { EncodedPacket } from './packet'; + +/** @public */ +export type PacketStats = { + packetCount: number; + averagePacketRate: number; + averageBitrate: number; +}; export interface InputTrackBacking { getId(): number; @@ -12,6 +19,12 @@ export interface InputTrackBacking { getTimeResolution(): number; getFirstTimestamp(): Promise; computeDuration(): Promise; + + getFirstPacket(options: PacketRetrievalOptions): Promise; + getPacket(timestamp: number, options: PacketRetrievalOptions): Promise; + getNextPacket(packet: EncodedPacket, options: PacketRetrievalOptions): Promise; + getKeyPacket(timestamp: number, options: PacketRetrievalOptions): Promise; + getNextKeyPacket(packet: EncodedPacket, options: PacketRetrievalOptions): Promise; } /** @public */ @@ -28,7 +41,6 @@ export abstract class InputTrack { abstract get codec(): MediaCodec | null; abstract getCodecMimeType(): Promise; abstract canDecode(): Promise; - abstract computeSampleStats(): Promise; isVideoTrack(): this is InputVideoTrack { return this instanceof InputVideoTrack; @@ -57,6 +69,33 @@ export abstract class InputTrack { computeDuration() { return this._backing.computeDuration(); } + + async computePacketStats(): Promise { + const sink = new EncodedPacketSink(this); + + let startTimestamp = Infinity; + let endTimestamp = -Infinity; + let packetCount = 0; + let totalPacketBytes = 0; + + for await (const packet of sink.packets(undefined, undefined, { metadataOnly: true })) { + startTimestamp = Math.min(startTimestamp, packet.timestamp); + endTimestamp = Math.max(endTimestamp, packet.timestamp + packet.duration); + + packetCount++; + totalPacketBytes += packet.byteLength; + } + + return { + packetCount, + averagePacketRate: packetCount + ? Number((packetCount / (endTimestamp - startTimestamp)).toPrecision(16)) + : 0, + averageBitrate: packetCount + ? Number((8 * totalPacketBytes / (endTimestamp - startTimestamp)).toPrecision(16)) + : 0, + }; + } } export interface InputVideoTrackBacking extends InputTrackBacking { @@ -66,11 +105,6 @@ export interface InputVideoTrackBacking extends InputTrackBacking { getRotation(): Rotation; getColorSpace(): Promise; getDecoderConfig(): Promise; - getFirstSample(options: SampleRetrievalOptions): Promise; - getSample(timestamp: number, options: SampleRetrievalOptions): Promise; - getNextSample(sample: EncodedVideoSample, options: SampleRetrievalOptions): Promise; - getKeySample(timestamp: number, options: SampleRetrievalOptions): Promise; - getNextKeySample(sample: EncodedVideoSample, options: SampleRetrievalOptions): Promise; } /** @public */ @@ -161,10 +195,6 @@ export class InputVideoTrack extends InputTrack { return false; } } - - computeSampleStats() { - return computeSampleStats(new EncodedVideoSampleSink(this)); - } } export interface InputAudioTrackBacking extends InputTrackBacking { @@ -172,11 +202,6 @@ export interface InputAudioTrackBacking extends InputTrackBacking { getNumberOfChannels(): number; getSampleRate(): number; getDecoderConfig(): Promise; - getFirstSample(options: SampleRetrievalOptions): Promise; - getSample(timestamp: number, options: SampleRetrievalOptions): Promise; - getNextSample(sample: EncodedAudioSample, options: SampleRetrievalOptions): Promise; - getKeySample(timestamp: number, options: SampleRetrievalOptions): Promise; - getNextKeySample(sample: EncodedAudioSample, options: SampleRetrievalOptions): Promise; } /** @public */ @@ -245,40 +270,4 @@ export class InputAudioTrack extends InputTrack { return false; } } - - computeSampleStats() { - return computeSampleStats(new EncodedAudioSampleSink(this)); - } } - -/** @public */ -export type SampleStats = { - sampleCount: number; - averageSampleRate: number; - averageBitrate: number; -}; - -const computeSampleStats = async (sink: EncodedVideoSampleSink | EncodedAudioSampleSink): Promise => { - let startTimestamp = Infinity; - let endTimestamp = -Infinity; - let sampleCount = 0; - let totalSampleBytes = 0; - - for await (const sample of sink.samples(undefined, undefined, { metadataOnly: true })) { - startTimestamp = Math.min(startTimestamp, sample.timestamp); - endTimestamp = Math.max(endTimestamp, sample.timestamp + sample.duration); - - sampleCount++; - totalSampleBytes += sample.byteLength; - } - - return { - sampleCount, - averageSampleRate: sampleCount - ? Number((sampleCount / (endTimestamp - startTimestamp)).toPrecision(16)) - : 0, - averageBitrate: sampleCount - ? Number((8 * totalSampleBytes / (endTimestamp - startTimestamp)).toPrecision(16)) - : 0, - }; -}; diff --git a/src/isobmff/isobmff-boxes.ts b/src/isobmff/isobmff-boxes.ts index d77218a..5047c66 100644 --- a/src/isobmff/isobmff-boxes.ts +++ b/src/isobmff/isobmff-boxes.ts @@ -287,14 +287,16 @@ export const ftyp = (details: { ]); } - if (details.fragmented) return box('ftyp', [ - ascii('iso5'), // Major brand - u32(minorVersion), // Minor version - // Compatible brands - ascii('iso5'), - ascii('iso6'), - ascii('mp41'), - ]); + if (details.fragmented) { + return box('ftyp', [ + ascii('iso5'), // Major brand + u32(minorVersion), // Minor version + // Compatible brands + ascii('iso5'), + ascii('iso6'), + ascii('mp41'), + ]); + } return box('ftyp', [ ascii('isom'), // Major brand diff --git a/src/isobmff/isobmff-demuxer.ts b/src/isobmff/isobmff-demuxer.ts index 2c863f3..0ed210e 100644 --- a/src/isobmff/isobmff-demuxer.ts +++ b/src/isobmff/isobmff-demuxer.ts @@ -21,7 +21,7 @@ import { InputVideoTrack, InputVideoTrackBacking, } from '../input-track'; -import { SampleRetrievalOptions } from '../media-sink'; +import { PacketRetrievalOptions } from '../media-sink'; import { assert, COLOR_PRIMARIES_MAP_INVERSE, @@ -40,8 +40,8 @@ import { isIso639Dash2LanguageCode, roundToMultiple, } from '../misc'; +import { EncodedPacket, PLACEHOLDER_DATA } from '../packet'; import { Reader } from '../reader'; -import { EncodedAudioSample, EncodedVideoSample, PLACEHOLDER_DATA, SampleType } from '../sample'; import { IsobmffReader, MAX_BOX_HEADER_SIZE } from './isobmff-reader'; type InternalTrack = { @@ -1768,11 +1768,9 @@ export class IsobmffDemuxer extends Demuxer { } } -abstract class IsobmffTrackBacking< - Sample extends EncodedVideoSample | EncodedAudioSample, -> implements InputTrackBacking { - sampleToSampleIndex = new WeakMap(); - sampleToFragmentLocation = new WeakMap(); + packetToFragmentLocation = new WeakMap(); @@ -1796,25 +1794,16 @@ abstract class IsobmffTrackBacking< } async computeDuration() { - const lastSample = await this.getSample(Infinity, { metadataOnly: true }); - return (lastSample?.timestamp ?? 0) + (lastSample?.duration ?? 0); + const lastPacket = await this.getPacket(Infinity, { metadataOnly: true }); + return (lastPacket?.timestamp ?? 0) + (lastPacket?.duration ?? 0); } async getFirstTimestamp() { - const firstSample = await this.getFirstSample({ metadataOnly: true }); - return firstSample?.timestamp ?? 0; + const firstPacket = await this.getFirstPacket({ metadataOnly: true }); + return firstPacket?.timestamp ?? 0; } - abstract createSample( - data: Uint8Array, - byteLength: number, - type: SampleType, - timestamp: number, - duration: number, - sequenceNumber: number - ): Sample; - - async getFirstSample(options: SampleRetrievalOptions) { + async getFirstPacket(options: PacketRetrievalOptions) { if (this.internalTrack.demuxer.isFragmented) { return this.performFragmentedLookup( () => { @@ -1852,7 +1841,7 @@ abstract class IsobmffTrackBacking< ); } - return this.fetchSampleForSampleIndex(0, options); + return this.fetchPacketForSampleIndex(0, options); } private mapTimestampIntoTimescale(timestamp: number) { @@ -1862,7 +1851,7 @@ abstract class IsobmffTrackBacking< return roundToPrecision(timestamp * this.internalTrack.timescale, 14) + this.internalTrack.editListOffset; } - async getSample(timestamp: number, options: SampleRetrievalOptions) { + async getPacket(timestamp: number, options: PacketRetrievalOptions) { const timestampInTimescale = this.mapTimestampIntoTimescale(timestamp); if (this.internalTrack.demuxer.isFragmented) { @@ -1875,15 +1864,15 @@ abstract class IsobmffTrackBacking< } else { const sampleTable = this.internalTrack.demuxer.getSampleTableForTrack(this.internalTrack); const sampleIndex = getSampleIndexForTimestamp(sampleTable, timestampInTimescale); - return this.fetchSampleForSampleIndex(sampleIndex, options); + return this.fetchPacketForSampleIndex(sampleIndex, options); } } - async getNextSample(sample: Sample, options: SampleRetrievalOptions) { + async getNextPacket(packet: EncodedPacket, options: PacketRetrievalOptions) { if (this.internalTrack.demuxer.isFragmented) { - const locationInFragment = this.sampleToFragmentLocation.get(sample); + const locationInFragment = this.packetToFragmentLocation.get(packet); if (locationInFragment === undefined) { - throw new Error('Sample was not created from this track.'); + throw new Error('Packet was not created from this track.'); } const trackData = locationInFragment.fragment.trackData.get(this.internalTrack.id)!; @@ -1941,14 +1930,14 @@ abstract class IsobmffTrackBacking< ); } - const sampleIndex = this.sampleToSampleIndex.get(sample); + const sampleIndex = this.packetToSampleIndex.get(packet); if (sampleIndex === undefined) { - throw new Error('Sample was not created from this track.'); + throw new Error('Packet was not created from this track.'); } - return this.fetchSampleForSampleIndex(sampleIndex + 1, options); + return this.fetchPacketForSampleIndex(sampleIndex + 1, options); } - async getKeySample(timestamp: number, options: SampleRetrievalOptions) { + async getKeyPacket(timestamp: number, options: PacketRetrievalOptions) { const timestampInTimescale = this.mapTimestampIntoTimescale(timestamp); if (this.internalTrack.demuxer.isFragmented) { @@ -1965,14 +1954,14 @@ abstract class IsobmffTrackBacking< const keyFrameSampleIndex = sampleIndex === -1 ? -1 : getRelevantKeyframeIndexForSample(sampleTable, sampleIndex); - return this.fetchSampleForSampleIndex(keyFrameSampleIndex, options); + return this.fetchPacketForSampleIndex(keyFrameSampleIndex, options); } - async getNextKeySample(sample: Sample, options: SampleRetrievalOptions) { + async getNextKeyPacket(packet: EncodedPacket, options: PacketRetrievalOptions) { if (this.internalTrack.demuxer.isFragmented) { - const locationInFragment = this.sampleToFragmentLocation.get(sample); + const locationInFragment = this.packetToFragmentLocation.get(packet); if (locationInFragment === undefined) { - throw new Error('Sample was not created from this track.'); + throw new Error('Packet was not created from this track.'); } const trackData = locationInFragment.fragment.trackData.get(this.internalTrack.id)!; @@ -2039,16 +2028,16 @@ abstract class IsobmffTrackBacking< ); } - const sampleIndex = this.sampleToSampleIndex.get(sample); + const sampleIndex = this.packetToSampleIndex.get(packet); if (sampleIndex === undefined) { - throw new Error('Sample was not created from this track.'); + throw new Error('Packet was not created from this track.'); } const sampleTable = this.internalTrack.demuxer.getSampleTableForTrack(this.internalTrack); const nextKeyFrameSampleIndex = getNextKeyframeIndexForSample(sampleTable, sampleIndex); - return this.fetchSampleForSampleIndex(nextKeyFrameSampleIndex, options); + return this.fetchPacketForSampleIndex(nextKeyFrameSampleIndex, options); } - private async fetchSampleForSampleIndex(sampleIndex: number, options: SampleRetrievalOptions) { + private async fetchPacketForSampleIndex(sampleIndex: number, options: PacketRetrievalOptions) { if (sampleIndex === -1) { return null; } @@ -2076,21 +2065,21 @@ abstract class IsobmffTrackBacking< const timestamp = (sampleInfo.presentationTimestamp - this.internalTrack.editListOffset) / this.internalTrack.timescale; const duration = sampleInfo.duration / this.internalTrack.timescale; - const sample = this.createSample( + const packet = new EncodedPacket( data, - sampleInfo.sampleSize, sampleInfo.isKeyFrame ? 'key' : 'delta', timestamp, duration, sampleIndex, + sampleInfo.sampleSize, ); - this.sampleToSampleIndex.set(sample, sampleIndex); + this.packetToSampleIndex.set(packet, sampleIndex); - return sample; + return packet; } - private async fetchSampleInFragment(fragment: Fragment, sampleIndex: number, options: SampleRetrievalOptions) { + private async fetchPacketInFragment(fragment: Fragment, sampleIndex: number, options: PacketRetrievalOptions) { if (sampleIndex === -1) { return null; } @@ -2113,18 +2102,18 @@ abstract class IsobmffTrackBacking< const timestamp = (fragmentSample.presentationTimestamp - this.internalTrack.editListOffset) / this.internalTrack.timescale; const duration = fragmentSample.duration / this.internalTrack.timescale; - const sample = this.createSample( + const packet = new EncodedPacket( data, - fragmentSample.byteSize, fragmentSample.isKeyFrame ? 'key' : 'delta', timestamp, duration, fragment.moofOffset + sampleIndex, + fragmentSample.byteSize, ); - this.sampleToFragmentLocation.set(sample, { fragment, sampleIndex }); + this.packetToFragmentLocation.set(packet, { fragment, sampleIndex }); - return sample; + return packet; } private findSampleInFragmentsForTimestamp(timestampInTimescale: number) { @@ -2183,13 +2172,13 @@ abstract class IsobmffTrackBacking< return { fragmentIndex, sampleIndex, correctSampleFound }; } - /** Looks for a sample in the fragments while trying to load as few fragments as possible to retrieve it. */ + /** Looks for a packet in the fragments while trying to load as few fragments as possible to retrieve it. */ private async performFragmentedLookup( getBestMatch: () => { fragmentIndex: number; sampleIndex: number; correctSampleFound: boolean }, searchTimestamp: number, latestTimestamp: number, - options: SampleRetrievalOptions, - ): Promise { + options: PacketRetrievalOptions, + ): Promise { const demuxer = this.internalTrack.demuxer; const release = await demuxer.fragmentLookupMutex.acquire(); // The algorithm requires exclusivity @@ -2198,7 +2187,7 @@ abstract class IsobmffTrackBacking< if (correctSampleFound) { // The correct sample already exists, easy path. const fragment = this.internalTrack.fragments[fragmentIndex]!; - return this.fetchSampleInFragment(fragment, sampleIndex, options); + return this.fetchPacketInFragment(fragment, sampleIndex, options); } const metadataReader = demuxer.metadataReader; @@ -2284,7 +2273,7 @@ abstract class IsobmffTrackBacking< const { fragmentIndex, sampleIndex, correctSampleFound } = getBestMatch(); if (correctSampleFound) { const fragment = this.internalTrack.fragments[fragmentIndex]!; - return this.fetchSampleInFragment(fragment, sampleIndex, options); + return this.fetchPacketInFragment(fragment, sampleIndex, options); } if (fragmentIndex !== -1) { bestFragmentIndex = fragmentIndex; @@ -2295,11 +2284,11 @@ abstract class IsobmffTrackBacking< metadataReader.pos = startPos + boxInfo.totalSize; } - let result: Sample | null = null; + let result: EncodedPacket | null = null; const bestFragment = bestFragmentIndex !== -1 ? this.internalTrack.fragments[bestFragmentIndex]! : null; if (bestFragment) { // If we finished looping but didn't find a perfect match, still return the best match we found - result = await this.fetchSampleInFragment(bestFragment, bestSampleIndex, options); + result = await this.fetchPacketInFragment(bestFragment, bestSampleIndex, options); } // Catch faulty lookup table entries @@ -2318,7 +2307,7 @@ abstract class IsobmffTrackBacking< } } -class IsobmffVideoTrackBacking extends IsobmffTrackBacking implements InputVideoTrackBacking { +class IsobmffVideoTrackBacking extends IsobmffTrackBacking implements InputVideoTrackBacking { override internalTrack: InternalVideoTrack; decoderConfigPromise: Promise | null = null; @@ -2359,8 +2348,8 @@ class IsobmffVideoTrackBacking extends IsobmffTrackBacking i return this.decoderConfigPromise ??= (async (): Promise => { if (this.internalTrack.info.codec === 'vp9' && !this.internalTrack.info.vp9CodecInfo) { - const firstSample = await this.getFirstSample({}); - this.internalTrack.info.vp9CodecInfo = firstSample && extractVp9CodecInfoFromFrame(firstSample.data); + const firstPacket = await this.getFirstPacket({}); + this.internalTrack.info.vp9CodecInfo = firstPacket && extractVp9CodecInfoFromFrame(firstPacket.data); } return { @@ -2372,20 +2361,9 @@ class IsobmffVideoTrackBacking extends IsobmffTrackBacking i }; })(); } - - createSample( - data: Uint8Array, - byteLength: number, - type: SampleType, - timestamp: number, - duration: number, - sequenceNumber: number, - ) { - return new EncodedVideoSample(data, type, timestamp, duration, sequenceNumber, byteLength); - } } -class IsobmffAudioTrackBacking extends IsobmffTrackBacking implements InputAudioTrackBacking { +class IsobmffAudioTrackBacking extends IsobmffTrackBacking implements InputAudioTrackBacking { override internalTrack: InternalAudioTrack; decoderConfig: AudioDecoderConfig | null = null; @@ -2418,16 +2396,6 @@ class IsobmffAudioTrackBacking extends IsobmffTrackBacking i description: this.internalTrack.info.codecDescription ?? undefined, }; } - - createSample( - data: Uint8Array, - byteLength: number, - type: SampleType, - timestamp: number, - duration: number, - ) { - return new EncodedAudioSample(data, type, timestamp, duration, byteLength); - } } const getSampleIndexForTimestamp = (sampleTable: SampleTable, timescaleUnits: number) => { diff --git a/src/isobmff/isobmff-muxer.ts b/src/isobmff/isobmff-muxer.ts index 3f56b2c..3b0d239 100644 --- a/src/isobmff/isobmff-muxer.ts +++ b/src/isobmff/isobmff-muxer.ts @@ -13,8 +13,8 @@ import { validateSubtitleMetadata, validateVideoChunkMetadata, } from '../codec'; -import { EncodedAudioSample, EncodedVideoSample, SampleType } from '../sample'; import { BufferTarget } from '../target'; +import { EncodedPacket, PacketType } from '../packet'; export const GLOBAL_TIMESCALE = 1000; const TIMESTAMP_OFFSET = 2_082_844_800; // Seconds between Jan 1 1904 and Jan 1 1970 @@ -25,7 +25,7 @@ export type Sample = { duration: number; data: Uint8Array | null; size: number; - type: SampleType; + type: PacketType; timescaleUnitsToNextSample: number; }; @@ -290,7 +290,7 @@ export class IsobmffMuxer extends Muxer { return newTrackData; } - async addEncodedVideoSample(track: OutputVideoTrack, sample: EncodedVideoSample, meta?: EncodedVideoChunkMetadata) { + async addEncodedVideoPacket(track: OutputVideoTrack, packet: EncodedPacket, meta?: EncodedVideoChunkMetadata) { const release = await this.mutex.acquire(); try { @@ -298,15 +298,15 @@ export class IsobmffMuxer extends Muxer { const timestamp = this.validateAndNormalizeTimestamp( trackData.track, - sample.timestamp, - sample.type === 'key', + packet.timestamp, + packet.type === 'key', ); const internalSample = this.createSampleForTrack( trackData, - sample.data, + packet.data, timestamp, - sample.duration, - sample.type, + packet.duration, + packet.type, ); await this.registerSample(trackData, internalSample); @@ -315,7 +315,7 @@ export class IsobmffMuxer extends Muxer { } } - async addEncodedAudioSample(track: OutputAudioTrack, sample: EncodedAudioSample, meta?: EncodedAudioChunkMetadata) { + async addEncodedAudioPacket(track: OutputAudioTrack, packet: EncodedPacket, meta?: EncodedAudioChunkMetadata) { const release = await this.mutex.acquire(); try { @@ -323,15 +323,15 @@ export class IsobmffMuxer extends Muxer { const timestamp = this.validateAndNormalizeTimestamp( trackData.track, - sample.timestamp, - sample.type === 'key', + packet.timestamp, + packet.type === 'key', ); const internalSample = this.createSampleForTrack( trackData, - sample.data, + packet.data, timestamp, - sample.duration, - sample.type, + packet.duration, + packet.type, ); if (trackData.requiresPcmTransformation) { @@ -492,7 +492,7 @@ export class IsobmffMuxer extends Muxer { data: Uint8Array, timestamp: number, duration: number, - type: SampleType, + type: PacketType, ) { const sample: Sample = { timestamp, diff --git a/src/matroska/matroska-demuxer.ts b/src/matroska/matroska-demuxer.ts index 8cd9d4c..abf2f1b 100644 --- a/src/matroska/matroska-demuxer.ts +++ b/src/matroska/matroska-demuxer.ts @@ -18,7 +18,7 @@ import { InputVideoTrack, InputVideoTrackBacking, } from '../input-track'; -import { SampleRetrievalOptions } from '../media-sink'; +import { PacketRetrievalOptions } from '../media-sink'; import { assert, AsyncMutex, @@ -34,8 +34,8 @@ import { TRANSFER_CHARACTERISTICS_MAP_INVERSE, UNDETERMINED_LANGUAGE, } from '../misc'; +import { EncodedPacket, PLACEHOLDER_DATA } from '../packet'; import { Reader } from '../reader'; -import { EncodedAudioSample, EncodedVideoSample, PLACEHOLDER_DATA, SampleType } from '../sample'; import { CODEC_STRING_MAP, EBMLId, EBMLReader, MAX_HEADER_SIZE, MIN_HEADER_SIZE } from './ebml'; type Segment = { @@ -971,10 +971,8 @@ export class MatroskaDemuxer extends Demuxer { } } -abstract class MatroskaTrackBacking< - Sample extends EncodedVideoSample | EncodedAudioSample, -> implements InputTrackBacking { - sampleToClusterLocation = new WeakMap(); @@ -990,8 +988,8 @@ abstract class MatroskaTrackBacking< } async computeDuration() { - const lastSample = await this.getSample(Infinity, { metadataOnly: true }); - return (lastSample?.timestamp ?? 0) + (lastSample?.duration ?? 0); + const lastPacket = await this.getPacket(Infinity, { metadataOnly: true }); + return (lastPacket?.timestamp ?? 0) + (lastPacket?.duration ?? 0); } getLanguageCode() { @@ -999,24 +997,15 @@ abstract class MatroskaTrackBacking< } async getFirstTimestamp() { - const firstSample = await this.getFirstSample({ metadataOnly: true }); - return firstSample?.timestamp ?? 0; + const firstPacket = await this.getFirstPacket({ metadataOnly: true }); + return firstPacket?.timestamp ?? 0; } getTimeResolution() { return this.internalTrack.segment.timestampFactor; } - abstract createSample( - data: Uint8Array, - byteLength: number, - type: SampleType, - timestamp: number, - duration: number, - sequenceNumber: number, - ): Sample; - - async getFirstSample(options: SampleRetrievalOptions) { + async getFirstPacket(options: PacketRetrievalOptions) { return this.performClusterLookup( () => { const startCluster = this.internalTrack.segment.clusters[0] ?? null; @@ -1060,7 +1049,7 @@ abstract class MatroskaTrackBacking< return roundToPrecision(timestamp * this.internalTrack.segment.timestampFactor, 14); } - async getSample(timestamp: number, options: SampleRetrievalOptions) { + async getPacket(timestamp: number, options: PacketRetrievalOptions) { const timestampInTimescale = this.intoTimescale(timestamp); return this.performClusterLookup( @@ -1071,10 +1060,10 @@ abstract class MatroskaTrackBacking< ); } - async getNextSample(sample: Sample, options: SampleRetrievalOptions) { - const locationInCluster = this.sampleToClusterLocation.get(sample); + async getNextPacket(packet: EncodedPacket, options: PacketRetrievalOptions) { + const locationInCluster = this.packetToClusterLocation.get(packet); if (locationInCluster === undefined) { - throw new Error('Sample was not created from this track.'); + throw new Error('Packet was not created from this track.'); } const trackData = locationInCluster.cluster.trackData.get(this.internalTrack.id)!; @@ -1132,7 +1121,7 @@ abstract class MatroskaTrackBacking< ); } - async getKeySample(timestamp: number, options: SampleRetrievalOptions) { + async getKeyPacket(timestamp: number, options: PacketRetrievalOptions) { const timestampInTimescale = this.intoTimescale(timestamp); return this.performClusterLookup( @@ -1143,10 +1132,10 @@ abstract class MatroskaTrackBacking< ); } - async getNextKeySample(sample: Sample, options: SampleRetrievalOptions) { - const locationInCluster = this.sampleToClusterLocation.get(sample); + async getNextKeyPacket(packet: EncodedPacket, options: PacketRetrievalOptions) { + const locationInCluster = this.packetToClusterLocation.get(packet); if (locationInCluster === undefined) { - throw new Error('Sample was not created from this track.'); + throw new Error('Packet was not created from this track.'); } const trackData = locationInCluster.cluster.trackData.get(this.internalTrack.id)!; @@ -1211,7 +1200,7 @@ abstract class MatroskaTrackBacking< ); } - private async fetchSampleInCluster(cluster: Cluster, blockIndex: number, options: SampleRetrievalOptions) { + private async fetchPacketInCluster(cluster: Cluster, blockIndex: number, options: PacketRetrievalOptions) { if (blockIndex === -1) { return null; } @@ -1223,18 +1212,18 @@ abstract class MatroskaTrackBacking< const data = options.metadataOnly ? PLACEHOLDER_DATA : block.data; const timestamp = block.timestamp / this.internalTrack.segment.timestampFactor; const duration = block.duration / this.internalTrack.segment.timestampFactor; - const sample = this.createSample( + const packet = new EncodedPacket( data, - block.data.byteLength, block.isKeyFrame ? 'key' : 'delta', timestamp, duration, cluster.dataStartPos + blockIndex, + block.data.byteLength, ); - this.sampleToClusterLocation.set(sample, { cluster, blockIndex }); + this.packetToClusterLocation.set(packet, { cluster, blockIndex }); - return sample; + return packet; } private findBlockInClustersForTimestamp(timestampInTimescale: number) { @@ -1301,7 +1290,7 @@ abstract class MatroskaTrackBacking< return { clusterIndex, blockIndex, correctBlockFound }; } - /** Looks for a sample in the clusters while trying to load as few clusters as possible to retrieve it. */ + /** Looks for a packet in the clusters while trying to load as few clusters as possible to retrieve it. */ private async performClusterLookup( // This function returns the best-matching block that is currently loaded. Based on this information, we know // which clusters we need to load to find the actual match. @@ -1310,8 +1299,8 @@ abstract class MatroskaTrackBacking< searchTimestamp: number, // The timestamp for which we know the correct block will not come after it latestTimestamp: number, - options: SampleRetrievalOptions, - ): Promise { + options: PacketRetrievalOptions, + ): Promise { const { demuxer, segment } = this.internalTrack; const release = await segment.clusterLookupMutex.acquire(); // The algorithm requires exclusivity @@ -1320,7 +1309,7 @@ abstract class MatroskaTrackBacking< if (correctBlockFound) { // The correct block already exists, easy path. const cluster = this.internalTrack.clusters[clusterIndex]!; - return this.fetchSampleInCluster(cluster, blockIndex, options); + return this.fetchPacketInCluster(cluster, blockIndex, options); } // We use the metadata reader to find the cluster, but the cluster reader to load the cluster @@ -1403,7 +1392,7 @@ abstract class MatroskaTrackBacking< const { clusterIndex, blockIndex, correctBlockFound } = getBestMatch(); if (correctBlockFound) { const cluster = this.internalTrack.clusters[clusterIndex]!; - return this.fetchSampleInCluster(cluster, blockIndex, options); + return this.fetchPacketInCluster(cluster, blockIndex, options); } if (clusterIndex !== -1) { bestClusterIndex = clusterIndex; @@ -1414,11 +1403,11 @@ abstract class MatroskaTrackBacking< metadataReader.pos = dataStartPos + size; } - let result: Sample | null = null; + let result: EncodedPacket | null = null; const bestCluster = bestClusterIndex !== -1 ? this.internalTrack.clusters[bestClusterIndex]! : null; if (bestCluster) { // If we finished looping but didn't find a perfect match, still return the best match we found - result = await this.fetchSampleInCluster(bestCluster, bestBlockIndex, options); + result = await this.fetchPacketInCluster(bestCluster, bestBlockIndex, options); } // Catch faulty cue points @@ -1437,7 +1426,7 @@ abstract class MatroskaTrackBacking< } } -class MatroskaVideoTrackBacking extends MatroskaTrackBacking implements InputVideoTrackBacking { +class MatroskaVideoTrackBacking extends MatroskaTrackBacking implements InputVideoTrackBacking { override internalTrack: InternalVideoTrack; decoderConfigPromise: Promise | null = null; @@ -1477,12 +1466,12 @@ class MatroskaVideoTrackBacking extends MatroskaTrackBacking } return this.decoderConfigPromise ??= (async (): Promise => { - let firstSample: EncodedVideoSample | null = null; - const needsSampleForAdditionalInfo + let firstPacket: EncodedPacket | null = null; + const needsPacketForAdditionalInfo = this.internalTrack.info.codec === 'vp9' || this.internalTrack.info.codec === 'av1'; - if (needsSampleForAdditionalInfo) { - firstSample = await this.getFirstSample({}); + if (needsPacketForAdditionalInfo) { + firstPacket = await this.getFirstPacket({}); } return { @@ -1492,11 +1481,11 @@ class MatroskaVideoTrackBacking extends MatroskaTrackBacking codec: this.internalTrack.info.codec, codecDescription: this.internalTrack.codecPrivate, colorSpace: this.internalTrack.info.colorSpace, - vp9CodecInfo: this.internalTrack.info.codec === 'vp9' && firstSample - ? extractVp9CodecInfoFromFrame(firstSample.data) + vp9CodecInfo: this.internalTrack.info.codec === 'vp9' && firstPacket + ? extractVp9CodecInfoFromFrame(firstPacket.data) : null, - av1CodecInfo: this.internalTrack.info.codec === 'av1' && firstSample - ? extractAv1CodecInfoFromFrame(firstSample.data) + av1CodecInfo: this.internalTrack.info.codec === 'av1' && firstPacket + ? extractAv1CodecInfoFromFrame(firstPacket.data) : null, }), codedWidth: this.internalTrack.info.width, @@ -1506,20 +1495,9 @@ class MatroskaVideoTrackBacking extends MatroskaTrackBacking }; })(); } - - createSample( - data: Uint8Array, - byteLength: number, - type: SampleType, - timestamp: number, - duration: number, - sequenceNumber: number, - ) { - return new EncodedVideoSample(data, type, timestamp, duration, sequenceNumber, byteLength); - } } -class MatroskaAudioTrackBacking extends MatroskaTrackBacking implements InputAudioTrackBacking { +class MatroskaAudioTrackBacking extends MatroskaTrackBacking implements InputAudioTrackBacking { override internalTrack: InternalAudioTrack; decoderConfig: AudioDecoderConfig | null = null; @@ -1556,16 +1534,6 @@ class MatroskaAudioTrackBacking extends MatroskaTrackBacking description: this.internalTrack.codecPrivate ?? undefined, }; } - - createSample( - data: Uint8Array, - byteLength: number, - type: SampleType, - timestamp: number, - duration: number, - ) { - return new EncodedAudioSample(data, type, timestamp, duration, byteLength); - } } /** Sorts blocks such that referenced blocks come before the blocks that reference them. */ diff --git a/src/matroska/matroska-muxer.ts b/src/matroska/matroska-muxer.ts index 1efebe3..9b1ae70 100644 --- a/src/matroska/matroska-muxer.ts +++ b/src/matroska/matroska-muxer.ts @@ -43,7 +43,7 @@ import { } from '../codec'; import { Muxer } from '../muxer'; import { Writer } from '../writer'; -import { EncodedAudioSample, EncodedVideoSample } from '../sample'; +import { EncodedPacket } from '../packet'; const MAX_CHUNK_LENGTH_MS = 2 ** 15; const APP_NAME = 'https://github.com/Vanilagy/webm-muxer'; // TODO @@ -477,15 +477,15 @@ export class MatroskaMuxer extends Muxer { return newTrackData; } - async addEncodedVideoSample(track: OutputVideoTrack, sample: EncodedVideoSample, meta?: EncodedVideoChunkMetadata) { + async addEncodedVideoPacket(track: OutputVideoTrack, packet: EncodedPacket, meta?: EncodedVideoChunkMetadata) { const release = await this.mutex.acquire(); try { const trackData = this.getVideoTrackData(track, meta); - const isKeyFrame = sample.type === 'key'; - const timestamp = this.validateAndNormalizeTimestamp(trackData.track, sample.timestamp, isKeyFrame); - const videoChunk = this.createInternalChunk(sample.data, timestamp, sample.duration, sample.type); + const isKeyFrame = packet.type === 'key'; + const timestamp = this.validateAndNormalizeTimestamp(trackData.track, packet.timestamp, isKeyFrame); + const videoChunk = this.createInternalChunk(packet.data, timestamp, packet.duration, packet.type); if (track.source._codec === 'vp9') this.fixVP9ColorSpace(trackData, videoChunk); trackData.chunkQueue.push(videoChunk); @@ -495,15 +495,15 @@ export class MatroskaMuxer extends Muxer { } } - async addEncodedAudioSample(track: OutputAudioTrack, sample: EncodedAudioSample, meta?: EncodedAudioChunkMetadata) { + async addEncodedAudioPacket(track: OutputAudioTrack, packet: EncodedPacket, meta?: EncodedAudioChunkMetadata) { const release = await this.mutex.acquire(); try { const trackData = this.getAudioTrackData(track, meta); - const isKeyFrame = sample.type === 'key'; - const timestamp = this.validateAndNormalizeTimestamp(trackData.track, sample.timestamp, isKeyFrame); - const audioChunk = this.createInternalChunk(sample.data, timestamp, sample.duration, sample.type); + const isKeyFrame = packet.type === 'key'; + const timestamp = this.validateAndNormalizeTimestamp(trackData.track, packet.timestamp, isKeyFrame); + const audioChunk = this.createInternalChunk(packet.data, timestamp, packet.duration, packet.type); trackData.chunkQueue.push(audioChunk); await this.interleaveChunks(); diff --git a/src/media-sink.ts b/src/media-sink.ts index 3d1912a..7fce553 100644 --- a/src/media-sink.ts +++ b/src/media-sink.ts @@ -1,6 +1,6 @@ import { parsePcmCodec, PCM_AUDIO_CODECS, PcmAudioCodec, VideoCodec, AudioCodec } from './codec'; import { CustomVideoDecoder, customVideoDecoders, CustomAudioDecoder, customAudioDecoders } from './custom-coder'; -import { InputAudioTrack, InputVideoTrack } from './input-track'; +import { InputAudioTrack, InputTrack, InputVideoTrack } from './input-track'; import { AnyIterable, assert, @@ -14,15 +14,15 @@ import { toDataView, validateAnyIterable, } from './misc'; +import { EncodedPacket } from './packet'; import { fromAlaw, fromUlaw } from './pcm'; -import { EncodedAudioSample, EncodedVideoSample } from './sample'; /** @public */ -export type SampleRetrievalOptions = { +export type PacketRetrievalOptions = { metadataOnly?: boolean; }; -const validateSampleRetrievalOptions = (options: SampleRetrievalOptions) => { +const validatePacketRetrievalOptions = (options: PacketRetrievalOptions) => { if (!options || typeof options !== 'object') { throw new TypeError('options must be an object.'); } @@ -38,19 +38,57 @@ const validateTimestamp = (timestamp: number) => { }; /** @public */ -export abstract class BaseSampleSink { - abstract getFirstSample(options?: SampleRetrievalOptions): Promise; - abstract getSample(timestamp: number, options?: SampleRetrievalOptions): Promise; - abstract getNextSample(sample: Sample, options?: SampleRetrievalOptions): Promise; - abstract getKeySample(timestamp: number, options?: SampleRetrievalOptions): Promise; - abstract getNextKeySample(sample: Sample, options?: SampleRetrievalOptions): Promise; +export class EncodedPacketSink { + /** @internal */ + _track: InputTrack; - samples( - startSample?: Sample, + constructor(track: InputTrack) { + if (!(track instanceof InputTrack)) { + throw new TypeError('track must be an InputTrack.'); + } + + this._track = track; + } + + getFirstPacket(options: PacketRetrievalOptions = {}) { + validatePacketRetrievalOptions(options); + return this._track._backing.getFirstPacket(options); + } + + getPacket(timestamp: number, options: PacketRetrievalOptions = {}) { + validateTimestamp(timestamp); + validatePacketRetrievalOptions(options); + return this._track._backing.getPacket(timestamp, options); + } + + getNextPacket(packet: EncodedPacket, options: PacketRetrievalOptions = {}) { + if (!(packet instanceof EncodedPacket)) { + throw new TypeError('packet must be an EncodedPacket.'); + } + validatePacketRetrievalOptions(options); + return this._track._backing.getNextPacket(packet, options); + } + + getKeyPacket(timestamp: number, options: PacketRetrievalOptions = {}) { + validateTimestamp(timestamp); + validatePacketRetrievalOptions(options); + return this._track._backing.getKeyPacket(timestamp, options); + } + + getNextKeyPacket(packet: EncodedPacket, options: PacketRetrievalOptions = {}) { + if (!(packet instanceof EncodedPacket)) { + throw new TypeError('packet must be an EncodedPacket.'); + } + validatePacketRetrievalOptions(options); + return this._track._backing.getNextKeyPacket(packet, options); + } + + packets( + startPacket?: EncodedPacket, endTimestamp = Infinity, - options?: SampleRetrievalOptions, - ): AsyncGenerator { - const sampleQueue: Sample[] = []; + options?: PacketRetrievalOptions, + ): AsyncGenerator { + const packetQueue: EncodedPacket[] = []; let { promise: queueNotEmpty, resolve: onQueueNotEmpty } = promiseWithResolvers(); let { promise: queueDequeue, resolve: onQueueDequeue } = promiseWithResolvers(); @@ -63,30 +101,30 @@ export abstract class BaseSampleSink Math.max(2, timestamps.length); - // The following is the "pump" process that keeps pumping samples into the queue + // The following is the "pump" process that keeps pumping packets into the queue (async () => { - let sample = startSample ?? await this.getFirstSample(options); + let packet = startPacket ?? await this.getFirstPacket(options); - while (sample && !terminated) { - if (sample.timestamp >= endTimestamp) { + while (packet && !terminated) { + if (packet.timestamp >= endTimestamp) { break; } - if (sampleQueue.length > maxQueueSize()) { + if (packetQueue.length > maxQueueSize()) { ({ promise: queueDequeue, resolve: onQueueDequeue } = promiseWithResolvers()); await queueDequeue; continue; } - sampleQueue.push(sample); + packetQueue.push(packet); onQueueNotEmpty(); ({ promise: queueNotEmpty, resolve: onQueueNotEmpty } = promiseWithResolvers()); - sample = await this.getNextSample(sample, options); + packet = await this.getNextPacket(packet, options); } ended = true; @@ -105,8 +143,8 @@ export abstract class BaseSampleSink 0) { - const value = sampleQueue.shift()!; + } else if (packetQueue.length > 0) { + const value = packetQueue.shift()!; const now = performance.now(); timestamps.push(now); @@ -148,7 +186,6 @@ export type WrappedMediaFrame = { }; abstract class DecoderWrapper< - Sample extends EncodedVideoSample | EncodedAudioSample, MediaFrame extends VideoFrame | AudioData, WrappedFrame extends WrappedMediaFrame = WrappedMediaFrame, > { @@ -158,14 +195,13 @@ abstract class DecoderWrapper< ) {} abstract getDecodeQueueSize(): number; - abstract decode(sample: Sample): void; + abstract decode(packet: EncodedPacket): void; abstract flush(): Promise; abstract close(): void; } /** @public */ export abstract class BaseMediaFrameSink< - Sample extends EncodedVideoSample | EncodedAudioSample, MediaFrame extends VideoFrame | AudioData, /** @internal */ WrappedFrame extends WrappedMediaFrame = WrappedMediaFrame, @@ -174,9 +210,9 @@ export abstract class BaseMediaFrameSink< abstract _createDecoder( onFrame: (frame: WrappedFrame) => unknown, onError: (error: DOMException) => unknown - ): Promise>; + ): Promise>; /** @internal */ - abstract _createSampleSink(): BaseSampleSink; + abstract _createPacketSink(): EncodedPacketSink; /** @internal */ private _duplicateFrame(frame: WrappedFrame) { @@ -206,7 +242,7 @@ export abstract class BaseMediaFrameSink< // the consumer. let outOfBandError = null as Error | null; - // The following is the "pump" process that keeps pumping samples into the decoder + // The following is the "pump" process that keeps pumping packets into the decoder (async () => { const decoderError = new Error(); const decoder = await this._createDecoder((wrappedFrame) => { @@ -252,53 +288,53 @@ export abstract class BaseMediaFrameSink< } }); - const sampleSink = this._createSampleSink(); - const keySample = await sampleSink.getKeySample(startTimestamp) ?? await sampleSink.getFirstSample(); - if (!keySample) { + const packetSink = this._createPacketSink(); + const keyPacket = await packetSink.getKeyPacket(startTimestamp) ?? await packetSink.getFirstPacket(); + if (!keyPacket) { return; } - let currentSample: Sample | null = keySample; + let currentPacket: EncodedPacket | null = keyPacket; - let samplesEndTimestamp = Infinity; + let packetsEndTimestamp = Infinity; if (endTimestamp < Infinity) { - // When an end timestamp is set, we cannot simply use that for the sample iterator due to out-of-order - // frames (B-frames). Instead, we'll need to keep decoding samples until we get a frame that exceeds + // When an end timestamp is set, we cannot simply use that for the packet iterator due to out-of-order + // frames (B-frames). Instead, we'll need to keep decoding packets until we get a frame that exceeds // this end time. However, we can still put a bound on it: Since key frames are by definition never // out of order, we can stop at the first key frame after the end timestamp. - const endSample = await sampleSink.getSample(endTimestamp); - const endKeySample = !endSample + const endPacket = await packetSink.getPacket(endTimestamp); + const endKeyPacket = !endPacket ? null - : endSample.type === 'key' && endSample.timestamp === endTimestamp - ? endSample - : await sampleSink.getNextKeySample(endSample); + : endPacket.type === 'key' && endPacket.timestamp === endTimestamp + ? endPacket + : await packetSink.getNextKeyPacket(endPacket); - if (endKeySample) { - samplesEndTimestamp = endKeySample.timestamp; + if (endKeyPacket) { + packetsEndTimestamp = endKeyPacket.timestamp; } } - const samples = sampleSink.samples(keySample, samplesEndTimestamp); - await samples.next(); // Skip the start sample as we already have it + const packets = packetSink.packets(keyPacket, packetsEndTimestamp); + await packets.next(); // Skip the start packet as we already have it - while (currentSample && !ended) { + while (currentPacket && !ended) { if (frameQueue.length + decoder.getDecodeQueueSize() > MAX_QUEUE_SIZE) { ({ promise: queueDequeue, resolve: onQueueDequeue } = promiseWithResolvers()); await queueDequeue; continue; } - decoder.decode(currentSample); + decoder.decode(currentPacket); - const sampleResult = await samples.next(); - if (sampleResult.done) { + const packetResult = await packets.next(); + if (packetResult.done) { break; } - currentSample = sampleResult.value; + currentPacket = packetResult.value; } - await samples.return(); + await packets.return(); if (!terminated) await decoder.flush(); decoder.close(); @@ -384,7 +420,7 @@ export abstract class BaseMediaFrameSink< ({ promise: queueNotEmpty, resolve: onQueueNotEmpty } = promiseWithResolvers()); }; - // The following is the "pump" process that keeps pumping samples into the decoder + // The following is the "pump" process that keeps pumping packets into the decoder (async () => { const decoderError = new Error(); const decoder = await this._createDecoder((wrappedFrame) => { @@ -419,9 +455,9 @@ export abstract class BaseMediaFrameSink< } }); - const sampleSink = this._createSampleSink(); - let lastKeySample: Sample | null = null; - let lastSample: Sample | null = null; + const packetSink = this._createPacketSink(); + let lastKeyPacket: EncodedPacket | null = null; + let lastPacket: EncodedPacket | null = null; for await (const timestamp of timestampIterator) { validateTimestamp(timestamp); @@ -435,57 +471,57 @@ export abstract class BaseMediaFrameSink< break; } - const targetSample = await sampleSink.getSample(timestamp); - if (!targetSample) { + const targetPacket = await packetSink.getPacket(timestamp); + if (!targetPacket) { pushToQueue(null); continue; } - const keySample = await sampleSink.getKeySample(timestamp); - if (!keySample) { + const keyPacket = await packetSink.getKeyPacket(timestamp); + if (!keyPacket) { pushToQueue(null); continue; } - if (lastSample && targetSample.sequenceNumber < lastSample.sequenceNumber) { + if (lastPacket && targetPacket.sequenceNumber < lastPacket.sequenceNumber) { // We're going back in time with this one, let's flush and reset to an clean state await decoder.flush(); timestampsOfInterest.length = 0; } if ( - lastKeySample - && keySample.sequenceNumber === lastKeySample.sequenceNumber - && targetSample.timestamp >= lastSample!.timestamp + lastKeyPacket + && keyPacket.sequenceNumber === lastKeyPacket.sequenceNumber + && targetPacket.timestamp >= lastPacket!.timestamp ) { - assert(lastSample); + assert(lastPacket); if ( - targetSample.sequenceNumber === lastSample.sequenceNumber + targetPacket.sequenceNumber === lastPacket.sequenceNumber && timestampsOfInterest.length === 0 ) { - // Special case: We have a repeat sample, but the frame for that sample has already been + // Special case: We have a repeat packet, but the frame for that packet has already been // decoded. Therefore, we need to push the frame here instead of in the decoder callback. if (lastUsedFrame) { pushToQueue(this._duplicateFrame(lastUsedFrame)); } } else { - timestampsOfInterest.push(targetSample.timestamp); + timestampsOfInterest.push(targetPacket.timestamp); } } else { - // The key sample has changed - lastKeySample = keySample; - lastSample = keySample; - decoder.decode(keySample); - timestampsOfInterest.push(targetSample.timestamp); + // The key packet has changed + lastKeyPacket = keyPacket; + lastPacket = keyPacket; + decoder.decode(keyPacket); + timestampsOfInterest.push(targetPacket.timestamp); } - while (lastSample.sequenceNumber < targetSample.sequenceNumber) { - const nextSample = await sampleSink.getNextSample(lastSample); - assert(nextSample); + while (lastPacket.sequenceNumber < targetPacket.sequenceNumber) { + const nextPacket = await packetSink.getNextPacket(lastPacket); + assert(nextPacket); - lastSample = nextSample; - decoder.decode(nextSample); + lastPacket = nextPacket; + decoder.decode(nextPacket); } } @@ -545,56 +581,7 @@ export abstract class BaseMediaFrameSink< } } -/** @public */ -export class EncodedVideoSampleSink extends BaseSampleSink { - /** @internal */ - _videoTrack: InputVideoTrack; - - constructor(videoTrack: InputVideoTrack) { - if (!(videoTrack instanceof InputVideoTrack)) { - throw new TypeError('videoTrack must be an InputVideoTrack.'); - } - - super(); - - this._videoTrack = videoTrack; - } - - getFirstSample(options: SampleRetrievalOptions = {}) { - validateSampleRetrievalOptions(options); - return this._videoTrack._backing.getFirstSample(options); - } - - getSample(timestamp: number, options: SampleRetrievalOptions = {}) { - validateTimestamp(timestamp); - validateSampleRetrievalOptions(options); - return this._videoTrack._backing.getSample(timestamp, options); - } - - getNextSample(sample: EncodedVideoSample, options: SampleRetrievalOptions = {}) { - if (!(sample instanceof EncodedVideoSample)) { - throw new TypeError('sample must be an EncodedVideoSample.'); - } - validateSampleRetrievalOptions(options); - return this._videoTrack._backing.getNextSample(sample, options); - } - - getKeySample(timestamp: number, options: SampleRetrievalOptions = {}) { - validateTimestamp(timestamp); - validateSampleRetrievalOptions(options); - return this._videoTrack._backing.getKeySample(timestamp, options); - } - - getNextKeySample(sample: EncodedVideoSample, options: SampleRetrievalOptions = {}) { - if (!(sample instanceof EncodedVideoSample)) { - throw new TypeError('sample must be an EncodedVideoSample.'); - } - validateSampleRetrievalOptions(options); - return this._videoTrack._backing.getNextKeySample(sample, options); - } -} - -class VideoDecoderWrapper extends DecoderWrapper { +class VideoDecoderWrapper extends DecoderWrapper { decoder: VideoDecoder | null = null; customDecoder: CustomVideoDecoder | null = null; @@ -673,17 +660,17 @@ class VideoDecoderWrapper extends DecoderWrapper } } - decode(sample: EncodedVideoSample) { + decode(packet: EncodedPacket) { if (this.customDecoder) { this.customDecoderQueueSize++; this.lastCustomDecoderPromise = this.lastCustomDecoderPromise.then(() => { - return this.customDecoder!.decode(sample); + return this.customDecoder!.decode(packet); }); void this.lastCustomDecoderPromise.then(() => this.customDecoderQueueSize--); } else { assert(this.decoder); - this.decoder.decode(sample.toEncodedVideoChunk()); + this.decoder.decode(packet.toEncodedVideoChunk()); } } @@ -724,7 +711,7 @@ export type WrappedVideoFrame = { }; /** @public */ -export class VideoFrameSink extends BaseMediaFrameSink { +export class VideoFrameSink extends BaseMediaFrameSink { /** @internal */ _videoTrack: InputVideoTrack; @@ -759,8 +746,8 @@ export class VideoFrameSink extends BaseMediaFrameSink { - /** @internal */ - _audioTrack: InputAudioTrack; - - constructor(audioTrack: InputAudioTrack) { - if (!(audioTrack instanceof InputAudioTrack)) { - throw new TypeError('audioTrack must be an InputAudioTrack.'); - } - - super(); - - this._audioTrack = audioTrack; - } - - getFirstSample(options: SampleRetrievalOptions = {}) { - validateSampleRetrievalOptions(options); - return this._audioTrack._backing.getFirstSample(options); - } - - getSample(timestamp: number, options: SampleRetrievalOptions = {}) { - validateTimestamp(timestamp); - validateSampleRetrievalOptions(options); - return this._audioTrack._backing.getSample(timestamp, options); - } - - getNextSample(sample: EncodedAudioSample, options: SampleRetrievalOptions = {}) { - if (!(sample instanceof EncodedAudioSample)) { - throw new TypeError('sample must be an EncodedAudioSample.'); - } - validateSampleRetrievalOptions(options); - return this._audioTrack._backing.getNextSample(sample, options); - } - - getKeySample(timestamp: number, options: SampleRetrievalOptions = {}) { - validateTimestamp(timestamp); - validateSampleRetrievalOptions(options); - return this._audioTrack._backing.getKeySample(timestamp, options); - } - - getNextKeySample(sample: EncodedAudioSample, options: SampleRetrievalOptions = {}) { - if (!(sample instanceof EncodedAudioSample)) { - throw new TypeError('sample must be an EncodedAudioSample.'); - } - validateSampleRetrievalOptions(options); - return this._audioTrack._backing.getNextKeySample(sample, options); - } -} - -class AudioDecoderWrapper extends DecoderWrapper { +class AudioDecoderWrapper extends DecoderWrapper { decoder: AudioDecoder | null = null; customDecoder: CustomAudioDecoder | null = null; @@ -989,17 +927,17 @@ class AudioDecoderWrapper extends DecoderWrapper } } - decode(sample: EncodedAudioSample) { + decode(packet: EncodedPacket) { if (this.customDecoder) { this.customDecoderQueueSize++; this.lastCustomDecoderPromise = this.lastCustomDecoderPromise.then(() => { - return this.customDecoder!.decode(sample); + return this.customDecoder!.decode(packet); }); void this.lastCustomDecoderPromise.then(() => this.customDecoderQueueSize--); } else { assert(this.decoder); - this.decoder.decode(sample.toEncodedAudioChunk()); + this.decoder.decode(packet.toEncodedAudioChunk()); } } @@ -1024,7 +962,7 @@ class AudioDecoderWrapper extends DecoderWrapper // There are a lot of PCM variants not natively supported by the browser and by AudioData. Therefore we need a simple // decoder that maps any input PCM format into a PCM format supported by the browser. -class PcmAudioDecoderWrapper extends DecoderWrapper { +class PcmAudioDecoderWrapper extends DecoderWrapper { codec: PcmAudioCodec; inputSampleSize: 1 | 2 | 3 | 4; @@ -1139,10 +1077,10 @@ class PcmAudioDecoderWrapper extends DecoderWrapper= preciseDuration) { - // We need to sync with the sample timestamp again - this.currentTimestamp = sample.timestamp; + if (this.currentTimestamp === null || Math.abs(packet.timestamp - this.currentTimestamp) >= preciseDuration) { + // We need to sync with the packet timestamp again + this.currentTimestamp = packet.timestamp; } const preciseTimestamp = this.currentTimestamp; @@ -1199,7 +1137,7 @@ export type WrappedAudioData = { }; /** @public */ -export class AudioDataSink extends BaseMediaFrameSink { +export class AudioDataSink extends BaseMediaFrameSink { /** @internal */ _audioTrack: InputAudioTrack; @@ -1246,8 +1184,8 @@ export class AudioDataSink extends BaseMediaFrameSink unknown; + onEncodedPacket?: (packet: EncodedPacket, meta: EncodedVideoChunkMetadata | undefined) => unknown; onEncodingError?: (error: Error) => unknown; }; @@ -171,7 +171,7 @@ const validateVideoEncodingConfig = (config: VideoEncodingConfig) => { ) { throw new TypeError('config.keyFrameInterval, when provided, must be a non-negative number.'); } - if (config.onEncodedSample !== undefined && typeof config.onEncodedSample !== 'function') { + if (config.onEncodedPacket !== undefined && typeof config.onEncodedPacket !== 'function') { throw new TypeError('config.onEncodedChunk, when provided, must be a function.'); } if (config.onEncodingError !== undefined && typeof config.onEncodingError !== 'function') { @@ -306,9 +306,9 @@ class VideoEncoderWrapper { this.customEncoder = new MatchingCustomEncoder() as CustomVideoEncoder; this.customEncoder.codec = this.encodingConfig.codec; this.customEncoder.config = encoderConfig; - this.customEncoder.onSample = (sample, meta) => { - this.encodingConfig.onEncodedSample?.(sample, meta); - void this.muxer!.addEncodedVideoSample(this.source._connectedTrack!, sample, meta); + this.customEncoder.onPacket = (packet, meta) => { + this.encodingConfig.onEncodedPacket?.(packet, meta); + void this.muxer!.addEncodedVideoPacket(this.source._connectedTrack!, packet, meta); }; this.customEncoder.init(); @@ -327,10 +327,10 @@ class VideoEncoderWrapper { this.encoder = new VideoEncoder({ output: (chunk, meta) => { - const sample = EncodedVideoSample.fromEncodedVideoChunk(chunk); + const packet = EncodedPacket.fromEncodedChunk(chunk); - this.encodingConfig.onEncodedSample?.(sample, meta); - void this.muxer!.addEncodedVideoSample(this.source._connectedTrack!, sample, meta); + this.encodingConfig.onEncodedPacket?.(packet, meta); + void this.muxer!.addEncodedVideoPacket(this.source._connectedTrack!, packet, meta); }, error: this.encodingConfig.onEncodingError ?? (error => console.error('VideoEncoder error:', error)), }); @@ -516,28 +516,28 @@ export abstract class AudioSource extends MediaSource { } /** @public */ -export class EncodedAudioSampleSource extends AudioSource { +export class EncodedAudioPacketSource extends AudioSource { constructor(codec: AudioCodec) { super(codec); } - add(sample: EncodedAudioSample, meta?: EncodedAudioChunkMetadata) { - if (!(sample instanceof EncodedAudioSample)) { - throw new TypeError('chunk must be an EncodedAudioSample.'); + add(packet: EncodedPacket, meta?: EncodedAudioChunkMetadata) { + if (!(packet instanceof EncodedPacket)) { + throw new TypeError('packet must be an EncodedPacket.'); } - if (sample.isMetadataOnly) { - throw new TypeError('Metadata-only samples cannot be added.'); + if (packet.isMetadataOnly) { + throw new TypeError('Metadata-only packets cannot be added.'); } this._ensureValidAdd(); - return this._connectedTrack!.output._muxer.addEncodedAudioSample(this._connectedTrack!, sample, meta); + return this._connectedTrack!.output._muxer.addEncodedAudioPacket(this._connectedTrack!, packet, meta); } } /** @public */ export type AudioEncodingConfig = { codec: AudioCodec; bitrate?: number | Quality; - onEncodedSample?: (sample: EncodedAudioSample, meta: EncodedAudioChunkMetadata | undefined) => unknown; + onEncodedPacket?: (packet: EncodedPacket, meta: EncodedAudioChunkMetadata | undefined) => unknown; onEncodingError?: (error: Error) => unknown; }; @@ -663,7 +663,7 @@ class AudioEncoderWrapper { view: DataView; }[] = []; - // Prepare all of the output buffers, each being bounded by CHUNK_SIZE so we don't generate huge samples + // Prepare all of the output buffers, each being bounded by CHUNK_SIZE so we don't generate huge packets for (let frame = 0; frame < numberOfFrames; frame += CHUNK_SIZE) { const frameCount = Math.min(CHUNK_SIZE, audioData.numberOfFrames - frame); const outputSize = frameCount * numberOfChannels * this.outputSampleSize; @@ -710,15 +710,15 @@ class AudioEncoderWrapper { const outputBuffer = view.buffer; const startFrame = i * CHUNK_SIZE; - const sample = new EncodedAudioSample( + const packet = new EncodedPacket( new Uint8Array(outputBuffer), 'key', timestamp / 1e6 + startFrame / sampleRate, frameCount / sampleRate, ); - this.encodingConfig.onEncodedSample?.(sample, meta); - await this.muxer!.addEncodedAudioSample(this.source._connectedTrack!, sample, meta); // With backpressure + this.encodingConfig.onEncodedPacket?.(packet, meta); + await this.muxer!.addEncodedAudioPacket(this.source._connectedTrack!, packet, meta); // With backpressure } } @@ -757,9 +757,9 @@ class AudioEncoderWrapper { this.customEncoder = new MatchingCustomEncoder() as CustomAudioEncoder; this.customEncoder.codec = this.encodingConfig.codec; this.customEncoder.config = encoderConfig; - this.customEncoder.onSample = (sample, meta) => { - this.encodingConfig.onEncodedSample?.(sample, meta); - void this.muxer!.addEncodedAudioSample(this.source._connectedTrack!, sample, meta); + this.customEncoder.onPacket = (packet, meta) => { + this.encodingConfig.onEncodedPacket?.(packet, meta); + void this.muxer!.addEncodedAudioPacket(this.source._connectedTrack!, packet, meta); }; this.customEncoder.init(); @@ -780,10 +780,10 @@ class AudioEncoderWrapper { this.encoder = new AudioEncoder({ output: (chunk, meta) => { - const sample = EncodedAudioSample.fromEncodedAudioChunk(chunk); + const packet = EncodedPacket.fromEncodedChunk(chunk); - this.encodingConfig.onEncodedSample?.(sample, meta); - void this.muxer!.addEncodedAudioSample(this.source._connectedTrack!, sample, meta); + this.encodingConfig.onEncodedPacket?.(packet, meta); + void this.muxer!.addEncodedAudioPacket(this.source._connectedTrack!, packet, meta); }, error: this.encodingConfig.onEncodingError ?? (error => console.error('AudioEncoder error:', error)), }); diff --git a/src/misc.ts b/src/misc.ts index d88b5fb..7a8467e 100644 --- a/src/misc.ts +++ b/src/misc.ts @@ -410,3 +410,6 @@ const ISO_639_2_REGEX = /^[a-z]{3}$/; export const isIso639Dash2LanguageCode = (x: string) => { return ISO_639_2_REGEX.test(x); }; + +// Since the result will be floored, add a bit of eps to compensate for floating point errors +export const SECOND_TO_MICROSECOND_FACTOR = 1e6 * (1 + Number.EPSILON); diff --git a/src/mp3/mp3-demuxer.ts b/src/mp3/mp3-demuxer.ts index 13a7655..83a519d 100644 --- a/src/mp3/mp3-demuxer.ts +++ b/src/mp3/mp3-demuxer.ts @@ -2,9 +2,9 @@ import { AudioCodec } from '../codec'; import { Demuxer } from '../demuxer'; import { Input } from '../input'; import { InputAudioTrack, InputAudioTrackBacking } from '../input-track'; -import { SampleRetrievalOptions } from '../media-sink'; +import { PacketRetrievalOptions } from '../media-sink'; import { assert, binarySearchExact, binarySearchLessOrEqual, last, UNDETERMINED_LANGUAGE } from '../misc'; -import { EncodedAudioSample, PLACEHOLDER_DATA } from '../sample'; +import { EncodedPacket, PLACEHOLDER_DATA } from '../packet'; import { FrameHeader, getXingOffset, INFO, XING } from './mp3-misc'; import { Mp3Reader } from './mp3-reader'; @@ -156,7 +156,7 @@ class Mp3AudioTrackBacking implements InputAudioTrackBacking { }; } - getSampleAtIndex(sampleIndex: number, options: SampleRetrievalOptions) { + getPacketAtIndex(sampleIndex: number, options: PacketRetrievalOptions) { if (sampleIndex === -1) { return null; } @@ -174,7 +174,7 @@ class Mp3AudioTrackBacking implements InputAudioTrackBacking { data = this.demuxer.reader.readBytes(rawSample.dataSize); } - return new EncodedAudioSample( + return new EncodedPacket( data, 'key', rawSample.timestamp, @@ -183,37 +183,37 @@ class Mp3AudioTrackBacking implements InputAudioTrackBacking { ); } - async getFirstSample(options: SampleRetrievalOptions) { - return this.getSampleAtIndex(0, options); + async getFirstPacket(options: PacketRetrievalOptions) { + return this.getPacketAtIndex(0, options); } - async getNextSample(sample: EncodedAudioSample, options: SampleRetrievalOptions) { + async getNextPacket(packet: EncodedPacket, options: PacketRetrievalOptions) { const sampleIndex = binarySearchExact( this.demuxer.allSamples, - sample.timestamp, + packet.timestamp, x => x.timestamp, ); if (sampleIndex === -1) { - throw new Error('Sample was not created from this track.'); + throw new Error('Packet was not created from this track.'); } - return this.getSampleAtIndex(sampleIndex + 1, options); + return this.getPacketAtIndex(sampleIndex + 1, options); } - async getSample(timestamp: number, options: SampleRetrievalOptions) { + async getPacket(timestamp: number, options: PacketRetrievalOptions) { const index = binarySearchLessOrEqual( this.demuxer.allSamples, timestamp, x => x.timestamp, ); - return this.getSampleAtIndex(index, options); + return this.getPacketAtIndex(index, options); } - getKeySample(timestamp: number, options: SampleRetrievalOptions) { - return this.getSample(timestamp, options); + getKeyPacket(timestamp: number, options: PacketRetrievalOptions) { + return this.getPacket(timestamp, options); } - getNextKeySample(sample: EncodedAudioSample, options: SampleRetrievalOptions) { - return this.getNextSample(sample, options); + getNextKeyPacket(packet: EncodedPacket, options: PacketRetrievalOptions) { + return this.getNextPacket(packet, options); } } diff --git a/src/mp3/mp3-muxer.ts b/src/mp3/mp3-muxer.ts index 29176c5..1bee238 100644 --- a/src/mp3/mp3-muxer.ts +++ b/src/mp3/mp3-muxer.ts @@ -1,7 +1,7 @@ import { assert, toDataView } from '../misc'; import { Muxer } from '../muxer'; import { Output, OutputAudioTrack } from '../output'; -import { EncodedAudioSample } from '../sample'; +import { EncodedPacket } from '../packet'; import { Writer } from '../writer'; import { getXingOffset, INFO, readFrameHeader, XING } from './mp3-misc'; import { Mp3Writer, XingFrameData } from './mp3-writer'; @@ -24,19 +24,19 @@ export class Mp3Muxer extends Muxer { // Nothing needed here } - async addEncodedVideoSample() { + async addEncodedVideoPacket() { throw new Error('MP3 does not support video.'); } - async addEncodedAudioSample( + async addEncodedAudioPacket( track: OutputAudioTrack, - sample: EncodedAudioSample, + packet: EncodedPacket, ) { const release = await this.mutex.acquire(); try { if (!this.xingFrameData) { - const view = toDataView(sample.data); + const view = toDataView(packet.data); if (view.byteLength < 4) { throw new Error('Invalid MP3 header in sample.'); } @@ -80,10 +80,10 @@ export class Mp3Muxer extends Muxer { this.frameCount++; } - this.validateAndNormalizeTimestamp(track, sample.timestamp, sample.type === 'key'); + this.validateAndNormalizeTimestamp(track, packet.timestamp, packet.type === 'key'); this.framePositions.push(this.writer.getPos()); - this.writer.write(sample.data); + this.writer.write(packet.data); this.frameCount++; await this.writer.flush(); diff --git a/src/muxer.ts b/src/muxer.ts index d7bf13d..7add20a 100644 --- a/src/muxer.ts +++ b/src/muxer.ts @@ -1,6 +1,6 @@ import { AsyncMutex } from './misc'; import { Output, OutputAudioTrack, OutputSubtitleTrack, OutputTrack, OutputVideoTrack } from './output'; -import { EncodedAudioSample, EncodedVideoSample } from './sample'; +import { EncodedPacket } from './packet'; import { SubtitleCue, SubtitleMetadata } from './subtitles'; export abstract class Muxer { @@ -12,14 +12,14 @@ export abstract class Muxer { } abstract start(): Promise; - abstract addEncodedVideoSample( + abstract addEncodedVideoPacket( track: OutputVideoTrack, - sample: EncodedVideoSample, + packet: EncodedPacket, meta?: EncodedVideoChunkMetadata ): Promise; - abstract addEncodedAudioSample( + abstract addEncodedAudioPacket( track: OutputAudioTrack, - sample: EncodedAudioSample, + packet: EncodedPacket, meta?: EncodedAudioChunkMetadata ): Promise; abstract addSubtitleCue(track: OutputSubtitleTrack, cue: SubtitleCue, meta?: SubtitleMetadata): Promise; diff --git a/src/ogg/ogg-demuxer.ts b/src/ogg/ogg-demuxer.ts index 2859477..e916c66 100644 --- a/src/ogg/ogg-demuxer.ts +++ b/src/ogg/ogg-demuxer.ts @@ -2,10 +2,10 @@ import { OPUS_INTERNAL_SAMPLE_RATE, parseOpusIdentificationHeader } from '../cod import { Demuxer } from '../demuxer'; import { Input } from '../input'; import { InputAudioTrack, InputAudioTrackBacking } from '../input-track'; -import { SampleRetrievalOptions } from '../media-sink'; +import { PacketRetrievalOptions } from '../media-sink'; import { assert, findLast, roundToPrecision, toDataView, UNDETERMINED_LANGUAGE } from '../misc'; +import { EncodedPacket, PLACEHOLDER_DATA } from '../packet'; import { Reader } from '../reader'; -import { EncodedAudioSample, PLACEHOLDER_DATA } from '../sample'; import { computeOggPageCrc, extractSampleMetadata, OggCodecInfo, parseModesFromVorbisSetupPacket } from './ogg-misc'; import { MAX_PAGE_HEADER_SIZE, MAX_PAGE_SIZE, MIN_PAGE_HEADER_SIZE, OggReader, Page } from './ogg-reader'; @@ -384,7 +384,7 @@ export class OggDemuxer extends Demuxer { } } -type SampleMetadata = { +type EncodedPacketMetadata = { packet: Packet; timestampInSamples: number; durationInSamples: number; @@ -393,7 +393,7 @@ type SampleMetadata = { class OggAudioTrackBacking implements InputAudioTrackBacking { internalSampleRate: number; - sampleToMetadata = new WeakMap(); + encodedPacketToMetadata = new WeakMap(); constructor(public bitstream: LogicalBitstream, public demuxer: OggDemuxer) { // Opus always uses a fixed sample rate for its internal calculations, even if the actual rate is different @@ -442,8 +442,8 @@ class OggAudioTrackBacking implements InputAudioTrackBacking { } async computeDuration() { - const lastSample = await this.getSample(Infinity, { metadataOnly: true }); - return (lastSample?.timestamp ?? 0) + (lastSample?.duration ?? 0); + const lastPacket = await this.getPacket(Infinity, { metadataOnly: true }); + return (lastPacket?.timestamp ?? 0) + (lastPacket?.duration ?? 0); } granulePositionToTimestampInSamples(granulePosition: number) { @@ -455,13 +455,13 @@ class OggAudioTrackBacking implements InputAudioTrackBacking { return granulePosition; } - createSampleFromPacket( + createEncodedPacketFromOggPacket( packet: Packet | null, additional: { timestampInSamples: number; vorbisLastBlocksize: number | null; }, - options: SampleRetrievalOptions, + options: PacketRetrievalOptions, ) { if (!packet) { return null; @@ -473,7 +473,7 @@ class OggAudioTrackBacking implements InputAudioTrackBacking { additional.vorbisLastBlocksize, ); - const sample = new EncodedAudioSample( + const encodedPacket = new EncodedPacket( options.metadataOnly ? PLACEHOLDER_DATA : packet.data, 'key', Math.max(0, additional.timestampInSamples) / this.internalSampleRate, @@ -481,16 +481,16 @@ class OggAudioTrackBacking implements InputAudioTrackBacking { packet.endPage.headerStartPos + packet.endSegmentIndex, ); - this.sampleToMetadata.set(sample, { + this.encodedPacketToMetadata.set(encodedPacket, { packet, timestampInSamples: additional.timestampInSamples, durationInSamples, vorbisBlockSize, }); - return sample; + return encodedPacket; } - async getFirstSample(options: SampleRetrievalOptions) { + async getFirstPacket(options: PacketRetrievalOptions) { assert(this.bitstream.lastMetadataPacket); const packetPosition = await this.demuxer.findNextPacketStart( this.demuxer.reader, @@ -512,7 +512,7 @@ class OggAudioTrackBacking implements InputAudioTrackBacking { packetPosition.startSegmentIndex, ); - return this.createSampleFromPacket( + return this.createEncodedPacketFromOggPacket( packet, { timestampInSamples, @@ -522,10 +522,10 @@ class OggAudioTrackBacking implements InputAudioTrackBacking { ); } - async getNextSample(prevSample: EncodedAudioSample, options: SampleRetrievalOptions) { - const prevMetadata = this.sampleToMetadata.get(prevSample); + async getNextPacket(prevPacket: EncodedPacket, options: PacketRetrievalOptions) { + const prevMetadata = this.encodedPacketToMetadata.get(prevPacket); if (!prevMetadata) { - throw new Error('Sample was not created from this track.'); + throw new Error('Packet was not created from this track.'); } const packetPosition = await this.demuxer.findNextPacketStart(this.demuxer.reader, prevMetadata.packet); @@ -541,7 +541,7 @@ class OggAudioTrackBacking implements InputAudioTrackBacking { packetPosition.startSegmentIndex, ); - return this.createSampleFromPacket( + return this.createEncodedPacketFromOggPacket( packet, { timestampInSamples, @@ -551,13 +551,13 @@ class OggAudioTrackBacking implements InputAudioTrackBacking { ); } - async getSample(timestamp: number, options: SampleRetrievalOptions) { + async getPacket(timestamp: number, options: PacketRetrievalOptions) { assert(this.demuxer.fileSize !== null); const timestampInSamples = roundToPrecision(timestamp * this.internalSampleRate, 14); if (timestampInSamples === 0) { // Fast path for timestamp 0 - avoids binary search when playing back from the start - return this.getFirstSample(options); + return this.getFirstPacket(options); } if (timestampInSamples < 0) { // There's nothing here @@ -581,8 +581,8 @@ class OggAudioTrackBacking implements InputAudioTrackBacking { const lowPages: Page[] = [lowPage]; // First, let's perform a binary serach (bisection search) on the file to find the approximate page where we'll - // find the sample. We want to find a page whose end sample position is less than or equal to the - // sample position we're searching for. + // find the packet. We want to find a page whose end packet position is less than or equal to the + // packet position we're searching for. // Outer loop: Does the binary serach outer: @@ -644,7 +644,7 @@ class OggAudioTrackBacking implements InputAudioTrackBacking { const isContinuationPage = page.granulePosition === -1; if (isContinuationPage) { - // No sample ends on this page - keep looking + // No packet ends on this page - keep looking searchStartPos = page.headerStartPos + page.totalSize; continue; } @@ -662,9 +662,9 @@ class OggAudioTrackBacking implements InputAudioTrackBacking { } } - // Now we have the last page with a sample position <= the sample position we're looking for, but there might - // be multiple pages with the sample position, in which case we actually need to find the first of such pages. - // We'll do this in two steps: First, let's find the latest page we know with an earlier sample position, and + // Now we have the last page with a packet position <= the packet position we're looking for, but there might + // be multiple pages with the packet position, in which case we actually need to find the first of such pages. + // We'll do this in two steps: First, let's find the latest page we know with an earlier packet position, and // then linear scan ourselves forward until we find the correct page. let lowerPage = startPosition.startPage; @@ -760,7 +760,7 @@ class OggAudioTrackBacking implements InputAudioTrackBacking { currentSegmentIndex = startPosition.segmentIndex; } } else { - // There is no next position, which means we're looking for the last sample in the bitstream. The + // There is no next position, which means we're looking for the last packet in the bitstream. The // granule position on the last page tends to be fucky, so let's instead start the search on the page // before that. So let's loop until we find a packet that ends in a previous page. while (true) { @@ -788,8 +788,8 @@ class OggAudioTrackBacking implements InputAudioTrackBacking { } } - let lastSample: EncodedAudioSample | null = null; - let lastSampleMetadata: SampleMetadata | null = null; + let lastEncodedPacket: EncodedPacket | null = null; + let lastEncodedPacketMetadata: EncodedPacketMetadata | null = null; // Alright, now it's time for the final, granular seek: We keep iterating over packets until we've found the one // with the correct timestamp - i.e., the last one with a timestamp <= the timestamp we're looking for. @@ -806,18 +806,18 @@ class OggAudioTrackBacking implements InputAudioTrackBacking { && currentSegmentIndex < startPosition.startSegmentIndex; if (!skipPacket) { - let sample = this.createSampleFromPacket( + let encodedPacket = this.createEncodedPacketFromOggPacket( packet, { timestampInSamples: currentTimestampInSamples, - vorbisLastBlocksize: lastSampleMetadata?.vorbisBlockSize ?? null, + vorbisLastBlocksize: lastEncodedPacketMetadata?.vorbisBlockSize ?? null, }, options, ); - assert(sample); + assert(encodedPacket); - let sampleMetadata = this.sampleToMetadata.get(sample); - assert(sampleMetadata); + let encodedPacketMetadata = this.encodedPacketToMetadata.get(encodedPacket); + assert(encodedPacketMetadata); if ( !currentTimestampIsCorrect @@ -828,25 +828,25 @@ class OggAudioTrackBacking implements InputAudioTrackBacking { currentTimestampInSamples = this.granulePositionToTimestampInSamples(currentPage.granulePosition); currentTimestampIsCorrect = true; - // Let's backpatch the sample we just created with the correct timestamp - sample = this.createSampleFromPacket( + // Let's backpatch the packet we just created with the correct timestamp + encodedPacket = this.createEncodedPacketFromOggPacket( packet, { - timestampInSamples: currentTimestampInSamples - sampleMetadata.durationInSamples, - vorbisLastBlocksize: lastSampleMetadata?.vorbisBlockSize ?? null, + timestampInSamples: currentTimestampInSamples - encodedPacketMetadata.durationInSamples, + vorbisLastBlocksize: lastEncodedPacketMetadata?.vorbisBlockSize ?? null, }, options, ); - assert(sample); + assert(encodedPacket); - sampleMetadata = this.sampleToMetadata.get(sample); - assert(sampleMetadata); + encodedPacketMetadata = this.encodedPacketToMetadata.get(encodedPacket); + assert(encodedPacketMetadata); } else { - currentTimestampInSamples += sampleMetadata.durationInSamples; + currentTimestampInSamples += encodedPacketMetadata.durationInSamples; } - lastSample = sample; - lastSampleMetadata = sampleMetadata; + lastEncodedPacket = encodedPacket; + lastEncodedPacketMetadata = encodedPacketMetadata; if ( currentTimestampIsCorrect @@ -854,7 +854,7 @@ class OggAudioTrackBacking implements InputAudioTrackBacking { // Next timestamp will be too late Math.max(currentTimestampInSamples, 0) > timestampInSamples // This timestamp already matches - || Math.max(sampleMetadata.timestampInSamples, 0) === timestampInSamples + || Math.max(encodedPacketMetadata.timestampInSamples, 0) === timestampInSamples ) ) { break; @@ -870,15 +870,15 @@ class OggAudioTrackBacking implements InputAudioTrackBacking { currentSegmentIndex = nextPosition.startSegmentIndex; } - return lastSample; + return lastEncodedPacket; } - getKeySample(timestamp: number, options: SampleRetrievalOptions) { - return this.getSample(timestamp, options); + getKeyPacket(timestamp: number, options: PacketRetrievalOptions) { + return this.getPacket(timestamp, options); } - getNextKeySample(sample: EncodedAudioSample, options: SampleRetrievalOptions) { - return this.getNextSample(sample, options); + getNextKeyPacket(packet: EncodedPacket, options: PacketRetrievalOptions) { + return this.getNextPacket(packet, options); } } @@ -894,7 +894,7 @@ const findPacketStartPosition = (pageList: Page[], endPage: Page, endSegmentInde for (segmentIndex; segmentIndex >= 0; segmentIndex--) { const lacingValue = page.lacingValues[segmentIndex]!; if (lacingValue < 255) { - segmentIndex++; // We know the last sample starts here + segmentIndex++; // We know the last packet starts here break outer; } } diff --git a/src/ogg/ogg-misc.ts b/src/ogg/ogg-misc.ts index 1185239..d8c18ff 100644 --- a/src/ogg/ogg-misc.ts +++ b/src/ogg/ogg-misc.ts @@ -182,8 +182,9 @@ export const parseModesFromVorbisSetupPacket = (setupHeader: Uint8Array) => { } bs.skip(1); modeCount++; - if (modeCount > 64) + if (modeCount > 64) { break; + } const bsClone = bs.clone(); const candidate = bsClone.read(6) + 1; if (candidate === modeCount) { diff --git a/src/ogg/ogg-muxer.ts b/src/ogg/ogg-muxer.ts index cc86580..6e06dbb 100644 --- a/src/ogg/ogg-muxer.ts +++ b/src/ogg/ogg-muxer.ts @@ -2,7 +2,7 @@ import { OPUS_INTERNAL_SAMPLE_RATE, parseOpusIdentificationHeader } from '../cod import { assert, setInt64, toDataView, toUint8Array } from '../misc'; import { Muxer } from '../muxer'; import { Output, OutputAudioTrack } from '../output'; -import { EncodedAudioSample } from '../sample'; +import { EncodedPacket } from '../packet'; import { Writer } from '../writer'; import { computeOggPageCrc, @@ -58,7 +58,7 @@ export class OggMuxer extends Muxer { // Nothin' } - addEncodedVideoSample(): never { + addEncodedVideoPacket(): never { throw new Error('Video tracks are not supported.'); } @@ -209,18 +209,18 @@ export class OggMuxer extends Muxer { } } - async addEncodedAudioSample(track: OutputAudioTrack, sample: EncodedAudioSample, meta?: EncodedAudioChunkMetadata) { + async addEncodedAudioPacket(track: OutputAudioTrack, packet: EncodedPacket, meta?: EncodedAudioChunkMetadata) { const release = await this.mutex.acquire(); try { const trackData = this.getTrackData(track, meta); - this.validateAndNormalizeTimestamp(trackData.track, sample.timestamp, sample.type === 'key'); + this.validateAndNormalizeTimestamp(trackData.track, packet.timestamp, packet.type === 'key'); const currentTimestampInSamples = trackData.currentTimestampInSamples; const { durationInSamples, vorbisBlockSize } = extractSampleMetadata( - sample.data, + packet.data, trackData.codecInfo, trackData.vorbisLastBlocksize, ); @@ -228,7 +228,7 @@ export class OggMuxer extends Muxer { trackData.vorbisLastBlocksize = vorbisBlockSize; trackData.packetQueue.push({ - data: sample.data, + data: packet.data, endGranulePosition: trackData.currentTimestampInSamples, timestamp: currentTimestampInSamples / trackData.internalSampleRate, forcePageFlush: false, diff --git a/src/packet.ts b/src/packet.ts new file mode 100644 index 0000000..e2ad5a0 --- /dev/null +++ b/src/packet.ts @@ -0,0 +1,116 @@ +import { SECOND_TO_MICROSECOND_FACTOR } from './misc'; + +export const PLACEHOLDER_DATA = new Uint8Array(0); + +export type PacketType = 'key' | 'delta'; + +export class EncodedPacket { + constructor( + public readonly data: Uint8Array, + public readonly type: PacketType, + public readonly timestamp: number, + public readonly duration: number, + public readonly sequenceNumber = -1, + public readonly byteLength = data.byteLength, + ) { + if (!(data instanceof Uint8Array)) { + throw new TypeError('data must be a Uint8Array.'); + } + if (type !== 'key' && type !== 'delta') { + throw new TypeError('type must be either "key" or "delta".'); + } + if (!Number.isFinite(timestamp)) { + throw new TypeError('timestamp must be a number.'); + } + if (!Number.isFinite(duration) || duration < 0) { + throw new TypeError('duration must be a non-negative number.'); + } + if (!Number.isFinite(sequenceNumber)) { + throw new TypeError('sequenceNumber must be a number.'); + } + if (!Number.isInteger(byteLength) || byteLength < 0) { + throw new TypeError('byteLength must be a non-negative integer.'); + } + } + + get isMetadataOnly() { + return this.data === PLACEHOLDER_DATA; + } + + get microsecondTimestamp() { + return Math.floor(SECOND_TO_MICROSECOND_FACTOR * this.timestamp); + } + + get microsecondDuration() { + return Math.floor(SECOND_TO_MICROSECOND_FACTOR * this.duration); + } + + toEncodedVideoChunk() { + if (this.isMetadataOnly) { + throw new TypeError('Metadata-only packets cannot be converted to a video chunk.'); + } + if (typeof EncodedVideoChunk === 'undefined') { + throw new Error('Your browser does not support EncodedVideoChunk.'); + } + + return new EncodedVideoChunk({ + data: this.data, + type: this.type, + timestamp: this.microsecondTimestamp, + duration: this.microsecondDuration, + }); + } + + toEncodedAudioChunk() { + if (this.isMetadataOnly) { + throw new TypeError('Metadata-only packets cannot be converted to an audio chunk.'); + } + if (typeof EncodedAudioChunk === 'undefined') { + throw new Error('Your browser does not support EncodedAudioChunk.'); + } + + return new EncodedAudioChunk({ + data: this.data, + type: this.type, + timestamp: this.microsecondTimestamp, + duration: this.microsecondDuration, + }); + } + + static fromEncodedChunk(chunk: EncodedVideoChunk | EncodedAudioChunk): EncodedPacket { + if (!(chunk instanceof EncodedVideoChunk || chunk instanceof EncodedAudioChunk)) { + throw new TypeError('chunk must be an EncodedVideoChunk or EncodedAudioChunk.'); + } + + const data = new Uint8Array(chunk.byteLength); + chunk.copyTo(data); + + return new EncodedPacket( + data, + chunk.type as PacketType, + chunk.timestamp / 1e6, + (chunk.duration ?? 0) / 1e6, + ); + } + + clone(options?: { timestamp?: number; duration?: number }): EncodedPacket { + if (options !== undefined && (typeof options !== 'object' || options === null)) { + throw new TypeError('options, when provided, must be an object.'); + } + if (options?.timestamp !== undefined && !Number.isFinite(options.timestamp)) { + throw new TypeError('options.timestamp, when provided, must be a number.'); + } + if (options?.duration !== undefined && !Number.isFinite(options.duration)) { + throw new TypeError('options.duration, when provided, must be a number.'); + } + + return new EncodedPacket( + this.data, + this.type, + options?.timestamp ?? this.timestamp, + options?.duration ?? this.duration, + this.sequenceNumber, + this.byteLength, + ); + } +} diff --git a/src/wave/wave-demuxer.ts b/src/wave/wave-demuxer.ts index eec771a..79b1167 100644 --- a/src/wave/wave-demuxer.ts +++ b/src/wave/wave-demuxer.ts @@ -2,10 +2,10 @@ import { AudioCodec } from '../codec'; import { Demuxer } from '../demuxer'; import { Input } from '../input'; import { InputAudioTrack, InputAudioTrackBacking } from '../input-track'; -import { SampleRetrievalOptions } from '../media-sink'; +import { PacketRetrievalOptions } from '../media-sink'; import { assert, UNDETERMINED_LANGUAGE } from '../misc'; +import { EncodedPacket, PLACEHOLDER_DATA } from '../packet'; import { Reader } from '../reader'; -import { EncodedAudioSample, PLACEHOLDER_DATA } from '../sample'; import { RiffReader } from './riff-reader'; export enum WaveFormat { @@ -178,7 +178,7 @@ export class WaveDemuxer extends Demuxer { } } -const SAMPLE_SIZE_IN_FRAMES = 2048; +const PACKET_SIZE_IN_FRAMES = 2048; class WaveAudioTrackBacking implements InputAudioTrackBacking { constructor(public demuxer: WaveDemuxer) {} @@ -232,18 +232,18 @@ class WaveAudioTrackBacking implements InputAudioTrackBacking { return 0; } - private async getSampleAtIndex( - sampleIndex: number, - options: SampleRetrievalOptions, - ): Promise { + private async getPacketAtIndex( + packetIndex: number, + options: PacketRetrievalOptions, + ): Promise { assert(this.demuxer.audioInfo); - const startOffset = sampleIndex * SAMPLE_SIZE_IN_FRAMES * this.demuxer.audioInfo.blockSizeInBytes; + const startOffset = packetIndex * PACKET_SIZE_IN_FRAMES * this.demuxer.audioInfo.blockSizeInBytes; if (startOffset >= this.demuxer.dataSize) { return null; } const sizeInBytes = Math.min( - SAMPLE_SIZE_IN_FRAMES * this.demuxer.audioInfo.blockSizeInBytes, + PACKET_SIZE_IN_FRAMES * this.demuxer.audioInfo.blockSizeInBytes, this.demuxer.dataSize - startOffset, ); @@ -251,12 +251,12 @@ class WaveAudioTrackBacking implements InputAudioTrackBacking { if (options.metadataOnly) { data = PLACEHOLDER_DATA; } else { - const sizeOfOneSample = SAMPLE_SIZE_IN_FRAMES * this.demuxer.audioInfo.blockSizeInBytes; - const chunkSize = Math.ceil(2 ** 19 / sizeOfOneSample) * sizeOfOneSample; + const sizeOfOnePacket = PACKET_SIZE_IN_FRAMES * this.demuxer.audioInfo.blockSizeInBytes; + const chunkSize = Math.ceil(2 ** 19 / sizeOfOnePacket) * sizeOfOnePacket; const chunkStart = Math.floor(startOffset / chunkSize) * chunkSize; const chunkEnd = chunkStart + chunkSize; - // Always load large 0.5 MiB chunks instead of just the required sample + // Always load large 0.5 MiB chunks instead of just the required packet await this.demuxer.chunkReader.reader.loadRange( this.demuxer.dataStart + chunkStart, this.demuxer.dataStart + chunkEnd, @@ -266,41 +266,41 @@ class WaveAudioTrackBacking implements InputAudioTrackBacking { data = this.demuxer.chunkReader.readBytes(sizeInBytes); } - const timestamp = sampleIndex * SAMPLE_SIZE_IN_FRAMES / this.demuxer.audioInfo.sampleRate; + const timestamp = packetIndex * PACKET_SIZE_IN_FRAMES / this.demuxer.audioInfo.sampleRate; const duration = sizeInBytes / this.demuxer.audioInfo.blockSizeInBytes / this.demuxer.audioInfo.sampleRate; - return new EncodedAudioSample( + return new EncodedPacket( data, 'key', timestamp, duration, - sampleIndex, + packetIndex, ); } - getFirstSample(options: SampleRetrievalOptions) { - return this.getSampleAtIndex(0, options); + getFirstPacket(options: PacketRetrievalOptions) { + return this.getPacketAtIndex(0, options); } - getSample(timestamp: number, options: SampleRetrievalOptions) { + getPacket(timestamp: number, options: PacketRetrievalOptions) { assert(this.demuxer.audioInfo); - const sampleIndex = Math.floor(timestamp * this.demuxer.audioInfo.sampleRate / SAMPLE_SIZE_IN_FRAMES); + const packetIndex = Math.floor(timestamp * this.demuxer.audioInfo.sampleRate / PACKET_SIZE_IN_FRAMES); - return this.getSampleAtIndex(sampleIndex, options); + return this.getPacketAtIndex(packetIndex, options); } - getNextSample(sample: EncodedAudioSample, options: SampleRetrievalOptions) { + getNextPacket(packet: EncodedPacket, options: PacketRetrievalOptions) { assert(this.demuxer.audioInfo); - const sampleIndex = Math.round(sample.timestamp * this.demuxer.audioInfo.sampleRate / SAMPLE_SIZE_IN_FRAMES); + const packetIndex = Math.round(packet.timestamp * this.demuxer.audioInfo.sampleRate / PACKET_SIZE_IN_FRAMES); - return this.getSampleAtIndex(sampleIndex + 1, options); + return this.getPacketAtIndex(packetIndex + 1, options); } - getKeySample(timestamp: number, options: SampleRetrievalOptions) { - return this.getSample(timestamp, options); + getKeyPacket(timestamp: number, options: PacketRetrievalOptions) { + return this.getPacket(timestamp, options); } - getNextKeySample(sample: EncodedAudioSample, options: SampleRetrievalOptions) { - return this.getNextSample(sample, options); + getNextKeyPacket(packet: EncodedPacket, options: PacketRetrievalOptions) { + return this.getNextPacket(packet, options); } } diff --git a/src/wave/wave-muxer.ts b/src/wave/wave-muxer.ts index 43212f1..218ae1b 100644 --- a/src/wave/wave-muxer.ts +++ b/src/wave/wave-muxer.ts @@ -1,10 +1,10 @@ import { Muxer } from '../muxer'; import { Output, OutputAudioTrack } from '../output'; -import { EncodedAudioSample } from '../sample'; import { parsePcmCodec, PcmAudioCodec } from '../codec'; import { WaveFormat } from './wave-demuxer'; import { RiffWriter } from './riff-writer'; import { Writer } from '../writer'; +import { EncodedPacket } from '../packet'; export class WaveMuxer extends Muxer { private writer: Writer; @@ -23,13 +23,13 @@ export class WaveMuxer extends Muxer { // Nothing needed here - we'll write the header with the first sample } - async addEncodedVideoSample() { + async addEncodedVideoPacket() { throw new Error('WAVE does not support video.'); } - async addEncodedAudioSample( + async addEncodedAudioPacket( track: OutputAudioTrack, - sample: EncodedAudioSample, + packet: EncodedPacket, meta?: EncodedAudioChunkMetadata, ) { const release = await this.mutex.acquire(); @@ -44,10 +44,10 @@ export class WaveMuxer extends Muxer { this.headerWritten = true; } - this.validateAndNormalizeTimestamp(track, sample.timestamp, sample.type === 'key'); + this.validateAndNormalizeTimestamp(track, packet.timestamp, packet.type === 'key'); - this.writer.write(sample.data); - this.dataSize += sample.data.byteLength; + this.writer.write(packet.data); + this.dataSize += packet.data.byteLength; await this.writer.flush(); } finally { diff --git a/todo.txt b/todo.txt index b4ea554..07fc482 100644 --- a/todo.txt +++ b/todo.txt @@ -3,4 +3,5 @@ - is this fixed? https://github.com/Vanilagy/webm-muxer/issues/50 - cross-track offset for streaming sources - configurable fragmented mp4 fragment size, like the mp4-muxer PR -- fix rotation matrix thing, make sure its counter-clockwise everywhere (is this a breaking change from mp4-muxer?) \ No newline at end of file +- fix rotation matrix thing, make sure its counter-clockwise everywhere (is this a breaking change from mp4-muxer?) +- canvassink ring buffer \ No newline at end of file