diff --git a/dev/demux.html b/dev/demux.html index cd9415d..a64b60c 100644 --- a/dev/demux.html +++ b/dev/demux.html @@ -13,6 +13,12 @@ source }); + const audioTrack = await input.getPrimaryAudioTrack(); + const drain = new Metamuxer.EncodedAudioChunkDrain(audioTrack); + + console.log(await drain.getFirstChunk()); + + /* const videoTrack = await input.getPrimaryVideoTrack(); const drain = new Metamuxer.VideoFrameDrain(videoTrack); @@ -44,6 +50,7 @@ } download(new Blob([target.buffer]), 'converted.mp4'); + */ /* const videoTrack = await input.getPrimaryVideoTrack(); diff --git a/dev/player.html b/dev/player.html index 44e6526..f3ad168 100644 --- a/dev/player.html +++ b/dev/player.html @@ -134,7 +134,9 @@ async function seek() { currentFrame = newCurrentFrame; nextFrame = newNextFrame; - context.drawImage(currentFrame.canvas, 0, 0); + if (currentFrame) { + context.drawImage(currentFrame.canvas, 0, 0); + } seeking = false; } diff --git a/src/codec.ts b/src/codec.ts index a2e4e85..3565027 100644 --- a/src/codec.ts +++ b/src/codec.ts @@ -11,9 +11,32 @@ import { import { SubtitleMetadata } from './subtitles'; /** @public */ -export const VIDEO_CODECS = ['avc', 'hevc', 'vp8', 'vp9', 'av1'] as const; +export const VIDEO_CODECS = [ + 'avc', + 'hevc', + 'vp8', + 'vp9', + 'av1', +] as const; /** @public */ -export const AUDIO_CODECS = ['aac', 'mp3', 'opus'] as const; // TODO add the rest +export const PCM_CODECS = [ + 'pcm-u8', + 'pcm-s8', + 'pcm-s16be', + 'pcm-s16le', + 'pcm-s24be', + 'pcm-s24le', + 'pcm-s32be', + 'pcm-s32le', + 'pcm-f32be', + 'pcm-f32le', +] as const; +export const AUDIO_CODECS = [ + 'aac', + 'mp3', + 'opus', + ...PCM_CODECS, +] as const; // TODO add the rest /** @public */ export const SUBTITLE_CODECS = ['webvtt'] as const; // TODO add the rest @@ -21,6 +44,7 @@ export const SUBTITLE_CODECS = ['webvtt'] as const; // TODO add the rest export type VideoCodec = typeof VIDEO_CODECS[number]; /** @public */ export type AudioCodec = typeof AUDIO_CODECS[number]; +export type PcmAudioCodec = typeof PCM_CODECS[number]; /** @public */ export type SubtitleCodec = typeof SUBTITLE_CODECS[number]; /** @public */ @@ -361,7 +385,6 @@ export const buildAudioCodecString = (codec: AudioCodec, numberOfChannels: numbe return 'vorbis'; // Also easy, this one } - // eslint-disable-next-line @typescript-eslint/restrict-template-expressions throw new TypeError(`Unhandled codec '${codec}'.`); }; @@ -373,11 +396,12 @@ export const extractAudioCodecString = (codec: AudioCodec, description: Uint8Arr return 'mp3'; } else if (codec === 'opus') { return 'opus'; + } else if (codec.startsWith('pcm-')) { + return codec; } else if (codec === 'vorbis') { return 'vorbis'; } - // eslint-disable-next-line @typescript-eslint/restrict-template-expressions throw new TypeError(`Unhandled codec '${codec}'.`); }; diff --git a/src/isobmff/isobmff-boxes.ts b/src/isobmff/isobmff-boxes.ts index bda787e..a5e9604 100644 --- a/src/isobmff/isobmff-boxes.ts +++ b/src/isobmff/isobmff-boxes.ts @@ -681,14 +681,14 @@ export const soundSampleDescription = ( ) => box(compressionType, [ Array(6).fill(0), // Reserved u16(1), // Data reference index - u16(0), // Version + u16(0), // Version (AudioSampleEntry, not AudioSampleEntryV1) u16(0), // Revision level u32(0), // Vendor u16(trackData.info.numberOfChannels), // Number of channels u16(16), // Sample size (bits) u16(0), // Compression ID u16(0), // Packet size - fixed_16_16(trackData.info.sampleRate), // Sample rate + u32(2 ** 16 * trackData.info.sampleRate), // Sample rate ], [ AUDIO_CODEC_TO_CONFIGURATION_BOX[trackData.track.source._codec](trackData), ]); diff --git a/src/isobmff/isobmff-demuxer.ts b/src/isobmff/isobmff-demuxer.ts index 0708ef9..cf0cbe6 100644 --- a/src/isobmff/isobmff-demuxer.ts +++ b/src/isobmff/isobmff-demuxer.ts @@ -83,7 +83,7 @@ type SampleTable = { presentationTimestamps: { presentationTimestamp: number; sampleIndex: number; - }[]; + }[] | null; }; type SampleTimingEntry = { startIndex: number; @@ -263,7 +263,7 @@ export class IsobmffDemuxer extends Demuxer { keySampleIndices: null, chunkOffsets: [], sampleToChunk: [], - presentationTimestamps: [], + presentationTimestamps: null, }; internalTrack.sampleTable = sampleTable; @@ -272,28 +272,116 @@ export class IsobmffDemuxer extends Demuxer { this.traverseBox(); this.currentTrack = null; - for (const entry of sampleTable.sampleTimingEntries) { - for (let i = 0; i < entry.count; i++) { - sampleTable.presentationTimestamps.push({ - presentationTimestamp: entry.startDecodeTimestamp + i * entry.delta, - sampleIndex: entry.startIndex + i, - }); - } - } + if ( + internalTrack.info?.type === 'audio' + && internalTrack.info.codec?.startsWith('pcm-') + && sampleTable.sampleCompositionTimeOffsets.length === 0 + ) { + // If the audio has PCM samples, the way the samples are defined in the sample table is somewhat + // suboptimal: Each individual audio sample is its own sample, meaning we can have 48000 samples per second. + // Because we treat each sample as its own atomic unit that can be decoded, this would lead to a huge + // amount of very short samples for PCM audio. So instead, we make a transformation: If the audio is in PCM, + // we say that each chunk (that normally holds many samples) now is one big sample. We can this because + // the samples in the chunk are contiguous and the format is PCM, so the entire chunk as one thing still + // encodes valid audio information. - for (const entry of sampleTable.sampleCompositionTimeOffsets) { - for (let i = 0; i < entry.count; i++) { - const sampleIndex = entry.startIndex + i; - const sample = sampleTable.presentationTimestamps[sampleIndex]; - if (!sample) { - continue; + const newSampleTimingEntries: SampleTimingEntry[] = []; + const newSampleSizes: number[] = []; + + for (let i = 0; i < sampleTable.sampleToChunk.length; i++) { + const chunkEntry = sampleTable.sampleToChunk[i]!; + const nextEntry = sampleTable.sampleToChunk[i + 1]; + const chunkCount = (nextEntry ? nextEntry.startChunkIndex : sampleTable.chunkOffsets.length) + - chunkEntry.startChunkIndex; + + for (let j = 0; j < chunkCount; j++) { + const startSampleIndex = chunkEntry.startSampleIndex + j * chunkEntry.samplesPerChunk; + const endSampleIndex = startSampleIndex + chunkEntry.samplesPerChunk; // Exclusive, outside of chunk + + const startTimingEntryIndex = binarySearchLessOrEqual( + sampleTable.sampleTimingEntries, + chunkEntry.startSampleIndex, + x => x.startIndex, + ); + const startTimingEntry = sampleTable.sampleTimingEntries[startTimingEntryIndex]!; + const endTimingEntryIndex = binarySearchLessOrEqual( + sampleTable.sampleTimingEntries, + endSampleIndex, + x => x.startIndex, + ); + const endTimingEntry = sampleTable.sampleTimingEntries[endTimingEntryIndex]!; + + const firstSampleTimestamp = startTimingEntry.startDecodeTimestamp + + (startSampleIndex - startTimingEntry.startIndex) * startTimingEntry.delta; + const lastSampleTimestamp = endTimingEntry.startDecodeTimestamp + + (endSampleIndex - endTimingEntry.startIndex) * endTimingEntry.delta; + const delta = lastSampleTimestamp - firstSampleTimestamp; + + const lastSampleTimingEntry = last(newSampleTimingEntries); + if (lastSampleTimingEntry && lastSampleTimingEntry.delta === delta) { + lastSampleTimingEntry.count++; + } else { + // One sample for the entire chunk + newSampleTimingEntries.push({ + startIndex: chunkEntry.startChunkIndex, + startDecodeTimestamp: firstSampleTimestamp, + count: 1, + delta, + }); + } + + // Compute the chunk size by summing the sample sizes + let chunkSize = 0; + if (sampleTable.sampleSizes.length === 1) { + // Given PCM, this branch should be the likely one + chunkSize = sampleTable.sampleSizes[0]! * chunkEntry.samplesPerChunk; + } else { + for (let k = startSampleIndex; k < endSampleIndex; k++) { + chunkSize += sampleTable.sampleSizes[k]!; + } + } + + newSampleSizes.push(chunkSize); } - sample.presentationTimestamp += entry.offset; + chunkEntry.startSampleIndex = chunkEntry.startChunkIndex; + chunkEntry.samplesPerChunk = 1; } + + sampleTable.sampleTimingEntries = newSampleTimingEntries; + sampleTable.sampleSizes = newSampleSizes; } - sampleTable.presentationTimestamps.sort((a, b) => a.presentationTimestamp - b.presentationTimestamp); + if (sampleTable.sampleCompositionTimeOffsets.length > 0) { + // If composition time offsets are defined, we must build a list of all presentation timestamps and then + // sort them + sampleTable.presentationTimestamps = []; + + for (const entry of sampleTable.sampleTimingEntries) { + for (let i = 0; i < entry.count; i++) { + sampleTable.presentationTimestamps.push({ + presentationTimestamp: entry.startDecodeTimestamp + i * entry.delta, + sampleIndex: entry.startIndex + i, + }); + } + } + + for (const entry of sampleTable.sampleCompositionTimeOffsets) { + for (let i = 0; i < entry.count; i++) { + const sampleIndex = entry.startIndex + i; + const sample = sampleTable.presentationTimestamps[sampleIndex]; + if (!sample) { + continue; + } + + sample.presentationTimestamp += entry.offset; + } + } + + sampleTable.presentationTimestamps.sort((a, b) => a.presentationTimestamp - b.presentationTimestamp); + } else { + // If they're not defined, we can simply use the decode timestamps as presentation timestamps + } return internalTrack.sampleTable; } @@ -613,9 +701,20 @@ export class IsobmffDemuxer extends Demuxer { this.readContiguousBoxes(startPos + sampleBoxInfo.totalSize - this.isobmffReader.pos); } else { if (sampleBoxInfo.name === 'mp4a') { - // We don't know the codec yet, need to read the esds box + // We don't know the codec yet (might be AAC, might be MP3), need to read the esds box } else if (sampleBoxInfo.name.toLowerCase() === 'opus') { track.info.codec = 'opus'; + } else if ( + sampleBoxInfo.name === 'twos' + || sampleBoxInfo.name === 'sowt' + || sampleBoxInfo.name === 'raw ' + || sampleBoxInfo.name === 'in24' + || sampleBoxInfo.name === 'in32' + || sampleBoxInfo.name === 'fl32' + || sampleBoxInfo.name === 'lpcm' + ) { + // It's PCM + // developer.apple.com/documentation/quicktime-file-format/sound_sample_descriptions/ } else { const { name } = sampleBoxInfo; console.warn(`Unsupported audio codec (sample entry type '${name}') - discarding track.`); @@ -628,8 +727,9 @@ export class IsobmffDemuxer extends Demuxer { this.isobmffReader.pos += 3 * 2; let channelCount = this.isobmffReader.readU16(); + let sampleSize = this.isobmffReader.readU16(); - this.isobmffReader.pos += 2 + 2 + 2; + this.isobmffReader.pos += 2 * 2; // Can't use fixed16_16 as that's signed let sampleRate = this.isobmffReader.readU32() / 0x10000; @@ -643,18 +743,13 @@ export class IsobmffDemuxer extends Demuxer { sampleRate = this.isobmffReader.readF64(); channelCount = this.isobmffReader.readU32(); this.isobmffReader.pos += 4; // Always 0x7F000000 - // eslint-disable-next-line @typescript-eslint/no-unused-vars - const sampleSize = this.isobmffReader.readU32(); - // eslint-disable-next-line @typescript-eslint/no-unused-vars + sampleSize = this.isobmffReader.readU32(); + const flags = this.isobmffReader.readU32(); - // eslint-disable-next-line @typescript-eslint/no-unused-vars - const bytesPerFrame = this.isobmffReader.readU32(); - // eslint-disable-next-line @typescript-eslint/no-unused-vars - const samplesPerFrame = this.isobmffReader.readU32(); + this.isobmffReader.pos += 2 * 4; - /* if (sampleBoxInfo.name === 'lpcm') { const bytesPerSample = (sampleSize + 7) >> 3; const isFloat = Boolean(flags & 1); @@ -664,36 +759,64 @@ export class IsobmffDemuxer extends Demuxer { if (sampleSize > 0 && sampleSize <= 64) { if (isFloat) { if (sampleSize === 32 && !isBigEndian) { - track.pcmType = 'pcm-f32'; + track.info.codec = isBigEndian ? 'pcm-f32be' : 'pcm-f32le'; } } else { if (sFlags & (1 << (bytesPerSample - 1))) { - if (bytesPerSample === 2 && !isBigEndian) { - track.pcmType = 'pcm-s16'; - } else if (bytesPerSample === 3 && !isBigEndian) { - track.pcmType = 'pcm-s24'; - } else if (bytesPerSample === 4 && !isBigEndian) { - track.pcmType = 'pcm-s32'; + if (bytesPerSample === 1) { + track.info.codec = 'pcm-s8'; + } else if (bytesPerSample === 2) { + track.info.codec = isBigEndian ? 'pcm-s16be' : 'pcm-s16le'; + } else if (bytesPerSample === 3) { + track.info.codec = isBigEndian ? 'pcm-s24be' : 'pcm-s24le'; + } else if (bytesPerSample === 4) { + track.info.codec = isBigEndian ? 'pcm-s32be' : 'pcm-s32le'; } } else { if (bytesPerSample === 1) { - track.pcmType = 'pcm-u8'; + track.info.codec = 'pcm-u8'; } } } } - if (track.pcmType === null) { - throw new Error(`Unsupported linear PCM type.`); + if (track.info.codec === null) { + console.warn('Unsupportedd PCM format.'); + break; } } - */ } } track.info.numberOfChannels = channelCount; track.info.sampleRate = sampleRate; + if (sampleBoxInfo.name === 'twos') { + if (sampleSize === 8) { + track.info.codec = 'pcm-s8'; + } else if (sampleSize === 16) { + track.info.codec = 'pcm-s16be'; + } else { + throw new Error(`Unsupported sample size ${sampleSize} for codec 'twos'.`); + } + } else if (sampleBoxInfo.name === 'sowt') { + if (sampleSize === 8) { + track.info.codec = 'pcm-s8'; + } else if (sampleSize === 16) { + track.info.codec = 'pcm-s16le'; + } else { + throw new Error(`Unsupported sample size ${sampleSize} for codec 'sowt'.`); + } + } else if (sampleBoxInfo.name === 'raw ') { + track.info.codec = 'pcm-u8'; + } else if (sampleBoxInfo.name === 'in24') { + track.info.codec = 'pcm-s24be'; + } else if (sampleBoxInfo.name === 'in32') { + track.info.codec = 'pcm-s32be'; + } else if (sampleBoxInfo.name === 'fl32') { + track.info.codec = 'pcm-f32be'; + } + this.readContiguousBoxes(startPos + sampleBoxInfo.totalSize - this.isobmffReader.pos); } } @@ -873,6 +996,25 @@ export class IsobmffDemuxer extends Demuxer { } }; break; + case 'enda': { + const track = this.currentTrack; + assert(track && track.info?.type === 'audio'); + + const littleEndian = this.isobmffReader.readU16() & 0xff; // 0xff is from FFmpeg + + if (littleEndian) { + if (track.info.codec === 'pcm-s16be') { + track.info.codec = 'pcm-s16le'; + } else if (track.info.codec === 'pcm-s24be') { + track.info.codec = 'pcm-s24le'; + } else if (track.info.codec === 'pcm-s32be') { + track.info.codec = 'pcm-s32le'; + } else if (track.info.codec === 'pcm-f32be') { + track.info.codec = 'pcm-f32le'; + } + } + }; break; + case 'stts': { const track = this.currentTrack; assert(track); @@ -1966,16 +2108,30 @@ class IsobmffAudioTrackBacking extends IsobmffTrackBacking im } const getSampleIndexForTimestamp = (sampleTable: SampleTable, timescaleUnits: number) => { - const index = binarySearchLessOrEqual( - sampleTable.presentationTimestamps, - timescaleUnits, - x => x.presentationTimestamp, - ); - if (index === -1) { - return -1; - } + if (sampleTable.presentationTimestamps) { + const index = binarySearchLessOrEqual( + sampleTable.presentationTimestamps, + timescaleUnits, + x => x.presentationTimestamp, + ); + if (index === -1) { + return -1; + } - return sampleTable.presentationTimestamps[index]!.sampleIndex; + return sampleTable.presentationTimestamps[index]!.sampleIndex; + } else { + const index = binarySearchLessOrEqual( + sampleTable.sampleTimingEntries, + timescaleUnits, + x => x.startDecodeTimestamp, + ); + if (index === -1) { + return -1; + } + + const entry = sampleTable.sampleTimingEntries[index]!; + return entry.startIndex + Math.floor((timescaleUnits - entry.startDecodeTimestamp) / entry.delta); + } }; type SampleInfo = { diff --git a/src/media-drain.ts b/src/media-drain.ts index 5454a23..12e9116 100644 --- a/src/media-drain.ts +++ b/src/media-drain.ts @@ -1,5 +1,14 @@ +import { PCM_CODECS, PcmAudioCodec } from './codec'; import { InputAudioTrack, InputVideoTrack } from './input-track'; -import { AnyIterable, assert, promiseWithResolvers, toAsyncIterator, validateAnyIterable } from './misc'; +import { + AnyIterable, + assert, + getInt24, + getUint24, + promiseWithResolvers, + toAsyncIterator, + validateAnyIterable, +} from './misc'; /** @public */ export type ChunkRetrievalOptions = { @@ -104,6 +113,21 @@ export abstract class BaseChunkDrain { + constructor( + public onMedia: (media: MediaFrame) => unknown, + public onError: (error: DOMException) => unknown, + ) {} + + abstract getDecodeQueueSize(): number; + abstract decode(chunk: Chunk): void; + abstract flush(): Promise; + abstract close(): void; +} + /** @public */ export abstract class BaseMediaFrameDrain< Chunk extends EncodedVideoChunk | EncodedAudioChunk, @@ -113,7 +137,7 @@ export abstract class BaseMediaFrameDrain< abstract _createDecoder( onMedia: (media: MediaFrame) => unknown, onError: (error: DOMException) => unknown - ): Promise; + ): Promise>; /** @internal */ abstract _createChunkDrain(): BaseChunkDrain; @@ -185,7 +209,7 @@ export abstract class BaseMediaFrameDrain< for await (const timestamp of timestampIterator) { validateTimestamp(timestamp); - while (frameQueue.length + decoder.decodeQueueSize > MAX_QUEUE_SIZE) { + while (frameQueue.length + decoder.getDecodeQueueSize() > MAX_QUEUE_SIZE) { ({ promise: queueDequeue, resolve: onQueueDequeue } = promiseWithResolvers()); await queueDequeue; } @@ -236,10 +260,6 @@ export abstract class BaseMediaFrameDrain< lastChunk = nextChunk; decoder.decode(nextChunk); } - - if (decoder.decodeQueueSize >= 10) { - await new Promise(resolve => decoder.addEventListener('dequeue', resolve, { once: true })); - } } await decoder.flush(); @@ -376,7 +396,7 @@ export abstract class BaseMediaFrameDrain< await chunks.next(); while (currentChunk && !ended) { - if (frameQueue.length + decoder.decodeQueueSize > MAX_QUEUE_SIZE) { + if (frameQueue.length + decoder.getDecodeQueueSize() > MAX_QUEUE_SIZE) { ({ promise: queueDequeue, resolve: onQueueDequeue } = promiseWithResolvers()); await queueDequeue; continue; @@ -483,6 +503,37 @@ export class EncodedVideoChunkDrain extends BaseChunkDrain { } } +class VideoDecoderWrapper extends DecoderWrapper { + decoder: VideoDecoder; + + constructor( + onFrame: (frame: VideoFrame) => unknown, + onError: (error: DOMException) => unknown, + decoderConfig: VideoDecoderConfig, + ) { + super(onFrame, onError); + + this.decoder = new VideoDecoder({ output: onFrame, error: onError }); + this.decoder.configure(decoderConfig); + } + + getDecodeQueueSize() { + return this.decoder.decodeQueueSize; + } + + decode(chunk: EncodedVideoChunk) { + this.decoder.decode(chunk); + } + + flush() { + return this.decoder.flush(); + } + + close() { + this.decoder.close(); + } +} + /** @public */ export class VideoFrameDrain extends BaseMediaFrameDrain { /** @internal */ @@ -503,11 +554,7 @@ export class VideoFrameDrain extends BaseMediaFrameDrain unknown, onError: (error: DOMException) => unknown) { this._decoderConfig ??= await this._videoTrack.getDecoderConfig(); - - const decoder = new VideoDecoder({ output: onFrame, error: onError }); - decoder.configure(this._decoderConfig); - - return decoder; + return new VideoDecoderWrapper(onFrame, onError, this._decoderConfig); } /** @internal */ @@ -668,6 +715,185 @@ export class EncodedAudioChunkDrain extends BaseChunkDrain { } } +class AudioDecoderWrapper extends DecoderWrapper { + decoder: AudioDecoder; + + constructor( + onData: (data: AudioData) => unknown, + onError: (error: DOMException) => unknown, + decoderConfig: AudioDecoderConfig, + ) { + super(onData, onError); + + this.decoder = new AudioDecoder({ output: onData, error: onError }); + this.decoder.configure(decoderConfig); + } + + getDecodeQueueSize() { + return this.decoder.decodeQueueSize; + } + + decode(chunk: EncodedAudioChunk) { + this.decoder.decode(chunk); + } + + flush() { + return this.decoder.flush(); + } + + close() { + this.decoder.close(); + } +} + +const PCM_CODEC_REGEX = /^pcm-([usf])(\d+)+(be|le)?$/; + +// There are a lot of PCM variants not natively supported by the browser and by AudioData. Therefore we need a simple +// decoder that maps any input PCM format into a PCM format supported by the browser. +class PcmAudioDecoderWrapper extends DecoderWrapper { + codec: PcmAudioCodec; + + inputSampleSize: 1 | 2 | 3 | 4; + readInputValue: (view: DataView, byteOffset: number) => number; + + outputSampleSize: 1 | 2 | 4; + outputFormat: 'u8' | 's16' | 's32' | 'f32'; + writeOutputValue: (view: DataView, byteOffset: number, value: number) => void; + + constructor( + onData: (data: AudioData) => unknown, + onError: (error: DOMException) => unknown, + public decoderConfig: AudioDecoderConfig, + ) { + super(onData, onError); + + assert((PCM_CODECS as readonly string[]).includes(decoderConfig.codec)); + this.codec = decoderConfig.codec as PcmAudioCodec; + + const match = this.codec.match(PCM_CODEC_REGEX); + assert(match); + + let dataType: 'unsigned' | 'signed' | 'float'; + if (match[1] === 'u') { + dataType = 'unsigned'; + } else if (match[1] === 's') { + dataType = 'signed'; + } else { + dataType = 'float'; + } + + this.inputSampleSize = (Number(match[2]) / 8) as 1 | 2 | 3 | 4; + const littleEndian = match[3] === 'le'; + + switch (this.inputSampleSize) { + case 1: { + if (dataType === 'unsigned') { + this.readInputValue = (view, byteOffset) => view.getUint8(byteOffset) - 2 ** 7; + } else { + this.readInputValue = (view, byteOffset) => view.getInt8(byteOffset); + } + }; break; + case 2: { + if (dataType === 'unsigned') { + this.readInputValue = (view, byteOffset) => view.getUint16(byteOffset, littleEndian) - 2 ** 15; + } else { + this.readInputValue = (view, byteOffset) => view.getInt16(byteOffset, littleEndian); + } + }; break; + case 3: { + if (dataType === 'unsigned') { + this.readInputValue = (view, byteOffset) => getUint24(view, byteOffset, littleEndian) - 2 ** 23; + } else { + this.readInputValue = (view, byteOffset) => getInt24(view, byteOffset, littleEndian); + } + }; break; + case 4: { + if (dataType === 'unsigned') { + this.readInputValue = (view, byteOffset) => view.getUint32(byteOffset, littleEndian) - 2 ** 31; + } else if (dataType === 'signed') { + this.readInputValue = (view, byteOffset) => view.getInt32(byteOffset, littleEndian); + } else { + this.readInputValue = (view, byteOffset) => view.getFloat32(byteOffset, littleEndian); + } + }; break; + } + + switch (this.inputSampleSize) { + case 1: { + this.outputSampleSize = 1; + this.outputFormat = 'u8'; + this.writeOutputValue = (view, byteOffset, value) => view.setUint8(byteOffset, value + 2 ** 7); + }; break; + case 2: { + this.outputSampleSize = 2; + this.outputFormat = 's16'; + this.writeOutputValue = (view, byteOffset, value) => view.setInt16(byteOffset, value, true); + }; break; + case 3: { + this.outputSampleSize = 4; + this.outputFormat = 's32'; + // From https://www.w3.org/TR/webcodecs: + // AudioData containing 24-bit samples SHOULD store those samples in s32 or f32. When samples are + // stored in s32, each sample MUST be left-shifted by 8 bits. + this.writeOutputValue = (view, byteOffset, value) => view.setInt32(byteOffset, value << 8, true); + }; break; + case 4: { + this.outputSampleSize = 4; + + if (dataType === 'float') { + this.outputFormat = 'f32'; + this.writeOutputValue = (view, byteOffset, value) => view.setFloat32(byteOffset, value, true); + } else { + this.outputFormat = 's32'; + this.writeOutputValue = (view, byteOffset, value) => view.setInt32(byteOffset, value, true); + } + }; break; + }; + } + + getDecodeQueueSize() { + return 0; + } + + decode(chunk: EncodedAudioChunk) { + const inputBuffer = new ArrayBuffer(chunk.byteLength); + const inputView = new DataView(inputBuffer); + chunk.copyTo(inputBuffer); + + const numberOfFrames = chunk.byteLength / this.decoderConfig.numberOfChannels / this.inputSampleSize; + + const outputBufferSize = numberOfFrames * this.decoderConfig.numberOfChannels * this.outputSampleSize; + const outputBuffer = new ArrayBuffer(outputBufferSize); + const outputView = new DataView(outputBuffer); + + for (let i = 0; i < numberOfFrames * this.decoderConfig.numberOfChannels; i++) { + const inputIndex = i * this.inputSampleSize; + const outputIndex = i * this.outputSampleSize; + + const value = this.readInputValue(inputView, inputIndex); + this.writeOutputValue(outputView, outputIndex, value); + } + + const audioData = new AudioData({ + format: this.outputFormat, + data: outputBuffer, + numberOfChannels: this.decoderConfig.numberOfChannels, + sampleRate: this.decoderConfig.sampleRate, + numberOfFrames, + timestamp: chunk.timestamp, + }); + this.onMedia(audioData); + } + + async flush() { + // Do nothing + } + + close() { + // Do nothing + } +} + /** @public */ export class AudioDataDrain extends BaseMediaFrameDrain { /** @internal */ @@ -689,10 +915,11 @@ export class AudioDataDrain extends BaseMediaFrameDrain unknown, onError: (error: DOMException) => unknown) { this._decoderConfig ??= await this._audioTrack.getDecoderConfig(); - const decoder = new AudioDecoder({ output: onData, error: onError }); - decoder.configure(this._decoderConfig); - - return decoder; + if (this._decoderConfig.codec.startsWith('pcm-')) { + return new PcmAudioDecoderWrapper(onData, onError, this._decoderConfig); + } else { + return new AudioDecoderWrapper(onData, onError, this._decoderConfig); + } } /** @internal */ diff --git a/src/misc.ts b/src/misc.ts index 3df6e34..cb770af 100644 --- a/src/misc.ts +++ b/src/misc.ts @@ -245,3 +245,26 @@ export const validateAnyIterable = (iterable: AnyIterable) => { throw new TypeError('Argument must be an iterable or async iterable.'); } }; + +export const assertNever = (x: never) => { + // eslint-disable-next-line @typescript-eslint/restrict-template-expressions + throw new Error(`Unexpected value: ${x}`); +}; + +export const getUint24 = (view: DataView, byteOffset: number, littleEndian: boolean) => { + const byte1 = view.getUint8(byteOffset); + const byte2 = view.getUint8(byteOffset + 1); + const byte3 = view.getUint8(byteOffset + 2); + + if (littleEndian) { + return byte1 | (byte2 << 8) | (byte3 << 16); + } else { + return (byte1 << 16) | (byte2 << 8) | byte3; + } +}; + +export const getInt24 = (view: DataView, byteOffset: number, littleEndian: boolean) => { + // The left shift pushes the most significant bit into the sign bit region, and the subsequent right shift + // then correctly interprets the sign bit. + return getUint24(view, byteOffset, littleEndian) << 8 >> 8; +};