Implement PCM decoder, add PCM support to ISOBMFF demuxer

dev/demux.html

@@ -13,6 +13,12 @@
 			source
 		});
 
+		const audioTrack = await input.getPrimaryAudioTrack();
+		const drain = new Metamuxer.EncodedAudioChunkDrain(audioTrack);
+
+		console.log(await drain.getFirstChunk());
+
+		/*
 		const videoTrack = await input.getPrimaryVideoTrack();
 		const drain = new Metamuxer.VideoFrameDrain(videoTrack);
 
@@ -44,6 +50,7 @@
 		}
 
 		download(new Blob([target.buffer]), 'converted.mp4');
+		*/
 
 		/*
 		const videoTrack = await input.getPrimaryVideoTrack();

dev/player.html

@@ -134,7 +134,9 @@ async function seek() {
 	currentFrame = newCurrentFrame;
 	nextFrame = newNextFrame;
 
-	context.drawImage(currentFrame.canvas, 0, 0);
+	if (currentFrame) {
+		context.drawImage(currentFrame.canvas, 0, 0);
+	}
 	
 	seeking = false;
 }

src/codec.ts

@@ -11,9 +11,32 @@ import {
 import { SubtitleMetadata } from './subtitles';
 
 /** @public */
-export const VIDEO_CODECS = ['avc', 'hevc', 'vp8', 'vp9', 'av1'] as const;
+export const VIDEO_CODECS = [
+	'avc',
+	'hevc',
+	'vp8',
+	'vp9',
+	'av1',
+] as const;
 /** @public */
-export const AUDIO_CODECS = ['aac', 'mp3', 'opus'] as const; // TODO add the rest
+export const PCM_CODECS = [
+	'pcm-u8',
+	'pcm-s8',
+	'pcm-s16be',
+	'pcm-s16le',
+	'pcm-s24be',
+	'pcm-s24le',
+	'pcm-s32be',
+	'pcm-s32le',
+	'pcm-f32be',
+	'pcm-f32le',
+] as const;
+export const AUDIO_CODECS = [
+	'aac',
+	'mp3',
+	'opus',
+	...PCM_CODECS,
+] as const; // TODO add the rest
 /** @public */
 export const SUBTITLE_CODECS = ['webvtt'] as const; // TODO add the rest
 
@@ -21,6 +44,7 @@ export const SUBTITLE_CODECS = ['webvtt'] as const; // TODO add the rest
 export type VideoCodec = typeof VIDEO_CODECS[number];
 /** @public */
 export type AudioCodec = typeof AUDIO_CODECS[number];
+export type PcmAudioCodec = typeof PCM_CODECS[number];
 /** @public */
 export type SubtitleCodec = typeof SUBTITLE_CODECS[number];
 /** @public */
@@ -361,7 +385,6 @@ export const buildAudioCodecString = (codec: AudioCodec, numberOfChannels: numbe
 		return 'vorbis'; // Also easy, this one
 	}
 
-	// eslint-disable-next-line @typescript-eslint/restrict-template-expressions
 	throw new TypeError(`Unhandled codec '${codec}'.`);
 };
 
@@ -373,11 +396,12 @@ export const extractAudioCodecString = (codec: AudioCodec, description: Uint8Arr
 		return 'mp3';
 	} else if (codec === 'opus') {
 		return 'opus';
+	} else if (codec.startsWith('pcm-')) {
+		return codec;
 	} else if (codec === 'vorbis') {
 		return 'vorbis';
 	}
 
-	// eslint-disable-next-line @typescript-eslint/restrict-template-expressions
 	throw new TypeError(`Unhandled codec '${codec}'.`);
 };
 

src/isobmff/isobmff-boxes.ts

@@ -681,14 +681,14 @@ export const soundSampleDescription = (
 ) => box(compressionType, [
 	Array(6).fill(0), // Reserved
 	u16(1), // Data reference index
-	u16(0), // Version
+	u16(0), // Version (AudioSampleEntry, not AudioSampleEntryV1)
 	u16(0), // Revision level
 	u32(0), // Vendor
 	u16(trackData.info.numberOfChannels), // Number of channels
 	u16(16), // Sample size (bits)
 	u16(0), // Compression ID
 	u16(0), // Packet size
-	fixed_16_16(trackData.info.sampleRate), // Sample rate
+	u32(2 ** 16 * trackData.info.sampleRate), // Sample rate
 ], [
 	AUDIO_CODEC_TO_CONFIGURATION_BOX[trackData.track.source._codec](trackData),
 ]);

src/isobmff/isobmff-demuxer.ts

@@ -83,7 +83,7 @@ type SampleTable = {
 	presentationTimestamps: {
 		presentationTimestamp: number;
 		sampleIndex: number;
-	}[];
+	}[] | null;
 };
 type SampleTimingEntry = {
 	startIndex: number;
@@ -263,7 +263,7 @@ export class IsobmffDemuxer extends Demuxer {
 			keySampleIndices: null,
 			chunkOffsets: [],
 			sampleToChunk: [],
-			presentationTimestamps: [],
+			presentationTimestamps: null,
 		};
 		internalTrack.sampleTable = sampleTable;
 
@@ -272,28 +272,116 @@ export class IsobmffDemuxer extends Demuxer {
 		this.traverseBox();
 		this.currentTrack = null;
 
-		for (const entry of sampleTable.sampleTimingEntries) {
+		if (
-			for (let i = 0; i < entry.count; i++) {
+			internalTrack.info?.type === 'audio'
-				sampleTable.presentationTimestamps.push({
+			&& internalTrack.info.codec?.startsWith('pcm-')
-					presentationTimestamp: entry.startDecodeTimestamp + i * entry.delta,
+			&& sampleTable.sampleCompositionTimeOffsets.length === 0
-					sampleIndex: entry.startIndex + i,
+		) {
-				});
+			// If the audio has PCM samples, the way the samples are defined in the sample table is somewhat
-			}
+			// suboptimal: Each individual audio sample is its own sample, meaning we can have 48000 samples per second.
-		}
+			// Because we treat each sample as its own atomic unit that can be decoded, this would lead to a huge
+			// amount of very short samples for PCM audio. So instead, we make a transformation: If the audio is in PCM,
+			// we say that each chunk (that normally holds many samples) now is one big sample. We can this because
+			// the samples in the chunk are contiguous and the format is PCM, so the entire chunk as one thing still
+			// encodes valid audio information.
 
-		for (const entry of sampleTable.sampleCompositionTimeOffsets) {
+			const newSampleTimingEntries: SampleTimingEntry[] = [];
-			for (let i = 0; i < entry.count; i++) {
+			const newSampleSizes: number[] = [];
-				const sampleIndex = entry.startIndex + i;
+
-				const sample = sampleTable.presentationTimestamps[sampleIndex];
+			for (let i = 0; i < sampleTable.sampleToChunk.length; i++) {
-				if (!sample) {
+				const chunkEntry = sampleTable.sampleToChunk[i]!;
-					continue;
+				const nextEntry = sampleTable.sampleToChunk[i + 1];
+				const chunkCount = (nextEntry ? nextEntry.startChunkIndex : sampleTable.chunkOffsets.length)
+					- chunkEntry.startChunkIndex;
+
+				for (let j = 0; j < chunkCount; j++) {
+					const startSampleIndex = chunkEntry.startSampleIndex + j * chunkEntry.samplesPerChunk;
+					const endSampleIndex = startSampleIndex + chunkEntry.samplesPerChunk; // Exclusive, outside of chunk
+
+					const startTimingEntryIndex = binarySearchLessOrEqual(
+						sampleTable.sampleTimingEntries,
+						chunkEntry.startSampleIndex,
+						x => x.startIndex,
+					);
+					const startTimingEntry = sampleTable.sampleTimingEntries[startTimingEntryIndex]!;
+					const endTimingEntryIndex = binarySearchLessOrEqual(
+						sampleTable.sampleTimingEntries,
+						endSampleIndex,
+						x => x.startIndex,
+					);
+					const endTimingEntry = sampleTable.sampleTimingEntries[endTimingEntryIndex]!;
+
+					const firstSampleTimestamp = startTimingEntry.startDecodeTimestamp
+						+ (startSampleIndex - startTimingEntry.startIndex) * startTimingEntry.delta;
+					const lastSampleTimestamp = endTimingEntry.startDecodeTimestamp
+						+ (endSampleIndex - endTimingEntry.startIndex) * endTimingEntry.delta;
+					const delta = lastSampleTimestamp - firstSampleTimestamp;
+
+					const lastSampleTimingEntry = last(newSampleTimingEntries);
+					if (lastSampleTimingEntry && lastSampleTimingEntry.delta === delta) {
+						lastSampleTimingEntry.count++;
+					} else {
+						// One sample for the entire chunk
+						newSampleTimingEntries.push({
+							startIndex: chunkEntry.startChunkIndex,
+							startDecodeTimestamp: firstSampleTimestamp,
+							count: 1,
+							delta,
+						});
+					}
+
+					// Compute the chunk size by summing the sample sizes
+					let chunkSize = 0;
+					if (sampleTable.sampleSizes.length === 1) {
+						// Given PCM, this branch should be the likely one
+						chunkSize = sampleTable.sampleSizes[0]! * chunkEntry.samplesPerChunk;
+					} else {
+						for (let k = startSampleIndex; k < endSampleIndex; k++) {
+							chunkSize += sampleTable.sampleSizes[k]!;
+						}
+					}
+
+					newSampleSizes.push(chunkSize);
 				}
 
-				sample.presentationTimestamp += entry.offset;
+				chunkEntry.startSampleIndex = chunkEntry.startChunkIndex;
+				chunkEntry.samplesPerChunk = 1;
 			}
+
+			sampleTable.sampleTimingEntries = newSampleTimingEntries;
+			sampleTable.sampleSizes = newSampleSizes;
 		}
 
-		sampleTable.presentationTimestamps.sort((a, b) => a.presentationTimestamp - b.presentationTimestamp);
+		if (sampleTable.sampleCompositionTimeOffsets.length > 0) {
+			// If composition time offsets are defined, we must build a list of all presentation timestamps and then
+			// sort them
+			sampleTable.presentationTimestamps = [];
+
+			for (const entry of sampleTable.sampleTimingEntries) {
+				for (let i = 0; i < entry.count; i++) {
+					sampleTable.presentationTimestamps.push({
+						presentationTimestamp: entry.startDecodeTimestamp + i * entry.delta,
+						sampleIndex: entry.startIndex + i,
+					});
+				}
+			}
+
+			for (const entry of sampleTable.sampleCompositionTimeOffsets) {
+				for (let i = 0; i < entry.count; i++) {
+					const sampleIndex = entry.startIndex + i;
+					const sample = sampleTable.presentationTimestamps[sampleIndex];
+					if (!sample) {
+						continue;
+					}
+
+					sample.presentationTimestamp += entry.offset;
+				}
+			}
+
+			sampleTable.presentationTimestamps.sort((a, b) => a.presentationTimestamp - b.presentationTimestamp);
+		} else {
+			// If they're not defined, we can simply use the decode timestamps as presentation timestamps
+		}
 
 		return internalTrack.sampleTable;
 	}
@@ -613,9 +701,20 @@ export class IsobmffDemuxer extends Demuxer {
 						this.readContiguousBoxes(startPos + sampleBoxInfo.totalSize - this.isobmffReader.pos);
 					} else {
 						if (sampleBoxInfo.name === 'mp4a') {
-							// We don't know the codec yet, need to read the esds box
+							// We don't know the codec yet (might be AAC, might be MP3), need to read the esds box
 						} else if (sampleBoxInfo.name.toLowerCase() === 'opus') {
 							track.info.codec = 'opus';
+						} else if (
+							sampleBoxInfo.name === 'twos'
+							|| sampleBoxInfo.name === 'sowt'
+							|| sampleBoxInfo.name === 'raw '
+							|| sampleBoxInfo.name === 'in24'
+							|| sampleBoxInfo.name === 'in32'
+							|| sampleBoxInfo.name === 'fl32'
+							|| sampleBoxInfo.name === 'lpcm'
+						) {
+							// It's PCM
+							// developer.apple.com/documentation/quicktime-file-format/sound_sample_descriptions/
 						} else {
 							const { name } = sampleBoxInfo;
 							console.warn(`Unsupported audio codec (sample entry type '${name}') - discarding track.`);
@@ -628,8 +727,9 @@ export class IsobmffDemuxer extends Demuxer {
 						this.isobmffReader.pos += 3 * 2;
 
 						let channelCount = this.isobmffReader.readU16();
+						let sampleSize = this.isobmffReader.readU16();
 
-						this.isobmffReader.pos += 2 + 2 + 2;
+						this.isobmffReader.pos += 2 * 2;
 
 						// Can't use fixed16_16 as that's signed
 						let sampleRate = this.isobmffReader.readU32() / 0x10000;
@@ -643,18 +743,13 @@ export class IsobmffDemuxer extends Demuxer {
 								sampleRate = this.isobmffReader.readF64();
 								channelCount = this.isobmffReader.readU32();
 								this.isobmffReader.pos += 4; // Always 0x7F000000
-								// eslint-disable-next-line @typescript-eslint/no-unused-vars
-								const sampleSize = this.isobmffReader.readU32();
 
-								// eslint-disable-next-line @typescript-eslint/no-unused-vars
+								sampleSize = this.isobmffReader.readU32();
+
 								const flags = this.isobmffReader.readU32();
 
-								// eslint-disable-next-line @typescript-eslint/no-unused-vars
+								this.isobmffReader.pos += 2 * 4;
-								const bytesPerFrame = this.isobmffReader.readU32();
-								// eslint-disable-next-line @typescript-eslint/no-unused-vars
-								const samplesPerFrame = this.isobmffReader.readU32();
 
-								/*
 								if (sampleBoxInfo.name === 'lpcm') {
 									const bytesPerSample = (sampleSize + 7) >> 3;
 									const isFloat = Boolean(flags & 1);
@@ -664,36 +759,64 @@ export class IsobmffDemuxer extends Demuxer {
 									if (sampleSize > 0 && sampleSize <= 64) {
 										if (isFloat) {
 											if (sampleSize === 32 && !isBigEndian) {
-												track.pcmType = 'pcm-f32';
+												track.info.codec = isBigEndian ? 'pcm-f32be' : 'pcm-f32le';
 											}
 										} else {
 											if (sFlags & (1 << (bytesPerSample - 1))) {
-												if (bytesPerSample === 2 && !isBigEndian) {
+												if (bytesPerSample === 1) {
-													track.pcmType = 'pcm-s16';
+													track.info.codec = 'pcm-s8';
-												} else if (bytesPerSample === 3 && !isBigEndian) {
+												} else if (bytesPerSample === 2) {
-													track.pcmType = 'pcm-s24';
+													track.info.codec = isBigEndian ? 'pcm-s16be' : 'pcm-s16le';
-												} else if (bytesPerSample === 4 && !isBigEndian) {
+												} else if (bytesPerSample === 3) {
-													track.pcmType = 'pcm-s32';
+													track.info.codec = isBigEndian ? 'pcm-s24be' : 'pcm-s24le';
+												} else if (bytesPerSample === 4) {
+													track.info.codec = isBigEndian ? 'pcm-s32be' : 'pcm-s32le';
 												}
 											} else {
 												if (bytesPerSample === 1) {
-													track.pcmType = 'pcm-u8';
+													track.info.codec = 'pcm-u8';
 												}
 											}
 										}
 									}
 
-									if (track.pcmType === null) {
+									if (track.info.codec === null) {
-										throw new Error(`Unsupported linear PCM type.`);
+										console.warn('Unsupportedd PCM format.');
+										break;
 									}
 								}
-								*/
 							}
 						}
 
 						track.info.numberOfChannels = channelCount;
 						track.info.sampleRate = sampleRate;
 
+						if (sampleBoxInfo.name === 'twos') {
+							if (sampleSize === 8) {
+								track.info.codec = 'pcm-s8';
+							} else if (sampleSize === 16) {
+								track.info.codec = 'pcm-s16be';
+							} else {
+								throw new Error(`Unsupported sample size ${sampleSize} for codec 'twos'.`);
+							}
+						} else if (sampleBoxInfo.name === 'sowt') {
+							if (sampleSize === 8) {
+								track.info.codec = 'pcm-s8';
+							} else if (sampleSize === 16) {
+								track.info.codec = 'pcm-s16le';
+							} else {
+								throw new Error(`Unsupported sample size ${sampleSize} for codec 'sowt'.`);
+							}
+						} else if (sampleBoxInfo.name === 'raw ') {
+							track.info.codec = 'pcm-u8';
+						} else if (sampleBoxInfo.name === 'in24') {
+							track.info.codec = 'pcm-s24be';
+						} else if (sampleBoxInfo.name === 'in32') {
+							track.info.codec = 'pcm-s32be';
+						} else if (sampleBoxInfo.name === 'fl32') {
+							track.info.codec = 'pcm-f32be';
+						}
+
 						this.readContiguousBoxes(startPos + sampleBoxInfo.totalSize - this.isobmffReader.pos);
 					}
 				}
@@ -873,6 +996,25 @@ export class IsobmffDemuxer extends Demuxer {
 				}
 			}; break;
 
+			case 'enda': {
+				const track = this.currentTrack;
+				assert(track && track.info?.type === 'audio');
+
+				const littleEndian = this.isobmffReader.readU16() & 0xff; // 0xff is from FFmpeg
+
+				if (littleEndian) {
+					if (track.info.codec === 'pcm-s16be') {
+						track.info.codec = 'pcm-s16le';
+					} else if (track.info.codec === 'pcm-s24be') {
+						track.info.codec = 'pcm-s24le';
+					} else if (track.info.codec === 'pcm-s32be') {
+						track.info.codec = 'pcm-s32le';
+					} else if (track.info.codec === 'pcm-f32be') {
+						track.info.codec = 'pcm-f32le';
+					}
+				}
+			}; break;
+
 			case 'stts': {
 				const track = this.currentTrack;
 				assert(track);
@@ -1966,16 +2108,30 @@ class IsobmffAudioTrackBacking extends IsobmffTrackBacking<EncodedAudioChunk> im
 }
 
 const getSampleIndexForTimestamp = (sampleTable: SampleTable, timescaleUnits: number) => {
-	const index = binarySearchLessOrEqual(
+	if (sampleTable.presentationTimestamps) {
-		sampleTable.presentationTimestamps,
+		const index = binarySearchLessOrEqual(
-		timescaleUnits,
+			sampleTable.presentationTimestamps,
-		x => x.presentationTimestamp,
+			timescaleUnits,
-	);
+			x => x.presentationTimestamp,
-	if (index === -1) {
+		);
-		return -1;
+		if (index === -1) {
-	}
+			return -1;
+		}
 
-	return sampleTable.presentationTimestamps[index]!.sampleIndex;
+		return sampleTable.presentationTimestamps[index]!.sampleIndex;
+	} else {
+		const index = binarySearchLessOrEqual(
+			sampleTable.sampleTimingEntries,
+			timescaleUnits,
+			x => x.startDecodeTimestamp,
+		);
+		if (index === -1) {
+			return -1;
+		}
+
+		const entry = sampleTable.sampleTimingEntries[index]!;
+		return entry.startIndex + Math.floor((timescaleUnits - entry.startDecodeTimestamp) / entry.delta);
+	}
 };
 
 type SampleInfo = {

src/media-drain.ts

@@ -1,5 +1,14 @@
+import { PCM_CODECS, PcmAudioCodec } from './codec';
 import { InputAudioTrack, InputVideoTrack } from './input-track';
-import { AnyIterable, assert, promiseWithResolvers, toAsyncIterator, validateAnyIterable } from './misc';
+import {
+	AnyIterable,
+	assert,
+	getInt24,
+	getUint24,
+	promiseWithResolvers,
+	toAsyncIterator,
+	validateAnyIterable,
+} from './misc';
 
 /** @public */
 export type ChunkRetrievalOptions = {
@@ -104,6 +113,21 @@ export abstract class BaseChunkDrain<Chunk extends EncodedVideoChunk | EncodedAu
 	}
 }
 
+abstract class DecoderWrapper<
+	Chunk extends EncodedVideoChunk | EncodedAudioChunk,
+	MediaFrame extends VideoFrame | AudioData,
+> {
+	constructor(
+		public onMedia: (media: MediaFrame) => unknown,
+		public onError: (error: DOMException) => unknown,
+	) {}
+
+	abstract getDecodeQueueSize(): number;
+	abstract decode(chunk: Chunk): void;
+	abstract flush(): Promise<void>;
+	abstract close(): void;
+}
+
 /** @public */
 export abstract class BaseMediaFrameDrain<
 	Chunk extends EncodedVideoChunk | EncodedAudioChunk,
@@ -113,7 +137,7 @@ export abstract class BaseMediaFrameDrain<
 	abstract _createDecoder(
 		onMedia: (media: MediaFrame) => unknown,
 		onError: (error: DOMException) => unknown
-	): Promise<VideoDecoder | AudioDecoder>;
+	): Promise<DecoderWrapper<Chunk, MediaFrame>>;
 	/** @internal */
 	abstract _createChunkDrain(): BaseChunkDrain<Chunk>;
 
@@ -185,7 +209,7 @@ export abstract class BaseMediaFrameDrain<
 			for await (const timestamp of timestampIterator) {
 				validateTimestamp(timestamp);
 
-				while (frameQueue.length + decoder.decodeQueueSize > MAX_QUEUE_SIZE) {
+				while (frameQueue.length + decoder.getDecodeQueueSize() > MAX_QUEUE_SIZE) {
 					({ promise: queueDequeue, resolve: onQueueDequeue } = promiseWithResolvers());
 					await queueDequeue;
 				}
@@ -236,10 +260,6 @@ export abstract class BaseMediaFrameDrain<
 					lastChunk = nextChunk;
 					decoder.decode(nextChunk);
 				}
-
-				if (decoder.decodeQueueSize >= 10) {
-					await new Promise(resolve => decoder.addEventListener('dequeue', resolve, { once: true }));
-				}
 			}
 
 			await decoder.flush();
@@ -376,7 +396,7 @@ export abstract class BaseMediaFrameDrain<
 			await chunks.next();
 
 			while (currentChunk && !ended) {
-				if (frameQueue.length + decoder.decodeQueueSize > MAX_QUEUE_SIZE) {
+				if (frameQueue.length + decoder.getDecodeQueueSize() > MAX_QUEUE_SIZE) {
 					({ promise: queueDequeue, resolve: onQueueDequeue } = promiseWithResolvers());
 					await queueDequeue;
 					continue;
@@ -483,6 +503,37 @@ export class EncodedVideoChunkDrain extends BaseChunkDrain<EncodedVideoChunk> {
 	}
 }
 
+class VideoDecoderWrapper extends DecoderWrapper<EncodedVideoChunk, VideoFrame> {
+	decoder: VideoDecoder;
+
+	constructor(
+		onFrame: (frame: VideoFrame) => unknown,
+		onError: (error: DOMException) => unknown,
+		decoderConfig: VideoDecoderConfig,
+	) {
+		super(onFrame, onError);
+
+		this.decoder = new VideoDecoder({ output: onFrame, error: onError });
+		this.decoder.configure(decoderConfig);
+	}
+
+	getDecodeQueueSize() {
+		return this.decoder.decodeQueueSize;
+	}
+
+	decode(chunk: EncodedVideoChunk) {
+		this.decoder.decode(chunk);
+	}
+
+	flush() {
+		return this.decoder.flush();
+	}
+
+	close() {
+		this.decoder.close();
+	}
+}
+
 /** @public */
 export class VideoFrameDrain extends BaseMediaFrameDrain<EncodedVideoChunk, VideoFrame> {
 	/** @internal */
@@ -503,11 +554,7 @@ export class VideoFrameDrain extends BaseMediaFrameDrain<EncodedVideoChunk, Vide
 	/** @internal */
 	async _createDecoder(onFrame: (frame: VideoFrame) => unknown, onError: (error: DOMException) => unknown) {
 		this._decoderConfig ??= await this._videoTrack.getDecoderConfig();
-
+		return new VideoDecoderWrapper(onFrame, onError, this._decoderConfig);
-		const decoder = new VideoDecoder({ output: onFrame, error: onError });
-		decoder.configure(this._decoderConfig);
-
-		return decoder;
 	}
 
 	/** @internal */
@@ -668,6 +715,185 @@ export class EncodedAudioChunkDrain extends BaseChunkDrain<EncodedAudioChunk> {
 	}
 }
 
+class AudioDecoderWrapper extends DecoderWrapper<EncodedAudioChunk, AudioData> {
+	decoder: AudioDecoder;
+
+	constructor(
+		onData: (data: AudioData) => unknown,
+		onError: (error: DOMException) => unknown,
+		decoderConfig: AudioDecoderConfig,
+	) {
+		super(onData, onError);
+
+		this.decoder = new AudioDecoder({ output: onData, error: onError });
+		this.decoder.configure(decoderConfig);
+	}
+
+	getDecodeQueueSize() {
+		return this.decoder.decodeQueueSize;
+	}
+
+	decode(chunk: EncodedAudioChunk) {
+		this.decoder.decode(chunk);
+	}
+
+	flush() {
+		return this.decoder.flush();
+	}
+
+	close() {
+		this.decoder.close();
+	}
+}
+
+const PCM_CODEC_REGEX = /^pcm-([usf])(\d+)+(be|le)?$/;
+
+// There are a lot of PCM variants not natively supported by the browser and by AudioData. Therefore we need a simple
+// decoder that maps any input PCM format into a PCM format supported by the browser.
+class PcmAudioDecoderWrapper extends DecoderWrapper<EncodedAudioChunk, AudioData> {
+	codec: PcmAudioCodec;
+
+	inputSampleSize: 1 | 2 | 3 | 4;
+	readInputValue: (view: DataView, byteOffset: number) => number;
+
+	outputSampleSize: 1 | 2 | 4;
+	outputFormat: 'u8' | 's16' | 's32' | 'f32';
+	writeOutputValue: (view: DataView, byteOffset: number, value: number) => void;
+
+	constructor(
+		onData: (data: AudioData) => unknown,
+		onError: (error: DOMException) => unknown,
+		public decoderConfig: AudioDecoderConfig,
+	) {
+		super(onData, onError);
+
+		assert((PCM_CODECS as readonly string[]).includes(decoderConfig.codec));
+		this.codec = decoderConfig.codec as PcmAudioCodec;
+
+		const match = this.codec.match(PCM_CODEC_REGEX);
+		assert(match);
+
+		let dataType: 'unsigned' | 'signed' | 'float';
+		if (match[1] === 'u') {
+			dataType = 'unsigned';
+		} else if (match[1] === 's') {
+			dataType = 'signed';
+		} else {
+			dataType = 'float';
+		}
+
+		this.inputSampleSize = (Number(match[2]) / 8) as 1 | 2 | 3 | 4;
+		const littleEndian = match[3] === 'le';
+
+		switch (this.inputSampleSize) {
+			case 1: {
+				if (dataType === 'unsigned') {
+					this.readInputValue = (view, byteOffset) => view.getUint8(byteOffset) - 2 ** 7;
+				} else {
+					this.readInputValue = (view, byteOffset) => view.getInt8(byteOffset);
+				}
+			}; break;
+			case 2: {
+				if (dataType === 'unsigned') {
+					this.readInputValue = (view, byteOffset) => view.getUint16(byteOffset, littleEndian) - 2 ** 15;
+				} else {
+					this.readInputValue = (view, byteOffset) => view.getInt16(byteOffset, littleEndian);
+				}
+			}; break;
+			case 3: {
+				if (dataType === 'unsigned') {
+					this.readInputValue = (view, byteOffset) => getUint24(view, byteOffset, littleEndian) - 2 ** 23;
+				} else {
+					this.readInputValue = (view, byteOffset) => getInt24(view, byteOffset, littleEndian);
+				}
+			}; break;
+			case 4: {
+				if (dataType === 'unsigned') {
+					this.readInputValue = (view, byteOffset) => view.getUint32(byteOffset, littleEndian) - 2 ** 31;
+				} else if (dataType === 'signed') {
+					this.readInputValue = (view, byteOffset) => view.getInt32(byteOffset, littleEndian);
+				} else {
+					this.readInputValue = (view, byteOffset) => view.getFloat32(byteOffset, littleEndian);
+				}
+			}; break;
+		}
+
+		switch (this.inputSampleSize) {
+			case 1: {
+				this.outputSampleSize = 1;
+				this.outputFormat = 'u8';
+				this.writeOutputValue = (view, byteOffset, value) => view.setUint8(byteOffset, value + 2 ** 7);
+			}; break;
+			case 2: {
+				this.outputSampleSize = 2;
+				this.outputFormat = 's16';
+				this.writeOutputValue = (view, byteOffset, value) => view.setInt16(byteOffset, value, true);
+			}; break;
+			case 3: {
+				this.outputSampleSize = 4;
+				this.outputFormat = 's32';
+				// From https://www.w3.org/TR/webcodecs:
+				// AudioData containing 24-bit samples SHOULD store those samples in s32 or f32. When samples are
+				// stored in s32, each sample MUST be left-shifted by 8 bits.
+				this.writeOutputValue = (view, byteOffset, value) => view.setInt32(byteOffset, value << 8, true);
+			}; break;
+			case 4: {
+				this.outputSampleSize = 4;
+
+				if (dataType === 'float') {
+					this.outputFormat = 'f32';
+					this.writeOutputValue = (view, byteOffset, value) => view.setFloat32(byteOffset, value, true);
+				} else {
+					this.outputFormat = 's32';
+					this.writeOutputValue = (view, byteOffset, value) => view.setInt32(byteOffset, value, true);
+				}
+			}; break;
+		};
+	}
+
+	getDecodeQueueSize() {
+		return 0;
+	}
+
+	decode(chunk: EncodedAudioChunk) {
+		const inputBuffer = new ArrayBuffer(chunk.byteLength);
+		const inputView = new DataView(inputBuffer);
+		chunk.copyTo(inputBuffer);
+
+		const numberOfFrames = chunk.byteLength / this.decoderConfig.numberOfChannels / this.inputSampleSize;
+
+		const outputBufferSize = numberOfFrames * this.decoderConfig.numberOfChannels * this.outputSampleSize;
+		const outputBuffer = new ArrayBuffer(outputBufferSize);
+		const outputView = new DataView(outputBuffer);
+
+		for (let i = 0; i < numberOfFrames * this.decoderConfig.numberOfChannels; i++) {
+			const inputIndex = i * this.inputSampleSize;
+			const outputIndex = i * this.outputSampleSize;
+
+			const value = this.readInputValue(inputView, inputIndex);
+			this.writeOutputValue(outputView, outputIndex, value);
+		}
+
+		const audioData = new AudioData({
+			format: this.outputFormat,
+			data: outputBuffer,
+			numberOfChannels: this.decoderConfig.numberOfChannels,
+			sampleRate: this.decoderConfig.sampleRate,
+			numberOfFrames,
+			timestamp: chunk.timestamp,
+		});
+		this.onMedia(audioData);
+	}
+
+	async flush() {
+		// Do nothing
+	}
+
+	close() {
+		// Do nothing
+	}
+}
+
 /** @public */
 export class AudioDataDrain extends BaseMediaFrameDrain<EncodedAudioChunk, AudioData> {
 	/** @internal */
@@ -689,10 +915,11 @@ export class AudioDataDrain extends BaseMediaFrameDrain<EncodedAudioChunk, Audio
 	async _createDecoder(onData: (data: AudioData) => unknown, onError: (error: DOMException) => unknown) {
 		this._decoderConfig ??= await this._audioTrack.getDecoderConfig();
 
-		const decoder = new AudioDecoder({ output: onData, error: onError });
+		if (this._decoderConfig.codec.startsWith('pcm-')) {
-		decoder.configure(this._decoderConfig);
+			return new PcmAudioDecoderWrapper(onData, onError, this._decoderConfig);
-
+		} else {
-		return decoder;
+			return new AudioDecoderWrapper(onData, onError, this._decoderConfig);
+		}
 	}
 
 	/** @internal */

src/misc.ts

@@ -245,3 +245,26 @@ export const validateAnyIterable = (iterable: AnyIterable<unknown>) => {
 		throw new TypeError('Argument must be an iterable or async iterable.');
 	}
 };
+
+export const assertNever = (x: never) => {
+	// eslint-disable-next-line @typescript-eslint/restrict-template-expressions
+	throw new Error(`Unexpected value: ${x}`);
+};
+
+export const getUint24 = (view: DataView, byteOffset: number, littleEndian: boolean) => {
+	const byte1 = view.getUint8(byteOffset);
+	const byte2 = view.getUint8(byteOffset + 1);
+	const byte3 = view.getUint8(byteOffset + 2);
+
+	if (littleEndian) {
+		return byte1 | (byte2 << 8) | (byte3 << 16);
+	} else {
+		return (byte1 << 16) | (byte2 << 8) | byte3;
+	}
+};
+
+export const getInt24 = (view: DataView, byteOffset: number, littleEndian: boolean) => {
+	// The left shift pushes the most significant bit into the sign bit region, and the subsequent right shift
+	// then correctly interprets the sign bit.
+	return getUint24(view, byteOffset, littleEndian) << 8 >> 8;
+};