diff --git a/dev/convert.html b/dev/convert.html index fbff6d4..4af9517 100644 --- a/dev/convert.html +++ b/dev/convert.html @@ -43,8 +43,12 @@ codec: 'aac', }, */ + video: { + codec: 'avc', + }, audio: { - discard: true + codec: 'mp3', + bitrate: 320000 }, trim: { start: 0, diff --git a/dev/demux.html b/dev/demux.html index f0fac30..43b58ba 100644 --- a/dev/demux.html +++ b/dev/demux.html @@ -22,15 +22,18 @@ }); const audioTrack = await input.getPrimaryAudioTrack(); - const sink = new Metamuxer.EncodedAudioSampleSink(audioTrack); + const sink = new Metamuxer.AudioDataSink(audioTrack); - const mediaSource = new Metamuxer.EncodedAudioSampleSource(await audioTrack.getCodec()); + const mediaSource = new Metamuxer.AudioDataSource({ + codec: 'mp3', + bitrate: 192000 + }); output.addAudioTrack(mediaSource); output.start(); - for await (const sample of sink.samples()) { - await mediaSource.digest(sample); + for await (const { data } of sink.data()) { + await mediaSource.digest(data); } await output.finalize(); diff --git a/src/codec.ts b/src/codec.ts index 62fa5c0..5d58e51 100644 --- a/src/codec.ts +++ b/src/codec.ts @@ -1,3 +1,4 @@ +import { customAudioEncoders, customVideoEncoders } from './custom-coder'; import { COLOR_PRIMARIES_MAP, MATRIX_COEFFICIENTS_MAP, @@ -1098,7 +1099,13 @@ export class Quality { } else if (codec === 'opus' || codec === 'vorbis') { finalBitrate = Math.max(6000, finalBitrate); } else if (codec === 'mp3') { - finalBitrate = Math.round(finalBitrate / 32000) * 32000; + const validRates = [ + 8000, 16000, 24000, 32000, 40000, 48000, 64000, 80000, + 96000, 112000, 128000, 160000, 192000, 224000, 256000, 320000, + ]; + finalBitrate = validRates.reduce((prev, curr) => + Math.abs(curr - finalBitrate) < Math.abs(prev - finalBitrate) ? curr : prev, + ); } return Math.round(finalBitrate / 1000) * 1000; @@ -1441,6 +1448,26 @@ export const canEncodeVideo = async (codec: VideoCodec, { width = 1280, height = throw new TypeError('bitrate must be a positive integer.'); } + if (customVideoEncoders.length > 0) { + const encoderConfig: VideoEncoderConfig = { + codec: buildVideoCodecString( + codec, + width, + height, + bitrate, + ), + width, + height, + bitrate, + ...getVideoEncoderConfigExtension(codec), + }; + + if (customVideoEncoders.some(x => x.supports(codec, encoderConfig))) { + // There's a custom encoder + return true; + } + } + if (typeof VideoEncoder === 'undefined') { return false; } @@ -1475,6 +1502,25 @@ export const canEncodeAudio = async (codec: AudioCodec, { numberOfChannels = 2, throw new TypeError('bitrate must be a positive integer.'); } + if (customAudioEncoders.length > 0) { + const encoderConfig: AudioEncoderConfig = { + codec: buildAudioCodecString( + codec, + numberOfChannels, + sampleRate, + ), + numberOfChannels, + sampleRate, + bitrate, + ...getAudioEncoderConfigExtension(codec), + }; + + if (customAudioEncoders.some(x => x.supports(codec, encoderConfig))) { + // There's a custom encoder + return true; + } + } + if ((PCM_AUDIO_CODECS as readonly string[]).includes(codec)) { return true; // Because we encode these ourselves } diff --git a/src/custom-coder.ts b/src/custom-coder.ts new file mode 100644 index 0000000..e528387 --- /dev/null +++ b/src/custom-coder.ts @@ -0,0 +1,101 @@ +import { AudioCodec, VideoCodec } from './codec'; +import { EncodedAudioSample, EncodedVideoSample } from './sample'; + +/** @public */ +export class CustomVideoDecoder { + constructor( + public codec: VideoCodec, + public config: VideoDecoderConfig, + public onFrame: (frame: VideoFrame) => unknown, + ) {} + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + static supports(codec: VideoCodec, config: VideoDecoderConfig): boolean { + return false; + } + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + decode(sample: EncodedVideoSample): Promise | void {} + flush(): Promise | void {} +} + +/** @public */ +export class CustomAudioDecoder { + constructor( + public codec: AudioCodec, + public config: AudioDecoderConfig, + public onData: (data: AudioData) => unknown, + ) {} + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + static supports(codec: AudioCodec, config: AudioDecoderConfig): boolean { + return false; + } + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + decode(sample: EncodedAudioSample): Promise | void {} + flush(): Promise | void {} +} + +/** @public */ +export class CustomVideoEncoder { + constructor( + public codec: VideoCodec, + public config: VideoEncoderConfig, + public onSample: (sample: EncodedVideoSample, meta?: EncodedVideoChunkMetadata) => unknown, + ) {} + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + static supports(codec: VideoCodec, config: VideoEncoderConfig): boolean { + return false; + } + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + encode(videoFrame: VideoFrame, options: VideoEncoderEncodeOptions): Promise | void {} + flush(): Promise | void {} +} + +/** @public */ +export class CustomAudioEncoder { + constructor( + public codec: AudioCodec, + public config: AudioEncoderConfig, + public onSample: (sample: EncodedAudioSample, meta?: EncodedAudioChunkMetadata) => unknown, + ) {} + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + static supports(codec: AudioCodec, config: AudioEncoderConfig): boolean { + return false; + } + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + encode(audioData: AudioData): Promise | void {} + flush(): Promise | void {} +} + +export const customVideoDecoders: typeof CustomVideoDecoder[] = []; +export const customAudioDecoders: typeof CustomAudioDecoder[] = []; +export const customVideoEncoders: typeof CustomVideoEncoder[] = []; +export const customAudioEncoders: typeof CustomAudioEncoder[] = []; + +/** @public */ +export const registerDecoder = (decoder: typeof CustomVideoDecoder | typeof CustomAudioDecoder) => { + if (decoder.prototype instanceof CustomVideoDecoder) { + customVideoDecoders.push(decoder as typeof CustomVideoDecoder); + } else if (decoder.prototype instanceof CustomAudioDecoder) { + customAudioDecoders.push(decoder as typeof CustomAudioDecoder); + } else { + throw new TypeError('Decoder must be a CustomVideoDecoder or CustomAudioDecoder.'); + } +}; + +/** @public */ +export const registerEncoder = (encoder: typeof CustomVideoEncoder | typeof CustomAudioEncoder) => { + if (encoder.prototype instanceof CustomVideoEncoder) { + customVideoEncoders.push(encoder as typeof CustomVideoEncoder); + } else if (encoder.prototype instanceof CustomAudioEncoder) { + customAudioEncoders.push(encoder as typeof CustomAudioEncoder); + } else { + throw new TypeError('Encoder must be a CustomVideoEncoder or CustomAudioEncoder.'); + } +}; diff --git a/src/index.ts b/src/index.ts index f7a45cb..28caf50 100644 --- a/src/index.ts +++ b/src/index.ts @@ -108,5 +108,13 @@ export { WrappedAudioBuffer, } from './media-sink'; export { convert, ConversionOptions, ConversionInfo } from './conversion'; +export { + CustomVideoDecoder, + CustomAudioDecoder, + CustomVideoEncoder, + CustomAudioEncoder, + registerDecoder, + registerEncoder, +} from './custom-coder'; // 🐡🦔 diff --git a/src/input-track.ts b/src/input-track.ts index fb228ce..9d512a6 100644 --- a/src/input-track.ts +++ b/src/input-track.ts @@ -1,6 +1,7 @@ import { AudioCodec, MediaCodec, VideoCodec } from './codec'; +import { customAudioDecoders, customVideoDecoders } from './custom-coder'; import { EncodedAudioSampleSink, EncodedVideoSampleSink, SampleRetrievalOptions } from './media-sink'; -import { Rotation } from './misc'; +import { assert, Rotation } from './misc'; import { TrackType } from './output'; import { EncodedAudioSample, EncodedVideoSample } from './sample'; @@ -137,6 +138,17 @@ export class InputVideoTrack extends InputTrack { return false; } + const codec = await this.getCodec(); + assert(codec !== null); + + if (customVideoDecoders.some(x => x.supports(codec, decoderConfig))) { + return true; + } + + if (typeof VideoDecoder === 'undefined') { + return false; + } + const support = await VideoDecoder.isConfigSupported(decoderConfig); return support.supported === true; } catch (error) { @@ -206,9 +218,20 @@ export class InputAudioTrack extends InputTrack { return false; } + const codec = await this.getCodec(); + assert(codec !== null); + + if (customAudioDecoders.some(x => x.supports(codec, decoderConfig))) { + return true; + } + if (decoderConfig.codec.startsWith('pcm-')) { return true; // Since we decode it ourselves } else { + if (typeof AudioDecoder === 'undefined') { + return false; + } + const support = await AudioDecoder.isConfigSupported(decoderConfig); return support.supported === true; } diff --git a/src/media-sink.ts b/src/media-sink.ts index 3572c8d..3951137 100644 --- a/src/media-sink.ts +++ b/src/media-sink.ts @@ -1,4 +1,5 @@ -import { parsePcmCodec, PCM_AUDIO_CODECS, PcmAudioCodec } from './codec'; +import { parsePcmCodec, PCM_AUDIO_CODECS, PcmAudioCodec, VideoCodec, AudioCodec } from './codec'; +import { CustomVideoDecoder, customVideoDecoders, CustomAudioDecoder, customAudioDecoders } from './custom-coder'; import { InputAudioTrack, InputVideoTrack } from './input-track'; import { AnyIterable, @@ -579,40 +580,57 @@ export class EncodedVideoSampleSink extends BaseSampleSink { } class VideoDecoderWrapper extends DecoderWrapper { - decoder: VideoDecoder; + decoder: VideoDecoder | null = null; pendingSamples: EncodedVideoSample[] = []; + customDecoder: CustomVideoDecoder | null = null; + lastCustomDecoderPromise = Promise.resolve(); + customDecoderQueueSize = 0; + constructor( onFrame: (frame: WrappedMediaFrame) => unknown, onError: (error: DOMException) => unknown, + codec: VideoCodec, decoderConfig: VideoDecoderConfig, ) { super(onFrame, onError); - this.decoder = new VideoDecoder({ - output: (frame) => { - const sample = this.pendingSamples.shift(); - assert(sample); + const frameHandler = (frame: VideoFrame) => { + const sample = this.pendingSamples.shift(); + assert(sample); - // Let's get these from the sample instead of the frame, as the frame has no innate timing info - // (unlike AudioData), so the sample will always be more accurate. - const timestamp = sample.timestamp; - const duration = sample.duration; + // Let's get these from the sample instead of the frame, as the frame has no innate timing info + // (unlike AudioData), so the sample will always be more accurate. + const timestamp = sample.timestamp; + const duration = sample.duration; - onFrame({ - frame, - sample, - timestamp, - duration, - }); - }, - error: onError, - }); - this.decoder.configure(decoderConfig); + onFrame({ + frame, + sample, + timestamp, + duration, + }); + }; + + const MatchingCustomDecoder = customVideoDecoders.find(x => x.supports(codec, decoderConfig)); + if (MatchingCustomDecoder) { + this.customDecoder = new MatchingCustomDecoder(codec, decoderConfig, frameHandler); + } else { + this.decoder = new VideoDecoder({ + output: frameHandler, + error: onError, + }); + this.decoder.configure(decoderConfig); + } } getDecodeQueueSize() { - return this.decoder.decodeQueueSize; + if (this.customDecoder) { + return this.customDecoderQueueSize; + } else { + assert(this.decoder); + return this.decoder.decodeQueueSize; + } } decode(sample: EncodedVideoSample) { @@ -620,15 +638,35 @@ class VideoDecoderWrapper extends DecoderWrapper const insertionIndex = binarySearchLessOrEqual(this.pendingSamples, sample.timestamp, x => x.timestamp); this.pendingSamples.splice(insertionIndex + 1, 0, sample); - this.decoder.decode(sample.toEncodedVideoChunk()); + if (this.customDecoder) { + this.customDecoderQueueSize++; + this.lastCustomDecoderPromise = this.lastCustomDecoderPromise.then(() => { + return this.customDecoder!.decode(sample); + }); + + void this.lastCustomDecoderPromise.then(() => this.customDecoderQueueSize--); + } else { + assert(this.decoder); + this.decoder.decode(sample.toEncodedVideoChunk()); + } } flush() { - return this.decoder.flush(); + if (this.customDecoder) { + return this.lastCustomDecoderPromise.then(() => this.customDecoder!.flush()); + } else { + assert(this.decoder); + return this.decoder.flush(); + } } close() { - this.decoder.close(); + if (this.customDecoder) { + void this.lastCustomDecoderPromise.then(() => this.customDecoder!.flush()); + } else { + assert(this.decoder); + this.decoder.close(); + } } } @@ -666,10 +704,11 @@ export class VideoFrameSink extends BaseMediaFrameSink { } class AudioDecoderWrapper extends DecoderWrapper { - decoder: AudioDecoder; + decoder: AudioDecoder | null = null; pendingSamples: EncodedAudioSample[] = []; + customDecoder: CustomAudioDecoder | null = null; + lastCustomDecoderPromise = Promise.resolve(); + customDecoderQueueSize = 0; + constructor( onData: (data: WrappedMediaFrame) => unknown, onError: (error: DOMException) => unknown, + codec: AudioCodec, decoderConfig: AudioDecoderConfig, ) { super(onData, onError); - this.decoder = new AudioDecoder({ output: (data) => { + const dataHandler = (data: AudioData) => { const sample = this.pendingSamples.shift(); assert(sample); - // We use the timing information from the data instead of sample as it will be more accurate. However, - // we also know these need to be multiple of the sample length, so let's round: + // We use the timing information from the data instead of sample as it will be more accurate const timestamp = Math.round(data.timestamp / 1e6 * decoderConfig.sampleRate) / decoderConfig.sampleRate; const duration = Math.round(data.duration / 1e6 * decoderConfig.sampleRate) / decoderConfig.sampleRate; @@ -873,12 +916,27 @@ class AudioDecoderWrapper extends DecoderWrapper timestamp, duration, }); - }, error: onError }); - this.decoder.configure(decoderConfig); + }; + + const MatchingCustomDecoder = customAudioDecoders.find(x => x.supports(codec, decoderConfig)); + if (MatchingCustomDecoder) { + this.customDecoder = new MatchingCustomDecoder(codec, decoderConfig, dataHandler); + } else { + this.decoder = new AudioDecoder({ + output: dataHandler, + error: onError, + }); + this.decoder.configure(decoderConfig); + } } getDecodeQueueSize() { - return this.decoder.decodeQueueSize; + if (this.customDecoder) { + return this.customDecoderQueueSize; + } else { + assert(this.decoder); + return this.decoder.decodeQueueSize; + } } decode(sample: EncodedAudioSample) { @@ -886,15 +944,35 @@ class AudioDecoderWrapper extends DecoderWrapper const insertionIndex = binarySearchLessOrEqual(this.pendingSamples, sample.timestamp, x => x.timestamp); this.pendingSamples.splice(insertionIndex + 1, 0, sample); - this.decoder.decode(sample.toEncodedAudioChunk()); + if (this.customDecoder) { + this.customDecoderQueueSize++; + this.lastCustomDecoderPromise = this.lastCustomDecoderPromise.then(() => { + return this.customDecoder!.decode(sample); + }); + + void this.lastCustomDecoderPromise.then(() => this.customDecoderQueueSize--); + } else { + assert(this.decoder); + this.decoder.decode(sample.toEncodedAudioChunk()); + } } flush() { - return this.decoder.flush(); + if (this.customDecoder) { + return this.lastCustomDecoderPromise.then(() => this.customDecoder!.flush()); + } else { + assert(this.decoder); + return this.decoder.flush(); + } } close() { - this.decoder.close(); + if (this.customDecoder) { + void this.lastCustomDecoderPromise.then(() => this.customDecoder!.flush()); + } else { + assert(this.decoder); + this.decoder.close(); + } } } @@ -1102,13 +1180,14 @@ export class AudioDataSink extends BaseMediaFrameSink { class VideoEncoderWrapper { private ensureEncoderPromise: Promise | null = null; + private encoderInitialized = false; private encoder: VideoEncoder | null = null; private muxer: Muxer | null = null; private lastMultipleOfKeyFrameInterval = -1; private lastWidth: number | null = null; private lastHeight: number | null = null; - constructor(private source: VideoSource, private encodingConfig: VideoEncodingConfig) { - validateVideoEncodingConfig(encodingConfig); - } + private customEncoder: CustomVideoEncoder | null = null; + private lastCustomEncoderPromise = Promise.resolve(); + private customEncoderQueueSize = 0; + + constructor(private source: VideoSource, private encodingConfig: VideoEncodingConfig) {} async digest(videoFrame: VideoFrame, shouldClose: boolean, encodeOptions?: VideoEncoderEncodeOptions) { this.source._ensureValidDigest(); @@ -201,14 +210,14 @@ class VideoEncoderWrapper { this.lastHeight = videoFrame.codedHeight; } - if (!this.encoder) { + if (!this.encoderInitialized) { if (this.ensureEncoderPromise) { await this.ensureEncoderPromise; } else { await this.ensureEncoder(videoFrame); } } - assert(this.encoder); + assert(this.encoderInitialized); const keyFrameInterval = this.encodingConfig.keyFrameInterval ?? 5; const multipleOfKeyFrameInterval = Math.floor((videoFrame.timestamp / 1e6) / keyFrameInterval); @@ -216,22 +225,43 @@ class VideoEncoderWrapper { // Ensure a key frame every KEY_FRAME_INTERVAL seconds. It is important that all video tracks follow the same // "key frame" rhythm, because aligned key frames are required to start new fragments in ISOBMFF or clusters // in Matroska. - this.encoder.encode(videoFrame, { + const finalEncodeOptions = { ...encodeOptions, keyFrame: encodeOptions?.keyFrame || keyFrameInterval === 0 || multipleOfKeyFrameInterval !== this.lastMultipleOfKeyFrameInterval, - }); - - if (shouldClose) { - videoFrame.close(); - } - + }; this.lastMultipleOfKeyFrameInterval = multipleOfKeyFrameInterval; - // We need to do this after sending the frame to the encoder as the frame otherwise might be closed - if (this.encoder.encodeQueueSize >= 4) { - await new Promise(resolve => this.encoder!.addEventListener('dequeue', resolve, { once: true })); + if (this.customEncoder) { + this.customEncoderQueueSize++; + this.lastCustomEncoderPromise = this.lastCustomEncoderPromise.then(() => { + return this.customEncoder!.encode(videoFrame, finalEncodeOptions); + }); + + void this.lastCustomEncoderPromise.then(() => { + this.customEncoderQueueSize--; + + if (shouldClose) { + videoFrame.close(); + } + }); + + if (this.customEncoderQueueSize >= 4) { + await this.lastCustomEncoderPromise; + } + } else { + assert(this.encoder); + this.encoder.encode(videoFrame, finalEncodeOptions); + + if (shouldClose) { + videoFrame.close(); + } + + // We need to do this after sending the frame to the encoder as the frame otherwise might be closed + if (this.encoder.encodeQueueSize >= 4) { + await new Promise(resolve => this.encoder!.addEventListener('dequeue', resolve, { once: true })); + } } await this.muxer!.mutex.currentPromise; // Allow the writer to apply backpressure @@ -242,10 +272,6 @@ class VideoEncoderWrapper { return; } - if (typeof VideoEncoder === 'undefined') { - throw new Error('VideoEncoder is not supported by this browser.'); - } - const { promise, resolve } = promiseWithResolvers(); this.ensureEncoderPromise = promise; @@ -269,33 +295,58 @@ class VideoEncoderWrapper { latencyMode: this.encodingConfig.latencyMode, ...getVideoEncoderConfigExtension(this.encodingConfig.codec), }; - const support = await VideoEncoder.isConfigSupported(encoderConfig); - if (!support.supported) { - throw new Error( - 'This specific encoder configuration is not supported by this browser. Consider using another codec or' - + ' changing your video parameters.', + + const MatchingCustomEncoder = customVideoEncoders.find(x => x.supports( + this.encodingConfig.codec, + encoderConfig, + )); + + if (MatchingCustomEncoder) { + this.customEncoder = new MatchingCustomEncoder( + this.encodingConfig.codec, + encoderConfig, + (sample, meta) => { + this.encodingConfig.onEncodedSample?.(sample, meta); + void this.muxer!.addEncodedVideoSample(this.source._connectedTrack!, sample, meta); + }, ); + } else { + if (typeof VideoEncoder === 'undefined') { + throw new Error('VideoEncoder is not supported by this browser.'); + } + + const support = await VideoEncoder.isConfigSupported(encoderConfig); + if (!support.supported) { + throw new Error( + 'This specific encoder configuration is not supported by this browser. Consider using another codec' + + ' or changing your video parameters.', + ); + } + + this.encoder = new VideoEncoder({ + output: (chunk, meta) => { + const sample = EncodedVideoSample.fromEncodedVideoChunk(chunk); + + this.encodingConfig.onEncodedSample?.(sample, meta); + void this.muxer!.addEncodedVideoSample(this.source._connectedTrack!, sample, meta); + }, + error: this.encodingConfig.onEncodingError ?? (error => console.error('VideoEncoder error:', error)), + }); + this.encoder.configure(encoderConfig); } - this.encoder = new VideoEncoder({ - output: (chunk, meta) => { - const sample = EncodedVideoSample.fromEncodedVideoChunk(chunk); - - this.encodingConfig.onEncodedSample?.(sample, meta); - void this.muxer!.addEncodedVideoSample(this.source._connectedTrack!, sample, meta); - }, - error: this.encodingConfig.onEncodingError ?? (error => console.error('VideoEncoder error:', error)), - }); - this.encoder.configure(encoderConfig); - assert(this.source._connectedTrack); this.muxer = this.source._connectedTrack.output._muxer; + this.encoderInitialized = true; + resolve(); } async flush() { - if (this.encoder) { + if (this.customEncoder) { + await this.lastCustomEncoderPromise.then(() => this.customEncoder!.flush()); + } else if (this.encoder) { await this.encoder.flush(); this.encoder.close(); } @@ -308,6 +359,8 @@ export class VideoFrameSource extends VideoSource { private _encoder: VideoEncoderWrapper; constructor(encodingConfig: VideoEncodingConfig) { + validateVideoEncodingConfig(encodingConfig); + super(encodingConfig.codec); this._encoder = new VideoEncoderWrapper(this, encodingConfig); } @@ -337,6 +390,7 @@ export class CanvasSource extends VideoSource { if (!(canvas instanceof HTMLCanvasElement)) { throw new TypeError('canvas must be an HTMLCanvasElement.'); } + validateVideoEncodingConfig(encodingConfig); super(encodingConfig.codec); this._encoder = new VideoEncoderWrapper(this, encodingConfig); @@ -382,6 +436,7 @@ export class MediaStreamVideoTrackSource extends VideoSource { if (!(track instanceof MediaStreamTrack) || track.kind !== 'video') { throw new TypeError('track must be a video MediaStreamTrack.'); } + validateVideoEncodingConfig(encodingConfig); encodingConfig = { ...encodingConfig, @@ -507,9 +562,11 @@ class AudioEncoderWrapper { private outputSampleSize: number | null = null; private writeOutputValue: ((view: DataView, byteOffset: number, value: number) => void) | null = null; - constructor(private source: AudioSource, private encodingConfig: AudioEncodingConfig) { - validateAudioEncodingConfig(encodingConfig); - } + private customEncoder: CustomAudioEncoder | null = null; + private lastCustomEncoderPromise = Promise.resolve(); + private customEncoderQueueSize = 0; + + constructor(private source: AudioSource, private encodingConfig: AudioEncodingConfig) {} async digest(audioData: AudioData, shouldClose: boolean) { this.source._ensureValidDigest(); @@ -540,7 +597,26 @@ class AudioEncoderWrapper { } assert(this.encoderInitialized); - if (this.isPcmEncoder) { + if (this.customEncoder) { + this.customEncoderQueueSize++; + this.lastCustomEncoderPromise = this.lastCustomEncoderPromise.then(() => { + return this.customEncoder!.encode(audioData); + }); + + void this.lastCustomEncoderPromise.then(() => { + this.customEncoderQueueSize--; + + if (shouldClose) { + audioData.close(); + } + }); + + if (this.customEncoderQueueSize >= 4) { + await this.lastCustomEncoderPromise; + } + + await this.muxer!.mutex.currentPromise; // Allow the writer to apply backpressure + } else if (this.isPcmEncoder) { await this.doPcmEncoding(audioData, shouldClose); } else { assert(this.encoder); @@ -638,29 +714,44 @@ class AudioEncoderWrapper { const { promise, resolve } = promiseWithResolvers(); this.ensureEncoderPromise = promise; - if ((PCM_AUDIO_CODECS as readonly string[]).includes(this.encodingConfig.codec)) { + const { numberOfChannels, sampleRate } = audioData; + const bitrate = this.encodingConfig.bitrate instanceof Quality + ? this.encodingConfig.bitrate._toAudioBitrate(this.encodingConfig.codec) + : this.encodingConfig.bitrate; + + const encoderConfig: AudioEncoderConfig = { + codec: buildAudioCodecString( + this.encodingConfig.codec, + numberOfChannels, + sampleRate, + ), + numberOfChannels, + sampleRate, + bitrate, + ...getAudioEncoderConfigExtension(this.encodingConfig.codec), + }; + + const MatchingCustomEncoder = customAudioEncoders.find(x => x.supports( + this.encodingConfig.codec, + encoderConfig, + )); + + if (MatchingCustomEncoder) { + this.customEncoder = new MatchingCustomEncoder( + this.encodingConfig.codec, + encoderConfig, + (sample, meta) => { + this.encodingConfig.onEncodedSample?.(sample, meta); + void this.muxer!.addEncodedAudioSample(this.source._connectedTrack!, sample, meta); + }, + ); + } else if ((PCM_AUDIO_CODECS as readonly string[]).includes(this.encodingConfig.codec)) { this.initPcmEncoder(); } else { if (typeof AudioEncoder === 'undefined') { throw new Error('AudioEncoder is not supported by this browser.'); } - const { numberOfChannels, sampleRate } = audioData; - const bitrate = this.encodingConfig.bitrate instanceof Quality - ? this.encodingConfig.bitrate._toAudioBitrate(this.encodingConfig.codec) - : this.encodingConfig.bitrate; - - const encoderConfig: AudioEncoderConfig = { - codec: buildAudioCodecString( - this.encodingConfig.codec, - numberOfChannels, - sampleRate, - ), - numberOfChannels, - sampleRate, - bitrate, - ...getAudioEncoderConfigExtension(this.encodingConfig.codec), - }; const support = await AudioEncoder.isConfigSupported(encoderConfig); if (!support.supported) { throw new Error( @@ -770,7 +861,9 @@ class AudioEncoderWrapper { } async flush() { - if (this.encoder) { + if (this.customEncoder) { + await this.lastCustomEncoderPromise.then(() => this.customEncoder!.flush()); + } else if (this.encoder) { await this.encoder.flush(); this.encoder.close(); } @@ -783,6 +876,8 @@ export class AudioDataSource extends AudioSource { private _encoder: AudioEncoderWrapper; constructor(encodingConfig: AudioEncodingConfig) { + validateAudioEncodingConfig(encodingConfig); + super(encodingConfig.codec); this._encoder = new AudioEncoderWrapper(this, encodingConfig); } @@ -809,6 +904,8 @@ export class AudioBufferSource extends AudioSource { private _accumulatedFrameCount = 0; constructor(encodingConfig: AudioEncodingConfig) { + validateAudioEncodingConfig(encodingConfig); + super(encodingConfig.codec); this._encoder = new AudioEncoderWrapper(this, encodingConfig); } @@ -884,6 +981,7 @@ export class MediaStreamAudioTrackSource extends AudioSource { if (!(track instanceof MediaStreamTrack) || track.kind !== 'audio') { throw new TypeError('track must be an audio MediaStreamTrack.'); } + validateAudioEncodingConfig(encodingConfig); super(encodingConfig.codec); this._encoder = new AudioEncoderWrapper(this, encodingConfig);