diff --git a/examples/procedural-generation/procedural-generation.ts b/examples/procedural-generation/procedural-generation.ts index 97a6181..93efdb2 100644 --- a/examples/procedural-generation/procedural-generation.ts +++ b/examples/procedural-generation/procedural-generation.ts @@ -147,7 +147,7 @@ const generateVideo = async () => { bitrate: QUALITY_HIGH, keyFrameInterval: 2, transform: { - frameRate: 5, + // frameRate: 5, }, }); output.addVideoTrack(canvasSource, { frameRate }); @@ -167,8 +167,11 @@ const generateVideo = async () => { audioBufferSource = new AudioBufferSource({ codec: audioCodec, bitrate: QUALITY_HIGH, + transform: { + + }, }); - // output.addAudioTrack(audioBufferSource); + output.addAudioTrack(audioBufferSource); /* audioBufferSource2 = new AudioBufferSource({ @@ -219,8 +222,8 @@ const generateVideo = async () => { // Let's render the audio. Ideally, the audio is rendered before the video (or concurrently to it), but for // simplicity, we're rendering it after we've cranked through all frames. const audioBuffer = await audioContext.startRendering(); - // await audioBufferSource.add(audioBuffer); - // audioBufferSource.close(); + await audioBufferSource.add(audioBuffer); + audioBufferSource.close(); // await audioBufferSource2!.add(audioBuffer); // audioBufferSource2!.close(); diff --git a/src/conversion.ts b/src/conversion.ts index 786d54f..cb90ec1 100644 --- a/src/conversion.ts +++ b/src/conversion.ts @@ -52,6 +52,7 @@ import { Mp4OutputFormat } from './output-format'; import { AudioSample, clampCropRectangle, validateCropRectangle, VideoSample } from './sample'; import { MetadataTags, validateMetadataTags } from './metadata'; import { NullTarget } from './target'; +import { AudioResampler } from './resample'; /** * The options for media file conversion. @@ -1611,6 +1612,8 @@ export class Conversion { startTime: this._startTimestamp, endTime: this._endTimestamp, onSample: async (sample) => { + sample.setTimestamp(sample.timestamp - this._startTimestamp); + await this._registerAudioSample(track, trackOptions, source, sample); sample.close(); }, @@ -1731,280 +1734,3 @@ class TrackSynchronizer { this.computeMinAndMaybeResolve(); } } - -/** - * Utility class to handle audio resampling, handling both sample rate resampling as well as channel up/downmixing. - * The advantage over doing this manually rather than using OfflineAudioContext to do it for us is the artifact-free - * handling of putting multiple resampled audio samples back to back, which produces flaky results using - * OfflineAudioContext. - */ -export class AudioResampler { - sourceSampleRate: number | null = null; - targetSampleRate: number; - sourceNumberOfChannels: number | null = null; - targetNumberOfChannels: number; - startTime: number; - endTime: number; - onSample: (sample: AudioSample) => Promise; - - bufferSizeInFrames: number; - bufferSizeInSamples: number; - outputBuffer: Float32Array; - /** Start frame of current buffer */ - bufferStartFrame: number; - /** The highest index written to in the current buffer */ - maxWrittenFrame: number; - channelMixer!: (sourceData: Float32Array, sourceFrameIndex: number, targetChannelIndex: number) => number; - tempSourceBuffer!: Float32Array; - - constructor(options: { - targetSampleRate: number; - targetNumberOfChannels: number; - startTime: number; - endTime: number; - onSample: (sample: AudioSample) => Promise; - }) { - this.targetSampleRate = options.targetSampleRate; - this.targetNumberOfChannels = options.targetNumberOfChannels; - this.startTime = options.startTime; - this.endTime = options.endTime; - this.onSample = options.onSample; - - this.bufferSizeInFrames = Math.floor(this.targetSampleRate * 5.0); // 5 seconds - this.bufferSizeInSamples = this.bufferSizeInFrames * this.targetNumberOfChannels; - - this.outputBuffer = new Float32Array(this.bufferSizeInSamples); - this.bufferStartFrame = 0; - this.maxWrittenFrame = -1; - } - - /** - * Sets up the channel mixer to handle up/downmixing in the case where input and output channel counts don't match. - */ - doChannelMixerSetup(): void { - assert(this.sourceNumberOfChannels !== null); - - const sourceNum = this.sourceNumberOfChannels; - const targetNum = this.targetNumberOfChannels; - - // Logic taken from - // https://developer.mozilla.org/en-US/docs/Web/API/Web_Audio_API/Basic_concepts_behind_Web_Audio_API - // Most of the mapping functions are branchless. - - if (sourceNum === 1 && targetNum === 2) { - // Mono to Stereo: M -> L, M -> R - this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number) => { - return sourceData[sourceFrameIndex * sourceNum]!; - }; - } else if (sourceNum === 1 && targetNum === 4) { - // Mono to Quad: M -> L, M -> R, 0 -> SL, 0 -> SR - this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number, targetChannelIndex: number) => { - return sourceData[sourceFrameIndex * sourceNum]! * +(targetChannelIndex < 2); - }; - } else if (sourceNum === 1 && targetNum === 6) { - // Mono to 5.1: 0 -> L, 0 -> R, M -> C, 0 -> LFE, 0 -> SL, 0 -> SR - this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number, targetChannelIndex: number) => { - return sourceData[sourceFrameIndex * sourceNum]! * +(targetChannelIndex === 2); - }; - } else if (sourceNum === 2 && targetNum === 1) { - // Stereo to Mono: 0.5 * (L + R) - this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number) => { - const baseIdx = sourceFrameIndex * sourceNum; - return 0.5 * (sourceData[baseIdx]! + sourceData[baseIdx + 1]!); - }; - } else if (sourceNum === 2 && targetNum === 4) { - // Stereo to Quad: L -> L, R -> R, 0 -> SL, 0 -> SR - this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number, targetChannelIndex: number) => { - return sourceData[sourceFrameIndex * sourceNum + targetChannelIndex]! * +(targetChannelIndex < 2); - }; - } else if (sourceNum === 2 && targetNum === 6) { - // Stereo to 5.1: L -> L, R -> R, 0 -> C, 0 -> LFE, 0 -> SL, 0 -> SR - this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number, targetChannelIndex: number) => { - return sourceData[sourceFrameIndex * sourceNum + targetChannelIndex]! * +(targetChannelIndex < 2); - }; - } else if (sourceNum === 4 && targetNum === 1) { - // Quad to Mono: 0.25 * (L + R + SL + SR) - this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number) => { - const baseIdx = sourceFrameIndex * sourceNum; - return 0.25 * ( - sourceData[baseIdx]! + sourceData[baseIdx + 1]! - + sourceData[baseIdx + 2]! + sourceData[baseIdx + 3]! - ); - }; - } else if (sourceNum === 4 && targetNum === 2) { - // Quad to Stereo: 0.5 * (L + SL), 0.5 * (R + SR) - this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number, targetChannelIndex: number) => { - const baseIdx = sourceFrameIndex * sourceNum; - return 0.5 * ( - sourceData[baseIdx + targetChannelIndex]! - + sourceData[baseIdx + targetChannelIndex + 2]! - ); - }; - } else if (sourceNum === 4 && targetNum === 6) { - // Quad to 5.1: L -> L, R -> R, 0 -> C, 0 -> LFE, SL -> SL, SR -> SR - this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number, targetChannelIndex: number) => { - const baseIdx = sourceFrameIndex * sourceNum; - - // It's a bit harder to do this one branchlessly - if (targetChannelIndex < 2) return sourceData[baseIdx + targetChannelIndex]!; // L, R - if (targetChannelIndex === 2 || targetChannelIndex === 3) return 0; // C, LFE - return sourceData[baseIdx + targetChannelIndex - 2]!; // SL, SR - }; - } else if (sourceNum === 6 && targetNum === 1) { - // 5.1 to Mono: sqrt(1/2) * (L + R) + C + 0.5 * (SL + SR) - this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number) => { - const baseIdx = sourceFrameIndex * sourceNum; - return Math.SQRT1_2 * (sourceData[baseIdx]! + sourceData[baseIdx + 1]!) - + sourceData[baseIdx + 2]! - + 0.5 * (sourceData[baseIdx + 4]! + sourceData[baseIdx + 5]!); - }; - } else if (sourceNum === 6 && targetNum === 2) { - // 5.1 to Stereo: L + sqrt(1/2) * (C + SL), R + sqrt(1/2) * (C + SR) - this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number, targetChannelIndex: number) => { - const baseIdx = sourceFrameIndex * sourceNum; - return sourceData[baseIdx + targetChannelIndex]! - + Math.SQRT1_2 * (sourceData[baseIdx + 2]! + sourceData[baseIdx + targetChannelIndex + 4]!); - }; - } else if (sourceNum === 6 && targetNum === 4) { - // 5.1 to Quad: L + sqrt(1/2) * C, R + sqrt(1/2) * C, SL, SR - this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number, targetChannelIndex: number) => { - const baseIdx = sourceFrameIndex * sourceNum; - - // It's a bit harder to do this one branchlessly - if (targetChannelIndex < 2) { - return sourceData[baseIdx + targetChannelIndex]! + Math.SQRT1_2 * sourceData[baseIdx + 2]!; - } - return sourceData[baseIdx + targetChannelIndex + 2]!; // SL, SR - }; - } else { - // Discrete fallback: direct mapping with zero-fill or drop - this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number, targetChannelIndex: number) => { - return targetChannelIndex < sourceNum - ? sourceData[sourceFrameIndex * sourceNum + targetChannelIndex]! - : 0; - }; - } - } - - ensureTempBufferSize(requiredSamples: number): void { - let length = this.tempSourceBuffer.length; - - while (length < requiredSamples) { - length *= 2; - } - - if (length !== this.tempSourceBuffer.length) { - const newBuffer = new Float32Array(length); - newBuffer.set(this.tempSourceBuffer); - this.tempSourceBuffer = newBuffer; - } - } - - async add(audioSample: AudioSample) { - if (this.sourceSampleRate === null) { - // This is the first sample, so let's init the missing data. Initting the sample rate from the decoded - // sample is more reliable than using the file's metadata, because decoders are free to emit any sample rate - // they see fit. - this.sourceSampleRate = audioSample.sampleRate; - this.sourceNumberOfChannels = audioSample.numberOfChannels; - - // Pre-allocate temporary buffer for source data - this.tempSourceBuffer = new Float32Array(this.sourceSampleRate * this.sourceNumberOfChannels); - - this.doChannelMixerSetup(); - } - - const requiredSamples = audioSample.numberOfFrames * audioSample.numberOfChannels; - this.ensureTempBufferSize(requiredSamples); - - // Copy the audio data to the temp buffer - const sourceDataSize = audioSample.allocationSize({ planeIndex: 0, format: 'f32' }); - const sourceView = new Float32Array(this.tempSourceBuffer.buffer, 0, sourceDataSize / 4); - audioSample.copyTo(sourceView, { planeIndex: 0, format: 'f32' }); - - const inputStartTime = audioSample.timestamp - this.startTime; - const inputDuration = audioSample.numberOfFrames / this.sourceSampleRate; - const inputEndTime = Math.min(inputStartTime + inputDuration, this.endTime - this.startTime); - - // Compute which output frames are affected by this sample - const outputStartFrame = Math.floor(inputStartTime * this.targetSampleRate); - const outputEndFrame = Math.ceil(inputEndTime * this.targetSampleRate); - - for (let outputFrame = outputStartFrame; outputFrame < outputEndFrame; outputFrame++) { - if (outputFrame < this.bufferStartFrame) { - continue; // Skip writes to the past - } - - while (outputFrame >= this.bufferStartFrame + this.bufferSizeInFrames) { - // The write is after the current buffer, so finalize it - await this.finalizeCurrentBuffer(); - this.bufferStartFrame += this.bufferSizeInFrames; - } - - const bufferFrameIndex = outputFrame - this.bufferStartFrame; - assert(bufferFrameIndex < this.bufferSizeInFrames); - - const outputTime = outputFrame / this.targetSampleRate; - const inputTime = outputTime - inputStartTime; - const sourcePosition = inputTime * this.sourceSampleRate; - - const sourceLowerFrame = Math.floor(sourcePosition); - const sourceUpperFrame = Math.ceil(sourcePosition); - const fraction = sourcePosition - sourceLowerFrame; - - // Process each output channel - for (let targetChannel = 0; targetChannel < this.targetNumberOfChannels; targetChannel++) { - let lowerSample = 0; - let upperSample = 0; - - if (sourceLowerFrame >= 0 && sourceLowerFrame < audioSample.numberOfFrames) { - lowerSample = this.channelMixer(sourceView, sourceLowerFrame, targetChannel); - } - - if (sourceUpperFrame >= 0 && sourceUpperFrame < audioSample.numberOfFrames) { - upperSample = this.channelMixer(sourceView, sourceUpperFrame, targetChannel); - } - - // For resampling, we do naive linear interpolation to find the in-between sample. This produces - // suboptimal results especially for downsampling (for which a low-pass filter would first need to be - // applied), but AudioContext doesn't do this either, so, whatever, for now. - const outputSample = lowerSample + fraction * (upperSample - lowerSample); - - // Write to output buffer (interleaved) - const outputIndex = bufferFrameIndex * this.targetNumberOfChannels + targetChannel; - this.outputBuffer[outputIndex]! += outputSample; // Add in case of overlapping samples - } - - this.maxWrittenFrame = Math.max(this.maxWrittenFrame, bufferFrameIndex); - } - } - - async finalizeCurrentBuffer() { - if (this.maxWrittenFrame < 0) { - return; // Nothing to finalize - } - - const samplesWritten = (this.maxWrittenFrame + 1) * this.targetNumberOfChannels; - - const outputData = new Float32Array(samplesWritten); - outputData.set(this.outputBuffer.subarray(0, samplesWritten)); - - const timestampSeconds = this.bufferStartFrame / this.targetSampleRate; - const audioSample = new AudioSample({ - format: 'f32', - sampleRate: this.targetSampleRate, - numberOfChannels: this.targetNumberOfChannels, - timestamp: timestampSeconds, - data: outputData, - }); - - await this.onSample(audioSample); - - this.outputBuffer.fill(0); - this.maxWrittenFrame = -1; - } - - finalize() { - return this.finalizeCurrentBuffer(); - } -} diff --git a/src/encode.ts b/src/encode.ts index 0cc2f52..b3048c9 100644 --- a/src/encode.ts +++ b/src/encode.ts @@ -24,7 +24,7 @@ import { import { customAudioEncoders, customVideoEncoders } from './custom-coder'; import { isFirefox, MaybePromise, Rotation } from './misc'; import { EncodedPacket } from './packet'; -import { CropRectangle, validateCropRectangle, VideoSample } from './sample'; +import { AudioSample, CropRectangle, validateCropRectangle, VideoSample } from './sample'; const canEncodeVideoMemo = new Map>(); const canEncodeAudioMemo = new Map>(); @@ -355,6 +355,25 @@ export type AudioEncodingConfig = { */ bitrate?: number | Quality; + /** + * Optional transformations to apply to the audio samples before they are passed to the encoder. + */ + transform?: { + /** The desired number of output channels to up/downmix to. */ + numberOfChannels?: number; + /** The desired output sample rate in hertz to resample to. */ + sampleRate?: number; + /** + * Allows for custom user-defined processing of audio samples, e.g. for applying audio effects or timestamp + * modifications. Called for each audio sample after resampling and remixing. + * + * Must return an {@link AudioSample}, an array of them, or `null` for dropping the sample. + */ + process?: (sample: AudioSample) => MaybePromise< + AudioSample | AudioSample[] | null + >; + }; + /** Called for each successfully encoded packet. Both the packet and the encoding metadata are passed. */ onEncodedPacket?: (packet: EncodedPacket, meta: EncodedAudioChunkMetadata | undefined) => unknown; /** @@ -384,6 +403,26 @@ export const validateAudioEncodingConfig = (config: AudioEncodingConfig) => { ) { throw new TypeError('config.bitrate, when provided, must be a positive integer or a quality.'); } + if (config.transform !== undefined) { + if (typeof config.transform !== 'object' || !config.transform) { + throw new TypeError('config.transform, when provided, must be an object.'); + } + if ( + config.transform.numberOfChannels !== undefined + && (!Number.isInteger(config.transform.numberOfChannels) || config.transform.numberOfChannels <= 0) + ) { + throw new TypeError('config.transform.numberOfChannels, when provided, must be a positive integer.'); + } + if ( + config.transform.sampleRate !== undefined + && (!Number.isInteger(config.transform.sampleRate) || config.transform.sampleRate <= 0) + ) { + throw new TypeError('config.transform.sampleRate, when provided, must be a positive integer.'); + } + if (config.transform.process !== undefined && typeof config.transform.process !== 'function') { + throw new TypeError('config.transform.process, when provided, must be a function.'); + } + } if (config.onEncodedPacket !== undefined && typeof config.onEncodedPacket !== 'function') { throw new TypeError('config.onEncodedChunk, when provided, must be a function.'); } diff --git a/src/media-source.ts b/src/media-source.ts index 9ad8440..9a48e7b 100644 --- a/src/media-source.ts +++ b/src/media-source.ts @@ -59,6 +59,7 @@ import { validateVideoEncodingConfig, VideoEncodingConfig, } from './encode'; +import { AudioResampler } from './resample'; /** * Base class for media sources. Media sources are used to add media samples to an output file. @@ -1778,6 +1779,8 @@ class AudioEncoderWrapper { private lastEndSampleIndex: number | null = null; + private resampler: AudioResampler | null = null; + /** * Encoders typically throw their errors "out of band", meaning asynchronously in some other execution context. * However, we want to surface these errors to the user within the normal control flow, so they don't go uncaught. @@ -1809,6 +1812,75 @@ class AudioEncoderWrapper { this.lastSampleRate = audioSample.sampleRate; } + const config = this.encodingConfig; + const needsResample = config.transform?.numberOfChannels !== undefined + || config.transform?.sampleRate !== undefined; + + if (needsResample) { + if (!this.resampler) { + // Initialize the resampler on first sample + this.resampler = new AudioResampler({ + targetNumberOfChannels: config.transform!.numberOfChannels + ?? audioSample.numberOfChannels, + targetSampleRate: config.transform!.sampleRate + ?? audioSample.sampleRate, + startTime: audioSample.timestamp, + endTime: Infinity, + onSample: async (sample) => { + await this.processAndEncode(sample, true); + }, + }); + } + + await this.resampler.add(audioSample); + } else { + await this.processAndEncode(audioSample, shouldClose); + } + } finally { + if (shouldClose) { + audioSample.close(); + } + } + } + + /** + * Runs the process function (if any) and encodes the resulting samples. + */ + private async processAndEncode(audioSample: AudioSample, shouldClose: boolean) { + const config = this.encodingConfig; + + if (config.transform?.process) { + let processed = config.transform.process(audioSample); + if (processed instanceof Promise) { + processed = await processed; + } + + if (processed === null) { + return; + } + + if (!Array.isArray(processed)) { + processed = [processed]; + } + + for (const sample of processed) { + if (!(sample instanceof AudioSample)) { + throw new TypeError( + 'The audio process function must return an AudioSample, null, or an array of AudioSamples.', + ); + } + await this.encodeSample(sample, true); + } + } else { + await this.encodeSample(audioSample, shouldClose); + } + } + + /** + * Encodes a single audio sample, handling encoder init, gap padding, and backpressure. + */ + private async encodeSample(audioSample: AudioSample, shouldClose: boolean) { + try { if (!this.encoderInitialized) { if (!this.ensureEncoderPromise) { this.ensureEncoder(audioSample); @@ -1853,7 +1925,7 @@ class AudioEncoderWrapper { timestamp: this.lastEndSampleIndex / audioSample.sampleRate, }); - await this.add(fillSample, true); // Recursive call + await this.encodeSample(fillSample, true); // Recursive call } this.lastEndSampleIndex += audioSample.numberOfFrames; @@ -1872,7 +1944,6 @@ class AudioEncoderWrapper { .catch((error: Error) => this.error ??= error) .finally(() => { clonedSample.close(); - // `audioSample` gets closed in the finally block at the end of the method }); if (this.customEncoderQueueSize >= 4) { @@ -1900,7 +1971,6 @@ class AudioEncoderWrapper { } } finally { if (shouldClose) { - // Make sure it's always closed, even if there was an error audioSample.close(); } } @@ -2189,7 +2259,15 @@ class AudioEncoderWrapper { } async flushAndClose(forceClose: boolean) { - if (!forceClose) this.checkForEncoderError(); + if (!forceClose) { + this.checkForEncoderError(); + } + + // Finalize the resampler to flush any buffered audio + if (!forceClose && this.resampler) { + await this.resampler.finalize(); + } + this.resampler = null; if (this.customEncoder) { if (!forceClose) { @@ -2207,7 +2285,9 @@ class AudioEncoderWrapper { } } - if (!forceClose) this.checkForEncoderError(); + if (!forceClose) { + this.checkForEncoderError(); + } } getQueueSize() { diff --git a/src/resample.ts b/src/resample.ts new file mode 100644 index 0000000..19c4606 --- /dev/null +++ b/src/resample.ts @@ -0,0 +1,281 @@ +import { assert } from './misc'; +import { AudioSample } from './sample'; + +/** + * Utility class to handle audio resampling, handling both sample rate resampling as well as channel up/downmixing. + * The advantage over doing this manually rather than using OfflineAudioContext to do it for us is the artifact-free + * handling of putting multiple resampled audio samples back to back, which produces flaky results using + * OfflineAudioContext. + */ +export class AudioResampler { + sourceSampleRate: number | null = null; + targetSampleRate: number; + sourceNumberOfChannels: number | null = null; + targetNumberOfChannels: number; + startTime: number; + endTime: number; + onSample: (sample: AudioSample) => Promise; + + bufferSizeInFrames: number; + bufferSizeInSamples: number; + outputBuffer: Float32Array; + /** Start frame of current buffer */ + bufferStartFrame: number; + /** The highest index written to in the current buffer */ + maxWrittenFrame: number | null = null; + channelMixer!: (sourceData: Float32Array, sourceFrameIndex: number, targetChannelIndex: number) => number; + tempSourceBuffer!: Float32Array; + + constructor(options: { + targetSampleRate: number; + targetNumberOfChannels: number; + startTime: number; + endTime: number; + onSample: (sample: AudioSample) => Promise; + }) { + this.targetSampleRate = options.targetSampleRate; + this.targetNumberOfChannels = options.targetNumberOfChannels; + this.startTime = options.startTime; + this.endTime = options.endTime; + this.onSample = options.onSample; + + this.bufferSizeInFrames = Math.floor(this.targetSampleRate * 5.0); // 5 seconds + this.bufferSizeInSamples = this.bufferSizeInFrames * this.targetNumberOfChannels; + + this.outputBuffer = new Float32Array(this.bufferSizeInSamples); + this.bufferStartFrame = Math.floor(this.startTime * this.targetSampleRate); + } + + /** + * Sets up the channel mixer to handle up/downmixing in the case where input and output channel counts don't match. + */ + doChannelMixerSetup(): void { + assert(this.sourceNumberOfChannels !== null); + + const sourceNum = this.sourceNumberOfChannels; + const targetNum = this.targetNumberOfChannels; + + // Logic taken from + // https://developer.mozilla.org/en-US/docs/Web/API/Web_Audio_API/Basic_concepts_behind_Web_Audio_API + // Most of the mapping functions are branchless. + + if (sourceNum === 1 && targetNum === 2) { + // Mono to Stereo: M -> L, M -> R + this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number) => { + return sourceData[sourceFrameIndex * sourceNum]!; + }; + } else if (sourceNum === 1 && targetNum === 4) { + // Mono to Quad: M -> L, M -> R, 0 -> SL, 0 -> SR + this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number, targetChannelIndex: number) => { + return sourceData[sourceFrameIndex * sourceNum]! * +(targetChannelIndex < 2); + }; + } else if (sourceNum === 1 && targetNum === 6) { + // Mono to 5.1: 0 -> L, 0 -> R, M -> C, 0 -> LFE, 0 -> SL, 0 -> SR + this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number, targetChannelIndex: number) => { + return sourceData[sourceFrameIndex * sourceNum]! * +(targetChannelIndex === 2); + }; + } else if (sourceNum === 2 && targetNum === 1) { + // Stereo to Mono: 0.5 * (L + R) + this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number) => { + const baseIdx = sourceFrameIndex * sourceNum; + return 0.5 * (sourceData[baseIdx]! + sourceData[baseIdx + 1]!); + }; + } else if (sourceNum === 2 && targetNum === 4) { + // Stereo to Quad: L -> L, R -> R, 0 -> SL, 0 -> SR + this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number, targetChannelIndex: number) => { + return sourceData[sourceFrameIndex * sourceNum + targetChannelIndex]! * +(targetChannelIndex < 2); + }; + } else if (sourceNum === 2 && targetNum === 6) { + // Stereo to 5.1: L -> L, R -> R, 0 -> C, 0 -> LFE, 0 -> SL, 0 -> SR + this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number, targetChannelIndex: number) => { + return sourceData[sourceFrameIndex * sourceNum + targetChannelIndex]! * +(targetChannelIndex < 2); + }; + } else if (sourceNum === 4 && targetNum === 1) { + // Quad to Mono: 0.25 * (L + R + SL + SR) + this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number) => { + const baseIdx = sourceFrameIndex * sourceNum; + return 0.25 * ( + sourceData[baseIdx]! + sourceData[baseIdx + 1]! + + sourceData[baseIdx + 2]! + sourceData[baseIdx + 3]! + ); + }; + } else if (sourceNum === 4 && targetNum === 2) { + // Quad to Stereo: 0.5 * (L + SL), 0.5 * (R + SR) + this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number, targetChannelIndex: number) => { + const baseIdx = sourceFrameIndex * sourceNum; + return 0.5 * ( + sourceData[baseIdx + targetChannelIndex]! + + sourceData[baseIdx + targetChannelIndex + 2]! + ); + }; + } else if (sourceNum === 4 && targetNum === 6) { + // Quad to 5.1: L -> L, R -> R, 0 -> C, 0 -> LFE, SL -> SL, SR -> SR + this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number, targetChannelIndex: number) => { + const baseIdx = sourceFrameIndex * sourceNum; + + // It's a bit harder to do this one branchlessly + if (targetChannelIndex < 2) return sourceData[baseIdx + targetChannelIndex]!; // L, R + if (targetChannelIndex === 2 || targetChannelIndex === 3) return 0; // C, LFE + return sourceData[baseIdx + targetChannelIndex - 2]!; // SL, SR + }; + } else if (sourceNum === 6 && targetNum === 1) { + // 5.1 to Mono: sqrt(1/2) * (L + R) + C + 0.5 * (SL + SR) + this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number) => { + const baseIdx = sourceFrameIndex * sourceNum; + return Math.SQRT1_2 * (sourceData[baseIdx]! + sourceData[baseIdx + 1]!) + + sourceData[baseIdx + 2]! + + 0.5 * (sourceData[baseIdx + 4]! + sourceData[baseIdx + 5]!); + }; + } else if (sourceNum === 6 && targetNum === 2) { + // 5.1 to Stereo: L + sqrt(1/2) * (C + SL), R + sqrt(1/2) * (C + SR) + this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number, targetChannelIndex: number) => { + const baseIdx = sourceFrameIndex * sourceNum; + return sourceData[baseIdx + targetChannelIndex]! + + Math.SQRT1_2 * (sourceData[baseIdx + 2]! + sourceData[baseIdx + targetChannelIndex + 4]!); + }; + } else if (sourceNum === 6 && targetNum === 4) { + // 5.1 to Quad: L + sqrt(1/2) * C, R + sqrt(1/2) * C, SL, SR + this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number, targetChannelIndex: number) => { + const baseIdx = sourceFrameIndex * sourceNum; + + // It's a bit harder to do this one branchlessly + if (targetChannelIndex < 2) { + return sourceData[baseIdx + targetChannelIndex]! + Math.SQRT1_2 * sourceData[baseIdx + 2]!; + } + return sourceData[baseIdx + targetChannelIndex + 2]!; // SL, SR + }; + } else { + // Discrete fallback: direct mapping with zero-fill or drop + this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number, targetChannelIndex: number) => { + return targetChannelIndex < sourceNum + ? sourceData[sourceFrameIndex * sourceNum + targetChannelIndex]! + : 0; + }; + } + } + + ensureTempBufferSize(requiredSamples: number): void { + let length = this.tempSourceBuffer.length; + + while (length < requiredSamples) { + length *= 2; + } + + if (length !== this.tempSourceBuffer.length) { + const newBuffer = new Float32Array(length); + newBuffer.set(this.tempSourceBuffer); + this.tempSourceBuffer = newBuffer; + } + } + + async add(audioSample: AudioSample) { + if (this.sourceSampleRate === null) { + // This is the first sample, so let's init the missing data. Initting the sample rate from the decoded + // sample is more reliable than using the file's metadata, because decoders are free to emit any sample rate + // they see fit. + this.sourceSampleRate = audioSample.sampleRate; + this.sourceNumberOfChannels = audioSample.numberOfChannels; + + // Pre-allocate temporary buffer for source data + this.tempSourceBuffer = new Float32Array(this.sourceSampleRate * this.sourceNumberOfChannels); + + this.doChannelMixerSetup(); + } + + const requiredSamples = audioSample.numberOfFrames * audioSample.numberOfChannels; + this.ensureTempBufferSize(requiredSamples); + + // Copy the audio data to the temp buffer + const sourceDataSize = audioSample.allocationSize({ planeIndex: 0, format: 'f32' }); + const sourceView = new Float32Array(this.tempSourceBuffer.buffer, 0, sourceDataSize / 4); + audioSample.copyTo(sourceView, { planeIndex: 0, format: 'f32' }); + + const inputStartTime = audioSample.timestamp; + const inputEndTime = Math.min(audioSample.timestamp + audioSample.duration, this.endTime); + + // Compute which output frames are affected by this sample + const outputStartFrame = Math.floor(inputStartTime * this.targetSampleRate); + const outputEndFrame = Math.ceil(inputEndTime * this.targetSampleRate); + + for (let outputFrame = outputStartFrame; outputFrame < outputEndFrame; outputFrame++) { + if (outputFrame < this.bufferStartFrame) { + continue; // Skip writes to the past + } + + while (outputFrame >= this.bufferStartFrame + this.bufferSizeInFrames) { + // The write is after the current buffer, so finalize it + await this.finalizeCurrentBuffer(); + this.bufferStartFrame += this.bufferSizeInFrames; + } + + const bufferFrameIndex = outputFrame - this.bufferStartFrame; + assert(bufferFrameIndex < this.bufferSizeInFrames); + + const outputTime = outputFrame / this.targetSampleRate; + const inputTime = outputTime - inputStartTime; + const sourcePosition = inputTime * this.sourceSampleRate; + + const sourceLowerFrame = Math.floor(sourcePosition); + const sourceUpperFrame = Math.ceil(sourcePosition); + const fraction = sourcePosition - sourceLowerFrame; + + // Process each output channel + for (let targetChannel = 0; targetChannel < this.targetNumberOfChannels; targetChannel++) { + let lowerSample = 0; + let upperSample = 0; + + if (sourceLowerFrame >= 0 && sourceLowerFrame < audioSample.numberOfFrames) { + lowerSample = this.channelMixer(sourceView, sourceLowerFrame, targetChannel); + } + + if (sourceUpperFrame >= 0 && sourceUpperFrame < audioSample.numberOfFrames) { + upperSample = this.channelMixer(sourceView, sourceUpperFrame, targetChannel); + } + + // For resampling, we do naive linear interpolation to find the in-between sample. This produces + // suboptimal results especially for downsampling (for which a low-pass filter would first need to be + // applied), but AudioContext doesn't do this either, so, whatever, for now. + const outputSample = lowerSample + fraction * (upperSample - lowerSample); + + // Write to output buffer (interleaved) + const outputIndex = bufferFrameIndex * this.targetNumberOfChannels + targetChannel; + this.outputBuffer[outputIndex]! += outputSample; // Add in case of overlapping samples + } + + if (this.maxWrittenFrame === null) { + this.maxWrittenFrame = bufferFrameIndex; + } else { + this.maxWrittenFrame = Math.max(this.maxWrittenFrame, bufferFrameIndex); + } + } + } + + async finalizeCurrentBuffer() { + if (this.maxWrittenFrame === null) { + return; // Nothing to finalize + } + + const samplesWritten = (this.maxWrittenFrame + 1) * this.targetNumberOfChannels; + + const outputData = new Float32Array(samplesWritten); + outputData.set(this.outputBuffer.subarray(0, samplesWritten)); + + const timestampSeconds = this.bufferStartFrame / this.targetSampleRate; + const audioSample = new AudioSample({ + format: 'f32', + sampleRate: this.targetSampleRate, + numberOfChannels: this.targetNumberOfChannels, + timestamp: timestampSeconds, + data: outputData, + }); + + await this.onSample(audioSample); + + this.outputBuffer.fill(0); + this.maxWrittenFrame = null; + } + + finalize() { + return this.finalizeCurrentBuffer(); + } +} diff --git a/test/browser/media-sources.test.ts b/test/browser/media-sources.test.ts index 2f6aa79..0a70eb4 100644 --- a/test/browser/media-sources.test.ts +++ b/test/browser/media-sources.test.ts @@ -2,17 +2,32 @@ import { expect, test } from 'vitest'; import { Output } from '../../src/output.js'; import { Mp4OutputFormat, WebMOutputFormat } from '../../src/output-format.js'; import { BufferTarget } from '../../src/target.js'; -import { VideoSampleSource } from '../../src/media-source.js'; -import { VideoSample } from '../../src/sample.js'; +import { AudioSampleSource, VideoSampleSource } from '../../src/media-source.js'; +import { AudioSample, VideoSample } from '../../src/sample.js'; import { QUALITY_MEDIUM } from '../../src/encode.js'; import { Input } from '../../src/input.js'; import { ALL_FORMATS } from '../../src/input-format.js'; import { BufferSource } from '../../src/source.js'; import { VideoSampleSink } from '../../src/media-sink.js'; import { assert, Rotation } from '../../src/misc.js'; -import { InputVideoTrack } from '../../src/input-track.js'; +import { InputAudioTrack, InputVideoTrack } from '../../src/input-track.js'; -test('VideoSampleSource.close() should be idempotent after finalize()', async () => { +test('VideoSampleSource, normal usage', async () => { + const buffer = await encodeFrames( + { codec: 'vp8', bitrate: QUALITY_MEDIUM }, + [{ width: 100, height: 100 }, { width: 100, height: 100 }, { width: 100, height: 100 }], + ); + + const samples = await readBackSamples(buffer); + expect(samples).toHaveLength(3); + + for (const sample of samples) { + expect(sample.codedWidth).toBe(100); + expect(sample.codedHeight).toBe(100); + } +}); + +test('VideoSampleSource, .close() should be idempotent after finalize()', async () => { const output = new Output({ format: new WebMOutputFormat(), target: new BufferTarget(), @@ -40,7 +55,7 @@ test('VideoSampleSource.close() should be idempotent after finalize()', async () videoSource.close(); // This previously threw }); -test('Changing input dimensions throws with deny (default)', async () => { +test('VideoSampleSource, changing input dimensions throws with deny (default)', async () => { const output = new Output({ format: new Mp4OutputFormat(), target: new BufferTarget(), @@ -65,7 +80,7 @@ test('Changing input dimensions throws with deny (default)', async () => { videoSource.close(); }); -test('Changing input dimensions with passThrough preserves per-frame dimensions', async () => { +test('VideoSampleSource, changing input dimensions with passThrough preserves per-frame dimensions', async () => { const buffer = await encodeFrames( { codec: 'vp8', bitrate: QUALITY_MEDIUM, sizeChangeBehavior: 'passThrough' }, [{ width: 100, height: 100 }, { width: 200, height: 150 }], @@ -81,21 +96,24 @@ test('Changing input dimensions with passThrough preserves per-frame dimensions' ).toBe(true); }); -test('Changing input dimensions with fill/contain/cover locks output to first frame dimensions', async () => { - for (const behavior of ['fill', 'contain', 'cover'] as const) { - const buffer = await encodeFrames( - { codec: 'vp8', bitrate: QUALITY_MEDIUM, sizeChangeBehavior: behavior }, - [{ width: 100, height: 100 }, { width: 200, height: 150 }], - ); +test( + 'VideoSampleSource, changing input dimensions with fill/contain/cover locks output to first frame dimensions', + async () => { + for (const behavior of ['fill', 'contain', 'cover'] as const) { + const buffer = await encodeFrames( + { codec: 'vp8', bitrate: QUALITY_MEDIUM, sizeChangeBehavior: behavior }, + [{ width: 100, height: 100 }, { width: 200, height: 150 }], + ); - const { input, track } = await readBackTrack(buffer); - expect(track.codedWidth).toBe(100); - expect(track.codedHeight).toBe(100); - input.dispose(); - } -}); + const { input, track } = await readBackTrack(buffer); + expect(track.codedWidth).toBe(100); + expect(track.codedHeight).toBe(100); + input.dispose(); + } + }, +); -test('Same-sized frames with width and height set', async () => { +test('VideoSampleSource, same-sized frames with width and height set', async () => { const buffer = await encodeFrames( { codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { width: 50, height: 80, fit: 'fill' } }, [{ width: 100, height: 100 }, { width: 100, height: 100 }], @@ -107,7 +125,7 @@ test('Same-sized frames with width and height set', async () => { input.dispose(); }); -test('Same-sized frames with rotation set to 90', async () => { +test('VideoSampleSource, same-sized frames with rotation set to 90', async () => { const buffer = await encodeFrames( { codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { rotate: 90 } }, [{ width: 200, height: 100 }, { width: 200, height: 100 }], @@ -119,7 +137,7 @@ test('Same-sized frames with rotation set to 90', async () => { input.dispose(); }); -test('Same-sized frames with rotation, width and height', async () => { +test('VideoSampleSource, same-sized frames with rotation, width and height', async () => { const buffer = await encodeFrames( { codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { rotate: 90, width: 50, height: 80, fit: 'contain' } }, [{ width: 200, height: 100 }, { width: 200, height: 100 }], @@ -131,7 +149,7 @@ test('Same-sized frames with rotation, width and height', async () => { input.dispose(); }); -test('Changing dimensions with passThrough and rotation 90', async () => { +test('VideoSampleSource, changing dimensions with passThrough and rotation 90', async () => { const buffer = await encodeFrames( { codec: 'vp8', bitrate: QUALITY_MEDIUM, sizeChangeBehavior: 'passThrough', transform: { rotate: 90 } }, [{ width: 200, height: 100 }, { width: 300, height: 150 }], @@ -147,7 +165,7 @@ test('Changing dimensions with passThrough and rotation 90', async () => { ).toBe(true); }); -test('Changing dimensions with passThrough, width and height set', async () => { +test('VideoSampleSource, changing dimensions with passThrough, width and height set', async () => { const buffer = await encodeFrames( { codec: 'vp8', @@ -169,7 +187,7 @@ test('Changing dimensions with passThrough, width and height set', async () => { expect(samples[1]).toMatchObject({ codedWidth: 50, codedHeight: 80 }); }); -test('Encoding rotated video frames', async () => { +test('VideoSampleSource, encoding rotated video frames', async () => { const buffer = await encodeFrames( { codec: 'vp8', bitrate: QUALITY_MEDIUM }, [{ width: 200, height: 100, rotation: 90 }, { width: 200, height: 100, rotation: 90 }], @@ -184,7 +202,7 @@ test('Encoding rotated video frames', async () => { } }); -test('Encoding rotated video frames with forced transform', async () => { +test('VideoSampleSource, encoding rotated video frames with forced transform', async () => { const buffer = await encodeFrames( { codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { force: true } }, [{ width: 200, height: 100, rotation: 90 }, { width: 200, height: 100, rotation: 90 }], @@ -199,7 +217,7 @@ test('Encoding rotated video frames with forced transform', async () => { } }); -test('transform.process identity function', async () => { +test('VideoSampleSource, transform.process identity function', async () => { const buffer = await encodeFrames( { codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { process: sample => sample } }, [{ width: 100, height: 100 }, { width: 100, height: 100 }, { width: 100, height: 100 }], @@ -213,7 +231,7 @@ test('transform.process identity function', async () => { } }); -test('transform.process manual resize', async () => { +test('VideoSampleSource, transform.process manual resize', async () => { const buffer = await encodeFrames( { codec: 'vp8', @@ -241,7 +259,7 @@ test('transform.process manual resize', async () => { input.dispose(); }); -test('transform.process receives pre-transformed frames', async () => { +test('VideoSampleSource, transform.process receives pre-transformed frames', async () => { const receivedDimensions: { width: number; height: number }[] = []; const buffer = await encodeFrames( @@ -276,7 +294,7 @@ test('transform.process receives pre-transformed frames', async () => { input.dispose(); }); -test('transform.process drops all frames after the first', async () => { +test('VideoSampleSource, transform.process drops all frames after the first', async () => { let frameIndex = 0; const buffer = await encodeFrames( @@ -299,7 +317,7 @@ test('transform.process drops all frames after the first', async () => { expect(samples).toHaveLength(1); }); -test('transform.process expands every frame into two', async () => { +test('VideoSampleSource, transform.process expands every frame into two', async () => { const buffer = await encodeFrames( { codec: 'vp8', @@ -331,7 +349,7 @@ test('transform.process expands every frame into two', async () => { } }); -test('transform.frameRate normalizes variable-rate input to fixed rate', async () => { +test('VideoSampleSource, transform.frameRate normalizes variable-rate input to fixed rate', async () => { const buffer = await encodeFrames( { codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { frameRate: 10 } }, [ @@ -349,7 +367,7 @@ test('transform.frameRate normalizes variable-rate input to fixed rate', async ( } }); -test('transform.frameRate pads gaps by repeating last frame', async () => { +test('VideoSampleSource, transform.frameRate pads gaps by repeating last frame', async () => { const buffer = await encodeFrames( { codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { frameRate: 10 } }, [ @@ -366,7 +384,7 @@ test('transform.frameRate pads gaps by repeating last frame', async () => { } }); -test('transform.frameRate deduplicates frames in the same slot', async () => { +test('VideoSampleSource, transform.frameRate deduplicates frames in the same slot', async () => { const buffer = await encodeFrames( { codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { frameRate: 10 } }, [ @@ -383,7 +401,7 @@ test('transform.frameRate deduplicates frames in the same slot', async () => { expect(samples[1]!.timestamp).toBeCloseTo(0.1); }); -test('transform.frameRate final padding fills remaining duration', async () => { +test('VideoSampleSource, transform.frameRate final padding fills remaining duration', async () => { const buffer = await encodeFrames( { codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { frameRate: 10 } }, [ @@ -398,7 +416,7 @@ test('transform.frameRate final padding fills remaining duration', async () => { } }); -test('transform.frameRate skipping and padding combined', async () => { +test('VideoSampleSource, transform.frameRate skipping and padding combined', async () => { const buffer = await encodeFrames( { codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { frameRate: 10 } }, [ @@ -418,7 +436,7 @@ test('transform.frameRate skipping and padding combined', async () => { } }); -test('transform.frameRate works with transform', async () => { +test('VideoSampleSource, transform.frameRate works with transform', async () => { const buffer = await encodeFrames( { codec: 'vp8', @@ -440,7 +458,7 @@ test('transform.frameRate works with transform', async () => { } }); -test('transform.frameRate works with process', async () => { +test('VideoSampleSource, transform.frameRate works with process', async () => { const processedTimestamps: number[] = []; const buffer = await encodeFrames( @@ -552,3 +570,116 @@ const readBackSamples = async (buffer: ArrayBuffer) => { input.dispose(); return samples; }; + +test('AudioSampleSource, normal usage', async () => { + const sample = makeSineWave(48000, 2, 1); + const buffer = await encodeAudio({ codec: 'pcm-s16' }, sample); + + const { input, track } = await readBackAudioTrack(buffer); + expect(track.numberOfChannels).toBe(2); + expect(track.sampleRate).toBe(48000); + expect(await track.computeDuration()).toBe(1); + input.dispose(); +}); + +test('AudioSampleSource, remixed to mono', async () => { + const sample = makeSineWave(48000, 2, 1); + const buffer = await encodeAudio( + { codec: 'pcm-s16', transform: { numberOfChannels: 1 } }, + sample, + ); + + const { input, track } = await readBackAudioTrack(buffer); + expect(track.numberOfChannels).toBe(1); + expect(track.sampleRate).toBe(48000); + expect(await track.computeDuration()).toBe(1); + input.dispose(); +}); + +test('AudioSampleSource, resampled to 44100 Hz', async () => { + const sample = makeSineWave(48000, 2, 1); + const buffer = await encodeAudio( + { codec: 'pcm-s16', transform: { sampleRate: 44100 } }, + sample, + ); + + const { input, track } = await readBackAudioTrack(buffer); + expect(track.numberOfChannels).toBe(2); + expect(track.sampleRate).toBe(44100); + expect(await track.computeDuration()).toBe(1); + input.dispose(); +}); + +test('AudioSampleSource, resampled stereo with non-zero start timestamp', async () => { + const sample = makeSineWave(48000, 2, 1, 1); + const buffer = await encodeAudio( + { codec: 'pcm-s16', transform: { numberOfChannels: 2 } }, + sample, + ); + + const { input, track } = await readBackAudioTrack(buffer); + expect(track.numberOfChannels).toBe(2); + expect(track.sampleRate).toBe(48000); + + expect(await track.getFirstTimestamp()).toBe(1); + expect(await track.computeDuration()).toBe(2); + input.dispose(); +}); + +const makeSineWave = ( + sampleRate: number, + numberOfChannels: number, + durationSeconds: number, + timestamp = 0, +) => { + const numberOfFrames = Math.round(sampleRate * durationSeconds); + const data = new Float32Array(numberOfFrames * numberOfChannels); + + for (let frame = 0; frame < numberOfFrames; frame++) { + const value = Math.sin(2 * Math.PI * 440 * frame / sampleRate); + for (let ch = 0; ch < numberOfChannels; ch++) { + data[frame * numberOfChannels + ch] = value; + } + } + + return new AudioSample({ + data, + format: 'f32-planar', + sampleRate, + numberOfChannels, + numberOfFrames, + timestamp, + }); +}; + +const encodeAudio = async ( + encodingConfig: ConstructorParameters[0], + sample: AudioSample, +) => { + const output = new Output({ + format: new Mp4OutputFormat({ fastStart: 'fragmented' }), // Fragmented to avoid the PCM transformation + target: new BufferTarget(), + }); + + const audioSource = new AudioSampleSource(encodingConfig); + output.addAudioTrack(audioSource); + await output.start(); + + await audioSource.add(sample); + sample.close(); + + await output.finalize(); + return output.target.buffer!; +}; + +const readBackAudioTrack = async (buffer: ArrayBuffer) => { + const input = new Input({ + source: new BufferSource(buffer), + formats: ALL_FORMATS, + }); + + const track = await input.getPrimaryAudioTrack() as InputAudioTrack; + assert(track); + + return { input, track }; +};