arcodange/mediabunny
1
0
Fork 0
mirror of https://github.com/arcodange-org/mediabunny.git synced 2026-09-27 10:53:50 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add audio resampling, remixing and processing options to AudioEncodingConfig, small AudioResampler adjustments, add more tests

Browse Source
This commit is contained in:
Vanilagy
2026-04-01 15:46:02 +02:00
parent 1c08589021
commit b86e527dc6
6 changed files with 584 additions and 324 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/procedural-generation/procedural-generation.ts
+7 -4
View File
@@ -147,7 +147,7 @@ const generateVideo = async () => {
bitrate: QUALITY_HIGH,
keyFrameInterval: 2,
transform: {
frameRate: 5,
// frameRate: 5,
},
});
output.addVideoTrack(canvasSource, { frameRate });
@@ -167,8 +167,11 @@ const generateVideo = async () => {
audioBufferSource = new AudioBufferSource({
codec: audioCodec,
bitrate: QUALITY_HIGH,
transform: {
},
});
// output.addAudioTrack(audioBufferSource);
output.addAudioTrack(audioBufferSource);
/*
audioBufferSource2 = new AudioBufferSource({
@@ -219,8 +222,8 @@ const generateVideo = async () => {
// Let's render the audio. Ideally, the audio is rendered before the video (or concurrently to it), but for
// simplicity, we're rendering it after we've cranked through all frames.
const audioBuffer = await audioContext.startRendering();
// await audioBufferSource.add(audioBuffer);
// audioBufferSource.close();
await audioBufferSource.add(audioBuffer);
audioBufferSource.close();
// await audioBufferSource2!.add(audioBuffer);
// audioBufferSource2!.close();
src/conversion.ts
+3 -277
View File
@@ -52,6 +52,7 @@ import { Mp4OutputFormat } from './output-format';
import { AudioSample, clampCropRectangle, validateCropRectangle, VideoSample } from './sample';
import { MetadataTags, validateMetadataTags } from './metadata';
import { NullTarget } from './target';
import { AudioResampler } from './resample';
/**
* The options for media file conversion.
@@ -1611,6 +1612,8 @@ export class Conversion {
startTime: this._startTimestamp,
endTime: this._endTimestamp,
onSample: async (sample) => {
sample.setTimestamp(sample.timestamp - this._startTimestamp);
await this._registerAudioSample(track, trackOptions, source, sample);
sample.close();
},
@@ -1731,280 +1734,3 @@ class TrackSynchronizer {
this.computeMinAndMaybeResolve();
}
}
/**
* Utility class to handle audio resampling, handling both sample rate resampling as well as channel up/downmixing.
* The advantage over doing this manually rather than using OfflineAudioContext to do it for us is the artifact-free
* handling of putting multiple resampled audio samples back to back, which produces flaky results using
* OfflineAudioContext.
*/
export class AudioResampler {
sourceSampleRate: number | null = null;
targetSampleRate: number;
sourceNumberOfChannels: number | null = null;
targetNumberOfChannels: number;
startTime: number;
endTime: number;
onSample: (sample: AudioSample) => Promise<void>;
bufferSizeInFrames: number;
bufferSizeInSamples: number;
outputBuffer: Float32Array;
/** Start frame of current buffer */
bufferStartFrame: number;
/** The highest index written to in the current buffer */
maxWrittenFrame: number;
channelMixer!: (sourceData: Float32Array, sourceFrameIndex: number, targetChannelIndex: number) => number;
tempSourceBuffer!: Float32Array;
constructor(options: {
targetSampleRate: number;
targetNumberOfChannels: number;
startTime: number;
endTime: number;
onSample: (sample: AudioSample) => Promise<void>;
}) {
this.targetSampleRate = options.targetSampleRate;
this.targetNumberOfChannels = options.targetNumberOfChannels;
this.startTime = options.startTime;
this.endTime = options.endTime;
this.onSample = options.onSample;
this.bufferSizeInFrames = Math.floor(this.targetSampleRate * 5.0); // 5 seconds
this.bufferSizeInSamples = this.bufferSizeInFrames * this.targetNumberOfChannels;
this.outputBuffer = new Float32Array(this.bufferSizeInSamples);
this.bufferStartFrame = 0;
this.maxWrittenFrame = -1;
}
/**
* Sets up the channel mixer to handle up/downmixing in the case where input and output channel counts don't match.
*/
doChannelMixerSetup(): void {
assert(this.sourceNumberOfChannels !== null);
const sourceNum = this.sourceNumberOfChannels;
const targetNum = this.targetNumberOfChannels;
// Logic taken from
// https://developer.mozilla.org/en-US/docs/Web/API/Web_Audio_API/Basic_concepts_behind_Web_Audio_API
// Most of the mapping functions are branchless.
if (sourceNum === 1 && targetNum === 2) {
// Mono to Stereo: M -> L, M -> R
this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number) => {
return sourceData[sourceFrameIndex * sourceNum]!;
};
} else if (sourceNum === 1 && targetNum === 4) {
// Mono to Quad: M -> L, M -> R, 0 -> SL, 0 -> SR
this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number, targetChannelIndex: number) => {
return sourceData[sourceFrameIndex * sourceNum]! * +(targetChannelIndex < 2);
};
} else if (sourceNum === 1 && targetNum === 6) {
// Mono to 5.1: 0 -> L, 0 -> R, M -> C, 0 -> LFE, 0 -> SL, 0 -> SR
this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number, targetChannelIndex: number) => {
return sourceData[sourceFrameIndex * sourceNum]! * +(targetChannelIndex === 2);
};
} else if (sourceNum === 2 && targetNum === 1) {
// Stereo to Mono: 0.5 * (L + R)
this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number) => {
const baseIdx = sourceFrameIndex * sourceNum;
return 0.5 * (sourceData[baseIdx]! + sourceData[baseIdx + 1]!);
};
} else if (sourceNum === 2 && targetNum === 4) {
// Stereo to Quad: L -> L, R -> R, 0 -> SL, 0 -> SR
this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number, targetChannelIndex: number) => {
return sourceData[sourceFrameIndex * sourceNum + targetChannelIndex]! * +(targetChannelIndex < 2);
};
} else if (sourceNum === 2 && targetNum === 6) {
// Stereo to 5.1: L -> L, R -> R, 0 -> C, 0 -> LFE, 0 -> SL, 0 -> SR
this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number, targetChannelIndex: number) => {
return sourceData[sourceFrameIndex * sourceNum + targetChannelIndex]! * +(targetChannelIndex < 2);
};
} else if (sourceNum === 4 && targetNum === 1) {
// Quad to Mono: 0.25 * (L + R + SL + SR)
this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number) => {
const baseIdx = sourceFrameIndex * sourceNum;
return 0.25 * (
sourceData[baseIdx]! + sourceData[baseIdx + 1]!
+ sourceData[baseIdx + 2]! + sourceData[baseIdx + 3]!
);
};
} else if (sourceNum === 4 && targetNum === 2) {
// Quad to Stereo: 0.5 * (L + SL), 0.5 * (R + SR)
this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number, targetChannelIndex: number) => {
const baseIdx = sourceFrameIndex * sourceNum;
return 0.5 * (
sourceData[baseIdx + targetChannelIndex]!
+ sourceData[baseIdx + targetChannelIndex + 2]!
);
};
} else if (sourceNum === 4 && targetNum === 6) {
// Quad to 5.1: L -> L, R -> R, 0 -> C, 0 -> LFE, SL -> SL, SR -> SR
this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number, targetChannelIndex: number) => {
const baseIdx = sourceFrameIndex * sourceNum;
// It's a bit harder to do this one branchlessly
if (targetChannelIndex < 2) return sourceData[baseIdx + targetChannelIndex]!; // L, R
if (targetChannelIndex === 2 || targetChannelIndex === 3) return 0; // C, LFE
return sourceData[baseIdx + targetChannelIndex - 2]!; // SL, SR
};
} else if (sourceNum === 6 && targetNum === 1) {
// 5.1 to Mono: sqrt(1/2) * (L + R) + C + 0.5 * (SL + SR)
this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number) => {
const baseIdx = sourceFrameIndex * sourceNum;
return Math.SQRT1_2 * (sourceData[baseIdx]! + sourceData[baseIdx + 1]!)
+ sourceData[baseIdx + 2]!
+ 0.5 * (sourceData[baseIdx + 4]! + sourceData[baseIdx + 5]!);
};
} else if (sourceNum === 6 && targetNum === 2) {
// 5.1 to Stereo: L + sqrt(1/2) * (C + SL), R + sqrt(1/2) * (C + SR)
this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number, targetChannelIndex: number) => {
const baseIdx = sourceFrameIndex * sourceNum;
return sourceData[baseIdx + targetChannelIndex]!
+ Math.SQRT1_2 * (sourceData[baseIdx + 2]! + sourceData[baseIdx + targetChannelIndex + 4]!);
};
} else if (sourceNum === 6 && targetNum === 4) {
// 5.1 to Quad: L + sqrt(1/2) * C, R + sqrt(1/2) * C, SL, SR
this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number, targetChannelIndex: number) => {
const baseIdx = sourceFrameIndex * sourceNum;
// It's a bit harder to do this one branchlessly
if (targetChannelIndex < 2) {
return sourceData[baseIdx + targetChannelIndex]! + Math.SQRT1_2 * sourceData[baseIdx + 2]!;
}
return sourceData[baseIdx + targetChannelIndex + 2]!; // SL, SR
};
} else {
// Discrete fallback: direct mapping with zero-fill or drop
this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number, targetChannelIndex: number) => {
return targetChannelIndex < sourceNum
? sourceData[sourceFrameIndex * sourceNum + targetChannelIndex]!
: 0;
};
}
}
ensureTempBufferSize(requiredSamples: number): void {
let length = this.tempSourceBuffer.length;
while (length < requiredSamples) {
length *= 2;
}
if (length !== this.tempSourceBuffer.length) {
const newBuffer = new Float32Array(length);
newBuffer.set(this.tempSourceBuffer);
this.tempSourceBuffer = newBuffer;
}
}
async add(audioSample: AudioSample) {
if (this.sourceSampleRate === null) {
// This is the first sample, so let's init the missing data. Initting the sample rate from the decoded
// sample is more reliable than using the file's metadata, because decoders are free to emit any sample rate
// they see fit.
this.sourceSampleRate = audioSample.sampleRate;
this.sourceNumberOfChannels = audioSample.numberOfChannels;
// Pre-allocate temporary buffer for source data
this.tempSourceBuffer = new Float32Array(this.sourceSampleRate * this.sourceNumberOfChannels);
this.doChannelMixerSetup();
}
const requiredSamples = audioSample.numberOfFrames * audioSample.numberOfChannels;
this.ensureTempBufferSize(requiredSamples);
// Copy the audio data to the temp buffer
const sourceDataSize = audioSample.allocationSize({ planeIndex: 0, format: 'f32' });
const sourceView = new Float32Array(this.tempSourceBuffer.buffer, 0, sourceDataSize / 4);
audioSample.copyTo(sourceView, { planeIndex: 0, format: 'f32' });
const inputStartTime = audioSample.timestamp - this.startTime;
const inputDuration = audioSample.numberOfFrames / this.sourceSampleRate;
const inputEndTime = Math.min(inputStartTime + inputDuration, this.endTime - this.startTime);
// Compute which output frames are affected by this sample
const outputStartFrame = Math.floor(inputStartTime * this.targetSampleRate);
const outputEndFrame = Math.ceil(inputEndTime * this.targetSampleRate);
for (let outputFrame = outputStartFrame; outputFrame < outputEndFrame; outputFrame++) {
if (outputFrame < this.bufferStartFrame) {
continue; // Skip writes to the past
}
while (outputFrame >= this.bufferStartFrame + this.bufferSizeInFrames) {
// The write is after the current buffer, so finalize it
await this.finalizeCurrentBuffer();
this.bufferStartFrame += this.bufferSizeInFrames;
}
const bufferFrameIndex = outputFrame - this.bufferStartFrame;
assert(bufferFrameIndex < this.bufferSizeInFrames);
const outputTime = outputFrame / this.targetSampleRate;
const inputTime = outputTime - inputStartTime;
const sourcePosition = inputTime * this.sourceSampleRate;
const sourceLowerFrame = Math.floor(sourcePosition);
const sourceUpperFrame = Math.ceil(sourcePosition);
const fraction = sourcePosition - sourceLowerFrame;
// Process each output channel
for (let targetChannel = 0; targetChannel < this.targetNumberOfChannels; targetChannel++) {
let lowerSample = 0;
let upperSample = 0;
if (sourceLowerFrame >= 0 && sourceLowerFrame < audioSample.numberOfFrames) {
lowerSample = this.channelMixer(sourceView, sourceLowerFrame, targetChannel);
}
if (sourceUpperFrame >= 0 && sourceUpperFrame < audioSample.numberOfFrames) {
upperSample = this.channelMixer(sourceView, sourceUpperFrame, targetChannel);
}
// For resampling, we do naive linear interpolation to find the in-between sample. This produces
// suboptimal results especially for downsampling (for which a low-pass filter would first need to be
// applied), but AudioContext doesn't do this either, so, whatever, for now.
const outputSample = lowerSample + fraction * (upperSample - lowerSample);
// Write to output buffer (interleaved)
const outputIndex = bufferFrameIndex * this.targetNumberOfChannels + targetChannel;
this.outputBuffer[outputIndex]! += outputSample; // Add in case of overlapping samples
}
this.maxWrittenFrame = Math.max(this.maxWrittenFrame, bufferFrameIndex);
}
}
async finalizeCurrentBuffer() {
if (this.maxWrittenFrame < 0) {
return; // Nothing to finalize
}
const samplesWritten = (this.maxWrittenFrame + 1) * this.targetNumberOfChannels;
const outputData = new Float32Array(samplesWritten);
outputData.set(this.outputBuffer.subarray(0, samplesWritten));
const timestampSeconds = this.bufferStartFrame / this.targetSampleRate;
const audioSample = new AudioSample({
format: 'f32',
sampleRate: this.targetSampleRate,
numberOfChannels: this.targetNumberOfChannels,
timestamp: timestampSeconds,
data: outputData,
});
await this.onSample(audioSample);
this.outputBuffer.fill(0);
this.maxWrittenFrame = -1;
}
finalize() {
return this.finalizeCurrentBuffer();
}
}
src/encode.ts
+40 -1
View File
@@ -24,7 +24,7 @@ import {
import { customAudioEncoders, customVideoEncoders } from './custom-coder';
import { isFirefox, MaybePromise, Rotation } from './misc';
import { EncodedPacket } from './packet';
import { CropRectangle, validateCropRectangle, VideoSample } from './sample';
import { AudioSample, CropRectangle, validateCropRectangle, VideoSample } from './sample';
const canEncodeVideoMemo = new Map<string, Promise<boolean>>();
const canEncodeAudioMemo = new Map<string, Promise<boolean>>();
@@ -355,6 +355,25 @@ export type AudioEncodingConfig = {
*/
bitrate?: number | Quality;
/**
* Optional transformations to apply to the audio samples before they are passed to the encoder.
*/
transform?: {
/** The desired number of output channels to up/downmix to. */
numberOfChannels?: number;
/** The desired output sample rate in hertz to resample to. */
sampleRate?: number;
/**
* Allows for custom user-defined processing of audio samples, e.g. for applying audio effects or timestamp
* modifications. Called for each audio sample after resampling and remixing.
*
* Must return an {@link AudioSample}, an array of them, or `null` for dropping the sample.
*/
process?: (sample: AudioSample) => MaybePromise<
AudioSample | AudioSample[] | null
>;
};
/** Called for each successfully encoded packet. Both the packet and the encoding metadata are passed. */
onEncodedPacket?: (packet: EncodedPacket, meta: EncodedAudioChunkMetadata | undefined) => unknown;
/**
@@ -384,6 +403,26 @@ export const validateAudioEncodingConfig = (config: AudioEncodingConfig) => {
) {
throw new TypeError('config.bitrate, when provided, must be a positive integer or a quality.');
}
if (config.transform !== undefined) {
if (typeof config.transform !== 'object' || !config.transform) {
throw new TypeError('config.transform, when provided, must be an object.');
}
if (
config.transform.numberOfChannels !== undefined
&& (!Number.isInteger(config.transform.numberOfChannels) || config.transform.numberOfChannels <= 0)
) {
throw new TypeError('config.transform.numberOfChannels, when provided, must be a positive integer.');
}
if (
config.transform.sampleRate !== undefined
&& (!Number.isInteger(config.transform.sampleRate) || config.transform.sampleRate <= 0)
) {
throw new TypeError('config.transform.sampleRate, when provided, must be a positive integer.');
}
if (config.transform.process !== undefined && typeof config.transform.process !== 'function') {
throw new TypeError('config.transform.process, when provided, must be a function.');
}
}
if (config.onEncodedPacket !== undefined && typeof config.onEncodedPacket !== 'function') {
throw new TypeError('config.onEncodedChunk, when provided, must be a function.');
}
src/media-source.ts
+85 -5
View File
@@ -59,6 +59,7 @@ import {
validateVideoEncodingConfig,
VideoEncodingConfig,
} from './encode';
import { AudioResampler } from './resample';
/**
* Base class for media sources. Media sources are used to add media samples to an output file.
@@ -1778,6 +1779,8 @@ class AudioEncoderWrapper {
private lastEndSampleIndex: number | null = null;
private resampler: AudioResampler | null = null;
/**
* Encoders typically throw their errors "out of band", meaning asynchronously in some other execution context.
* However, we want to surface these errors to the user within the normal control flow, so they don't go uncaught.
@@ -1809,6 +1812,75 @@ class AudioEncoderWrapper {
this.lastSampleRate = audioSample.sampleRate;
}
const config = this.encodingConfig;
const needsResample = config.transform?.numberOfChannels !== undefined
|| config.transform?.sampleRate !== undefined;
if (needsResample) {
if (!this.resampler) {
// Initialize the resampler on first sample
this.resampler = new AudioResampler({
targetNumberOfChannels: config.transform!.numberOfChannels
?? audioSample.numberOfChannels,
targetSampleRate: config.transform!.sampleRate
?? audioSample.sampleRate,
startTime: audioSample.timestamp,
endTime: Infinity,
onSample: async (sample) => {
await this.processAndEncode(sample, true);
},
});
}
await this.resampler.add(audioSample);
} else {
await this.processAndEncode(audioSample, shouldClose);
}
} finally {
if (shouldClose) {
audioSample.close();
}
}
}
/**
* Runs the process function (if any) and encodes the resulting samples.
*/
private async processAndEncode(audioSample: AudioSample, shouldClose: boolean) {
const config = this.encodingConfig;
if (config.transform?.process) {
let processed = config.transform.process(audioSample);
if (processed instanceof Promise) {
processed = await processed;
}
if (processed === null) {
return;
}
if (!Array.isArray(processed)) {
processed = [processed];
}
for (const sample of processed) {
if (!(sample instanceof AudioSample)) {
throw new TypeError(
'The audio process function must return an AudioSample, null, or an array of AudioSamples.',
);
}
await this.encodeSample(sample, true);
}
} else {
await this.encodeSample(audioSample, shouldClose);
}
}
/**
* Encodes a single audio sample, handling encoder init, gap padding, and backpressure.
*/
private async encodeSample(audioSample: AudioSample, shouldClose: boolean) {
try {
if (!this.encoderInitialized) {
if (!this.ensureEncoderPromise) {
this.ensureEncoder(audioSample);
@@ -1853,7 +1925,7 @@ class AudioEncoderWrapper {
timestamp: this.lastEndSampleIndex / audioSample.sampleRate,
});
await this.add(fillSample, true); // Recursive call
await this.encodeSample(fillSample, true); // Recursive call
}
this.lastEndSampleIndex += audioSample.numberOfFrames;
@@ -1872,7 +1944,6 @@ class AudioEncoderWrapper {
.catch((error: Error) => this.error ??= error)
.finally(() => {
clonedSample.close();
// `audioSample` gets closed in the finally block at the end of the method
});
if (this.customEncoderQueueSize >= 4) {
@@ -1900,7 +1971,6 @@ class AudioEncoderWrapper {
}
} finally {
if (shouldClose) {
// Make sure it's always closed, even if there was an error
audioSample.close();
}
}
@@ -2189,7 +2259,15 @@ class AudioEncoderWrapper {
}
async flushAndClose(forceClose: boolean) {
if (!forceClose) this.checkForEncoderError();
if (!forceClose) {
this.checkForEncoderError();
}
// Finalize the resampler to flush any buffered audio
if (!forceClose && this.resampler) {
await this.resampler.finalize();
}
this.resampler = null;
if (this.customEncoder) {
if (!forceClose) {
@@ -2207,7 +2285,9 @@ class AudioEncoderWrapper {
}
}
if (!forceClose) this.checkForEncoderError();
if (!forceClose) {
this.checkForEncoderError();
}
}
getQueueSize() {
src/resample.ts
+281
View File
@@ -0,0 +1,281 @@
import { assert } from './misc';
import { AudioSample } from './sample';
/**
* Utility class to handle audio resampling, handling both sample rate resampling as well as channel up/downmixing.
* The advantage over doing this manually rather than using OfflineAudioContext to do it for us is the artifact-free
* handling of putting multiple resampled audio samples back to back, which produces flaky results using
* OfflineAudioContext.
*/
export class AudioResampler {
sourceSampleRate: number | null = null;
targetSampleRate: number;
sourceNumberOfChannels: number | null = null;
targetNumberOfChannels: number;
startTime: number;
endTime: number;
onSample: (sample: AudioSample) => Promise<void>;
bufferSizeInFrames: number;
bufferSizeInSamples: number;
outputBuffer: Float32Array;
/** Start frame of current buffer */
bufferStartFrame: number;
/** The highest index written to in the current buffer */
maxWrittenFrame: number | null = null;
channelMixer!: (sourceData: Float32Array, sourceFrameIndex: number, targetChannelIndex: number) => number;
tempSourceBuffer!: Float32Array;
constructor(options: {
targetSampleRate: number;
targetNumberOfChannels: number;
startTime: number;
endTime: number;
onSample: (sample: AudioSample) => Promise<void>;
}) {
this.targetSampleRate = options.targetSampleRate;
this.targetNumberOfChannels = options.targetNumberOfChannels;
this.startTime = options.startTime;
this.endTime = options.endTime;
this.onSample = options.onSample;
this.bufferSizeInFrames = Math.floor(this.targetSampleRate * 5.0); // 5 seconds
this.bufferSizeInSamples = this.bufferSizeInFrames * this.targetNumberOfChannels;
this.outputBuffer = new Float32Array(this.bufferSizeInSamples);
this.bufferStartFrame = Math.floor(this.startTime * this.targetSampleRate);
}
/**
* Sets up the channel mixer to handle up/downmixing in the case where input and output channel counts don't match.
*/
doChannelMixerSetup(): void {
assert(this.sourceNumberOfChannels !== null);
const sourceNum = this.sourceNumberOfChannels;
const targetNum = this.targetNumberOfChannels;
// Logic taken from
// https://developer.mozilla.org/en-US/docs/Web/API/Web_Audio_API/Basic_concepts_behind_Web_Audio_API
// Most of the mapping functions are branchless.
if (sourceNum === 1 && targetNum === 2) {
// Mono to Stereo: M -> L, M -> R
this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number) => {
return sourceData[sourceFrameIndex * sourceNum]!;
};
} else if (sourceNum === 1 && targetNum === 4) {
// Mono to Quad: M -> L, M -> R, 0 -> SL, 0 -> SR
this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number, targetChannelIndex: number) => {
return sourceData[sourceFrameIndex * sourceNum]! * +(targetChannelIndex < 2);
};
} else if (sourceNum === 1 && targetNum === 6) {
// Mono to 5.1: 0 -> L, 0 -> R, M -> C, 0 -> LFE, 0 -> SL, 0 -> SR
this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number, targetChannelIndex: number) => {
return sourceData[sourceFrameIndex * sourceNum]! * +(targetChannelIndex === 2);
};
} else if (sourceNum === 2 && targetNum === 1) {
// Stereo to Mono: 0.5 * (L + R)
this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number) => {
const baseIdx = sourceFrameIndex * sourceNum;
return 0.5 * (sourceData[baseIdx]! + sourceData[baseIdx + 1]!);
};
} else if (sourceNum === 2 && targetNum === 4) {
// Stereo to Quad: L -> L, R -> R, 0 -> SL, 0 -> SR
this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number, targetChannelIndex: number) => {
return sourceData[sourceFrameIndex * sourceNum + targetChannelIndex]! * +(targetChannelIndex < 2);
};
} else if (sourceNum === 2 && targetNum === 6) {
// Stereo to 5.1: L -> L, R -> R, 0 -> C, 0 -> LFE, 0 -> SL, 0 -> SR
this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number, targetChannelIndex: number) => {
return sourceData[sourceFrameIndex * sourceNum + targetChannelIndex]! * +(targetChannelIndex < 2);
};
} else if (sourceNum === 4 && targetNum === 1) {
// Quad to Mono: 0.25 * (L + R + SL + SR)
this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number) => {
const baseIdx = sourceFrameIndex * sourceNum;
return 0.25 * (
sourceData[baseIdx]! + sourceData[baseIdx + 1]!
+ sourceData[baseIdx + 2]! + sourceData[baseIdx + 3]!
);
};
} else if (sourceNum === 4 && targetNum === 2) {
// Quad to Stereo: 0.5 * (L + SL), 0.5 * (R + SR)
this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number, targetChannelIndex: number) => {
const baseIdx = sourceFrameIndex * sourceNum;
return 0.5 * (
sourceData[baseIdx + targetChannelIndex]!
+ sourceData[baseIdx + targetChannelIndex + 2]!
);
};
} else if (sourceNum === 4 && targetNum === 6) {
// Quad to 5.1: L -> L, R -> R, 0 -> C, 0 -> LFE, SL -> SL, SR -> SR
this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number, targetChannelIndex: number) => {
const baseIdx = sourceFrameIndex * sourceNum;
// It's a bit harder to do this one branchlessly
if (targetChannelIndex < 2) return sourceData[baseIdx + targetChannelIndex]!; // L, R
if (targetChannelIndex === 2 || targetChannelIndex === 3) return 0; // C, LFE
return sourceData[baseIdx + targetChannelIndex - 2]!; // SL, SR
};
} else if (sourceNum === 6 && targetNum === 1) {
// 5.1 to Mono: sqrt(1/2) * (L + R) + C + 0.5 * (SL + SR)
this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number) => {
const baseIdx = sourceFrameIndex * sourceNum;
return Math.SQRT1_2 * (sourceData[baseIdx]! + sourceData[baseIdx + 1]!)
+ sourceData[baseIdx + 2]!
+ 0.5 * (sourceData[baseIdx + 4]! + sourceData[baseIdx + 5]!);
};
} else if (sourceNum === 6 && targetNum === 2) {
// 5.1 to Stereo: L + sqrt(1/2) * (C + SL), R + sqrt(1/2) * (C + SR)
this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number, targetChannelIndex: number) => {
const baseIdx = sourceFrameIndex * sourceNum;
return sourceData[baseIdx + targetChannelIndex]!
+ Math.SQRT1_2 * (sourceData[baseIdx + 2]! + sourceData[baseIdx + targetChannelIndex + 4]!);
};
} else if (sourceNum === 6 && targetNum === 4) {
// 5.1 to Quad: L + sqrt(1/2) * C, R + sqrt(1/2) * C, SL, SR
this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number, targetChannelIndex: number) => {
const baseIdx = sourceFrameIndex * sourceNum;
// It's a bit harder to do this one branchlessly
if (targetChannelIndex < 2) {
return sourceData[baseIdx + targetChannelIndex]! + Math.SQRT1_2 * sourceData[baseIdx + 2]!;
}
return sourceData[baseIdx + targetChannelIndex + 2]!; // SL, SR
};
} else {
// Discrete fallback: direct mapping with zero-fill or drop
this.channelMixer = (sourceData: Float32Array, sourceFrameIndex: number, targetChannelIndex: number) => {
return targetChannelIndex < sourceNum
? sourceData[sourceFrameIndex * sourceNum + targetChannelIndex]!
: 0;
};
}
}
ensureTempBufferSize(requiredSamples: number): void {
let length = this.tempSourceBuffer.length;
while (length < requiredSamples) {
length *= 2;
}
if (length !== this.tempSourceBuffer.length) {
const newBuffer = new Float32Array(length);
newBuffer.set(this.tempSourceBuffer);
this.tempSourceBuffer = newBuffer;
}
}
async add(audioSample: AudioSample) {
if (this.sourceSampleRate === null) {
// This is the first sample, so let's init the missing data. Initting the sample rate from the decoded
// sample is more reliable than using the file's metadata, because decoders are free to emit any sample rate
// they see fit.
this.sourceSampleRate = audioSample.sampleRate;
this.sourceNumberOfChannels = audioSample.numberOfChannels;
// Pre-allocate temporary buffer for source data
this.tempSourceBuffer = new Float32Array(this.sourceSampleRate * this.sourceNumberOfChannels);
this.doChannelMixerSetup();
}
const requiredSamples = audioSample.numberOfFrames * audioSample.numberOfChannels;
this.ensureTempBufferSize(requiredSamples);
// Copy the audio data to the temp buffer
const sourceDataSize = audioSample.allocationSize({ planeIndex: 0, format: 'f32' });
const sourceView = new Float32Array(this.tempSourceBuffer.buffer, 0, sourceDataSize / 4);
audioSample.copyTo(sourceView, { planeIndex: 0, format: 'f32' });
const inputStartTime = audioSample.timestamp;
const inputEndTime = Math.min(audioSample.timestamp + audioSample.duration, this.endTime);
// Compute which output frames are affected by this sample
const outputStartFrame = Math.floor(inputStartTime * this.targetSampleRate);
const outputEndFrame = Math.ceil(inputEndTime * this.targetSampleRate);
for (let outputFrame = outputStartFrame; outputFrame < outputEndFrame; outputFrame++) {
if (outputFrame < this.bufferStartFrame) {
continue; // Skip writes to the past
}
while (outputFrame >= this.bufferStartFrame + this.bufferSizeInFrames) {
// The write is after the current buffer, so finalize it
await this.finalizeCurrentBuffer();
this.bufferStartFrame += this.bufferSizeInFrames;
}
const bufferFrameIndex = outputFrame - this.bufferStartFrame;
assert(bufferFrameIndex < this.bufferSizeInFrames);
const outputTime = outputFrame / this.targetSampleRate;
const inputTime = outputTime - inputStartTime;
const sourcePosition = inputTime * this.sourceSampleRate;
const sourceLowerFrame = Math.floor(sourcePosition);
const sourceUpperFrame = Math.ceil(sourcePosition);
const fraction = sourcePosition - sourceLowerFrame;
// Process each output channel
for (let targetChannel = 0; targetChannel < this.targetNumberOfChannels; targetChannel++) {
let lowerSample = 0;
let upperSample = 0;
if (sourceLowerFrame >= 0 && sourceLowerFrame < audioSample.numberOfFrames) {
lowerSample = this.channelMixer(sourceView, sourceLowerFrame, targetChannel);
}
if (sourceUpperFrame >= 0 && sourceUpperFrame < audioSample.numberOfFrames) {
upperSample = this.channelMixer(sourceView, sourceUpperFrame, targetChannel);
}
// For resampling, we do naive linear interpolation to find the in-between sample. This produces
// suboptimal results especially for downsampling (for which a low-pass filter would first need to be
// applied), but AudioContext doesn't do this either, so, whatever, for now.
const outputSample = lowerSample + fraction * (upperSample - lowerSample);
// Write to output buffer (interleaved)
const outputIndex = bufferFrameIndex * this.targetNumberOfChannels + targetChannel;
this.outputBuffer[outputIndex]! += outputSample; // Add in case of overlapping samples
}
if (this.maxWrittenFrame === null) {
this.maxWrittenFrame = bufferFrameIndex;
} else {
this.maxWrittenFrame = Math.max(this.maxWrittenFrame, bufferFrameIndex);
}
}
}
async finalizeCurrentBuffer() {
if (this.maxWrittenFrame === null) {
return; // Nothing to finalize
}
const samplesWritten = (this.maxWrittenFrame + 1) * this.targetNumberOfChannels;
const outputData = new Float32Array(samplesWritten);
outputData.set(this.outputBuffer.subarray(0, samplesWritten));
const timestampSeconds = this.bufferStartFrame / this.targetSampleRate;
const audioSample = new AudioSample({
format: 'f32',
sampleRate: this.targetSampleRate,
numberOfChannels: this.targetNumberOfChannels,
timestamp: timestampSeconds,
data: outputData,
});
await this.onSample(audioSample);
this.outputBuffer.fill(0);
this.maxWrittenFrame = null;
}
finalize() {
return this.finalizeCurrentBuffer();
}
}
test/browser/media-sources.test.ts
+158 -27
View File
@@ -2,17 +2,32 @@ import { expect, test } from 'vitest';
import { Output } from '../../src/output.js';
import { Mp4OutputFormat, WebMOutputFormat } from '../../src/output-format.js';
import { BufferTarget } from '../../src/target.js';
import { VideoSampleSource } from '../../src/media-source.js';
import { VideoSample } from '../../src/sample.js';
import { AudioSampleSource, VideoSampleSource } from '../../src/media-source.js';
import { AudioSample, VideoSample } from '../../src/sample.js';
import { QUALITY_MEDIUM } from '../../src/encode.js';
import { Input } from '../../src/input.js';
import { ALL_FORMATS } from '../../src/input-format.js';
import { BufferSource } from '../../src/source.js';
import { VideoSampleSink } from '../../src/media-sink.js';
import { assert, Rotation } from '../../src/misc.js';
import { InputVideoTrack } from '../../src/input-track.js';
import { InputAudioTrack, InputVideoTrack } from '../../src/input-track.js';
test('VideoSampleSource.close() should be idempotent after finalize()', async () => {
test('VideoSampleSource, normal usage', async () => {
const buffer = await encodeFrames(
{ codec: 'vp8', bitrate: QUALITY_MEDIUM },
[{ width: 100, height: 100 }, { width: 100, height: 100 }, { width: 100, height: 100 }],
);
const samples = await readBackSamples(buffer);
expect(samples).toHaveLength(3);
for (const sample of samples) {
expect(sample.codedWidth).toBe(100);
expect(sample.codedHeight).toBe(100);
}
});
test('VideoSampleSource, .close() should be idempotent after finalize()', async () => {
const output = new Output({
format: new WebMOutputFormat(),
target: new BufferTarget(),
@@ -40,7 +55,7 @@ test('VideoSampleSource.close() should be idempotent after finalize()', async ()
videoSource.close(); // This previously threw
});
test('Changing input dimensions throws with deny (default)', async () => {
test('VideoSampleSource, changing input dimensions throws with deny (default)', async () => {
const output = new Output({
format: new Mp4OutputFormat(),
target: new BufferTarget(),
@@ -65,7 +80,7 @@ test('Changing input dimensions throws with deny (default)', async () => {
videoSource.close();
});
test('Changing input dimensions with passThrough preserves per-frame dimensions', async () => {
test('VideoSampleSource, changing input dimensions with passThrough preserves per-frame dimensions', async () => {
const buffer = await encodeFrames(
{ codec: 'vp8', bitrate: QUALITY_MEDIUM, sizeChangeBehavior: 'passThrough' },
[{ width: 100, height: 100 }, { width: 200, height: 150 }],
@@ -81,7 +96,9 @@ test('Changing input dimensions with passThrough preserves per-frame dimensions'
).toBe(true);
});
test('Changing input dimensions with fill/contain/cover locks output to first frame dimensions', async () => {
test(
'VideoSampleSource, changing input dimensions with fill/contain/cover locks output to first frame dimensions',
async () => {
for (const behavior of ['fill', 'contain', 'cover'] as const) {
const buffer = await encodeFrames(
{ codec: 'vp8', bitrate: QUALITY_MEDIUM, sizeChangeBehavior: behavior },
@@ -93,9 +110,10 @@ test('Changing input dimensions with fill/contain/cover locks output to first fr
expect(track.codedHeight).toBe(100);
input.dispose();
}
});
},
);
test('Same-sized frames with width and height set', async () => {
test('VideoSampleSource, same-sized frames with width and height set', async () => {
const buffer = await encodeFrames(
{ codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { width: 50, height: 80, fit: 'fill' } },
[{ width: 100, height: 100 }, { width: 100, height: 100 }],
@@ -107,7 +125,7 @@ test('Same-sized frames with width and height set', async () => {
input.dispose();
});
test('Same-sized frames with rotation set to 90', async () => {
test('VideoSampleSource, same-sized frames with rotation set to 90', async () => {
const buffer = await encodeFrames(
{ codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { rotate: 90 } },
[{ width: 200, height: 100 }, { width: 200, height: 100 }],
@@ -119,7 +137,7 @@ test('Same-sized frames with rotation set to 90', async () => {
input.dispose();
});
test('Same-sized frames with rotation, width and height', async () => {
test('VideoSampleSource, same-sized frames with rotation, width and height', async () => {
const buffer = await encodeFrames(
{ codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { rotate: 90, width: 50, height: 80, fit: 'contain' } },
[{ width: 200, height: 100 }, { width: 200, height: 100 }],
@@ -131,7 +149,7 @@ test('Same-sized frames with rotation, width and height', async () => {
input.dispose();
});
test('Changing dimensions with passThrough and rotation 90', async () => {
test('VideoSampleSource, changing dimensions with passThrough and rotation 90', async () => {
const buffer = await encodeFrames(
{ codec: 'vp8', bitrate: QUALITY_MEDIUM, sizeChangeBehavior: 'passThrough', transform: { rotate: 90 } },
[{ width: 200, height: 100 }, { width: 300, height: 150 }],
@@ -147,7 +165,7 @@ test('Changing dimensions with passThrough and rotation 90', async () => {
).toBe(true);
});
test('Changing dimensions with passThrough, width and height set', async () => {
test('VideoSampleSource, changing dimensions with passThrough, width and height set', async () => {
const buffer = await encodeFrames(
{
codec: 'vp8',
@@ -169,7 +187,7 @@ test('Changing dimensions with passThrough, width and height set', async () => {
expect(samples[1]).toMatchObject({ codedWidth: 50, codedHeight: 80 });
});
test('Encoding rotated video frames', async () => {
test('VideoSampleSource, encoding rotated video frames', async () => {
const buffer = await encodeFrames(
{ codec: 'vp8', bitrate: QUALITY_MEDIUM },
[{ width: 200, height: 100, rotation: 90 }, { width: 200, height: 100, rotation: 90 }],
@@ -184,7 +202,7 @@ test('Encoding rotated video frames', async () => {
}
});
test('Encoding rotated video frames with forced transform', async () => {
test('VideoSampleSource, encoding rotated video frames with forced transform', async () => {
const buffer = await encodeFrames(
{ codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { force: true } },
[{ width: 200, height: 100, rotation: 90 }, { width: 200, height: 100, rotation: 90 }],
@@ -199,7 +217,7 @@ test('Encoding rotated video frames with forced transform', async () => {
}
});
test('transform.process identity function', async () => {
test('VideoSampleSource, transform.process identity function', async () => {
const buffer = await encodeFrames(
{ codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { process: sample => sample } },
[{ width: 100, height: 100 }, { width: 100, height: 100 }, { width: 100, height: 100 }],
@@ -213,7 +231,7 @@ test('transform.process identity function', async () => {
}
});
test('transform.process manual resize', async () => {
test('VideoSampleSource, transform.process manual resize', async () => {
const buffer = await encodeFrames(
{
codec: 'vp8',
@@ -241,7 +259,7 @@ test('transform.process manual resize', async () => {
input.dispose();
});
test('transform.process receives pre-transformed frames', async () => {
test('VideoSampleSource, transform.process receives pre-transformed frames', async () => {
const receivedDimensions: { width: number; height: number }[] = [];
const buffer = await encodeFrames(
@@ -276,7 +294,7 @@ test('transform.process receives pre-transformed frames', async () => {
input.dispose();
});
test('transform.process drops all frames after the first', async () => {
test('VideoSampleSource, transform.process drops all frames after the first', async () => {
let frameIndex = 0;
const buffer = await encodeFrames(
@@ -299,7 +317,7 @@ test('transform.process drops all frames after the first', async () => {
expect(samples).toHaveLength(1);
});
test('transform.process expands every frame into two', async () => {
test('VideoSampleSource, transform.process expands every frame into two', async () => {
const buffer = await encodeFrames(
{
codec: 'vp8',
@@ -331,7 +349,7 @@ test('transform.process expands every frame into two', async () => {
}
});
test('transform.frameRate normalizes variable-rate input to fixed rate', async () => {
test('VideoSampleSource, transform.frameRate normalizes variable-rate input to fixed rate', async () => {
const buffer = await encodeFrames(
{ codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { frameRate: 10 } },
[
@@ -349,7 +367,7 @@ test('transform.frameRate normalizes variable-rate input to fixed rate', async (
}
});
test('transform.frameRate pads gaps by repeating last frame', async () => {
test('VideoSampleSource, transform.frameRate pads gaps by repeating last frame', async () => {
const buffer = await encodeFrames(
{ codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { frameRate: 10 } },
[
@@ -366,7 +384,7 @@ test('transform.frameRate pads gaps by repeating last frame', async () => {
}
});
test('transform.frameRate deduplicates frames in the same slot', async () => {
test('VideoSampleSource, transform.frameRate deduplicates frames in the same slot', async () => {
const buffer = await encodeFrames(
{ codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { frameRate: 10 } },
[
@@ -383,7 +401,7 @@ test('transform.frameRate deduplicates frames in the same slot', async () => {
expect(samples[1]!.timestamp).toBeCloseTo(0.1);
});
test('transform.frameRate final padding fills remaining duration', async () => {
test('VideoSampleSource, transform.frameRate final padding fills remaining duration', async () => {
const buffer = await encodeFrames(
{ codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { frameRate: 10 } },
[
@@ -398,7 +416,7 @@ test('transform.frameRate final padding fills remaining duration', async () => {
}
});
test('transform.frameRate skipping and padding combined', async () => {
test('VideoSampleSource, transform.frameRate skipping and padding combined', async () => {
const buffer = await encodeFrames(
{ codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { frameRate: 10 } },
[
@@ -418,7 +436,7 @@ test('transform.frameRate skipping and padding combined', async () => {
}
});
test('transform.frameRate works with transform', async () => {
test('VideoSampleSource, transform.frameRate works with transform', async () => {
const buffer = await encodeFrames(
{
codec: 'vp8',
@@ -440,7 +458,7 @@ test('transform.frameRate works with transform', async () => {
}
});
test('transform.frameRate works with process', async () => {
test('VideoSampleSource, transform.frameRate works with process', async () => {
const processedTimestamps: number[] = [];
const buffer = await encodeFrames(
@@ -552,3 +570,116 @@ const readBackSamples = async (buffer: ArrayBuffer) => {
input.dispose();
return samples;
};
test('AudioSampleSource, normal usage', async () => {
const sample = makeSineWave(48000, 2, 1);
const buffer = await encodeAudio({ codec: 'pcm-s16' }, sample);
const { input, track } = await readBackAudioTrack(buffer);
expect(track.numberOfChannels).toBe(2);
expect(track.sampleRate).toBe(48000);
expect(await track.computeDuration()).toBe(1);
input.dispose();
});
test('AudioSampleSource, remixed to mono', async () => {
const sample = makeSineWave(48000, 2, 1);
const buffer = await encodeAudio(
{ codec: 'pcm-s16', transform: { numberOfChannels: 1 } },
sample,
);
const { input, track } = await readBackAudioTrack(buffer);
expect(track.numberOfChannels).toBe(1);
expect(track.sampleRate).toBe(48000);
expect(await track.computeDuration()).toBe(1);
input.dispose();
});
test('AudioSampleSource, resampled to 44100 Hz', async () => {
const sample = makeSineWave(48000, 2, 1);
const buffer = await encodeAudio(
{ codec: 'pcm-s16', transform: { sampleRate: 44100 } },
sample,
);
const { input, track } = await readBackAudioTrack(buffer);
expect(track.numberOfChannels).toBe(2);
expect(track.sampleRate).toBe(44100);
expect(await track.computeDuration()).toBe(1);
input.dispose();
});
test('AudioSampleSource, resampled stereo with non-zero start timestamp', async () => {
const sample = makeSineWave(48000, 2, 1, 1);
const buffer = await encodeAudio(
{ codec: 'pcm-s16', transform: { numberOfChannels: 2 } },
sample,
);
const { input, track } = await readBackAudioTrack(buffer);
expect(track.numberOfChannels).toBe(2);
expect(track.sampleRate).toBe(48000);
expect(await track.getFirstTimestamp()).toBe(1);
expect(await track.computeDuration()).toBe(2);
input.dispose();
});
const makeSineWave = (
sampleRate: number,
numberOfChannels: number,
durationSeconds: number,
timestamp = 0,
) => {
const numberOfFrames = Math.round(sampleRate * durationSeconds);
const data = new Float32Array(numberOfFrames * numberOfChannels);
for (let frame = 0; frame < numberOfFrames; frame++) {
const value = Math.sin(2 * Math.PI * 440 * frame / sampleRate);
for (let ch = 0; ch < numberOfChannels; ch++) {
data[frame * numberOfChannels + ch] = value;
}
}
return new AudioSample({
data,
format: 'f32-planar',
sampleRate,
numberOfChannels,
numberOfFrames,
timestamp,
});
};
const encodeAudio = async (
encodingConfig: ConstructorParameters<typeof AudioSampleSource>[0],
sample: AudioSample,
) => {
const output = new Output({
format: new Mp4OutputFormat({ fastStart: 'fragmented' }), // Fragmented to avoid the PCM transformation
target: new BufferTarget(),
});
const audioSource = new AudioSampleSource(encodingConfig);
output.addAudioTrack(audioSource);
await output.start();
await audioSource.add(sample);
sample.close();
await output.finalize();
return output.target.buffer!;
};
const readBackAudioTrack = async (buffer: ArrayBuffer) => {
const input = new Input({
source: new BufferSource(buffer),
formats: ALL_FORMATS,
});
const track = await input.getPrimaryAudioTrack() as InputAudioTrack;
assert(track);
return { input, track };
};