From 8b9a1acf161900d80ac436205be8ecd540bb2c84 Mon Sep 17 00:00:00 2001 From: Vanilagy <1696106+Vanilagy@users.noreply.github.com> Date: Mon, 15 Jun 2026 18:30:16 +0200 Subject: [PATCH] Fix Conversion API process callbacks being called *before* other transformations (fixes #403), restructure Conversion API to make full use of in-source transformations, add AudioSample.trim() --- dev/convert.html | 20 +- src/conversion.ts | 323 ++++++++++------------------- src/encode.ts | 10 + src/media-source.ts | 22 +- src/resample.ts | 25 +-- src/sample.ts | 68 ++++++ test/browser/audio-samples.test.ts | 139 +++++++++++++ 7 files changed, 369 insertions(+), 238 deletions(-) create mode 100644 test/browser/audio-samples.test.ts diff --git a/dev/convert.html b/dev/convert.html index c94dbb9..9b5117e 100644 --- a/dev/convert.html +++ b/dev/convert.html @@ -91,13 +91,16 @@ //const startTime = await primaryTrack.getFirstTimestamp(); //console.log(startTime) - let ctx = null; + const canvas = new OffscreenCanvas(1280, 720); + const ctx = canvas.getContext('2d'); + + //let ctx = null; let conversion = await Mediabunny.Conversion.init({ input, output, audio: { - codec: 'aac', - forceTranscode: true, + //codec: 'aac', + //forceTranscode: true, //forceTranscode: true, //sampleFormat: 's16', }, @@ -126,7 +129,10 @@ }, */ video: { - discard: true, + process: (sample) => { + sample.draw(ctx, 0, 0, 1280, 720); + return new Mediabunny.VideoSample(canvas, { timestamp: sample.timestamp, duration: sample.duration }); + }, }, tags: {} ?? { title: 'Bigggy', @@ -146,8 +152,10 @@ } }, trim: { - start: 300.14984567374756 - 100, - end: 310.1548298151939 - 100, + start: -2, + end: 10, + //start: 300.14984567374756 - 100, + //end: 310.1548298151939 - 100, //end: 10, //start: startTime, //end: startTime + 2, diff --git a/src/conversion.ts b/src/conversion.ts index f7106fe..49bee67 100644 --- a/src/conversion.ts +++ b/src/conversion.ts @@ -14,6 +14,7 @@ import { VideoCodec, } from './codec'; import { + AudioEncodingConfig, getEncodableAudioCodecs, getFirstEncodableVideoCodec, Quality, @@ -50,17 +51,14 @@ import { Output, OutputTrackGroup, TrackType } from './output'; import { Mp4OutputFormat } from './output-format'; import { AudioSample, - audioSampleToInterleavedFormat, clampCropRectangle, CropRectangle, - toInterleavedAudioFormat, validateCropRectangle, VideoSample, VideoSampleResource, } from './sample'; import { MetadataTags, validateMetadataTags } from './metadata'; import { NullTarget } from './target'; -import { AudioResampler } from './resample'; /** * The options for media file conversion. @@ -1342,6 +1340,10 @@ export class Conversion { encodingConfig.transform.frameRate = trackOptions.frameRate; } + if (trackOptions.process) { + encodingConfig.transform.process = trackOptions.process; + } + if (needsRerender) { outputTrackRotation = 0; // Since the rotation is baked into the output @@ -1353,6 +1355,12 @@ export class Conversion { encodingConfig.transform.alpha = alpha; } + // We need to do this because `process` can emit new timestamps + let lastSampleTimestamp: number | null = null; + encodingConfig.onEncodedSample = (sample) => { + lastSampleTimestamp = sample.timestamp; + }; + const source = new VideoSampleSource(encodingConfig); videoSource = source; @@ -1370,7 +1378,14 @@ export class Conversion { const adjustedSampleTimestamp = Math.max(sample.timestamp - this._startTimestamp, 0); sample.setTimestamp(adjustedSampleTimestamp); - await this._registerVideoSample(trackOptions, outputTrackId, source, sample); + this._reportProgress(outputTrackId, sample.timestamp + sample.duration); + await source.add(sample); + + if (lastSampleTimestamp !== null) { + if (this._synchronizer.shouldWait(outputTrackId, lastSampleTimestamp)) { + await this._synchronizer.wait(lastSampleTimestamp); + } + } sample.close(); } @@ -1403,69 +1418,6 @@ export class Conversion { this._outputOwnTrackGroups.push(ownGroup); } - /** @internal */ - async _registerVideoSample( - trackOptions: ConversionVideoOptions, - outputTrackId: number, - source: VideoSampleSource, - sample: VideoSample, - ) { - if (this._canceled) { - return; - } - - this._reportProgress(outputTrackId, sample.timestamp + sample.duration); - - let finalSamples: VideoSample[]; - if (!trackOptions.process) { - finalSamples = [sample]; - } else { - let processed = trackOptions.process(sample); - if (processed instanceof Promise) processed = await processed; - - if (!Array.isArray(processed)) { - processed = processed === null ? [] : [processed]; - } - - finalSamples = processed.map((x) => { - if (x instanceof VideoSample) { - return x; - } - - if (typeof VideoFrame !== 'undefined' && x instanceof VideoFrame) { - return new VideoSample(x); - } - - // Calling the VideoSample constructor here will automatically handle input validation for us - // (it throws for any non-legal argument). - return new VideoSample(x as CanvasImageSource, { - timestamp: sample.timestamp, - duration: sample.duration, - }); - }); - } - - try { - for (const finalSample of finalSamples) { - if (this._canceled) { - break; - } - - await source.add(finalSample); - - if (this._synchronizer.shouldWait(outputTrackId, finalSample.timestamp)) { - await this._synchronizer.wait(finalSample.timestamp); - } - } - } finally { - for (const finalSample of finalSamples) { - if (finalSample !== sample) { - finalSample.close(); - } - } - } - } - /** @internal */ async _processAudioTrack(track: InputAudioTrack, trackOptions: ConversionAudioOptions, outputTrackId: number) { const sourceCodec = await track.getCodec(); @@ -1487,16 +1439,18 @@ export class Conversion { let numberOfChannels = trackOptions.numberOfChannels ?? originalNumberOfChannels; let sampleRate = trackOptions.sampleRate ?? originalSampleRate; - let needsResample = numberOfChannels !== originalNumberOfChannels - || sampleRate !== originalSampleRate - || firstTimestamp < this._startTimestamp - || (firstTimestamp > this._startTimestamp && !this.output.format.supportsTimestampedMediaData); + + const needsTrimming = firstTimestamp < this._startTimestamp; + const needsPadding = firstTimestamp > this._startTimestamp && !this.output.format.supportsTimestampedMediaData; let audioCodecs = this.output.format.getSupportedAudioCodecs(); if ( !trackOptions.forceTranscode && !trackOptions.bitrate - && !needsResample + && numberOfChannels === originalNumberOfChannels + && sampleRate === originalSampleRate + && !needsTrimming + && !needsPadding && audioCodecs.includes(sourceCodec) && (!trackOptions.codec || trackOptions.codec === sourceCodec) && !trackOptions.process @@ -1589,7 +1543,6 @@ export class Conversion { .find(codec => (NON_PCM_AUDIO_CODECS as readonly string[]).includes(codec)); if (nonPcmCodec) { // We are able to encode using a non-PCM codec, but it'll require resampling - needsResample = true; codecOfChoice = nonPcmCodec; numberOfChannels = FALLBACK_NUMBER_OF_CHANNELS; sampleRate = FALLBACK_SAMPLE_RATE; @@ -1607,44 +1560,86 @@ export class Conversion { return; } - if (needsResample) { - audioSource = this._resampleAudio( - track, - trackOptions, - outputTrackId, - codecOfChoice, - numberOfChannels, - sampleRate, - bitrate, - ); - } else { - const source = new AudioSampleSource({ - codec: codecOfChoice, - bitrate, - }); - audioSource = source; + const encodingConfig: AudioEncodingConfig = { + codec: codecOfChoice, + bitrate, + transform: { + sampleFormat: trackOptions.sampleFormat, + process: trackOptions.process, + }, + }; + assert(encodingConfig.transform); - this._trackPromises.push((async () => { - await this._started; + if (numberOfChannels !== originalNumberOfChannels) { + encodingConfig.transform.numberOfChannels = numberOfChannels; + } + if (sampleRate !== originalSampleRate) { + encodingConfig.transform.sampleRate = sampleRate; + } - const sink = new AudioSampleSink(track); - for await (const sample of sink.samples(undefined, this._endTimestamp)) { - if (this._canceled) { - sample.close(); - return; - } + let lastSampleTimestamp: number | null = null; + encodingConfig.onEncodedSample = (sample) => { + lastSampleTimestamp = sample.timestamp; + }; - // Offset the timestamp as needed - sample.setTimestamp(sample.timestamp - this._startTimestamp); + const source = new AudioSampleSource(encodingConfig); + audioSource = source; - await this._registerAudioSample(trackOptions, outputTrackId, source, sample); + this._trackPromises.push((async () => { + await this._started; + + if (needsPadding) { + const paddingLength = firstTimestamp - this._startTimestamp; + const paddingLengthSamples = Math.round(paddingLength * originalSampleRate); + + const silentSample = new AudioSample({ + data: new Float32Array(paddingLengthSamples * originalNumberOfChannels), + format: 'f32-planar', + numberOfChannels: originalNumberOfChannels, + sampleRate: originalSampleRate, + timestamp: 0, + }); + await this._registerAudioSample(silentSample, source, outputTrackId, () => lastSampleTimestamp); + } + + const sink = new AudioSampleSink(track); + for await (let sample of sink.samples(this._startTimestamp, this._endTimestamp)) { + if (this._canceled) { sample.close(); + return; } - source.close(); - this._synchronizer.closeTrack(outputTrackId); - })()); - } + let startFrame = 0; + let endFrame = sample.numberOfFrames; + + if (sample.timestamp < this._startTimestamp) { + startFrame = Math.round((this._startTimestamp - sample.timestamp) * sample.sampleRate); + } + if (sample.timestamp + sample.duration > this._endTimestamp) { + endFrame = Math.round((this._endTimestamp - sample.timestamp) * sample.sampleRate); + } + + if (startFrame > 0 || endFrame < sample.numberOfFrames) { + // Trim the sample if it sticks out of the trim region on either end + const trimmedSample = sample.trim(startFrame, endFrame); + sample.close(); + sample = trimmedSample; + + if (sample.numberOfFrames === 0) { + sample.close(); + continue; + } + } + + // Offset the timestamp as needed + sample.setTimestamp(sample.timestamp - this._startTimestamp); + + await this._registerAudioSample(sample, source, outputTrackId, () => lastSampleTimestamp); + } + + source.close(); + this._synchronizer.closeTrack(outputTrackId); + })()); } let ownGroup: OutputTrackGroup | null = null; @@ -1670,126 +1665,24 @@ export class Conversion { /** @internal */ async _registerAudioSample( - trackOptions: ConversionAudioOptions, - outputTrackId: number, + sample: AudioSample, source: AudioSampleSource, - inputSample: AudioSample, + outputTrackId: number, + getLastSampleTimestamp: () => number | null, ) { - if (this._canceled) { - return; - } - - let sample = inputSample; - - if ( - trackOptions.sampleFormat !== undefined - && toInterleavedAudioFormat(sample.format) !== trackOptions.sampleFormat - ) { - // Do a sample format conversion - sample = audioSampleToInterleavedFormat(sample, trackOptions.sampleFormat); - } - this._reportProgress(outputTrackId, sample.timestamp + sample.duration); - let finalSamples: AudioSample[]; - if (!trackOptions.process) { - finalSamples = [sample]; - } else { - let processed = trackOptions.process(sample); - if (processed instanceof Promise) processed = await processed; + await source.add(sample); + sample.close(); - if (!Array.isArray(processed)) { - processed = processed === null ? [] : [processed]; - } - - if (!processed.every(x => x instanceof AudioSample)) { - throw new TypeError( - 'The audio process function must return an AudioSample, null, or an array of AudioSamples.', - ); - } - - finalSamples = processed; - } - - try { - for (const finalSample of finalSamples) { - if (this._canceled) { - break; - } - - await source.add(finalSample); - - if (this._synchronizer.shouldWait(outputTrackId, finalSample.timestamp)) { - await this._synchronizer.wait(finalSample.timestamp); - } - } - } finally { - if (sample !== inputSample) { - sample.close(); - } - - for (const finalSample of finalSamples) { - if (finalSample !== inputSample) { - finalSample.close(); - } + const lastSampleTimestamp = getLastSampleTimestamp(); + if (lastSampleTimestamp !== null) { + if (this._synchronizer.shouldWait(outputTrackId, lastSampleTimestamp)) { + await this._synchronizer.wait(lastSampleTimestamp); } } } - /** @internal */ - _resampleAudio( - track: InputAudioTrack, - trackOptions: ConversionAudioOptions, - outputTrackId: number, - codec: AudioCodec, - targetNumberOfChannels: number, - targetSampleRate: number, - bitrate: number | Quality, - ) { - const source = new AudioSampleSource({ - codec, - bitrate, - }); - - this._trackPromises.push((async () => { - await this._started; - - const resampler = new AudioResampler({ - targetNumberOfChannels, - targetSampleRate, - startTime: this._startTimestamp, - endTime: this._endTimestamp, - onSample: async (sample) => { - assert(sample.timestamp >= this._startTimestamp); - sample.setTimestamp(sample.timestamp - this._startTimestamp); - - await this._registerAudioSample(trackOptions, outputTrackId, source, sample); - sample.close(); - }, - }); - - const sink = new AudioSampleSink(track); - const iterator = sink.samples(this._startTimestamp, this._endTimestamp); - - for await (const sample of iterator) { - if (this._canceled) { - sample.close(); - return; - } - - await resampler.add(sample); - sample.close(); - } - - await resampler.finalize(); - - source.close(); - this._synchronizer.closeTrack(outputTrackId); - })()); - - return source; - } - /** @internal */ _reportProgress(trackId: number, endTimestamp: number) { if (!this._computeProgress) { diff --git a/src/encode.ts b/src/encode.ts index 3565da1..4f52495 100644 --- a/src/encode.ts +++ b/src/encode.ts @@ -74,6 +74,8 @@ export type VideoEncodingConfig = { * WebCodecs API, is created. */ onEncoderConfig?: (config: VideoEncoderConfig) => unknown; + /** Called right before a sample is passed to the encoder. */ + onEncodedSample?: (sample: VideoSample) => unknown; } & VideoEncodingAdditionalOptions; /** @@ -232,6 +234,9 @@ export const validateVideoEncodingConfig = (config: VideoEncodingConfig) => { if (config.onEncoderConfig !== undefined && typeof config.onEncoderConfig !== 'function') { throw new TypeError('config.onEncoderConfig, when provided, must be a function.'); } + if (config.onEncodedSample !== undefined && typeof config.onEncodedSample !== 'function') { + throw new TypeError('config.onEncodedSample, when provided, must be a function.'); + } validateVideoEncodingAdditionalOptions(config.codec, config); }; @@ -382,6 +387,8 @@ export type AudioEncodingConfig = { * WebCodecs API, is created. */ onEncoderConfig?: (config: AudioEncoderConfig) => unknown; + /** Called right before a sample is passed to the encoder. */ + onEncodedSample?: (sample: AudioSample) => unknown; } & AudioEncodingAdditionalOptions; /** @@ -462,6 +469,9 @@ export const validateAudioEncodingConfig = (config: AudioEncodingConfig) => { if (config.onEncoderConfig !== undefined && typeof config.onEncoderConfig !== 'function') { throw new TypeError('config.onEncoderConfig, when provided, must be a function.'); } + if (config.onEncodedSample !== undefined && typeof config.onEncodedSample !== 'function') { + throw new TypeError('config.onEncodedSample, when provided, must be a function.'); + } validateAudioEncodingAdditionalOptions(config.codec, config); }; diff --git a/src/media-source.ts b/src/media-source.ts index ed6d29b..59786ca 100644 --- a/src/media-source.ts +++ b/src/media-source.ts @@ -265,6 +265,7 @@ class VideoEncoderWrapper { * So, we keep track of the encoder error and throw it as soon as we get the chance. */ private error: Error | null = null; + private closed = false; private lastMuxerPromise: Promise = Promise.resolve(); @@ -429,6 +430,8 @@ class VideoEncoderWrapper { return new VideoSample(x); } + // Calling the VideoSample constructor here will automatically handle input validation for us + // (it throws for any non-legal argument). return new VideoSample(x as CanvasImageSource, { timestamp: videoSample.timestamp, duration: videoSample.duration, @@ -457,6 +460,10 @@ class VideoEncoderWrapper { } assert(this.encoderInitialized); + if (this.closed) { + break; + } + const keyFrameInterval = this.encodingConfig.keyFrameInterval ?? 2; const multipleOfKeyFrameInterval = Math.floor(sampleToEncode.timestamp / keyFrameInterval); @@ -474,6 +481,8 @@ class VideoEncoderWrapper { }; this.lastMultipleOfKeyFrameInterval = multipleOfKeyFrameInterval; + this.encodingConfig.onEncodedSample?.(sampleToEncode); + if (this.customEncoder) { this.customEncoderQueueSize++; @@ -843,6 +852,8 @@ class VideoEncoderWrapper { await this.padFrameRate(alignedEnd); } + this.closed = true; + this.frameRateLastSample?.close(); this.frameRateLastSample = null; @@ -2033,6 +2044,7 @@ class AudioEncoderWrapper { */ private error: Error | null = null; private lastMuxerPromise: Promise = Promise.resolve(); + private closed = false; constructor(private source: AudioSource, private encodingConfig: AudioEncodingConfig) {} @@ -2070,8 +2082,6 @@ class AudioEncoderWrapper { ?? audioSample.numberOfChannels, targetSampleRate: config.transform!.sampleRate ?? audioSample.sampleRate, - startTime: audioSample.timestamp, - endTime: Infinity, onSample: async (sample) => { await this.processAndEncode(sample, true); }, @@ -2162,6 +2172,10 @@ class AudioEncoderWrapper { } assert(this.encoderInitialized); + if (this.closed) { + return; + } + // Handle padding of gaps with silence to avoid audio drift over time, like in // https://github.com/Vanilagy/mediabunny/issues/176 // TODO An open question is how encoders deal with the first AudioData having a non-zero timestamp, and with @@ -2197,6 +2211,8 @@ class AudioEncoderWrapper { } } + this.encodingConfig.onEncodedSample?.(audioSample); + if (this.customEncoder) { this.customEncoderQueueSize++; @@ -2536,6 +2552,8 @@ class AudioEncoderWrapper { } this.resampler = null; + this.closed = true; + if (this.customEncoder) { if (!forceClose) { void this.customEncoderCallSerializer.call(() => this.customEncoder!.flush()); diff --git a/src/resample.ts b/src/resample.ts index 4686a9d..bb78a1d 100644 --- a/src/resample.ts +++ b/src/resample.ts @@ -20,41 +20,32 @@ export class AudioResampler { targetSampleRate: number; sourceNumberOfChannels: number | null = null; targetNumberOfChannels: number; - endTime: number; + startTime: number | null = null; onSample: (sample: AudioSample) => Promise; bufferSizeInFrames: number; bufferSizeInSamples: number; outputBuffer: Float32Array; /** Start frame of current buffer */ - bufferStartFrame: number; + bufferStartFrame = 0; /** The highest index written to in the current buffer */ maxWrittenFrame: number | null = null; channelMixer!: (sourceData: Float32Array, sourceFrameIndex: number, targetChannelIndex: number) => number; tempSourceBuffer!: Float32Array; - timestampOffset: number; constructor(options: { targetSampleRate: number; targetNumberOfChannels: number; - startTime: number; - endTime: number; onSample: (sample: AudioSample) => Promise; }) { this.targetSampleRate = options.targetSampleRate; this.targetNumberOfChannels = options.targetNumberOfChannels; - this.endTime = options.endTime; this.onSample = options.onSample; this.bufferSizeInFrames = Math.floor(this.targetSampleRate * 5.0); // 5 seconds this.bufferSizeInSamples = this.bufferSizeInFrames * this.targetNumberOfChannels; this.outputBuffer = new Float32Array(this.bufferSizeInSamples); - - this.bufferStartFrame = Math.floor(options.startTime * this.targetSampleRate); - // Set to ensure that if the buffer start frame lands on a fractional sample, that the first timestamp still - // comes out as exactly startTime - this.timestampOffset = options.startTime - this.bufferStartFrame / this.targetSampleRate; } /** @@ -186,6 +177,7 @@ export class AudioResampler { // they see fit. this.sourceSampleRate = audioSample.sampleRate; this.sourceNumberOfChannels = audioSample.numberOfChannels; + this.startTime = audioSample.timestamp; // Pre-allocate temporary buffer for source data this.tempSourceBuffer = new Float32Array(this.sourceSampleRate * this.sourceNumberOfChannels); @@ -193,6 +185,8 @@ export class AudioResampler { this.doChannelMixerSetup(); } + assert(this.startTime !== null); + const requiredSamples = audioSample.numberOfFrames * audioSample.numberOfChannels; this.ensureTempBufferSize(requiredSamples); @@ -201,8 +195,8 @@ export class AudioResampler { const sourceView = new Float32Array(this.tempSourceBuffer.buffer, 0, sourceDataSize / 4); audioSample.copyTo(sourceView, { planeIndex: 0, format: 'f32' }); - const inputStartTime = audioSample.timestamp; - const inputEndTime = Math.min(audioSample.timestamp + audioSample.duration, this.endTime); + const inputStartTime = audioSample.timestamp - this.startTime; + const inputEndTime = inputStartTime + audioSample.duration; // Compute which output frames are affected by this sample const outputStartFrame = Math.floor(inputStartTime * this.targetSampleRate); @@ -266,17 +260,18 @@ export class AudioResampler { return; // Nothing to finalize } + assert(this.startTime !== null); + const samplesWritten = (this.maxWrittenFrame + 1) * this.targetNumberOfChannels; const outputData = new Float32Array(samplesWritten); outputData.set(this.outputBuffer.subarray(0, samplesWritten)); - const timestampSeconds = this.bufferStartFrame / this.targetSampleRate; const audioSample = new AudioSample({ format: 'f32', sampleRate: this.targetSampleRate, numberOfChannels: this.targetNumberOfChannels, - timestamp: timestampSeconds + this.timestampOffset, + timestamp: this.startTime + this.bufferStartFrame / this.targetSampleRate, data: outputData, }); diff --git a/src/sample.ts b/src/sample.ts index bd97ba8..d6fa6ef 100644 --- a/src/sample.ts +++ b/src/sample.ts @@ -2712,6 +2712,74 @@ export class AudioSample implements Disposable { } } + /** + * Returns a new {@link AudioSample} containing only the frames in the range [startSample, endSample). Both bounds + * must lie within this sample's range of frames. The returned sample's timestamp is shifted to match the start of + * the trimmed section. + */ + trim(startSample: number, endSample = this.numberOfFrames) { + if (!Number.isInteger(startSample) || startSample < 0) { + throw new TypeError('startSample must be a non-negative integer.'); + } + if (!Number.isInteger(endSample) || endSample < 0) { + throw new TypeError('endSample must be a non-negative integer.'); + } + if (startSample > this.numberOfFrames) { + throw new RangeError('startSample out of range.'); + } + if (endSample > this.numberOfFrames) { + throw new RangeError('endSample out of range.'); + } + if (endSample < startSample) { + throw new RangeError('endSample must not be less than startSample.'); + } + + if (this._closed) { + throw new Error('AudioSample is closed.'); + } + + const frameCount = endSample - startSample; + const bytesPerSample = getBytesPerSample(this.format); + + let data: Uint8Array; + if (formatIsPlanar(this.format)) { + const planeSize = frameCount * bytesPerSample; + data = new Uint8Array(planeSize * this.numberOfChannels); + + if (frameCount > 0) { + // Copy plane-by-plane + for (let i = 0; i < this.numberOfChannels; i++) { + this.copyTo(data.subarray(i * planeSize, (i + 1) * planeSize), { + planeIndex: i, + format: this.format, + frameOffset: startSample, + frameCount, + }); + } + } + } else { + // Trivial + data = new Uint8Array(frameCount * this.numberOfChannels * bytesPerSample); + + if (frameCount > 0) { + this.copyTo(data, { + planeIndex: 0, + format: this.format, + frameOffset: startSample, + frameCount, + }); + } + } + + return new AudioSample({ + data, + format: this.format, + sampleRate: this.sampleRate, + numberOfChannels: this.numberOfChannels, + timestamp: this.timestamp + startSample / this.sampleRate, + }); + } + /** * Closes this audio sample, releasing held resources. Audio samples should be closed as soon as they are not * needed anymore. diff --git a/test/browser/audio-samples.test.ts b/test/browser/audio-samples.test.ts new file mode 100644 index 0000000..dffa535 --- /dev/null +++ b/test/browser/audio-samples.test.ts @@ -0,0 +1,139 @@ +import { expect, test } from 'vitest'; +import { AudioSample } from '../../src/sample.js'; + +const SAMPLE_RATE = 48000; +const NUM_CHANNELS = 2; +const NUM_FRAMES = 10; +const TIMESTAMP = 1; + +const cellValue = (frame: number, channel: number) => frame * 10 + channel; + +const makeInterleavedSample = () => { + const data = new Int16Array(NUM_FRAMES * NUM_CHANNELS); + for (let f = 0; f < NUM_FRAMES; f++) { + for (let c = 0; c < NUM_CHANNELS; c++) { + data[f * NUM_CHANNELS + c] = cellValue(f, c); + } + } + + return new AudioSample({ + data, + format: 's16', + sampleRate: SAMPLE_RATE, + numberOfChannels: NUM_CHANNELS, + timestamp: TIMESTAMP, + }); +}; + +const makePlanarSample = () => { + const data = new Float32Array(NUM_FRAMES * NUM_CHANNELS); + for (let c = 0; c < NUM_CHANNELS; c++) { + for (let f = 0; f < NUM_FRAMES; f++) { + data[c * NUM_FRAMES + f] = cellValue(f, c); + } + } + + return new AudioSample({ + data, + format: 'f32-planar', + sampleRate: SAMPLE_RATE, + numberOfChannels: NUM_CHANNELS, + timestamp: TIMESTAMP, + }); +}; + +const readCell = (sample: AudioSample, frame: number, channel: number) => { + if (sample.format === 's16') { + const out = new Int16Array(sample.numberOfFrames * sample.numberOfChannels); + sample.copyTo(out, { planeIndex: 0, format: 's16' }); + return out[frame * sample.numberOfChannels + channel]; + } else { + const out = new Float32Array(sample.numberOfFrames); + sample.copyTo(out, { planeIndex: channel, format: 'f32-planar' }); + return out[frame]; + } +}; + +for (const [label, makeSample] of [ + ['interleaved', makeInterleavedSample], + ['planar', makePlanarSample], +] as const) { + test(`trim, normal case (${label})`, () => { + using sample = makeSample(); + using trimmed = sample.trim(3, 8); + + expect(trimmed).not.toBe(sample); + expect(trimmed.format).toBe(sample.format); + expect(trimmed.sampleRate).toBe(SAMPLE_RATE); + expect(trimmed.numberOfChannels).toBe(NUM_CHANNELS); + expect(trimmed.numberOfFrames).toBe(5); + expect(trimmed.duration).toBeCloseTo(5 / SAMPLE_RATE); + expect(trimmed.timestamp).toBeCloseTo(TIMESTAMP + 3 / SAMPLE_RATE); + + // The trimmed frame i corresponds to the original frame i + 3 + for (let f = 0; f < trimmed.numberOfFrames; f++) { + for (let c = 0; c < NUM_CHANNELS; c++) { + expect(readCell(trimmed, f, c)).toBe(cellValue(f + 3, c)); + } + } + + // The original sample must be untouched + expect(sample.numberOfFrames).toBe(NUM_FRAMES); + expect(readCell(sample, 3, 0)).toBe(cellValue(3, 0)); + }); + + test(`trim, full range is an independent copy (${label})`, () => { + using sample = makeSample(); + using trimmed = sample.trim(0, NUM_FRAMES); + + expect(trimmed.numberOfFrames).toBe(NUM_FRAMES); + expect(trimmed.timestamp).toBeCloseTo(TIMESTAMP); + for (let f = 0; f < NUM_FRAMES; f++) { + for (let c = 0; c < NUM_CHANNELS; c++) { + expect(readCell(trimmed, f, c)).toBe(cellValue(f, c)); + } + } + }); + + test(`trim, pathological zero-sample case (${label})`, () => { + using sample = makeSample(); + + // Empty range in the middle + using empty = sample.trim(4, 4); + expect(empty.numberOfFrames).toBe(0); + expect(empty.duration).toBe(0); + expect(empty.format).toBe(sample.format); + expect(empty.numberOfChannels).toBe(NUM_CHANNELS); + expect(empty.timestamp).toBeCloseTo(TIMESTAMP + 4 / SAMPLE_RATE); + + // Empty range right at the end (startSample === numberOfFrames) is still valid + using emptyAtEnd = sample.trim(NUM_FRAMES, NUM_FRAMES); + expect(emptyAtEnd.numberOfFrames).toBe(0); + expect(emptyAtEnd.timestamp).toBeCloseTo(TIMESTAMP + NUM_FRAMES / SAMPLE_RATE); + }); + + test(`trim, illegal params (${label})`, () => { + using sample = makeSample(); + + // Non-integer + expect(() => sample.trim(1.5, 5)).toThrow(TypeError); + expect(() => sample.trim(1, 5.5)).toThrow(TypeError); + + // Negative + expect(() => sample.trim(-1, 5)).toThrow(TypeError); + expect(() => sample.trim(0, -1)).toThrow(TypeError); + + // Out of range + expect(() => sample.trim(0, NUM_FRAMES + 1)).toThrow(RangeError); + expect(() => sample.trim(NUM_FRAMES + 1, NUM_FRAMES + 1)).toThrow(RangeError); + + // End before start + expect(() => sample.trim(6, 3)).toThrow(RangeError); + }); +} + +test('trim, throws on a closed sample', () => { + const sample = makeInterleavedSample(); + sample.close(); + expect(() => sample.trim(0, 5)).toThrow('AudioSample is closed.'); +});