From 1e200edc831501c971d073efc36aff7ad84b1740 Mon Sep 17 00:00:00 2001 From: Vanilagy <1696106+Vanilagy@users.noreply.github.com> Date: Sun, 10 May 2026 16:37:25 +0200 Subject: [PATCH] Implement audio decoder & encoder --- packages/server/src/audio-decoder.ts | 106 ++++++ packages/server/src/audio-encoder.ts | 370 +++++++++++++++++++ packages/server/src/audio-sample.ts | 55 +++ packages/server/src/index.ts | 9 +- packages/server/src/misc.ts | 6 + packages/server/src/video-encoder.ts | 6 +- src/sample.ts | 93 +++-- test/node/server-extension.test.ts | 523 ++++++++++++++++++++++++++- 8 files changed, 1128 insertions(+), 40 deletions(-) create mode 100644 packages/server/src/audio-decoder.ts create mode 100644 packages/server/src/audio-encoder.ts create mode 100644 packages/server/src/audio-sample.ts diff --git a/packages/server/src/audio-decoder.ts b/packages/server/src/audio-decoder.ts new file mode 100644 index 0000000..4157a37 --- /dev/null +++ b/packages/server/src/audio-decoder.ts @@ -0,0 +1,106 @@ +import { AudioCodec, AudioSample, CustomAudioDecoder, EncodedPacket, MaybePromise } from 'mediabunny'; +import * as NodeAv from 'node-av'; +import { CODEC_TO_CODEC_ID, getChannelLayout } from './misc'; +import { assert, roundToDivisor, toUint8Array } from '../../../src/misc'; +import { NodeAvFrameAudioSampleResource } from './audio-sample'; + +export class NodeAvAudioDecoder extends CustomAudioDecoder { + frame!: NodeAv.Frame; + packet!: NodeAv.Packet; + codecContext!: NodeAv.CodecContext; + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + static override supports(codec: AudioCodec, config: AudioDecoderConfig): boolean { + return codec === 'aac' + || codec === 'opus' + || codec === 'mp3' + || codec === 'vorbis' + || codec === 'flac' + || codec === 'ac3' + || codec === 'eac3'; + } + + async init(): Promise { + this.frame = new NodeAv.Frame(); + this.frame.alloc(); + this.packet = new NodeAv.Packet(); + this.packet.alloc(); + + const codecId = CODEC_TO_CODEC_ID[this.codec]; + assert(codecId !== undefined); + + const codec = NodeAv.Codec.findDecoder(codecId); + if (codec === null) { + throw new Error(`Unable to obtain libav codec for '${this.codec}'.`); + } + + const codecContext = new NodeAv.CodecContext(); + codecContext.allocContext3(codec); + + codecContext.sampleRate = this.config.sampleRate; + codecContext.channelLayout = getChannelLayout(this.config.numberOfChannels); + codecContext.codecType = NodeAv.AVMEDIA_TYPE_AUDIO; + codecContext.codecId = codecId; + codecContext.extraData = this.config.description + ? Buffer.from(toUint8Array(this.config.description)) + : null; + + const ret = await codecContext.open2(); + NodeAv.FFmpegError.throwIfError(ret, 'Open codec context'); + + this.codecContext = codecContext; + } + + async decode(packet: EncodedPacket): Promise { + this.packet.isKeyframe = packet.type === 'key'; + this.packet.data = Buffer.from(packet.data); + this.packet.timeBase = { num: 1, den: 1e6 }; + this.packet.pts = BigInt(packet.microsecondTimestamp); + this.packet.dts = NodeAv.AV_NOPTS_VALUE; + this.packet.duration = BigInt(packet.microsecondDuration); + + const ret = await this.codecContext.sendPacket(this.packet); + NodeAv.FFmpegError.throwIfError(ret, 'Send packet'); + + while (true) { + const receiveRet = await this.codecContext.receiveFrame(this.frame); + if (receiveRet === NodeAv.AVERROR_EAGAIN || receiveRet === NodeAv.AVERROR_EOF) { + break; + } + + this.receiveFrame(receiveRet); + } + } + + receiveFrame(ret: number) { + NodeAv.FFmpegError.throwIfError(ret, 'Receive frame'); + + const timestamp = roundToDivisor(Number(this.frame.pts) / 1e6, this.config.sampleRate); + this.onSample(new AudioSample(new NodeAvFrameAudioSampleResource(this.frame, timestamp))); + } + + async flush(): Promise { + // Send null packet to signal flush + const ret = await this.codecContext.sendPacket(null); + NodeAv.FFmpegError.throwIfError(ret, 'Flush decoder'); + + // Keep receiving frames until no more are available + while (true) { + const receiveRet = await this.codecContext.receiveFrame(this.frame); + if (receiveRet === NodeAv.AVERROR_EAGAIN || receiveRet === NodeAv.AVERROR_EOF) { + // No more frames available + break; + } + + this.receiveFrame(receiveRet); + } + + this.codecContext.flushBuffers(); + } + + close(): MaybePromise { + this.codecContext.freeContext(); + this.frame.free(); + this.packet.free(); + } +} diff --git a/packages/server/src/audio-encoder.ts b/packages/server/src/audio-encoder.ts new file mode 100644 index 0000000..1a655a0 --- /dev/null +++ b/packages/server/src/audio-encoder.ts @@ -0,0 +1,370 @@ +import { AudioCodec, AudioSample, CustomAudioEncoder, MaybePromise, QUALITY_MEDIUM } from 'mediabunny'; +import * as NodeAv from 'node-av'; +import { CODEC_TO_CODEC_ID, fromAudioSampleFormat, getChannelLayout } from './misc'; +import { assert, toUint8Array } from '../../../src/misc'; +import { NodeAvFrameAudioSampleResource } from './audio-sample'; +import { AdtsHeaderTemplate, buildAdtsHeaderTemplate, parseAacAudioSpecificConfig } from '../../../shared/aac-misc'; +import { EncodedPacket } from 'mediabunny'; + +const AAC_SAMPLE_RATES + = [96000, 88200, 64000, 48000, 44100, 32000, 24000, 22050, 16000, 12000, 11025, 8000, 7350]; +const OPUS_SAMPLE_RATES = [8000, 12000, 16000, 24000, 48000]; +const MP3_SAMPLE_RATES = [8000, 11025, 12000, 16000, 22050, 24000, 32000, 44100, 48000]; +const AC3_SAMPLE_RATES = [32000, 44100, 48000]; + +const FRAME_SIZE_FALLBACK = 1024; // Just 'cause + +export class NodeAvAudioEncoder extends CustomAudioEncoder { + frame!: NodeAv.Frame; + packet!: NodeAv.Packet; + avCodec!: NodeAv.Codec; + codecContext: NodeAv.CodecContext | null = null; + firstExpectedTimestamp: number | null = null; + outputTimestampOffset = 0; + + resampler: NodeAv.SoftwareResampleContext | null = null; + inputParametersKey: string | null = null; + resamplerInputSampleRate: number | null = null; + nextResamplerPts: bigint | null = null; + dstFrame: NodeAv.Frame | null = null; + packetEmitted = false; + adtsHeaderTemplate: AdtsHeaderTemplate | null = null; + + static override supports(codec: AudioCodec, config: AudioEncoderConfig): boolean { + const { numberOfChannels, sampleRate } = config; + + return ( + codec === 'aac' && numberOfChannels >= 1 && numberOfChannels <= 48 + && AAC_SAMPLE_RATES.includes(sampleRate) + ) || ( + codec === 'opus' && numberOfChannels >= 1 && numberOfChannels <= 255 + && OPUS_SAMPLE_RATES.includes(sampleRate) + ) || ( + codec === 'mp3' && numberOfChannels >= 1 && numberOfChannels <= 2 + && MP3_SAMPLE_RATES.includes(sampleRate) + ) || ( + codec === 'vorbis' && numberOfChannels >= 1 && numberOfChannels <= 255 + && sampleRate <= 200000 + ) || ( + codec === 'flac' && numberOfChannels >= 1 && numberOfChannels <= 8 + && sampleRate <= 655350 + ) || ( + codec === 'ac3' && numberOfChannels >= 1 && numberOfChannels <= 6 + && AC3_SAMPLE_RATES.includes(sampleRate) + ) || ( + codec === 'eac3' && numberOfChannels >= 1 && numberOfChannels <= 16 + && AC3_SAMPLE_RATES.includes(sampleRate) + ); + } + + async init(): Promise { + this.frame = new NodeAv.Frame(); + this.frame.alloc(); + this.packet = new NodeAv.Packet(); + this.packet.alloc(); + + const codecId = CODEC_TO_CODEC_ID[this.codec]; + assert(codecId !== undefined); + + const codec = NodeAv.Codec.findEncoder(codecId); + if (!codec) { + throw new Error(`Unable to obtain libav codec for '${this.codec}'.`); + } + + this.avCodec = codec; + + await this.createCodecContext(); + } + + async createCodecContext() { + assert(this.codecContext === null); + + const codecContext = new NodeAv.CodecContext(); + codecContext.allocContext3(this.avCodec); + + let sampleFormat = NodeAv.AV_SAMPLE_FMT_FLTP; + if (this.avCodec.sampleFormats && !this.avCodec.sampleFormats.includes(NodeAv.AV_SAMPLE_FMT_FLTP)) { + // Use a format that's supported + sampleFormat = this.avCodec.sampleFormats[0]!; + } + + codecContext.sampleRate = this.config.sampleRate; + codecContext.channelLayout = getChannelLayout(this.config.numberOfChannels); + codecContext.codecType = NodeAv.AVMEDIA_TYPE_AUDIO; + codecContext.codecId = CODEC_TO_CODEC_ID[this.codec]!; + codecContext.sampleFormat = sampleFormat; + codecContext.timeBase = new NodeAv.Rational(1, this.config.sampleRate); + codecContext.bitRate = BigInt(this.config.bitrate ?? QUALITY_MEDIUM._toAudioBitrate(this.codec) ?? 0); + + if (this.config.bitrateMode === 'constant') { + codecContext.rcMinRate = codecContext.bitRate; + codecContext.rcMaxRate = codecContext.bitRate; + } + + const ret = await codecContext.open2(); + NodeAv.FFmpegError.throwIfError(ret, 'Open codec context'); + + this.codecContext = codecContext; + } + + async encode(audioSample: AudioSample): Promise { + if (this.codecContext === null) { + await this.createCodecContext(); + assert(this.codecContext); + } + + this.firstExpectedTimestamp ??= audioSample.timestamp; + + if (audioSample._data instanceof NodeAvFrameAudioSampleResource) { + this.frame.ref(audioSample._data.frame); + } else { + // Copy audio data from AudioData to FFmpeg Frame + const format = fromAudioSampleFormat(audioSample.format); + this.frame.format = format; + this.frame.nbSamples = audioSample.numberOfFrames; + this.frame.sampleRate = audioSample.sampleRate; + this.frame.channelLayout = getChannelLayout(audioSample.numberOfChannels); + this.frame.pts = BigInt(Math.round(audioSample.timestamp * audioSample.sampleRate)); + this.frame.duration = BigInt(audioSample.numberOfFrames); + + this.frame.allocBuffer(); + assert(this.frame.data); + + for (let i = 0; i < this.frame.data.length; i++) { + audioSample.copyTo(this.frame.data[i]!, { planeIndex: i }); + } + } + + const key = `${this.frame.sampleRate}:${this.frame.channels}:${this.frame.format}`; + if (this.inputParametersKey !== null && this.inputParametersKey !== key) { + throw new Error( + 'Input audio parameters changed. For this audio encoder, you cannot change the input audio' + + ' parameters over time.', + ); + } + this.inputParametersKey = key; + + // We need the resampler when: + // 1. Format conversion is needed (sample format, sample rate, or channel count differs) + // 2. The codec requires fixed frame sizes + const requiresResampler + = this.codecContext.frameSize > 0 + || this.codecContext.sampleFormat !== this.frame.format + || this.codecContext.sampleRate !== this.frame.sampleRate + || this.codecContext.channels !== this.frame.channels; + + if (requiresResampler) { + if (!this.resampler) { + this.resampler = new NodeAv.SoftwareResampleContext(); + this.resamplerInputSampleRate = this.frame.sampleRate; + + const outLayout = getChannelLayout(this.codecContext.channels); + const inLayout = getChannelLayout(this.frame.channels); + + const ret = this.resampler.allocSetOpts2( + outLayout, this.codecContext.sampleFormat, this.codecContext.sampleRate, + inLayout, this.frame.format as NodeAv.AVSampleFormat, this.frame.sampleRate, + ); + NodeAv.FFmpegError.throwIfError(ret, 'allocSetOpts2'); + + const ret2 = this.resampler.init(); + NodeAv.FFmpegError.throwIfError(ret2, 'init'); + + this.dstFrame = new NodeAv.Frame(); + this.dstFrame.alloc(); + this.dstFrame.channelLayout = outLayout; + this.dstFrame.sampleRate = this.codecContext.sampleRate; + this.dstFrame.format = this.codecContext.sampleFormat; + this.dstFrame.nbSamples = this.codecContext.frameSize || FRAME_SIZE_FALLBACK; + this.dstFrame.duration = BigInt(this.dstFrame.nbSamples); + this.dstFrame.allocBuffer(); + + this.nextResamplerPts = this.frame.pts; + } + + const inputBuffers = this.frame.data; + if (!inputBuffers) { + throw new DOMException('Frame has no data', 'EncodingError'); + } + await this.resampler.convert(null, 0, inputBuffers, this.frame.nbSamples); + + await this.pullResampledFrames(); + } else { + await this.sendFrameAndReceivePackets(this.frame); + } + } + + async pullResampledFrames() { + assert(this.codecContext); + assert(this.resampler); + assert(this.dstFrame); + assert(this.nextResamplerPts !== null); + + const frameSize = this.codecContext.frameSize || FRAME_SIZE_FALLBACK; + + while (true) { + const available = this.resampler.getOutSamples(0); + if (available < frameSize) { + break; + } + + await this.resampler.convert(this.dstFrame.data, frameSize, null, 0); + + this.dstFrame.pts = this.nextResamplerPts; + + await this.sendFrameAndReceivePackets(this.dstFrame); + this.nextResamplerPts += BigInt(frameSize); + } + } + + async sendFrameAndReceivePackets(frame: NodeAv.Frame | null) { + assert(this.codecContext); + + const ret = await this.codecContext.sendFrame(frame); + NodeAv.FFmpegError.throwIfError(ret, 'Send frame'); + + while (true) { + const receiveRet = await this.codecContext.receivePacket(this.packet); + if (receiveRet === NodeAv.AVERROR_EAGAIN || receiveRet === NodeAv.AVERROR_EOF) { + break; + } + + this.receivePacket(receiveRet); + } + } + + receivePacket(ret: number) { + assert(this.codecContext); + assert(this.firstExpectedTimestamp !== null); + NodeAv.FFmpegError.throwIfError(ret, 'Receive packet'); + + if (!this.packet.data) { + return; + } + + let timestamp = Number(this.packet.pts) / this.codecContext.sampleRate; + const duration = Number(this.packet.duration) / this.codecContext.sampleRate; + + let data: Uint8Array = this.packet.data; + + let metadata: EncodedAudioChunkMetadata | undefined; + if (this.packetEmitted) { + metadata = {}; + } else { + // To compensate for any negative timestamp things that FFmpeg might do. It does these for a reason, to + // indicate encoder delay, but the notion of this is not yet supported in Mediabunny. + this.outputTimestampOffset = this.firstExpectedTimestamp - timestamp; + + const codecString = this.config.codec; + let description = this.codecContext.extraData + ? toUint8Array(this.codecContext.extraData) + : undefined; + + if ( + description + // eslint-disable-next-line @stylistic/max-len + // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access + && this.codec === 'aac' && (this.config as any).aac?.format === 'adts' + ) { + const config = parseAacAudioSpecificConfig(description); + this.adtsHeaderTemplate = buildAdtsHeaderTemplate(config); + description = undefined; // Not used with 'adts' format + } + + if (description && this.codec === 'flac') { + // FFmpeg uses the STREAMINFO block as the extradata, but WebCodecs wants a different format: + // 1. The bytes 0x66 0x4C 0x61 0x43 ("fLaC" in ASCII) + // 2. A metadata block (called the STREAMINFO block) as described in section 7 of [FLAC] + // 3. Other optional metadata blocks (not included here, because, well, they're optional) + description = new Uint8Array([ + 0x66, 0x4c, 0x61, 0x43, // 'fLaC' + 128, 0, 0, description.byteLength, + ...description, + ]); + } + + metadata = { + decoderConfig: { + codec: codecString, + sampleRate: this.codecContext.sampleRate, + numberOfChannels: this.codecContext.channels, + description, + }, + }; + } + + if (this.adtsHeaderTemplate) { + const frameLength = data.byteLength + this.adtsHeaderTemplate.header.byteLength; + this.adtsHeaderTemplate.bitstream.pos = 30; + this.adtsHeaderTemplate.bitstream.writeBits(13, frameLength); + + const final = new Uint8Array(this.adtsHeaderTemplate.header.byteLength + data.byteLength); + final.set(this.adtsHeaderTemplate.header, 0); + final.set(data, this.adtsHeaderTemplate.header.byteLength); + + data = final; + } + + timestamp += this.outputTimestampOffset; + + const packet = new EncodedPacket( + data, + 'key', + timestamp, + duration, + ); + + this.onPacket(packet, metadata); + this.packetEmitted = true; + } + + async flush(): Promise { + if (!this.codecContext) { + return; + } + + outer: + if (this.resampler) { + assert(this.resamplerInputSampleRate !== null); + + const currentOutSamples = this.resampler.getOutSamples(0); + if (currentOutSamples === 0) { + break outer; // Clean cut-off point + } + + const frameSize = this.codecContext.frameSize || FRAME_SIZE_FALLBACK; + assert(currentOutSamples < frameSize); // Because if it's more, it would've already been retrieved + + const inputSamplesNeeded = Math.ceil( + ((frameSize - currentOutSamples) / this.codecContext.sampleRate) * this.resamplerInputSampleRate, + ); + this.resampler.injectSilence(inputSamplesNeeded); + + await this.pullResampledFrames(); + } + + await this.sendFrameAndReceivePackets(null); + + this.codecContext.freeContext(); + this.codecContext = null; + this.packetEmitted = false; + this.firstExpectedTimestamp = null; + this.outputTimestampOffset = 0; + + this.resampler?.free(); + this.resampler = null; + this.inputParametersKey = null; + this.resamplerInputSampleRate = null; + this.nextResamplerPts = null; + this.dstFrame?.free(); + this.dstFrame = null; + } + + close(): MaybePromise { + this.codecContext?.freeContext(); + this.frame.free(); + this.packet.free(); + this.dstFrame?.free(); + this.resampler?.free(); + } +} diff --git a/packages/server/src/audio-sample.ts b/packages/server/src/audio-sample.ts new file mode 100644 index 0000000..1afc9b8 --- /dev/null +++ b/packages/server/src/audio-sample.ts @@ -0,0 +1,55 @@ +import { AudioSampleResource } from 'mediabunny'; +import * as NodeAv from 'node-av'; +import { toAudioSampleFormat } from './misc'; +import { assert, toUint8Array } from '../../../src/misc'; + +export class NodeAvFrameAudioSampleResource extends AudioSampleResource { + frame: NodeAv.Frame; + timestamp: number; + + constructor(frame: NodeAv.Frame, timestamp: number) { + super(); + + const clone = frame.clone(); + if (!clone) { + throw new Error('Allocation failure during frame clone.'); + } + + this.frame = clone; + this.timestamp = timestamp; + } + + getFormat(): AudioSampleFormat { + const result = toAudioSampleFormat(this.frame.format as NodeAv.AVSampleFormat); + if (result === null) { + throw new TypeError('Unsupported audio sample format: ' + this.frame.format); + } + + return result; + } + + getSampleRate(): number { + return this.frame.sampleRate; + } + + getNumberOfChannels(): number { + return this.frame.channels; + } + + getNumberOfFrames(): number { + return this.frame.nbSamples; + } + + getTimestamp(): number { + return this.timestamp; + } + + close(): void { + this.frame.free(); + } + + getDataPlane(planeIndex: number): Uint8Array { + assert(this.frame.data && planeIndex < this.frame.data.length); + return toUint8Array(this.frame.data[planeIndex]!); + } +} diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index bc6dd28..5471ac5 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -1,7 +1,9 @@ import { registerDecoder, registerEncoder } from 'mediabunny'; -import { NodeAvVideoDecoder } from './video-decoder'; import * as NodeAv from 'node-av'; +import { NodeAvVideoDecoder } from './video-decoder'; import { NodeAvVideoEncoder } from './video-encoder'; +import { NodeAvAudioDecoder } from './audio-decoder'; +import { NodeAvAudioEncoder } from './audio-encoder'; const SERVER_LOADED_SYMBOL = Symbol.for('@mediabunny/server loaded'); if ((globalThis as Record)[SERVER_LOADED_SYMBOL]) { @@ -23,6 +25,11 @@ export const registerMediabunnyServer = () => { NodeAv.Log.setLevel(NodeAv.AV_LOG_ERROR); + // Video registerDecoder(NodeAvVideoDecoder); registerEncoder(NodeAvVideoEncoder); + + // Audio + registerDecoder(NodeAvAudioDecoder); + registerEncoder(NodeAvAudioEncoder); }; diff --git a/packages/server/src/misc.ts b/packages/server/src/misc.ts index e3bb3b1..b458119 100644 --- a/packages/server/src/misc.ts +++ b/packages/server/src/misc.ts @@ -9,6 +9,12 @@ export const CODEC_TO_CODEC_ID: Partial> = av1: NodeAv.AV_CODEC_ID_AV1, aac: NodeAv.AV_CODEC_ID_AAC, + opus: NodeAv.AV_CODEC_ID_OPUS, + mp3: NodeAv.AV_CODEC_ID_MP3, + vorbis: NodeAv.AV_CODEC_ID_VORBIS, + flac: NodeAv.AV_CODEC_ID_FLAC, + ac3: NodeAv.AV_CODEC_ID_AC3, + eac3: NodeAv.AV_CODEC_ID_EAC3, }; let cachedHardwareContext: NodeAv.HardwareContext | null | undefined = undefined; diff --git a/packages/server/src/video-encoder.ts b/packages/server/src/video-encoder.ts index 752fbc2..2e932c7 100644 --- a/packages/server/src/video-encoder.ts +++ b/packages/server/src/video-encoder.ts @@ -165,10 +165,8 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder { assert(this.codecContext); } - const underlyingData = videoSample.getUnderlyingData(); - - if (underlyingData instanceof NodeAvFrameVideoSampleResource) { - this.frame.ref(underlyingData.frame); + if (videoSample._data instanceof NodeAvFrameVideoSampleResource) { + this.frame.ref(videoSample._data.frame); } else { if (videoSample.format === null) { throw new Error('Cannot encode foreign VideoSample with unknown (null) format.'); diff --git a/src/sample.ts b/src/sample.ts index f341553..d8bd447 100644 --- a/src/sample.ts +++ b/src/sample.ts @@ -615,15 +615,6 @@ export class VideoSample implements Disposable { finalizationRegistry?.register(this, { type: 'video', data: this._data }, this); } - getUnderlyingData() { - if (this._closed) { - throw new Error('VideoSample is closed.'); - } - - assert(this._data !== null); - return this._data; - } - /** Clones this video sample. */ clone() { if (this._closed) { @@ -1881,15 +1872,7 @@ export abstract class AudioSampleResource { */ abstract close(): void; - /** - * Returns the number of bytes required to hold the underlying audio data as specified by the given options. - */ - abstract allocationSize(options: AudioSampleCopyToOptions): number; - - /** - * Copies the underlying audio data to an ArrayBuffer or ArrayBufferView as specified by the given options. - */ - abstract copyTo(destination: AllowSharedBufferSource, options: AudioSampleCopyToOptions): void; + abstract getDataPlane(planeIndex: number): Uint8Array; } /** @@ -2004,10 +1987,30 @@ export class AudioSample implements Disposable { init._referenceCount++; this.format = init.getFormat(); + if (!AUDIO_SAMPLE_FORMATS.has(this.format)) { + throw new TypeError('getFormat() must return an AudioSampleFormat.'); + } + this.sampleRate = init.getSampleRate(); + if (!Number.isInteger(this.sampleRate) || this.sampleRate <= 0) { + throw new TypeError('getSampleRate() must return a positive integer.'); + } + this.numberOfFrames = init.getNumberOfFrames(); + if (!Number.isInteger(this.numberOfFrames) || this.numberOfFrames < 0) { + throw new TypeError('getNumberOfFrames() must return a non-negative integer.'); + } + this.numberOfChannels = init.getNumberOfChannels(); + if (!Number.isInteger(this.numberOfChannels) || this.numberOfChannels <= 0) { + throw new TypeError('getNumberOfChannels() must return a positive integer.'); + } + this.timestamp = init.getTimestamp(); + if (!Number.isFinite(this.timestamp)) { + throw new TypeError('getTimestamp() must return a finite number.'); + } + this.duration = this.numberOfFrames / this.sampleRate; } else { if (!init || typeof init !== 'object') { @@ -2084,10 +2087,6 @@ export class AudioSample implements Disposable { throw new Error('AudioSample is closed.'); } - if (this._data instanceof AudioSampleResource) { - return this._data.allocationSize(options); - } - const destFormat = options.format ?? this.format; const frameOffset = options.frameOffset ?? 0; @@ -2140,11 +2139,8 @@ export class AudioSample implements Disposable { throw new Error('AudioSample is closed.'); } - if (this._data instanceof AudioSampleResource) { - return this._data.copyTo(destination, options); - } - - const { planeIndex, format, frameCount: optFrameCount, frameOffset: optFrameOffset } = options; + const { format, frameCount: optFrameCount, frameOffset: optFrameOffset } = options; + let { planeIndex } = options; const srcFormat = this.format; const destFormat = format ?? this.format; @@ -2204,12 +2200,51 @@ export class AudioSample implements Disposable { }); } } else { - const uint8Data = this._data; - const srcView = toDataView(uint8Data); const readFn = getReadFunction(srcFormat); const srcBytesPerSample = getBytesPerSample(srcFormat); const srcIsPlanar = formatIsPlanar(srcFormat); + let uint8Data: Uint8Array; + if (this._data instanceof AudioSampleResource) { + const getDataPlaneValidated = (index: number) => { + const result = (this._data as AudioSampleResource).getDataPlane(index); + if (!(result instanceof Uint8Array)) { + throw new TypeError('getDataPlane() must return a Uint8Array.'); + } + + const expectedSize = numFrames * srcBytesPerSample * (srcIsPlanar ? 1 : numChannels); + if (result.byteLength !== expectedSize) { + throw new TypeError( + `Data plane ${index} has invalid size. Expected exactly ${expectedSize} bytes, got` + + ` ${result.byteLength} bytes.`, + ); + } + + return result; + }; + + if (srcIsPlanar) { + if (destIsPlanar) { + // Only one source plane will be extracted, so let's fetch only that one + uint8Data = getDataPlaneValidated(planeIndex); + planeIndex = 0; // To fix the subsequent access + } else { + // Pack all planes tightly together + uint8Data = new Uint8Array(numFrames * srcBytesPerSample * numChannels); + for (let ch = 0; ch < numChannels; ch++) { + const planeData = getDataPlaneValidated(ch); + uint8Data.set(planeData, ch * numFrames * srcBytesPerSample); + } + } + } else { + uint8Data = getDataPlaneValidated(0); // That's the only plane there is + } + } else { + uint8Data = this._data; + } + + const srcView = toDataView(uint8Data); + for (let i = 0; i < copyFrameCount; i++) { if (destIsPlanar) { const destOffset = i * destBytesPerSample; diff --git a/test/node/server-extension.test.ts b/test/node/server-extension.test.ts index 145d94c..5df9892 100644 --- a/test/node/server-extension.test.ts +++ b/test/node/server-extension.test.ts @@ -3,13 +3,14 @@ import { registerMediabunnyServer } from '@mediabunny/server'; import { Input } from '../../src/input.js'; import { BufferSource, FilePathSource } from '../../src/source.js'; import { ALL_FORMATS } from '../../src/input-format.js'; -import { assert, toUint8Array } from '../../src/misc.js'; -import { EncodedPacketSink, VideoSampleSink } from '../../src/media-sink.js'; +import { assert, last, toUint8Array } from '../../src/misc.js'; +import { AudioSampleSink, EncodedPacketSink, VideoSampleSink } from '../../src/media-sink.js'; import { NodeAvVideoDecoder } from '../../packages/server/src/video-decoder.js'; import { NodeAvVideoEncoder } from '../../packages/server/src/video-encoder.js'; import { NodeAvAudioDecoder } from '../../packages/server/src/audio-decoder.js'; +import { NodeAvAudioEncoder } from '../../packages/server/src/audio-encoder.js'; import { AudioSample, VideoSample } from '../../src/sample.js'; -import { buildVideoCodecString, VideoCodec } from '../../src/codec.js'; +import { AudioCodec, buildAudioCodecString, buildVideoCodecString, VideoCodec } from '../../src/codec.js'; import { EncodedPacket } from '../../src/packet.js'; import { AvcNalUnitType, @@ -24,6 +25,7 @@ import { Mp4OutputFormat } from '../../src/output-format.js'; import { BufferTarget } from '../../src/target.js'; import { Conversion } from '../../src/conversion.js'; import { NodeAvFrameVideoSampleResource } from '../../packages/server/src/video-sample.js'; +import { NodeAvFrameAudioSampleResource } from '../../packages/server/src/audio-sample.js'; beforeAll(() => { registerMediabunnyServer(); @@ -600,7 +602,7 @@ describe('Video', async () => { using sample = await sink.getSample(0); assert(sample); - expect(sample.getUnderlyingData()).toBeInstanceOf(NodeAvFrameVideoSampleResource); + expect(sample._data).toBeInstanceOf(NodeAvFrameVideoSampleResource); expect(sample.format).toBe('I420'); expect(sample.codedWidth).toBe(1920); expect(sample.codedHeight).toBe(1080); @@ -736,7 +738,6 @@ describe('Video', async () => { }); }); -/* describe('Audio', async () => { test('Decoder lifecycle', async () => { using input = new Input({ @@ -807,5 +808,515 @@ describe('Audio', async () => { await decoder.close(); }); + + test('Encoder lifecycle', async () => { + const encoder = new NodeAvAudioEncoder(); + // @ts-expect-error Readonly + encoder.codec = 'aac'; + // @ts-expect-error Readonly + encoder.config = { + codec: buildAudioCodecString('aac', 2, 48000), + numberOfChannels: 2, + sampleRate: 48000, + bitrate: 128000, + } satisfies AudioEncoderConfig; + + let packetCount = 0; + + // @ts-expect-error Readonly + encoder.onPacket = (packet: EncodedPacket, meta: EncodedAudioChunkMetadata) => { + expect(packet.duration).toBe(1024 / 48000); + expect(packet.type).toBe('key'); + + if (packetCount === 0) { + expect(packet.timestamp).toBe(0); + + expect(meta.decoderConfig).toBeDefined(); + expect(meta.decoderConfig!.codec).toBe('mp4a.40.2'); + expect(meta.decoderConfig!.numberOfChannels).toBe(2); + expect(meta.decoderConfig!.sampleRate).toBe(48000); + expect(meta.decoderConfig?.description).toBeDefined(); + } + + packetCount++; + }; + + await encoder.init(); + + const data = createF32SineWave(48000, 2, 2); + using sample1 = new AudioSample({ + data, + format: 'f32', + timestamp: 0, + numberOfChannels: 2, + sampleRate: 48000, + }); + await encoder.encode(sample1); + + await encoder.flush(); + + const minPacketCount = 2 * 48000 / 1024; + expect(packetCount).toBeGreaterThan(minPacketCount); + + packetCount = 0; + + using sample2 = new AudioSample({ + data, + format: 'f32', + timestamp: 0, + numberOfChannels: 2, + sampleRate: 48000, + }); + await encoder.encode(sample2); + + await encoder.flush(); + + expect(packetCount).toBeGreaterThan(minPacketCount); + + await encoder.close(); + }); + + const createF32SineWave = (sampleRate: number, channels: number, durationSeconds: number) => { + const totalFrames = sampleRate * durationSeconds; + const data = new Float32Array(totalFrames * channels); + + for (let i = 0; i < totalFrames; i++) { + const value = Math.sin(2 * Math.PI * 440 * i / sampleRate); + for (let ch = 0; ch < channels; ch++) { + data[i * channels + ch] = value; + } + } + + return data; + }; + + test('AAC encode & decode, AAC format', async () => { + await encodeDecodeTest('aac', { + // @ts-expect-error Fucky type + aac: { format: 'aac' }, + }, async (packet, meta, i) => { + expect(packet.type).toBe('key'); + expect(packet.duration).toBe(1024 / 48000); + + if (i === 0) { + expect(meta.decoderConfig).toBeDefined(); + expect(meta.decoderConfig!.codec).toBe('mp4a.40.2'); + expect(meta.decoderConfig!.numberOfChannels).toBe(2); + expect(meta.decoderConfig!.sampleRate).toBe(48000); + expect(meta.decoderConfig!.description).toBeDefined(); + } + }, async (sample) => { + expect(sample.duration).toBe(1024 / 48000); + expect(sample.numberOfChannels).toBe(2); + expect(sample.sampleRate).toBe(48000); + }); + }); + + test('AAC encode & decode, ADTS format', async () => { + await encodeDecodeTest('aac', { + // @ts-expect-error Fucky type + aac: { format: 'adts' }, + }, async (packet, meta, i) => { + expect(packet.type).toBe('key'); + expect(packet.duration).toBe(1024 / 48000); + expect(packet.data[0]).toBe(0xff); + + if (i === 0) { + expect(meta.decoderConfig).toBeDefined(); + expect(meta.decoderConfig!.codec).toBe('mp4a.40.2'); + expect(meta.decoderConfig!.numberOfChannels).toBe(2); + expect(meta.decoderConfig!.sampleRate).toBe(48000); + expect(meta.decoderConfig!.description).toBeUndefined(); + } + }, async (sample) => { + expect(sample.duration).toBe(1024 / 48000); + expect(sample.numberOfChannels).toBe(2); + expect(sample.sampleRate).toBe(48000); + }); + }); + + test('Opus encode & decode', async () => { + await encodeDecodeTest('opus', {}, async (packet, meta, i, n) => { + expect(packet.type).toBe('key'); + + if (i < n - 1) { + expect(packet.duration).toBe(0.02); + } + + if (i === 0) { + expect(meta.decoderConfig).toBeDefined(); + expect(meta.decoderConfig!.codec).toBe('opus'); + expect(meta.decoderConfig!.numberOfChannels).toBe(2); + expect(meta.decoderConfig!.sampleRate).toBe(48000); + expect(meta.decoderConfig!.description).toBeDefined(); + expect([...toUint8Array(meta.decoderConfig!.description!).slice(0, 8)]).toEqual([ + // OpusHead + 0x4f, 0x70, 0x75, 0x73, 0x48, 0x65, 0x61, 0x64, + ]); + } + }, async (sample, i, n) => { + expect(sample.numberOfChannels).toBe(2); + expect(sample.sampleRate).toBe(48000); + + if (i > 0 && i < n - 1) { + expect(sample.duration).toBe(0.02); + } + }); + }); + + test('MP3 encode & decode', async () => { + await encodeDecodeTest('mp3', {}, async (packet, meta, i, n) => { + expect(packet.type).toBe('key'); + + if (i < n - 1) { + expect(packet.duration).toBe(1152 / 48000); + } + + if (i === 0) { + expect(meta.decoderConfig).toBeDefined(); + expect(meta.decoderConfig!.codec).toBe('mp3'); + expect(meta.decoderConfig!.numberOfChannels).toBe(2); + expect(meta.decoderConfig!.sampleRate).toBe(48000); + expect(meta.decoderConfig!.description).toBeUndefined(); + } + }, async (sample) => { + expect(sample.numberOfChannels).toBe(2); + expect(sample.sampleRate).toBe(48000); + expect(sample.duration).toBe(1152 / 48000); + }); + }); + + test('Vorbis encode & decode', async () => { + await encodeDecodeTest('vorbis', {}, async (packet, meta, i) => { + expect(packet.type).toBe('key'); + + if (i === 0) { + expect(meta.decoderConfig).toBeDefined(); + expect(meta.decoderConfig!.codec).toBe('vorbis'); + expect(meta.decoderConfig!.numberOfChannels).toBe(2); + expect(meta.decoderConfig!.sampleRate).toBe(48000); + expect(meta.decoderConfig!.description).toBeDefined(); + expect(toUint8Array(meta.decoderConfig!.description!)[0]).toBe(2); // "Xiph extradata format" + } + }, async (sample) => { + expect(sample.numberOfChannels).toBe(2); + expect(sample.sampleRate).toBe(48000); + }); + }); + + test('FLAC encode & decode', async () => { + await encodeDecodeTest('flac', {}, async (packet, meta, i) => { + expect(packet.type).toBe('key'); + expect(packet.duration).toBe(0.096); + + if (i === 0) { + expect(meta.decoderConfig).toBeDefined(); + expect(meta.decoderConfig!.codec).toBe('flac'); + expect(meta.decoderConfig!.numberOfChannels).toBe(2); + expect(meta.decoderConfig!.sampleRate).toBe(48000); + expect(meta.decoderConfig!.description).toBeDefined(); + expect([...toUint8Array(meta.decoderConfig!.description!).slice(0, 4)]).toEqual([ + // fLaC + 0x66, 0x4c, 0x61, 0x43, + ]); + } + }, async (sample) => { + expect(sample.numberOfChannels).toBe(2); + expect(sample.sampleRate).toBe(48000); + expect(sample.duration).toBe(0.096); + }); + }); + + test('AC-3 encode & decode', async () => { + await encodeDecodeTest('ac3', {}, async (packet, meta, i) => { + expect(packet.type).toBe('key'); + expect(packet.duration).toBe(0.032); + + if (i === 0) { + expect(meta.decoderConfig).toBeDefined(); + expect(meta.decoderConfig!.codec).toBe('ac-3'); + expect(meta.decoderConfig!.numberOfChannels).toBe(2); + expect(meta.decoderConfig!.sampleRate).toBe(48000); + expect(meta.decoderConfig!.description).toBeUndefined(); + } + }, async (sample) => { + expect(sample.numberOfChannels).toBe(2); + expect(sample.sampleRate).toBe(48000); + expect(sample.duration).toBe(0.032); + }); + }); + + test('E-AC-3 encode & decode', async () => { + await encodeDecodeTest('eac3', {}, async (packet, meta, i) => { + expect(packet.type).toBe('key'); + expect(packet.duration).toBe(0.032); + + if (i === 0) { + expect(meta.decoderConfig).toBeDefined(); + expect(meta.decoderConfig!.codec).toBe('ec-3'); + expect(meta.decoderConfig!.numberOfChannels).toBe(2); + expect(meta.decoderConfig!.sampleRate).toBe(48000); + expect(meta.decoderConfig!.description).toBeUndefined(); + } + }, async (sample) => { + expect(sample.numberOfChannels).toBe(2); + expect(sample.sampleRate).toBe(48000); + expect(sample.duration).toBe(0.032); + }); + }); + + const encodeDecodeTest = async ( + codec: AudioCodec, + extraConfig: Partial, + onPacket: (packet: EncodedPacket, meta: EncodedAudioChunkMetadata, i: number, n: number) => Promise, + onSample: (sample: AudioSample, i: number, n: number) => Promise, + ) => { + const sampleRate = 48000; + const numberOfChannels = 2; + + const encoder = new NodeAvAudioEncoder(); + // @ts-expect-error Readonly + encoder.codec = codec; + // @ts-expect-error Readonly + encoder.config = { + codec: buildAudioCodecString(codec, numberOfChannels, sampleRate), + numberOfChannels, + sampleRate, + bitrate: 128000, + ...extraConfig, + } satisfies AudioEncoderConfig; + + const packets: EncodedPacket[] = []; + const metas: EncodedAudioChunkMetadata[] = []; + + // @ts-expect-error Readonly + encoder.onPacket = (packet: EncodedPacket, meta: EncodedAudioChunkMetadata) => { + packets.push(packet); + metas.push(meta); + }; + + await encoder.init(); + + const data = createF32SineWave(sampleRate, numberOfChannels, 2); + using inputSample = new AudioSample({ + data, + format: 'f32', + timestamp: 0, + numberOfChannels, + sampleRate, + }); + await encoder.encode(inputSample); + + await encoder.flush(); + + for (let i = 0; i < packets.length; i++) { + await onPacket(packets[i]!, metas[i]!, i, packets.length); + } + + const decoder = new NodeAvAudioDecoder(); + // @ts-expect-error Readonly + decoder.codec = codec; + // @ts-expect-error Readonly + decoder.config = metas[0]!.decoderConfig!; + + const decodedSamples: AudioSample[] = []; + + // @ts-expect-error Readonly + decoder.onSample = (sample: AudioSample) => { + decodedSamples.push(sample); + }; + + await decoder.init(); + + for (const packet of packets) { + await decoder.decode(packet); + } + + await decoder.flush(); + + expect(decodedSamples.length).toBeGreaterThan(0); + expect(last(decodedSamples)!.timestamp + last(decodedSamples)!.duration).toBeGreaterThanOrEqual(2); + + for (let i = 0; i < decodedSamples.length; i++) { + await onSample(decodedSamples[i]!, i, decodedSamples.length); + } + + const signalChunks: Float32Array[] = []; + + for (const sample of decodedSamples) { + const buf = new Float32Array(new ArrayBuffer(sample.allocationSize({ + format: 'f32-planar', planeIndex: 0, + }))); + sample.copyTo(buf, { format: 'f32-planar', planeIndex: 0 }); + signalChunks.push(buf); + + sample.close(); + } + + const totalSize = signalChunks.reduce((sum, chunk) => sum + chunk.length, 0); + const signal = new Float32Array(totalSize); + let offset = 0; + for (const chunk of signalChunks) { + signal.set(chunk, offset); + offset += chunk.length; + } + + const score = sine440Score(signal, sampleRate, 440); + expect(score).toBeGreaterThan(0.98); + + await encoder.close(); + await decoder.close(); + }; + + const sine440Score = (x: Float32Array, sampleRate: number, freq: number) => { + const w = 2 * Math.PI * freq / sampleRate; + + let ss = 0, cc = 0, sc = 0; + let xs = 0, xc = 0; + let xx = 0; + + for (let n = 0; n < x.length; n++) { + const s = Math.sin(w * n); + const c = Math.cos(w * n); + + ss += s * s; + cc += c * c; + sc += s * c; + + xs += x[n]! * s; + xc += x[n]! * c; + xx += x[n]! * x[n]!; + } + + const det = ss * cc - sc * sc; + + const a = (xs * cc - xc * sc) / det; + const b = (xc * ss - xs * sc) / det; + + let fitEnergy = 0; + + for (let n = 0; n < x.length; n++) { + const y = a * Math.sin(w * n) + b * Math.cos(w * n); + fitEnergy += y * y; + } + + const score = fitEnergy / xx; + return score; + }; + + test('AAC conversion roundtrip', { timeout: 10_000 }, async () => { + await conversionRoundtrip('aac'); + }); + + test('Opus conversion roundtrip', { timeout: 10_000 }, async () => { + await conversionRoundtrip('opus'); + }); + + test('MP3 conversion roundtrip', { timeout: 10_000 }, async () => { + await conversionRoundtrip('mp3'); + }); + + test('Vorbis conversion roundtrip', { timeout: 10_000 }, async () => { + await conversionRoundtrip('vorbis'); + }); + + test('FLAC conversion roundtrip', { timeout: 10_000 }, async () => { + await conversionRoundtrip('flac'); + }); + + test('AC-3 conversion roundtrip', { timeout: 10_000 }, async () => { + await conversionRoundtrip('ac3'); + }); + + test('E-AC-3 conversion roundtrip', { timeout: 10_000 }, async () => { + await conversionRoundtrip('eac3'); + }); + + const conversionRoundtrip = async (codec: AudioCodec) => { + using input = new Input({ + source: new FilePathSource('./test/public/trim-buck-bunny-ffmpeg.ts'), + formats: ALL_FORMATS, + }); + + const output = new Output({ + format: new Mp4OutputFormat(), + target: new BufferTarget(), + }); + + const inputTrack = await input.getPrimaryAudioTrack(); + assert(inputTrack); + + const conversion = await Conversion.init({ + input, + output, + video: { + discard: true, + }, audio: { + codec, + forceTranscode: true, + }, + trim: { + start: 0, + }, + }); + await conversion.execute(); + + using newInput = new Input({ + source: new BufferSource(output.target.buffer!), + formats: ALL_FORMATS, + }); + + const newInputTrack = await newInput.getPrimaryAudioTrack(); + assert(newInputTrack); + + expect(await newInputTrack.getCodec()).toBe(codec); + expect(await newInputTrack.computeDuration()).toBeCloseTo(await inputTrack.computeDuration(), 0); + + const sink = new AudioSampleSink(newInputTrack); + + let sampleCount = 0; + for await (using sample of sink.samples()) { + expect([await inputTrack.getNumberOfChannels(), 2].includes(sample.numberOfChannels)).toBe(true); + expect(sample.sampleRate).toBe(await inputTrack.getSampleRate()); + + sampleCount++; + } + + expect(sampleCount).toBeGreaterThan(0); + }; + + test('Custom AudioSample resource', async () => { + using input = new Input({ + source: new FilePathSource('./test/public/trim-buck-bunny-ffmpeg.ts'), + formats: ALL_FORMATS, + }); + + const audioTrack = await input.getPrimaryAudioTrack(); + assert(audioTrack); + + const sink = new AudioSampleSink(audioTrack); + using sample = await sink.getSample(await audioTrack.getFirstTimestamp()); + assert(sample); + + expect(sample._data).toBeInstanceOf(NodeAvFrameAudioSampleResource); + expect(sample.format).toBe('f32-planar'); + expect(sample.numberOfChannels).toBe(6); + expect(sample.sampleRate).toBe(48000); + expect(sample.timestamp).toBe(await audioTrack.getFirstTimestamp()); + expect(sample.duration).toBe(1024 / 48000); + expect(sample.numberOfFrames).toBe(1024); + + const size = sample.allocationSize({ planeIndex: 0 }); + expect(size).toBe(1024 * Float32Array.BYTES_PER_ELEMENT); + + // Test a bunch of copyTo()s: + const buf = new ArrayBuffer(1e6); + sample.copyTo(buf, { planeIndex: 0 }); + sample.copyTo(buf, { planeIndex: 1 }); + sample.copyTo(buf, { planeIndex: 2 }); + sample.copyTo(buf, { planeIndex: 3 }); + sample.copyTo(buf, { planeIndex: 4 }); + sample.copyTo(buf, { planeIndex: 5 }); + sample.copyTo(buf, { format: 'f32', planeIndex: 0 }); + }); }); -*/