Implement audio decoder & encoder

This commit is contained in:
Vanilagy
2026-05-10 16:37:25 +02:00
parent ab9958234a
commit 1e200edc83
8 changed files with 1128 additions and 40 deletions
+106
View File
@@ -0,0 +1,106 @@
import { AudioCodec, AudioSample, CustomAudioDecoder, EncodedPacket, MaybePromise } from 'mediabunny';
import * as NodeAv from 'node-av';
import { CODEC_TO_CODEC_ID, getChannelLayout } from './misc';
import { assert, roundToDivisor, toUint8Array } from '../../../src/misc';
import { NodeAvFrameAudioSampleResource } from './audio-sample';
export class NodeAvAudioDecoder extends CustomAudioDecoder {
frame!: NodeAv.Frame;
packet!: NodeAv.Packet;
codecContext!: NodeAv.CodecContext;
// eslint-disable-next-line @typescript-eslint/no-unused-vars
static override supports(codec: AudioCodec, config: AudioDecoderConfig): boolean {
return codec === 'aac'
|| codec === 'opus'
|| codec === 'mp3'
|| codec === 'vorbis'
|| codec === 'flac'
|| codec === 'ac3'
|| codec === 'eac3';
}
async init(): Promise<void> {
this.frame = new NodeAv.Frame();
this.frame.alloc();
this.packet = new NodeAv.Packet();
this.packet.alloc();
const codecId = CODEC_TO_CODEC_ID[this.codec];
assert(codecId !== undefined);
const codec = NodeAv.Codec.findDecoder(codecId);
if (codec === null) {
throw new Error(`Unable to obtain libav codec for '${this.codec}'.`);
}
const codecContext = new NodeAv.CodecContext();
codecContext.allocContext3(codec);
codecContext.sampleRate = this.config.sampleRate;
codecContext.channelLayout = getChannelLayout(this.config.numberOfChannels);
codecContext.codecType = NodeAv.AVMEDIA_TYPE_AUDIO;
codecContext.codecId = codecId;
codecContext.extraData = this.config.description
? Buffer.from(toUint8Array(this.config.description))
: null;
const ret = await codecContext.open2();
NodeAv.FFmpegError.throwIfError(ret, 'Open codec context');
this.codecContext = codecContext;
}
async decode(packet: EncodedPacket): Promise<void> {
this.packet.isKeyframe = packet.type === 'key';
this.packet.data = Buffer.from(packet.data);
this.packet.timeBase = { num: 1, den: 1e6 };
this.packet.pts = BigInt(packet.microsecondTimestamp);
this.packet.dts = NodeAv.AV_NOPTS_VALUE;
this.packet.duration = BigInt(packet.microsecondDuration);
const ret = await this.codecContext.sendPacket(this.packet);
NodeAv.FFmpegError.throwIfError(ret, 'Send packet');
while (true) {
const receiveRet = await this.codecContext.receiveFrame(this.frame);
if (receiveRet === NodeAv.AVERROR_EAGAIN || receiveRet === NodeAv.AVERROR_EOF) {
break;
}
this.receiveFrame(receiveRet);
}
}
receiveFrame(ret: number) {
NodeAv.FFmpegError.throwIfError(ret, 'Receive frame');
const timestamp = roundToDivisor(Number(this.frame.pts) / 1e6, this.config.sampleRate);
this.onSample(new AudioSample(new NodeAvFrameAudioSampleResource(this.frame, timestamp)));
}
async flush(): Promise<void> {
// Send null packet to signal flush
const ret = await this.codecContext.sendPacket(null);
NodeAv.FFmpegError.throwIfError(ret, 'Flush decoder');
// Keep receiving frames until no more are available
while (true) {
const receiveRet = await this.codecContext.receiveFrame(this.frame);
if (receiveRet === NodeAv.AVERROR_EAGAIN || receiveRet === NodeAv.AVERROR_EOF) {
// No more frames available
break;
}
this.receiveFrame(receiveRet);
}
this.codecContext.flushBuffers();
}
close(): MaybePromise<void> {
this.codecContext.freeContext();
this.frame.free();
this.packet.free();
}
}
+370
View File
@@ -0,0 +1,370 @@
import { AudioCodec, AudioSample, CustomAudioEncoder, MaybePromise, QUALITY_MEDIUM } from 'mediabunny';
import * as NodeAv from 'node-av';
import { CODEC_TO_CODEC_ID, fromAudioSampleFormat, getChannelLayout } from './misc';
import { assert, toUint8Array } from '../../../src/misc';
import { NodeAvFrameAudioSampleResource } from './audio-sample';
import { AdtsHeaderTemplate, buildAdtsHeaderTemplate, parseAacAudioSpecificConfig } from '../../../shared/aac-misc';
import { EncodedPacket } from 'mediabunny';
const AAC_SAMPLE_RATES
= [96000, 88200, 64000, 48000, 44100, 32000, 24000, 22050, 16000, 12000, 11025, 8000, 7350];
const OPUS_SAMPLE_RATES = [8000, 12000, 16000, 24000, 48000];
const MP3_SAMPLE_RATES = [8000, 11025, 12000, 16000, 22050, 24000, 32000, 44100, 48000];
const AC3_SAMPLE_RATES = [32000, 44100, 48000];
const FRAME_SIZE_FALLBACK = 1024; // Just 'cause
export class NodeAvAudioEncoder extends CustomAudioEncoder {
frame!: NodeAv.Frame;
packet!: NodeAv.Packet;
avCodec!: NodeAv.Codec;
codecContext: NodeAv.CodecContext | null = null;
firstExpectedTimestamp: number | null = null;
outputTimestampOffset = 0;
resampler: NodeAv.SoftwareResampleContext | null = null;
inputParametersKey: string | null = null;
resamplerInputSampleRate: number | null = null;
nextResamplerPts: bigint | null = null;
dstFrame: NodeAv.Frame | null = null;
packetEmitted = false;
adtsHeaderTemplate: AdtsHeaderTemplate | null = null;
static override supports(codec: AudioCodec, config: AudioEncoderConfig): boolean {
const { numberOfChannels, sampleRate } = config;
return (
codec === 'aac' && numberOfChannels >= 1 && numberOfChannels <= 48
&& AAC_SAMPLE_RATES.includes(sampleRate)
) || (
codec === 'opus' && numberOfChannels >= 1 && numberOfChannels <= 255
&& OPUS_SAMPLE_RATES.includes(sampleRate)
) || (
codec === 'mp3' && numberOfChannels >= 1 && numberOfChannels <= 2
&& MP3_SAMPLE_RATES.includes(sampleRate)
) || (
codec === 'vorbis' && numberOfChannels >= 1 && numberOfChannels <= 255
&& sampleRate <= 200000
) || (
codec === 'flac' && numberOfChannels >= 1 && numberOfChannels <= 8
&& sampleRate <= 655350
) || (
codec === 'ac3' && numberOfChannels >= 1 && numberOfChannels <= 6
&& AC3_SAMPLE_RATES.includes(sampleRate)
) || (
codec === 'eac3' && numberOfChannels >= 1 && numberOfChannels <= 16
&& AC3_SAMPLE_RATES.includes(sampleRate)
);
}
async init(): Promise<void> {
this.frame = new NodeAv.Frame();
this.frame.alloc();
this.packet = new NodeAv.Packet();
this.packet.alloc();
const codecId = CODEC_TO_CODEC_ID[this.codec];
assert(codecId !== undefined);
const codec = NodeAv.Codec.findEncoder(codecId);
if (!codec) {
throw new Error(`Unable to obtain libav codec for '${this.codec}'.`);
}
this.avCodec = codec;
await this.createCodecContext();
}
async createCodecContext() {
assert(this.codecContext === null);
const codecContext = new NodeAv.CodecContext();
codecContext.allocContext3(this.avCodec);
let sampleFormat = NodeAv.AV_SAMPLE_FMT_FLTP;
if (this.avCodec.sampleFormats && !this.avCodec.sampleFormats.includes(NodeAv.AV_SAMPLE_FMT_FLTP)) {
// Use a format that's supported
sampleFormat = this.avCodec.sampleFormats[0]!;
}
codecContext.sampleRate = this.config.sampleRate;
codecContext.channelLayout = getChannelLayout(this.config.numberOfChannels);
codecContext.codecType = NodeAv.AVMEDIA_TYPE_AUDIO;
codecContext.codecId = CODEC_TO_CODEC_ID[this.codec]!;
codecContext.sampleFormat = sampleFormat;
codecContext.timeBase = new NodeAv.Rational(1, this.config.sampleRate);
codecContext.bitRate = BigInt(this.config.bitrate ?? QUALITY_MEDIUM._toAudioBitrate(this.codec) ?? 0);
if (this.config.bitrateMode === 'constant') {
codecContext.rcMinRate = codecContext.bitRate;
codecContext.rcMaxRate = codecContext.bitRate;
}
const ret = await codecContext.open2();
NodeAv.FFmpegError.throwIfError(ret, 'Open codec context');
this.codecContext = codecContext;
}
async encode(audioSample: AudioSample): Promise<void> {
if (this.codecContext === null) {
await this.createCodecContext();
assert(this.codecContext);
}
this.firstExpectedTimestamp ??= audioSample.timestamp;
if (audioSample._data instanceof NodeAvFrameAudioSampleResource) {
this.frame.ref(audioSample._data.frame);
} else {
// Copy audio data from AudioData to FFmpeg Frame
const format = fromAudioSampleFormat(audioSample.format);
this.frame.format = format;
this.frame.nbSamples = audioSample.numberOfFrames;
this.frame.sampleRate = audioSample.sampleRate;
this.frame.channelLayout = getChannelLayout(audioSample.numberOfChannels);
this.frame.pts = BigInt(Math.round(audioSample.timestamp * audioSample.sampleRate));
this.frame.duration = BigInt(audioSample.numberOfFrames);
this.frame.allocBuffer();
assert(this.frame.data);
for (let i = 0; i < this.frame.data.length; i++) {
audioSample.copyTo(this.frame.data[i]!, { planeIndex: i });
}
}
const key = `${this.frame.sampleRate}:${this.frame.channels}:${this.frame.format}`;
if (this.inputParametersKey !== null && this.inputParametersKey !== key) {
throw new Error(
'Input audio parameters changed. For this audio encoder, you cannot change the input audio'
+ ' parameters over time.',
);
}
this.inputParametersKey = key;
// We need the resampler when:
// 1. Format conversion is needed (sample format, sample rate, or channel count differs)
// 2. The codec requires fixed frame sizes
const requiresResampler
= this.codecContext.frameSize > 0
|| this.codecContext.sampleFormat !== this.frame.format
|| this.codecContext.sampleRate !== this.frame.sampleRate
|| this.codecContext.channels !== this.frame.channels;
if (requiresResampler) {
if (!this.resampler) {
this.resampler = new NodeAv.SoftwareResampleContext();
this.resamplerInputSampleRate = this.frame.sampleRate;
const outLayout = getChannelLayout(this.codecContext.channels);
const inLayout = getChannelLayout(this.frame.channels);
const ret = this.resampler.allocSetOpts2(
outLayout, this.codecContext.sampleFormat, this.codecContext.sampleRate,
inLayout, this.frame.format as NodeAv.AVSampleFormat, this.frame.sampleRate,
);
NodeAv.FFmpegError.throwIfError(ret, 'allocSetOpts2');
const ret2 = this.resampler.init();
NodeAv.FFmpegError.throwIfError(ret2, 'init');
this.dstFrame = new NodeAv.Frame();
this.dstFrame.alloc();
this.dstFrame.channelLayout = outLayout;
this.dstFrame.sampleRate = this.codecContext.sampleRate;
this.dstFrame.format = this.codecContext.sampleFormat;
this.dstFrame.nbSamples = this.codecContext.frameSize || FRAME_SIZE_FALLBACK;
this.dstFrame.duration = BigInt(this.dstFrame.nbSamples);
this.dstFrame.allocBuffer();
this.nextResamplerPts = this.frame.pts;
}
const inputBuffers = this.frame.data;
if (!inputBuffers) {
throw new DOMException('Frame has no data', 'EncodingError');
}
await this.resampler.convert(null, 0, inputBuffers, this.frame.nbSamples);
await this.pullResampledFrames();
} else {
await this.sendFrameAndReceivePackets(this.frame);
}
}
async pullResampledFrames() {
assert(this.codecContext);
assert(this.resampler);
assert(this.dstFrame);
assert(this.nextResamplerPts !== null);
const frameSize = this.codecContext.frameSize || FRAME_SIZE_FALLBACK;
while (true) {
const available = this.resampler.getOutSamples(0);
if (available < frameSize) {
break;
}
await this.resampler.convert(this.dstFrame.data, frameSize, null, 0);
this.dstFrame.pts = this.nextResamplerPts;
await this.sendFrameAndReceivePackets(this.dstFrame);
this.nextResamplerPts += BigInt(frameSize);
}
}
async sendFrameAndReceivePackets(frame: NodeAv.Frame | null) {
assert(this.codecContext);
const ret = await this.codecContext.sendFrame(frame);
NodeAv.FFmpegError.throwIfError(ret, 'Send frame');
while (true) {
const receiveRet = await this.codecContext.receivePacket(this.packet);
if (receiveRet === NodeAv.AVERROR_EAGAIN || receiveRet === NodeAv.AVERROR_EOF) {
break;
}
this.receivePacket(receiveRet);
}
}
receivePacket(ret: number) {
assert(this.codecContext);
assert(this.firstExpectedTimestamp !== null);
NodeAv.FFmpegError.throwIfError(ret, 'Receive packet');
if (!this.packet.data) {
return;
}
let timestamp = Number(this.packet.pts) / this.codecContext.sampleRate;
const duration = Number(this.packet.duration) / this.codecContext.sampleRate;
let data: Uint8Array = this.packet.data;
let metadata: EncodedAudioChunkMetadata | undefined;
if (this.packetEmitted) {
metadata = {};
} else {
// To compensate for any negative timestamp things that FFmpeg might do. It does these for a reason, to
// indicate encoder delay, but the notion of this is not yet supported in Mediabunny.
this.outputTimestampOffset = this.firstExpectedTimestamp - timestamp;
const codecString = this.config.codec;
let description = this.codecContext.extraData
? toUint8Array(this.codecContext.extraData)
: undefined;
if (
description
// eslint-disable-next-line @stylistic/max-len
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access
&& this.codec === 'aac' && (this.config as any).aac?.format === 'adts'
) {
const config = parseAacAudioSpecificConfig(description);
this.adtsHeaderTemplate = buildAdtsHeaderTemplate(config);
description = undefined; // Not used with 'adts' format
}
if (description && this.codec === 'flac') {
// FFmpeg uses the STREAMINFO block as the extradata, but WebCodecs wants a different format:
// 1. The bytes 0x66 0x4C 0x61 0x43 ("fLaC" in ASCII)
// 2. A metadata block (called the STREAMINFO block) as described in section 7 of [FLAC]
// 3. Other optional metadata blocks (not included here, because, well, they're optional)
description = new Uint8Array([
0x66, 0x4c, 0x61, 0x43, // 'fLaC'
128, 0, 0, description.byteLength,
...description,
]);
}
metadata = {
decoderConfig: {
codec: codecString,
sampleRate: this.codecContext.sampleRate,
numberOfChannels: this.codecContext.channels,
description,
},
};
}
if (this.adtsHeaderTemplate) {
const frameLength = data.byteLength + this.adtsHeaderTemplate.header.byteLength;
this.adtsHeaderTemplate.bitstream.pos = 30;
this.adtsHeaderTemplate.bitstream.writeBits(13, frameLength);
const final = new Uint8Array(this.adtsHeaderTemplate.header.byteLength + data.byteLength);
final.set(this.adtsHeaderTemplate.header, 0);
final.set(data, this.adtsHeaderTemplate.header.byteLength);
data = final;
}
timestamp += this.outputTimestampOffset;
const packet = new EncodedPacket(
data,
'key',
timestamp,
duration,
);
this.onPacket(packet, metadata);
this.packetEmitted = true;
}
async flush(): Promise<void> {
if (!this.codecContext) {
return;
}
outer:
if (this.resampler) {
assert(this.resamplerInputSampleRate !== null);
const currentOutSamples = this.resampler.getOutSamples(0);
if (currentOutSamples === 0) {
break outer; // Clean cut-off point
}
const frameSize = this.codecContext.frameSize || FRAME_SIZE_FALLBACK;
assert(currentOutSamples < frameSize); // Because if it's more, it would've already been retrieved
const inputSamplesNeeded = Math.ceil(
((frameSize - currentOutSamples) / this.codecContext.sampleRate) * this.resamplerInputSampleRate,
);
this.resampler.injectSilence(inputSamplesNeeded);
await this.pullResampledFrames();
}
await this.sendFrameAndReceivePackets(null);
this.codecContext.freeContext();
this.codecContext = null;
this.packetEmitted = false;
this.firstExpectedTimestamp = null;
this.outputTimestampOffset = 0;
this.resampler?.free();
this.resampler = null;
this.inputParametersKey = null;
this.resamplerInputSampleRate = null;
this.nextResamplerPts = null;
this.dstFrame?.free();
this.dstFrame = null;
}
close(): MaybePromise<void> {
this.codecContext?.freeContext();
this.frame.free();
this.packet.free();
this.dstFrame?.free();
this.resampler?.free();
}
}
+55
View File
@@ -0,0 +1,55 @@
import { AudioSampleResource } from 'mediabunny';
import * as NodeAv from 'node-av';
import { toAudioSampleFormat } from './misc';
import { assert, toUint8Array } from '../../../src/misc';
export class NodeAvFrameAudioSampleResource extends AudioSampleResource {
frame: NodeAv.Frame;
timestamp: number;
constructor(frame: NodeAv.Frame, timestamp: number) {
super();
const clone = frame.clone();
if (!clone) {
throw new Error('Allocation failure during frame clone.');
}
this.frame = clone;
this.timestamp = timestamp;
}
getFormat(): AudioSampleFormat {
const result = toAudioSampleFormat(this.frame.format as NodeAv.AVSampleFormat);
if (result === null) {
throw new TypeError('Unsupported audio sample format: ' + this.frame.format);
}
return result;
}
getSampleRate(): number {
return this.frame.sampleRate;
}
getNumberOfChannels(): number {
return this.frame.channels;
}
getNumberOfFrames(): number {
return this.frame.nbSamples;
}
getTimestamp(): number {
return this.timestamp;
}
close(): void {
this.frame.free();
}
getDataPlane(planeIndex: number): Uint8Array {
assert(this.frame.data && planeIndex < this.frame.data.length);
return toUint8Array(this.frame.data[planeIndex]!);
}
}
+8 -1
View File
@@ -1,7 +1,9 @@
import { registerDecoder, registerEncoder } from 'mediabunny'; import { registerDecoder, registerEncoder } from 'mediabunny';
import { NodeAvVideoDecoder } from './video-decoder';
import * as NodeAv from 'node-av'; import * as NodeAv from 'node-av';
import { NodeAvVideoDecoder } from './video-decoder';
import { NodeAvVideoEncoder } from './video-encoder'; import { NodeAvVideoEncoder } from './video-encoder';
import { NodeAvAudioDecoder } from './audio-decoder';
import { NodeAvAudioEncoder } from './audio-encoder';
const SERVER_LOADED_SYMBOL = Symbol.for('@mediabunny/server loaded'); const SERVER_LOADED_SYMBOL = Symbol.for('@mediabunny/server loaded');
if ((globalThis as Record<symbol, unknown>)[SERVER_LOADED_SYMBOL]) { if ((globalThis as Record<symbol, unknown>)[SERVER_LOADED_SYMBOL]) {
@@ -23,6 +25,11 @@ export const registerMediabunnyServer = () => {
NodeAv.Log.setLevel(NodeAv.AV_LOG_ERROR); NodeAv.Log.setLevel(NodeAv.AV_LOG_ERROR);
// Video
registerDecoder(NodeAvVideoDecoder); registerDecoder(NodeAvVideoDecoder);
registerEncoder(NodeAvVideoEncoder); registerEncoder(NodeAvVideoEncoder);
// Audio
registerDecoder(NodeAvAudioDecoder);
registerEncoder(NodeAvAudioEncoder);
}; };
+6
View File
@@ -9,6 +9,12 @@ export const CODEC_TO_CODEC_ID: Partial<Record<MediaCodec, NodeAv.AVCodecID>> =
av1: NodeAv.AV_CODEC_ID_AV1, av1: NodeAv.AV_CODEC_ID_AV1,
aac: NodeAv.AV_CODEC_ID_AAC, aac: NodeAv.AV_CODEC_ID_AAC,
opus: NodeAv.AV_CODEC_ID_OPUS,
mp3: NodeAv.AV_CODEC_ID_MP3,
vorbis: NodeAv.AV_CODEC_ID_VORBIS,
flac: NodeAv.AV_CODEC_ID_FLAC,
ac3: NodeAv.AV_CODEC_ID_AC3,
eac3: NodeAv.AV_CODEC_ID_EAC3,
}; };
let cachedHardwareContext: NodeAv.HardwareContext | null | undefined = undefined; let cachedHardwareContext: NodeAv.HardwareContext | null | undefined = undefined;
+2 -4
View File
@@ -165,10 +165,8 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
assert(this.codecContext); assert(this.codecContext);
} }
const underlyingData = videoSample.getUnderlyingData(); if (videoSample._data instanceof NodeAvFrameVideoSampleResource) {
this.frame.ref(videoSample._data.frame);
if (underlyingData instanceof NodeAvFrameVideoSampleResource) {
this.frame.ref(underlyingData.frame);
} else { } else {
if (videoSample.format === null) { if (videoSample.format === null) {
throw new Error('Cannot encode foreign VideoSample with unknown (null) format.'); throw new Error('Cannot encode foreign VideoSample with unknown (null) format.');
+64 -29
View File
@@ -615,15 +615,6 @@ export class VideoSample implements Disposable {
finalizationRegistry?.register(this, { type: 'video', data: this._data }, this); finalizationRegistry?.register(this, { type: 'video', data: this._data }, this);
} }
getUnderlyingData() {
if (this._closed) {
throw new Error('VideoSample is closed.');
}
assert(this._data !== null);
return this._data;
}
/** Clones this video sample. */ /** Clones this video sample. */
clone() { clone() {
if (this._closed) { if (this._closed) {
@@ -1881,15 +1872,7 @@ export abstract class AudioSampleResource {
*/ */
abstract close(): void; abstract close(): void;
/** abstract getDataPlane(planeIndex: number): Uint8Array;
* Returns the number of bytes required to hold the underlying audio data as specified by the given options.
*/
abstract allocationSize(options: AudioSampleCopyToOptions): number;
/**
* Copies the underlying audio data to an ArrayBuffer or ArrayBufferView as specified by the given options.
*/
abstract copyTo(destination: AllowSharedBufferSource, options: AudioSampleCopyToOptions): void;
} }
/** /**
@@ -2004,10 +1987,30 @@ export class AudioSample implements Disposable {
init._referenceCount++; init._referenceCount++;
this.format = init.getFormat(); this.format = init.getFormat();
if (!AUDIO_SAMPLE_FORMATS.has(this.format)) {
throw new TypeError('getFormat() must return an AudioSampleFormat.');
}
this.sampleRate = init.getSampleRate(); this.sampleRate = init.getSampleRate();
if (!Number.isInteger(this.sampleRate) || this.sampleRate <= 0) {
throw new TypeError('getSampleRate() must return a positive integer.');
}
this.numberOfFrames = init.getNumberOfFrames(); this.numberOfFrames = init.getNumberOfFrames();
if (!Number.isInteger(this.numberOfFrames) || this.numberOfFrames < 0) {
throw new TypeError('getNumberOfFrames() must return a non-negative integer.');
}
this.numberOfChannels = init.getNumberOfChannels(); this.numberOfChannels = init.getNumberOfChannels();
if (!Number.isInteger(this.numberOfChannels) || this.numberOfChannels <= 0) {
throw new TypeError('getNumberOfChannels() must return a positive integer.');
}
this.timestamp = init.getTimestamp(); this.timestamp = init.getTimestamp();
if (!Number.isFinite(this.timestamp)) {
throw new TypeError('getTimestamp() must return a finite number.');
}
this.duration = this.numberOfFrames / this.sampleRate; this.duration = this.numberOfFrames / this.sampleRate;
} else { } else {
if (!init || typeof init !== 'object') { if (!init || typeof init !== 'object') {
@@ -2084,10 +2087,6 @@ export class AudioSample implements Disposable {
throw new Error('AudioSample is closed.'); throw new Error('AudioSample is closed.');
} }
if (this._data instanceof AudioSampleResource) {
return this._data.allocationSize(options);
}
const destFormat = options.format ?? this.format; const destFormat = options.format ?? this.format;
const frameOffset = options.frameOffset ?? 0; const frameOffset = options.frameOffset ?? 0;
@@ -2140,11 +2139,8 @@ export class AudioSample implements Disposable {
throw new Error('AudioSample is closed.'); throw new Error('AudioSample is closed.');
} }
if (this._data instanceof AudioSampleResource) { const { format, frameCount: optFrameCount, frameOffset: optFrameOffset } = options;
return this._data.copyTo(destination, options); let { planeIndex } = options;
}
const { planeIndex, format, frameCount: optFrameCount, frameOffset: optFrameOffset } = options;
const srcFormat = this.format; const srcFormat = this.format;
const destFormat = format ?? this.format; const destFormat = format ?? this.format;
@@ -2204,12 +2200,51 @@ export class AudioSample implements Disposable {
}); });
} }
} else { } else {
const uint8Data = this._data;
const srcView = toDataView(uint8Data);
const readFn = getReadFunction(srcFormat); const readFn = getReadFunction(srcFormat);
const srcBytesPerSample = getBytesPerSample(srcFormat); const srcBytesPerSample = getBytesPerSample(srcFormat);
const srcIsPlanar = formatIsPlanar(srcFormat); const srcIsPlanar = formatIsPlanar(srcFormat);
let uint8Data: Uint8Array;
if (this._data instanceof AudioSampleResource) {
const getDataPlaneValidated = (index: number) => {
const result = (this._data as AudioSampleResource).getDataPlane(index);
if (!(result instanceof Uint8Array)) {
throw new TypeError('getDataPlane() must return a Uint8Array.');
}
const expectedSize = numFrames * srcBytesPerSample * (srcIsPlanar ? 1 : numChannels);
if (result.byteLength !== expectedSize) {
throw new TypeError(
`Data plane ${index} has invalid size. Expected exactly ${expectedSize} bytes, got`
+ ` ${result.byteLength} bytes.`,
);
}
return result;
};
if (srcIsPlanar) {
if (destIsPlanar) {
// Only one source plane will be extracted, so let's fetch only that one
uint8Data = getDataPlaneValidated(planeIndex);
planeIndex = 0; // To fix the subsequent access
} else {
// Pack all planes tightly together
uint8Data = new Uint8Array(numFrames * srcBytesPerSample * numChannels);
for (let ch = 0; ch < numChannels; ch++) {
const planeData = getDataPlaneValidated(ch);
uint8Data.set(planeData, ch * numFrames * srcBytesPerSample);
}
}
} else {
uint8Data = getDataPlaneValidated(0); // That's the only plane there is
}
} else {
uint8Data = this._data;
}
const srcView = toDataView(uint8Data);
for (let i = 0; i < copyFrameCount; i++) { for (let i = 0; i < copyFrameCount; i++) {
if (destIsPlanar) { if (destIsPlanar) {
const destOffset = i * destBytesPerSample; const destOffset = i * destBytesPerSample;
+517 -6
View File
@@ -3,13 +3,14 @@ import { registerMediabunnyServer } from '@mediabunny/server';
import { Input } from '../../src/input.js'; import { Input } from '../../src/input.js';
import { BufferSource, FilePathSource } from '../../src/source.js'; import { BufferSource, FilePathSource } from '../../src/source.js';
import { ALL_FORMATS } from '../../src/input-format.js'; import { ALL_FORMATS } from '../../src/input-format.js';
import { assert, toUint8Array } from '../../src/misc.js'; import { assert, last, toUint8Array } from '../../src/misc.js';
import { EncodedPacketSink, VideoSampleSink } from '../../src/media-sink.js'; import { AudioSampleSink, EncodedPacketSink, VideoSampleSink } from '../../src/media-sink.js';
import { NodeAvVideoDecoder } from '../../packages/server/src/video-decoder.js'; import { NodeAvVideoDecoder } from '../../packages/server/src/video-decoder.js';
import { NodeAvVideoEncoder } from '../../packages/server/src/video-encoder.js'; import { NodeAvVideoEncoder } from '../../packages/server/src/video-encoder.js';
import { NodeAvAudioDecoder } from '../../packages/server/src/audio-decoder.js'; import { NodeAvAudioDecoder } from '../../packages/server/src/audio-decoder.js';
import { NodeAvAudioEncoder } from '../../packages/server/src/audio-encoder.js';
import { AudioSample, VideoSample } from '../../src/sample.js'; import { AudioSample, VideoSample } from '../../src/sample.js';
import { buildVideoCodecString, VideoCodec } from '../../src/codec.js'; import { AudioCodec, buildAudioCodecString, buildVideoCodecString, VideoCodec } from '../../src/codec.js';
import { EncodedPacket } from '../../src/packet.js'; import { EncodedPacket } from '../../src/packet.js';
import { import {
AvcNalUnitType, AvcNalUnitType,
@@ -24,6 +25,7 @@ import { Mp4OutputFormat } from '../../src/output-format.js';
import { BufferTarget } from '../../src/target.js'; import { BufferTarget } from '../../src/target.js';
import { Conversion } from '../../src/conversion.js'; import { Conversion } from '../../src/conversion.js';
import { NodeAvFrameVideoSampleResource } from '../../packages/server/src/video-sample.js'; import { NodeAvFrameVideoSampleResource } from '../../packages/server/src/video-sample.js';
import { NodeAvFrameAudioSampleResource } from '../../packages/server/src/audio-sample.js';
beforeAll(() => { beforeAll(() => {
registerMediabunnyServer(); registerMediabunnyServer();
@@ -600,7 +602,7 @@ describe('Video', async () => {
using sample = await sink.getSample(0); using sample = await sink.getSample(0);
assert(sample); assert(sample);
expect(sample.getUnderlyingData()).toBeInstanceOf(NodeAvFrameVideoSampleResource); expect(sample._data).toBeInstanceOf(NodeAvFrameVideoSampleResource);
expect(sample.format).toBe('I420'); expect(sample.format).toBe('I420');
expect(sample.codedWidth).toBe(1920); expect(sample.codedWidth).toBe(1920);
expect(sample.codedHeight).toBe(1080); expect(sample.codedHeight).toBe(1080);
@@ -736,7 +738,6 @@ describe('Video', async () => {
}); });
}); });
/*
describe('Audio', async () => { describe('Audio', async () => {
test('Decoder lifecycle', async () => { test('Decoder lifecycle', async () => {
using input = new Input({ using input = new Input({
@@ -807,5 +808,515 @@ describe('Audio', async () => {
await decoder.close(); await decoder.close();
}); });
test('Encoder lifecycle', async () => {
const encoder = new NodeAvAudioEncoder();
// @ts-expect-error Readonly
encoder.codec = 'aac';
// @ts-expect-error Readonly
encoder.config = {
codec: buildAudioCodecString('aac', 2, 48000),
numberOfChannels: 2,
sampleRate: 48000,
bitrate: 128000,
} satisfies AudioEncoderConfig;
let packetCount = 0;
// @ts-expect-error Readonly
encoder.onPacket = (packet: EncodedPacket, meta: EncodedAudioChunkMetadata) => {
expect(packet.duration).toBe(1024 / 48000);
expect(packet.type).toBe('key');
if (packetCount === 0) {
expect(packet.timestamp).toBe(0);
expect(meta.decoderConfig).toBeDefined();
expect(meta.decoderConfig!.codec).toBe('mp4a.40.2');
expect(meta.decoderConfig!.numberOfChannels).toBe(2);
expect(meta.decoderConfig!.sampleRate).toBe(48000);
expect(meta.decoderConfig?.description).toBeDefined();
}
packetCount++;
};
await encoder.init();
const data = createF32SineWave(48000, 2, 2);
using sample1 = new AudioSample({
data,
format: 'f32',
timestamp: 0,
numberOfChannels: 2,
sampleRate: 48000,
});
await encoder.encode(sample1);
await encoder.flush();
const minPacketCount = 2 * 48000 / 1024;
expect(packetCount).toBeGreaterThan(minPacketCount);
packetCount = 0;
using sample2 = new AudioSample({
data,
format: 'f32',
timestamp: 0,
numberOfChannels: 2,
sampleRate: 48000,
});
await encoder.encode(sample2);
await encoder.flush();
expect(packetCount).toBeGreaterThan(minPacketCount);
await encoder.close();
});
const createF32SineWave = (sampleRate: number, channels: number, durationSeconds: number) => {
const totalFrames = sampleRate * durationSeconds;
const data = new Float32Array(totalFrames * channels);
for (let i = 0; i < totalFrames; i++) {
const value = Math.sin(2 * Math.PI * 440 * i / sampleRate);
for (let ch = 0; ch < channels; ch++) {
data[i * channels + ch] = value;
}
}
return data;
};
test('AAC encode & decode, AAC format', async () => {
await encodeDecodeTest('aac', {
// @ts-expect-error Fucky type
aac: { format: 'aac' },
}, async (packet, meta, i) => {
expect(packet.type).toBe('key');
expect(packet.duration).toBe(1024 / 48000);
if (i === 0) {
expect(meta.decoderConfig).toBeDefined();
expect(meta.decoderConfig!.codec).toBe('mp4a.40.2');
expect(meta.decoderConfig!.numberOfChannels).toBe(2);
expect(meta.decoderConfig!.sampleRate).toBe(48000);
expect(meta.decoderConfig!.description).toBeDefined();
}
}, async (sample) => {
expect(sample.duration).toBe(1024 / 48000);
expect(sample.numberOfChannels).toBe(2);
expect(sample.sampleRate).toBe(48000);
});
});
test('AAC encode & decode, ADTS format', async () => {
await encodeDecodeTest('aac', {
// @ts-expect-error Fucky type
aac: { format: 'adts' },
}, async (packet, meta, i) => {
expect(packet.type).toBe('key');
expect(packet.duration).toBe(1024 / 48000);
expect(packet.data[0]).toBe(0xff);
if (i === 0) {
expect(meta.decoderConfig).toBeDefined();
expect(meta.decoderConfig!.codec).toBe('mp4a.40.2');
expect(meta.decoderConfig!.numberOfChannels).toBe(2);
expect(meta.decoderConfig!.sampleRate).toBe(48000);
expect(meta.decoderConfig!.description).toBeUndefined();
}
}, async (sample) => {
expect(sample.duration).toBe(1024 / 48000);
expect(sample.numberOfChannels).toBe(2);
expect(sample.sampleRate).toBe(48000);
});
});
test('Opus encode & decode', async () => {
await encodeDecodeTest('opus', {}, async (packet, meta, i, n) => {
expect(packet.type).toBe('key');
if (i < n - 1) {
expect(packet.duration).toBe(0.02);
}
if (i === 0) {
expect(meta.decoderConfig).toBeDefined();
expect(meta.decoderConfig!.codec).toBe('opus');
expect(meta.decoderConfig!.numberOfChannels).toBe(2);
expect(meta.decoderConfig!.sampleRate).toBe(48000);
expect(meta.decoderConfig!.description).toBeDefined();
expect([...toUint8Array(meta.decoderConfig!.description!).slice(0, 8)]).toEqual([
// OpusHead
0x4f, 0x70, 0x75, 0x73, 0x48, 0x65, 0x61, 0x64,
]);
}
}, async (sample, i, n) => {
expect(sample.numberOfChannels).toBe(2);
expect(sample.sampleRate).toBe(48000);
if (i > 0 && i < n - 1) {
expect(sample.duration).toBe(0.02);
}
});
});
test('MP3 encode & decode', async () => {
await encodeDecodeTest('mp3', {}, async (packet, meta, i, n) => {
expect(packet.type).toBe('key');
if (i < n - 1) {
expect(packet.duration).toBe(1152 / 48000);
}
if (i === 0) {
expect(meta.decoderConfig).toBeDefined();
expect(meta.decoderConfig!.codec).toBe('mp3');
expect(meta.decoderConfig!.numberOfChannels).toBe(2);
expect(meta.decoderConfig!.sampleRate).toBe(48000);
expect(meta.decoderConfig!.description).toBeUndefined();
}
}, async (sample) => {
expect(sample.numberOfChannels).toBe(2);
expect(sample.sampleRate).toBe(48000);
expect(sample.duration).toBe(1152 / 48000);
});
});
test('Vorbis encode & decode', async () => {
await encodeDecodeTest('vorbis', {}, async (packet, meta, i) => {
expect(packet.type).toBe('key');
if (i === 0) {
expect(meta.decoderConfig).toBeDefined();
expect(meta.decoderConfig!.codec).toBe('vorbis');
expect(meta.decoderConfig!.numberOfChannels).toBe(2);
expect(meta.decoderConfig!.sampleRate).toBe(48000);
expect(meta.decoderConfig!.description).toBeDefined();
expect(toUint8Array(meta.decoderConfig!.description!)[0]).toBe(2); // "Xiph extradata format"
}
}, async (sample) => {
expect(sample.numberOfChannels).toBe(2);
expect(sample.sampleRate).toBe(48000);
});
});
test('FLAC encode & decode', async () => {
await encodeDecodeTest('flac', {}, async (packet, meta, i) => {
expect(packet.type).toBe('key');
expect(packet.duration).toBe(0.096);
if (i === 0) {
expect(meta.decoderConfig).toBeDefined();
expect(meta.decoderConfig!.codec).toBe('flac');
expect(meta.decoderConfig!.numberOfChannels).toBe(2);
expect(meta.decoderConfig!.sampleRate).toBe(48000);
expect(meta.decoderConfig!.description).toBeDefined();
expect([...toUint8Array(meta.decoderConfig!.description!).slice(0, 4)]).toEqual([
// fLaC
0x66, 0x4c, 0x61, 0x43,
]);
}
}, async (sample) => {
expect(sample.numberOfChannels).toBe(2);
expect(sample.sampleRate).toBe(48000);
expect(sample.duration).toBe(0.096);
});
});
test('AC-3 encode & decode', async () => {
await encodeDecodeTest('ac3', {}, async (packet, meta, i) => {
expect(packet.type).toBe('key');
expect(packet.duration).toBe(0.032);
if (i === 0) {
expect(meta.decoderConfig).toBeDefined();
expect(meta.decoderConfig!.codec).toBe('ac-3');
expect(meta.decoderConfig!.numberOfChannels).toBe(2);
expect(meta.decoderConfig!.sampleRate).toBe(48000);
expect(meta.decoderConfig!.description).toBeUndefined();
}
}, async (sample) => {
expect(sample.numberOfChannels).toBe(2);
expect(sample.sampleRate).toBe(48000);
expect(sample.duration).toBe(0.032);
});
});
test('E-AC-3 encode & decode', async () => {
await encodeDecodeTest('eac3', {}, async (packet, meta, i) => {
expect(packet.type).toBe('key');
expect(packet.duration).toBe(0.032);
if (i === 0) {
expect(meta.decoderConfig).toBeDefined();
expect(meta.decoderConfig!.codec).toBe('ec-3');
expect(meta.decoderConfig!.numberOfChannels).toBe(2);
expect(meta.decoderConfig!.sampleRate).toBe(48000);
expect(meta.decoderConfig!.description).toBeUndefined();
}
}, async (sample) => {
expect(sample.numberOfChannels).toBe(2);
expect(sample.sampleRate).toBe(48000);
expect(sample.duration).toBe(0.032);
});
});
const encodeDecodeTest = async (
codec: AudioCodec,
extraConfig: Partial<AudioEncoderConfig>,
onPacket: (packet: EncodedPacket, meta: EncodedAudioChunkMetadata, i: number, n: number) => Promise<void>,
onSample: (sample: AudioSample, i: number, n: number) => Promise<void>,
) => {
const sampleRate = 48000;
const numberOfChannels = 2;
const encoder = new NodeAvAudioEncoder();
// @ts-expect-error Readonly
encoder.codec = codec;
// @ts-expect-error Readonly
encoder.config = {
codec: buildAudioCodecString(codec, numberOfChannels, sampleRate),
numberOfChannels,
sampleRate,
bitrate: 128000,
...extraConfig,
} satisfies AudioEncoderConfig;
const packets: EncodedPacket[] = [];
const metas: EncodedAudioChunkMetadata[] = [];
// @ts-expect-error Readonly
encoder.onPacket = (packet: EncodedPacket, meta: EncodedAudioChunkMetadata) => {
packets.push(packet);
metas.push(meta);
};
await encoder.init();
const data = createF32SineWave(sampleRate, numberOfChannels, 2);
using inputSample = new AudioSample({
data,
format: 'f32',
timestamp: 0,
numberOfChannels,
sampleRate,
});
await encoder.encode(inputSample);
await encoder.flush();
for (let i = 0; i < packets.length; i++) {
await onPacket(packets[i]!, metas[i]!, i, packets.length);
}
const decoder = new NodeAvAudioDecoder();
// @ts-expect-error Readonly
decoder.codec = codec;
// @ts-expect-error Readonly
decoder.config = metas[0]!.decoderConfig!;
const decodedSamples: AudioSample[] = [];
// @ts-expect-error Readonly
decoder.onSample = (sample: AudioSample) => {
decodedSamples.push(sample);
};
await decoder.init();
for (const packet of packets) {
await decoder.decode(packet);
}
await decoder.flush();
expect(decodedSamples.length).toBeGreaterThan(0);
expect(last(decodedSamples)!.timestamp + last(decodedSamples)!.duration).toBeGreaterThanOrEqual(2);
for (let i = 0; i < decodedSamples.length; i++) {
await onSample(decodedSamples[i]!, i, decodedSamples.length);
}
const signalChunks: Float32Array[] = [];
for (const sample of decodedSamples) {
const buf = new Float32Array(new ArrayBuffer(sample.allocationSize({
format: 'f32-planar', planeIndex: 0,
})));
sample.copyTo(buf, { format: 'f32-planar', planeIndex: 0 });
signalChunks.push(buf);
sample.close();
}
const totalSize = signalChunks.reduce((sum, chunk) => sum + chunk.length, 0);
const signal = new Float32Array(totalSize);
let offset = 0;
for (const chunk of signalChunks) {
signal.set(chunk, offset);
offset += chunk.length;
}
const score = sine440Score(signal, sampleRate, 440);
expect(score).toBeGreaterThan(0.98);
await encoder.close();
await decoder.close();
};
const sine440Score = (x: Float32Array, sampleRate: number, freq: number) => {
const w = 2 * Math.PI * freq / sampleRate;
let ss = 0, cc = 0, sc = 0;
let xs = 0, xc = 0;
let xx = 0;
for (let n = 0; n < x.length; n++) {
const s = Math.sin(w * n);
const c = Math.cos(w * n);
ss += s * s;
cc += c * c;
sc += s * c;
xs += x[n]! * s;
xc += x[n]! * c;
xx += x[n]! * x[n]!;
}
const det = ss * cc - sc * sc;
const a = (xs * cc - xc * sc) / det;
const b = (xc * ss - xs * sc) / det;
let fitEnergy = 0;
for (let n = 0; n < x.length; n++) {
const y = a * Math.sin(w * n) + b * Math.cos(w * n);
fitEnergy += y * y;
}
const score = fitEnergy / xx;
return score;
};
test('AAC conversion roundtrip', { timeout: 10_000 }, async () => {
await conversionRoundtrip('aac');
});
test('Opus conversion roundtrip', { timeout: 10_000 }, async () => {
await conversionRoundtrip('opus');
});
test('MP3 conversion roundtrip', { timeout: 10_000 }, async () => {
await conversionRoundtrip('mp3');
});
test('Vorbis conversion roundtrip', { timeout: 10_000 }, async () => {
await conversionRoundtrip('vorbis');
});
test('FLAC conversion roundtrip', { timeout: 10_000 }, async () => {
await conversionRoundtrip('flac');
});
test('AC-3 conversion roundtrip', { timeout: 10_000 }, async () => {
await conversionRoundtrip('ac3');
});
test('E-AC-3 conversion roundtrip', { timeout: 10_000 }, async () => {
await conversionRoundtrip('eac3');
});
const conversionRoundtrip = async (codec: AudioCodec) => {
using input = new Input({
source: new FilePathSource('./test/public/trim-buck-bunny-ffmpeg.ts'),
formats: ALL_FORMATS,
});
const output = new Output({
format: new Mp4OutputFormat(),
target: new BufferTarget(),
});
const inputTrack = await input.getPrimaryAudioTrack();
assert(inputTrack);
const conversion = await Conversion.init({
input,
output,
video: {
discard: true,
}, audio: {
codec,
forceTranscode: true,
},
trim: {
start: 0,
},
});
await conversion.execute();
using newInput = new Input({
source: new BufferSource(output.target.buffer!),
formats: ALL_FORMATS,
});
const newInputTrack = await newInput.getPrimaryAudioTrack();
assert(newInputTrack);
expect(await newInputTrack.getCodec()).toBe(codec);
expect(await newInputTrack.computeDuration()).toBeCloseTo(await inputTrack.computeDuration(), 0);
const sink = new AudioSampleSink(newInputTrack);
let sampleCount = 0;
for await (using sample of sink.samples()) {
expect([await inputTrack.getNumberOfChannels(), 2].includes(sample.numberOfChannels)).toBe(true);
expect(sample.sampleRate).toBe(await inputTrack.getSampleRate());
sampleCount++;
}
expect(sampleCount).toBeGreaterThan(0);
};
test('Custom AudioSample resource', async () => {
using input = new Input({
source: new FilePathSource('./test/public/trim-buck-bunny-ffmpeg.ts'),
formats: ALL_FORMATS,
});
const audioTrack = await input.getPrimaryAudioTrack();
assert(audioTrack);
const sink = new AudioSampleSink(audioTrack);
using sample = await sink.getSample(await audioTrack.getFirstTimestamp());
assert(sample);
expect(sample._data).toBeInstanceOf(NodeAvFrameAudioSampleResource);
expect(sample.format).toBe('f32-planar');
expect(sample.numberOfChannels).toBe(6);
expect(sample.sampleRate).toBe(48000);
expect(sample.timestamp).toBe(await audioTrack.getFirstTimestamp());
expect(sample.duration).toBe(1024 / 48000);
expect(sample.numberOfFrames).toBe(1024);
const size = sample.allocationSize({ planeIndex: 0 });
expect(size).toBe(1024 * Float32Array.BYTES_PER_ELEMENT);
// Test a bunch of copyTo()s:
const buf = new ArrayBuffer(1e6);
sample.copyTo(buf, { planeIndex: 0 });
sample.copyTo(buf, { planeIndex: 1 });
sample.copyTo(buf, { planeIndex: 2 });
sample.copyTo(buf, { planeIndex: 3 });
sample.copyTo(buf, { planeIndex: 4 });
sample.copyTo(buf, { planeIndex: 5 });
sample.copyTo(buf, { format: 'f32', planeIndex: 0 });
});
}); });
*/