Add @mediabunny/aac-encoder extension package

This commit is contained in:
Vanilagy
2026-03-04 17:05:34 +01:00
parent 76e0703b13
commit c4398fb88c
51 changed files with 1976 additions and 274 deletions
+190
View File
@@ -0,0 +1,190 @@
/*!
* Copyright (c) 2026-present, Vanilagy and contributors
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/.
*/
#include <emscripten.h>
#include <stdlib.h>
#include <string.h>
#include "libavcodec/avcodec.h"
#include "libavutil/opt.h"
#include "libavutil/channel_layout.h"
#include "libavutil/log.h"
typedef struct {
AVCodecContext *codec_ctx;
AVPacket *packet;
AVFrame *frame;
float *input_buffer;
int input_buffer_size;
int encoded_pts;
int encoded_duration;
} EncoderContext;
EMSCRIPTEN_KEEPALIVE
EncoderContext *init_encoder(int channels, int sample_rate, int bitrate) {
av_log_set_level(AV_LOG_ERROR);
const AVCodec *codec = avcodec_find_encoder(AV_CODEC_ID_AAC);
if (!codec) return NULL;
AVCodecContext *codec_ctx = avcodec_alloc_context3(codec);
if (!codec_ctx) return NULL;
codec_ctx->sample_fmt = AV_SAMPLE_FMT_FLTP;
codec_ctx->sample_rate = sample_rate;
codec_ctx->bit_rate = bitrate;
codec_ctx->time_base = (AVRational){1, sample_rate};
AVChannelLayout layout;
av_channel_layout_default(&layout, channels);
av_channel_layout_copy(&codec_ctx->ch_layout, &layout);
av_channel_layout_uninit(&layout);
if (avcodec_open2(codec_ctx, codec, NULL) < 0) {
avcodec_free_context(&codec_ctx);
return NULL;
}
AVPacket *packet = av_packet_alloc();
if (!packet) {
avcodec_free_context(&codec_ctx);
return NULL;
}
AVFrame *frame = av_frame_alloc();
if (!frame) {
av_packet_free(&packet);
avcodec_free_context(&codec_ctx);
return NULL;
}
frame->format = AV_SAMPLE_FMT_FLTP;
frame->sample_rate = sample_rate;
frame->nb_samples = codec_ctx->frame_size;
av_channel_layout_copy(&frame->ch_layout, &codec_ctx->ch_layout);
if (av_frame_get_buffer(frame, 0) < 0) {
av_frame_free(&frame);
av_packet_free(&packet);
avcodec_free_context(&codec_ctx);
return NULL;
}
EncoderContext *ctx = malloc(sizeof(EncoderContext));
if (!ctx) {
av_frame_free(&frame);
av_packet_free(&packet);
avcodec_free_context(&codec_ctx);
return NULL;
}
ctx->codec_ctx = codec_ctx;
ctx->packet = packet;
ctx->frame = frame;
ctx->input_buffer = NULL;
ctx->input_buffer_size = 0;
ctx->encoded_pts = 0;
ctx->encoded_duration = 0;
return ctx;
}
EMSCRIPTEN_KEEPALIVE
int get_encoder_frame_size(EncoderContext *ctx) {
return ctx->codec_ctx->frame_size;
}
EMSCRIPTEN_KEEPALIVE
uint8_t *get_encoder_extradata(EncoderContext *ctx) {
return ctx->codec_ctx->extradata;
}
EMSCRIPTEN_KEEPALIVE
int get_encoder_extradata_size(EncoderContext *ctx) {
return ctx->codec_ctx->extradata_size;
}
EMSCRIPTEN_KEEPALIVE
float *get_encode_input_ptr(EncoderContext *ctx, int size) {
if (ctx->input_buffer_size < size) {
free(ctx->input_buffer);
ctx->input_buffer = malloc(size);
if (!ctx->input_buffer) {
ctx->input_buffer_size = 0;
return NULL;
}
ctx->input_buffer_size = size;
}
return ctx->input_buffer;
}
EMSCRIPTEN_KEEPALIVE
int send_frame(EncoderContext *ctx, int pts) {
int channels = ctx->codec_ctx->ch_layout.nb_channels;
int frame_size = ctx->frame->nb_samples;
ctx->frame->pts = pts;
// Deinterleave f32 input into the frame's f32-planar planes
float *input = ctx->input_buffer;
for (int ch = 0; ch < channels; ch++) {
float *plane = (float *)ctx->frame->data[ch];
for (int i = 0; i < frame_size; i++) {
plane[i] = input[i * channels + ch];
}
}
return avcodec_send_frame(ctx->codec_ctx, ctx->frame);
}
EMSCRIPTEN_KEEPALIVE
int receive_packet(EncoderContext *ctx) {
int ret = avcodec_receive_packet(ctx->codec_ctx, ctx->packet);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
return 0;
}
if (ret < 0) return ret;
ctx->encoded_pts = ctx->packet->pts;
ctx->encoded_duration = ctx->packet->duration;
return ctx->packet->size;
}
EMSCRIPTEN_KEEPALIVE
void flush_encoder_start(EncoderContext *ctx) {
avcodec_send_frame(ctx->codec_ctx, NULL);
}
EMSCRIPTEN_KEEPALIVE
void reset_encoder(EncoderContext *ctx) {
avcodec_flush_buffers(ctx->codec_ctx);
}
EMSCRIPTEN_KEEPALIVE
uint8_t *get_encoded_data(EncoderContext *ctx) {
return ctx->packet->data;
}
EMSCRIPTEN_KEEPALIVE
int get_encoded_pts(EncoderContext *ctx) {
return ctx->encoded_pts;
}
EMSCRIPTEN_KEEPALIVE
int get_encoded_duration(EncoderContext *ctx) {
return ctx->encoded_duration;
}
EMSCRIPTEN_KEEPALIVE
void close_encoder(EncoderContext *ctx) {
free(ctx->input_buffer);
av_frame_free(&ctx->frame);
av_packet_free(&ctx->packet);
avcodec_free_context(&ctx->codec_ctx);
free(ctx);
}
+204
View File
@@ -0,0 +1,204 @@
/*!
* Copyright (c) 2026-present, Vanilagy and contributors
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/.
*/
import createModule from '../build/aac';
import type { WorkerCommand, WorkerResponse, WorkerResponseData } from './shared';
type ExtendedEmscriptenModule = EmscriptenModule & {
cwrap: typeof cwrap;
};
let module: ExtendedEmscriptenModule;
let modulePromise: Promise<ExtendedEmscriptenModule> | null = null;
let initEncoderFn: (channels: number, sampleRate: number, bitrate: number) => number;
let getEncoderFrameSize: (ctx: number) => number;
let getEncoderExtradata: (ctx: number) => number;
let getEncoderExtradataSize: (ctx: number) => number;
let getEncodeInputPtr: (ctx: number, size: number) => number;
let sendFrameFn: (ctx: number, pts: number) => number;
let receivePacketFn: (ctx: number) => number;
let flushEncoderStartFn: (ctx: number) => void;
let resetEncoderFn: (ctx: number) => void;
let getEncodedData: (ctx: number) => number;
let getEncodedPts: (ctx: number) => number;
let getEncodedDuration: (ctx: number) => number;
let closeEncoderFn: (ctx: number) => void;
const ensureModule = async () => {
if (!module) {
if (modulePromise) {
return modulePromise;
}
modulePromise = createModule() as Promise<ExtendedEmscriptenModule>;
module = await modulePromise;
modulePromise = null;
initEncoderFn = module.cwrap('init_encoder', 'number', ['number', 'number', 'number']);
getEncoderFrameSize = module.cwrap('get_encoder_frame_size', 'number', ['number']);
getEncoderExtradata = module.cwrap('get_encoder_extradata', 'number', ['number']);
getEncoderExtradataSize = module.cwrap('get_encoder_extradata_size', 'number', ['number']);
getEncodeInputPtr = module.cwrap('get_encode_input_ptr', 'number', ['number', 'number']);
sendFrameFn = module.cwrap('send_frame', 'number', ['number', 'number']);
receivePacketFn = module.cwrap('receive_packet', 'number', ['number']);
flushEncoderStartFn = module.cwrap('flush_encoder_start', null, ['number']);
resetEncoderFn = module.cwrap('reset_encoder', null, ['number']);
getEncodedData = module.cwrap('get_encoded_data', 'number', ['number']);
getEncodedPts = module.cwrap('get_encoded_pts', 'number', ['number']);
getEncodedDuration = module.cwrap('get_encoded_duration', 'number', ['number']);
closeEncoderFn = module.cwrap('close_encoder', null, ['number']);
}
};
const initEncoder = async (
numberOfChannels: number,
sampleRate: number,
bitrate: number,
) => {
await ensureModule();
const ctx = initEncoderFn(numberOfChannels, sampleRate, bitrate);
if (ctx === 0) {
throw new Error('Failed to initialize AAC encoder.');
}
const frameSize = getEncoderFrameSize(ctx);
const extradataPtr = getEncoderExtradata(ctx);
const extradataSize = getEncoderExtradataSize(ctx);
const extradata = module.HEAPU8.slice(extradataPtr, extradataPtr + extradataSize).buffer;
return { ctx, frameSize, extradata };
};
type PacketInfo = { encodedData: ArrayBuffer; pts: number; duration: number };
const drainPackets = (ctx: number) => {
const packets: PacketInfo[] = [];
let size: number;
while ((size = receivePacketFn(ctx)) > 0) {
const ptr = getEncodedData(ctx);
const encodedData = module.HEAPU8.slice(ptr, ptr + size).buffer;
const pts = getEncodedPts(ctx);
const duration = getEncodedDuration(ctx);
packets.push({ encodedData, pts, duration });
}
return packets;
};
const encode = (ctx: number, audioData: ArrayBuffer, timestamp: number) => {
const audioBytes = new Uint8Array(audioData);
const inputPtr = getEncodeInputPtr(ctx, audioBytes.length);
if (inputPtr === 0) {
throw new Error('Failed to allocate encoder input buffer.');
}
module.HEAPU8.set(audioBytes, inputPtr);
const ret = sendFrameFn(ctx, timestamp);
if (ret < 0) {
throw new Error(`Encode failed with error code ${ret}.`);
}
return drainPackets(ctx);
};
const onMessage = (data: { id: number; command: WorkerCommand }) => {
const { id, command } = data;
const handleCommand = async (): Promise<void> => {
try {
let result: WorkerResponseData;
const transferables: Transferable[] = [];
switch (command.type) {
case 'init': {
const { ctx, frameSize, extradata } = await initEncoder(
command.data.numberOfChannels,
command.data.sampleRate,
command.data.bitrate,
);
result = { type: command.type, ctx, frameSize, extradata };
transferables.push(extradata);
}; break;
case 'encode': {
const packets = encode(
command.data.ctx,
command.data.audioData,
command.data.timestamp,
);
for (const p of packets) {
transferables.push(p.encodedData);
}
result = { type: command.type, packets };
}; break;
case 'flush': {
flushEncoderStartFn(command.data.ctx);
const packets = drainPackets(command.data.ctx);
for (const p of packets) {
transferables.push(p.encodedData);
}
resetEncoderFn(command.data.ctx);
result = { type: command.type, packets };
}; break;
case 'close': {
closeEncoderFn(command.data.ctx);
result = { type: command.type };
}; break;
}
const response: WorkerResponse = {
id,
success: true,
data: result,
};
sendMessage(response, transferables);
} catch (error: unknown) {
const response: WorkerResponse = {
id,
success: false,
error,
};
sendMessage(response);
}
};
void handleCommand();
};
const sendMessage = (data: unknown, transferables?: Transferable[]) => {
if (parentPort) {
parentPort.postMessage(data, transferables ?? []);
} else {
self.postMessage(data, { transfer: transferables ?? [] });
}
};
let parentPort: {
postMessage: (data: unknown, transferables?: Transferable[]) => void;
on: (event: string, listener: (data: never) => void) => void;
} | null = null;
if (typeof self === 'undefined') {
const workerModule = 'worker_threads';
// eslint-disable-next-line @stylistic/max-len
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-require-imports, @typescript-eslint/no-unsafe-member-access
parentPort = require(workerModule).parentPort;
}
if (parentPort) {
parentPort.on('message', onMessage);
} else {
self.addEventListener('message', event => onMessage(event.data as { id: number; command: WorkerCommand }));
}
+306
View File
@@ -0,0 +1,306 @@
/*!
* Copyright (c) 2026-present, Vanilagy and contributors
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/.
*/
import {
type AdtsHeaderTemplate,
buildAdtsHeaderTemplate,
parseAacAudioSpecificConfig,
writeAdtsFrameLength,
} from '../../../shared/aac-misc';
import {
CustomAudioEncoder,
AudioCodec,
AudioSample,
EncodedPacket,
registerEncoder,
} from 'mediabunny';
import type { WorkerCommand, WorkerResponse, WorkerResponseData } from './shared';
// @ts-expect-error An esbuild plugin handles this, TypeScript doesn't need to understand
import createWorker from './encode.worker';
const AAC_SAMPLE_RATES = [
96000, 88200, 64000, 48000, 44100, 32000,
24000, 22050, 16000, 12000, 11025, 8000, 7350,
];
class AacEncoder extends CustomAudioEncoder {
private worker: Worker | null = null;
private nextMessageId = 0;
private pendingMessages = new Map<number, {
resolve: (value: WorkerResponseData) => void;
reject: (reason?: unknown) => void;
}>();
private ctx = 0;
private encoderFrameSize = 0;
private sampleRate = 0;
private numberOfChannels = 0;
private chunkMetadata: EncodedAudioChunkMetadata = {};
private useAdts = false;
private adtsHeaderTemplate: AdtsHeaderTemplate | null = null;
private description: Uint8Array | null = null;
// Accumulate interleaved f32 samples until we have a full frame
private pendingBuffer = new Float32Array(2 ** 16);
private pendingFrames = 0;
private nextSampleTimestampInSamples: number | null = null;
private nextPacketTimestampInSamples: number | null = null;
static override supports(codec: AudioCodec, config: AudioEncoderConfig): boolean {
return codec === 'aac'
&& config.numberOfChannels >= 1
&& config.numberOfChannels <= 8
&& AAC_SAMPLE_RATES.includes(config.sampleRate)
&& config.bitrate !== undefined;
}
async init() {
// eslint-disable-next-line @typescript-eslint/no-unsafe-call
this.worker = (await createWorker()) as Worker;
const onMessage = (data: WorkerResponse) => {
const pending = this.pendingMessages.get(data.id);
assert(pending !== undefined);
this.pendingMessages.delete(data.id);
if (data.success) {
pending.resolve(data.data);
} else {
pending.reject(data.error);
}
};
if (this.worker.addEventListener) {
this.worker.addEventListener('message', event => onMessage(event.data as WorkerResponse));
} else {
const nodeWorker = this.worker as unknown as {
on: (event: string, listener: (data: never) => void) => void;
};
nodeWorker.on('message', onMessage);
}
assert(this.config.bitrate !== undefined);
this.sampleRate = this.config.sampleRate;
this.numberOfChannels = this.config.numberOfChannels;
const result = await this.sendCommand({
type: 'init',
data: {
numberOfChannels: this.config.numberOfChannels,
sampleRate: this.config.sampleRate,
bitrate: this.config.bitrate,
},
});
this.ctx = result.ctx;
this.encoderFrameSize = result.frameSize;
// The ffmpeg encoder provides an AudioSpecificConfig as extradata after init
const description = new Uint8Array(result.extradata);
const aacConfig = (this.config as { aac?: { format?: 'aac' | 'adts' } }).aac;
this.useAdts = aacConfig?.format === 'adts';
if (this.useAdts) {
const audioSpecificConfig = parseAacAudioSpecificConfig(description);
this.adtsHeaderTemplate = buildAdtsHeaderTemplate(audioSpecificConfig);
}
this.description = this.useAdts ? null : description;
this.resetInternalState();
}
private resetInternalState() {
this.pendingFrames = 0;
this.nextSampleTimestampInSamples = null;
this.nextPacketTimestampInSamples = null;
this.chunkMetadata = {
decoderConfig: {
codec: 'mp4a.40.2',
numberOfChannels: this.config.numberOfChannels,
sampleRate: this.config.sampleRate,
...(this.description ? { description: this.description } : {}),
},
};
}
async encode(audioSample: AudioSample) {
if (this.nextSampleTimestampInSamples === null) {
this.nextSampleTimestampInSamples = Math.round(audioSample.timestamp * this.sampleRate);
this.nextPacketTimestampInSamples = this.nextSampleTimestampInSamples;
}
const channels = this.numberOfChannels;
const incomingFrames = audioSample.numberOfFrames;
// Extract interleaved f32 data
const totalBytes = audioSample.allocationSize({ format: 'f32', planeIndex: 0 });
const audioBytes = new Uint8Array(totalBytes);
audioSample.copyTo(audioBytes, { format: 'f32', planeIndex: 0 });
const incomingData = new Float32Array(audioBytes.buffer);
const requiredSamples = (this.pendingFrames + incomingFrames) * channels;
if (requiredSamples > this.pendingBuffer.length) {
let newSize = this.pendingBuffer.length;
while (newSize < requiredSamples) {
newSize *= 2;
}
const newBuffer = new Float32Array(newSize);
newBuffer.set(this.pendingBuffer.subarray(0, this.pendingFrames * channels));
this.pendingBuffer = newBuffer;
}
this.pendingBuffer.set(incomingData, this.pendingFrames * channels);
this.pendingFrames += incomingFrames;
while (this.pendingFrames >= this.encoderFrameSize) {
await this.encodeOneFrame();
}
}
async flush() {
// Pad remaining samples with silence to fill a full frame
if (this.pendingFrames > 0) {
const channels = this.numberOfChannels;
const frameSize = this.encoderFrameSize;
const usedSamples = this.pendingFrames * channels;
const frameSamples = frameSize * channels;
this.pendingBuffer.fill(0, usedSamples, frameSamples);
this.pendingFrames = frameSize;
await this.encodeOneFrame();
}
const result = await this.sendCommand({ type: 'flush', data: { ctx: this.ctx } });
this.emitPackets(result.packets);
this.resetInternalState();
}
close() {
void this.sendCommand({ type: 'close', data: { ctx: this.ctx } });
this.worker?.terminate();
}
private async encodeOneFrame() {
assert(this.nextSampleTimestampInSamples !== null);
assert(this.nextPacketTimestampInSamples !== null);
const channels = this.numberOfChannels;
const frameSize = this.encoderFrameSize;
const frameSamples = frameSize * channels;
const frameData = this.pendingBuffer.slice(0, frameSamples);
// Shift remaining using copyWithin
this.pendingFrames -= frameSize;
if (this.pendingFrames > 0) {
this.pendingBuffer.copyWithin(0, frameSamples, frameSamples + this.pendingFrames * channels);
}
const audioData = frameData.buffer;
const result = await this.sendCommand({
type: 'encode',
data: {
ctx: this.ctx,
audioData,
timestamp: this.nextSampleTimestampInSamples,
},
}, [audioData]);
this.nextSampleTimestampInSamples += frameSize;
this.emitPackets(result.packets);
}
private emitPackets(packets: Array<{ encodedData: ArrayBuffer; pts: number; duration: number }>) {
assert(this.nextPacketTimestampInSamples !== null);
for (const p of packets) {
let data = new Uint8Array(p.encodedData);
if (this.useAdts) {
assert(this.adtsHeaderTemplate !== null);
const { header, bitstream } = this.adtsHeaderTemplate;
const frameLength = header.byteLength + data.byteLength;
writeAdtsFrameLength(bitstream, frameLength);
const adtsFrame = new Uint8Array(frameLength);
adtsFrame.set(header, 0);
adtsFrame.set(data, header.byteLength);
data = adtsFrame;
}
const packet = new EncodedPacket(
data,
'key',
this.nextPacketTimestampInSamples / this.sampleRate,
p.duration / this.sampleRate,
);
this.nextPacketTimestampInSamples += p.duration;
this.onPacket(
packet,
this.chunkMetadata,
);
this.chunkMetadata = {};
}
}
private sendCommand<T extends string>(
command: WorkerCommand & { type: T },
transferables?: Transferable[],
) {
return new Promise<WorkerResponseData & { type: T }>((resolve, reject) => {
const id = this.nextMessageId++;
this.pendingMessages.set(id, {
resolve: resolve as (value: WorkerResponseData) => void,
reject,
});
assert(this.worker);
if (transferables) {
this.worker.postMessage({ id, command }, transferables);
} else {
this.worker.postMessage({ id, command });
}
});
}
}
/**
* Registers the AAC encoder, which Mediabunny will then use automatically when applicable. Make sure to call this
* function before starting any encoding task.
*
* Preferably, wrap the call in a condition to avoid overriding any native AAC encoder:
*
* ```ts
* import { canEncodeAudio } from 'mediabunny';
* import { registerAacEncoder } from '@mediabunny/aac-encoder';
*
* if (!(await canEncodeAudio('aac'))) {
* registerAacEncoder();
* }
* ```
*
* @group \@mediabunny/aac-encoder
* @public
*/
export const registerAacEncoder = () => {
registerEncoder(AacEncoder);
};
function assert(x: unknown): asserts x {
if (!x) {
throw new Error('Assertion failed.');
}
}
+20
View File
@@ -0,0 +1,20 @@
/*!
* Copyright (c) 2026-present, Vanilagy and contributors
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/.
*/
const AAC_ENCODER_LOADED_SYMBOL = Symbol.for('@mediabunny/aac-encoder loaded');
if ((globalThis as Record<symbol, unknown>)[AAC_ENCODER_LOADED_SYMBOL]) {
console.error(
'[WARNING]\n@mediabunny/aac-encoder was loaded twice.'
+ ' This will likely cause the encoder not to work correctly.'
+ ' Check if multiple dependencies are importing different versions of @mediabunny/aac-encoder,'
+ ' or if something is being bundled incorrectly.',
);
}
(globalThis as Record<symbol, unknown>)[AAC_ENCODER_LOADED_SYMBOL] = true;
export { registerAacEncoder } from './encoder';
+66
View File
@@ -0,0 +1,66 @@
/*!
* Copyright (c) 2026-present, Vanilagy and contributors
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/.
*/
export type WorkerCommand = {
type: 'init';
data: {
numberOfChannels: number;
sampleRate: number;
bitrate: number;
};
} | {
type: 'encode';
data: {
ctx: number;
audioData: ArrayBuffer;
timestamp: number;
};
} | {
type: 'flush';
data: {
ctx: number;
};
} | {
type: 'close';
data: {
ctx: number;
};
};
export type WorkerResponseData = {
type: 'init';
ctx: number;
frameSize: number;
extradata: ArrayBuffer;
} | {
type: 'encode';
packets: Array<{
encodedData: ArrayBuffer;
pts: number;
duration: number;
}>;
} | {
type: 'flush';
packets: Array<{
encodedData: ArrayBuffer;
pts: number;
duration: number;
}>;
} | {
type: 'close';
};
export type WorkerResponse = {
id: number;
} & ({
success: true;
data: WorkerResponseData;
} | {
success: false;
error: unknown;
});