mirror of
https://github.com/arcodange-org/mediabunny.git
synced 2026-10-03 13:53:53 +02:00
Implement audio decoder & encoder
This commit is contained in:
@@ -0,0 +1,106 @@
|
||||
import { AudioCodec, AudioSample, CustomAudioDecoder, EncodedPacket, MaybePromise } from 'mediabunny';
|
||||
import * as NodeAv from 'node-av';
|
||||
import { CODEC_TO_CODEC_ID, getChannelLayout } from './misc';
|
||||
import { assert, roundToDivisor, toUint8Array } from '../../../src/misc';
|
||||
import { NodeAvFrameAudioSampleResource } from './audio-sample';
|
||||
|
||||
export class NodeAvAudioDecoder extends CustomAudioDecoder {
|
||||
frame!: NodeAv.Frame;
|
||||
packet!: NodeAv.Packet;
|
||||
codecContext!: NodeAv.CodecContext;
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
static override supports(codec: AudioCodec, config: AudioDecoderConfig): boolean {
|
||||
return codec === 'aac'
|
||||
|| codec === 'opus'
|
||||
|| codec === 'mp3'
|
||||
|| codec === 'vorbis'
|
||||
|| codec === 'flac'
|
||||
|| codec === 'ac3'
|
||||
|| codec === 'eac3';
|
||||
}
|
||||
|
||||
async init(): Promise<void> {
|
||||
this.frame = new NodeAv.Frame();
|
||||
this.frame.alloc();
|
||||
this.packet = new NodeAv.Packet();
|
||||
this.packet.alloc();
|
||||
|
||||
const codecId = CODEC_TO_CODEC_ID[this.codec];
|
||||
assert(codecId !== undefined);
|
||||
|
||||
const codec = NodeAv.Codec.findDecoder(codecId);
|
||||
if (codec === null) {
|
||||
throw new Error(`Unable to obtain libav codec for '${this.codec}'.`);
|
||||
}
|
||||
|
||||
const codecContext = new NodeAv.CodecContext();
|
||||
codecContext.allocContext3(codec);
|
||||
|
||||
codecContext.sampleRate = this.config.sampleRate;
|
||||
codecContext.channelLayout = getChannelLayout(this.config.numberOfChannels);
|
||||
codecContext.codecType = NodeAv.AVMEDIA_TYPE_AUDIO;
|
||||
codecContext.codecId = codecId;
|
||||
codecContext.extraData = this.config.description
|
||||
? Buffer.from(toUint8Array(this.config.description))
|
||||
: null;
|
||||
|
||||
const ret = await codecContext.open2();
|
||||
NodeAv.FFmpegError.throwIfError(ret, 'Open codec context');
|
||||
|
||||
this.codecContext = codecContext;
|
||||
}
|
||||
|
||||
async decode(packet: EncodedPacket): Promise<void> {
|
||||
this.packet.isKeyframe = packet.type === 'key';
|
||||
this.packet.data = Buffer.from(packet.data);
|
||||
this.packet.timeBase = { num: 1, den: 1e6 };
|
||||
this.packet.pts = BigInt(packet.microsecondTimestamp);
|
||||
this.packet.dts = NodeAv.AV_NOPTS_VALUE;
|
||||
this.packet.duration = BigInt(packet.microsecondDuration);
|
||||
|
||||
const ret = await this.codecContext.sendPacket(this.packet);
|
||||
NodeAv.FFmpegError.throwIfError(ret, 'Send packet');
|
||||
|
||||
while (true) {
|
||||
const receiveRet = await this.codecContext.receiveFrame(this.frame);
|
||||
if (receiveRet === NodeAv.AVERROR_EAGAIN || receiveRet === NodeAv.AVERROR_EOF) {
|
||||
break;
|
||||
}
|
||||
|
||||
this.receiveFrame(receiveRet);
|
||||
}
|
||||
}
|
||||
|
||||
receiveFrame(ret: number) {
|
||||
NodeAv.FFmpegError.throwIfError(ret, 'Receive frame');
|
||||
|
||||
const timestamp = roundToDivisor(Number(this.frame.pts) / 1e6, this.config.sampleRate);
|
||||
this.onSample(new AudioSample(new NodeAvFrameAudioSampleResource(this.frame, timestamp)));
|
||||
}
|
||||
|
||||
async flush(): Promise<void> {
|
||||
// Send null packet to signal flush
|
||||
const ret = await this.codecContext.sendPacket(null);
|
||||
NodeAv.FFmpegError.throwIfError(ret, 'Flush decoder');
|
||||
|
||||
// Keep receiving frames until no more are available
|
||||
while (true) {
|
||||
const receiveRet = await this.codecContext.receiveFrame(this.frame);
|
||||
if (receiveRet === NodeAv.AVERROR_EAGAIN || receiveRet === NodeAv.AVERROR_EOF) {
|
||||
// No more frames available
|
||||
break;
|
||||
}
|
||||
|
||||
this.receiveFrame(receiveRet);
|
||||
}
|
||||
|
||||
this.codecContext.flushBuffers();
|
||||
}
|
||||
|
||||
close(): MaybePromise<void> {
|
||||
this.codecContext.freeContext();
|
||||
this.frame.free();
|
||||
this.packet.free();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,370 @@
|
||||
import { AudioCodec, AudioSample, CustomAudioEncoder, MaybePromise, QUALITY_MEDIUM } from 'mediabunny';
|
||||
import * as NodeAv from 'node-av';
|
||||
import { CODEC_TO_CODEC_ID, fromAudioSampleFormat, getChannelLayout } from './misc';
|
||||
import { assert, toUint8Array } from '../../../src/misc';
|
||||
import { NodeAvFrameAudioSampleResource } from './audio-sample';
|
||||
import { AdtsHeaderTemplate, buildAdtsHeaderTemplate, parseAacAudioSpecificConfig } from '../../../shared/aac-misc';
|
||||
import { EncodedPacket } from 'mediabunny';
|
||||
|
||||
const AAC_SAMPLE_RATES
|
||||
= [96000, 88200, 64000, 48000, 44100, 32000, 24000, 22050, 16000, 12000, 11025, 8000, 7350];
|
||||
const OPUS_SAMPLE_RATES = [8000, 12000, 16000, 24000, 48000];
|
||||
const MP3_SAMPLE_RATES = [8000, 11025, 12000, 16000, 22050, 24000, 32000, 44100, 48000];
|
||||
const AC3_SAMPLE_RATES = [32000, 44100, 48000];
|
||||
|
||||
const FRAME_SIZE_FALLBACK = 1024; // Just 'cause
|
||||
|
||||
export class NodeAvAudioEncoder extends CustomAudioEncoder {
|
||||
frame!: NodeAv.Frame;
|
||||
packet!: NodeAv.Packet;
|
||||
avCodec!: NodeAv.Codec;
|
||||
codecContext: NodeAv.CodecContext | null = null;
|
||||
firstExpectedTimestamp: number | null = null;
|
||||
outputTimestampOffset = 0;
|
||||
|
||||
resampler: NodeAv.SoftwareResampleContext | null = null;
|
||||
inputParametersKey: string | null = null;
|
||||
resamplerInputSampleRate: number | null = null;
|
||||
nextResamplerPts: bigint | null = null;
|
||||
dstFrame: NodeAv.Frame | null = null;
|
||||
packetEmitted = false;
|
||||
adtsHeaderTemplate: AdtsHeaderTemplate | null = null;
|
||||
|
||||
static override supports(codec: AudioCodec, config: AudioEncoderConfig): boolean {
|
||||
const { numberOfChannels, sampleRate } = config;
|
||||
|
||||
return (
|
||||
codec === 'aac' && numberOfChannels >= 1 && numberOfChannels <= 48
|
||||
&& AAC_SAMPLE_RATES.includes(sampleRate)
|
||||
) || (
|
||||
codec === 'opus' && numberOfChannels >= 1 && numberOfChannels <= 255
|
||||
&& OPUS_SAMPLE_RATES.includes(sampleRate)
|
||||
) || (
|
||||
codec === 'mp3' && numberOfChannels >= 1 && numberOfChannels <= 2
|
||||
&& MP3_SAMPLE_RATES.includes(sampleRate)
|
||||
) || (
|
||||
codec === 'vorbis' && numberOfChannels >= 1 && numberOfChannels <= 255
|
||||
&& sampleRate <= 200000
|
||||
) || (
|
||||
codec === 'flac' && numberOfChannels >= 1 && numberOfChannels <= 8
|
||||
&& sampleRate <= 655350
|
||||
) || (
|
||||
codec === 'ac3' && numberOfChannels >= 1 && numberOfChannels <= 6
|
||||
&& AC3_SAMPLE_RATES.includes(sampleRate)
|
||||
) || (
|
||||
codec === 'eac3' && numberOfChannels >= 1 && numberOfChannels <= 16
|
||||
&& AC3_SAMPLE_RATES.includes(sampleRate)
|
||||
);
|
||||
}
|
||||
|
||||
async init(): Promise<void> {
|
||||
this.frame = new NodeAv.Frame();
|
||||
this.frame.alloc();
|
||||
this.packet = new NodeAv.Packet();
|
||||
this.packet.alloc();
|
||||
|
||||
const codecId = CODEC_TO_CODEC_ID[this.codec];
|
||||
assert(codecId !== undefined);
|
||||
|
||||
const codec = NodeAv.Codec.findEncoder(codecId);
|
||||
if (!codec) {
|
||||
throw new Error(`Unable to obtain libav codec for '${this.codec}'.`);
|
||||
}
|
||||
|
||||
this.avCodec = codec;
|
||||
|
||||
await this.createCodecContext();
|
||||
}
|
||||
|
||||
async createCodecContext() {
|
||||
assert(this.codecContext === null);
|
||||
|
||||
const codecContext = new NodeAv.CodecContext();
|
||||
codecContext.allocContext3(this.avCodec);
|
||||
|
||||
let sampleFormat = NodeAv.AV_SAMPLE_FMT_FLTP;
|
||||
if (this.avCodec.sampleFormats && !this.avCodec.sampleFormats.includes(NodeAv.AV_SAMPLE_FMT_FLTP)) {
|
||||
// Use a format that's supported
|
||||
sampleFormat = this.avCodec.sampleFormats[0]!;
|
||||
}
|
||||
|
||||
codecContext.sampleRate = this.config.sampleRate;
|
||||
codecContext.channelLayout = getChannelLayout(this.config.numberOfChannels);
|
||||
codecContext.codecType = NodeAv.AVMEDIA_TYPE_AUDIO;
|
||||
codecContext.codecId = CODEC_TO_CODEC_ID[this.codec]!;
|
||||
codecContext.sampleFormat = sampleFormat;
|
||||
codecContext.timeBase = new NodeAv.Rational(1, this.config.sampleRate);
|
||||
codecContext.bitRate = BigInt(this.config.bitrate ?? QUALITY_MEDIUM._toAudioBitrate(this.codec) ?? 0);
|
||||
|
||||
if (this.config.bitrateMode === 'constant') {
|
||||
codecContext.rcMinRate = codecContext.bitRate;
|
||||
codecContext.rcMaxRate = codecContext.bitRate;
|
||||
}
|
||||
|
||||
const ret = await codecContext.open2();
|
||||
NodeAv.FFmpegError.throwIfError(ret, 'Open codec context');
|
||||
|
||||
this.codecContext = codecContext;
|
||||
}
|
||||
|
||||
async encode(audioSample: AudioSample): Promise<void> {
|
||||
if (this.codecContext === null) {
|
||||
await this.createCodecContext();
|
||||
assert(this.codecContext);
|
||||
}
|
||||
|
||||
this.firstExpectedTimestamp ??= audioSample.timestamp;
|
||||
|
||||
if (audioSample._data instanceof NodeAvFrameAudioSampleResource) {
|
||||
this.frame.ref(audioSample._data.frame);
|
||||
} else {
|
||||
// Copy audio data from AudioData to FFmpeg Frame
|
||||
const format = fromAudioSampleFormat(audioSample.format);
|
||||
this.frame.format = format;
|
||||
this.frame.nbSamples = audioSample.numberOfFrames;
|
||||
this.frame.sampleRate = audioSample.sampleRate;
|
||||
this.frame.channelLayout = getChannelLayout(audioSample.numberOfChannels);
|
||||
this.frame.pts = BigInt(Math.round(audioSample.timestamp * audioSample.sampleRate));
|
||||
this.frame.duration = BigInt(audioSample.numberOfFrames);
|
||||
|
||||
this.frame.allocBuffer();
|
||||
assert(this.frame.data);
|
||||
|
||||
for (let i = 0; i < this.frame.data.length; i++) {
|
||||
audioSample.copyTo(this.frame.data[i]!, { planeIndex: i });
|
||||
}
|
||||
}
|
||||
|
||||
const key = `${this.frame.sampleRate}:${this.frame.channels}:${this.frame.format}`;
|
||||
if (this.inputParametersKey !== null && this.inputParametersKey !== key) {
|
||||
throw new Error(
|
||||
'Input audio parameters changed. For this audio encoder, you cannot change the input audio'
|
||||
+ ' parameters over time.',
|
||||
);
|
||||
}
|
||||
this.inputParametersKey = key;
|
||||
|
||||
// We need the resampler when:
|
||||
// 1. Format conversion is needed (sample format, sample rate, or channel count differs)
|
||||
// 2. The codec requires fixed frame sizes
|
||||
const requiresResampler
|
||||
= this.codecContext.frameSize > 0
|
||||
|| this.codecContext.sampleFormat !== this.frame.format
|
||||
|| this.codecContext.sampleRate !== this.frame.sampleRate
|
||||
|| this.codecContext.channels !== this.frame.channels;
|
||||
|
||||
if (requiresResampler) {
|
||||
if (!this.resampler) {
|
||||
this.resampler = new NodeAv.SoftwareResampleContext();
|
||||
this.resamplerInputSampleRate = this.frame.sampleRate;
|
||||
|
||||
const outLayout = getChannelLayout(this.codecContext.channels);
|
||||
const inLayout = getChannelLayout(this.frame.channels);
|
||||
|
||||
const ret = this.resampler.allocSetOpts2(
|
||||
outLayout, this.codecContext.sampleFormat, this.codecContext.sampleRate,
|
||||
inLayout, this.frame.format as NodeAv.AVSampleFormat, this.frame.sampleRate,
|
||||
);
|
||||
NodeAv.FFmpegError.throwIfError(ret, 'allocSetOpts2');
|
||||
|
||||
const ret2 = this.resampler.init();
|
||||
NodeAv.FFmpegError.throwIfError(ret2, 'init');
|
||||
|
||||
this.dstFrame = new NodeAv.Frame();
|
||||
this.dstFrame.alloc();
|
||||
this.dstFrame.channelLayout = outLayout;
|
||||
this.dstFrame.sampleRate = this.codecContext.sampleRate;
|
||||
this.dstFrame.format = this.codecContext.sampleFormat;
|
||||
this.dstFrame.nbSamples = this.codecContext.frameSize || FRAME_SIZE_FALLBACK;
|
||||
this.dstFrame.duration = BigInt(this.dstFrame.nbSamples);
|
||||
this.dstFrame.allocBuffer();
|
||||
|
||||
this.nextResamplerPts = this.frame.pts;
|
||||
}
|
||||
|
||||
const inputBuffers = this.frame.data;
|
||||
if (!inputBuffers) {
|
||||
throw new DOMException('Frame has no data', 'EncodingError');
|
||||
}
|
||||
await this.resampler.convert(null, 0, inputBuffers, this.frame.nbSamples);
|
||||
|
||||
await this.pullResampledFrames();
|
||||
} else {
|
||||
await this.sendFrameAndReceivePackets(this.frame);
|
||||
}
|
||||
}
|
||||
|
||||
async pullResampledFrames() {
|
||||
assert(this.codecContext);
|
||||
assert(this.resampler);
|
||||
assert(this.dstFrame);
|
||||
assert(this.nextResamplerPts !== null);
|
||||
|
||||
const frameSize = this.codecContext.frameSize || FRAME_SIZE_FALLBACK;
|
||||
|
||||
while (true) {
|
||||
const available = this.resampler.getOutSamples(0);
|
||||
if (available < frameSize) {
|
||||
break;
|
||||
}
|
||||
|
||||
await this.resampler.convert(this.dstFrame.data, frameSize, null, 0);
|
||||
|
||||
this.dstFrame.pts = this.nextResamplerPts;
|
||||
|
||||
await this.sendFrameAndReceivePackets(this.dstFrame);
|
||||
this.nextResamplerPts += BigInt(frameSize);
|
||||
}
|
||||
}
|
||||
|
||||
async sendFrameAndReceivePackets(frame: NodeAv.Frame | null) {
|
||||
assert(this.codecContext);
|
||||
|
||||
const ret = await this.codecContext.sendFrame(frame);
|
||||
NodeAv.FFmpegError.throwIfError(ret, 'Send frame');
|
||||
|
||||
while (true) {
|
||||
const receiveRet = await this.codecContext.receivePacket(this.packet);
|
||||
if (receiveRet === NodeAv.AVERROR_EAGAIN || receiveRet === NodeAv.AVERROR_EOF) {
|
||||
break;
|
||||
}
|
||||
|
||||
this.receivePacket(receiveRet);
|
||||
}
|
||||
}
|
||||
|
||||
receivePacket(ret: number) {
|
||||
assert(this.codecContext);
|
||||
assert(this.firstExpectedTimestamp !== null);
|
||||
NodeAv.FFmpegError.throwIfError(ret, 'Receive packet');
|
||||
|
||||
if (!this.packet.data) {
|
||||
return;
|
||||
}
|
||||
|
||||
let timestamp = Number(this.packet.pts) / this.codecContext.sampleRate;
|
||||
const duration = Number(this.packet.duration) / this.codecContext.sampleRate;
|
||||
|
||||
let data: Uint8Array = this.packet.data;
|
||||
|
||||
let metadata: EncodedAudioChunkMetadata | undefined;
|
||||
if (this.packetEmitted) {
|
||||
metadata = {};
|
||||
} else {
|
||||
// To compensate for any negative timestamp things that FFmpeg might do. It does these for a reason, to
|
||||
// indicate encoder delay, but the notion of this is not yet supported in Mediabunny.
|
||||
this.outputTimestampOffset = this.firstExpectedTimestamp - timestamp;
|
||||
|
||||
const codecString = this.config.codec;
|
||||
let description = this.codecContext.extraData
|
||||
? toUint8Array(this.codecContext.extraData)
|
||||
: undefined;
|
||||
|
||||
if (
|
||||
description
|
||||
// eslint-disable-next-line @stylistic/max-len
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access
|
||||
&& this.codec === 'aac' && (this.config as any).aac?.format === 'adts'
|
||||
) {
|
||||
const config = parseAacAudioSpecificConfig(description);
|
||||
this.adtsHeaderTemplate = buildAdtsHeaderTemplate(config);
|
||||
description = undefined; // Not used with 'adts' format
|
||||
}
|
||||
|
||||
if (description && this.codec === 'flac') {
|
||||
// FFmpeg uses the STREAMINFO block as the extradata, but WebCodecs wants a different format:
|
||||
// 1. The bytes 0x66 0x4C 0x61 0x43 ("fLaC" in ASCII)
|
||||
// 2. A metadata block (called the STREAMINFO block) as described in section 7 of [FLAC]
|
||||
// 3. Other optional metadata blocks (not included here, because, well, they're optional)
|
||||
description = new Uint8Array([
|
||||
0x66, 0x4c, 0x61, 0x43, // 'fLaC'
|
||||
128, 0, 0, description.byteLength,
|
||||
...description,
|
||||
]);
|
||||
}
|
||||
|
||||
metadata = {
|
||||
decoderConfig: {
|
||||
codec: codecString,
|
||||
sampleRate: this.codecContext.sampleRate,
|
||||
numberOfChannels: this.codecContext.channels,
|
||||
description,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
if (this.adtsHeaderTemplate) {
|
||||
const frameLength = data.byteLength + this.adtsHeaderTemplate.header.byteLength;
|
||||
this.adtsHeaderTemplate.bitstream.pos = 30;
|
||||
this.adtsHeaderTemplate.bitstream.writeBits(13, frameLength);
|
||||
|
||||
const final = new Uint8Array(this.adtsHeaderTemplate.header.byteLength + data.byteLength);
|
||||
final.set(this.adtsHeaderTemplate.header, 0);
|
||||
final.set(data, this.adtsHeaderTemplate.header.byteLength);
|
||||
|
||||
data = final;
|
||||
}
|
||||
|
||||
timestamp += this.outputTimestampOffset;
|
||||
|
||||
const packet = new EncodedPacket(
|
||||
data,
|
||||
'key',
|
||||
timestamp,
|
||||
duration,
|
||||
);
|
||||
|
||||
this.onPacket(packet, metadata);
|
||||
this.packetEmitted = true;
|
||||
}
|
||||
|
||||
async flush(): Promise<void> {
|
||||
if (!this.codecContext) {
|
||||
return;
|
||||
}
|
||||
|
||||
outer:
|
||||
if (this.resampler) {
|
||||
assert(this.resamplerInputSampleRate !== null);
|
||||
|
||||
const currentOutSamples = this.resampler.getOutSamples(0);
|
||||
if (currentOutSamples === 0) {
|
||||
break outer; // Clean cut-off point
|
||||
}
|
||||
|
||||
const frameSize = this.codecContext.frameSize || FRAME_SIZE_FALLBACK;
|
||||
assert(currentOutSamples < frameSize); // Because if it's more, it would've already been retrieved
|
||||
|
||||
const inputSamplesNeeded = Math.ceil(
|
||||
((frameSize - currentOutSamples) / this.codecContext.sampleRate) * this.resamplerInputSampleRate,
|
||||
);
|
||||
this.resampler.injectSilence(inputSamplesNeeded);
|
||||
|
||||
await this.pullResampledFrames();
|
||||
}
|
||||
|
||||
await this.sendFrameAndReceivePackets(null);
|
||||
|
||||
this.codecContext.freeContext();
|
||||
this.codecContext = null;
|
||||
this.packetEmitted = false;
|
||||
this.firstExpectedTimestamp = null;
|
||||
this.outputTimestampOffset = 0;
|
||||
|
||||
this.resampler?.free();
|
||||
this.resampler = null;
|
||||
this.inputParametersKey = null;
|
||||
this.resamplerInputSampleRate = null;
|
||||
this.nextResamplerPts = null;
|
||||
this.dstFrame?.free();
|
||||
this.dstFrame = null;
|
||||
}
|
||||
|
||||
close(): MaybePromise<void> {
|
||||
this.codecContext?.freeContext();
|
||||
this.frame.free();
|
||||
this.packet.free();
|
||||
this.dstFrame?.free();
|
||||
this.resampler?.free();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
import { AudioSampleResource } from 'mediabunny';
|
||||
import * as NodeAv from 'node-av';
|
||||
import { toAudioSampleFormat } from './misc';
|
||||
import { assert, toUint8Array } from '../../../src/misc';
|
||||
|
||||
export class NodeAvFrameAudioSampleResource extends AudioSampleResource {
|
||||
frame: NodeAv.Frame;
|
||||
timestamp: number;
|
||||
|
||||
constructor(frame: NodeAv.Frame, timestamp: number) {
|
||||
super();
|
||||
|
||||
const clone = frame.clone();
|
||||
if (!clone) {
|
||||
throw new Error('Allocation failure during frame clone.');
|
||||
}
|
||||
|
||||
this.frame = clone;
|
||||
this.timestamp = timestamp;
|
||||
}
|
||||
|
||||
getFormat(): AudioSampleFormat {
|
||||
const result = toAudioSampleFormat(this.frame.format as NodeAv.AVSampleFormat);
|
||||
if (result === null) {
|
||||
throw new TypeError('Unsupported audio sample format: ' + this.frame.format);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
getSampleRate(): number {
|
||||
return this.frame.sampleRate;
|
||||
}
|
||||
|
||||
getNumberOfChannels(): number {
|
||||
return this.frame.channels;
|
||||
}
|
||||
|
||||
getNumberOfFrames(): number {
|
||||
return this.frame.nbSamples;
|
||||
}
|
||||
|
||||
getTimestamp(): number {
|
||||
return this.timestamp;
|
||||
}
|
||||
|
||||
close(): void {
|
||||
this.frame.free();
|
||||
}
|
||||
|
||||
getDataPlane(planeIndex: number): Uint8Array {
|
||||
assert(this.frame.data && planeIndex < this.frame.data.length);
|
||||
return toUint8Array(this.frame.data[planeIndex]!);
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,9 @@
|
||||
import { registerDecoder, registerEncoder } from 'mediabunny';
|
||||
import { NodeAvVideoDecoder } from './video-decoder';
|
||||
import * as NodeAv from 'node-av';
|
||||
import { NodeAvVideoDecoder } from './video-decoder';
|
||||
import { NodeAvVideoEncoder } from './video-encoder';
|
||||
import { NodeAvAudioDecoder } from './audio-decoder';
|
||||
import { NodeAvAudioEncoder } from './audio-encoder';
|
||||
|
||||
const SERVER_LOADED_SYMBOL = Symbol.for('@mediabunny/server loaded');
|
||||
if ((globalThis as Record<symbol, unknown>)[SERVER_LOADED_SYMBOL]) {
|
||||
@@ -23,6 +25,11 @@ export const registerMediabunnyServer = () => {
|
||||
|
||||
NodeAv.Log.setLevel(NodeAv.AV_LOG_ERROR);
|
||||
|
||||
// Video
|
||||
registerDecoder(NodeAvVideoDecoder);
|
||||
registerEncoder(NodeAvVideoEncoder);
|
||||
|
||||
// Audio
|
||||
registerDecoder(NodeAvAudioDecoder);
|
||||
registerEncoder(NodeAvAudioEncoder);
|
||||
};
|
||||
|
||||
@@ -9,6 +9,12 @@ export const CODEC_TO_CODEC_ID: Partial<Record<MediaCodec, NodeAv.AVCodecID>> =
|
||||
av1: NodeAv.AV_CODEC_ID_AV1,
|
||||
|
||||
aac: NodeAv.AV_CODEC_ID_AAC,
|
||||
opus: NodeAv.AV_CODEC_ID_OPUS,
|
||||
mp3: NodeAv.AV_CODEC_ID_MP3,
|
||||
vorbis: NodeAv.AV_CODEC_ID_VORBIS,
|
||||
flac: NodeAv.AV_CODEC_ID_FLAC,
|
||||
ac3: NodeAv.AV_CODEC_ID_AC3,
|
||||
eac3: NodeAv.AV_CODEC_ID_EAC3,
|
||||
};
|
||||
|
||||
let cachedHardwareContext: NodeAv.HardwareContext | null | undefined = undefined;
|
||||
|
||||
@@ -165,10 +165,8 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
|
||||
assert(this.codecContext);
|
||||
}
|
||||
|
||||
const underlyingData = videoSample.getUnderlyingData();
|
||||
|
||||
if (underlyingData instanceof NodeAvFrameVideoSampleResource) {
|
||||
this.frame.ref(underlyingData.frame);
|
||||
if (videoSample._data instanceof NodeAvFrameVideoSampleResource) {
|
||||
this.frame.ref(videoSample._data.frame);
|
||||
} else {
|
||||
if (videoSample.format === null) {
|
||||
throw new Error('Cannot encode foreign VideoSample with unknown (null) format.');
|
||||
|
||||
+64
-29
@@ -615,15 +615,6 @@ export class VideoSample implements Disposable {
|
||||
finalizationRegistry?.register(this, { type: 'video', data: this._data }, this);
|
||||
}
|
||||
|
||||
getUnderlyingData() {
|
||||
if (this._closed) {
|
||||
throw new Error('VideoSample is closed.');
|
||||
}
|
||||
|
||||
assert(this._data !== null);
|
||||
return this._data;
|
||||
}
|
||||
|
||||
/** Clones this video sample. */
|
||||
clone() {
|
||||
if (this._closed) {
|
||||
@@ -1881,15 +1872,7 @@ export abstract class AudioSampleResource {
|
||||
*/
|
||||
abstract close(): void;
|
||||
|
||||
/**
|
||||
* Returns the number of bytes required to hold the underlying audio data as specified by the given options.
|
||||
*/
|
||||
abstract allocationSize(options: AudioSampleCopyToOptions): number;
|
||||
|
||||
/**
|
||||
* Copies the underlying audio data to an ArrayBuffer or ArrayBufferView as specified by the given options.
|
||||
*/
|
||||
abstract copyTo(destination: AllowSharedBufferSource, options: AudioSampleCopyToOptions): void;
|
||||
abstract getDataPlane(planeIndex: number): Uint8Array;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -2004,10 +1987,30 @@ export class AudioSample implements Disposable {
|
||||
init._referenceCount++;
|
||||
|
||||
this.format = init.getFormat();
|
||||
if (!AUDIO_SAMPLE_FORMATS.has(this.format)) {
|
||||
throw new TypeError('getFormat() must return an AudioSampleFormat.');
|
||||
}
|
||||
|
||||
this.sampleRate = init.getSampleRate();
|
||||
if (!Number.isInteger(this.sampleRate) || this.sampleRate <= 0) {
|
||||
throw new TypeError('getSampleRate() must return a positive integer.');
|
||||
}
|
||||
|
||||
this.numberOfFrames = init.getNumberOfFrames();
|
||||
if (!Number.isInteger(this.numberOfFrames) || this.numberOfFrames < 0) {
|
||||
throw new TypeError('getNumberOfFrames() must return a non-negative integer.');
|
||||
}
|
||||
|
||||
this.numberOfChannels = init.getNumberOfChannels();
|
||||
if (!Number.isInteger(this.numberOfChannels) || this.numberOfChannels <= 0) {
|
||||
throw new TypeError('getNumberOfChannels() must return a positive integer.');
|
||||
}
|
||||
|
||||
this.timestamp = init.getTimestamp();
|
||||
if (!Number.isFinite(this.timestamp)) {
|
||||
throw new TypeError('getTimestamp() must return a finite number.');
|
||||
}
|
||||
|
||||
this.duration = this.numberOfFrames / this.sampleRate;
|
||||
} else {
|
||||
if (!init || typeof init !== 'object') {
|
||||
@@ -2084,10 +2087,6 @@ export class AudioSample implements Disposable {
|
||||
throw new Error('AudioSample is closed.');
|
||||
}
|
||||
|
||||
if (this._data instanceof AudioSampleResource) {
|
||||
return this._data.allocationSize(options);
|
||||
}
|
||||
|
||||
const destFormat = options.format ?? this.format;
|
||||
|
||||
const frameOffset = options.frameOffset ?? 0;
|
||||
@@ -2140,11 +2139,8 @@ export class AudioSample implements Disposable {
|
||||
throw new Error('AudioSample is closed.');
|
||||
}
|
||||
|
||||
if (this._data instanceof AudioSampleResource) {
|
||||
return this._data.copyTo(destination, options);
|
||||
}
|
||||
|
||||
const { planeIndex, format, frameCount: optFrameCount, frameOffset: optFrameOffset } = options;
|
||||
const { format, frameCount: optFrameCount, frameOffset: optFrameOffset } = options;
|
||||
let { planeIndex } = options;
|
||||
|
||||
const srcFormat = this.format;
|
||||
const destFormat = format ?? this.format;
|
||||
@@ -2204,12 +2200,51 @@ export class AudioSample implements Disposable {
|
||||
});
|
||||
}
|
||||
} else {
|
||||
const uint8Data = this._data;
|
||||
const srcView = toDataView(uint8Data);
|
||||
const readFn = getReadFunction(srcFormat);
|
||||
const srcBytesPerSample = getBytesPerSample(srcFormat);
|
||||
const srcIsPlanar = formatIsPlanar(srcFormat);
|
||||
|
||||
let uint8Data: Uint8Array;
|
||||
if (this._data instanceof AudioSampleResource) {
|
||||
const getDataPlaneValidated = (index: number) => {
|
||||
const result = (this._data as AudioSampleResource).getDataPlane(index);
|
||||
if (!(result instanceof Uint8Array)) {
|
||||
throw new TypeError('getDataPlane() must return a Uint8Array.');
|
||||
}
|
||||
|
||||
const expectedSize = numFrames * srcBytesPerSample * (srcIsPlanar ? 1 : numChannels);
|
||||
if (result.byteLength !== expectedSize) {
|
||||
throw new TypeError(
|
||||
`Data plane ${index} has invalid size. Expected exactly ${expectedSize} bytes, got`
|
||||
+ ` ${result.byteLength} bytes.`,
|
||||
);
|
||||
}
|
||||
|
||||
return result;
|
||||
};
|
||||
|
||||
if (srcIsPlanar) {
|
||||
if (destIsPlanar) {
|
||||
// Only one source plane will be extracted, so let's fetch only that one
|
||||
uint8Data = getDataPlaneValidated(planeIndex);
|
||||
planeIndex = 0; // To fix the subsequent access
|
||||
} else {
|
||||
// Pack all planes tightly together
|
||||
uint8Data = new Uint8Array(numFrames * srcBytesPerSample * numChannels);
|
||||
for (let ch = 0; ch < numChannels; ch++) {
|
||||
const planeData = getDataPlaneValidated(ch);
|
||||
uint8Data.set(planeData, ch * numFrames * srcBytesPerSample);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
uint8Data = getDataPlaneValidated(0); // That's the only plane there is
|
||||
}
|
||||
} else {
|
||||
uint8Data = this._data;
|
||||
}
|
||||
|
||||
const srcView = toDataView(uint8Data);
|
||||
|
||||
for (let i = 0; i < copyFrameCount; i++) {
|
||||
if (destIsPlanar) {
|
||||
const destOffset = i * destBytesPerSample;
|
||||
|
||||
@@ -3,13 +3,14 @@ import { registerMediabunnyServer } from '@mediabunny/server';
|
||||
import { Input } from '../../src/input.js';
|
||||
import { BufferSource, FilePathSource } from '../../src/source.js';
|
||||
import { ALL_FORMATS } from '../../src/input-format.js';
|
||||
import { assert, toUint8Array } from '../../src/misc.js';
|
||||
import { EncodedPacketSink, VideoSampleSink } from '../../src/media-sink.js';
|
||||
import { assert, last, toUint8Array } from '../../src/misc.js';
|
||||
import { AudioSampleSink, EncodedPacketSink, VideoSampleSink } from '../../src/media-sink.js';
|
||||
import { NodeAvVideoDecoder } from '../../packages/server/src/video-decoder.js';
|
||||
import { NodeAvVideoEncoder } from '../../packages/server/src/video-encoder.js';
|
||||
import { NodeAvAudioDecoder } from '../../packages/server/src/audio-decoder.js';
|
||||
import { NodeAvAudioEncoder } from '../../packages/server/src/audio-encoder.js';
|
||||
import { AudioSample, VideoSample } from '../../src/sample.js';
|
||||
import { buildVideoCodecString, VideoCodec } from '../../src/codec.js';
|
||||
import { AudioCodec, buildAudioCodecString, buildVideoCodecString, VideoCodec } from '../../src/codec.js';
|
||||
import { EncodedPacket } from '../../src/packet.js';
|
||||
import {
|
||||
AvcNalUnitType,
|
||||
@@ -24,6 +25,7 @@ import { Mp4OutputFormat } from '../../src/output-format.js';
|
||||
import { BufferTarget } from '../../src/target.js';
|
||||
import { Conversion } from '../../src/conversion.js';
|
||||
import { NodeAvFrameVideoSampleResource } from '../../packages/server/src/video-sample.js';
|
||||
import { NodeAvFrameAudioSampleResource } from '../../packages/server/src/audio-sample.js';
|
||||
|
||||
beforeAll(() => {
|
||||
registerMediabunnyServer();
|
||||
@@ -600,7 +602,7 @@ describe('Video', async () => {
|
||||
using sample = await sink.getSample(0);
|
||||
assert(sample);
|
||||
|
||||
expect(sample.getUnderlyingData()).toBeInstanceOf(NodeAvFrameVideoSampleResource);
|
||||
expect(sample._data).toBeInstanceOf(NodeAvFrameVideoSampleResource);
|
||||
expect(sample.format).toBe('I420');
|
||||
expect(sample.codedWidth).toBe(1920);
|
||||
expect(sample.codedHeight).toBe(1080);
|
||||
@@ -736,7 +738,6 @@ describe('Video', async () => {
|
||||
});
|
||||
});
|
||||
|
||||
/*
|
||||
describe('Audio', async () => {
|
||||
test('Decoder lifecycle', async () => {
|
||||
using input = new Input({
|
||||
@@ -807,5 +808,515 @@ describe('Audio', async () => {
|
||||
|
||||
await decoder.close();
|
||||
});
|
||||
|
||||
test('Encoder lifecycle', async () => {
|
||||
const encoder = new NodeAvAudioEncoder();
|
||||
// @ts-expect-error Readonly
|
||||
encoder.codec = 'aac';
|
||||
// @ts-expect-error Readonly
|
||||
encoder.config = {
|
||||
codec: buildAudioCodecString('aac', 2, 48000),
|
||||
numberOfChannels: 2,
|
||||
sampleRate: 48000,
|
||||
bitrate: 128000,
|
||||
} satisfies AudioEncoderConfig;
|
||||
|
||||
let packetCount = 0;
|
||||
|
||||
// @ts-expect-error Readonly
|
||||
encoder.onPacket = (packet: EncodedPacket, meta: EncodedAudioChunkMetadata) => {
|
||||
expect(packet.duration).toBe(1024 / 48000);
|
||||
expect(packet.type).toBe('key');
|
||||
|
||||
if (packetCount === 0) {
|
||||
expect(packet.timestamp).toBe(0);
|
||||
|
||||
expect(meta.decoderConfig).toBeDefined();
|
||||
expect(meta.decoderConfig!.codec).toBe('mp4a.40.2');
|
||||
expect(meta.decoderConfig!.numberOfChannels).toBe(2);
|
||||
expect(meta.decoderConfig!.sampleRate).toBe(48000);
|
||||
expect(meta.decoderConfig?.description).toBeDefined();
|
||||
}
|
||||
|
||||
packetCount++;
|
||||
};
|
||||
|
||||
await encoder.init();
|
||||
|
||||
const data = createF32SineWave(48000, 2, 2);
|
||||
using sample1 = new AudioSample({
|
||||
data,
|
||||
format: 'f32',
|
||||
timestamp: 0,
|
||||
numberOfChannels: 2,
|
||||
sampleRate: 48000,
|
||||
});
|
||||
await encoder.encode(sample1);
|
||||
|
||||
await encoder.flush();
|
||||
|
||||
const minPacketCount = 2 * 48000 / 1024;
|
||||
expect(packetCount).toBeGreaterThan(minPacketCount);
|
||||
|
||||
packetCount = 0;
|
||||
|
||||
using sample2 = new AudioSample({
|
||||
data,
|
||||
format: 'f32',
|
||||
timestamp: 0,
|
||||
numberOfChannels: 2,
|
||||
sampleRate: 48000,
|
||||
});
|
||||
await encoder.encode(sample2);
|
||||
|
||||
await encoder.flush();
|
||||
|
||||
expect(packetCount).toBeGreaterThan(minPacketCount);
|
||||
|
||||
await encoder.close();
|
||||
});
|
||||
|
||||
const createF32SineWave = (sampleRate: number, channels: number, durationSeconds: number) => {
|
||||
const totalFrames = sampleRate * durationSeconds;
|
||||
const data = new Float32Array(totalFrames * channels);
|
||||
|
||||
for (let i = 0; i < totalFrames; i++) {
|
||||
const value = Math.sin(2 * Math.PI * 440 * i / sampleRate);
|
||||
for (let ch = 0; ch < channels; ch++) {
|
||||
data[i * channels + ch] = value;
|
||||
}
|
||||
}
|
||||
|
||||
return data;
|
||||
};
|
||||
|
||||
test('AAC encode & decode, AAC format', async () => {
|
||||
await encodeDecodeTest('aac', {
|
||||
// @ts-expect-error Fucky type
|
||||
aac: { format: 'aac' },
|
||||
}, async (packet, meta, i) => {
|
||||
expect(packet.type).toBe('key');
|
||||
expect(packet.duration).toBe(1024 / 48000);
|
||||
|
||||
if (i === 0) {
|
||||
expect(meta.decoderConfig).toBeDefined();
|
||||
expect(meta.decoderConfig!.codec).toBe('mp4a.40.2');
|
||||
expect(meta.decoderConfig!.numberOfChannels).toBe(2);
|
||||
expect(meta.decoderConfig!.sampleRate).toBe(48000);
|
||||
expect(meta.decoderConfig!.description).toBeDefined();
|
||||
}
|
||||
}, async (sample) => {
|
||||
expect(sample.duration).toBe(1024 / 48000);
|
||||
expect(sample.numberOfChannels).toBe(2);
|
||||
expect(sample.sampleRate).toBe(48000);
|
||||
});
|
||||
});
|
||||
|
||||
test('AAC encode & decode, ADTS format', async () => {
|
||||
await encodeDecodeTest('aac', {
|
||||
// @ts-expect-error Fucky type
|
||||
aac: { format: 'adts' },
|
||||
}, async (packet, meta, i) => {
|
||||
expect(packet.type).toBe('key');
|
||||
expect(packet.duration).toBe(1024 / 48000);
|
||||
expect(packet.data[0]).toBe(0xff);
|
||||
|
||||
if (i === 0) {
|
||||
expect(meta.decoderConfig).toBeDefined();
|
||||
expect(meta.decoderConfig!.codec).toBe('mp4a.40.2');
|
||||
expect(meta.decoderConfig!.numberOfChannels).toBe(2);
|
||||
expect(meta.decoderConfig!.sampleRate).toBe(48000);
|
||||
expect(meta.decoderConfig!.description).toBeUndefined();
|
||||
}
|
||||
}, async (sample) => {
|
||||
expect(sample.duration).toBe(1024 / 48000);
|
||||
expect(sample.numberOfChannels).toBe(2);
|
||||
expect(sample.sampleRate).toBe(48000);
|
||||
});
|
||||
});
|
||||
|
||||
test('Opus encode & decode', async () => {
|
||||
await encodeDecodeTest('opus', {}, async (packet, meta, i, n) => {
|
||||
expect(packet.type).toBe('key');
|
||||
|
||||
if (i < n - 1) {
|
||||
expect(packet.duration).toBe(0.02);
|
||||
}
|
||||
|
||||
if (i === 0) {
|
||||
expect(meta.decoderConfig).toBeDefined();
|
||||
expect(meta.decoderConfig!.codec).toBe('opus');
|
||||
expect(meta.decoderConfig!.numberOfChannels).toBe(2);
|
||||
expect(meta.decoderConfig!.sampleRate).toBe(48000);
|
||||
expect(meta.decoderConfig!.description).toBeDefined();
|
||||
expect([...toUint8Array(meta.decoderConfig!.description!).slice(0, 8)]).toEqual([
|
||||
// OpusHead
|
||||
0x4f, 0x70, 0x75, 0x73, 0x48, 0x65, 0x61, 0x64,
|
||||
]);
|
||||
}
|
||||
}, async (sample, i, n) => {
|
||||
expect(sample.numberOfChannels).toBe(2);
|
||||
expect(sample.sampleRate).toBe(48000);
|
||||
|
||||
if (i > 0 && i < n - 1) {
|
||||
expect(sample.duration).toBe(0.02);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test('MP3 encode & decode', async () => {
|
||||
await encodeDecodeTest('mp3', {}, async (packet, meta, i, n) => {
|
||||
expect(packet.type).toBe('key');
|
||||
|
||||
if (i < n - 1) {
|
||||
expect(packet.duration).toBe(1152 / 48000);
|
||||
}
|
||||
|
||||
if (i === 0) {
|
||||
expect(meta.decoderConfig).toBeDefined();
|
||||
expect(meta.decoderConfig!.codec).toBe('mp3');
|
||||
expect(meta.decoderConfig!.numberOfChannels).toBe(2);
|
||||
expect(meta.decoderConfig!.sampleRate).toBe(48000);
|
||||
expect(meta.decoderConfig!.description).toBeUndefined();
|
||||
}
|
||||
}, async (sample) => {
|
||||
expect(sample.numberOfChannels).toBe(2);
|
||||
expect(sample.sampleRate).toBe(48000);
|
||||
expect(sample.duration).toBe(1152 / 48000);
|
||||
});
|
||||
});
|
||||
|
||||
test('Vorbis encode & decode', async () => {
|
||||
await encodeDecodeTest('vorbis', {}, async (packet, meta, i) => {
|
||||
expect(packet.type).toBe('key');
|
||||
|
||||
if (i === 0) {
|
||||
expect(meta.decoderConfig).toBeDefined();
|
||||
expect(meta.decoderConfig!.codec).toBe('vorbis');
|
||||
expect(meta.decoderConfig!.numberOfChannels).toBe(2);
|
||||
expect(meta.decoderConfig!.sampleRate).toBe(48000);
|
||||
expect(meta.decoderConfig!.description).toBeDefined();
|
||||
expect(toUint8Array(meta.decoderConfig!.description!)[0]).toBe(2); // "Xiph extradata format"
|
||||
}
|
||||
}, async (sample) => {
|
||||
expect(sample.numberOfChannels).toBe(2);
|
||||
expect(sample.sampleRate).toBe(48000);
|
||||
});
|
||||
});
|
||||
|
||||
test('FLAC encode & decode', async () => {
|
||||
await encodeDecodeTest('flac', {}, async (packet, meta, i) => {
|
||||
expect(packet.type).toBe('key');
|
||||
expect(packet.duration).toBe(0.096);
|
||||
|
||||
if (i === 0) {
|
||||
expect(meta.decoderConfig).toBeDefined();
|
||||
expect(meta.decoderConfig!.codec).toBe('flac');
|
||||
expect(meta.decoderConfig!.numberOfChannels).toBe(2);
|
||||
expect(meta.decoderConfig!.sampleRate).toBe(48000);
|
||||
expect(meta.decoderConfig!.description).toBeDefined();
|
||||
expect([...toUint8Array(meta.decoderConfig!.description!).slice(0, 4)]).toEqual([
|
||||
// fLaC
|
||||
0x66, 0x4c, 0x61, 0x43,
|
||||
]);
|
||||
}
|
||||
}, async (sample) => {
|
||||
expect(sample.numberOfChannels).toBe(2);
|
||||
expect(sample.sampleRate).toBe(48000);
|
||||
expect(sample.duration).toBe(0.096);
|
||||
});
|
||||
});
|
||||
|
||||
test('AC-3 encode & decode', async () => {
|
||||
await encodeDecodeTest('ac3', {}, async (packet, meta, i) => {
|
||||
expect(packet.type).toBe('key');
|
||||
expect(packet.duration).toBe(0.032);
|
||||
|
||||
if (i === 0) {
|
||||
expect(meta.decoderConfig).toBeDefined();
|
||||
expect(meta.decoderConfig!.codec).toBe('ac-3');
|
||||
expect(meta.decoderConfig!.numberOfChannels).toBe(2);
|
||||
expect(meta.decoderConfig!.sampleRate).toBe(48000);
|
||||
expect(meta.decoderConfig!.description).toBeUndefined();
|
||||
}
|
||||
}, async (sample) => {
|
||||
expect(sample.numberOfChannels).toBe(2);
|
||||
expect(sample.sampleRate).toBe(48000);
|
||||
expect(sample.duration).toBe(0.032);
|
||||
});
|
||||
});
|
||||
|
||||
test('E-AC-3 encode & decode', async () => {
|
||||
await encodeDecodeTest('eac3', {}, async (packet, meta, i) => {
|
||||
expect(packet.type).toBe('key');
|
||||
expect(packet.duration).toBe(0.032);
|
||||
|
||||
if (i === 0) {
|
||||
expect(meta.decoderConfig).toBeDefined();
|
||||
expect(meta.decoderConfig!.codec).toBe('ec-3');
|
||||
expect(meta.decoderConfig!.numberOfChannels).toBe(2);
|
||||
expect(meta.decoderConfig!.sampleRate).toBe(48000);
|
||||
expect(meta.decoderConfig!.description).toBeUndefined();
|
||||
}
|
||||
}, async (sample) => {
|
||||
expect(sample.numberOfChannels).toBe(2);
|
||||
expect(sample.sampleRate).toBe(48000);
|
||||
expect(sample.duration).toBe(0.032);
|
||||
});
|
||||
});
|
||||
|
||||
const encodeDecodeTest = async (
|
||||
codec: AudioCodec,
|
||||
extraConfig: Partial<AudioEncoderConfig>,
|
||||
onPacket: (packet: EncodedPacket, meta: EncodedAudioChunkMetadata, i: number, n: number) => Promise<void>,
|
||||
onSample: (sample: AudioSample, i: number, n: number) => Promise<void>,
|
||||
) => {
|
||||
const sampleRate = 48000;
|
||||
const numberOfChannels = 2;
|
||||
|
||||
const encoder = new NodeAvAudioEncoder();
|
||||
// @ts-expect-error Readonly
|
||||
encoder.codec = codec;
|
||||
// @ts-expect-error Readonly
|
||||
encoder.config = {
|
||||
codec: buildAudioCodecString(codec, numberOfChannels, sampleRate),
|
||||
numberOfChannels,
|
||||
sampleRate,
|
||||
bitrate: 128000,
|
||||
...extraConfig,
|
||||
} satisfies AudioEncoderConfig;
|
||||
|
||||
const packets: EncodedPacket[] = [];
|
||||
const metas: EncodedAudioChunkMetadata[] = [];
|
||||
|
||||
// @ts-expect-error Readonly
|
||||
encoder.onPacket = (packet: EncodedPacket, meta: EncodedAudioChunkMetadata) => {
|
||||
packets.push(packet);
|
||||
metas.push(meta);
|
||||
};
|
||||
|
||||
await encoder.init();
|
||||
|
||||
const data = createF32SineWave(sampleRate, numberOfChannels, 2);
|
||||
using inputSample = new AudioSample({
|
||||
data,
|
||||
format: 'f32',
|
||||
timestamp: 0,
|
||||
numberOfChannels,
|
||||
sampleRate,
|
||||
});
|
||||
await encoder.encode(inputSample);
|
||||
|
||||
await encoder.flush();
|
||||
|
||||
for (let i = 0; i < packets.length; i++) {
|
||||
await onPacket(packets[i]!, metas[i]!, i, packets.length);
|
||||
}
|
||||
|
||||
const decoder = new NodeAvAudioDecoder();
|
||||
// @ts-expect-error Readonly
|
||||
decoder.codec = codec;
|
||||
// @ts-expect-error Readonly
|
||||
decoder.config = metas[0]!.decoderConfig!;
|
||||
|
||||
const decodedSamples: AudioSample[] = [];
|
||||
|
||||
// @ts-expect-error Readonly
|
||||
decoder.onSample = (sample: AudioSample) => {
|
||||
decodedSamples.push(sample);
|
||||
};
|
||||
|
||||
await decoder.init();
|
||||
|
||||
for (const packet of packets) {
|
||||
await decoder.decode(packet);
|
||||
}
|
||||
|
||||
await decoder.flush();
|
||||
|
||||
expect(decodedSamples.length).toBeGreaterThan(0);
|
||||
expect(last(decodedSamples)!.timestamp + last(decodedSamples)!.duration).toBeGreaterThanOrEqual(2);
|
||||
|
||||
for (let i = 0; i < decodedSamples.length; i++) {
|
||||
await onSample(decodedSamples[i]!, i, decodedSamples.length);
|
||||
}
|
||||
|
||||
const signalChunks: Float32Array[] = [];
|
||||
|
||||
for (const sample of decodedSamples) {
|
||||
const buf = new Float32Array(new ArrayBuffer(sample.allocationSize({
|
||||
format: 'f32-planar', planeIndex: 0,
|
||||
})));
|
||||
sample.copyTo(buf, { format: 'f32-planar', planeIndex: 0 });
|
||||
signalChunks.push(buf);
|
||||
|
||||
sample.close();
|
||||
}
|
||||
|
||||
const totalSize = signalChunks.reduce((sum, chunk) => sum + chunk.length, 0);
|
||||
const signal = new Float32Array(totalSize);
|
||||
let offset = 0;
|
||||
for (const chunk of signalChunks) {
|
||||
signal.set(chunk, offset);
|
||||
offset += chunk.length;
|
||||
}
|
||||
|
||||
const score = sine440Score(signal, sampleRate, 440);
|
||||
expect(score).toBeGreaterThan(0.98);
|
||||
|
||||
await encoder.close();
|
||||
await decoder.close();
|
||||
};
|
||||
|
||||
const sine440Score = (x: Float32Array, sampleRate: number, freq: number) => {
|
||||
const w = 2 * Math.PI * freq / sampleRate;
|
||||
|
||||
let ss = 0, cc = 0, sc = 0;
|
||||
let xs = 0, xc = 0;
|
||||
let xx = 0;
|
||||
|
||||
for (let n = 0; n < x.length; n++) {
|
||||
const s = Math.sin(w * n);
|
||||
const c = Math.cos(w * n);
|
||||
|
||||
ss += s * s;
|
||||
cc += c * c;
|
||||
sc += s * c;
|
||||
|
||||
xs += x[n]! * s;
|
||||
xc += x[n]! * c;
|
||||
xx += x[n]! * x[n]!;
|
||||
}
|
||||
|
||||
const det = ss * cc - sc * sc;
|
||||
|
||||
const a = (xs * cc - xc * sc) / det;
|
||||
const b = (xc * ss - xs * sc) / det;
|
||||
|
||||
let fitEnergy = 0;
|
||||
|
||||
for (let n = 0; n < x.length; n++) {
|
||||
const y = a * Math.sin(w * n) + b * Math.cos(w * n);
|
||||
fitEnergy += y * y;
|
||||
}
|
||||
|
||||
const score = fitEnergy / xx;
|
||||
return score;
|
||||
};
|
||||
|
||||
test('AAC conversion roundtrip', { timeout: 10_000 }, async () => {
|
||||
await conversionRoundtrip('aac');
|
||||
});
|
||||
|
||||
test('Opus conversion roundtrip', { timeout: 10_000 }, async () => {
|
||||
await conversionRoundtrip('opus');
|
||||
});
|
||||
|
||||
test('MP3 conversion roundtrip', { timeout: 10_000 }, async () => {
|
||||
await conversionRoundtrip('mp3');
|
||||
});
|
||||
|
||||
test('Vorbis conversion roundtrip', { timeout: 10_000 }, async () => {
|
||||
await conversionRoundtrip('vorbis');
|
||||
});
|
||||
|
||||
test('FLAC conversion roundtrip', { timeout: 10_000 }, async () => {
|
||||
await conversionRoundtrip('flac');
|
||||
});
|
||||
|
||||
test('AC-3 conversion roundtrip', { timeout: 10_000 }, async () => {
|
||||
await conversionRoundtrip('ac3');
|
||||
});
|
||||
|
||||
test('E-AC-3 conversion roundtrip', { timeout: 10_000 }, async () => {
|
||||
await conversionRoundtrip('eac3');
|
||||
});
|
||||
|
||||
const conversionRoundtrip = async (codec: AudioCodec) => {
|
||||
using input = new Input({
|
||||
source: new FilePathSource('./test/public/trim-buck-bunny-ffmpeg.ts'),
|
||||
formats: ALL_FORMATS,
|
||||
});
|
||||
|
||||
const output = new Output({
|
||||
format: new Mp4OutputFormat(),
|
||||
target: new BufferTarget(),
|
||||
});
|
||||
|
||||
const inputTrack = await input.getPrimaryAudioTrack();
|
||||
assert(inputTrack);
|
||||
|
||||
const conversion = await Conversion.init({
|
||||
input,
|
||||
output,
|
||||
video: {
|
||||
discard: true,
|
||||
}, audio: {
|
||||
codec,
|
||||
forceTranscode: true,
|
||||
},
|
||||
trim: {
|
||||
start: 0,
|
||||
},
|
||||
});
|
||||
await conversion.execute();
|
||||
|
||||
using newInput = new Input({
|
||||
source: new BufferSource(output.target.buffer!),
|
||||
formats: ALL_FORMATS,
|
||||
});
|
||||
|
||||
const newInputTrack = await newInput.getPrimaryAudioTrack();
|
||||
assert(newInputTrack);
|
||||
|
||||
expect(await newInputTrack.getCodec()).toBe(codec);
|
||||
expect(await newInputTrack.computeDuration()).toBeCloseTo(await inputTrack.computeDuration(), 0);
|
||||
|
||||
const sink = new AudioSampleSink(newInputTrack);
|
||||
|
||||
let sampleCount = 0;
|
||||
for await (using sample of sink.samples()) {
|
||||
expect([await inputTrack.getNumberOfChannels(), 2].includes(sample.numberOfChannels)).toBe(true);
|
||||
expect(sample.sampleRate).toBe(await inputTrack.getSampleRate());
|
||||
|
||||
sampleCount++;
|
||||
}
|
||||
|
||||
expect(sampleCount).toBeGreaterThan(0);
|
||||
};
|
||||
|
||||
test('Custom AudioSample resource', async () => {
|
||||
using input = new Input({
|
||||
source: new FilePathSource('./test/public/trim-buck-bunny-ffmpeg.ts'),
|
||||
formats: ALL_FORMATS,
|
||||
});
|
||||
|
||||
const audioTrack = await input.getPrimaryAudioTrack();
|
||||
assert(audioTrack);
|
||||
|
||||
const sink = new AudioSampleSink(audioTrack);
|
||||
using sample = await sink.getSample(await audioTrack.getFirstTimestamp());
|
||||
assert(sample);
|
||||
|
||||
expect(sample._data).toBeInstanceOf(NodeAvFrameAudioSampleResource);
|
||||
expect(sample.format).toBe('f32-planar');
|
||||
expect(sample.numberOfChannels).toBe(6);
|
||||
expect(sample.sampleRate).toBe(48000);
|
||||
expect(sample.timestamp).toBe(await audioTrack.getFirstTimestamp());
|
||||
expect(sample.duration).toBe(1024 / 48000);
|
||||
expect(sample.numberOfFrames).toBe(1024);
|
||||
|
||||
const size = sample.allocationSize({ planeIndex: 0 });
|
||||
expect(size).toBe(1024 * Float32Array.BYTES_PER_ELEMENT);
|
||||
|
||||
// Test a bunch of copyTo()s:
|
||||
const buf = new ArrayBuffer(1e6);
|
||||
sample.copyTo(buf, { planeIndex: 0 });
|
||||
sample.copyTo(buf, { planeIndex: 1 });
|
||||
sample.copyTo(buf, { planeIndex: 2 });
|
||||
sample.copyTo(buf, { planeIndex: 3 });
|
||||
sample.copyTo(buf, { planeIndex: 4 });
|
||||
sample.copyTo(buf, { planeIndex: 5 });
|
||||
sample.copyTo(buf, { format: 'f32', planeIndex: 0 });
|
||||
});
|
||||
});
|
||||
*/
|
||||
|
||||
Reference in New Issue
Block a user