mirror of
https://github.com/arcodange-org/mediabunny.git
synced 2026-10-03 05:43:50 +02:00
Implement audio decoder & encoder
This commit is contained in:
@@ -0,0 +1,106 @@
|
|||||||
|
import { AudioCodec, AudioSample, CustomAudioDecoder, EncodedPacket, MaybePromise } from 'mediabunny';
|
||||||
|
import * as NodeAv from 'node-av';
|
||||||
|
import { CODEC_TO_CODEC_ID, getChannelLayout } from './misc';
|
||||||
|
import { assert, roundToDivisor, toUint8Array } from '../../../src/misc';
|
||||||
|
import { NodeAvFrameAudioSampleResource } from './audio-sample';
|
||||||
|
|
||||||
|
export class NodeAvAudioDecoder extends CustomAudioDecoder {
|
||||||
|
frame!: NodeAv.Frame;
|
||||||
|
packet!: NodeAv.Packet;
|
||||||
|
codecContext!: NodeAv.CodecContext;
|
||||||
|
|
||||||
|
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||||
|
static override supports(codec: AudioCodec, config: AudioDecoderConfig): boolean {
|
||||||
|
return codec === 'aac'
|
||||||
|
|| codec === 'opus'
|
||||||
|
|| codec === 'mp3'
|
||||||
|
|| codec === 'vorbis'
|
||||||
|
|| codec === 'flac'
|
||||||
|
|| codec === 'ac3'
|
||||||
|
|| codec === 'eac3';
|
||||||
|
}
|
||||||
|
|
||||||
|
async init(): Promise<void> {
|
||||||
|
this.frame = new NodeAv.Frame();
|
||||||
|
this.frame.alloc();
|
||||||
|
this.packet = new NodeAv.Packet();
|
||||||
|
this.packet.alloc();
|
||||||
|
|
||||||
|
const codecId = CODEC_TO_CODEC_ID[this.codec];
|
||||||
|
assert(codecId !== undefined);
|
||||||
|
|
||||||
|
const codec = NodeAv.Codec.findDecoder(codecId);
|
||||||
|
if (codec === null) {
|
||||||
|
throw new Error(`Unable to obtain libav codec for '${this.codec}'.`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const codecContext = new NodeAv.CodecContext();
|
||||||
|
codecContext.allocContext3(codec);
|
||||||
|
|
||||||
|
codecContext.sampleRate = this.config.sampleRate;
|
||||||
|
codecContext.channelLayout = getChannelLayout(this.config.numberOfChannels);
|
||||||
|
codecContext.codecType = NodeAv.AVMEDIA_TYPE_AUDIO;
|
||||||
|
codecContext.codecId = codecId;
|
||||||
|
codecContext.extraData = this.config.description
|
||||||
|
? Buffer.from(toUint8Array(this.config.description))
|
||||||
|
: null;
|
||||||
|
|
||||||
|
const ret = await codecContext.open2();
|
||||||
|
NodeAv.FFmpegError.throwIfError(ret, 'Open codec context');
|
||||||
|
|
||||||
|
this.codecContext = codecContext;
|
||||||
|
}
|
||||||
|
|
||||||
|
async decode(packet: EncodedPacket): Promise<void> {
|
||||||
|
this.packet.isKeyframe = packet.type === 'key';
|
||||||
|
this.packet.data = Buffer.from(packet.data);
|
||||||
|
this.packet.timeBase = { num: 1, den: 1e6 };
|
||||||
|
this.packet.pts = BigInt(packet.microsecondTimestamp);
|
||||||
|
this.packet.dts = NodeAv.AV_NOPTS_VALUE;
|
||||||
|
this.packet.duration = BigInt(packet.microsecondDuration);
|
||||||
|
|
||||||
|
const ret = await this.codecContext.sendPacket(this.packet);
|
||||||
|
NodeAv.FFmpegError.throwIfError(ret, 'Send packet');
|
||||||
|
|
||||||
|
while (true) {
|
||||||
|
const receiveRet = await this.codecContext.receiveFrame(this.frame);
|
||||||
|
if (receiveRet === NodeAv.AVERROR_EAGAIN || receiveRet === NodeAv.AVERROR_EOF) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
this.receiveFrame(receiveRet);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
receiveFrame(ret: number) {
|
||||||
|
NodeAv.FFmpegError.throwIfError(ret, 'Receive frame');
|
||||||
|
|
||||||
|
const timestamp = roundToDivisor(Number(this.frame.pts) / 1e6, this.config.sampleRate);
|
||||||
|
this.onSample(new AudioSample(new NodeAvFrameAudioSampleResource(this.frame, timestamp)));
|
||||||
|
}
|
||||||
|
|
||||||
|
async flush(): Promise<void> {
|
||||||
|
// Send null packet to signal flush
|
||||||
|
const ret = await this.codecContext.sendPacket(null);
|
||||||
|
NodeAv.FFmpegError.throwIfError(ret, 'Flush decoder');
|
||||||
|
|
||||||
|
// Keep receiving frames until no more are available
|
||||||
|
while (true) {
|
||||||
|
const receiveRet = await this.codecContext.receiveFrame(this.frame);
|
||||||
|
if (receiveRet === NodeAv.AVERROR_EAGAIN || receiveRet === NodeAv.AVERROR_EOF) {
|
||||||
|
// No more frames available
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
this.receiveFrame(receiveRet);
|
||||||
|
}
|
||||||
|
|
||||||
|
this.codecContext.flushBuffers();
|
||||||
|
}
|
||||||
|
|
||||||
|
close(): MaybePromise<void> {
|
||||||
|
this.codecContext.freeContext();
|
||||||
|
this.frame.free();
|
||||||
|
this.packet.free();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,370 @@
|
|||||||
|
import { AudioCodec, AudioSample, CustomAudioEncoder, MaybePromise, QUALITY_MEDIUM } from 'mediabunny';
|
||||||
|
import * as NodeAv from 'node-av';
|
||||||
|
import { CODEC_TO_CODEC_ID, fromAudioSampleFormat, getChannelLayout } from './misc';
|
||||||
|
import { assert, toUint8Array } from '../../../src/misc';
|
||||||
|
import { NodeAvFrameAudioSampleResource } from './audio-sample';
|
||||||
|
import { AdtsHeaderTemplate, buildAdtsHeaderTemplate, parseAacAudioSpecificConfig } from '../../../shared/aac-misc';
|
||||||
|
import { EncodedPacket } from 'mediabunny';
|
||||||
|
|
||||||
|
const AAC_SAMPLE_RATES
|
||||||
|
= [96000, 88200, 64000, 48000, 44100, 32000, 24000, 22050, 16000, 12000, 11025, 8000, 7350];
|
||||||
|
const OPUS_SAMPLE_RATES = [8000, 12000, 16000, 24000, 48000];
|
||||||
|
const MP3_SAMPLE_RATES = [8000, 11025, 12000, 16000, 22050, 24000, 32000, 44100, 48000];
|
||||||
|
const AC3_SAMPLE_RATES = [32000, 44100, 48000];
|
||||||
|
|
||||||
|
const FRAME_SIZE_FALLBACK = 1024; // Just 'cause
|
||||||
|
|
||||||
|
export class NodeAvAudioEncoder extends CustomAudioEncoder {
|
||||||
|
frame!: NodeAv.Frame;
|
||||||
|
packet!: NodeAv.Packet;
|
||||||
|
avCodec!: NodeAv.Codec;
|
||||||
|
codecContext: NodeAv.CodecContext | null = null;
|
||||||
|
firstExpectedTimestamp: number | null = null;
|
||||||
|
outputTimestampOffset = 0;
|
||||||
|
|
||||||
|
resampler: NodeAv.SoftwareResampleContext | null = null;
|
||||||
|
inputParametersKey: string | null = null;
|
||||||
|
resamplerInputSampleRate: number | null = null;
|
||||||
|
nextResamplerPts: bigint | null = null;
|
||||||
|
dstFrame: NodeAv.Frame | null = null;
|
||||||
|
packetEmitted = false;
|
||||||
|
adtsHeaderTemplate: AdtsHeaderTemplate | null = null;
|
||||||
|
|
||||||
|
static override supports(codec: AudioCodec, config: AudioEncoderConfig): boolean {
|
||||||
|
const { numberOfChannels, sampleRate } = config;
|
||||||
|
|
||||||
|
return (
|
||||||
|
codec === 'aac' && numberOfChannels >= 1 && numberOfChannels <= 48
|
||||||
|
&& AAC_SAMPLE_RATES.includes(sampleRate)
|
||||||
|
) || (
|
||||||
|
codec === 'opus' && numberOfChannels >= 1 && numberOfChannels <= 255
|
||||||
|
&& OPUS_SAMPLE_RATES.includes(sampleRate)
|
||||||
|
) || (
|
||||||
|
codec === 'mp3' && numberOfChannels >= 1 && numberOfChannels <= 2
|
||||||
|
&& MP3_SAMPLE_RATES.includes(sampleRate)
|
||||||
|
) || (
|
||||||
|
codec === 'vorbis' && numberOfChannels >= 1 && numberOfChannels <= 255
|
||||||
|
&& sampleRate <= 200000
|
||||||
|
) || (
|
||||||
|
codec === 'flac' && numberOfChannels >= 1 && numberOfChannels <= 8
|
||||||
|
&& sampleRate <= 655350
|
||||||
|
) || (
|
||||||
|
codec === 'ac3' && numberOfChannels >= 1 && numberOfChannels <= 6
|
||||||
|
&& AC3_SAMPLE_RATES.includes(sampleRate)
|
||||||
|
) || (
|
||||||
|
codec === 'eac3' && numberOfChannels >= 1 && numberOfChannels <= 16
|
||||||
|
&& AC3_SAMPLE_RATES.includes(sampleRate)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async init(): Promise<void> {
|
||||||
|
this.frame = new NodeAv.Frame();
|
||||||
|
this.frame.alloc();
|
||||||
|
this.packet = new NodeAv.Packet();
|
||||||
|
this.packet.alloc();
|
||||||
|
|
||||||
|
const codecId = CODEC_TO_CODEC_ID[this.codec];
|
||||||
|
assert(codecId !== undefined);
|
||||||
|
|
||||||
|
const codec = NodeAv.Codec.findEncoder(codecId);
|
||||||
|
if (!codec) {
|
||||||
|
throw new Error(`Unable to obtain libav codec for '${this.codec}'.`);
|
||||||
|
}
|
||||||
|
|
||||||
|
this.avCodec = codec;
|
||||||
|
|
||||||
|
await this.createCodecContext();
|
||||||
|
}
|
||||||
|
|
||||||
|
async createCodecContext() {
|
||||||
|
assert(this.codecContext === null);
|
||||||
|
|
||||||
|
const codecContext = new NodeAv.CodecContext();
|
||||||
|
codecContext.allocContext3(this.avCodec);
|
||||||
|
|
||||||
|
let sampleFormat = NodeAv.AV_SAMPLE_FMT_FLTP;
|
||||||
|
if (this.avCodec.sampleFormats && !this.avCodec.sampleFormats.includes(NodeAv.AV_SAMPLE_FMT_FLTP)) {
|
||||||
|
// Use a format that's supported
|
||||||
|
sampleFormat = this.avCodec.sampleFormats[0]!;
|
||||||
|
}
|
||||||
|
|
||||||
|
codecContext.sampleRate = this.config.sampleRate;
|
||||||
|
codecContext.channelLayout = getChannelLayout(this.config.numberOfChannels);
|
||||||
|
codecContext.codecType = NodeAv.AVMEDIA_TYPE_AUDIO;
|
||||||
|
codecContext.codecId = CODEC_TO_CODEC_ID[this.codec]!;
|
||||||
|
codecContext.sampleFormat = sampleFormat;
|
||||||
|
codecContext.timeBase = new NodeAv.Rational(1, this.config.sampleRate);
|
||||||
|
codecContext.bitRate = BigInt(this.config.bitrate ?? QUALITY_MEDIUM._toAudioBitrate(this.codec) ?? 0);
|
||||||
|
|
||||||
|
if (this.config.bitrateMode === 'constant') {
|
||||||
|
codecContext.rcMinRate = codecContext.bitRate;
|
||||||
|
codecContext.rcMaxRate = codecContext.bitRate;
|
||||||
|
}
|
||||||
|
|
||||||
|
const ret = await codecContext.open2();
|
||||||
|
NodeAv.FFmpegError.throwIfError(ret, 'Open codec context');
|
||||||
|
|
||||||
|
this.codecContext = codecContext;
|
||||||
|
}
|
||||||
|
|
||||||
|
async encode(audioSample: AudioSample): Promise<void> {
|
||||||
|
if (this.codecContext === null) {
|
||||||
|
await this.createCodecContext();
|
||||||
|
assert(this.codecContext);
|
||||||
|
}
|
||||||
|
|
||||||
|
this.firstExpectedTimestamp ??= audioSample.timestamp;
|
||||||
|
|
||||||
|
if (audioSample._data instanceof NodeAvFrameAudioSampleResource) {
|
||||||
|
this.frame.ref(audioSample._data.frame);
|
||||||
|
} else {
|
||||||
|
// Copy audio data from AudioData to FFmpeg Frame
|
||||||
|
const format = fromAudioSampleFormat(audioSample.format);
|
||||||
|
this.frame.format = format;
|
||||||
|
this.frame.nbSamples = audioSample.numberOfFrames;
|
||||||
|
this.frame.sampleRate = audioSample.sampleRate;
|
||||||
|
this.frame.channelLayout = getChannelLayout(audioSample.numberOfChannels);
|
||||||
|
this.frame.pts = BigInt(Math.round(audioSample.timestamp * audioSample.sampleRate));
|
||||||
|
this.frame.duration = BigInt(audioSample.numberOfFrames);
|
||||||
|
|
||||||
|
this.frame.allocBuffer();
|
||||||
|
assert(this.frame.data);
|
||||||
|
|
||||||
|
for (let i = 0; i < this.frame.data.length; i++) {
|
||||||
|
audioSample.copyTo(this.frame.data[i]!, { planeIndex: i });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const key = `${this.frame.sampleRate}:${this.frame.channels}:${this.frame.format}`;
|
||||||
|
if (this.inputParametersKey !== null && this.inputParametersKey !== key) {
|
||||||
|
throw new Error(
|
||||||
|
'Input audio parameters changed. For this audio encoder, you cannot change the input audio'
|
||||||
|
+ ' parameters over time.',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
this.inputParametersKey = key;
|
||||||
|
|
||||||
|
// We need the resampler when:
|
||||||
|
// 1. Format conversion is needed (sample format, sample rate, or channel count differs)
|
||||||
|
// 2. The codec requires fixed frame sizes
|
||||||
|
const requiresResampler
|
||||||
|
= this.codecContext.frameSize > 0
|
||||||
|
|| this.codecContext.sampleFormat !== this.frame.format
|
||||||
|
|| this.codecContext.sampleRate !== this.frame.sampleRate
|
||||||
|
|| this.codecContext.channels !== this.frame.channels;
|
||||||
|
|
||||||
|
if (requiresResampler) {
|
||||||
|
if (!this.resampler) {
|
||||||
|
this.resampler = new NodeAv.SoftwareResampleContext();
|
||||||
|
this.resamplerInputSampleRate = this.frame.sampleRate;
|
||||||
|
|
||||||
|
const outLayout = getChannelLayout(this.codecContext.channels);
|
||||||
|
const inLayout = getChannelLayout(this.frame.channels);
|
||||||
|
|
||||||
|
const ret = this.resampler.allocSetOpts2(
|
||||||
|
outLayout, this.codecContext.sampleFormat, this.codecContext.sampleRate,
|
||||||
|
inLayout, this.frame.format as NodeAv.AVSampleFormat, this.frame.sampleRate,
|
||||||
|
);
|
||||||
|
NodeAv.FFmpegError.throwIfError(ret, 'allocSetOpts2');
|
||||||
|
|
||||||
|
const ret2 = this.resampler.init();
|
||||||
|
NodeAv.FFmpegError.throwIfError(ret2, 'init');
|
||||||
|
|
||||||
|
this.dstFrame = new NodeAv.Frame();
|
||||||
|
this.dstFrame.alloc();
|
||||||
|
this.dstFrame.channelLayout = outLayout;
|
||||||
|
this.dstFrame.sampleRate = this.codecContext.sampleRate;
|
||||||
|
this.dstFrame.format = this.codecContext.sampleFormat;
|
||||||
|
this.dstFrame.nbSamples = this.codecContext.frameSize || FRAME_SIZE_FALLBACK;
|
||||||
|
this.dstFrame.duration = BigInt(this.dstFrame.nbSamples);
|
||||||
|
this.dstFrame.allocBuffer();
|
||||||
|
|
||||||
|
this.nextResamplerPts = this.frame.pts;
|
||||||
|
}
|
||||||
|
|
||||||
|
const inputBuffers = this.frame.data;
|
||||||
|
if (!inputBuffers) {
|
||||||
|
throw new DOMException('Frame has no data', 'EncodingError');
|
||||||
|
}
|
||||||
|
await this.resampler.convert(null, 0, inputBuffers, this.frame.nbSamples);
|
||||||
|
|
||||||
|
await this.pullResampledFrames();
|
||||||
|
} else {
|
||||||
|
await this.sendFrameAndReceivePackets(this.frame);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async pullResampledFrames() {
|
||||||
|
assert(this.codecContext);
|
||||||
|
assert(this.resampler);
|
||||||
|
assert(this.dstFrame);
|
||||||
|
assert(this.nextResamplerPts !== null);
|
||||||
|
|
||||||
|
const frameSize = this.codecContext.frameSize || FRAME_SIZE_FALLBACK;
|
||||||
|
|
||||||
|
while (true) {
|
||||||
|
const available = this.resampler.getOutSamples(0);
|
||||||
|
if (available < frameSize) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
await this.resampler.convert(this.dstFrame.data, frameSize, null, 0);
|
||||||
|
|
||||||
|
this.dstFrame.pts = this.nextResamplerPts;
|
||||||
|
|
||||||
|
await this.sendFrameAndReceivePackets(this.dstFrame);
|
||||||
|
this.nextResamplerPts += BigInt(frameSize);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async sendFrameAndReceivePackets(frame: NodeAv.Frame | null) {
|
||||||
|
assert(this.codecContext);
|
||||||
|
|
||||||
|
const ret = await this.codecContext.sendFrame(frame);
|
||||||
|
NodeAv.FFmpegError.throwIfError(ret, 'Send frame');
|
||||||
|
|
||||||
|
while (true) {
|
||||||
|
const receiveRet = await this.codecContext.receivePacket(this.packet);
|
||||||
|
if (receiveRet === NodeAv.AVERROR_EAGAIN || receiveRet === NodeAv.AVERROR_EOF) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
this.receivePacket(receiveRet);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
receivePacket(ret: number) {
|
||||||
|
assert(this.codecContext);
|
||||||
|
assert(this.firstExpectedTimestamp !== null);
|
||||||
|
NodeAv.FFmpegError.throwIfError(ret, 'Receive packet');
|
||||||
|
|
||||||
|
if (!this.packet.data) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let timestamp = Number(this.packet.pts) / this.codecContext.sampleRate;
|
||||||
|
const duration = Number(this.packet.duration) / this.codecContext.sampleRate;
|
||||||
|
|
||||||
|
let data: Uint8Array = this.packet.data;
|
||||||
|
|
||||||
|
let metadata: EncodedAudioChunkMetadata | undefined;
|
||||||
|
if (this.packetEmitted) {
|
||||||
|
metadata = {};
|
||||||
|
} else {
|
||||||
|
// To compensate for any negative timestamp things that FFmpeg might do. It does these for a reason, to
|
||||||
|
// indicate encoder delay, but the notion of this is not yet supported in Mediabunny.
|
||||||
|
this.outputTimestampOffset = this.firstExpectedTimestamp - timestamp;
|
||||||
|
|
||||||
|
const codecString = this.config.codec;
|
||||||
|
let description = this.codecContext.extraData
|
||||||
|
? toUint8Array(this.codecContext.extraData)
|
||||||
|
: undefined;
|
||||||
|
|
||||||
|
if (
|
||||||
|
description
|
||||||
|
// eslint-disable-next-line @stylistic/max-len
|
||||||
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access
|
||||||
|
&& this.codec === 'aac' && (this.config as any).aac?.format === 'adts'
|
||||||
|
) {
|
||||||
|
const config = parseAacAudioSpecificConfig(description);
|
||||||
|
this.adtsHeaderTemplate = buildAdtsHeaderTemplate(config);
|
||||||
|
description = undefined; // Not used with 'adts' format
|
||||||
|
}
|
||||||
|
|
||||||
|
if (description && this.codec === 'flac') {
|
||||||
|
// FFmpeg uses the STREAMINFO block as the extradata, but WebCodecs wants a different format:
|
||||||
|
// 1. The bytes 0x66 0x4C 0x61 0x43 ("fLaC" in ASCII)
|
||||||
|
// 2. A metadata block (called the STREAMINFO block) as described in section 7 of [FLAC]
|
||||||
|
// 3. Other optional metadata blocks (not included here, because, well, they're optional)
|
||||||
|
description = new Uint8Array([
|
||||||
|
0x66, 0x4c, 0x61, 0x43, // 'fLaC'
|
||||||
|
128, 0, 0, description.byteLength,
|
||||||
|
...description,
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
|
||||||
|
metadata = {
|
||||||
|
decoderConfig: {
|
||||||
|
codec: codecString,
|
||||||
|
sampleRate: this.codecContext.sampleRate,
|
||||||
|
numberOfChannels: this.codecContext.channels,
|
||||||
|
description,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (this.adtsHeaderTemplate) {
|
||||||
|
const frameLength = data.byteLength + this.adtsHeaderTemplate.header.byteLength;
|
||||||
|
this.adtsHeaderTemplate.bitstream.pos = 30;
|
||||||
|
this.adtsHeaderTemplate.bitstream.writeBits(13, frameLength);
|
||||||
|
|
||||||
|
const final = new Uint8Array(this.adtsHeaderTemplate.header.byteLength + data.byteLength);
|
||||||
|
final.set(this.adtsHeaderTemplate.header, 0);
|
||||||
|
final.set(data, this.adtsHeaderTemplate.header.byteLength);
|
||||||
|
|
||||||
|
data = final;
|
||||||
|
}
|
||||||
|
|
||||||
|
timestamp += this.outputTimestampOffset;
|
||||||
|
|
||||||
|
const packet = new EncodedPacket(
|
||||||
|
data,
|
||||||
|
'key',
|
||||||
|
timestamp,
|
||||||
|
duration,
|
||||||
|
);
|
||||||
|
|
||||||
|
this.onPacket(packet, metadata);
|
||||||
|
this.packetEmitted = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
async flush(): Promise<void> {
|
||||||
|
if (!this.codecContext) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
outer:
|
||||||
|
if (this.resampler) {
|
||||||
|
assert(this.resamplerInputSampleRate !== null);
|
||||||
|
|
||||||
|
const currentOutSamples = this.resampler.getOutSamples(0);
|
||||||
|
if (currentOutSamples === 0) {
|
||||||
|
break outer; // Clean cut-off point
|
||||||
|
}
|
||||||
|
|
||||||
|
const frameSize = this.codecContext.frameSize || FRAME_SIZE_FALLBACK;
|
||||||
|
assert(currentOutSamples < frameSize); // Because if it's more, it would've already been retrieved
|
||||||
|
|
||||||
|
const inputSamplesNeeded = Math.ceil(
|
||||||
|
((frameSize - currentOutSamples) / this.codecContext.sampleRate) * this.resamplerInputSampleRate,
|
||||||
|
);
|
||||||
|
this.resampler.injectSilence(inputSamplesNeeded);
|
||||||
|
|
||||||
|
await this.pullResampledFrames();
|
||||||
|
}
|
||||||
|
|
||||||
|
await this.sendFrameAndReceivePackets(null);
|
||||||
|
|
||||||
|
this.codecContext.freeContext();
|
||||||
|
this.codecContext = null;
|
||||||
|
this.packetEmitted = false;
|
||||||
|
this.firstExpectedTimestamp = null;
|
||||||
|
this.outputTimestampOffset = 0;
|
||||||
|
|
||||||
|
this.resampler?.free();
|
||||||
|
this.resampler = null;
|
||||||
|
this.inputParametersKey = null;
|
||||||
|
this.resamplerInputSampleRate = null;
|
||||||
|
this.nextResamplerPts = null;
|
||||||
|
this.dstFrame?.free();
|
||||||
|
this.dstFrame = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
close(): MaybePromise<void> {
|
||||||
|
this.codecContext?.freeContext();
|
||||||
|
this.frame.free();
|
||||||
|
this.packet.free();
|
||||||
|
this.dstFrame?.free();
|
||||||
|
this.resampler?.free();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,55 @@
|
|||||||
|
import { AudioSampleResource } from 'mediabunny';
|
||||||
|
import * as NodeAv from 'node-av';
|
||||||
|
import { toAudioSampleFormat } from './misc';
|
||||||
|
import { assert, toUint8Array } from '../../../src/misc';
|
||||||
|
|
||||||
|
export class NodeAvFrameAudioSampleResource extends AudioSampleResource {
|
||||||
|
frame: NodeAv.Frame;
|
||||||
|
timestamp: number;
|
||||||
|
|
||||||
|
constructor(frame: NodeAv.Frame, timestamp: number) {
|
||||||
|
super();
|
||||||
|
|
||||||
|
const clone = frame.clone();
|
||||||
|
if (!clone) {
|
||||||
|
throw new Error('Allocation failure during frame clone.');
|
||||||
|
}
|
||||||
|
|
||||||
|
this.frame = clone;
|
||||||
|
this.timestamp = timestamp;
|
||||||
|
}
|
||||||
|
|
||||||
|
getFormat(): AudioSampleFormat {
|
||||||
|
const result = toAudioSampleFormat(this.frame.format as NodeAv.AVSampleFormat);
|
||||||
|
if (result === null) {
|
||||||
|
throw new TypeError('Unsupported audio sample format: ' + this.frame.format);
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
getSampleRate(): number {
|
||||||
|
return this.frame.sampleRate;
|
||||||
|
}
|
||||||
|
|
||||||
|
getNumberOfChannels(): number {
|
||||||
|
return this.frame.channels;
|
||||||
|
}
|
||||||
|
|
||||||
|
getNumberOfFrames(): number {
|
||||||
|
return this.frame.nbSamples;
|
||||||
|
}
|
||||||
|
|
||||||
|
getTimestamp(): number {
|
||||||
|
return this.timestamp;
|
||||||
|
}
|
||||||
|
|
||||||
|
close(): void {
|
||||||
|
this.frame.free();
|
||||||
|
}
|
||||||
|
|
||||||
|
getDataPlane(planeIndex: number): Uint8Array {
|
||||||
|
assert(this.frame.data && planeIndex < this.frame.data.length);
|
||||||
|
return toUint8Array(this.frame.data[planeIndex]!);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,7 +1,9 @@
|
|||||||
import { registerDecoder, registerEncoder } from 'mediabunny';
|
import { registerDecoder, registerEncoder } from 'mediabunny';
|
||||||
import { NodeAvVideoDecoder } from './video-decoder';
|
|
||||||
import * as NodeAv from 'node-av';
|
import * as NodeAv from 'node-av';
|
||||||
|
import { NodeAvVideoDecoder } from './video-decoder';
|
||||||
import { NodeAvVideoEncoder } from './video-encoder';
|
import { NodeAvVideoEncoder } from './video-encoder';
|
||||||
|
import { NodeAvAudioDecoder } from './audio-decoder';
|
||||||
|
import { NodeAvAudioEncoder } from './audio-encoder';
|
||||||
|
|
||||||
const SERVER_LOADED_SYMBOL = Symbol.for('@mediabunny/server loaded');
|
const SERVER_LOADED_SYMBOL = Symbol.for('@mediabunny/server loaded');
|
||||||
if ((globalThis as Record<symbol, unknown>)[SERVER_LOADED_SYMBOL]) {
|
if ((globalThis as Record<symbol, unknown>)[SERVER_LOADED_SYMBOL]) {
|
||||||
@@ -23,6 +25,11 @@ export const registerMediabunnyServer = () => {
|
|||||||
|
|
||||||
NodeAv.Log.setLevel(NodeAv.AV_LOG_ERROR);
|
NodeAv.Log.setLevel(NodeAv.AV_LOG_ERROR);
|
||||||
|
|
||||||
|
// Video
|
||||||
registerDecoder(NodeAvVideoDecoder);
|
registerDecoder(NodeAvVideoDecoder);
|
||||||
registerEncoder(NodeAvVideoEncoder);
|
registerEncoder(NodeAvVideoEncoder);
|
||||||
|
|
||||||
|
// Audio
|
||||||
|
registerDecoder(NodeAvAudioDecoder);
|
||||||
|
registerEncoder(NodeAvAudioEncoder);
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -9,6 +9,12 @@ export const CODEC_TO_CODEC_ID: Partial<Record<MediaCodec, NodeAv.AVCodecID>> =
|
|||||||
av1: NodeAv.AV_CODEC_ID_AV1,
|
av1: NodeAv.AV_CODEC_ID_AV1,
|
||||||
|
|
||||||
aac: NodeAv.AV_CODEC_ID_AAC,
|
aac: NodeAv.AV_CODEC_ID_AAC,
|
||||||
|
opus: NodeAv.AV_CODEC_ID_OPUS,
|
||||||
|
mp3: NodeAv.AV_CODEC_ID_MP3,
|
||||||
|
vorbis: NodeAv.AV_CODEC_ID_VORBIS,
|
||||||
|
flac: NodeAv.AV_CODEC_ID_FLAC,
|
||||||
|
ac3: NodeAv.AV_CODEC_ID_AC3,
|
||||||
|
eac3: NodeAv.AV_CODEC_ID_EAC3,
|
||||||
};
|
};
|
||||||
|
|
||||||
let cachedHardwareContext: NodeAv.HardwareContext | null | undefined = undefined;
|
let cachedHardwareContext: NodeAv.HardwareContext | null | undefined = undefined;
|
||||||
|
|||||||
@@ -165,10 +165,8 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
|
|||||||
assert(this.codecContext);
|
assert(this.codecContext);
|
||||||
}
|
}
|
||||||
|
|
||||||
const underlyingData = videoSample.getUnderlyingData();
|
if (videoSample._data instanceof NodeAvFrameVideoSampleResource) {
|
||||||
|
this.frame.ref(videoSample._data.frame);
|
||||||
if (underlyingData instanceof NodeAvFrameVideoSampleResource) {
|
|
||||||
this.frame.ref(underlyingData.frame);
|
|
||||||
} else {
|
} else {
|
||||||
if (videoSample.format === null) {
|
if (videoSample.format === null) {
|
||||||
throw new Error('Cannot encode foreign VideoSample with unknown (null) format.');
|
throw new Error('Cannot encode foreign VideoSample with unknown (null) format.');
|
||||||
|
|||||||
+64
-29
@@ -615,15 +615,6 @@ export class VideoSample implements Disposable {
|
|||||||
finalizationRegistry?.register(this, { type: 'video', data: this._data }, this);
|
finalizationRegistry?.register(this, { type: 'video', data: this._data }, this);
|
||||||
}
|
}
|
||||||
|
|
||||||
getUnderlyingData() {
|
|
||||||
if (this._closed) {
|
|
||||||
throw new Error('VideoSample is closed.');
|
|
||||||
}
|
|
||||||
|
|
||||||
assert(this._data !== null);
|
|
||||||
return this._data;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Clones this video sample. */
|
/** Clones this video sample. */
|
||||||
clone() {
|
clone() {
|
||||||
if (this._closed) {
|
if (this._closed) {
|
||||||
@@ -1881,15 +1872,7 @@ export abstract class AudioSampleResource {
|
|||||||
*/
|
*/
|
||||||
abstract close(): void;
|
abstract close(): void;
|
||||||
|
|
||||||
/**
|
abstract getDataPlane(planeIndex: number): Uint8Array;
|
||||||
* Returns the number of bytes required to hold the underlying audio data as specified by the given options.
|
|
||||||
*/
|
|
||||||
abstract allocationSize(options: AudioSampleCopyToOptions): number;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Copies the underlying audio data to an ArrayBuffer or ArrayBufferView as specified by the given options.
|
|
||||||
*/
|
|
||||||
abstract copyTo(destination: AllowSharedBufferSource, options: AudioSampleCopyToOptions): void;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -2004,10 +1987,30 @@ export class AudioSample implements Disposable {
|
|||||||
init._referenceCount++;
|
init._referenceCount++;
|
||||||
|
|
||||||
this.format = init.getFormat();
|
this.format = init.getFormat();
|
||||||
|
if (!AUDIO_SAMPLE_FORMATS.has(this.format)) {
|
||||||
|
throw new TypeError('getFormat() must return an AudioSampleFormat.');
|
||||||
|
}
|
||||||
|
|
||||||
this.sampleRate = init.getSampleRate();
|
this.sampleRate = init.getSampleRate();
|
||||||
|
if (!Number.isInteger(this.sampleRate) || this.sampleRate <= 0) {
|
||||||
|
throw new TypeError('getSampleRate() must return a positive integer.');
|
||||||
|
}
|
||||||
|
|
||||||
this.numberOfFrames = init.getNumberOfFrames();
|
this.numberOfFrames = init.getNumberOfFrames();
|
||||||
|
if (!Number.isInteger(this.numberOfFrames) || this.numberOfFrames < 0) {
|
||||||
|
throw new TypeError('getNumberOfFrames() must return a non-negative integer.');
|
||||||
|
}
|
||||||
|
|
||||||
this.numberOfChannels = init.getNumberOfChannels();
|
this.numberOfChannels = init.getNumberOfChannels();
|
||||||
|
if (!Number.isInteger(this.numberOfChannels) || this.numberOfChannels <= 0) {
|
||||||
|
throw new TypeError('getNumberOfChannels() must return a positive integer.');
|
||||||
|
}
|
||||||
|
|
||||||
this.timestamp = init.getTimestamp();
|
this.timestamp = init.getTimestamp();
|
||||||
|
if (!Number.isFinite(this.timestamp)) {
|
||||||
|
throw new TypeError('getTimestamp() must return a finite number.');
|
||||||
|
}
|
||||||
|
|
||||||
this.duration = this.numberOfFrames / this.sampleRate;
|
this.duration = this.numberOfFrames / this.sampleRate;
|
||||||
} else {
|
} else {
|
||||||
if (!init || typeof init !== 'object') {
|
if (!init || typeof init !== 'object') {
|
||||||
@@ -2084,10 +2087,6 @@ export class AudioSample implements Disposable {
|
|||||||
throw new Error('AudioSample is closed.');
|
throw new Error('AudioSample is closed.');
|
||||||
}
|
}
|
||||||
|
|
||||||
if (this._data instanceof AudioSampleResource) {
|
|
||||||
return this._data.allocationSize(options);
|
|
||||||
}
|
|
||||||
|
|
||||||
const destFormat = options.format ?? this.format;
|
const destFormat = options.format ?? this.format;
|
||||||
|
|
||||||
const frameOffset = options.frameOffset ?? 0;
|
const frameOffset = options.frameOffset ?? 0;
|
||||||
@@ -2140,11 +2139,8 @@ export class AudioSample implements Disposable {
|
|||||||
throw new Error('AudioSample is closed.');
|
throw new Error('AudioSample is closed.');
|
||||||
}
|
}
|
||||||
|
|
||||||
if (this._data instanceof AudioSampleResource) {
|
const { format, frameCount: optFrameCount, frameOffset: optFrameOffset } = options;
|
||||||
return this._data.copyTo(destination, options);
|
let { planeIndex } = options;
|
||||||
}
|
|
||||||
|
|
||||||
const { planeIndex, format, frameCount: optFrameCount, frameOffset: optFrameOffset } = options;
|
|
||||||
|
|
||||||
const srcFormat = this.format;
|
const srcFormat = this.format;
|
||||||
const destFormat = format ?? this.format;
|
const destFormat = format ?? this.format;
|
||||||
@@ -2204,12 +2200,51 @@ export class AudioSample implements Disposable {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
const uint8Data = this._data;
|
|
||||||
const srcView = toDataView(uint8Data);
|
|
||||||
const readFn = getReadFunction(srcFormat);
|
const readFn = getReadFunction(srcFormat);
|
||||||
const srcBytesPerSample = getBytesPerSample(srcFormat);
|
const srcBytesPerSample = getBytesPerSample(srcFormat);
|
||||||
const srcIsPlanar = formatIsPlanar(srcFormat);
|
const srcIsPlanar = formatIsPlanar(srcFormat);
|
||||||
|
|
||||||
|
let uint8Data: Uint8Array;
|
||||||
|
if (this._data instanceof AudioSampleResource) {
|
||||||
|
const getDataPlaneValidated = (index: number) => {
|
||||||
|
const result = (this._data as AudioSampleResource).getDataPlane(index);
|
||||||
|
if (!(result instanceof Uint8Array)) {
|
||||||
|
throw new TypeError('getDataPlane() must return a Uint8Array.');
|
||||||
|
}
|
||||||
|
|
||||||
|
const expectedSize = numFrames * srcBytesPerSample * (srcIsPlanar ? 1 : numChannels);
|
||||||
|
if (result.byteLength !== expectedSize) {
|
||||||
|
throw new TypeError(
|
||||||
|
`Data plane ${index} has invalid size. Expected exactly ${expectedSize} bytes, got`
|
||||||
|
+ ` ${result.byteLength} bytes.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
};
|
||||||
|
|
||||||
|
if (srcIsPlanar) {
|
||||||
|
if (destIsPlanar) {
|
||||||
|
// Only one source plane will be extracted, so let's fetch only that one
|
||||||
|
uint8Data = getDataPlaneValidated(planeIndex);
|
||||||
|
planeIndex = 0; // To fix the subsequent access
|
||||||
|
} else {
|
||||||
|
// Pack all planes tightly together
|
||||||
|
uint8Data = new Uint8Array(numFrames * srcBytesPerSample * numChannels);
|
||||||
|
for (let ch = 0; ch < numChannels; ch++) {
|
||||||
|
const planeData = getDataPlaneValidated(ch);
|
||||||
|
uint8Data.set(planeData, ch * numFrames * srcBytesPerSample);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
uint8Data = getDataPlaneValidated(0); // That's the only plane there is
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
uint8Data = this._data;
|
||||||
|
}
|
||||||
|
|
||||||
|
const srcView = toDataView(uint8Data);
|
||||||
|
|
||||||
for (let i = 0; i < copyFrameCount; i++) {
|
for (let i = 0; i < copyFrameCount; i++) {
|
||||||
if (destIsPlanar) {
|
if (destIsPlanar) {
|
||||||
const destOffset = i * destBytesPerSample;
|
const destOffset = i * destBytesPerSample;
|
||||||
|
|||||||
@@ -3,13 +3,14 @@ import { registerMediabunnyServer } from '@mediabunny/server';
|
|||||||
import { Input } from '../../src/input.js';
|
import { Input } from '../../src/input.js';
|
||||||
import { BufferSource, FilePathSource } from '../../src/source.js';
|
import { BufferSource, FilePathSource } from '../../src/source.js';
|
||||||
import { ALL_FORMATS } from '../../src/input-format.js';
|
import { ALL_FORMATS } from '../../src/input-format.js';
|
||||||
import { assert, toUint8Array } from '../../src/misc.js';
|
import { assert, last, toUint8Array } from '../../src/misc.js';
|
||||||
import { EncodedPacketSink, VideoSampleSink } from '../../src/media-sink.js';
|
import { AudioSampleSink, EncodedPacketSink, VideoSampleSink } from '../../src/media-sink.js';
|
||||||
import { NodeAvVideoDecoder } from '../../packages/server/src/video-decoder.js';
|
import { NodeAvVideoDecoder } from '../../packages/server/src/video-decoder.js';
|
||||||
import { NodeAvVideoEncoder } from '../../packages/server/src/video-encoder.js';
|
import { NodeAvVideoEncoder } from '../../packages/server/src/video-encoder.js';
|
||||||
import { NodeAvAudioDecoder } from '../../packages/server/src/audio-decoder.js';
|
import { NodeAvAudioDecoder } from '../../packages/server/src/audio-decoder.js';
|
||||||
|
import { NodeAvAudioEncoder } from '../../packages/server/src/audio-encoder.js';
|
||||||
import { AudioSample, VideoSample } from '../../src/sample.js';
|
import { AudioSample, VideoSample } from '../../src/sample.js';
|
||||||
import { buildVideoCodecString, VideoCodec } from '../../src/codec.js';
|
import { AudioCodec, buildAudioCodecString, buildVideoCodecString, VideoCodec } from '../../src/codec.js';
|
||||||
import { EncodedPacket } from '../../src/packet.js';
|
import { EncodedPacket } from '../../src/packet.js';
|
||||||
import {
|
import {
|
||||||
AvcNalUnitType,
|
AvcNalUnitType,
|
||||||
@@ -24,6 +25,7 @@ import { Mp4OutputFormat } from '../../src/output-format.js';
|
|||||||
import { BufferTarget } from '../../src/target.js';
|
import { BufferTarget } from '../../src/target.js';
|
||||||
import { Conversion } from '../../src/conversion.js';
|
import { Conversion } from '../../src/conversion.js';
|
||||||
import { NodeAvFrameVideoSampleResource } from '../../packages/server/src/video-sample.js';
|
import { NodeAvFrameVideoSampleResource } from '../../packages/server/src/video-sample.js';
|
||||||
|
import { NodeAvFrameAudioSampleResource } from '../../packages/server/src/audio-sample.js';
|
||||||
|
|
||||||
beforeAll(() => {
|
beforeAll(() => {
|
||||||
registerMediabunnyServer();
|
registerMediabunnyServer();
|
||||||
@@ -600,7 +602,7 @@ describe('Video', async () => {
|
|||||||
using sample = await sink.getSample(0);
|
using sample = await sink.getSample(0);
|
||||||
assert(sample);
|
assert(sample);
|
||||||
|
|
||||||
expect(sample.getUnderlyingData()).toBeInstanceOf(NodeAvFrameVideoSampleResource);
|
expect(sample._data).toBeInstanceOf(NodeAvFrameVideoSampleResource);
|
||||||
expect(sample.format).toBe('I420');
|
expect(sample.format).toBe('I420');
|
||||||
expect(sample.codedWidth).toBe(1920);
|
expect(sample.codedWidth).toBe(1920);
|
||||||
expect(sample.codedHeight).toBe(1080);
|
expect(sample.codedHeight).toBe(1080);
|
||||||
@@ -736,7 +738,6 @@ describe('Video', async () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
/*
|
|
||||||
describe('Audio', async () => {
|
describe('Audio', async () => {
|
||||||
test('Decoder lifecycle', async () => {
|
test('Decoder lifecycle', async () => {
|
||||||
using input = new Input({
|
using input = new Input({
|
||||||
@@ -807,5 +808,515 @@ describe('Audio', async () => {
|
|||||||
|
|
||||||
await decoder.close();
|
await decoder.close();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('Encoder lifecycle', async () => {
|
||||||
|
const encoder = new NodeAvAudioEncoder();
|
||||||
|
// @ts-expect-error Readonly
|
||||||
|
encoder.codec = 'aac';
|
||||||
|
// @ts-expect-error Readonly
|
||||||
|
encoder.config = {
|
||||||
|
codec: buildAudioCodecString('aac', 2, 48000),
|
||||||
|
numberOfChannels: 2,
|
||||||
|
sampleRate: 48000,
|
||||||
|
bitrate: 128000,
|
||||||
|
} satisfies AudioEncoderConfig;
|
||||||
|
|
||||||
|
let packetCount = 0;
|
||||||
|
|
||||||
|
// @ts-expect-error Readonly
|
||||||
|
encoder.onPacket = (packet: EncodedPacket, meta: EncodedAudioChunkMetadata) => {
|
||||||
|
expect(packet.duration).toBe(1024 / 48000);
|
||||||
|
expect(packet.type).toBe('key');
|
||||||
|
|
||||||
|
if (packetCount === 0) {
|
||||||
|
expect(packet.timestamp).toBe(0);
|
||||||
|
|
||||||
|
expect(meta.decoderConfig).toBeDefined();
|
||||||
|
expect(meta.decoderConfig!.codec).toBe('mp4a.40.2');
|
||||||
|
expect(meta.decoderConfig!.numberOfChannels).toBe(2);
|
||||||
|
expect(meta.decoderConfig!.sampleRate).toBe(48000);
|
||||||
|
expect(meta.decoderConfig?.description).toBeDefined();
|
||||||
|
}
|
||||||
|
|
||||||
|
packetCount++;
|
||||||
|
};
|
||||||
|
|
||||||
|
await encoder.init();
|
||||||
|
|
||||||
|
const data = createF32SineWave(48000, 2, 2);
|
||||||
|
using sample1 = new AudioSample({
|
||||||
|
data,
|
||||||
|
format: 'f32',
|
||||||
|
timestamp: 0,
|
||||||
|
numberOfChannels: 2,
|
||||||
|
sampleRate: 48000,
|
||||||
|
});
|
||||||
|
await encoder.encode(sample1);
|
||||||
|
|
||||||
|
await encoder.flush();
|
||||||
|
|
||||||
|
const minPacketCount = 2 * 48000 / 1024;
|
||||||
|
expect(packetCount).toBeGreaterThan(minPacketCount);
|
||||||
|
|
||||||
|
packetCount = 0;
|
||||||
|
|
||||||
|
using sample2 = new AudioSample({
|
||||||
|
data,
|
||||||
|
format: 'f32',
|
||||||
|
timestamp: 0,
|
||||||
|
numberOfChannels: 2,
|
||||||
|
sampleRate: 48000,
|
||||||
|
});
|
||||||
|
await encoder.encode(sample2);
|
||||||
|
|
||||||
|
await encoder.flush();
|
||||||
|
|
||||||
|
expect(packetCount).toBeGreaterThan(minPacketCount);
|
||||||
|
|
||||||
|
await encoder.close();
|
||||||
|
});
|
||||||
|
|
||||||
|
const createF32SineWave = (sampleRate: number, channels: number, durationSeconds: number) => {
|
||||||
|
const totalFrames = sampleRate * durationSeconds;
|
||||||
|
const data = new Float32Array(totalFrames * channels);
|
||||||
|
|
||||||
|
for (let i = 0; i < totalFrames; i++) {
|
||||||
|
const value = Math.sin(2 * Math.PI * 440 * i / sampleRate);
|
||||||
|
for (let ch = 0; ch < channels; ch++) {
|
||||||
|
data[i * channels + ch] = value;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return data;
|
||||||
|
};
|
||||||
|
|
||||||
|
test('AAC encode & decode, AAC format', async () => {
|
||||||
|
await encodeDecodeTest('aac', {
|
||||||
|
// @ts-expect-error Fucky type
|
||||||
|
aac: { format: 'aac' },
|
||||||
|
}, async (packet, meta, i) => {
|
||||||
|
expect(packet.type).toBe('key');
|
||||||
|
expect(packet.duration).toBe(1024 / 48000);
|
||||||
|
|
||||||
|
if (i === 0) {
|
||||||
|
expect(meta.decoderConfig).toBeDefined();
|
||||||
|
expect(meta.decoderConfig!.codec).toBe('mp4a.40.2');
|
||||||
|
expect(meta.decoderConfig!.numberOfChannels).toBe(2);
|
||||||
|
expect(meta.decoderConfig!.sampleRate).toBe(48000);
|
||||||
|
expect(meta.decoderConfig!.description).toBeDefined();
|
||||||
|
}
|
||||||
|
}, async (sample) => {
|
||||||
|
expect(sample.duration).toBe(1024 / 48000);
|
||||||
|
expect(sample.numberOfChannels).toBe(2);
|
||||||
|
expect(sample.sampleRate).toBe(48000);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test('AAC encode & decode, ADTS format', async () => {
|
||||||
|
await encodeDecodeTest('aac', {
|
||||||
|
// @ts-expect-error Fucky type
|
||||||
|
aac: { format: 'adts' },
|
||||||
|
}, async (packet, meta, i) => {
|
||||||
|
expect(packet.type).toBe('key');
|
||||||
|
expect(packet.duration).toBe(1024 / 48000);
|
||||||
|
expect(packet.data[0]).toBe(0xff);
|
||||||
|
|
||||||
|
if (i === 0) {
|
||||||
|
expect(meta.decoderConfig).toBeDefined();
|
||||||
|
expect(meta.decoderConfig!.codec).toBe('mp4a.40.2');
|
||||||
|
expect(meta.decoderConfig!.numberOfChannels).toBe(2);
|
||||||
|
expect(meta.decoderConfig!.sampleRate).toBe(48000);
|
||||||
|
expect(meta.decoderConfig!.description).toBeUndefined();
|
||||||
|
}
|
||||||
|
}, async (sample) => {
|
||||||
|
expect(sample.duration).toBe(1024 / 48000);
|
||||||
|
expect(sample.numberOfChannels).toBe(2);
|
||||||
|
expect(sample.sampleRate).toBe(48000);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test('Opus encode & decode', async () => {
|
||||||
|
await encodeDecodeTest('opus', {}, async (packet, meta, i, n) => {
|
||||||
|
expect(packet.type).toBe('key');
|
||||||
|
|
||||||
|
if (i < n - 1) {
|
||||||
|
expect(packet.duration).toBe(0.02);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (i === 0) {
|
||||||
|
expect(meta.decoderConfig).toBeDefined();
|
||||||
|
expect(meta.decoderConfig!.codec).toBe('opus');
|
||||||
|
expect(meta.decoderConfig!.numberOfChannels).toBe(2);
|
||||||
|
expect(meta.decoderConfig!.sampleRate).toBe(48000);
|
||||||
|
expect(meta.decoderConfig!.description).toBeDefined();
|
||||||
|
expect([...toUint8Array(meta.decoderConfig!.description!).slice(0, 8)]).toEqual([
|
||||||
|
// OpusHead
|
||||||
|
0x4f, 0x70, 0x75, 0x73, 0x48, 0x65, 0x61, 0x64,
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
}, async (sample, i, n) => {
|
||||||
|
expect(sample.numberOfChannels).toBe(2);
|
||||||
|
expect(sample.sampleRate).toBe(48000);
|
||||||
|
|
||||||
|
if (i > 0 && i < n - 1) {
|
||||||
|
expect(sample.duration).toBe(0.02);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test('MP3 encode & decode', async () => {
|
||||||
|
await encodeDecodeTest('mp3', {}, async (packet, meta, i, n) => {
|
||||||
|
expect(packet.type).toBe('key');
|
||||||
|
|
||||||
|
if (i < n - 1) {
|
||||||
|
expect(packet.duration).toBe(1152 / 48000);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (i === 0) {
|
||||||
|
expect(meta.decoderConfig).toBeDefined();
|
||||||
|
expect(meta.decoderConfig!.codec).toBe('mp3');
|
||||||
|
expect(meta.decoderConfig!.numberOfChannels).toBe(2);
|
||||||
|
expect(meta.decoderConfig!.sampleRate).toBe(48000);
|
||||||
|
expect(meta.decoderConfig!.description).toBeUndefined();
|
||||||
|
}
|
||||||
|
}, async (sample) => {
|
||||||
|
expect(sample.numberOfChannels).toBe(2);
|
||||||
|
expect(sample.sampleRate).toBe(48000);
|
||||||
|
expect(sample.duration).toBe(1152 / 48000);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test('Vorbis encode & decode', async () => {
|
||||||
|
await encodeDecodeTest('vorbis', {}, async (packet, meta, i) => {
|
||||||
|
expect(packet.type).toBe('key');
|
||||||
|
|
||||||
|
if (i === 0) {
|
||||||
|
expect(meta.decoderConfig).toBeDefined();
|
||||||
|
expect(meta.decoderConfig!.codec).toBe('vorbis');
|
||||||
|
expect(meta.decoderConfig!.numberOfChannels).toBe(2);
|
||||||
|
expect(meta.decoderConfig!.sampleRate).toBe(48000);
|
||||||
|
expect(meta.decoderConfig!.description).toBeDefined();
|
||||||
|
expect(toUint8Array(meta.decoderConfig!.description!)[0]).toBe(2); // "Xiph extradata format"
|
||||||
|
}
|
||||||
|
}, async (sample) => {
|
||||||
|
expect(sample.numberOfChannels).toBe(2);
|
||||||
|
expect(sample.sampleRate).toBe(48000);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test('FLAC encode & decode', async () => {
|
||||||
|
await encodeDecodeTest('flac', {}, async (packet, meta, i) => {
|
||||||
|
expect(packet.type).toBe('key');
|
||||||
|
expect(packet.duration).toBe(0.096);
|
||||||
|
|
||||||
|
if (i === 0) {
|
||||||
|
expect(meta.decoderConfig).toBeDefined();
|
||||||
|
expect(meta.decoderConfig!.codec).toBe('flac');
|
||||||
|
expect(meta.decoderConfig!.numberOfChannels).toBe(2);
|
||||||
|
expect(meta.decoderConfig!.sampleRate).toBe(48000);
|
||||||
|
expect(meta.decoderConfig!.description).toBeDefined();
|
||||||
|
expect([...toUint8Array(meta.decoderConfig!.description!).slice(0, 4)]).toEqual([
|
||||||
|
// fLaC
|
||||||
|
0x66, 0x4c, 0x61, 0x43,
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
}, async (sample) => {
|
||||||
|
expect(sample.numberOfChannels).toBe(2);
|
||||||
|
expect(sample.sampleRate).toBe(48000);
|
||||||
|
expect(sample.duration).toBe(0.096);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test('AC-3 encode & decode', async () => {
|
||||||
|
await encodeDecodeTest('ac3', {}, async (packet, meta, i) => {
|
||||||
|
expect(packet.type).toBe('key');
|
||||||
|
expect(packet.duration).toBe(0.032);
|
||||||
|
|
||||||
|
if (i === 0) {
|
||||||
|
expect(meta.decoderConfig).toBeDefined();
|
||||||
|
expect(meta.decoderConfig!.codec).toBe('ac-3');
|
||||||
|
expect(meta.decoderConfig!.numberOfChannels).toBe(2);
|
||||||
|
expect(meta.decoderConfig!.sampleRate).toBe(48000);
|
||||||
|
expect(meta.decoderConfig!.description).toBeUndefined();
|
||||||
|
}
|
||||||
|
}, async (sample) => {
|
||||||
|
expect(sample.numberOfChannels).toBe(2);
|
||||||
|
expect(sample.sampleRate).toBe(48000);
|
||||||
|
expect(sample.duration).toBe(0.032);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test('E-AC-3 encode & decode', async () => {
|
||||||
|
await encodeDecodeTest('eac3', {}, async (packet, meta, i) => {
|
||||||
|
expect(packet.type).toBe('key');
|
||||||
|
expect(packet.duration).toBe(0.032);
|
||||||
|
|
||||||
|
if (i === 0) {
|
||||||
|
expect(meta.decoderConfig).toBeDefined();
|
||||||
|
expect(meta.decoderConfig!.codec).toBe('ec-3');
|
||||||
|
expect(meta.decoderConfig!.numberOfChannels).toBe(2);
|
||||||
|
expect(meta.decoderConfig!.sampleRate).toBe(48000);
|
||||||
|
expect(meta.decoderConfig!.description).toBeUndefined();
|
||||||
|
}
|
||||||
|
}, async (sample) => {
|
||||||
|
expect(sample.numberOfChannels).toBe(2);
|
||||||
|
expect(sample.sampleRate).toBe(48000);
|
||||||
|
expect(sample.duration).toBe(0.032);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
const encodeDecodeTest = async (
|
||||||
|
codec: AudioCodec,
|
||||||
|
extraConfig: Partial<AudioEncoderConfig>,
|
||||||
|
onPacket: (packet: EncodedPacket, meta: EncodedAudioChunkMetadata, i: number, n: number) => Promise<void>,
|
||||||
|
onSample: (sample: AudioSample, i: number, n: number) => Promise<void>,
|
||||||
|
) => {
|
||||||
|
const sampleRate = 48000;
|
||||||
|
const numberOfChannels = 2;
|
||||||
|
|
||||||
|
const encoder = new NodeAvAudioEncoder();
|
||||||
|
// @ts-expect-error Readonly
|
||||||
|
encoder.codec = codec;
|
||||||
|
// @ts-expect-error Readonly
|
||||||
|
encoder.config = {
|
||||||
|
codec: buildAudioCodecString(codec, numberOfChannels, sampleRate),
|
||||||
|
numberOfChannels,
|
||||||
|
sampleRate,
|
||||||
|
bitrate: 128000,
|
||||||
|
...extraConfig,
|
||||||
|
} satisfies AudioEncoderConfig;
|
||||||
|
|
||||||
|
const packets: EncodedPacket[] = [];
|
||||||
|
const metas: EncodedAudioChunkMetadata[] = [];
|
||||||
|
|
||||||
|
// @ts-expect-error Readonly
|
||||||
|
encoder.onPacket = (packet: EncodedPacket, meta: EncodedAudioChunkMetadata) => {
|
||||||
|
packets.push(packet);
|
||||||
|
metas.push(meta);
|
||||||
|
};
|
||||||
|
|
||||||
|
await encoder.init();
|
||||||
|
|
||||||
|
const data = createF32SineWave(sampleRate, numberOfChannels, 2);
|
||||||
|
using inputSample = new AudioSample({
|
||||||
|
data,
|
||||||
|
format: 'f32',
|
||||||
|
timestamp: 0,
|
||||||
|
numberOfChannels,
|
||||||
|
sampleRate,
|
||||||
|
});
|
||||||
|
await encoder.encode(inputSample);
|
||||||
|
|
||||||
|
await encoder.flush();
|
||||||
|
|
||||||
|
for (let i = 0; i < packets.length; i++) {
|
||||||
|
await onPacket(packets[i]!, metas[i]!, i, packets.length);
|
||||||
|
}
|
||||||
|
|
||||||
|
const decoder = new NodeAvAudioDecoder();
|
||||||
|
// @ts-expect-error Readonly
|
||||||
|
decoder.codec = codec;
|
||||||
|
// @ts-expect-error Readonly
|
||||||
|
decoder.config = metas[0]!.decoderConfig!;
|
||||||
|
|
||||||
|
const decodedSamples: AudioSample[] = [];
|
||||||
|
|
||||||
|
// @ts-expect-error Readonly
|
||||||
|
decoder.onSample = (sample: AudioSample) => {
|
||||||
|
decodedSamples.push(sample);
|
||||||
|
};
|
||||||
|
|
||||||
|
await decoder.init();
|
||||||
|
|
||||||
|
for (const packet of packets) {
|
||||||
|
await decoder.decode(packet);
|
||||||
|
}
|
||||||
|
|
||||||
|
await decoder.flush();
|
||||||
|
|
||||||
|
expect(decodedSamples.length).toBeGreaterThan(0);
|
||||||
|
expect(last(decodedSamples)!.timestamp + last(decodedSamples)!.duration).toBeGreaterThanOrEqual(2);
|
||||||
|
|
||||||
|
for (let i = 0; i < decodedSamples.length; i++) {
|
||||||
|
await onSample(decodedSamples[i]!, i, decodedSamples.length);
|
||||||
|
}
|
||||||
|
|
||||||
|
const signalChunks: Float32Array[] = [];
|
||||||
|
|
||||||
|
for (const sample of decodedSamples) {
|
||||||
|
const buf = new Float32Array(new ArrayBuffer(sample.allocationSize({
|
||||||
|
format: 'f32-planar', planeIndex: 0,
|
||||||
|
})));
|
||||||
|
sample.copyTo(buf, { format: 'f32-planar', planeIndex: 0 });
|
||||||
|
signalChunks.push(buf);
|
||||||
|
|
||||||
|
sample.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
const totalSize = signalChunks.reduce((sum, chunk) => sum + chunk.length, 0);
|
||||||
|
const signal = new Float32Array(totalSize);
|
||||||
|
let offset = 0;
|
||||||
|
for (const chunk of signalChunks) {
|
||||||
|
signal.set(chunk, offset);
|
||||||
|
offset += chunk.length;
|
||||||
|
}
|
||||||
|
|
||||||
|
const score = sine440Score(signal, sampleRate, 440);
|
||||||
|
expect(score).toBeGreaterThan(0.98);
|
||||||
|
|
||||||
|
await encoder.close();
|
||||||
|
await decoder.close();
|
||||||
|
};
|
||||||
|
|
||||||
|
const sine440Score = (x: Float32Array, sampleRate: number, freq: number) => {
|
||||||
|
const w = 2 * Math.PI * freq / sampleRate;
|
||||||
|
|
||||||
|
let ss = 0, cc = 0, sc = 0;
|
||||||
|
let xs = 0, xc = 0;
|
||||||
|
let xx = 0;
|
||||||
|
|
||||||
|
for (let n = 0; n < x.length; n++) {
|
||||||
|
const s = Math.sin(w * n);
|
||||||
|
const c = Math.cos(w * n);
|
||||||
|
|
||||||
|
ss += s * s;
|
||||||
|
cc += c * c;
|
||||||
|
sc += s * c;
|
||||||
|
|
||||||
|
xs += x[n]! * s;
|
||||||
|
xc += x[n]! * c;
|
||||||
|
xx += x[n]! * x[n]!;
|
||||||
|
}
|
||||||
|
|
||||||
|
const det = ss * cc - sc * sc;
|
||||||
|
|
||||||
|
const a = (xs * cc - xc * sc) / det;
|
||||||
|
const b = (xc * ss - xs * sc) / det;
|
||||||
|
|
||||||
|
let fitEnergy = 0;
|
||||||
|
|
||||||
|
for (let n = 0; n < x.length; n++) {
|
||||||
|
const y = a * Math.sin(w * n) + b * Math.cos(w * n);
|
||||||
|
fitEnergy += y * y;
|
||||||
|
}
|
||||||
|
|
||||||
|
const score = fitEnergy / xx;
|
||||||
|
return score;
|
||||||
|
};
|
||||||
|
|
||||||
|
test('AAC conversion roundtrip', { timeout: 10_000 }, async () => {
|
||||||
|
await conversionRoundtrip('aac');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('Opus conversion roundtrip', { timeout: 10_000 }, async () => {
|
||||||
|
await conversionRoundtrip('opus');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('MP3 conversion roundtrip', { timeout: 10_000 }, async () => {
|
||||||
|
await conversionRoundtrip('mp3');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('Vorbis conversion roundtrip', { timeout: 10_000 }, async () => {
|
||||||
|
await conversionRoundtrip('vorbis');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('FLAC conversion roundtrip', { timeout: 10_000 }, async () => {
|
||||||
|
await conversionRoundtrip('flac');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('AC-3 conversion roundtrip', { timeout: 10_000 }, async () => {
|
||||||
|
await conversionRoundtrip('ac3');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('E-AC-3 conversion roundtrip', { timeout: 10_000 }, async () => {
|
||||||
|
await conversionRoundtrip('eac3');
|
||||||
|
});
|
||||||
|
|
||||||
|
const conversionRoundtrip = async (codec: AudioCodec) => {
|
||||||
|
using input = new Input({
|
||||||
|
source: new FilePathSource('./test/public/trim-buck-bunny-ffmpeg.ts'),
|
||||||
|
formats: ALL_FORMATS,
|
||||||
|
});
|
||||||
|
|
||||||
|
const output = new Output({
|
||||||
|
format: new Mp4OutputFormat(),
|
||||||
|
target: new BufferTarget(),
|
||||||
|
});
|
||||||
|
|
||||||
|
const inputTrack = await input.getPrimaryAudioTrack();
|
||||||
|
assert(inputTrack);
|
||||||
|
|
||||||
|
const conversion = await Conversion.init({
|
||||||
|
input,
|
||||||
|
output,
|
||||||
|
video: {
|
||||||
|
discard: true,
|
||||||
|
}, audio: {
|
||||||
|
codec,
|
||||||
|
forceTranscode: true,
|
||||||
|
},
|
||||||
|
trim: {
|
||||||
|
start: 0,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await conversion.execute();
|
||||||
|
|
||||||
|
using newInput = new Input({
|
||||||
|
source: new BufferSource(output.target.buffer!),
|
||||||
|
formats: ALL_FORMATS,
|
||||||
|
});
|
||||||
|
|
||||||
|
const newInputTrack = await newInput.getPrimaryAudioTrack();
|
||||||
|
assert(newInputTrack);
|
||||||
|
|
||||||
|
expect(await newInputTrack.getCodec()).toBe(codec);
|
||||||
|
expect(await newInputTrack.computeDuration()).toBeCloseTo(await inputTrack.computeDuration(), 0);
|
||||||
|
|
||||||
|
const sink = new AudioSampleSink(newInputTrack);
|
||||||
|
|
||||||
|
let sampleCount = 0;
|
||||||
|
for await (using sample of sink.samples()) {
|
||||||
|
expect([await inputTrack.getNumberOfChannels(), 2].includes(sample.numberOfChannels)).toBe(true);
|
||||||
|
expect(sample.sampleRate).toBe(await inputTrack.getSampleRate());
|
||||||
|
|
||||||
|
sampleCount++;
|
||||||
|
}
|
||||||
|
|
||||||
|
expect(sampleCount).toBeGreaterThan(0);
|
||||||
|
};
|
||||||
|
|
||||||
|
test('Custom AudioSample resource', async () => {
|
||||||
|
using input = new Input({
|
||||||
|
source: new FilePathSource('./test/public/trim-buck-bunny-ffmpeg.ts'),
|
||||||
|
formats: ALL_FORMATS,
|
||||||
|
});
|
||||||
|
|
||||||
|
const audioTrack = await input.getPrimaryAudioTrack();
|
||||||
|
assert(audioTrack);
|
||||||
|
|
||||||
|
const sink = new AudioSampleSink(audioTrack);
|
||||||
|
using sample = await sink.getSample(await audioTrack.getFirstTimestamp());
|
||||||
|
assert(sample);
|
||||||
|
|
||||||
|
expect(sample._data).toBeInstanceOf(NodeAvFrameAudioSampleResource);
|
||||||
|
expect(sample.format).toBe('f32-planar');
|
||||||
|
expect(sample.numberOfChannels).toBe(6);
|
||||||
|
expect(sample.sampleRate).toBe(48000);
|
||||||
|
expect(sample.timestamp).toBe(await audioTrack.getFirstTimestamp());
|
||||||
|
expect(sample.duration).toBe(1024 / 48000);
|
||||||
|
expect(sample.numberOfFrames).toBe(1024);
|
||||||
|
|
||||||
|
const size = sample.allocationSize({ planeIndex: 0 });
|
||||||
|
expect(size).toBe(1024 * Float32Array.BYTES_PER_ELEMENT);
|
||||||
|
|
||||||
|
// Test a bunch of copyTo()s:
|
||||||
|
const buf = new ArrayBuffer(1e6);
|
||||||
|
sample.copyTo(buf, { planeIndex: 0 });
|
||||||
|
sample.copyTo(buf, { planeIndex: 1 });
|
||||||
|
sample.copyTo(buf, { planeIndex: 2 });
|
||||||
|
sample.copyTo(buf, { planeIndex: 3 });
|
||||||
|
sample.copyTo(buf, { planeIndex: 4 });
|
||||||
|
sample.copyTo(buf, { planeIndex: 5 });
|
||||||
|
sample.copyTo(buf, { format: 'f32', planeIndex: 0 });
|
||||||
|
});
|
||||||
});
|
});
|
||||||
*/
|
|
||||||
|
|||||||
Reference in New Issue
Block a user