arcodange/mediabunny
1
0
Fork 0
mirror of https://github.com/arcodange-org/mediabunny.git synced 2026-10-09 16:53:47 +02:00
Code Issues Packages Projects Releases Wiki Activity

Remove NodeAv FinaliziationRegistry logic (#392)

Browse Source
This commit is contained in:
Vanilagy
2026-06-02 10:24:33 +02:00
parent 96d0d9fb22
commit e0ced9901e
4 changed files with 212 additions and 293 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/server/src/audio-decoder.ts
+26 -46
View File
@@ -12,28 +12,10 @@ import { CODEC_TO_CODEC_ID, getChannelLayout } from './misc';
import { assert, toUint8Array } from '../../../src/misc'; import { assert, toUint8Array } from '../../../src/misc';
import { AvFrameAudioSampleResource } from './audio-sample'; import { AvFrameAudioSampleResource } from './audio-sample';
type NodeAvState = {
frame: NodeAv.Frame;
packet: NodeAv.Packet;
codecContext: NodeAv.CodecContext | null;
};
const freeState = (state: NodeAvState) => {
state.codecContext?.freeContext();
state.frame.free();
state.packet.free();
};
// Needed for proper freeing if close isn't called
let finalizationRegistry: FinalizationRegistry<NodeAvState> | null = null;
if (typeof FinalizationRegistry !== 'undefined') {
finalizationRegistry = new FinalizationRegistry<NodeAvState>((state) => {
freeState(state);
});
}
export class NodeAvAudioDecoder extends CustomAudioDecoder { export class NodeAvAudioDecoder extends CustomAudioDecoder {
state!: NodeAvState; frame!: NodeAv.Frame;
packet!: NodeAv.Packet;
codecContext: NodeAv.CodecContext | null = null;
// eslint-disable-next-line @typescript-eslint/no-unused-vars // eslint-disable-next-line @typescript-eslint/no-unused-vars
static override supports(codec: AudioCodec, config: AudioDecoderConfig): boolean { static override supports(codec: AudioCodec, config: AudioDecoderConfig): boolean {
@@ -47,13 +29,10 @@ export class NodeAvAudioDecoder extends CustomAudioDecoder {
} }
async init(): Promise<void> { async init(): Promise<void> {
const frame = new NodeAv.Frame(); this.frame = new NodeAv.Frame();
frame.alloc(); this.frame.alloc();
const packet = new NodeAv.Packet(); this.packet = new NodeAv.Packet();
packet.alloc(); this.packet.alloc();
this.state = { frame, packet, codecContext: null };
finalizationRegistry?.register(this, this.state, this);
const codecId = CODEC_TO_CODEC_ID[this.codec]; const codecId = CODEC_TO_CODEC_ID[this.codec];
assert(codecId !== undefined); assert(codecId !== undefined);
@@ -78,26 +57,26 @@ export class NodeAvAudioDecoder extends CustomAudioDecoder {
const ret = await codecContext.open2(); const ret = await codecContext.open2();
NodeAv.FFmpegError.throwIfError(ret, 'Open codec context'); NodeAv.FFmpegError.throwIfError(ret, 'Open codec context');
this.state.codecContext = codecContext; this.codecContext = codecContext;
} }
async decode(packet: EncodedPacket): Promise<void> { async decode(packet: EncodedPacket): Promise<void> {
assert(this.state.codecContext); assert(this.codecContext);
this.state.packet.isKeyframe = packet.type === 'key'; this.packet.isKeyframe = packet.type === 'key';
this.state.packet.data = Buffer.from(packet.data); this.packet.data = Buffer.from(packet.data);
this.state.packet.timeBase = { num: 1, den: this.config.sampleRate }; this.packet.timeBase = { num: 1, den: this.config.sampleRate };
this.state.packet.pts = BigInt(Math.round(packet.timestamp * this.config.sampleRate)); this.packet.pts = BigInt(Math.round(packet.timestamp * this.config.sampleRate));
this.state.packet.dts = NodeAv.AV_NOPTS_VALUE; this.packet.dts = NodeAv.AV_NOPTS_VALUE;
this.state.packet.duration = BigInt(Math.round(packet.duration * this.config.sampleRate)); this.packet.duration = BigInt(Math.round(packet.duration * this.config.sampleRate));
const ret = await this.state.codecContext.sendPacket(this.state.packet); const ret = await this.codecContext.sendPacket(this.packet);
NodeAv.FFmpegError.throwIfError(ret, 'Send packet'); NodeAv.FFmpegError.throwIfError(ret, 'Send packet');
this.state.packet.unref(); // Don't need the data again, so just unref it this.packet.unref(); // Don't need the data again, so just unref it
while (true) { while (true) {
const receiveRet = await this.state.codecContext.receiveFrame(this.state.frame); const receiveRet = await this.codecContext.receiveFrame(this.frame);
if (receiveRet === NodeAv.AVERROR_EAGAIN || receiveRet === NodeAv.AVERROR_EOF) { if (receiveRet === NodeAv.AVERROR_EAGAIN || receiveRet === NodeAv.AVERROR_EOF) {
break; break;
} }
@@ -109,7 +88,7 @@ export class NodeAvAudioDecoder extends CustomAudioDecoder {
receiveFrame(ret: number) { receiveFrame(ret: number) {
NodeAv.FFmpegError.throwIfError(ret, 'Receive frame'); NodeAv.FFmpegError.throwIfError(ret, 'Receive frame');
const clone = this.state.frame.clone(); const clone = this.frame.clone();
if (!clone) { if (!clone) {
throw new Error('Allocation failure during frame clone.'); throw new Error('Allocation failure during frame clone.');
} }
@@ -119,15 +98,15 @@ export class NodeAvAudioDecoder extends CustomAudioDecoder {
} }
async flush(): Promise<void> { async flush(): Promise<void> {
assert(this.state.codecContext); assert(this.codecContext);
// Send null packet to signal flush // Send null packet to signal flush
const ret = await this.state.codecContext.sendPacket(null); const ret = await this.codecContext.sendPacket(null);
NodeAv.FFmpegError.throwIfError(ret, 'Flush decoder'); NodeAv.FFmpegError.throwIfError(ret, 'Flush decoder');
// Keep receiving frames until no more are available // Keep receiving frames until no more are available
while (true) { while (true) {
const receiveRet = await this.state.codecContext.receiveFrame(this.state.frame); const receiveRet = await this.codecContext.receiveFrame(this.frame);
if (receiveRet === NodeAv.AVERROR_EAGAIN || receiveRet === NodeAv.AVERROR_EOF) { if (receiveRet === NodeAv.AVERROR_EAGAIN || receiveRet === NodeAv.AVERROR_EOF) {
// No more frames available // No more frames available
break; break;
@@ -136,11 +115,12 @@ export class NodeAvAudioDecoder extends CustomAudioDecoder {
this.receiveFrame(receiveRet); this.receiveFrame(receiveRet);
} }
this.state.codecContext.flushBuffers(); this.codecContext.flushBuffers();
} }
close(): MaybePromise<void> { close(): MaybePromise<void> {
finalizationRegistry?.unregister(this); this.codecContext?.freeContext();
freeState(this.state); this.frame.free();
this.packet.free();
} }
} }
packages/server/src/audio-encoder.ts
+82 -102
View File
@@ -32,32 +32,12 @@ const AC3_SAMPLE_RATES = [32000, 44100, 48000];
const FRAME_SIZE_FALLBACK = 1024; // Just 'cause const FRAME_SIZE_FALLBACK = 1024; // Just 'cause
type NodeAvState = {
frame: NodeAv.Frame;
packet: NodeAv.Packet;
codecContext: NodeAv.CodecContext | null;
resampler: NodeAv.SoftwareResampleContext | null;
dstFrame: NodeAv.Frame | null;
};
const freeState = (state: NodeAvState) => {
state.codecContext?.freeContext();
state.frame.free();
state.packet.free();
state.dstFrame?.free();
state.resampler?.free();
};
// Needed for proper freeing if close isn't called
let finalizationRegistry: FinalizationRegistry<NodeAvState> | null = null;
if (typeof FinalizationRegistry !== 'undefined') {
finalizationRegistry = new FinalizationRegistry<NodeAvState>((state) => {
freeState(state);
});
}
export class NodeAvAudioEncoder extends CustomAudioEncoder { export class NodeAvAudioEncoder extends CustomAudioEncoder {
state!: NodeAvState; frame!: NodeAv.Frame;
packet!: NodeAv.Packet;
codecContext: NodeAv.CodecContext | null = null;
resampler: NodeAv.SoftwareResampleContext | null = null;
dstFrame: NodeAv.Frame | null = null;
avCodec!: NodeAv.Codec; avCodec!: NodeAv.Codec;
firstExpectedTimestamp: number | null = null; firstExpectedTimestamp: number | null = null;
outputTimestampOffset = 0; outputTimestampOffset = 0;
@@ -96,13 +76,10 @@ export class NodeAvAudioEncoder extends CustomAudioEncoder {
} }
async init(): Promise<void> { async init(): Promise<void> {
const frame = new NodeAv.Frame(); this.frame = new NodeAv.Frame();
frame.alloc(); this.frame.alloc();
const packet = new NodeAv.Packet(); this.packet = new NodeAv.Packet();
packet.alloc(); this.packet.alloc();
this.state = { frame, packet, codecContext: null, resampler: null, dstFrame: null };
finalizationRegistry?.register(this, this.state, this);
const codecId = CODEC_TO_CODEC_ID[this.codec]; const codecId = CODEC_TO_CODEC_ID[this.codec];
assert(codecId !== undefined); assert(codecId !== undefined);
@@ -118,7 +95,7 @@ export class NodeAvAudioEncoder extends CustomAudioEncoder {
} }
async createCodecContext() { async createCodecContext() {
assert(this.state.codecContext === null); assert(this.codecContext === null);
const codecContext = new NodeAv.CodecContext(); const codecContext = new NodeAv.CodecContext();
codecContext.allocContext3(this.avCodec); codecContext.allocContext3(this.avCodec);
@@ -145,13 +122,13 @@ export class NodeAvAudioEncoder extends CustomAudioEncoder {
const ret = await codecContext.open2(); const ret = await codecContext.open2();
NodeAv.FFmpegError.throwIfError(ret, 'Open codec context'); NodeAv.FFmpegError.throwIfError(ret, 'Open codec context');
this.state.codecContext = codecContext; this.codecContext = codecContext;
} }
async encode(audioSample: AudioSample): Promise<void> { async encode(audioSample: AudioSample): Promise<void> {
if (this.state.codecContext === null) { if (this.codecContext === null) {
await this.createCodecContext(); await this.createCodecContext();
assert(this.state.codecContext); assert(this.codecContext);
} }
this.firstExpectedTimestamp ??= audioSample.timestamp; this.firstExpectedTimestamp ??= audioSample.timestamp;
@@ -160,17 +137,17 @@ export class NodeAvAudioEncoder extends CustomAudioEncoder {
// Release any buffers still referenced from the previous encode before reffing the new frame, otherwise // Release any buffers still referenced from the previous encode before reffing the new frame, otherwise
// av_frame_ref leaks them // av_frame_ref leaks them
// https://github.com/Vanilagy/mediabunny/issues/392 // https://github.com/Vanilagy/mediabunny/issues/392
this.state.frame.unref(); this.frame.unref();
this.state.frame.ref(audioSample._data.frame); this.frame.ref(audioSample._data.frame);
} else { } else {
copyAudioSampleToAvFrame(audioSample, this.state.frame); copyAudioSampleToAvFrame(audioSample, this.frame);
} }
this.state.frame.pts = BigInt(Math.round(audioSample.timestamp * this.config.sampleRate)); this.frame.pts = BigInt(Math.round(audioSample.timestamp * this.config.sampleRate));
this.state.frame.duration = BigInt(Math.round(audioSample.duration * this.config.sampleRate)); this.frame.duration = BigInt(Math.round(audioSample.duration * this.config.sampleRate));
this.state.frame.timeBase = new NodeAv.Rational(1, this.config.sampleRate); this.frame.timeBase = new NodeAv.Rational(1, this.config.sampleRate);
const key = `${this.state.frame.sampleRate}:${this.state.frame.channels}:${this.state.frame.format}`; const key = `${this.frame.sampleRate}:${this.frame.channels}:${this.frame.format}`;
if (this.inputParametersKey !== null && this.inputParametersKey !== key) { if (this.inputParametersKey !== null && this.inputParametersKey !== key) {
throw new Error( throw new Error(
'Input audio parameters changed. For this audio encoder, you cannot change the input audio' 'Input audio parameters changed. For this audio encoder, you cannot change the input audio'
@@ -183,83 +160,83 @@ export class NodeAvAudioEncoder extends CustomAudioEncoder {
// 1. Format conversion is needed (sample format, sample rate, or channel count differs) // 1. Format conversion is needed (sample format, sample rate, or channel count differs)
// 2. The codec requires fixed frame sizes // 2. The codec requires fixed frame sizes
const requiresResampler const requiresResampler
= this.state.codecContext.frameSize > 0 = this.codecContext.frameSize > 0
|| this.state.codecContext.sampleFormat !== this.state.frame.format || this.codecContext.sampleFormat !== this.frame.format
|| this.state.codecContext.sampleRate !== this.state.frame.sampleRate || this.codecContext.sampleRate !== this.frame.sampleRate
|| this.state.codecContext.channels !== this.state.frame.channels; || this.codecContext.channels !== this.frame.channels;
if (requiresResampler) { if (requiresResampler) {
if (!this.state.resampler) { if (!this.resampler) {
this.state.resampler = new NodeAv.SoftwareResampleContext(); this.resampler = new NodeAv.SoftwareResampleContext();
this.resamplerInputSampleRate = this.state.frame.sampleRate; this.resamplerInputSampleRate = this.frame.sampleRate;
const outLayout = getChannelLayout(this.state.codecContext.channels); const outLayout = getChannelLayout(this.codecContext.channels);
const inLayout = getChannelLayout(this.state.frame.channels); const inLayout = getChannelLayout(this.frame.channels);
const ret = this.state.resampler.allocSetOpts2( const ret = this.resampler.allocSetOpts2(
outLayout, this.state.codecContext.sampleFormat, this.state.codecContext.sampleRate, outLayout, this.codecContext.sampleFormat, this.codecContext.sampleRate,
inLayout, this.state.frame.format as NodeAv.AVSampleFormat, this.state.frame.sampleRate, inLayout, this.frame.format as NodeAv.AVSampleFormat, this.frame.sampleRate,
); );
NodeAv.FFmpegError.throwIfError(ret, 'allocSetOpts2'); NodeAv.FFmpegError.throwIfError(ret, 'allocSetOpts2');
const ret2 = this.state.resampler.init(); const ret2 = this.resampler.init();
NodeAv.FFmpegError.throwIfError(ret2, 'init'); NodeAv.FFmpegError.throwIfError(ret2, 'init');
this.state.dstFrame = new NodeAv.Frame(); this.dstFrame = new NodeAv.Frame();
this.state.dstFrame.alloc(); this.dstFrame.alloc();
this.state.dstFrame.channelLayout = outLayout; this.dstFrame.channelLayout = outLayout;
this.state.dstFrame.sampleRate = this.state.codecContext.sampleRate; this.dstFrame.sampleRate = this.codecContext.sampleRate;
this.state.dstFrame.format = this.state.codecContext.sampleFormat; this.dstFrame.format = this.codecContext.sampleFormat;
this.state.dstFrame.nbSamples = this.state.codecContext.frameSize || FRAME_SIZE_FALLBACK; this.dstFrame.nbSamples = this.codecContext.frameSize || FRAME_SIZE_FALLBACK;
this.state.dstFrame.duration = BigInt(this.state.dstFrame.nbSamples); this.dstFrame.duration = BigInt(this.dstFrame.nbSamples);
this.state.dstFrame.allocBuffer(); this.dstFrame.allocBuffer();
this.nextResamplerPts = this.state.frame.pts; this.nextResamplerPts = this.frame.pts;
} }
const inputBuffers = this.state.frame.data; const inputBuffers = this.frame.data;
if (!inputBuffers) { if (!inputBuffers) {
throw new DOMException('Frame has no data', 'EncodingError'); throw new DOMException('Frame has no data', 'EncodingError');
} }
await this.state.resampler.convert(null, 0, inputBuffers, this.state.frame.nbSamples); await this.resampler.convert(null, 0, inputBuffers, this.frame.nbSamples);
await this.pullResampledFrames(); await this.pullResampledFrames();
} else { } else {
await this.sendFrameAndReceivePackets(this.state.frame); await this.sendFrameAndReceivePackets(this.frame);
} }
} }
async pullResampledFrames() { async pullResampledFrames() {
assert(this.state.codecContext); assert(this.codecContext);
assert(this.state.resampler); assert(this.resampler);
assert(this.state.dstFrame); assert(this.dstFrame);
assert(this.nextResamplerPts !== null); assert(this.nextResamplerPts !== null);
const frameSize = this.state.codecContext.frameSize || FRAME_SIZE_FALLBACK; const frameSize = this.codecContext.frameSize || FRAME_SIZE_FALLBACK;
while (true) { while (true) {
const available = this.state.resampler.getOutSamples(0); const available = this.resampler.getOutSamples(0);
if (available < frameSize) { if (available < frameSize) {
break; break;
} }
await this.state.resampler.convert(this.state.dstFrame.data, frameSize, null, 0); await this.resampler.convert(this.dstFrame.data, frameSize, null, 0);
this.state.dstFrame.pts = this.nextResamplerPts; this.dstFrame.pts = this.nextResamplerPts;
await this.sendFrameAndReceivePackets(this.state.dstFrame); await this.sendFrameAndReceivePackets(this.dstFrame);
this.nextResamplerPts += BigInt(frameSize); this.nextResamplerPts += BigInt(frameSize);
} }
} }
async sendFrameAndReceivePackets(frame: NodeAv.Frame | null) { async sendFrameAndReceivePackets(frame: NodeAv.Frame | null) {
assert(this.state.codecContext); assert(this.codecContext);
const ret = await this.state.codecContext.sendFrame(frame); const ret = await this.codecContext.sendFrame(frame);
NodeAv.FFmpegError.throwIfError(ret, 'Send frame'); NodeAv.FFmpegError.throwIfError(ret, 'Send frame');
while (true) { while (true) {
const receiveRet = await this.state.codecContext.receivePacket(this.state.packet); const receiveRet = await this.codecContext.receivePacket(this.packet);
if (receiveRet === NodeAv.AVERROR_EAGAIN || receiveRet === NodeAv.AVERROR_EOF) { if (receiveRet === NodeAv.AVERROR_EAGAIN || receiveRet === NodeAv.AVERROR_EOF) {
break; break;
} }
@@ -269,18 +246,18 @@ export class NodeAvAudioEncoder extends CustomAudioEncoder {
} }
receivePacket(ret: number) { receivePacket(ret: number) {
assert(this.state.codecContext); assert(this.codecContext);
assert(this.firstExpectedTimestamp !== null); assert(this.firstExpectedTimestamp !== null);
NodeAv.FFmpegError.throwIfError(ret, 'Receive packet'); NodeAv.FFmpegError.throwIfError(ret, 'Receive packet');
if (!this.state.packet.data) { if (!this.packet.data) {
return; return;
} }
let timestamp = Number(this.state.packet.pts) / this.state.codecContext.sampleRate; let timestamp = Number(this.packet.pts) / this.codecContext.sampleRate;
const duration = Number(this.state.packet.duration) / this.state.codecContext.sampleRate; const duration = Number(this.packet.duration) / this.codecContext.sampleRate;
let data: Uint8Array = this.state.packet.data; let data: Uint8Array = this.packet.data;
let metadata: EncodedAudioChunkMetadata | undefined; let metadata: EncodedAudioChunkMetadata | undefined;
if (this.packetEmitted) { if (this.packetEmitted) {
@@ -292,8 +269,8 @@ export class NodeAvAudioEncoder extends CustomAudioEncoder {
this.outputTimestampOffset = Math.max(this.firstExpectedTimestamp - timestamp, 0); this.outputTimestampOffset = Math.max(this.firstExpectedTimestamp - timestamp, 0);
const codecString = this.config.codec; const codecString = this.config.codec;
let description = this.state.codecContext.extraData let description = this.codecContext.extraData
? toUint8Array(this.state.codecContext.extraData) ? toUint8Array(this.codecContext.extraData)
: undefined; : undefined;
if (this.codec === 'aac') { if (this.codec === 'aac') {
@@ -338,8 +315,8 @@ export class NodeAvAudioEncoder extends CustomAudioEncoder {
metadata = { metadata = {
decoderConfig: { decoderConfig: {
codec: codecString, codec: codecString,
sampleRate: this.state.codecContext.sampleRate, sampleRate: this.codecContext.sampleRate,
numberOfChannels: this.state.codecContext.channels, numberOfChannels: this.codecContext.channels,
description, description,
}, },
}; };
@@ -371,50 +348,53 @@ export class NodeAvAudioEncoder extends CustomAudioEncoder {
} }
async flush(): Promise<void> { async flush(): Promise<void> {
if (!this.state.codecContext) { if (!this.codecContext) {
return; return;
} }
outer: outer:
if (this.state.resampler) { if (this.resampler) {
assert(this.resamplerInputSampleRate !== null); assert(this.resamplerInputSampleRate !== null);
const currentOutSamples = this.state.resampler.getOutSamples(0); const currentOutSamples = this.resampler.getOutSamples(0);
if (currentOutSamples === 0) { if (currentOutSamples === 0) {
break outer; // Clean cut-off point break outer; // Clean cut-off point
} }
const frameSize = this.state.codecContext.frameSize || FRAME_SIZE_FALLBACK; const frameSize = this.codecContext.frameSize || FRAME_SIZE_FALLBACK;
assert(currentOutSamples < frameSize); // Because if it's more, it would've already been retrieved assert(currentOutSamples < frameSize); // Because if it's more, it would've already been retrieved
const inputSamplesNeeded = Math.ceil( const inputSamplesNeeded = Math.ceil(
((frameSize - currentOutSamples) / this.state.codecContext.sampleRate) * this.resamplerInputSampleRate, ((frameSize - currentOutSamples) / this.codecContext.sampleRate) * this.resamplerInputSampleRate,
); );
this.state.resampler.injectSilence(inputSamplesNeeded); this.resampler.injectSilence(inputSamplesNeeded);
await this.pullResampledFrames(); await this.pullResampledFrames();
} }
await this.sendFrameAndReceivePackets(null); await this.sendFrameAndReceivePackets(null);
this.state.codecContext.freeContext(); this.codecContext.freeContext();
this.state.codecContext = null; this.codecContext = null;
this.packetEmitted = false; this.packetEmitted = false;
this.firstExpectedTimestamp = null; this.firstExpectedTimestamp = null;
this.outputTimestampOffset = 0; this.outputTimestampOffset = 0;
this.adtsHeaderTemplate = null; this.adtsHeaderTemplate = null;
this.state.resampler?.free(); this.resampler?.free();
this.state.resampler = null; this.resampler = null;
this.inputParametersKey = null; this.inputParametersKey = null;
this.resamplerInputSampleRate = null; this.resamplerInputSampleRate = null;
this.nextResamplerPts = null; this.nextResamplerPts = null;
this.state.dstFrame?.free(); this.dstFrame?.free();
this.state.dstFrame = null; this.dstFrame = null;
} }
close(): MaybePromise<void> { close(): MaybePromise<void> {
finalizationRegistry?.unregister(this); this.codecContext?.freeContext();
freeState(this.state); this.frame.free();
this.packet.free();
this.dstFrame?.free();
this.resampler?.free();
} }
} }
packages/server/src/video-decoder.ts
+34 -54
View File
@@ -12,28 +12,10 @@ import { CODEC_TO_CODEC_ID, getHardwareDecoderCodec } from './misc';
import { assert, binarySearchLessOrEqual, simplifyRational, toUint8Array } from '../../../src/misc'; import { assert, binarySearchLessOrEqual, simplifyRational, toUint8Array } from '../../../src/misc';
import { AvFrameVideoSampleResource } from './video-sample'; import { AvFrameVideoSampleResource } from './video-sample';
type NodeAvState = {
frame: NodeAv.Frame;
packet: NodeAv.Packet;
codecContext: NodeAv.CodecContext | null;
};
const freeState = (state: NodeAvState) => {
state.codecContext?.freeContext();
state.frame.free();
state.packet.free();
};
// Needed for proper freeing if close isn't called
let finalizationRegistry: FinalizationRegistry<NodeAvState> | null = null;
if (typeof FinalizationRegistry !== 'undefined') {
finalizationRegistry = new FinalizationRegistry<NodeAvState>((state) => {
freeState(state);
});
}
export class NodeAvVideoDecoder extends CustomVideoDecoder { export class NodeAvVideoDecoder extends CustomVideoDecoder {
state!: NodeAvState; frame!: NodeAv.Frame;
packet!: NodeAv.Packet;
codecContext: NodeAv.CodecContext | null = null;
pixelAspectRatio!: Rational; pixelAspectRatio!: Rational;
// Bookkeeping to restore the original timing information // Bookkeeping to restore the original timing information
@@ -51,17 +33,14 @@ export class NodeAvVideoDecoder extends CustomVideoDecoder {
} }
async init() { async init() {
const frame = new NodeAv.Frame(); this.frame = new NodeAv.Frame();
frame.alloc(); this.frame.alloc();
const packet = new NodeAv.Packet(); this.packet = new NodeAv.Packet();
packet.alloc(); this.packet.alloc();
this.state = { frame, packet, codecContext: null };
finalizationRegistry?.register(this, this.state, this);
} }
async initCodecContext(packet: EncodedPacket) { async initCodecContext(packet: EncodedPacket) {
assert(this.state.codecContext === null); assert(this.codecContext === null);
const codecId = CODEC_TO_CODEC_ID[this.codec]; const codecId = CODEC_TO_CODEC_ID[this.codec];
assert(codecId !== undefined); assert(codecId !== undefined);
@@ -106,29 +85,29 @@ export class NodeAvVideoDecoder extends CustomVideoDecoder {
const ret = await codecContext.open2(); const ret = await codecContext.open2();
NodeAv.FFmpegError.throwIfError(ret, 'Open codec context'); NodeAv.FFmpegError.throwIfError(ret, 'Open codec context');
this.state.codecContext = codecContext; this.codecContext = codecContext;
} }
async decode(packet: EncodedPacket) { async decode(packet: EncodedPacket) {
if (this.state.codecContext === null) { if (this.codecContext === null) {
await this.initCodecContext(packet); await this.initCodecContext(packet);
} }
assert(this.state.codecContext); assert(this.codecContext);
this.state.packet.isKeyframe = packet.type === 'key'; this.packet.isKeyframe = packet.type === 'key';
this.state.packet.data = Buffer.from(packet.data); this.packet.data = Buffer.from(packet.data);
this.state.packet.timeBase = { num: 1, den: 1e6 }; this.packet.timeBase = { num: 1, den: 1e6 };
this.state.packet.pts = BigInt(packet.microsecondTimestamp); this.packet.pts = BigInt(packet.microsecondTimestamp);
this.state.packet.dts = NodeAv.AV_NOPTS_VALUE; this.packet.dts = NodeAv.AV_NOPTS_VALUE;
this.state.packet.duration = BigInt(packet.microsecondDuration); this.packet.duration = BigInt(packet.microsecondDuration);
if (packet.sideData.alpha) { if (packet.sideData.alpha) {
const matroskaBlockAdditional = Buffer.alloc(8 + packet.sideData.alpha.byteLength); const matroskaBlockAdditional = Buffer.alloc(8 + packet.sideData.alpha.byteLength);
matroskaBlockAdditional[7] = 1; // BlockAddId matroskaBlockAdditional[7] = 1; // BlockAddId
matroskaBlockAdditional.set(packet.sideData.alpha, 8); matroskaBlockAdditional.set(packet.sideData.alpha, 8);
this.state.packet.addSideData(NodeAv.AV_PKT_DATA_MATROSKA_BLOCKADDITIONAL, matroskaBlockAdditional); this.packet.addSideData(NodeAv.AV_PKT_DATA_MATROSKA_BLOCKADDITIONAL, matroskaBlockAdditional);
} }
const preciseTimingIndex = binarySearchLessOrEqual( const preciseTimingIndex = binarySearchLessOrEqual(
@@ -163,13 +142,13 @@ export class NodeAvVideoDecoder extends CustomVideoDecoder {
} }
} }
const ret = await this.state.codecContext.sendPacket(this.state.packet); const ret = await this.codecContext.sendPacket(this.packet);
NodeAv.FFmpegError.throwIfError(ret, 'Send packet'); NodeAv.FFmpegError.throwIfError(ret, 'Send packet');
this.state.packet.unref(); // Don't need the data again, so just unref it this.packet.unref(); // Don't need the data again, so just unref it
while (true) { while (true) {
const receiveRet = await this.state.codecContext.receiveFrame(this.state.frame); const receiveRet = await this.codecContext.receiveFrame(this.frame);
if (receiveRet === NodeAv.AVERROR_EAGAIN || receiveRet === NodeAv.AVERROR_EOF) { if (receiveRet === NodeAv.AVERROR_EAGAIN || receiveRet === NodeAv.AVERROR_EOF) {
break; break;
} }
@@ -181,14 +160,14 @@ export class NodeAvVideoDecoder extends CustomVideoDecoder {
receiveFrame(ret: number) { receiveFrame(ret: number) {
NodeAv.FFmpegError.throwIfError(ret, 'Receive frame'); NodeAv.FFmpegError.throwIfError(ret, 'Receive frame');
this.state.frame.sampleAspectRatio = new NodeAv.Rational(this.pixelAspectRatio.num, this.pixelAspectRatio.den); this.frame.sampleAspectRatio = new NodeAv.Rational(this.pixelAspectRatio.num, this.pixelAspectRatio.den);
let timestamp = Number(this.state.frame.pts) / 1e6; let timestamp = Number(this.frame.pts) / 1e6;
let duration = Number(this.state.frame.duration) / 1e6; let duration = Number(this.frame.duration) / 1e6;
const preciseTimingIndex = binarySearchLessOrEqual( const preciseTimingIndex = binarySearchLessOrEqual(
this.preciseTimings, this.preciseTimings,
Number(this.state.frame.pts), Number(this.frame.pts),
x => x.microsecondTimestamp, x => x.microsecondTimestamp,
); );
const entry = preciseTimingIndex !== -1 const entry = preciseTimingIndex !== -1
@@ -197,7 +176,7 @@ export class NodeAvVideoDecoder extends CustomVideoDecoder {
// If there's a relevant timing entry, refine the frame's timing data to get better accuracy than // If there's a relevant timing entry, refine the frame's timing data to get better accuracy than
// microseconds // microseconds
if (entry && entry.microsecondTimestamp === Number(this.state.frame.pts)) { if (entry && entry.microsecondTimestamp === Number(this.frame.pts)) {
if (entry.timestampIsValid) { if (entry.timestampIsValid) {
timestamp = entry.timestamp; timestamp = entry.timestamp;
} }
@@ -206,7 +185,7 @@ export class NodeAvVideoDecoder extends CustomVideoDecoder {
} }
} }
const clone = this.state.frame.clone(); const clone = this.frame.clone();
if (!clone) { if (!clone) {
throw new Error('Frame clone allocation failed.'); throw new Error('Frame clone allocation failed.');
} }
@@ -218,17 +197,17 @@ export class NodeAvVideoDecoder extends CustomVideoDecoder {
} }
async flush() { async flush() {
if (!this.state.codecContext) { if (!this.codecContext) {
return; return;
} }
// Send null packet to signal flush // Send null packet to signal flush
const ret = await this.state.codecContext.sendPacket(null); const ret = await this.codecContext.sendPacket(null);
NodeAv.FFmpegError.throwIfError(ret, 'Flush decoder'); NodeAv.FFmpegError.throwIfError(ret, 'Flush decoder');
// Keep receiving frames until no more are available // Keep receiving frames until no more are available
while (true) { while (true) {
const receiveRet = await this.state.codecContext.receiveFrame(this.state.frame); const receiveRet = await this.codecContext.receiveFrame(this.frame);
if (receiveRet === NodeAv.AVERROR_EAGAIN || receiveRet === NodeAv.AVERROR_EOF) { if (receiveRet === NodeAv.AVERROR_EAGAIN || receiveRet === NodeAv.AVERROR_EOF) {
// No more frames available // No more frames available
break; break;
@@ -237,11 +216,12 @@ export class NodeAvVideoDecoder extends CustomVideoDecoder {
this.receiveFrame(receiveRet); this.receiveFrame(receiveRet);
} }
this.state.codecContext.flushBuffers(); this.codecContext.flushBuffers();
} }
close(): MaybePromise<void> { close(): MaybePromise<void> {
finalizationRegistry?.unregister(this); this.codecContext?.freeContext();
freeState(this.state); this.frame.free();
this.packet.free();
} }
} }
packages/server/src/video-encoder.ts
+70 -91
View File
@@ -41,32 +41,12 @@ import {
import { extractVideoCodecString } from '../../../src/codec'; import { extractVideoCodecString } from '../../../src/codec';
import { assert, binarySearchLessOrEqual, simplifyRational, toUint8Array } from '../../../src/misc'; import { assert, binarySearchLessOrEqual, simplifyRational, toUint8Array } from '../../../src/misc';
type NodeAvState = {
frame: NodeAv.Frame;
packet: NodeAv.Packet;
codecContext: NodeAv.CodecContext | null;
scaler: NodeAv.SoftwareScaleContext | null;
dstFrame: NodeAv.Frame | null;
};
const freeState = (state: NodeAvState) => {
state.codecContext?.freeContext();
state.frame.free();
state.packet.free();
state.scaler?.freeContext();
state.dstFrame?.free();
};
// Needed for proper freeing if close isn't called
let finalizationRegistry: FinalizationRegistry<NodeAvState> | null = null;
if (typeof FinalizationRegistry !== 'undefined') {
finalizationRegistry = new FinalizationRegistry<NodeAvState>((state) => {
freeState(state);
});
}
export class NodeAvVideoEncoder extends CustomVideoEncoder { export class NodeAvVideoEncoder extends CustomVideoEncoder {
state!: NodeAvState; frame!: NodeAv.Frame;
packet!: NodeAv.Packet;
codecContext: NodeAv.CodecContext | null = null;
scaler: NodeAv.SoftwareScaleContext | null = null;
dstFrame: NodeAv.Frame | null = null;
avCodec!: NodeAv.Codec; avCodec!: NodeAv.Codec;
lastBuffer: Buffer | null = null; lastBuffer: Buffer | null = null;
packetEmitted = false; packetEmitted = false;
@@ -87,16 +67,12 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
} }
async init(): Promise<void> { async init(): Promise<void> {
const frame = new NodeAv.Frame(); this.frame = new NodeAv.Frame();
frame.alloc(); this.frame.alloc();
frame.timeBase = new NodeAv.Rational(1, 1e6); this.frame.timeBase = new NodeAv.Rational(1, 1e6);
const packet = new NodeAv.Packet(); this.packet = new NodeAv.Packet();
packet.alloc(); this.packet.alloc();
this.state = { frame, packet, codecContext: null, scaler: null, dstFrame: null };
finalizationRegistry?.register(this, this.state, this);
const codecId = CODEC_TO_CODEC_ID[this.codec]; const codecId = CODEC_TO_CODEC_ID[this.codec];
assert(codecId !== undefined); assert(codecId !== undefined);
@@ -120,7 +96,7 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
} }
async createCodecContext() { async createCodecContext() {
assert(this.state.codecContext === null); assert(this.codecContext === null);
const codecContext = new NodeAv.CodecContext(); const codecContext = new NodeAv.CodecContext();
codecContext.allocContext3(this.avCodec); codecContext.allocContext3(this.avCodec);
@@ -200,69 +176,69 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
const ret = await codecContext.open2(); const ret = await codecContext.open2();
NodeAv.FFmpegError.throwIfError(ret, 'Open codec context'); NodeAv.FFmpegError.throwIfError(ret, 'Open codec context');
this.state.codecContext = codecContext; this.codecContext = codecContext;
} }
async encode(videoSample: VideoSample, options: VideoEncoderEncodeOptions): Promise<void> { async encode(videoSample: VideoSample, options: VideoEncoderEncodeOptions): Promise<void> {
if (this.state.codecContext === null) { if (this.codecContext === null) {
await this.createCodecContext(); await this.createCodecContext();
} }
assert(this.state.codecContext); assert(this.codecContext);
if (videoSample._data instanceof AvFrameVideoSampleResource) { if (videoSample._data instanceof AvFrameVideoSampleResource) {
// Release any buffers still referenced from the previous encode before reffing the new frame, otherwise // Release any buffers still referenced from the previous encode before reffing the new frame, otherwise
// av_frame_ref leaks them // av_frame_ref leaks them
// https://github.com/Vanilagy/mediabunny/issues/392 // https://github.com/Vanilagy/mediabunny/issues/392
this.state.frame.unref(); this.frame.unref();
this.state.frame.ref(videoSample._data.frame); this.frame.ref(videoSample._data.frame);
} else { } else {
if (videoSample.format === null) { if (videoSample.format === null) {
throw new Error('Cannot encode foreign VideoSample with unknown (null) format.'); throw new Error('Cannot encode foreign VideoSample with unknown (null) format.');
} }
this.lastBuffer = await copyVideoSampleToAvFrame(videoSample, this.state.frame, this.lastBuffer); this.lastBuffer = await copyVideoSampleToAvFrame(videoSample, this.frame, this.lastBuffer);
} }
let frameToEncode = this.state.frame; let frameToEncode = this.frame;
const requiresScaler const requiresScaler
= this.state.codecContext.pixelFormat !== this.state.frame.format = this.codecContext.pixelFormat !== this.frame.format
|| this.state.codecContext.width !== this.state.frame.width || this.codecContext.width !== this.frame.width
|| this.state.codecContext.height !== this.state.frame.height; || this.codecContext.height !== this.frame.height;
if (requiresScaler) { if (requiresScaler) {
if (!this.state.scaler) { if (!this.scaler) {
this.state.scaler = new NodeAv.SoftwareScaleContext(); this.scaler = new NodeAv.SoftwareScaleContext();
} }
const key = `${this.state.frame.width}x${this.state.frame.height}:${this.state.frame.format}`; const key = `${this.frame.width}x${this.frame.height}:${this.frame.format}`;
const needsConfigure = key !== this.lastScalerKey; const needsConfigure = key !== this.lastScalerKey;
if (needsConfigure) { if (needsConfigure) {
this.state.scaler.getContext( this.scaler.getContext(
this.state.frame.width, this.state.frame.height, this.state.frame.format as NodeAv.AVPixelFormat, this.frame.width, this.frame.height, this.frame.format as NodeAv.AVPixelFormat,
this.state.codecContext.width, this.state.codecContext.height, this.state.codecContext.pixelFormat, this.codecContext.width, this.codecContext.height, this.codecContext.pixelFormat,
NodeAv.SWS_FAST_BILINEAR, NodeAv.SWS_FAST_BILINEAR,
); );
this.lastScalerKey = key; this.lastScalerKey = key;
const ret = this.state.scaler.initContext(); const ret = this.scaler.initContext();
NodeAv.FFmpegError.throwIfError(ret, 'initContext'); NodeAv.FFmpegError.throwIfError(ret, 'initContext');
} }
if (!this.state.dstFrame) { if (!this.dstFrame) {
this.state.dstFrame = new NodeAv.Frame(); this.dstFrame = new NodeAv.Frame();
this.state.dstFrame.alloc(); this.dstFrame.alloc();
this.state.dstFrame.width = this.state.codecContext.width; this.dstFrame.width = this.codecContext.width;
this.state.dstFrame.height = this.state.codecContext.height; this.dstFrame.height = this.codecContext.height;
this.state.dstFrame.format = this.state.codecContext.pixelFormat; this.dstFrame.format = this.codecContext.pixelFormat;
this.state.dstFrame.allocBuffer(); this.dstFrame.allocBuffer();
} }
await this.state.scaler.scaleFrame(this.state.dstFrame, this.state.frame); await this.scaler.scaleFrame(this.dstFrame, this.frame);
this.state.dstFrame.copyProps(this.state.frame); this.dstFrame.copyProps(this.frame);
frameToEncode = this.state.dstFrame; frameToEncode = this.dstFrame;
} }
frameToEncode.pts = BigInt(videoSample.microsecondTimestamp); frameToEncode.pts = BigInt(videoSample.microsecondTimestamp);
@@ -309,12 +285,12 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
} }
} }
const ret = await this.state.codecContext.sendFrame(frameToEncode); const ret = await this.codecContext.sendFrame(frameToEncode);
NodeAv.FFmpegError.throwIfError(ret, 'Send frame'); NodeAv.FFmpegError.throwIfError(ret, 'Send frame');
// Keep receiving packets until no more are available for this frame // Keep receiving packets until no more are available for this frame
while (true) { while (true) {
const receiveRet = await this.state.codecContext.receivePacket(this.state.packet); const receiveRet = await this.codecContext.receivePacket(this.packet);
if (receiveRet === NodeAv.AVERROR_EAGAIN || receiveRet === NodeAv.AVERROR_EOF) { if (receiveRet === NodeAv.AVERROR_EAGAIN || receiveRet === NodeAv.AVERROR_EOF) {
break; break;
} }
@@ -324,20 +300,20 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
} }
receivePacket(ret: number) { receivePacket(ret: number) {
assert(this.state.codecContext); assert(this.codecContext);
NodeAv.FFmpegError.throwIfError(ret, 'Receive packet'); NodeAv.FFmpegError.throwIfError(ret, 'Receive packet');
if (!this.state.packet.data) { if (!this.packet.data) {
return; return;
} }
let packetData = toUint8Array(this.state.packet.data); let packetData = toUint8Array(this.packet.data);
let timestamp = Number(this.state.packet.pts) / 1e6; let timestamp = Number(this.packet.pts) / 1e6;
let duration = Number(this.state.packet.duration) / 1e6; let duration = Number(this.packet.duration) / 1e6;
const preciseTimingIndex = binarySearchLessOrEqual( const preciseTimingIndex = binarySearchLessOrEqual(
this.preciseTimings, this.preciseTimings,
Number(this.state.packet.pts), Number(this.packet.pts),
x => x.microsecondTimestamp, x => x.microsecondTimestamp,
); );
const entry = preciseTimingIndex !== -1 const entry = preciseTimingIndex !== -1
@@ -346,7 +322,7 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
// If there's a relevant timing entry, refine the packet's timing data to get better accuracy than // If there's a relevant timing entry, refine the packet's timing data to get better accuracy than
// microseconds // microseconds
if (entry && entry.microsecondTimestamp === Number(this.state.packet.pts)) { if (entry && entry.microsecondTimestamp === Number(this.packet.pts)) {
if (entry.timestampIsValid) { if (entry.timestampIsValid) {
timestamp = entry.timestamp; timestamp = entry.timestamp;
} }
@@ -373,14 +349,14 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
let serializedRecord: Uint8Array; let serializedRecord: Uint8Array;
if (this.codec === 'avc') { if (this.codec === 'avc') {
const record = extractAvcDecoderConfigurationRecord(this.state.packet.data); const record = extractAvcDecoderConfigurationRecord(this.packet.data);
if (!record) { if (!record) {
throw new Error('Invalid AVC data, could not extract decoder configuration record.'); throw new Error('Invalid AVC data, could not extract decoder configuration record.');
} }
serializedRecord = serializeAvcDecoderConfigurationRecord(record); serializedRecord = serializeAvcDecoderConfigurationRecord(record);
} else { } else {
const record = extractHevcDecoderConfigurationRecord(this.state.packet.data); const record = extractHevcDecoderConfigurationRecord(this.packet.data);
if (!record) { if (!record) {
throw new Error('Invalid HEVC data, could not extract decoder configuration record.'); throw new Error('Invalid HEVC data, could not extract decoder configuration record.');
} }
@@ -515,14 +491,14 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
} }
const sideData: EncodedPacketSideData = {}; const sideData: EncodedPacketSideData = {};
const matroskaBlockAdditional = this.state.packet.getSideData(NodeAv.AV_PKT_DATA_MATROSKA_BLOCKADDITIONAL); const matroskaBlockAdditional = this.packet.getSideData(NodeAv.AV_PKT_DATA_MATROSKA_BLOCKADDITIONAL);
if (matroskaBlockAdditional) { if (matroskaBlockAdditional) {
sideData.alpha = toUint8Array(matroskaBlockAdditional).subarray(8); // Skip the BlockAddId sideData.alpha = toUint8Array(matroskaBlockAdditional).subarray(8); // Skip the BlockAddId
} }
const packet = new EncodedPacket( const packet = new EncodedPacket(
packetData, packetData,
this.state.packet.isKeyframe ? 'key' : 'delta', this.packet.isKeyframe ? 'key' : 'delta',
timestamp, timestamp,
duration, duration,
undefined, undefined,
@@ -534,19 +510,19 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
// Create the decoder config // Create the decoder config
metadata.decoderConfig = { metadata.decoderConfig = {
codec: decoderConfigCodecString, codec: decoderConfigCodecString,
codedWidth: this.state.codecContext.width, codedWidth: this.codecContext.width,
codedHeight: this.state.codecContext.height, codedHeight: this.codecContext.height,
displayAspectWidth: this.config.displayWidth ?? this.state.codecContext.width, displayAspectWidth: this.config.displayWidth ?? this.codecContext.width,
displayAspectHeight: this.config.displayHeight ?? this.state.codecContext.height, displayAspectHeight: this.config.displayHeight ?? this.codecContext.height,
description: decoderConfigDescription ?? undefined, description: decoderConfigDescription ?? undefined,
colorSpace: { colorSpace: {
primaries: unmapColorPrimaries(this.state.codecContext.colorPrimaries) as VideoColorPrimaries, primaries: unmapColorPrimaries(this.codecContext.colorPrimaries) as VideoColorPrimaries,
matrix: unmapMatrixCoefficients(this.state.codecContext.colorSpace) as VideoMatrixCoefficients, matrix: unmapMatrixCoefficients(this.codecContext.colorSpace) as VideoMatrixCoefficients,
transfer: transfer:
unmapTransferCharacteristics(this.state.codecContext.colorTrc) as VideoTransferCharacteristics, unmapTransferCharacteristics(this.codecContext.colorTrc) as VideoTransferCharacteristics,
fullRange: this.state.codecContext.colorRange === NodeAv.AVCOL_RANGE_JPEG fullRange: this.codecContext.colorRange === NodeAv.AVCOL_RANGE_JPEG
? true ? true
: this.state.codecContext.colorRange === NodeAv.AVCOL_RANGE_MPEG : this.codecContext.colorRange === NodeAv.AVCOL_RANGE_MPEG
? false ? false
: undefined, : undefined,
}, },
@@ -558,14 +534,14 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
} }
async flush(): Promise<void> { async flush(): Promise<void> {
if (this.state.codecContext) { if (this.codecContext) {
// Send null frame to signal flush // Send null frame to signal flush
const ret = await this.state.codecContext.sendFrame(null); const ret = await this.codecContext.sendFrame(null);
NodeAv.FFmpegError.throwIfError(ret, 'Send frame'); NodeAv.FFmpegError.throwIfError(ret, 'Send frame');
// Keep receiving packets until no more are available // Keep receiving packets until no more are available
while (true) { while (true) {
const receiveRet = await this.state.codecContext.receivePacket(this.state.packet); const receiveRet = await this.codecContext.receivePacket(this.packet);
if (receiveRet === NodeAv.AVERROR_EAGAIN || receiveRet === NodeAv.AVERROR_EOF) { if (receiveRet === NodeAv.AVERROR_EAGAIN || receiveRet === NodeAv.AVERROR_EOF) {
break; break;
} }
@@ -573,8 +549,8 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
this.receivePacket(receiveRet); this.receivePacket(receiveRet);
} }
this.state.codecContext.freeContext(); this.codecContext.freeContext();
this.state.codecContext = null; this.codecContext = null;
// The codec is done now and can't be reused. Any subsequent encode call will first need to recreate a // The codec is done now and can't be reused. Any subsequent encode call will first need to recreate a
// codec context. // codec context.
} }
@@ -583,7 +559,10 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
} }
close(): MaybePromise<void> { close(): MaybePromise<void> {
finalizationRegistry?.unregister(this); this.codecContext?.freeContext();
freeState(this.state); this.frame.free();
this.packet.free();
this.scaler?.freeContext();
this.dstFrame?.free();
} }
} }