mirror of
https://github.com/arcodange-org/mediabunny.git
synced 2026-10-01 21:03:46 +02:00
Fix NodeAv memory leak (fixes #392)
This commit is contained in:
Vanilagy
@@ -41,17 +41,36 @@ import {
|
||||
import { extractVideoCodecString } from '../../../src/codec';
|
||||
import { assert, binarySearchLessOrEqual, simplifyRational, toUint8Array } from '../../../src/misc';
|
||||
|
||||
type NodeAvState = {
|
||||
frame: NodeAv.Frame;
|
||||
packet: NodeAv.Packet;
|
||||
codecContext: NodeAv.CodecContext | null;
|
||||
scaler: NodeAv.SoftwareScaleContext | null;
|
||||
dstFrame: NodeAv.Frame | null;
|
||||
};
|
||||
|
||||
const freeState = (state: NodeAvState) => {
|
||||
state.codecContext?.freeContext();
|
||||
state.frame.free();
|
||||
state.packet.free();
|
||||
state.scaler?.freeContext();
|
||||
state.dstFrame?.free();
|
||||
};
|
||||
|
||||
// Needed for proper freeing if close isn't called
|
||||
let finalizationRegistry: FinalizationRegistry<NodeAvState> | null = null;
|
||||
if (typeof FinalizationRegistry !== 'undefined') {
|
||||
finalizationRegistry = new FinalizationRegistry<NodeAvState>((state) => {
|
||||
freeState(state);
|
||||
});
|
||||
}
|
||||
|
||||
export class NodeAvVideoEncoder extends CustomVideoEncoder {
|
||||
frame!: NodeAv.Frame;
|
||||
packet!: NodeAv.Packet;
|
||||
state!: NodeAvState;
|
||||
avCodec!: NodeAv.Codec;
|
||||
codecContext: NodeAv.CodecContext | null = null;
|
||||
lastBuffer: Buffer | null = null;
|
||||
packetEmitted = false;
|
||||
|
||||
scaler: NodeAv.SoftwareScaleContext | null = null;
|
||||
lastScalerKey: string | null = null;
|
||||
dstFrame: NodeAv.Frame | null = null;
|
||||
|
||||
// Bookkeeping to restore the original timing information
|
||||
preciseTimings: {
|
||||
@@ -68,12 +87,16 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
|
||||
}
|
||||
|
||||
async init(): Promise<void> {
|
||||
this.frame = new NodeAv.Frame();
|
||||
this.frame.alloc();
|
||||
this.frame.timeBase = new NodeAv.Rational(1, 1e6);
|
||||
const frame = new NodeAv.Frame();
|
||||
frame.alloc();
|
||||
frame.timeBase = new NodeAv.Rational(1, 1e6);
|
||||
|
||||
this.packet = new NodeAv.Packet();
|
||||
this.packet.alloc();
|
||||
const packet = new NodeAv.Packet();
|
||||
packet.alloc();
|
||||
|
||||
this.state = { frame, packet, codecContext: null, scaler: null, dstFrame: null };
|
||||
|
||||
finalizationRegistry?.register(this, this.state, this);
|
||||
|
||||
const codecId = CODEC_TO_CODEC_ID[this.codec];
|
||||
assert(codecId !== undefined);
|
||||
@@ -97,7 +120,7 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
|
||||
}
|
||||
|
||||
async createCodecContext() {
|
||||
assert(this.codecContext === null);
|
||||
assert(this.state.codecContext === null);
|
||||
|
||||
const codecContext = new NodeAv.CodecContext();
|
||||
codecContext.allocContext3(this.avCodec);
|
||||
@@ -177,65 +200,69 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
|
||||
const ret = await codecContext.open2();
|
||||
NodeAv.FFmpegError.throwIfError(ret, 'Open codec context');
|
||||
|
||||
this.codecContext = codecContext;
|
||||
this.state.codecContext = codecContext;
|
||||
}
|
||||
|
||||
async encode(videoSample: VideoSample, options: VideoEncoderEncodeOptions): Promise<void> {
|
||||
if (this.codecContext === null) {
|
||||
if (this.state.codecContext === null) {
|
||||
await this.createCodecContext();
|
||||
assert(this.codecContext);
|
||||
}
|
||||
assert(this.state.codecContext);
|
||||
|
||||
if (videoSample._data instanceof AvFrameVideoSampleResource) {
|
||||
this.frame.ref(videoSample._data.frame);
|
||||
// Release any buffers still referenced from the previous encode before reffing the new frame, otherwise
|
||||
// av_frame_ref leaks them
|
||||
// https://github.com/Vanilagy/mediabunny/issues/392
|
||||
this.state.frame.unref();
|
||||
this.state.frame.ref(videoSample._data.frame);
|
||||
} else {
|
||||
if (videoSample.format === null) {
|
||||
throw new Error('Cannot encode foreign VideoSample with unknown (null) format.');
|
||||
}
|
||||
|
||||
this.lastBuffer = await copyVideoSampleToAvFrame(videoSample, this.frame, this.lastBuffer);
|
||||
this.lastBuffer = await copyVideoSampleToAvFrame(videoSample, this.state.frame, this.lastBuffer);
|
||||
}
|
||||
|
||||
let frameToEncode = this.frame;
|
||||
let frameToEncode = this.state.frame;
|
||||
|
||||
const requiresScaler
|
||||
= this.codecContext.pixelFormat !== this.frame.format
|
||||
|| this.codecContext.width !== this.frame.width
|
||||
|| this.codecContext.height !== this.frame.height;
|
||||
= this.state.codecContext.pixelFormat !== this.state.frame.format
|
||||
|| this.state.codecContext.width !== this.state.frame.width
|
||||
|| this.state.codecContext.height !== this.state.frame.height;
|
||||
|
||||
if (requiresScaler) {
|
||||
if (!this.scaler) {
|
||||
this.scaler = new NodeAv.SoftwareScaleContext();
|
||||
if (!this.state.scaler) {
|
||||
this.state.scaler = new NodeAv.SoftwareScaleContext();
|
||||
}
|
||||
|
||||
const key = `${this.frame.width}x${this.frame.height}:${this.frame.format}`;
|
||||
const key = `${this.state.frame.width}x${this.state.frame.height}:${this.state.frame.format}`;
|
||||
const needsConfigure = key !== this.lastScalerKey;
|
||||
|
||||
if (needsConfigure) {
|
||||
this.scaler.getContext(
|
||||
this.frame.width, this.frame.height, this.frame.format as NodeAv.AVPixelFormat,
|
||||
this.codecContext.width, this.codecContext.height, this.codecContext.pixelFormat,
|
||||
this.state.scaler.getContext(
|
||||
this.state.frame.width, this.state.frame.height, this.state.frame.format as NodeAv.AVPixelFormat,
|
||||
this.state.codecContext.width, this.state.codecContext.height, this.state.codecContext.pixelFormat,
|
||||
NodeAv.SWS_FAST_BILINEAR,
|
||||
);
|
||||
|
||||
this.lastScalerKey = key;
|
||||
|
||||
const ret = this.scaler.initContext();
|
||||
const ret = this.state.scaler.initContext();
|
||||
NodeAv.FFmpegError.throwIfError(ret, 'initContext');
|
||||
}
|
||||
|
||||
if (!this.dstFrame) {
|
||||
this.dstFrame = new NodeAv.Frame();
|
||||
this.dstFrame.alloc();
|
||||
this.dstFrame.width = this.codecContext.width;
|
||||
this.dstFrame.height = this.codecContext.height;
|
||||
this.dstFrame.format = this.codecContext.pixelFormat;
|
||||
this.dstFrame.allocBuffer();
|
||||
if (!this.state.dstFrame) {
|
||||
this.state.dstFrame = new NodeAv.Frame();
|
||||
this.state.dstFrame.alloc();
|
||||
this.state.dstFrame.width = this.state.codecContext.width;
|
||||
this.state.dstFrame.height = this.state.codecContext.height;
|
||||
this.state.dstFrame.format = this.state.codecContext.pixelFormat;
|
||||
this.state.dstFrame.allocBuffer();
|
||||
}
|
||||
|
||||
await this.scaler.scaleFrame(this.dstFrame, this.frame);
|
||||
this.dstFrame.copyProps(this.frame);
|
||||
frameToEncode = this.dstFrame;
|
||||
await this.state.scaler.scaleFrame(this.state.dstFrame, this.state.frame);
|
||||
this.state.dstFrame.copyProps(this.state.frame);
|
||||
frameToEncode = this.state.dstFrame;
|
||||
}
|
||||
|
||||
frameToEncode.pts = BigInt(videoSample.microsecondTimestamp);
|
||||
@@ -282,12 +309,12 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
|
||||
}
|
||||
}
|
||||
|
||||
const ret = await this.codecContext.sendFrame(frameToEncode);
|
||||
const ret = await this.state.codecContext.sendFrame(frameToEncode);
|
||||
NodeAv.FFmpegError.throwIfError(ret, 'Send frame');
|
||||
|
||||
// Keep receiving packets until no more are available for this frame
|
||||
while (true) {
|
||||
const receiveRet = await this.codecContext.receivePacket(this.packet);
|
||||
const receiveRet = await this.state.codecContext.receivePacket(this.state.packet);
|
||||
if (receiveRet === NodeAv.AVERROR_EAGAIN || receiveRet === NodeAv.AVERROR_EOF) {
|
||||
break;
|
||||
}
|
||||
@@ -297,20 +324,20 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
|
||||
}
|
||||
|
||||
receivePacket(ret: number) {
|
||||
assert(this.codecContext);
|
||||
assert(this.state.codecContext);
|
||||
NodeAv.FFmpegError.throwIfError(ret, 'Receive packet');
|
||||
|
||||
if (!this.packet.data) {
|
||||
if (!this.state.packet.data) {
|
||||
return;
|
||||
}
|
||||
let packetData = toUint8Array(this.packet.data);
|
||||
let packetData = toUint8Array(this.state.packet.data);
|
||||
|
||||
let timestamp = Number(this.packet.pts) / 1e6;
|
||||
let duration = Number(this.packet.duration) / 1e6;
|
||||
let timestamp = Number(this.state.packet.pts) / 1e6;
|
||||
let duration = Number(this.state.packet.duration) / 1e6;
|
||||
|
||||
const preciseTimingIndex = binarySearchLessOrEqual(
|
||||
this.preciseTimings,
|
||||
Number(this.packet.pts),
|
||||
Number(this.state.packet.pts),
|
||||
x => x.microsecondTimestamp,
|
||||
);
|
||||
const entry = preciseTimingIndex !== -1
|
||||
@@ -319,7 +346,7 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
|
||||
|
||||
// If there's a relevant timing entry, refine the packet's timing data to get better accuracy than
|
||||
// microseconds
|
||||
if (entry && entry.microsecondTimestamp === Number(this.packet.pts)) {
|
||||
if (entry && entry.microsecondTimestamp === Number(this.state.packet.pts)) {
|
||||
if (entry.timestampIsValid) {
|
||||
timestamp = entry.timestamp;
|
||||
}
|
||||
@@ -346,14 +373,14 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
|
||||
let serializedRecord: Uint8Array;
|
||||
|
||||
if (this.codec === 'avc') {
|
||||
const record = extractAvcDecoderConfigurationRecord(this.packet.data);
|
||||
const record = extractAvcDecoderConfigurationRecord(this.state.packet.data);
|
||||
if (!record) {
|
||||
throw new Error('Invalid AVC data, could not extract decoder configuration record.');
|
||||
}
|
||||
|
||||
serializedRecord = serializeAvcDecoderConfigurationRecord(record);
|
||||
} else {
|
||||
const record = extractHevcDecoderConfigurationRecord(this.packet.data);
|
||||
const record = extractHevcDecoderConfigurationRecord(this.state.packet.data);
|
||||
if (!record) {
|
||||
throw new Error('Invalid HEVC data, could not extract decoder configuration record.');
|
||||
}
|
||||
@@ -488,14 +515,14 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
|
||||
}
|
||||
|
||||
const sideData: EncodedPacketSideData = {};
|
||||
const matroskaBlockAdditional = this.packet.getSideData(NodeAv.AV_PKT_DATA_MATROSKA_BLOCKADDITIONAL);
|
||||
const matroskaBlockAdditional = this.state.packet.getSideData(NodeAv.AV_PKT_DATA_MATROSKA_BLOCKADDITIONAL);
|
||||
if (matroskaBlockAdditional) {
|
||||
sideData.alpha = toUint8Array(matroskaBlockAdditional).subarray(8); // Skip the BlockAddId
|
||||
}
|
||||
|
||||
const packet = new EncodedPacket(
|
||||
packetData,
|
||||
this.packet.isKeyframe ? 'key' : 'delta',
|
||||
this.state.packet.isKeyframe ? 'key' : 'delta',
|
||||
timestamp,
|
||||
duration,
|
||||
undefined,
|
||||
@@ -507,21 +534,19 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
|
||||
// Create the decoder config
|
||||
metadata.decoderConfig = {
|
||||
codec: decoderConfigCodecString,
|
||||
codedWidth: this.codecContext.width,
|
||||
codedHeight: this.codecContext.height,
|
||||
displayAspectWidth: this.config.displayWidth ?? this.codecContext.width,
|
||||
displayAspectHeight: this.config.displayHeight ?? this.codecContext.height,
|
||||
codedWidth: this.state.codecContext.width,
|
||||
codedHeight: this.state.codecContext.height,
|
||||
displayAspectWidth: this.config.displayWidth ?? this.state.codecContext.width,
|
||||
displayAspectHeight: this.config.displayHeight ?? this.state.codecContext.height,
|
||||
description: decoderConfigDescription ?? undefined,
|
||||
colorSpace: {
|
||||
primaries:
|
||||
unmapColorPrimaries(this.codecContext.colorPrimaries) as VideoColorPrimaries,
|
||||
matrix:
|
||||
unmapMatrixCoefficients(this.codecContext.colorSpace) as VideoMatrixCoefficients,
|
||||
primaries: unmapColorPrimaries(this.state.codecContext.colorPrimaries) as VideoColorPrimaries,
|
||||
matrix: unmapMatrixCoefficients(this.state.codecContext.colorSpace) as VideoMatrixCoefficients,
|
||||
transfer:
|
||||
unmapTransferCharacteristics(this.codecContext.colorTrc) as VideoTransferCharacteristics,
|
||||
fullRange: this.codecContext.colorRange === NodeAv.AVCOL_RANGE_JPEG
|
||||
unmapTransferCharacteristics(this.state.codecContext.colorTrc) as VideoTransferCharacteristics,
|
||||
fullRange: this.state.codecContext.colorRange === NodeAv.AVCOL_RANGE_JPEG
|
||||
? true
|
||||
: this.codecContext.colorRange === NodeAv.AVCOL_RANGE_MPEG
|
||||
: this.state.codecContext.colorRange === NodeAv.AVCOL_RANGE_MPEG
|
||||
? false
|
||||
: undefined,
|
||||
},
|
||||
@@ -533,14 +558,14 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
|
||||
}
|
||||
|
||||
async flush(): Promise<void> {
|
||||
if (this.codecContext) {
|
||||
if (this.state.codecContext) {
|
||||
// Send null frame to signal flush
|
||||
const ret = await this.codecContext.sendFrame(null);
|
||||
const ret = await this.state.codecContext.sendFrame(null);
|
||||
NodeAv.FFmpegError.throwIfError(ret, 'Send frame');
|
||||
|
||||
// Keep receiving packets until no more are available
|
||||
while (true) {
|
||||
const receiveRet = await this.codecContext.receivePacket(this.packet);
|
||||
const receiveRet = await this.state.codecContext.receivePacket(this.state.packet);
|
||||
if (receiveRet === NodeAv.AVERROR_EAGAIN || receiveRet === NodeAv.AVERROR_EOF) {
|
||||
break;
|
||||
}
|
||||
@@ -548,8 +573,8 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
|
||||
this.receivePacket(receiveRet);
|
||||
}
|
||||
|
||||
this.codecContext.freeContext();
|
||||
this.codecContext = null;
|
||||
this.state.codecContext.freeContext();
|
||||
this.state.codecContext = null;
|
||||
// The codec is done now and can't be reused. Any subsequent encode call will first need to recreate a
|
||||
// codec context.
|
||||
}
|
||||
@@ -558,10 +583,7 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
|
||||
}
|
||||
|
||||
close(): MaybePromise<void> {
|
||||
this.codecContext?.freeContext();
|
||||
this.frame.free();
|
||||
this.packet.free();
|
||||
this.scaler?.freeContext();
|
||||
this.dstFrame?.free();
|
||||
finalizationRegistry?.unregister(this);
|
||||
freeState(this.state);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user