diff --git a/packages/server/src/audio-decoder.ts b/packages/server/src/audio-decoder.ts index ed9faf4..22bb420 100644 --- a/packages/server/src/audio-decoder.ts +++ b/packages/server/src/audio-decoder.ts @@ -12,28 +12,10 @@ import { CODEC_TO_CODEC_ID, getChannelLayout } from './misc'; import { assert, toUint8Array } from '../../../src/misc'; import { AvFrameAudioSampleResource } from './audio-sample'; -type NodeAvState = { - frame: NodeAv.Frame; - packet: NodeAv.Packet; - codecContext: NodeAv.CodecContext | null; -}; - -const freeState = (state: NodeAvState) => { - state.codecContext?.freeContext(); - state.frame.free(); - state.packet.free(); -}; - -// Needed for proper freeing if close isn't called -let finalizationRegistry: FinalizationRegistry | null = null; -if (typeof FinalizationRegistry !== 'undefined') { - finalizationRegistry = new FinalizationRegistry((state) => { - freeState(state); - }); -} - export class NodeAvAudioDecoder extends CustomAudioDecoder { - state!: NodeAvState; + frame!: NodeAv.Frame; + packet!: NodeAv.Packet; + codecContext: NodeAv.CodecContext | null = null; // eslint-disable-next-line @typescript-eslint/no-unused-vars static override supports(codec: AudioCodec, config: AudioDecoderConfig): boolean { @@ -47,13 +29,10 @@ export class NodeAvAudioDecoder extends CustomAudioDecoder { } async init(): Promise { - const frame = new NodeAv.Frame(); - frame.alloc(); - const packet = new NodeAv.Packet(); - packet.alloc(); - this.state = { frame, packet, codecContext: null }; - - finalizationRegistry?.register(this, this.state, this); + this.frame = new NodeAv.Frame(); + this.frame.alloc(); + this.packet = new NodeAv.Packet(); + this.packet.alloc(); const codecId = CODEC_TO_CODEC_ID[this.codec]; assert(codecId !== undefined); @@ -78,26 +57,26 @@ export class NodeAvAudioDecoder extends CustomAudioDecoder { const ret = await codecContext.open2(); NodeAv.FFmpegError.throwIfError(ret, 'Open codec context'); - this.state.codecContext = codecContext; + this.codecContext = codecContext; } async decode(packet: EncodedPacket): Promise { - assert(this.state.codecContext); + assert(this.codecContext); - this.state.packet.isKeyframe = packet.type === 'key'; - this.state.packet.data = Buffer.from(packet.data); - this.state.packet.timeBase = { num: 1, den: this.config.sampleRate }; - this.state.packet.pts = BigInt(Math.round(packet.timestamp * this.config.sampleRate)); - this.state.packet.dts = NodeAv.AV_NOPTS_VALUE; - this.state.packet.duration = BigInt(Math.round(packet.duration * this.config.sampleRate)); + this.packet.isKeyframe = packet.type === 'key'; + this.packet.data = Buffer.from(packet.data); + this.packet.timeBase = { num: 1, den: this.config.sampleRate }; + this.packet.pts = BigInt(Math.round(packet.timestamp * this.config.sampleRate)); + this.packet.dts = NodeAv.AV_NOPTS_VALUE; + this.packet.duration = BigInt(Math.round(packet.duration * this.config.sampleRate)); - const ret = await this.state.codecContext.sendPacket(this.state.packet); + const ret = await this.codecContext.sendPacket(this.packet); NodeAv.FFmpegError.throwIfError(ret, 'Send packet'); - this.state.packet.unref(); // Don't need the data again, so just unref it + this.packet.unref(); // Don't need the data again, so just unref it while (true) { - const receiveRet = await this.state.codecContext.receiveFrame(this.state.frame); + const receiveRet = await this.codecContext.receiveFrame(this.frame); if (receiveRet === NodeAv.AVERROR_EAGAIN || receiveRet === NodeAv.AVERROR_EOF) { break; } @@ -109,7 +88,7 @@ export class NodeAvAudioDecoder extends CustomAudioDecoder { receiveFrame(ret: number) { NodeAv.FFmpegError.throwIfError(ret, 'Receive frame'); - const clone = this.state.frame.clone(); + const clone = this.frame.clone(); if (!clone) { throw new Error('Allocation failure during frame clone.'); } @@ -119,15 +98,15 @@ export class NodeAvAudioDecoder extends CustomAudioDecoder { } async flush(): Promise { - assert(this.state.codecContext); + assert(this.codecContext); // Send null packet to signal flush - const ret = await this.state.codecContext.sendPacket(null); + const ret = await this.codecContext.sendPacket(null); NodeAv.FFmpegError.throwIfError(ret, 'Flush decoder'); // Keep receiving frames until no more are available while (true) { - const receiveRet = await this.state.codecContext.receiveFrame(this.state.frame); + const receiveRet = await this.codecContext.receiveFrame(this.frame); if (receiveRet === NodeAv.AVERROR_EAGAIN || receiveRet === NodeAv.AVERROR_EOF) { // No more frames available break; @@ -136,11 +115,12 @@ export class NodeAvAudioDecoder extends CustomAudioDecoder { this.receiveFrame(receiveRet); } - this.state.codecContext.flushBuffers(); + this.codecContext.flushBuffers(); } close(): MaybePromise { - finalizationRegistry?.unregister(this); - freeState(this.state); + this.codecContext?.freeContext(); + this.frame.free(); + this.packet.free(); } } diff --git a/packages/server/src/audio-encoder.ts b/packages/server/src/audio-encoder.ts index cdb2186..27eb759 100644 --- a/packages/server/src/audio-encoder.ts +++ b/packages/server/src/audio-encoder.ts @@ -32,32 +32,12 @@ const AC3_SAMPLE_RATES = [32000, 44100, 48000]; const FRAME_SIZE_FALLBACK = 1024; // Just 'cause -type NodeAvState = { - frame: NodeAv.Frame; - packet: NodeAv.Packet; - codecContext: NodeAv.CodecContext | null; - resampler: NodeAv.SoftwareResampleContext | null; - dstFrame: NodeAv.Frame | null; -}; - -const freeState = (state: NodeAvState) => { - state.codecContext?.freeContext(); - state.frame.free(); - state.packet.free(); - state.dstFrame?.free(); - state.resampler?.free(); -}; - -// Needed for proper freeing if close isn't called -let finalizationRegistry: FinalizationRegistry | null = null; -if (typeof FinalizationRegistry !== 'undefined') { - finalizationRegistry = new FinalizationRegistry((state) => { - freeState(state); - }); -} - export class NodeAvAudioEncoder extends CustomAudioEncoder { - state!: NodeAvState; + frame!: NodeAv.Frame; + packet!: NodeAv.Packet; + codecContext: NodeAv.CodecContext | null = null; + resampler: NodeAv.SoftwareResampleContext | null = null; + dstFrame: NodeAv.Frame | null = null; avCodec!: NodeAv.Codec; firstExpectedTimestamp: number | null = null; outputTimestampOffset = 0; @@ -96,13 +76,10 @@ export class NodeAvAudioEncoder extends CustomAudioEncoder { } async init(): Promise { - const frame = new NodeAv.Frame(); - frame.alloc(); - const packet = new NodeAv.Packet(); - packet.alloc(); - this.state = { frame, packet, codecContext: null, resampler: null, dstFrame: null }; - - finalizationRegistry?.register(this, this.state, this); + this.frame = new NodeAv.Frame(); + this.frame.alloc(); + this.packet = new NodeAv.Packet(); + this.packet.alloc(); const codecId = CODEC_TO_CODEC_ID[this.codec]; assert(codecId !== undefined); @@ -118,7 +95,7 @@ export class NodeAvAudioEncoder extends CustomAudioEncoder { } async createCodecContext() { - assert(this.state.codecContext === null); + assert(this.codecContext === null); const codecContext = new NodeAv.CodecContext(); codecContext.allocContext3(this.avCodec); @@ -145,13 +122,13 @@ export class NodeAvAudioEncoder extends CustomAudioEncoder { const ret = await codecContext.open2(); NodeAv.FFmpegError.throwIfError(ret, 'Open codec context'); - this.state.codecContext = codecContext; + this.codecContext = codecContext; } async encode(audioSample: AudioSample): Promise { - if (this.state.codecContext === null) { + if (this.codecContext === null) { await this.createCodecContext(); - assert(this.state.codecContext); + assert(this.codecContext); } this.firstExpectedTimestamp ??= audioSample.timestamp; @@ -160,17 +137,17 @@ export class NodeAvAudioEncoder extends CustomAudioEncoder { // Release any buffers still referenced from the previous encode before reffing the new frame, otherwise // av_frame_ref leaks them // https://github.com/Vanilagy/mediabunny/issues/392 - this.state.frame.unref(); - this.state.frame.ref(audioSample._data.frame); + this.frame.unref(); + this.frame.ref(audioSample._data.frame); } else { - copyAudioSampleToAvFrame(audioSample, this.state.frame); + copyAudioSampleToAvFrame(audioSample, this.frame); } - this.state.frame.pts = BigInt(Math.round(audioSample.timestamp * this.config.sampleRate)); - this.state.frame.duration = BigInt(Math.round(audioSample.duration * this.config.sampleRate)); - this.state.frame.timeBase = new NodeAv.Rational(1, this.config.sampleRate); + this.frame.pts = BigInt(Math.round(audioSample.timestamp * this.config.sampleRate)); + this.frame.duration = BigInt(Math.round(audioSample.duration * this.config.sampleRate)); + this.frame.timeBase = new NodeAv.Rational(1, this.config.sampleRate); - const key = `${this.state.frame.sampleRate}:${this.state.frame.channels}:${this.state.frame.format}`; + const key = `${this.frame.sampleRate}:${this.frame.channels}:${this.frame.format}`; if (this.inputParametersKey !== null && this.inputParametersKey !== key) { throw new Error( 'Input audio parameters changed. For this audio encoder, you cannot change the input audio' @@ -183,83 +160,83 @@ export class NodeAvAudioEncoder extends CustomAudioEncoder { // 1. Format conversion is needed (sample format, sample rate, or channel count differs) // 2. The codec requires fixed frame sizes const requiresResampler - = this.state.codecContext.frameSize > 0 - || this.state.codecContext.sampleFormat !== this.state.frame.format - || this.state.codecContext.sampleRate !== this.state.frame.sampleRate - || this.state.codecContext.channels !== this.state.frame.channels; + = this.codecContext.frameSize > 0 + || this.codecContext.sampleFormat !== this.frame.format + || this.codecContext.sampleRate !== this.frame.sampleRate + || this.codecContext.channels !== this.frame.channels; if (requiresResampler) { - if (!this.state.resampler) { - this.state.resampler = new NodeAv.SoftwareResampleContext(); - this.resamplerInputSampleRate = this.state.frame.sampleRate; + if (!this.resampler) { + this.resampler = new NodeAv.SoftwareResampleContext(); + this.resamplerInputSampleRate = this.frame.sampleRate; - const outLayout = getChannelLayout(this.state.codecContext.channels); - const inLayout = getChannelLayout(this.state.frame.channels); + const outLayout = getChannelLayout(this.codecContext.channels); + const inLayout = getChannelLayout(this.frame.channels); - const ret = this.state.resampler.allocSetOpts2( - outLayout, this.state.codecContext.sampleFormat, this.state.codecContext.sampleRate, - inLayout, this.state.frame.format as NodeAv.AVSampleFormat, this.state.frame.sampleRate, + const ret = this.resampler.allocSetOpts2( + outLayout, this.codecContext.sampleFormat, this.codecContext.sampleRate, + inLayout, this.frame.format as NodeAv.AVSampleFormat, this.frame.sampleRate, ); NodeAv.FFmpegError.throwIfError(ret, 'allocSetOpts2'); - const ret2 = this.state.resampler.init(); + const ret2 = this.resampler.init(); NodeAv.FFmpegError.throwIfError(ret2, 'init'); - this.state.dstFrame = new NodeAv.Frame(); - this.state.dstFrame.alloc(); - this.state.dstFrame.channelLayout = outLayout; - this.state.dstFrame.sampleRate = this.state.codecContext.sampleRate; - this.state.dstFrame.format = this.state.codecContext.sampleFormat; - this.state.dstFrame.nbSamples = this.state.codecContext.frameSize || FRAME_SIZE_FALLBACK; - this.state.dstFrame.duration = BigInt(this.state.dstFrame.nbSamples); - this.state.dstFrame.allocBuffer(); + this.dstFrame = new NodeAv.Frame(); + this.dstFrame.alloc(); + this.dstFrame.channelLayout = outLayout; + this.dstFrame.sampleRate = this.codecContext.sampleRate; + this.dstFrame.format = this.codecContext.sampleFormat; + this.dstFrame.nbSamples = this.codecContext.frameSize || FRAME_SIZE_FALLBACK; + this.dstFrame.duration = BigInt(this.dstFrame.nbSamples); + this.dstFrame.allocBuffer(); - this.nextResamplerPts = this.state.frame.pts; + this.nextResamplerPts = this.frame.pts; } - const inputBuffers = this.state.frame.data; + const inputBuffers = this.frame.data; if (!inputBuffers) { throw new DOMException('Frame has no data', 'EncodingError'); } - await this.state.resampler.convert(null, 0, inputBuffers, this.state.frame.nbSamples); + await this.resampler.convert(null, 0, inputBuffers, this.frame.nbSamples); await this.pullResampledFrames(); } else { - await this.sendFrameAndReceivePackets(this.state.frame); + await this.sendFrameAndReceivePackets(this.frame); } } async pullResampledFrames() { - assert(this.state.codecContext); - assert(this.state.resampler); - assert(this.state.dstFrame); + assert(this.codecContext); + assert(this.resampler); + assert(this.dstFrame); assert(this.nextResamplerPts !== null); - const frameSize = this.state.codecContext.frameSize || FRAME_SIZE_FALLBACK; + const frameSize = this.codecContext.frameSize || FRAME_SIZE_FALLBACK; while (true) { - const available = this.state.resampler.getOutSamples(0); + const available = this.resampler.getOutSamples(0); if (available < frameSize) { break; } - await this.state.resampler.convert(this.state.dstFrame.data, frameSize, null, 0); + await this.resampler.convert(this.dstFrame.data, frameSize, null, 0); - this.state.dstFrame.pts = this.nextResamplerPts; + this.dstFrame.pts = this.nextResamplerPts; - await this.sendFrameAndReceivePackets(this.state.dstFrame); + await this.sendFrameAndReceivePackets(this.dstFrame); this.nextResamplerPts += BigInt(frameSize); } } async sendFrameAndReceivePackets(frame: NodeAv.Frame | null) { - assert(this.state.codecContext); + assert(this.codecContext); - const ret = await this.state.codecContext.sendFrame(frame); + const ret = await this.codecContext.sendFrame(frame); NodeAv.FFmpegError.throwIfError(ret, 'Send frame'); while (true) { - const receiveRet = await this.state.codecContext.receivePacket(this.state.packet); + const receiveRet = await this.codecContext.receivePacket(this.packet); if (receiveRet === NodeAv.AVERROR_EAGAIN || receiveRet === NodeAv.AVERROR_EOF) { break; } @@ -269,18 +246,18 @@ export class NodeAvAudioEncoder extends CustomAudioEncoder { } receivePacket(ret: number) { - assert(this.state.codecContext); + assert(this.codecContext); assert(this.firstExpectedTimestamp !== null); NodeAv.FFmpegError.throwIfError(ret, 'Receive packet'); - if (!this.state.packet.data) { + if (!this.packet.data) { return; } - let timestamp = Number(this.state.packet.pts) / this.state.codecContext.sampleRate; - const duration = Number(this.state.packet.duration) / this.state.codecContext.sampleRate; + let timestamp = Number(this.packet.pts) / this.codecContext.sampleRate; + const duration = Number(this.packet.duration) / this.codecContext.sampleRate; - let data: Uint8Array = this.state.packet.data; + let data: Uint8Array = this.packet.data; let metadata: EncodedAudioChunkMetadata | undefined; if (this.packetEmitted) { @@ -292,8 +269,8 @@ export class NodeAvAudioEncoder extends CustomAudioEncoder { this.outputTimestampOffset = Math.max(this.firstExpectedTimestamp - timestamp, 0); const codecString = this.config.codec; - let description = this.state.codecContext.extraData - ? toUint8Array(this.state.codecContext.extraData) + let description = this.codecContext.extraData + ? toUint8Array(this.codecContext.extraData) : undefined; if (this.codec === 'aac') { @@ -338,8 +315,8 @@ export class NodeAvAudioEncoder extends CustomAudioEncoder { metadata = { decoderConfig: { codec: codecString, - sampleRate: this.state.codecContext.sampleRate, - numberOfChannels: this.state.codecContext.channels, + sampleRate: this.codecContext.sampleRate, + numberOfChannels: this.codecContext.channels, description, }, }; @@ -371,50 +348,53 @@ export class NodeAvAudioEncoder extends CustomAudioEncoder { } async flush(): Promise { - if (!this.state.codecContext) { + if (!this.codecContext) { return; } outer: - if (this.state.resampler) { + if (this.resampler) { assert(this.resamplerInputSampleRate !== null); - const currentOutSamples = this.state.resampler.getOutSamples(0); + const currentOutSamples = this.resampler.getOutSamples(0); if (currentOutSamples === 0) { break outer; // Clean cut-off point } - const frameSize = this.state.codecContext.frameSize || FRAME_SIZE_FALLBACK; + const frameSize = this.codecContext.frameSize || FRAME_SIZE_FALLBACK; assert(currentOutSamples < frameSize); // Because if it's more, it would've already been retrieved const inputSamplesNeeded = Math.ceil( - ((frameSize - currentOutSamples) / this.state.codecContext.sampleRate) * this.resamplerInputSampleRate, + ((frameSize - currentOutSamples) / this.codecContext.sampleRate) * this.resamplerInputSampleRate, ); - this.state.resampler.injectSilence(inputSamplesNeeded); + this.resampler.injectSilence(inputSamplesNeeded); await this.pullResampledFrames(); } await this.sendFrameAndReceivePackets(null); - this.state.codecContext.freeContext(); - this.state.codecContext = null; + this.codecContext.freeContext(); + this.codecContext = null; this.packetEmitted = false; this.firstExpectedTimestamp = null; this.outputTimestampOffset = 0; this.adtsHeaderTemplate = null; - this.state.resampler?.free(); - this.state.resampler = null; + this.resampler?.free(); + this.resampler = null; this.inputParametersKey = null; this.resamplerInputSampleRate = null; this.nextResamplerPts = null; - this.state.dstFrame?.free(); - this.state.dstFrame = null; + this.dstFrame?.free(); + this.dstFrame = null; } close(): MaybePromise { - finalizationRegistry?.unregister(this); - freeState(this.state); + this.codecContext?.freeContext(); + this.frame.free(); + this.packet.free(); + this.dstFrame?.free(); + this.resampler?.free(); } } diff --git a/packages/server/src/video-decoder.ts b/packages/server/src/video-decoder.ts index bbddef1..d79a943 100644 --- a/packages/server/src/video-decoder.ts +++ b/packages/server/src/video-decoder.ts @@ -12,28 +12,10 @@ import { CODEC_TO_CODEC_ID, getHardwareDecoderCodec } from './misc'; import { assert, binarySearchLessOrEqual, simplifyRational, toUint8Array } from '../../../src/misc'; import { AvFrameVideoSampleResource } from './video-sample'; -type NodeAvState = { - frame: NodeAv.Frame; - packet: NodeAv.Packet; - codecContext: NodeAv.CodecContext | null; -}; - -const freeState = (state: NodeAvState) => { - state.codecContext?.freeContext(); - state.frame.free(); - state.packet.free(); -}; - -// Needed for proper freeing if close isn't called -let finalizationRegistry: FinalizationRegistry | null = null; -if (typeof FinalizationRegistry !== 'undefined') { - finalizationRegistry = new FinalizationRegistry((state) => { - freeState(state); - }); -} - export class NodeAvVideoDecoder extends CustomVideoDecoder { - state!: NodeAvState; + frame!: NodeAv.Frame; + packet!: NodeAv.Packet; + codecContext: NodeAv.CodecContext | null = null; pixelAspectRatio!: Rational; // Bookkeeping to restore the original timing information @@ -51,17 +33,14 @@ export class NodeAvVideoDecoder extends CustomVideoDecoder { } async init() { - const frame = new NodeAv.Frame(); - frame.alloc(); - const packet = new NodeAv.Packet(); - packet.alloc(); - this.state = { frame, packet, codecContext: null }; - - finalizationRegistry?.register(this, this.state, this); + this.frame = new NodeAv.Frame(); + this.frame.alloc(); + this.packet = new NodeAv.Packet(); + this.packet.alloc(); } async initCodecContext(packet: EncodedPacket) { - assert(this.state.codecContext === null); + assert(this.codecContext === null); const codecId = CODEC_TO_CODEC_ID[this.codec]; assert(codecId !== undefined); @@ -106,29 +85,29 @@ export class NodeAvVideoDecoder extends CustomVideoDecoder { const ret = await codecContext.open2(); NodeAv.FFmpegError.throwIfError(ret, 'Open codec context'); - this.state.codecContext = codecContext; + this.codecContext = codecContext; } async decode(packet: EncodedPacket) { - if (this.state.codecContext === null) { + if (this.codecContext === null) { await this.initCodecContext(packet); } - assert(this.state.codecContext); + assert(this.codecContext); - this.state.packet.isKeyframe = packet.type === 'key'; - this.state.packet.data = Buffer.from(packet.data); - this.state.packet.timeBase = { num: 1, den: 1e6 }; - this.state.packet.pts = BigInt(packet.microsecondTimestamp); - this.state.packet.dts = NodeAv.AV_NOPTS_VALUE; - this.state.packet.duration = BigInt(packet.microsecondDuration); + this.packet.isKeyframe = packet.type === 'key'; + this.packet.data = Buffer.from(packet.data); + this.packet.timeBase = { num: 1, den: 1e6 }; + this.packet.pts = BigInt(packet.microsecondTimestamp); + this.packet.dts = NodeAv.AV_NOPTS_VALUE; + this.packet.duration = BigInt(packet.microsecondDuration); if (packet.sideData.alpha) { const matroskaBlockAdditional = Buffer.alloc(8 + packet.sideData.alpha.byteLength); matroskaBlockAdditional[7] = 1; // BlockAddId matroskaBlockAdditional.set(packet.sideData.alpha, 8); - this.state.packet.addSideData(NodeAv.AV_PKT_DATA_MATROSKA_BLOCKADDITIONAL, matroskaBlockAdditional); + this.packet.addSideData(NodeAv.AV_PKT_DATA_MATROSKA_BLOCKADDITIONAL, matroskaBlockAdditional); } const preciseTimingIndex = binarySearchLessOrEqual( @@ -163,13 +142,13 @@ export class NodeAvVideoDecoder extends CustomVideoDecoder { } } - const ret = await this.state.codecContext.sendPacket(this.state.packet); + const ret = await this.codecContext.sendPacket(this.packet); NodeAv.FFmpegError.throwIfError(ret, 'Send packet'); - this.state.packet.unref(); // Don't need the data again, so just unref it + this.packet.unref(); // Don't need the data again, so just unref it while (true) { - const receiveRet = await this.state.codecContext.receiveFrame(this.state.frame); + const receiveRet = await this.codecContext.receiveFrame(this.frame); if (receiveRet === NodeAv.AVERROR_EAGAIN || receiveRet === NodeAv.AVERROR_EOF) { break; } @@ -181,14 +160,14 @@ export class NodeAvVideoDecoder extends CustomVideoDecoder { receiveFrame(ret: number) { NodeAv.FFmpegError.throwIfError(ret, 'Receive frame'); - this.state.frame.sampleAspectRatio = new NodeAv.Rational(this.pixelAspectRatio.num, this.pixelAspectRatio.den); + this.frame.sampleAspectRatio = new NodeAv.Rational(this.pixelAspectRatio.num, this.pixelAspectRatio.den); - let timestamp = Number(this.state.frame.pts) / 1e6; - let duration = Number(this.state.frame.duration) / 1e6; + let timestamp = Number(this.frame.pts) / 1e6; + let duration = Number(this.frame.duration) / 1e6; const preciseTimingIndex = binarySearchLessOrEqual( this.preciseTimings, - Number(this.state.frame.pts), + Number(this.frame.pts), x => x.microsecondTimestamp, ); const entry = preciseTimingIndex !== -1 @@ -197,7 +176,7 @@ export class NodeAvVideoDecoder extends CustomVideoDecoder { // If there's a relevant timing entry, refine the frame's timing data to get better accuracy than // microseconds - if (entry && entry.microsecondTimestamp === Number(this.state.frame.pts)) { + if (entry && entry.microsecondTimestamp === Number(this.frame.pts)) { if (entry.timestampIsValid) { timestamp = entry.timestamp; } @@ -206,7 +185,7 @@ export class NodeAvVideoDecoder extends CustomVideoDecoder { } } - const clone = this.state.frame.clone(); + const clone = this.frame.clone(); if (!clone) { throw new Error('Frame clone allocation failed.'); } @@ -218,17 +197,17 @@ export class NodeAvVideoDecoder extends CustomVideoDecoder { } async flush() { - if (!this.state.codecContext) { + if (!this.codecContext) { return; } // Send null packet to signal flush - const ret = await this.state.codecContext.sendPacket(null); + const ret = await this.codecContext.sendPacket(null); NodeAv.FFmpegError.throwIfError(ret, 'Flush decoder'); // Keep receiving frames until no more are available while (true) { - const receiveRet = await this.state.codecContext.receiveFrame(this.state.frame); + const receiveRet = await this.codecContext.receiveFrame(this.frame); if (receiveRet === NodeAv.AVERROR_EAGAIN || receiveRet === NodeAv.AVERROR_EOF) { // No more frames available break; @@ -237,11 +216,12 @@ export class NodeAvVideoDecoder extends CustomVideoDecoder { this.receiveFrame(receiveRet); } - this.state.codecContext.flushBuffers(); + this.codecContext.flushBuffers(); } close(): MaybePromise { - finalizationRegistry?.unregister(this); - freeState(this.state); + this.codecContext?.freeContext(); + this.frame.free(); + this.packet.free(); } } diff --git a/packages/server/src/video-encoder.ts b/packages/server/src/video-encoder.ts index 5bea44d..4b5b82c 100644 --- a/packages/server/src/video-encoder.ts +++ b/packages/server/src/video-encoder.ts @@ -41,32 +41,12 @@ import { import { extractVideoCodecString } from '../../../src/codec'; import { assert, binarySearchLessOrEqual, simplifyRational, toUint8Array } from '../../../src/misc'; -type NodeAvState = { - frame: NodeAv.Frame; - packet: NodeAv.Packet; - codecContext: NodeAv.CodecContext | null; - scaler: NodeAv.SoftwareScaleContext | null; - dstFrame: NodeAv.Frame | null; -}; - -const freeState = (state: NodeAvState) => { - state.codecContext?.freeContext(); - state.frame.free(); - state.packet.free(); - state.scaler?.freeContext(); - state.dstFrame?.free(); -}; - -// Needed for proper freeing if close isn't called -let finalizationRegistry: FinalizationRegistry | null = null; -if (typeof FinalizationRegistry !== 'undefined') { - finalizationRegistry = new FinalizationRegistry((state) => { - freeState(state); - }); -} - export class NodeAvVideoEncoder extends CustomVideoEncoder { - state!: NodeAvState; + frame!: NodeAv.Frame; + packet!: NodeAv.Packet; + codecContext: NodeAv.CodecContext | null = null; + scaler: NodeAv.SoftwareScaleContext | null = null; + dstFrame: NodeAv.Frame | null = null; avCodec!: NodeAv.Codec; lastBuffer: Buffer | null = null; packetEmitted = false; @@ -87,16 +67,12 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder { } async init(): Promise { - const frame = new NodeAv.Frame(); - frame.alloc(); - frame.timeBase = new NodeAv.Rational(1, 1e6); + this.frame = new NodeAv.Frame(); + this.frame.alloc(); + this.frame.timeBase = new NodeAv.Rational(1, 1e6); - const packet = new NodeAv.Packet(); - packet.alloc(); - - this.state = { frame, packet, codecContext: null, scaler: null, dstFrame: null }; - - finalizationRegistry?.register(this, this.state, this); + this.packet = new NodeAv.Packet(); + this.packet.alloc(); const codecId = CODEC_TO_CODEC_ID[this.codec]; assert(codecId !== undefined); @@ -120,7 +96,7 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder { } async createCodecContext() { - assert(this.state.codecContext === null); + assert(this.codecContext === null); const codecContext = new NodeAv.CodecContext(); codecContext.allocContext3(this.avCodec); @@ -200,69 +176,69 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder { const ret = await codecContext.open2(); NodeAv.FFmpegError.throwIfError(ret, 'Open codec context'); - this.state.codecContext = codecContext; + this.codecContext = codecContext; } async encode(videoSample: VideoSample, options: VideoEncoderEncodeOptions): Promise { - if (this.state.codecContext === null) { + if (this.codecContext === null) { await this.createCodecContext(); } - assert(this.state.codecContext); + assert(this.codecContext); if (videoSample._data instanceof AvFrameVideoSampleResource) { // Release any buffers still referenced from the previous encode before reffing the new frame, otherwise // av_frame_ref leaks them // https://github.com/Vanilagy/mediabunny/issues/392 - this.state.frame.unref(); - this.state.frame.ref(videoSample._data.frame); + this.frame.unref(); + this.frame.ref(videoSample._data.frame); } else { if (videoSample.format === null) { throw new Error('Cannot encode foreign VideoSample with unknown (null) format.'); } - this.lastBuffer = await copyVideoSampleToAvFrame(videoSample, this.state.frame, this.lastBuffer); + this.lastBuffer = await copyVideoSampleToAvFrame(videoSample, this.frame, this.lastBuffer); } - let frameToEncode = this.state.frame; + let frameToEncode = this.frame; const requiresScaler - = this.state.codecContext.pixelFormat !== this.state.frame.format - || this.state.codecContext.width !== this.state.frame.width - || this.state.codecContext.height !== this.state.frame.height; + = this.codecContext.pixelFormat !== this.frame.format + || this.codecContext.width !== this.frame.width + || this.codecContext.height !== this.frame.height; if (requiresScaler) { - if (!this.state.scaler) { - this.state.scaler = new NodeAv.SoftwareScaleContext(); + if (!this.scaler) { + this.scaler = new NodeAv.SoftwareScaleContext(); } - const key = `${this.state.frame.width}x${this.state.frame.height}:${this.state.frame.format}`; + const key = `${this.frame.width}x${this.frame.height}:${this.frame.format}`; const needsConfigure = key !== this.lastScalerKey; if (needsConfigure) { - this.state.scaler.getContext( - this.state.frame.width, this.state.frame.height, this.state.frame.format as NodeAv.AVPixelFormat, - this.state.codecContext.width, this.state.codecContext.height, this.state.codecContext.pixelFormat, + this.scaler.getContext( + this.frame.width, this.frame.height, this.frame.format as NodeAv.AVPixelFormat, + this.codecContext.width, this.codecContext.height, this.codecContext.pixelFormat, NodeAv.SWS_FAST_BILINEAR, ); this.lastScalerKey = key; - const ret = this.state.scaler.initContext(); + const ret = this.scaler.initContext(); NodeAv.FFmpegError.throwIfError(ret, 'initContext'); } - if (!this.state.dstFrame) { - this.state.dstFrame = new NodeAv.Frame(); - this.state.dstFrame.alloc(); - this.state.dstFrame.width = this.state.codecContext.width; - this.state.dstFrame.height = this.state.codecContext.height; - this.state.dstFrame.format = this.state.codecContext.pixelFormat; - this.state.dstFrame.allocBuffer(); + if (!this.dstFrame) { + this.dstFrame = new NodeAv.Frame(); + this.dstFrame.alloc(); + this.dstFrame.width = this.codecContext.width; + this.dstFrame.height = this.codecContext.height; + this.dstFrame.format = this.codecContext.pixelFormat; + this.dstFrame.allocBuffer(); } - await this.state.scaler.scaleFrame(this.state.dstFrame, this.state.frame); - this.state.dstFrame.copyProps(this.state.frame); - frameToEncode = this.state.dstFrame; + await this.scaler.scaleFrame(this.dstFrame, this.frame); + this.dstFrame.copyProps(this.frame); + frameToEncode = this.dstFrame; } frameToEncode.pts = BigInt(videoSample.microsecondTimestamp); @@ -309,12 +285,12 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder { } } - const ret = await this.state.codecContext.sendFrame(frameToEncode); + const ret = await this.codecContext.sendFrame(frameToEncode); NodeAv.FFmpegError.throwIfError(ret, 'Send frame'); // Keep receiving packets until no more are available for this frame while (true) { - const receiveRet = await this.state.codecContext.receivePacket(this.state.packet); + const receiveRet = await this.codecContext.receivePacket(this.packet); if (receiveRet === NodeAv.AVERROR_EAGAIN || receiveRet === NodeAv.AVERROR_EOF) { break; } @@ -324,20 +300,20 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder { } receivePacket(ret: number) { - assert(this.state.codecContext); + assert(this.codecContext); NodeAv.FFmpegError.throwIfError(ret, 'Receive packet'); - if (!this.state.packet.data) { + if (!this.packet.data) { return; } - let packetData = toUint8Array(this.state.packet.data); + let packetData = toUint8Array(this.packet.data); - let timestamp = Number(this.state.packet.pts) / 1e6; - let duration = Number(this.state.packet.duration) / 1e6; + let timestamp = Number(this.packet.pts) / 1e6; + let duration = Number(this.packet.duration) / 1e6; const preciseTimingIndex = binarySearchLessOrEqual( this.preciseTimings, - Number(this.state.packet.pts), + Number(this.packet.pts), x => x.microsecondTimestamp, ); const entry = preciseTimingIndex !== -1 @@ -346,7 +322,7 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder { // If there's a relevant timing entry, refine the packet's timing data to get better accuracy than // microseconds - if (entry && entry.microsecondTimestamp === Number(this.state.packet.pts)) { + if (entry && entry.microsecondTimestamp === Number(this.packet.pts)) { if (entry.timestampIsValid) { timestamp = entry.timestamp; } @@ -373,14 +349,14 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder { let serializedRecord: Uint8Array; if (this.codec === 'avc') { - const record = extractAvcDecoderConfigurationRecord(this.state.packet.data); + const record = extractAvcDecoderConfigurationRecord(this.packet.data); if (!record) { throw new Error('Invalid AVC data, could not extract decoder configuration record.'); } serializedRecord = serializeAvcDecoderConfigurationRecord(record); } else { - const record = extractHevcDecoderConfigurationRecord(this.state.packet.data); + const record = extractHevcDecoderConfigurationRecord(this.packet.data); if (!record) { throw new Error('Invalid HEVC data, could not extract decoder configuration record.'); } @@ -515,14 +491,14 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder { } const sideData: EncodedPacketSideData = {}; - const matroskaBlockAdditional = this.state.packet.getSideData(NodeAv.AV_PKT_DATA_MATROSKA_BLOCKADDITIONAL); + const matroskaBlockAdditional = this.packet.getSideData(NodeAv.AV_PKT_DATA_MATROSKA_BLOCKADDITIONAL); if (matroskaBlockAdditional) { sideData.alpha = toUint8Array(matroskaBlockAdditional).subarray(8); // Skip the BlockAddId } const packet = new EncodedPacket( packetData, - this.state.packet.isKeyframe ? 'key' : 'delta', + this.packet.isKeyframe ? 'key' : 'delta', timestamp, duration, undefined, @@ -534,19 +510,19 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder { // Create the decoder config metadata.decoderConfig = { codec: decoderConfigCodecString, - codedWidth: this.state.codecContext.width, - codedHeight: this.state.codecContext.height, - displayAspectWidth: this.config.displayWidth ?? this.state.codecContext.width, - displayAspectHeight: this.config.displayHeight ?? this.state.codecContext.height, + codedWidth: this.codecContext.width, + codedHeight: this.codecContext.height, + displayAspectWidth: this.config.displayWidth ?? this.codecContext.width, + displayAspectHeight: this.config.displayHeight ?? this.codecContext.height, description: decoderConfigDescription ?? undefined, colorSpace: { - primaries: unmapColorPrimaries(this.state.codecContext.colorPrimaries) as VideoColorPrimaries, - matrix: unmapMatrixCoefficients(this.state.codecContext.colorSpace) as VideoMatrixCoefficients, + primaries: unmapColorPrimaries(this.codecContext.colorPrimaries) as VideoColorPrimaries, + matrix: unmapMatrixCoefficients(this.codecContext.colorSpace) as VideoMatrixCoefficients, transfer: - unmapTransferCharacteristics(this.state.codecContext.colorTrc) as VideoTransferCharacteristics, - fullRange: this.state.codecContext.colorRange === NodeAv.AVCOL_RANGE_JPEG + unmapTransferCharacteristics(this.codecContext.colorTrc) as VideoTransferCharacteristics, + fullRange: this.codecContext.colorRange === NodeAv.AVCOL_RANGE_JPEG ? true - : this.state.codecContext.colorRange === NodeAv.AVCOL_RANGE_MPEG + : this.codecContext.colorRange === NodeAv.AVCOL_RANGE_MPEG ? false : undefined, }, @@ -558,14 +534,14 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder { } async flush(): Promise { - if (this.state.codecContext) { + if (this.codecContext) { // Send null frame to signal flush - const ret = await this.state.codecContext.sendFrame(null); + const ret = await this.codecContext.sendFrame(null); NodeAv.FFmpegError.throwIfError(ret, 'Send frame'); // Keep receiving packets until no more are available while (true) { - const receiveRet = await this.state.codecContext.receivePacket(this.state.packet); + const receiveRet = await this.codecContext.receivePacket(this.packet); if (receiveRet === NodeAv.AVERROR_EAGAIN || receiveRet === NodeAv.AVERROR_EOF) { break; } @@ -573,8 +549,8 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder { this.receivePacket(receiveRet); } - this.state.codecContext.freeContext(); - this.state.codecContext = null; + this.codecContext.freeContext(); + this.codecContext = null; // The codec is done now and can't be reused. Any subsequent encode call will first need to recreate a // codec context. } @@ -583,7 +559,10 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder { } close(): MaybePromise { - finalizationRegistry?.unregister(this); - freeState(this.state); + this.codecContext?.freeContext(); + this.frame.free(); + this.packet.free(); + this.scaler?.freeContext(); + this.dstFrame?.free(); } }