diff --git a/packages/server/src/audio-decoder.ts b/packages/server/src/audio-decoder.ts index 4157a37..34186ed 100644 --- a/packages/server/src/audio-decoder.ts +++ b/packages/server/src/audio-decoder.ts @@ -1,7 +1,7 @@ import { AudioCodec, AudioSample, CustomAudioDecoder, EncodedPacket, MaybePromise } from 'mediabunny'; import * as NodeAv from 'node-av'; import { CODEC_TO_CODEC_ID, getChannelLayout } from './misc'; -import { assert, roundToDivisor, toUint8Array } from '../../../src/misc'; +import { assert, toUint8Array } from '../../../src/misc'; import { NodeAvFrameAudioSampleResource } from './audio-sample'; export class NodeAvAudioDecoder extends CustomAudioDecoder { @@ -39,6 +39,7 @@ export class NodeAvAudioDecoder extends CustomAudioDecoder { codecContext.sampleRate = this.config.sampleRate; codecContext.channelLayout = getChannelLayout(this.config.numberOfChannels); + codecContext.timeBase = new NodeAv.Rational(1, this.config.sampleRate); codecContext.codecType = NodeAv.AVMEDIA_TYPE_AUDIO; codecContext.codecId = codecId; codecContext.extraData = this.config.description @@ -54,10 +55,10 @@ export class NodeAvAudioDecoder extends CustomAudioDecoder { async decode(packet: EncodedPacket): Promise { this.packet.isKeyframe = packet.type === 'key'; this.packet.data = Buffer.from(packet.data); - this.packet.timeBase = { num: 1, den: 1e6 }; - this.packet.pts = BigInt(packet.microsecondTimestamp); + this.packet.timeBase = { num: 1, den: this.config.sampleRate }; + this.packet.pts = BigInt(Math.round(packet.timestamp * this.config.sampleRate)); this.packet.dts = NodeAv.AV_NOPTS_VALUE; - this.packet.duration = BigInt(packet.microsecondDuration); + this.packet.duration = BigInt(Math.round(packet.duration * this.config.sampleRate)); const ret = await this.codecContext.sendPacket(this.packet); NodeAv.FFmpegError.throwIfError(ret, 'Send packet'); @@ -75,7 +76,7 @@ export class NodeAvAudioDecoder extends CustomAudioDecoder { receiveFrame(ret: number) { NodeAv.FFmpegError.throwIfError(ret, 'Receive frame'); - const timestamp = roundToDivisor(Number(this.frame.pts) / 1e6, this.config.sampleRate); + const timestamp = Number(this.frame.pts) / this.config.sampleRate; this.onSample(new AudioSample(new NodeAvFrameAudioSampleResource(this.frame, timestamp))); } diff --git a/packages/server/src/audio-encoder.ts b/packages/server/src/audio-encoder.ts index 1a655a0..bc84184 100644 --- a/packages/server/src/audio-encoder.ts +++ b/packages/server/src/audio-encoder.ts @@ -124,8 +124,7 @@ export class NodeAvAudioEncoder extends CustomAudioEncoder { this.frame.nbSamples = audioSample.numberOfFrames; this.frame.sampleRate = audioSample.sampleRate; this.frame.channelLayout = getChannelLayout(audioSample.numberOfChannels); - this.frame.pts = BigInt(Math.round(audioSample.timestamp * audioSample.sampleRate)); - this.frame.duration = BigInt(audioSample.numberOfFrames); + this.frame.duration = BigInt(Math.round(audioSample.duration * this.config.sampleRate)); this.frame.allocBuffer(); assert(this.frame.data); @@ -135,6 +134,9 @@ export class NodeAvAudioEncoder extends CustomAudioEncoder { } } + this.frame.pts = BigInt(Math.round(audioSample.timestamp * this.config.sampleRate)); + this.frame.timeBase = new NodeAv.Rational(1, this.config.sampleRate); + const key = `${this.frame.sampleRate}:${this.frame.channels}:${this.frame.format}`; if (this.inputParametersKey !== null && this.inputParametersKey !== key) { throw new Error( @@ -314,8 +316,8 @@ export class NodeAvAudioEncoder extends CustomAudioEncoder { duration, ); - this.onPacket(packet, metadata); this.packetEmitted = true; + this.onPacket(packet, metadata); } async flush(): Promise { @@ -350,6 +352,7 @@ export class NodeAvAudioEncoder extends CustomAudioEncoder { this.packetEmitted = false; this.firstExpectedTimestamp = null; this.outputTimestampOffset = 0; + this.adtsHeaderTemplate = null; this.resampler?.free(); this.resampler = null; diff --git a/packages/server/src/video-encoder.ts b/packages/server/src/video-encoder.ts index 2a5524d..58fac99 100644 --- a/packages/server/src/video-encoder.ts +++ b/packages/server/src/video-encoder.ts @@ -1,4 +1,12 @@ -import { CustomVideoEncoder, MaybePromise, QUALITY_MEDIUM, VideoCodec, VideoSample, EncodedPacket } from 'mediabunny'; +import { + CustomVideoEncoder, + MaybePromise, + QUALITY_MEDIUM, + VideoCodec, + VideoSample, + EncodedPacket, + EncodedPacketSideData, +} from 'mediabunny'; import * as NodeAv from 'node-av'; import { CODEC_TO_CODEC_ID, @@ -25,7 +33,6 @@ import { } from '../../../src/codec-data'; import { extractVideoCodecString } from '../../../src/codec'; import { assert, binarySearchLessOrEqual, simplifyRational, toUint8Array } from '../../../src/misc'; -import { EncodedPacketSideData } from 'mediabunny'; export class NodeAvVideoEncoder extends CustomVideoEncoder { frame!: NodeAv.Frame; @@ -180,7 +187,6 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder { } this.lastBuffer = await copyVideoSampleToAvFrame(videoSample, this.frame, this.lastBuffer); - this.frame.keyFrame = options?.keyFrame ? 1 : 0; } let frameToEncode = this.frame; @@ -225,6 +231,18 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder { frameToEncode = this.dstFrame; } + frameToEncode.pts = BigInt(videoSample.microsecondTimestamp); + frameToEncode.duration = BigInt(videoSample.microsecondDuration); + frameToEncode.timeBase = new NodeAv.Rational(1, 1e6); + + // Let's just set both for good measure + frameToEncode.pictType = options?.keyFrame + ? NodeAv.AV_PICTURE_TYPE_I + : NodeAv.AV_PICTURE_TYPE_NONE; + frameToEncode.keyFrame = options?.keyFrame + ? 1 + : 0; + const preciseTimingIndex = binarySearchLessOrEqual( this.preciseTimings, videoSample.microsecondTimestamp, @@ -503,8 +521,8 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder { }; } - this.onPacket(packet, metadata); this.packetEmitted = true; + this.onPacket(packet, metadata); } async flush(): Promise { @@ -525,7 +543,7 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder { this.codecContext.freeContext(); this.codecContext = null; - // The codec is done now and can't be reused. Any subsequence encode call will first need to recreate a + // The codec is done now and can't be reused. Any subsequent encode call will first need to recreate a // codec context. } diff --git a/packages/server/src/video-sample.ts b/packages/server/src/video-sample.ts index 9d10a47..3d05459 100644 --- a/packages/server/src/video-sample.ts +++ b/packages/server/src/video-sample.ts @@ -1,7 +1,11 @@ import assert from 'assert'; -import { VideoSamplePixelFormat } from 'mediabunny'; -import { VideoSampleColorSpace } from 'mediabunny'; -import { VideoSampleResource } from 'mediabunny'; +import { + VideoSamplePixelFormat, + VideoSampleResource, + VideoSampleColorSpace, + SetRequired, + VideoSampleInit, +} from 'mediabunny'; import * as NodeAv from 'node-av'; import { MaybePromise, toUint8Array } from '../../../src/misc'; import { VideoSampleTransformationDescription, VideoDataPlane, VideoSample } from '../../../src/sample'; @@ -15,8 +19,6 @@ import { mapMatrixCoefficients, mapTransferCharacteristics, } from './misc'; -import { SetRequired } from 'mediabunny'; -import { VideoSampleInit } from 'mediabunny'; const JPEG_RANGE_PIX_FORMATS = new Set([ NodeAv.AV_PIX_FMT_YUVJ411P, @@ -140,8 +142,6 @@ export const copyVideoSampleToAvFrame = async (sample: VideoSample, frame: NodeA sample.pixelAspectRatio.num, sample.pixelAspectRatio.den, ); - frame.pts = BigInt(sample.microsecondTimestamp); - frame.duration = BigInt(sample.microsecondDuration); frame.colorPrimaries = mapColorPrimaries(sample.colorSpace.primaries ?? 'unknown') ?? NodeAv.AVCOL_PRI_UNSPECIFIED; frame.colorSpace = mapMatrixCoefficients(sample.colorSpace.matrix ?? 'unknown') diff --git a/src/media-sink.ts b/src/media-sink.ts index 4023fd5..ed5cc5d 100644 --- a/src/media-sink.ts +++ b/src/media-sink.ts @@ -1726,7 +1726,7 @@ const colorAlphaMergerWorkerCode = () => { const format = alpha.format; if (format === 'RGBA' || format === 'BGRA' || format === 'RGBX' || format === 'BGRX') { - // Pack alpha data tightly + // Pack alpha data tightly. Assume alpha is stored in RGB, so sample just from R for simplicity. const rOffset = (format === 'RGBA' || format === 'RGBX') ? 0 : 2; const pixelCount = width * height; for (let i = 0; i < pixelCount; i++) { diff --git a/src/sample.ts b/src/sample.ts index e61ffdd..76c10d7 100644 --- a/src/sample.ts +++ b/src/sample.ts @@ -1245,8 +1245,9 @@ export class VideoSample implements Disposable { ? [this.squarePixelWidth, this.squarePixelHeight] : [this.squarePixelHeight, this.squarePixelWidth]; - if (options.crop) { - clampCropRectangle(options.crop, rotatedWidth, rotatedHeight); + let finalCrop = options.crop; + if (finalCrop) { + finalCrop = clampCropRectangle(finalCrop, rotatedWidth, rotatedHeight); } // These variables specify where the final sample will be drawn on the canvas diff --git a/src/source.ts b/src/source.ts index cff524c..8b64e9e 100644 --- a/src/source.ts +++ b/src/source.ts @@ -591,11 +591,13 @@ const DEFAULT_RETRY_DELAY // Check if this could be a CORS error. If so, we cannot recover from it and // should not attempt to retry. // CORS errors are intentionally not opaque, so we need to rely on heuristics. - const couldBeCorsError = error instanceof Error && ( - error.message.includes('Failed to fetch') // Chrome - || error.message.includes('Load failed') // Safari - || error.message.includes('NetworkError when attempting to fetch resource') // Firefox - ); + const couldBeCorsError = error instanceof Error + && ( + error.message.includes('Failed to fetch') // Chrome + || error.message.includes('Load failed') // Safari + || error.message.includes('NetworkError when attempting to fetch resource') // Firefox + ) + && typeof window !== 'undefined'; // CORS only happens in browser environments if (couldBeCorsError) { let originOfSrc: string | null = null; diff --git a/test/node/server-extension.test.ts b/test/node/server-extension.test.ts index 2325aa6..2a5afe1 100644 --- a/test/node/server-extension.test.ts +++ b/test/node/server-extension.test.ts @@ -176,6 +176,45 @@ describe('Video', async () => { await encoder.close(); }); + test('Forced key frame encode', async () => { + const encoder = new NodeAvVideoEncoder(); + // @ts-expect-error Readonly + encoder.codec = 'avc'; + // @ts-expect-error Readonly + encoder.config = { + codec: buildVideoCodecString('avc', 1280, 720, 1e6), + width: 1280, + height: 720, + bitrate: 1e6, + } satisfies VideoEncoderConfig; + + let packetCount = 0; + + // @ts-expect-error Readonly + encoder.onPacket = (packet: EncodedPacket) => { + expect(packet.type).toBe(packetCount % 2 ? 'delta' : 'key'); + packetCount++; + }; + + await encoder.init(); + + const data = new Uint8Array(1280 * 720 * 4).fill(0xff); // White + for (let i = 0; i < 10; i++) { + using sample = new VideoSample(data, { + format: 'RGBX', + codedWidth: 1280, + codedHeight: 720, + timestamp: i / 30, + duration: 1 / 30, + }); + + await encoder.encode(sample, { keyFrame: i % 2 === 0 }); + } + + await encoder.flush(); + await encoder.close(); + }); + test('AVC encode and decode, length-prefixed', async () => { await encodeDecodeTest('avc', {}, async (packet, meta, i) => { expect(packet.timestamp).toBe(i / 30);