arcodange/mediabunny
1
0
Fork 0
mirror of https://github.com/arcodange-org/mediabunny.git synced 2026-09-29 11:53:52 +02:00
Code Issues Packages Projects Releases Wiki Activity

Fix some bugs & polish edge cases

Browse Source
This commit is contained in:
Vanilagy
2026-05-11 23:50:06 +02:00
parent d8c8606341
commit 81531cc510
8 changed files with 92 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/server/src/audio-decoder.ts
+6 -5
View File
@@ -1,7 +1,7 @@
import { AudioCodec, AudioSample, CustomAudioDecoder, EncodedPacket, MaybePromise } from 'mediabunny';
import * as NodeAv from 'node-av';
import { CODEC_TO_CODEC_ID, getChannelLayout } from './misc';
import { assert, roundToDivisor, toUint8Array } from '../../../src/misc';
import { assert, toUint8Array } from '../../../src/misc';
import { NodeAvFrameAudioSampleResource } from './audio-sample';
export class NodeAvAudioDecoder extends CustomAudioDecoder {
@@ -39,6 +39,7 @@ export class NodeAvAudioDecoder extends CustomAudioDecoder {
codecContext.sampleRate = this.config.sampleRate;
codecContext.channelLayout = getChannelLayout(this.config.numberOfChannels);
codecContext.timeBase = new NodeAv.Rational(1, this.config.sampleRate);
codecContext.codecType = NodeAv.AVMEDIA_TYPE_AUDIO;
codecContext.codecId = codecId;
codecContext.extraData = this.config.description
@@ -54,10 +55,10 @@ export class NodeAvAudioDecoder extends CustomAudioDecoder {
async decode(packet: EncodedPacket): Promise<void> {
this.packet.isKeyframe = packet.type === 'key';
this.packet.data = Buffer.from(packet.data);
this.packet.timeBase = { num: 1, den: 1e6 };
this.packet.pts = BigInt(packet.microsecondTimestamp);
this.packet.timeBase = { num: 1, den: this.config.sampleRate };
this.packet.pts = BigInt(Math.round(packet.timestamp * this.config.sampleRate));
this.packet.dts = NodeAv.AV_NOPTS_VALUE;
this.packet.duration = BigInt(packet.microsecondDuration);
this.packet.duration = BigInt(Math.round(packet.duration * this.config.sampleRate));
const ret = await this.codecContext.sendPacket(this.packet);
NodeAv.FFmpegError.throwIfError(ret, 'Send packet');
@@ -75,7 +76,7 @@ export class NodeAvAudioDecoder extends CustomAudioDecoder {
receiveFrame(ret: number) {
NodeAv.FFmpegError.throwIfError(ret, 'Receive frame');
const timestamp = roundToDivisor(Number(this.frame.pts) / 1e6, this.config.sampleRate);
const timestamp = Number(this.frame.pts) / this.config.sampleRate;
this.onSample(new AudioSample(new NodeAvFrameAudioSampleResource(this.frame, timestamp)));
}
packages/server/src/audio-encoder.ts
+6 -3
View File
@@ -124,8 +124,7 @@ export class NodeAvAudioEncoder extends CustomAudioEncoder {
this.frame.nbSamples = audioSample.numberOfFrames;
this.frame.sampleRate = audioSample.sampleRate;
this.frame.channelLayout = getChannelLayout(audioSample.numberOfChannels);
this.frame.pts = BigInt(Math.round(audioSample.timestamp * audioSample.sampleRate));
this.frame.duration = BigInt(audioSample.numberOfFrames);
this.frame.duration = BigInt(Math.round(audioSample.duration * this.config.sampleRate));
this.frame.allocBuffer();
assert(this.frame.data);
@@ -135,6 +134,9 @@ export class NodeAvAudioEncoder extends CustomAudioEncoder {
}
}
this.frame.pts = BigInt(Math.round(audioSample.timestamp * this.config.sampleRate));
this.frame.timeBase = new NodeAv.Rational(1, this.config.sampleRate);
const key = `${this.frame.sampleRate}:${this.frame.channels}:${this.frame.format}`;
if (this.inputParametersKey !== null && this.inputParametersKey !== key) {
throw new Error(
@@ -314,8 +316,8 @@ export class NodeAvAudioEncoder extends CustomAudioEncoder {
duration,
);
this.onPacket(packet, metadata);
this.packetEmitted = true;
this.onPacket(packet, metadata);
}
async flush(): Promise<void> {
@@ -350,6 +352,7 @@ export class NodeAvAudioEncoder extends CustomAudioEncoder {
this.packetEmitted = false;
this.firstExpectedTimestamp = null;
this.outputTimestampOffset = 0;
this.adtsHeaderTemplate = null;
this.resampler?.free();
this.resampler = null;
packages/server/src/video-encoder.ts
+23 -5
View File
@@ -1,4 +1,12 @@
import { CustomVideoEncoder, MaybePromise, QUALITY_MEDIUM, VideoCodec, VideoSample, EncodedPacket } from 'mediabunny';
import {
CustomVideoEncoder,
MaybePromise,
QUALITY_MEDIUM,
VideoCodec,
VideoSample,
EncodedPacket,
EncodedPacketSideData,
} from 'mediabunny';
import * as NodeAv from 'node-av';
import {
CODEC_TO_CODEC_ID,
@@ -25,7 +33,6 @@ import {
} from '../../../src/codec-data';
import { extractVideoCodecString } from '../../../src/codec';
import { assert, binarySearchLessOrEqual, simplifyRational, toUint8Array } from '../../../src/misc';
import { EncodedPacketSideData } from 'mediabunny';
export class NodeAvVideoEncoder extends CustomVideoEncoder {
frame!: NodeAv.Frame;
@@ -180,7 +187,6 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
}
this.lastBuffer = await copyVideoSampleToAvFrame(videoSample, this.frame, this.lastBuffer);
this.frame.keyFrame = options?.keyFrame ? 1 : 0;
}
let frameToEncode = this.frame;
@@ -225,6 +231,18 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
frameToEncode = this.dstFrame;
}
frameToEncode.pts = BigInt(videoSample.microsecondTimestamp);
frameToEncode.duration = BigInt(videoSample.microsecondDuration);
frameToEncode.timeBase = new NodeAv.Rational(1, 1e6);
// Let's just set both for good measure
frameToEncode.pictType = options?.keyFrame
? NodeAv.AV_PICTURE_TYPE_I
: NodeAv.AV_PICTURE_TYPE_NONE;
frameToEncode.keyFrame = options?.keyFrame
? 1
: 0;
const preciseTimingIndex = binarySearchLessOrEqual(
this.preciseTimings,
videoSample.microsecondTimestamp,
@@ -503,8 +521,8 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
};
}
this.onPacket(packet, metadata);
this.packetEmitted = true;
this.onPacket(packet, metadata);
}
async flush(): Promise<void> {
@@ -525,7 +543,7 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
this.codecContext.freeContext();
this.codecContext = null;
// The codec is done now and can't be reused. Any subsequence encode call will first need to recreate a
// The codec is done now and can't be reused. Any subsequent encode call will first need to recreate a
// codec context.
}
packages/server/src/video-sample.ts
+7 -7
View File
@@ -1,7 +1,11 @@
import assert from 'assert';
import { VideoSamplePixelFormat } from 'mediabunny';
import { VideoSampleColorSpace } from 'mediabunny';
import { VideoSampleResource } from 'mediabunny';
import {
VideoSamplePixelFormat,
VideoSampleResource,
VideoSampleColorSpace,
SetRequired,
VideoSampleInit,
} from 'mediabunny';
import * as NodeAv from 'node-av';
import { MaybePromise, toUint8Array } from '../../../src/misc';
import { VideoSampleTransformationDescription, VideoDataPlane, VideoSample } from '../../../src/sample';
@@ -15,8 +19,6 @@ import {
mapMatrixCoefficients,
mapTransferCharacteristics,
} from './misc';
import { SetRequired } from 'mediabunny';
import { VideoSampleInit } from 'mediabunny';
const JPEG_RANGE_PIX_FORMATS = new Set([
NodeAv.AV_PIX_FMT_YUVJ411P,
@@ -140,8 +142,6 @@ export const copyVideoSampleToAvFrame = async (sample: VideoSample, frame: NodeA
sample.pixelAspectRatio.num,
sample.pixelAspectRatio.den,
);
frame.pts = BigInt(sample.microsecondTimestamp);
frame.duration = BigInt(sample.microsecondDuration);
frame.colorPrimaries = mapColorPrimaries(sample.colorSpace.primaries ?? 'unknown')
?? NodeAv.AVCOL_PRI_UNSPECIFIED;
frame.colorSpace = mapMatrixCoefficients(sample.colorSpace.matrix ?? 'unknown')
src/media-sink.ts
+1 -1
View File
@@ -1726,7 +1726,7 @@ const colorAlphaMergerWorkerCode = () => {
const format = alpha.format;
if (format === 'RGBA' || format === 'BGRA' || format === 'RGBX' || format === 'BGRX') {
// Pack alpha data tightly
// Pack alpha data tightly. Assume alpha is stored in RGB, so sample just from R for simplicity.
const rOffset = (format === 'RGBA' || format === 'RGBX') ? 0 : 2;
const pixelCount = width * height;
for (let i = 0; i < pixelCount; i++) {
src/sample.ts
+3 -2
View File
@@ -1245,8 +1245,9 @@ export class VideoSample implements Disposable {
? [this.squarePixelWidth, this.squarePixelHeight]
: [this.squarePixelHeight, this.squarePixelWidth];
if (options.crop) {
clampCropRectangle(options.crop, rotatedWidth, rotatedHeight);
let finalCrop = options.crop;
if (finalCrop) {
finalCrop = clampCropRectangle(finalCrop, rotatedWidth, rotatedHeight);
}
// These variables specify where the final sample will be drawn on the canvas
src/source.ts
+7 -5
View File
@@ -591,11 +591,13 @@ const DEFAULT_RETRY_DELAY
// Check if this could be a CORS error. If so, we cannot recover from it and
// should not attempt to retry.
// CORS errors are intentionally not opaque, so we need to rely on heuristics.
const couldBeCorsError = error instanceof Error && (
error.message.includes('Failed to fetch') // Chrome
|| error.message.includes('Load failed') // Safari
|| error.message.includes('NetworkError when attempting to fetch resource') // Firefox
);
const couldBeCorsError = error instanceof Error
&& (
error.message.includes('Failed to fetch') // Chrome
|| error.message.includes('Load failed') // Safari
|| error.message.includes('NetworkError when attempting to fetch resource') // Firefox
)
&& typeof window !== 'undefined'; // CORS only happens in browser environments
if (couldBeCorsError) {
let originOfSrc: string | null = null;
test/node/server-extension.test.ts
+39
View File
@@ -176,6 +176,45 @@ describe('Video', async () => {
await encoder.close();
});
test('Forced key frame encode', async () => {
const encoder = new NodeAvVideoEncoder();
// @ts-expect-error Readonly
encoder.codec = 'avc';
// @ts-expect-error Readonly
encoder.config = {
codec: buildVideoCodecString('avc', 1280, 720, 1e6),
width: 1280,
height: 720,
bitrate: 1e6,
} satisfies VideoEncoderConfig;
let packetCount = 0;
// @ts-expect-error Readonly
encoder.onPacket = (packet: EncodedPacket) => {
expect(packet.type).toBe(packetCount % 2 ? 'delta' : 'key');
packetCount++;
};
await encoder.init();
const data = new Uint8Array(1280 * 720 * 4).fill(0xff); // White
for (let i = 0; i < 10; i++) {
using sample = new VideoSample(data, {
format: 'RGBX',
codedWidth: 1280,
codedHeight: 720,
timestamp: i / 30,
duration: 1 / 30,
});
await encoder.encode(sample, { keyFrame: i % 2 === 0 });
}
await encoder.flush();
await encoder.close();
});
test('AVC encode and decode, length-prefixed', async () => {
await encodeDecodeTest('avc', {}, async (packet, meta, i) => {
expect(packet.timestamp).toBe(i / 30);