Add the video portion of @mediabunny/server

packages/server/src/index.ts

@@ -0,0 +1,28 @@
+import { registerDecoder, registerEncoder } from 'mediabunny';
+import { NodeAvVideoDecoder } from './video-decoder';
+import * as NodeAv from 'node-av';
+import { NodeAvVideoEncoder } from './video-encoder';
+
+const SERVER_LOADED_SYMBOL = Symbol.for('@mediabunny/server loaded');
+if ((globalThis as Record<symbol, unknown>)[SERVER_LOADED_SYMBOL]) {
+	console.error(
+		'[WARNING]\n@mediabunny/server was loaded twice.'
+		+ ' This will likely cause the package not to work correctly.'
+		+ ' Check if multiple dependencies are importing different versions of @mediabunny/server,'
+		+ ' or if something is being bundled incorrectly.',
+	);
+}
+(globalThis as Record<symbol, unknown>)[SERVER_LOADED_SYMBOL] = true;
+
+let registered = false;
+export const registerMediabunnyServer = () => {
+	if (registered) {
+		return;
+	}
+	registered = true;
+
+	NodeAv.Log.setLevel(NodeAv.AV_LOG_ERROR);
+
+	registerDecoder(NodeAvVideoDecoder);
+	registerEncoder(NodeAvVideoEncoder);
+};

packages/server/src/misc.ts

@@ -0,0 +1,182 @@
+import { VideoSamplePixelFormat, VideoCodec } from 'mediabunny';
+import * as NodeAv from 'node-av';
+
+export const CODEC_TO_CODEC_ID: Record<VideoCodec, NodeAv.AVCodecID> = {
+	avc: NodeAv.AV_CODEC_ID_H264,
+	hevc: NodeAv.AV_CODEC_ID_HEVC,
+	vp8: NodeAv.AV_CODEC_ID_VP8,
+	vp9: NodeAv.AV_CODEC_ID_VP9,
+	av1: NodeAv.AV_CODEC_ID_AV1,
+};
+
+let cachedHardwareContext: NodeAv.HardwareContext | null | undefined = undefined;
+export const getHardwareContext = (): NodeAv.HardwareContext | null => {
+	if (cachedHardwareContext === undefined) {
+		cachedHardwareContext = NodeAv.HardwareContext.auto();
+	}
+	return cachedHardwareContext;
+};
+
+const hardwareDecoderCodecCache = new Map<NodeAv.AVCodecID, NodeAv.Codec | null>();
+export const getHardwareDecoderCodec = (codecId: NodeAv.AVCodecID): NodeAv.Codec | null => {
+	if (!hardwareDecoderCodecCache.has(codecId)) {
+		const hw = getHardwareContext();
+		hardwareDecoderCodecCache.set(codecId, hw?.getDecoderCodec(codecId) ?? null);
+	}
+	return hardwareDecoderCodecCache.get(codecId)!;
+};
+
+const hardwareEncoderCodecCache = new Map<NodeAv.AVCodecID, NodeAv.Codec | null>();
+export const getHardwareEncoderCodec = (codecId: NodeAv.AVCodecID): NodeAv.Codec | null => {
+	if (!hardwareEncoderCodecCache.has(codecId)) {
+		const hw = getHardwareContext();
+		hardwareEncoderCodecCache.set(codecId, hw?.getEncoderCodec(codecId) ?? null);
+	}
+	return hardwareEncoderCodecCache.get(codecId)!;
+};
+
+export const mapColorPrimaries = (primaries: string) => {
+	switch (primaries) {
+		case 'bt709': return NodeAv.AVCOL_PRI_BT709;
+		case 'bt470bg': return NodeAv.AVCOL_PRI_BT470BG;
+		case 'smpte170m': return NodeAv.AVCOL_PRI_SMPTE170M;
+		case 'bt2020': return NodeAv.AVCOL_PRI_BT2020;
+		case 'smpte432': return NodeAv.AVCOL_PRI_SMPTE432;
+	}
+
+	return null;
+};
+
+export const unmapColorPrimaries = (primaries: number) => {
+	switch (primaries) {
+		case NodeAv.AVCOL_PRI_BT709: return 'bt709';
+		case NodeAv.AVCOL_PRI_BT470BG: return 'bt470bg';
+		case NodeAv.AVCOL_PRI_SMPTE170M: return 'smpte170m';
+		case NodeAv.AVCOL_PRI_BT2020: return 'bt2020';
+		case NodeAv.AVCOL_PRI_SMPTE432: return 'smpte432';
+	}
+
+	return null;
+};
+
+export const mapTransferCharacteristics = (transfer: string) => {
+	switch (transfer) {
+		case 'bt709': return NodeAv.AVCOL_TRC_BT709;
+		case 'smpte170m': return NodeAv.AVCOL_TRC_SMPTE170M;
+		case 'iec61966-2-1': return NodeAv.AVCOL_TRC_IEC61966_2_1;
+		case 'linear': return NodeAv.AVCOL_TRC_LINEAR;
+		case 'pq': return NodeAv.AVCOL_TRC_SMPTE2084;
+		case 'hlg': return NodeAv.AVCOL_TRC_ARIB_STD_B67;
+	}
+
+	return null;
+};
+
+export const unmapTransferCharacteristics = (transfer: number) => {
+	switch (transfer) {
+		case NodeAv.AVCOL_TRC_BT709: return 'bt709';
+		case NodeAv.AVCOL_TRC_SMPTE170M: return 'smpte170m';
+		case NodeAv.AVCOL_TRC_IEC61966_2_1: return 'iec61966-2-1';
+		case NodeAv.AVCOL_TRC_LINEAR: return 'linear';
+		case NodeAv.AVCOL_TRC_SMPTE2084: return 'pq';
+		case NodeAv.AVCOL_TRC_ARIB_STD_B67: return 'hlg';
+	}
+
+	return null;
+};
+
+export const mapMatrixCoefficients = (matrix: string) => {
+	switch (matrix) {
+		case 'rgb': return NodeAv.AVCOL_SPC_RGB;
+		case 'bt709': return NodeAv.AVCOL_SPC_BT709;
+		case 'bt470bg': return NodeAv.AVCOL_SPC_BT470BG;
+		case 'smpte170m': return NodeAv.AVCOL_SPC_SMPTE170M;
+		case 'bt2020-ncl': return NodeAv.AVCOL_SPC_BT2020_NCL;
+	}
+
+	return null;
+};
+
+export const unmapMatrixCoefficients = (matrix: number) => {
+	switch (matrix) {
+		case NodeAv.AVCOL_SPC_RGB: return 'rgb';
+		case NodeAv.AVCOL_SPC_BT709: return 'bt709';
+		case NodeAv.AVCOL_SPC_BT470BG: return 'bt470bg';
+		case NodeAv.AVCOL_SPC_SMPTE170M: return 'smpte170m';
+		case NodeAv.AVCOL_SPC_BT2020_NCL: return 'bt2020-ncl';
+	}
+
+	return null;
+};
+
+export const toPixelFormat = (ffmpegPixelFormat: NodeAv.AVPixelFormat): VideoSamplePixelFormat | null => {
+	switch (ffmpegPixelFormat) {
+		case NodeAv.AV_PIX_FMT_YUV420P: return 'I420';
+		// "deprecated in favor of AV_PIX_FMT_YUV420P and setting color_range"
+		case NodeAv.AV_PIX_FMT_YUVJ420P: return 'I420';
+		case NodeAv.AV_PIX_FMT_YUV420P10LE: return 'I420P10';
+		case NodeAv.AV_PIX_FMT_YUV420P12LE: return 'I420P12';
+		case NodeAv.AV_PIX_FMT_YUVA420P: return 'I420A';
+		case NodeAv.AV_PIX_FMT_YUVA420P10LE: return 'I420AP10';
+
+		case NodeAv.AV_PIX_FMT_YUV422P: return 'I422';
+		// "deprecated in favor of AV_PIX_FMT_YUV422P and setting color_range"
+		case NodeAv.AV_PIX_FMT_YUVJ422P: return 'I422';
+		case NodeAv.AV_PIX_FMT_YUV422P10LE: return 'I422P10';
+		case NodeAv.AV_PIX_FMT_YUV422P12LE: return 'I422P12';
+		case NodeAv.AV_PIX_FMT_YUVA422P: return 'I422A';
+		case NodeAv.AV_PIX_FMT_YUVA422P10LE: return 'I422AP10';
+		case NodeAv.AV_PIX_FMT_YUVA422P12LE: return 'I422AP12';
+
+		case NodeAv.AV_PIX_FMT_YUV444P: return 'I444';
+		// "deprecated in favor of AV_PIX_FMT_YUV444P and setting color_range"
+		case NodeAv.AV_PIX_FMT_YUVJ444P: return 'I444';
+		case NodeAv.AV_PIX_FMT_YUV444P10LE: return 'I444P10';
+		case NodeAv.AV_PIX_FMT_YUV444P12LE: return 'I444P12';
+		case NodeAv.AV_PIX_FMT_YUVA444P: return 'I444A';
+		case NodeAv.AV_PIX_FMT_YUVA444P10LE: return 'I444AP10';
+		case NodeAv.AV_PIX_FMT_YUVA444P12LE: return 'I444AP12';
+
+		case NodeAv.AV_PIX_FMT_NV12: return 'NV12';
+
+		case NodeAv.AV_PIX_FMT_RGBA: return 'RGBA';
+		case NodeAv.AV_PIX_FMT_RGB0: return 'RGBX';
+		case NodeAv.AV_PIX_FMT_BGRA: return 'BGRA';
+		case NodeAv.AV_PIX_FMT_BGR0: return 'BGRX';
+
+		default: return null;
+	}
+};
+
+export const fromPixelFormat = (pixelFormat: VideoSamplePixelFormat) => {
+	switch (pixelFormat) {
+		case 'I420': return NodeAv.AV_PIX_FMT_YUV420P;
+		case 'I420P10': return NodeAv.AV_PIX_FMT_YUV420P10LE;
+		case 'I420P12': return NodeAv.AV_PIX_FMT_YUV420P12LE;
+		case 'I420A': return NodeAv.AV_PIX_FMT_YUVA420P;
+		case 'I420AP10': return NodeAv.AV_PIX_FMT_YUVA420P10LE;
+
+		case 'I422': return NodeAv.AV_PIX_FMT_YUV422P;
+		case 'I422P10': return NodeAv.AV_PIX_FMT_YUV422P10LE;
+		case 'I422P12': return NodeAv.AV_PIX_FMT_YUV422P12LE;
+		case 'I422A': return NodeAv.AV_PIX_FMT_YUVA422P;
+		case 'I422AP10': return NodeAv.AV_PIX_FMT_YUVA422P10LE;
+		case 'I422AP12': return NodeAv.AV_PIX_FMT_YUVA422P12LE;
+
+		case 'I444': return NodeAv.AV_PIX_FMT_YUV444P;
+		case 'I444P10': return NodeAv.AV_PIX_FMT_YUV444P10LE;
+		case 'I444P12': return NodeAv.AV_PIX_FMT_YUV444P12LE;
+		case 'I444A': return NodeAv.AV_PIX_FMT_YUVA444P;
+		case 'I444AP10': return NodeAv.AV_PIX_FMT_YUVA444P10LE;
+		case 'I444AP12': return NodeAv.AV_PIX_FMT_YUVA444P12LE;
+
+		case 'NV12': return NodeAv.AV_PIX_FMT_NV12;
+
+		case 'RGBA': return NodeAv.AV_PIX_FMT_RGBA;
+		case 'RGBX': return NodeAv.AV_PIX_FMT_RGB0;
+		case 'BGRA': return NodeAv.AV_PIX_FMT_BGRA;
+		case 'BGRX': return NodeAv.AV_PIX_FMT_BGR0;
+
+		default: return NodeAv.AV_PIX_FMT_NONE;
+	}
+};

packages/server/src/video-decoder.ts

@@ -0,0 +1,211 @@
+import { CustomVideoDecoder, VideoCodec, EncodedPacket, VideoSample, MaybePromise, Rational } from 'mediabunny';
+import * as NodeAv from 'node-av';
+import {
+	CODEC_TO_CODEC_ID,
+	getHardwareDecoderCodec,
+	mapColorPrimaries,
+	mapMatrixCoefficients,
+	mapTransferCharacteristics,
+} from './misc';
+import { binarySearchLessOrEqual, simplifyRational, toUint8Array } from '../../../src/misc';
+import { NodeAvFrameVideoSampleResource } from './video-sample';
+
+export class NodeAvVideoDecoder extends CustomVideoDecoder {
+	frame!: NodeAv.Frame;
+	packet!: NodeAv.Packet;
+	codecContext!: NodeAv.CodecContext;
+	pixelAspectRatio!: Rational;
+
+	// Bookkeeping to restore the original timing information
+	preciseTimings: {
+		microsecondTimestamp: number;
+		timestamp: number;
+		duration: number;
+		timestampIsValid: boolean;
+		durationIsValid: boolean;
+	}[] = [];
+
+	// eslint-disable-next-line @typescript-eslint/no-unused-vars
+	static override supports(codec: VideoCodec, config: VideoDecoderConfig): boolean {
+		return codec === 'avc' || codec === 'hevc' || codec === 'vp8' || codec === 'vp9' || codec === 'av1';
+	}
+
+	async init() {
+		this.frame = new NodeAv.Frame();
+		this.frame.alloc();
+		this.packet = new NodeAv.Packet();
+		this.packet.alloc();
+
+		const codecId = CODEC_TO_CODEC_ID[this.codec];
+
+		let codec: NodeAv.Codec | null;
+		if (this.config.hardwareAcceleration !== 'prefer-hardware' || this.codec === 'av1') {
+			// https://github.com/opencv/opencv/issues/24430
+			codec = NodeAv.Codec.findDecoder(codecId);
+		} else {
+			codec = getHardwareDecoderCodec(codecId) ?? NodeAv.Codec.findDecoder(codecId);
+		}
+
+		if (!codec) {
+			throw new Error(`Unable to obtain libav codec for '${this.codec}'.`);
+		}
+
+		const codecContext = new NodeAv.CodecContext();
+		codecContext.allocContext3(codec);
+
+		this.pixelAspectRatio = simplifyRational({
+			num: (this.config.displayAspectWidth ?? this.config.codedWidth ?? 0) * (this.config.codedHeight ?? 0),
+			den: (this.config.displayAspectHeight ?? this.config.codedHeight ?? 0) * (this.config.codedWidth ?? 0) || 1,
+		});
+
+		codecContext.width = this.config.codedWidth ?? 0; // Fucky that the dimensions can be optional, but oh well
+		codecContext.height = this.config.codedHeight ?? 0;
+		codecContext.codecType = NodeAv.AVMEDIA_TYPE_VIDEO;
+		codecContext.codecId = codecId;
+		codecContext.extraData = this.config.description
+			? Buffer.from(toUint8Array(this.config.description))
+			: null;
+		codecContext.sampleAspectRatio = new NodeAv.Rational(this.pixelAspectRatio.num, this.pixelAspectRatio.den);
+
+		if (this.config.colorSpace?.primaries) {
+			const mapped = mapColorPrimaries(this.config.colorSpace.primaries);
+			if (mapped !== null) {
+				codecContext.colorPrimaries = mapped;
+			}
+		}
+		if (this.config.colorSpace?.transfer) {
+			const mapped = mapTransferCharacteristics(this.config.colorSpace.transfer);
+			if (mapped !== null) {
+				codecContext.colorTrc = mapped;
+			}
+		}
+		if (this.config.colorSpace?.matrix) {
+			const mapped = mapMatrixCoefficients(this.config.colorSpace.matrix);
+			if (mapped !== null) {
+				codecContext.colorSpace = mapped;
+			}
+		}
+		if (this.config.colorSpace?.fullRange != null) {
+			codecContext.colorRange = this.config.colorSpace.fullRange
+				? NodeAv.AVCOL_RANGE_JPEG
+				: NodeAv.AVCOL_RANGE_MPEG;
+		}
+
+		const ret = await codecContext.open2();
+		NodeAv.FFmpegError.throwIfError(ret, 'Open codec context');
+
+		this.codecContext = codecContext;
+	}
+
+	async decode(packet: EncodedPacket) {
+		this.packet.isKeyframe = packet.type === 'key';
+		this.packet.data = Buffer.from(packet.data);
+		this.packet.timeBase = { num: 1, den: 1e6 };
+		this.packet.pts = BigInt(packet.microsecondTimestamp);
+		this.packet.dts = NodeAv.AV_NOPTS_VALUE;
+		this.packet.duration = BigInt(packet.microsecondDuration);
+
+		const preciseTimingIndex = binarySearchLessOrEqual(
+			this.preciseTimings,
+			packet.microsecondTimestamp,
+			x => x.microsecondTimestamp,
+		);
+		const existingEntry = preciseTimingIndex !== -1
+			? this.preciseTimings[preciseTimingIndex]
+			: null;
+		if (existingEntry && existingEntry.microsecondTimestamp === packet.microsecondTimestamp) {
+			if (existingEntry.timestamp !== packet.timestamp) {
+				// Mapping isn't unique, can't use the timestamp
+				existingEntry.timestampIsValid = false;
+			}
+			if (existingEntry.duration !== packet.duration) {
+				// Mapping isn't unique, can't use the duration
+				existingEntry.durationIsValid = false;
+			}
+		} else {
+			this.preciseTimings.splice(preciseTimingIndex + 1, 0, {
+				microsecondTimestamp: packet.microsecondTimestamp,
+				timestamp: packet.timestamp,
+				duration: packet.duration,
+				timestampIsValid: true,
+				durationIsValid: true,
+			});
+
+			// Make sure it doesn't grow indefinitely
+			if (this.preciseTimings.length > 128) {
+				this.preciseTimings.shift();
+			}
+		}
+
+		const ret = await this.codecContext.sendPacket(this.packet);
+		NodeAv.FFmpegError.throwIfError(ret, 'Send packet');
+
+		while (true) {
+			const receiveRet = await this.codecContext.receiveFrame(this.frame);
+			if (receiveRet === NodeAv.AVERROR_EAGAIN || receiveRet === NodeAv.AVERROR_EOF) {
+				break;
+			}
+
+			this.receiveFrame(receiveRet);
+		}
+	}
+
+	receiveFrame(ret: number) {
+		NodeAv.FFmpegError.throwIfError(ret, 'Receive frame');
+
+		this.frame.sampleAspectRatio = new NodeAv.Rational(this.pixelAspectRatio.num, this.pixelAspectRatio.den);
+
+		let timestamp = Number(this.frame.pts) / 1e6;
+		let duration = Number(this.frame.duration) / 1e6;
+
+		const preciseTimingIndex = binarySearchLessOrEqual(
+			this.preciseTimings,
+			Number(this.frame.pts),
+			x => x.microsecondTimestamp,
+		);
+		const entry = preciseTimingIndex !== -1
+			? this.preciseTimings[preciseTimingIndex]
+			: null;
+
+		// If there's a relevant timing entry, refine the frame's timing data to get better accuracy than
+		// microseconds
+		if (entry && entry.microsecondTimestamp === Number(this.frame.pts)) {
+			if (entry.timestampIsValid) {
+				timestamp = entry.timestamp;
+			}
+			if (entry.durationIsValid) {
+				duration = entry.duration;
+			}
+		}
+
+		this.onSample(new VideoSample(new NodeAvFrameVideoSampleResource(this.frame), {
+			timestamp,
+			duration,
+		}));
+	}
+
+	async flush() {
+		// Send null packet to signal flush
+		const ret = await this.codecContext.sendPacket(null);
+		NodeAv.FFmpegError.throwIfError(ret, 'Flush decoder');
+
+		// Keep receiving frames until no more are available
+		while (true) {
+			const receiveRet = await this.codecContext.receiveFrame(this.frame);
+			if (receiveRet === NodeAv.AVERROR_EAGAIN || receiveRet === NodeAv.AVERROR_EOF) {
+				// No more frames available
+				break;
+			}
+
+			this.receiveFrame(receiveRet);
+		}
+
+		this.codecContext.flushBuffers();
+	}
+
+	close(): MaybePromise<void> {
+		this.codecContext.freeContext();
+		this.frame.free();
+		this.packet.free();
+	}
+}

packages/server/src/video-encoder.ts

@@ -0,0 +1,563 @@
+import { CustomVideoEncoder, MaybePromise, QUALITY_MEDIUM, VideoCodec, VideoSample, EncodedPacket } from 'mediabunny';
+import * as NodeAv from 'node-av';
+import {
+	CODEC_TO_CODEC_ID,
+	fromPixelFormat,
+	getHardwareEncoderCodec,
+	mapColorPrimaries,
+	mapMatrixCoefficients,
+	mapTransferCharacteristics,
+	unmapColorPrimaries,
+	unmapMatrixCoefficients,
+	unmapTransferCharacteristics,
+} from './misc';
+import { NodeAvFrameVideoSampleResource } from './video-sample';
+import {
+	AvcNalUnitType,
+	extractAv1CodecInfoFromPacket,
+	extractAvcDecoderConfigurationRecord,
+	extractHevcDecoderConfigurationRecord,
+	extractNalUnitTypeForAvc,
+	extractNalUnitTypeForHevc,
+	extractVp9CodecInfoFromPacket,
+	HevcNalUnitType,
+	iterateNalUnitsInAnnexB,
+	NalUnitLocation,
+	serializeAvcDecoderConfigurationRecord,
+	serializeHevcDecoderConfigurationRecord,
+} from '../../../src/codec-data';
+import { extractVideoCodecString } from '../../../src/codec';
+import { assert, binarySearchLessOrEqual, simplifyRational, toUint8Array } from '../../../src/misc';
+
+export class NodeAvVideoEncoder extends CustomVideoEncoder {
+	frame!: NodeAv.Frame;
+	packet!: NodeAv.Packet;
+	avCodec!: NodeAv.Codec;
+	codecContext: NodeAv.CodecContext | null = null;
+	lastBuffer: Buffer | null = null;
+	colorSpaceSet = false;
+	packetEmitted = false;
+
+	scaler: NodeAv.SoftwareScaleContext | null = null;
+	lastScalerKey: string | null = null;
+	dstFrame: NodeAv.Frame | null = null;
+
+	// Bookkeeping to restore the original timing information
+	preciseTimings: {
+		microsecondTimestamp: number;
+		timestamp: number;
+		duration: number;
+		timestampIsValid: boolean;
+		durationIsValid: boolean;
+	}[] = [];
+
+	static override supports(codec: VideoCodec, config: VideoEncoderConfig): boolean {
+		return (codec === 'avc' || codec === 'hevc' || codec === 'vp8' || codec === 'vp9' || codec === 'av1')
+			&& !(config.alpha === 'keep' || config.bitrateMode === 'quantizer');
+	}
+
+	async init(): Promise<void> {
+		this.frame = new NodeAv.Frame();
+		this.frame.alloc();
+		this.frame.timeBase = new NodeAv.Rational(1, 1e6);
+
+		this.packet = new NodeAv.Packet();
+		this.packet.alloc();
+
+		const codecId = CODEC_TO_CODEC_ID[this.codec];
+
+		let codec: NodeAv.Codec | null = null;
+		if (this.config.hardwareAcceleration === 'prefer-software') {
+			codec = NodeAv.Codec.findEncoder(codecId);
+		} else {
+			codec = getHardwareEncoderCodec(codecId) ?? NodeAv.Codec.findEncoder(codecId);
+		}
+
+		if (!codec) {
+			throw new Error(`Unable to obtain libav codec for '${this.codec}'.`);
+		}
+
+		this.avCodec = codec;
+
+		await this.createCodecContext();
+	}
+
+	async createCodecContext() {
+		assert(this.codecContext === null);
+
+		const codecContext = new NodeAv.CodecContext();
+		codecContext.allocContext3(this.avCodec);
+
+		let pixelFormat = NodeAv.AV_PIX_FMT_YUV420P;
+		if (this.avCodec.pixelFormats && !this.avCodec.pixelFormats.includes(NodeAv.AV_PIX_FMT_YUV420P)) {
+			pixelFormat = this.avCodec.pixelFormats[0]!;
+		}
+
+		const pixelAspectRatio = simplifyRational({
+			num: (this.config.displayWidth ?? this.config.width) * this.config.height,
+			den: (this.config.displayHeight ?? this.config.height) * this.config.width,
+		});
+
+		codecContext.width = this.config.width;
+		codecContext.height = this.config.height;
+		codecContext.pixelFormat = pixelFormat;
+		codecContext.timeBase = new NodeAv.Rational(1, 1e6);
+		codecContext.gopSize = 60;
+		codecContext.framerate = new NodeAv.Rational(Math.round(this.config.framerate ?? 0) || 30, 1);
+		codecContext.bitRate = BigInt(
+			this.config.bitrate ?? QUALITY_MEDIUM._toVideoBitrate(this.codec, this.config.width, this.config.height),
+		);
+		codecContext.sampleAspectRatio = new NodeAv.Rational(pixelAspectRatio.num, pixelAspectRatio.den);
+
+		if (this.config.bitrateMode === 'constant') {
+			codecContext.rcMinRate = codecContext.bitRate;
+			codecContext.rcMaxRate = codecContext.bitRate;
+		}
+
+		const isRealtime = this.config.latencyMode === 'realtime';
+
+		if (this.avCodec.name === 'libx264') {
+			if (isRealtime) {
+				codecContext.setOption('tune', 'zerolatency');
+				codecContext.setOption('preset', 'ultrafast');
+			}
+		} else if (this.avCodec.name === 'libx265') {
+			codecContext.setOption('x265-params', 'log-level=error');
+
+			if (isRealtime) {
+				codecContext.setOption('tune', 'zerolatency');
+				codecContext.setOption('preset', 'ultrafast');
+			}
+		} else if (this.avCodec.name === 'libvpx') {
+			if (isRealtime) {
+				codecContext.setOption('deadline', 'realtime');
+				codecContext.setOption('cpu-used', '8');
+			} else {
+				codecContext.setOption('cpu-used', '8');
+			}
+		} else if (this.avCodec.name === 'libvpx-vp9') {
+			codecContext.setOption('deadline', 'realtime');
+
+			if (isRealtime) {
+				codecContext.setOption('cpu-used', '8');
+			} else {
+				codecContext.setOption('cpu-used', '5');
+			}
+		} else if (this.avCodec.name === 'libsvtav1') {
+			// SVTAV1 can be silenced by setting an environment variable:
+			// https://superuser.com/questions/1775236/how-to-remove-svt-av1-information-from-ffmpeg-output
+			process.env['SVT_LOG'] = '1';
+
+			if (isRealtime) {
+				codecContext.setOption('preset', '12');
+			}
+		}
+
+		const ret = await codecContext.open2();
+		NodeAv.FFmpegError.throwIfError(ret, 'Open codec context');
+
+		this.codecContext = codecContext;
+	}
+
+	async encode(videoSample: VideoSample, options: VideoEncoderEncodeOptions): Promise<void> {
+		if (this.codecContext === null) {
+			await this.createCodecContext();
+			assert(this.codecContext);
+		}
+
+		const underlyingData = videoSample.getUnderlyingData();
+
+		if (underlyingData instanceof NodeAvFrameVideoSampleResource) {
+			this.frame.ref(underlyingData.frame);
+		} else {
+			if (videoSample.format === null) {
+				throw new Error('Cannot encode foreign VideoSample with unknown (null) format.');
+			}
+
+			// Copy frame data from VideoSample to FFmpeg Frame
+			this.frame.format = fromPixelFormat(videoSample.format);
+			this.frame.width = videoSample.codedWidth;
+			this.frame.height = videoSample.codedHeight;
+			this.frame.pts = BigInt(videoSample.microsecondTimestamp);
+			this.frame.duration = BigInt(videoSample.microsecondDuration);
+			this.frame.colorPrimaries = mapColorPrimaries(videoSample.colorSpace.primaries ?? 'unknown')
+				?? NodeAv.AVCOL_PRI_UNSPECIFIED;
+			this.frame.colorSpace = mapMatrixCoefficients(videoSample.colorSpace.matrix ?? 'unknown')
+				?? NodeAv.AVCOL_SPC_UNSPECIFIED;
+			this.frame.colorTrc = mapTransferCharacteristics(videoSample.colorSpace.transfer ?? 'unknown')
+				?? NodeAv.AVCOL_TRC_UNSPECIFIED;
+			this.frame.colorRange = videoSample.colorSpace.fullRange === false
+				? NodeAv.AVCOL_RANGE_MPEG
+				: videoSample.colorSpace.fullRange === true
+					? NodeAv.AVCOL_RANGE_JPEG
+					: NodeAv.AVCOL_RANGE_UNSPECIFIED;
+
+			this.frame.keyFrame = options?.keyFrame ? 1 : 0;
+
+			const size = videoSample.allocationSize();
+			if (!this.lastBuffer || this.lastBuffer.byteLength !== size) {
+				this.lastBuffer = Buffer.from({ length: size });
+			}
+
+			await videoSample.copyTo(this.lastBuffer);
+			this.frame.fromBuffer(this.lastBuffer);
+		}
+
+		if (!this.colorSpaceSet) {
+			this.codecContext.colorPrimaries = this.frame.colorPrimaries;
+			this.codecContext.colorTrc = this.frame.colorTrc;
+			this.codecContext.colorSpace = this.frame.colorSpace;
+			this.codecContext.colorRange = this.frame.colorRange;
+
+			this.colorSpaceSet = true;
+		}
+
+		let frameToEncode = this.frame;
+
+		const requiresScaler
+			= this.codecContext.pixelFormat !== this.frame.format
+				|| this.codecContext.width !== this.frame.width
+				|| this.codecContext.height !== this.frame.height;
+
+		if (requiresScaler) {
+			if (!this.scaler) {
+				this.scaler = new NodeAv.SoftwareScaleContext();
+			}
+
+			const key = `${this.frame.width}x${this.frame.height}:${this.frame.format}`;
+			const needsConfigure = key !== this.lastScalerKey;
+
+			if (needsConfigure) {
+				this.scaler.getContext(
+					this.frame.width, this.frame.height, this.frame.format as NodeAv.AVPixelFormat,
+					this.codecContext.width, this.codecContext.height, this.codecContext.pixelFormat,
+					NodeAv.SWS_FAST_BILINEAR,
+				);
+
+				this.lastScalerKey = key;
+
+				const ret = this.scaler.initContext();
+				NodeAv.FFmpegError.throwIfError(ret, 'initContext');
+			}
+
+			if (!this.dstFrame) {
+				this.dstFrame = new NodeAv.Frame();
+				this.dstFrame.alloc();
+				this.dstFrame.width = this.codecContext.width;
+				this.dstFrame.height = this.codecContext.height;
+				this.dstFrame.format = this.codecContext.pixelFormat;
+				this.dstFrame.allocBuffer();
+			}
+
+			await this.scaler.scaleFrame(this.dstFrame, this.frame);
+			this.dstFrame.copyProps(this.frame);
+			frameToEncode = this.dstFrame;
+		}
+
+		const preciseTimingIndex = binarySearchLessOrEqual(
+			this.preciseTimings,
+			videoSample.microsecondTimestamp,
+			x => x.microsecondTimestamp,
+		);
+		const existingEntry = preciseTimingIndex !== -1
+			? this.preciseTimings[preciseTimingIndex]
+			: null;
+		if (existingEntry && existingEntry.microsecondTimestamp === videoSample.microsecondTimestamp) {
+			if (existingEntry.timestamp !== videoSample.timestamp) {
+				// Mapping isn't unique, can't use the timestamp
+				existingEntry.timestampIsValid = false;
+			}
+			if (existingEntry.duration !== videoSample.duration) {
+				// Mapping isn't unique, can't use the duration
+				existingEntry.durationIsValid = false;
+			}
+		} else {
+			this.preciseTimings.splice(preciseTimingIndex + 1, 0, {
+				microsecondTimestamp: videoSample.microsecondTimestamp,
+				timestamp: videoSample.timestamp,
+				duration: videoSample.duration,
+				timestampIsValid: true,
+				durationIsValid: true,
+			});
+
+			// Make sure it doesn't grow indefinitely
+			if (this.preciseTimings.length > 128) {
+				this.preciseTimings.shift();
+			}
+		}
+
+		const ret = await this.codecContext.sendFrame(frameToEncode);
+		NodeAv.FFmpegError.throwIfError(ret, 'Send frame');
+
+		// Keep receiving packets until no more are available for this frame
+		while (true) {
+			const receiveRet = await this.codecContext.receivePacket(this.packet);
+			if (receiveRet === NodeAv.AVERROR_EAGAIN || receiveRet === NodeAv.AVERROR_EOF) {
+				break;
+			}
+
+			this.receivePacket(ret);
+		}
+	}
+
+	receivePacket(ret: number) {
+		assert(this.codecContext);
+		NodeAv.FFmpegError.throwIfError(ret, 'Receive packet');
+
+		if (!this.packet.data) {
+			return;
+		}
+		let packetData = toUint8Array(this.packet.data);
+
+		let timestamp = Number(this.packet.pts) / 1e6;
+		let duration = Number(this.packet.duration) / 1e6;
+
+		const preciseTimingIndex = binarySearchLessOrEqual(
+			this.preciseTimings,
+			Number(this.packet.pts),
+			x => x.microsecondTimestamp,
+		);
+		const entry = preciseTimingIndex !== -1
+			? this.preciseTimings[preciseTimingIndex]
+			: null;
+
+		// If there's a relevant timing entry, refine the packet's timing data to get better accuracy than
+		// microseconds
+		if (entry && entry.microsecondTimestamp === Number(this.packet.pts)) {
+			if (entry.timestampIsValid) {
+				timestamp = entry.timestamp;
+			}
+			if (entry.durationIsValid) {
+				duration = entry.duration;
+			}
+		}
+
+		const metadata: EncodedVideoChunkMetadata = {};
+		let decoderConfigCodecString: string | null = null;
+		let decoderConfigDescription: Uint8Array | null = null;
+
+		if (this.codec === 'avc' || this.codec === 'hevc') {
+			let expectsAnnexB = false;
+			if (this.codec === 'avc') {
+				expectsAnnexB = this.config.avc?.format === 'annexb';
+			} else {
+				// eslint-disable-next-line @stylistic/max-len
+				// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access
+				expectsAnnexB = (this.config as any).hevc?.format === 'annexb';
+			}
+
+			if (!this.packetEmitted) {
+				let serializedRecord: Uint8Array;
+
+				if (this.codec === 'avc') {
+					const record = extractAvcDecoderConfigurationRecord(this.packet.data);
+					if (!record) {
+						throw new Error('Invalid AVC data, could not extract decoder configuration record.');
+					}
+
+					serializedRecord = serializeAvcDecoderConfigurationRecord(record);
+				} else {
+					const record = extractHevcDecoderConfigurationRecord(this.packet.data);
+					if (!record) {
+						throw new Error('Invalid HEVC data, could not extract decoder configuration record.');
+					}
+
+					serializedRecord = serializeHevcDecoderConfigurationRecord(record);
+				}
+
+				decoderConfigCodecString = extractVideoCodecString({
+					width: this.config.width,
+					height: this.config.height,
+					codec: this.codec,
+					codecDescription: serializedRecord,
+					colorSpace: null,
+					avcType: 1,
+					avcCodecInfo: null,
+					hevcCodecInfo: null,
+					vp9CodecInfo: null,
+					av1CodecInfo: null,
+				});
+
+				if (!expectsAnnexB) {
+					decoderConfigDescription = serializedRecord;
+				}
+			}
+
+			if (!expectsAnnexB) {
+				const NAL_UNIT_LENGTH_SIZE = 4;
+
+				const nalUnits: NalUnitLocation[] = [];
+				for (const loc of iterateNalUnitsInAnnexB(packetData)) {
+					if (this.codec === 'avc') {
+						const naluType = extractNalUnitTypeForAvc(packetData[loc.offset]!);
+
+						// Certain NALUs get stripped
+						if (
+							naluType !== AvcNalUnitType.SPS
+							&& naluType !== AvcNalUnitType.PPS
+							&& naluType !== AvcNalUnitType.SPS_EXT
+						) {
+							nalUnits.push(loc);
+						}
+					} else {
+						const naluType = extractNalUnitTypeForHevc(packetData[loc.offset]!);
+
+						// Certain NALUs get stripped
+						if (
+							naluType !== HevcNalUnitType.SPS_NUT
+							&& naluType !== HevcNalUnitType.PPS_NUT
+							&& naluType !== HevcNalUnitType.VPS_NUT
+						) {
+							nalUnits.push(loc);
+						}
+					}
+				}
+
+				let totalSize = 0;
+				for (const nalUnit of nalUnits) {
+					totalSize += NAL_UNIT_LENGTH_SIZE + nalUnit.length;
+				}
+
+				const lengthPrefixedData = new Uint8Array(totalSize);
+				const dataView = new DataView(lengthPrefixedData.buffer);
+				let offset = 0;
+
+				// Write each NAL unit with its length prefix
+				for (const nalUnit of nalUnits) {
+					const length = nalUnit.length;
+
+					dataView.setUint32(offset, length, false);
+					offset += 4;
+
+					lengthPrefixedData.set(
+						packetData.subarray(nalUnit.offset, nalUnit.offset + nalUnit.length),
+						offset,
+					);
+					offset += nalUnit.length;
+				}
+
+				packetData = lengthPrefixedData;
+			}
+		} else if (this.codec === 'vp8') {
+			if (!this.packetEmitted) {
+				decoderConfigCodecString = extractVideoCodecString({
+					width: this.config.width,
+					height: this.config.height,
+					codec: 'vp8',
+					codecDescription: null,
+					colorSpace: null,
+					avcType: null,
+					avcCodecInfo: null,
+					hevcCodecInfo: null,
+					vp9CodecInfo: null,
+					av1CodecInfo: null,
+				});
+			}
+		} else if (this.codec === 'vp9') {
+			if (!this.packetEmitted) {
+				const vp9CodecInfo = extractVp9CodecInfoFromPacket(packetData);
+
+				decoderConfigCodecString = extractVideoCodecString({
+					width: this.config.width,
+					height: this.config.height,
+					codec: 'vp9',
+					codecDescription: null,
+					colorSpace: null,
+					avcType: null,
+					avcCodecInfo: null,
+					hevcCodecInfo: null,
+					vp9CodecInfo,
+					av1CodecInfo: null,
+				});
+			}
+		} else if (this.codec === 'av1') {
+			if (!this.packetEmitted) {
+				const av1CodecInfo = extractAv1CodecInfoFromPacket(packetData);
+
+				decoderConfigCodecString = extractVideoCodecString({
+					width: this.config.width,
+					height: this.config.height,
+					codec: 'av1',
+					codecDescription: null,
+					colorSpace: null,
+					avcType: null,
+					avcCodecInfo: null,
+					hevcCodecInfo: null,
+					vp9CodecInfo: null,
+					av1CodecInfo,
+				});
+			}
+		} else {
+			throw new Error('Unreachable.');
+		}
+
+		const packet = new EncodedPacket(
+			packetData,
+			this.packet.isKeyframe ? 'key' : 'delta',
+			timestamp,
+			duration,
+		);
+
+		if (decoderConfigCodecString !== null) {
+			// Create the decoder config
+			metadata.decoderConfig = {
+				codec: decoderConfigCodecString,
+				codedWidth: this.codecContext.width,
+				codedHeight: this.codecContext.height,
+				displayAspectWidth: this.config.displayWidth ?? this.codecContext.width,
+				displayAspectHeight: this.config.displayHeight ?? this.codecContext.height,
+				description: decoderConfigDescription ?? undefined,
+				colorSpace: {
+					primaries:
+							unmapColorPrimaries(this.codecContext.colorPrimaries) as VideoColorPrimaries,
+					matrix:
+							unmapMatrixCoefficients(this.codecContext.colorSpace) as VideoMatrixCoefficients,
+					transfer:
+							unmapTransferCharacteristics(this.codecContext.colorTrc) as VideoTransferCharacteristics,
+					fullRange: this.codecContext.colorRange === NodeAv.AVCOL_RANGE_JPEG
+						? true
+						: this.codecContext.colorRange === NodeAv.AVCOL_RANGE_MPEG
+							? false
+							: undefined,
+				},
+			};
+		}
+
+		this.onPacket(packet, metadata);
+		this.packetEmitted = true;
+	}
+
+	async flush(): Promise<void> {
+		if (this.codecContext) {
+			// Send null frame to signal flush
+			const ret = await this.codecContext.sendFrame(null);
+			NodeAv.FFmpegError.throwIfError(ret, 'Send frame');
+
+			// Keep receiving packets until no more are available
+			while (true) {
+				const receiveRet = await this.codecContext.receivePacket(this.packet);
+				if (receiveRet === NodeAv.AVERROR_EAGAIN || receiveRet === NodeAv.AVERROR_EOF) {
+					break;
+				}
+
+				this.receivePacket(receiveRet);
+			}
+
+			this.codecContext.freeContext();
+			this.codecContext = null;
+			// The codec is done now and can't be reused. Any subsequence encode call will first need to recreate a
+			// codec context.
+		}
+
+		this.packetEmitted = false;
+		this.colorSpaceSet = false;
+	}
+
+	close(): MaybePromise<void> {
+		this.codecContext?.freeContext();
+		this.frame.free();
+		this.packet.free();
+		this.scaler?.freeContext();
+		this.dstFrame?.free();
+	}
+}

packages/server/src/video-sample.ts

@@ -0,0 +1,520 @@
+import assert from 'assert';
+import { Rectangle, VideoSamplePixelFormat } from 'mediabunny';
+import { VideoSampleColorSpace } from 'mediabunny';
+import { VideoSampleResource } from 'mediabunny';
+import * as NodeAv from 'node-av';
+import { toUint8Array } from '../../../src/misc';
+import { getPlaneConfigs } from '../../../src/sample';
+import {
+	toPixelFormat,
+	unmapColorPrimaries,
+	unmapTransferCharacteristics,
+	unmapMatrixCoefficients,
+	fromPixelFormat,
+	mapColorPrimaries,
+	mapTransferCharacteristics,
+	mapMatrixCoefficients,
+} from './misc';
+
+export class NodeAvFrameVideoSampleResource extends VideoSampleResource {
+	frame: NodeAv.Frame;
+
+	constructor(frame: NodeAv.Frame) {
+		super();
+
+		const clone = frame.clone();
+		if (!clone) {
+			throw new Error('Allocation failure during frame clone.');
+		}
+
+		this.frame = clone;
+	}
+
+	getFormat(): VideoSamplePixelFormat | null {
+		return toPixelFormat(this.frame.format as NodeAv.AVPixelFormat);
+	}
+
+	getCodedWidth(): number {
+		return this.frame.width;
+	}
+
+	getCodedHeight(): number {
+		return this.frame.height;
+	}
+
+	getSquarePixelWidth(): number {
+		if (this.frame.sampleAspectRatio.num > this.frame.sampleAspectRatio.den) {
+			return Math.round(this.frame.width * this.frame.sampleAspectRatio.num / this.frame.sampleAspectRatio.den);
+		} else {
+			return this.frame.width;
+		}
+	}
+
+	getSquarePixelHeight(): number {
+		if (this.frame.sampleAspectRatio.num > this.frame.sampleAspectRatio.den) {
+			return this.frame.height;
+		} else {
+			return Math.round(this.frame.height * this.frame.sampleAspectRatio.den / this.frame.sampleAspectRatio.num);
+		}
+	}
+
+	getColorSpace(): VideoSampleColorSpace {
+		return new VideoSampleColorSpace({
+			primaries: unmapColorPrimaries(this.frame.colorPrimaries) as VideoColorPrimaries | null,
+			transfer: unmapTransferCharacteristics(this.frame.colorTrc) as VideoTransferCharacteristics | null,
+			matrix: unmapMatrixCoefficients(this.frame.colorSpace) as VideoMatrixCoefficients | null,
+			fullRange: this.frame.colorRange === NodeAv.AVCOL_RANGE_JPEG,
+		});
+	}
+
+	close(): void {
+		this.frame.free();
+	}
+
+	allocationSize(options: VideoFrameCopyToOptions): number {
+		// 3. Let combinedLayout be the result of running the Parse VideoFrameCopyToOptions algorithm with options.
+		// 4. If combinedLayout is an exception, throw combinedLayout.
+		const combinedLayout = ParseVideoFrameCopyToOptions(this, options);
+
+		// 5. Return combinedLayout's allocationSize.
+		return combinedLayout.allocationSize;
+	}
+
+	async copyTo(
+		destination: AllowSharedBufferSource,
+		options: VideoFrameCopyToOptions,
+	): Promise<PlaneLayout[]> {
+		const format = this.getFormat();
+		assert(format !== null);
+
+		// 3. Let combinedLayout be the result of running the Parse VideoFrameCopyToOptions algorithm with options.
+		const combinedLayout = ParseVideoFrameCopyToOptions(this, options);
+
+		// 4. If destination.byteLength is less than combinedLayout’s allocationSize, return a promise rejected with
+		// // eslint-disable-next-line @stylistic/max-len
+		const destBytes = toUint8Array(destination);
+		if (destBytes.byteLength < combinedLayout.allocationSize) {
+			throw new TypeError(
+				`Destination buffer too small. Required: ${combinedLayout.allocationSize},`
+				+ ` Available: ${destBytes.byteLength}`,
+			);
+		}
+
+		// 5. If options.format is equal to one of RGBA, RGBX, BGRA, BGRX then:
+		if (options.format && ['RGBA', 'RGBX', 'BGRA', 'BGRX'].includes(options.format) && options.format !== format) {
+			// Let newOptions be the result of running the Clone Configuration algorithm with options.
+			// Assign undefined to newOptions.format.
+			const newOptions = { ...options };
+			delete newOptions.format;
+
+			// Let rgbFrame be the result of running the Convert to RGB frame algorithm with this, options.format,
+			// and options.colorSpace.
+			const rgbFrame = await ConvertToRGBFrame(this, options.format, options.colorSpace);
+
+			// Return the result of calling copyTo() on rgbFrame with destination and newOptions.
+			try {
+				const result = await rgbFrame.copyTo(destination, newOptions);
+				rgbFrame.close();
+				return result;
+			} catch (e) {
+				rgbFrame.close();
+				throw e;
+			}
+		}
+
+		// 6. Let p be a new Promise. (Implicit)
+		// 7. Let copyStepsQueue be the result of starting a new parallel queue. (Implicit)
+		// 8. Let planeLayouts be a new list.
+		const planeLayouts: PlaneLayout[] = [];
+
+		// Enqueue the following steps to copyStepsQueue: (fuck the queuing part)
+
+		// Let resource be the media resource referenced by [[resource reference]].
+		// (this.data)
+
+		// Let numPlanes be the number of planes as defined by [[format]].
+		const planes = getPlaneConfigs(format);
+		const numPlanes = planes.length;
+
+		// Let planeIndex be 0.
+		// While planeIndex is less than combinedLayout’s numPlanes:
+		for (let planeIndex = 0; planeIndex < numPlanes; planeIndex++) {
+			const computedLayout = combinedLayout.computedLayouts[planeIndex]!;
+
+			// Let sourceStride be the stride of the plane in resource as identified by planeIndex.
+			let sourceStride = 0;
+			let sourceStartOffset = 0;
+
+			sourceStride = this.frame.linesize[planeIndex]!;
+			const planeBuffer = this.frame.data![planeIndex]!;
+			if (!planeBuffer) {
+				throw new Error(`Missing data for plane ${planeIndex}`);
+			}
+			const sourceData = new Uint8Array(planeBuffer.buffer, planeBuffer.byteOffset, planeBuffer.byteLength);
+			sourceStartOffset = 0;
+
+			// Let sourceOffset be the product of multiplying computedLayout’s sourceTop by sourceStride
+			let sourceOffset = computedLayout.sourceTop * sourceStride;
+
+			// Add computedLayout’s sourceLeftBytes to sourceOffset.
+			sourceOffset += computedLayout.sourceLeftBytes;
+
+			// Adjust for the base offset of the plane in the source buffer
+			sourceOffset += sourceStartOffset;
+
+			// Let destinationOffset be computedLayout’s destinationOffset.
+			let destinationOffset = computedLayout.destinationOffset;
+
+			// Let rowBytes be computedLayout’s sourceWidthBytes.
+			const rowBytes = computedLayout.sourceWidthBytes;
+
+			// Let layout be a new PlaneLayout, with offset set to destinationOffset and stride set to rowBytes.
+			// This is a spec error actually (https://github.com/w3c/webcodecs/issues/918)
+			const layout: PlaneLayout = {
+				offset: destinationOffset,
+				stride: computedLayout.destinationStride,
+			};
+
+			// Let row be 0.
+			// While row is less than computedLayout’s sourceHeight:
+			for (let row = 0; row < computedLayout.sourceHeight; row++) {
+				// Copy rowBytes bytes from resource starting at sourceOffset to destination starting
+				// at destinationOffset.
+				if (sourceOffset + rowBytes > sourceData.byteLength) {
+					throw new Error(`Source buffer OOB read`);
+				}
+				if (destinationOffset + rowBytes > destBytes.byteLength) {
+					throw new Error(`Destination buffer OOB write`);
+				}
+
+				const srcSub = sourceData.subarray(sourceOffset, sourceOffset + rowBytes);
+				destBytes.set(srcSub, destinationOffset);
+
+				// Increment sourceOffset by sourceStride.
+				sourceOffset += sourceStride;
+
+				// Increment destinationOffset by computedLayout’s destinationStride.
+				destinationOffset += computedLayout.destinationStride;
+			}
+
+			// Append layout to planeLayouts.
+			planeLayouts.push(layout);
+		}
+
+		// Queue a task to resolve p with planeLayouts.
+		return planeLayouts;
+	}
+}
+
+type CombinedBufferLayout = {
+	allocationSize: number;
+	computedLayouts: ComputedPlaneLayout[];
+};
+
+type ComputedPlaneLayout = {
+	destinationOffset: number;
+	destinationStride: number;
+	sourceTop: number;
+	sourceHeight: number;
+	sourceLeftBytes: number;
+	sourceWidthBytes: number;
+};
+
+// Taken from the WebCodecs spec
+const ParseVideoFrameCopyToOptions = (
+	frame: VideoSampleResource,
+	options: VideoFrameCopyToOptions,
+): CombinedBufferLayout => {
+	// 1. Let defaultRect be the result of performing the getter steps for visibleRect.
+	const defaultRect: Rectangle = {
+		left: 0,
+		top: 0,
+		width: frame.getCodedWidth(),
+		height: frame.getCodedHeight(),
+	};
+
+	// 2. Let overrideRect be undefined.
+	// 3. If options.rect exists, assign the value of options.rect to overrideRect.
+	const overrideRect = options.rect;
+
+	// 4. Let parsedRect be the result of running the Parse Visible Rect algorithm...
+	const parsedRect = ParseVisibleRect(
+		defaultRect,
+		overrideRect,
+		frame.getCodedWidth(),
+		frame.getCodedHeight(),
+		frame.getFormat(),
+	);
+
+	// 5. If parsedRect is an exception, return parsedRect. (Handled by throw)
+
+	// 6. Let optLayout be undefined.
+	// 7. If options.layout exists, assign its value to optLayout.
+	const optLayout = options.layout;
+
+	// 8. Let format be undefined.
+	let format: VideoSamplePixelFormat | undefined;
+
+	// 9. If options.format does not exist, assign [[format]] to format.
+	if (!options.format || options.format === frame.getFormat()) {
+		format = frame.getFormat()!;
+	} else if (['RGBA', 'RGBX', 'BGRA', 'BGRX'].includes(options.format)) {
+		// 10. Otherwise, if options.format is equal to one of RGBA, RGBX, BGRA, BGRX, then assign options.format
+		//  to format...
+		format = options.format;
+	} else {
+		throw new Error('NotSupportedError: Invalid destination format');
+	}
+
+	// 11. Let combinedLayout be the result of running the Compute Layout and Allocation Size algorithm...
+	return ComputeLayoutAndAllocationSize(parsedRect, format, optLayout);
+};
+
+// Taken from the WebCodecs spec
+const ParseVisibleRect = (
+	defaultRect: DOMRectInit,
+	overrideRect: DOMRectInit | undefined,
+	codedWidth: number,
+	codedHeight: number,
+	format: VideoSamplePixelFormat | null,
+): DOMRectInit => {
+	// 1. Let sourceRect be defaultRect
+	const sourceRect = { ...defaultRect };
+
+	// 2. If overrideRect is not undefined:
+	if (overrideRect !== undefined) {
+		// If either of overrideRect.width or height is 0, return a TypeError.
+		if (overrideRect.width === 0 || overrideRect.height === 0) {
+			throw new TypeError('visibleRect dimensions cannot be zero');
+		}
+		// If the sum of overrideRect.x and overrideRect.width is greater than codedWidth, return a TypeError.
+		if ((overrideRect.x || 0) + (overrideRect.width || 0) > codedWidth) {
+			throw new TypeError('visibleRect exceeds codedWidth');
+		}
+		// If the sum of overrideRect.y and overrideRect.height is greater than codedHeight, return a TypeError.
+		if ((overrideRect.y || 0) + (overrideRect.height || 0) > codedHeight) {
+			throw new TypeError('visibleRect exceeds codedHeight');
+		}
+		// Assign overrideRect to sourceRect.
+		sourceRect.x = overrideRect.x || 0;
+		sourceRect.y = overrideRect.y || 0;
+		sourceRect.width = overrideRect.width || 0;
+		sourceRect.height = overrideRect.height || 0;
+	}
+
+	// 3. Let validAlignment be the result of running the Verify Rect Offset Alignment algorithm.
+	const validAlignment = VerifyRectOffsetAlignment(format, sourceRect);
+
+	// 4. If validAlignment is false, throw a TypeError.
+	if (!validAlignment) {
+		throw new TypeError('visibleRect alignment is invalid for the format');
+	}
+
+	// 5. Return sourceRect.
+	return sourceRect;
+};
+
+// Taken from the WebCodecs spec
+const VerifyRectOffsetAlignment = (format: VideoSamplePixelFormat | null, rect: DOMRectInit): boolean => {
+	// 1. If format is null, return true.
+	if (format === null) return true;
+
+	const planes = getPlaneConfigs(format);
+
+	// 2. Let planeIndex be 0.
+	// 3. Let numPlanes be the number of planes as defined by format.
+	// 4. While planeIndex is less than numPlanes:
+	for (let planeIndex = 0; planeIndex < planes.length; planeIndex++) {
+		const plane = planes[planeIndex]!;
+		const sampleWidth = plane.widthDivisor;
+		const sampleHeight = plane.heightDivisor;
+
+		// If rect.x is not a multiple of sampleWidth, return false.
+		if ((rect.x || 0) % sampleWidth !== 0) return false;
+		// If rect.y is not a multiple of sampleHeight, return false.
+		if ((rect.y || 0) % sampleHeight !== 0) return false;
+	}
+
+	return true;
+};
+
+// Taken from the WebCodecs spec
+const ComputeLayoutAndAllocationSize = (
+	parsedRect: DOMRectInit,
+	format: VideoSamplePixelFormat,
+	layout?: PlaneLayout[],
+): CombinedBufferLayout => {
+	const planes = getPlaneConfigs(format);
+
+	// 1. Let numPlanes be the number of planes as defined by format.
+	const numPlanes = planes.length;
+
+	// 2. If layout is not undefined and its length does not equal numPlanes, throw a TypeError.
+	if (layout !== undefined && layout.length !== numPlanes) {
+		throw new TypeError(`Layout must have ${numPlanes} planes`);
+	}
+
+	// 3. Let minAllocationSize be 0.
+	let minAllocationSize = 0;
+
+	// 4. Let computedLayouts be a new list.
+	const computedLayouts: ComputedPlaneLayout[] = [];
+
+	// 5. Let endOffsets be a new list.
+	const endOffsets: number[] = [];
+
+	// 6. Let planeIndex be 0.
+	// 7. While planeIndex < numPlanes:
+	for (let planeIndex = 0; planeIndex < numPlanes; planeIndex++) {
+		const plane = planes[planeIndex]!;
+		const sampleBytes = plane.sampleBytes;
+		const sampleWidth = plane.widthDivisor;
+		const sampleHeight = plane.heightDivisor;
+
+		// Let computedLayout be a new computed plane layout.
+		const computedLayout: ComputedPlaneLayout = {
+			destinationOffset: 0,
+			destinationStride: 0,
+			sourceTop: 0,
+			sourceHeight: 0,
+			sourceLeftBytes: 0,
+			sourceWidthBytes: 0,
+		};
+
+		// Set computedLayout’s sourceTop...
+		computedLayout.sourceTop = Math.ceil(Math.trunc(parsedRect.y || 0) / sampleHeight);
+		// Set computedLayout’s sourceHeight...
+		computedLayout.sourceHeight = Math.ceil(Math.trunc(parsedRect.height || 0) / sampleHeight);
+		// Set computedLayout’s sourceLeftBytes...
+		computedLayout.sourceLeftBytes = Math.floor(Math.trunc(parsedRect.x || 0) / sampleWidth) * sampleBytes;
+		// Set computedLayout’s sourceWidthBytes...
+		computedLayout.sourceWidthBytes = Math.floor(Math.trunc(parsedRect.width || 0) / sampleWidth) * sampleBytes;
+
+		// If layout is not undefined:
+		if (layout !== undefined) {
+			const planeLayout = layout[planeIndex]!;
+			// If planeLayout.stride is less than computedLayout’s sourceWidthBytes, return a TypeError.
+			if (planeLayout.stride < computedLayout.sourceWidthBytes) {
+				throw new TypeError(`Stride for plane ${planeIndex} is too small`);
+			}
+			// Assign planeLayout.offset to computedLayout’s destinationOffset.
+			computedLayout.destinationOffset = planeLayout.offset;
+			// Assign planeLayout.stride to computedLayout’s destinationStride.
+			computedLayout.destinationStride = planeLayout.stride;
+		} else {
+			// Otherwise:
+			// Assign minAllocationSize to computedLayout’s destinationOffset.
+			computedLayout.destinationOffset = minAllocationSize;
+			// Assign computedLayout’s sourceWidthBytes to computedLayout’s destinationStride.
+			computedLayout.destinationStride = computedLayout.sourceWidthBytes;
+		}
+
+		// Let planeSize be the product of multiplying computedLayout’s destinationStride and sourceHeight.
+		const planeSize = computedLayout.destinationStride * computedLayout.sourceHeight;
+
+		// Let planeEnd be the sum of planeSize and computedLayout’s destinationOffset.
+		const planeEnd = planeSize + computedLayout.destinationOffset;
+
+		// If planeSize or planeEnd is greater than maximum range of unsigned long, return a TypeError.
+		if (planeEnd > 4294967295) throw new TypeError('Allocation size exceeds limit');
+
+		// Append planeEnd to endOffsets.
+		endOffsets.push(planeEnd);
+
+		// Assign the maximum of minAllocationSize and planeEnd to minAllocationSize.
+		minAllocationSize = Math.max(minAllocationSize, planeEnd);
+
+		// Check for overlap
+		for (let earlierPlaneIndex = 0; earlierPlaneIndex < planeIndex; earlierPlaneIndex++) {
+			const earlierLayout = computedLayouts[earlierPlaneIndex]!;
+			// If plane A ends before plane B starts, they do not overlap.
+			if (
+				endOffsets[planeIndex]! <= earlierLayout.destinationOffset
+				|| endOffsets[earlierPlaneIndex]! <= computedLayout.destinationOffset
+			) {
+				continue;
+			}
+			throw new TypeError('Planes overlap');
+		}
+
+		computedLayouts.push(computedLayout);
+	}
+
+	// 12. Return combinedLayout.
+	return {
+		allocationSize: minAllocationSize,
+		computedLayouts: computedLayouts,
+	};
+};
+
+// Taken from the WebCodecs spec
+const ConvertToRGBFrame = async (
+	frame: NodeAvFrameVideoSampleResource,
+	format: VideoPixelFormat,
+	colorSpace?: PredefinedColorSpace,
+): Promise<NodeAvFrameVideoSampleResource> => {
+	// 1. Let convertedFrame be a new VideoFrame...
+	// (We construct it at the end, but we prepare the resource here)
+
+	const width = frame.getCodedWidth();
+	const height = frame.getCodedHeight();
+
+	const scaler = new NodeAv.SoftwareScaleContext();
+	const srcFmt = fromPixelFormat(frame.getFormat()!);
+	const dstFmt = fromPixelFormat(format);
+
+	// Configure scaling: Source -> Destination (Visible Size)
+	scaler.getContext(
+		width, height, srcFmt,
+		width, height, dstFmt,
+		NodeAv.SWS_BILINEAR,
+	);
+
+	// Allocate destination frame
+	const dstFrame = new NodeAv.Frame();
+	dstFrame.width = width;
+	dstFrame.height = height;
+	dstFrame.format = dstFmt;
+	dstFrame.alloc();
+	dstFrame.allocBuffer();
+
+	// Apply destination color space settings to dstFrame
+	// If colorSpace is not provided, srgb is used
+	const targetColorSpace = colorSpace || 'srgb';
+	if (targetColorSpace === 'srgb') {
+		dstFrame.colorPrimaries = NodeAv.AVCOL_PRI_BT709;
+		dstFrame.colorTrc = NodeAv.AVCOL_TRC_IEC61966_2_1;
+		dstFrame.colorSpace = NodeAv.AVCOL_SPC_RGB;
+		dstFrame.colorRange = NodeAv.AVCOL_RANGE_JPEG;
+	} else if (targetColorSpace === 'display-p3') {
+		dstFrame.colorPrimaries = NodeAv.AVCOL_PRI_SMPTE432;
+		dstFrame.colorTrc = NodeAv.AVCOL_TRC_IEC61966_2_1;
+		dstFrame.colorSpace = NodeAv.AVCOL_SPC_RGB;
+		dstFrame.colorRange = NodeAv.AVCOL_RANGE_JPEG;
+	}
+
+	const srcFrame = frame.frame;
+
+	// Apply source color space settings to srcFrame (if known)
+	// This ensures the scaler knows the input color space for correct conversion
+	const frameColorSpace = frame.getColorSpace();
+	if (colorSpace) {
+		srcFrame.colorPrimaries = mapColorPrimaries(frameColorSpace.primaries ?? 'unknown')
+			?? NodeAv.AVCOL_PRI_UNSPECIFIED;
+		srcFrame.colorTrc = mapTransferCharacteristics(frameColorSpace.transfer ?? 'unknown')
+			?? NodeAv.AVCOL_TRC_UNSPECIFIED;
+		srcFrame.colorSpace = mapMatrixCoefficients(frameColorSpace.matrix ?? 'unknown')
+			?? NodeAv.AVCOL_SPC_UNSPECIFIED;
+		srcFrame.colorRange = frameColorSpace.fullRange
+			? NodeAv.AVCOL_RANGE_JPEG
+			: NodeAv.AVCOL_RANGE_MPEG;
+	}
+
+	try {
+		await scaler.scaleFrame(dstFrame, srcFrame);
+	} finally {
+		scaler.freeContext();
+	}
+
+	return new NodeAvFrameVideoSampleResource(dstFrame);
+};