Add ProRes encode and decode support to @mediabunny/server

packages/prores/package.json

@@ -38,7 +38,7 @@
     "url": "https://github.com/sponsors/Vanilagy"
   },
   "dependencies": {
-    "turbores": "^1.0.0"
+    "turbores": "^1.1.0"
   },
   "peerDependencies": {
     "mediabunny": "^1.49.0"

packages/prores/src/index.ts

@@ -125,6 +125,19 @@ class ProresDecoder extends CustomVideoDecoder {
 			fullRange: result.colorRangeFull,
 		};
 
+		let displayWidth = result.visibleWidth;
+		let displayHeight = result.visibleHeight;
+
+		if (result.pixelAspectRatio.num > result.pixelAspectRatio.den) {
+			displayWidth = Math.round(
+				result.visibleWidth * result.pixelAspectRatio.num / result.pixelAspectRatio.den,
+			);
+		} else {
+			displayHeight = Math.round(
+				result.visibleHeight * result.pixelAspectRatio.den / result.pixelAspectRatio.num,
+			);
+		}
+
 		let sample: VideoSample;
 		if (typeof VideoFrame !== 'undefined') {
 			// Create a VideoFrame directly; this avoids the frame data being copied twice
@@ -132,6 +145,8 @@ class ProresDecoder extends CustomVideoDecoder {
 				format: result.pixelFormat as VideoPixelFormat,
 				codedWidth: result.codedWidth,
 				codedHeight: result.codedHeight,
+				displayWidth,
+				displayHeight,
 				visibleRect: {
 					x: 0,
 					y: 0,
@@ -152,6 +167,8 @@ class ProresDecoder extends CustomVideoDecoder {
 				format: result.pixelFormat,
 				codedWidth: result.codedWidth,
 				codedHeight: result.codedHeight,
+				displayWidth,
+				displayHeight,
 				visibleRect: {
 					left: 0,
 					top: 0,

packages/server/README.md

@@ -12,7 +12,7 @@
 By default, Mediabunny requires a browser environment for full access to decoders, encoders, and video processing features. `@mediabunny/server` uses [NodeAV](https://github.com/seydx/node-av) to polyfill this functionality for server-side environments such as Node, Bun, or Deno, enabling the usage of all Mediabunny features on the server. The result is a server-side media processing API that integrates naturally with TypeScript as opposed to the awkwardness and inefficiencies of calling out to the FFmpeg CLI.
 
 Features added by this package include:
-- Video decoders and encoders for AVC (H.264), HEVC (H.265), VP8, VP9, and AV1. Supports both length-prefixed and Annex B AVC/HEVC as well as transparent video via VP9.
+- Video decoders and encoders for AVC (H.264), HEVC (H.265), VP8, VP9, AV1, and ProRes. Supports both length-prefixed and Annex B AVC/HEVC as well as transparent video via VP9 and ProRes.
 - Audio decoders and encoders for AAC, MP3, Vorbis, Opus, FLAC, AC-3 and E-AC-3. Supports AAC in both AAC and ADTS formats.
 - Video frame transformation support (resize, rotate, crop)
 - Automatic hardware acceleration on all platforms (macOS, Linux, Windows)
@@ -321,6 +321,8 @@ For encoding, video frames and audio samples are transferred to FFmpeg by conver
 
 Whenever possible, `AVFrame`s are never copied over to JavaScript unless explicitly needed. This enables zero-copy decode -> transformation -> encode paths.
 
+ProRes decoding support is provided by [TurboRes](https://github.com/Vanilagy/turbores), as it is often faster than FFmpeg.
+
 ## License
 
 `@mediabunny/server` uses the same MPL-2.0 license as Mediabunny.

packages/server/package.json

@@ -34,7 +34,8 @@
     "url": "https://github.com/sponsors/Vanilagy"
   },
   "dependencies": {
-    "node-av": "^6.0.0"
+    "node-av": "^6.0.0",
+    "@mediabunny/prores": "^1.50.0"
   },
   "peerDependencies": {
     "mediabunny": "^1.45.0"

packages/server/src/index.ts

@@ -22,6 +22,7 @@ import { NodeAvAudioDecoder } from './audio-decoder';
 import { NodeAvAudioEncoder } from './audio-encoder';
 import { copyVideoSampleToAvFrame, AvFrameVideoSampleResource, transformVideoSample } from './video-sample';
 import { copyAudioSampleToAvFrame, AvFrameAudioSampleResource } from './audio-sample';
+import { registerProresDecoder } from '@mediabunny/prores';
 
 const SERVER_LOADED_SYMBOL = Symbol.for('@mediabunny/server loaded');
 if ((globalThis as Record<symbol, unknown>)[SERVER_LOADED_SYMBOL]) {
@@ -100,6 +101,7 @@ export const registerMediabunnyServer = (options: MediabunnyServerOptions = {})
 
 	// Video
 	registerDecoder(NodeAvVideoDecoder);
+	registerProresDecoder(); // Faster than FFmpeg, so we use it
 	registerEncoder(NodeAvVideoEncoder);
 
 	// Audio

packages/server/src/misc.ts

@@ -16,6 +16,7 @@ export const CODEC_TO_CODEC_ID: Partial<Record<MediaCodec, NodeAv.AVCodecID>> =
 	vp8: NodeAv.AV_CODEC_ID_VP8,
 	vp9: NodeAv.AV_CODEC_ID_VP9,
 	av1: NodeAv.AV_CODEC_ID_AV1,
+	prores: NodeAv.AV_CODEC_ID_PRORES,
 
 	aac: NodeAv.AV_CODEC_ID_AAC,
 	opus: NodeAv.AV_CODEC_ID_OPUS,

packages/server/src/video-encoder.ts

@@ -38,9 +38,18 @@ import {
 	serializeAvcDecoderConfigurationRecord,
 	serializeHevcDecoderConfigurationRecord,
 } from '../../../src/codec-data';
-import { extractVideoCodecString } from '../../../src/codec';
+import { extractVideoCodecString, ProresFourCc } from '../../../src/codec';
 import { assert, binarySearchLessOrEqual, simplifyRational, toUint8Array } from '../../../src/misc';
 
+const PRORES_FOURCC_TO_PROFILE: Record<ProresFourCc, NodeAv.AVProfile> = {
+	apco: NodeAv.AV_PROFILE_PRORES_PROXY,
+	apcs: NodeAv.AV_PROFILE_PRORES_LT,
+	apcn: NodeAv.AV_PROFILE_PRORES_STANDARD,
+	apch: NodeAv.AV_PROFILE_PRORES_HQ,
+	ap4h: NodeAv.AV_PROFILE_PRORES_4444,
+	ap4x: NodeAv.AV_PROFILE_PRORES_XQ,
+};
+
 export class NodeAvVideoEncoder extends CustomVideoEncoder {
 	frame!: NodeAv.Frame;
 	packet!: NodeAv.Packet;
@@ -62,8 +71,10 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
 	}[] = [];
 
 	static override supports(codec: VideoCodec, config: VideoEncoderConfig): boolean {
-		return (codec === 'avc' || codec === 'hevc' || codec === 'vp8' || codec === 'vp9' || codec === 'av1')
-			&& config.bitrateMode !== 'quantizer';
+		return (
+			codec === 'avc' || codec === 'hevc' || codec === 'vp8' || codec === 'vp9' || codec === 'av1'
+			|| codec === 'prores'
+		) && config.bitrateMode !== 'quantizer';
 	}
 
 	async init(): Promise<void> {
@@ -77,13 +88,25 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
 		const codecId = CODEC_TO_CODEC_ID[this.codec];
 		assert(codecId !== undefined);
 
+		const getSoftwareCodec = () => {
+			if (this.codec === 'prores') {
+				// Prefer prores_ks for ProRes
+				const proresKs = NodeAv.Codec.findEncoderByName(NodeAv.FF_ENCODER_PRORES_KS);
+				if (proresKs) {
+					return proresKs;
+				}
+			}
+
+			return NodeAv.Codec.findEncoder(codecId);
+		};
+
 		let codec: NodeAv.Codec | null = null;
 		if (this.codec === 'vp9' && this.config.alpha === 'keep') {
 			codec = NodeAv.Codec.findEncoderByName(NodeAv.FF_ENCODER_LIBVPX_VP9) ?? NodeAv.Codec.findEncoder(codecId);
 		} else if (this.config.hardwareAcceleration === 'prefer-software') {
-			codec = NodeAv.Codec.findEncoder(codecId);
+			codec = getSoftwareCodec();
 		} else {
-			codec = (await getHardwareEncoderCodec(codecId)) ?? NodeAv.Codec.findEncoder(codecId);
+			codec = (await getHardwareEncoderCodec(codecId)) ?? getSoftwareCodec();
 		}
 
 		if (!codec) {
@@ -108,8 +131,17 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
 				pixelFormat = this.avCodec.pixelFormats[0]!;
 			}
 
-			if (this.config.alpha === 'keep' && this.avCodec.pixelFormats.includes(NodeAv.AV_PIX_FMT_YUVA420P)) {
-				pixelFormat = NodeAv.AV_PIX_FMT_YUVA420P;
+			if (this.config.alpha === 'keep') {
+				if (this.avCodec.pixelFormats.includes(NodeAv.AV_PIX_FMT_YUVA420P)) {
+					pixelFormat = NodeAv.AV_PIX_FMT_YUVA420P;
+				} else {
+					// Let FFmpeg pick the best alpha-capable format it supports, minimizing data loss versus a
+					// high-quality YUVA source
+					pixelFormat = NodeAv.avcodecFindBestPixFmtOfList(
+						this.avCodec.pixelFormats,
+						NodeAv.AV_PIX_FMT_YUVA444P12LE,
+					);
+				}
 			}
 		}
 
@@ -173,6 +205,14 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
 			}
 		}
 
+		if (this.codec === 'prores') {
+			// Pick the encoder profile from the requested ProRes four-character code
+			const profile = PRORES_FOURCC_TO_PROFILE[this.config.codec as ProresFourCc];
+			assert(profile !== undefined);
+
+			codecContext.setOption('profile', String(profile));
+		}
+
 		const ret = await codecContext.open2();
 		NodeAv.FFmpegError.throwIfError(ret, 'Open codec context');
 
@@ -490,6 +530,22 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
 					proresFormat: null,
 				});
 			}
+		} else if (this.codec === 'prores') {
+			if (!this.packetEmitted) {
+				decoderConfigCodecString = extractVideoCodecString({
+					width: this.config.width,
+					height: this.config.height,
+					codec: 'prores',
+					codecDescription: null,
+					colorSpace: null,
+					avcType: null,
+					avcCodecInfo: null,
+					hevcCodecInfo: null,
+					vp9CodecInfo: null,
+					av1CodecInfo: null,
+					proresFormat: this.config.codec as ProresFourCc,
+				});
+			}
 		} else {
 			throw new Error('Unreachable.');
 		}