From 7d89c8de2d04959398ddb4a42c160c4d3f0a9499 Mon Sep 17 00:00:00 2001 From: Vanilagy <1696106+Vanilagy@users.noreply.github.com> Date: Tue, 30 Jun 2026 21:40:58 +0200 Subject: [PATCH] Add ProRes encode and decode support to @mediabunny/server --- docs/.vitepress/config.mts | 1 + docs/codec-registry/overview.md | 1 + docs/codec-registry/prores.md | 44 +++++ docs/guide/extensions/server.md | 2 +- docs/guide/supported-formats-and-codecs.md | 2 +- package-lock.json | 9 +- packages/prores/package.json | 2 +- packages/prores/src/index.ts | 17 ++ packages/server/README.md | 4 +- packages/server/package.json | 3 +- packages/server/src/index.ts | 2 + packages/server/src/misc.ts | 1 + packages/server/src/video-encoder.ts | 70 +++++++- src/codec.ts | 37 +++- src/decode.ts | 2 +- src/encode.ts | 6 +- test/node/prores.test.ts | 10 +- test/node/server-extension.test.ts | 197 ++++++++++++++++++++- 18 files changed, 383 insertions(+), 27 deletions(-) create mode 100644 docs/codec-registry/prores.md diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index d9d6b7b..836b76e 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -144,6 +144,7 @@ export default withMermaid({ { text: 'VP8', link: '/codec-registry/vp8' }, { text: 'VP9', link: '/codec-registry/vp9' }, { text: 'AV1', link: '/codec-registry/av1' }, + { text: 'ProRes', link: '/codec-registry/prores' }, ], }, { diff --git a/docs/codec-registry/overview.md b/docs/codec-registry/overview.md index cee3639..bae0fb4 100644 --- a/docs/codec-registry/overview.md +++ b/docs/codec-registry/overview.md @@ -15,6 +15,7 @@ The registry is an extension of the [WebCodecs Codec Registry](https://www.w3.or - [VP8](./vp8) - [VP9](./vp9) - [AV1](./av1) +- [ProRes](./prores) ## Audio codecs diff --git a/docs/codec-registry/prores.md b/docs/codec-registry/prores.md new file mode 100644 index 0000000..6a53984 --- /dev/null +++ b/docs/codec-registry/prores.md @@ -0,0 +1,44 @@ +--- +description: Apple ProRes video codec definition, defining legal codec strings, decoder configs, and packet data formats. +--- + + + + + +# ProRes codec registration + +## Description + +The Apple ProRes video codec, as specified in [SMPTE RDD 36](https://pub.smpte.org/doc/rdd36/). + +## Codec ID + +```ts +'prores' +``` + +## `EncodedPacket` data + +The packet's data must be a `frame()` as defined in [SMPTE RDD 36](https://pub.smpte.org/doc/rdd36/). + +## `EncodedPacket` type + +Since ProRes is intra-frame-only, every packet is a key frame and its type is therefore always `'key'`. + +## `VideoDecoderConfig` codec string + +The codec string must be one of the six four-character codes identifying the ProRes profile: + +- `'ap4x'` - ProRes 4444 XQ +- `'ap4h'` - ProRes 4444 +- `'apch'` - ProRes 422 High Quality +- `'apcn'` - ProRes 422 Standard Definition +- `'apcs'` - ProRes 422 LT +- `'apco'` - ProRes 422 Proxy + +## `VideoDecoderConfig` description + +`description` is not used for this codec. diff --git a/docs/guide/extensions/server.md b/docs/guide/extensions/server.md index 3ed3663..282ff56 100644 --- a/docs/guide/extensions/server.md +++ b/docs/guide/extensions/server.md @@ -7,7 +7,7 @@ description: Add full video and audio decoder and encoder support for use in ser By default, Mediabunny requires a browser environment for full access to decoders, encoders, and video processing features. `@mediabunny/server` uses [NodeAV](https://github.com/seydx/node-av) to polyfill this functionality for server-side environments such as Node, Bun, or Deno, enabling the usage of all Mediabunny features on the server. The result is a server-side media processing API that integrates naturally with TypeScript as opposed to the awkwardness and inefficiencies of calling out to the FFmpeg CLI. Features added by this package include: -- Video decoders and encoders for AVC (H.264), HEVC (H.265), VP8, VP9, and AV1. Supports both length-prefixed and Annex B AVC/HEVC as well as transparent video via VP9. +- Video decoders and encoders for AVC (H.264), HEVC (H.265), VP8, VP9, AV1, and ProRes. Supports both length-prefixed and Annex B AVC/HEVC as well as transparent video via VP9 and ProRes. - Audio decoders and encoders for AAC, MP3, Vorbis, Opus, FLAC, AC-3 and E-AC-3. Supports AAC in both AAC and ADTS formats. - Video frame transformation support (resize, rotate, crop) - Automatic hardware acceleration on all platforms (macOS, Linux, Windows) diff --git a/docs/guide/supported-formats-and-codecs.md b/docs/guide/supported-formats-and-codecs.md index 780f78b..75b9f9f 100644 --- a/docs/guide/supported-formats-and-codecs.md +++ b/docs/guide/supported-formats-and-codecs.md @@ -107,7 +107,7 @@ Not all codecs can be used with all containers. The following table specifies th For HLS, the supported codecs depend on the segment format chosen. -[^prores]: ProRes is not supported by WebCodecs. To decode it, use the [`@mediabunny/prores`](./extensions/prores) extension package. +[^prores]: ProRes is not supported by WebCodecs. To decode it, use the [`@mediabunny/prores`](./extensions/prores) extension package. The [`@mediabunny/server`](./extensions/server) extension package provides both decoding and encoding support for server-side environments. [^aac]: In some browsers, AAC encoding is not supported by WebCodecs. You can polyfill it with the [`@mediabunny/aac-encoder`](./extensions/aac-encoder) extension package. [^mp3]: MP3 encoding is not supported by WebCodecs. You can polyfill it with the [`@mediabunny/mp3-encoder`](./extensions/mp3-encoder) extension package. [^flac]: FLAC encoding is not supported by WebCodecs. You can polyfill it with the [`@mediabunny/flac-encoder`](./extensions/flac-encoder) extension package. diff --git a/package-lock.json b/package-lock.json index 2c241ef..bd1998f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10963,9 +10963,9 @@ "optional": true }, "node_modules/turbores": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/turbores/-/turbores-1.0.0.tgz", - "integrity": "sha512-pSa+DVX7X7LIZnU86gMEvFMR6ioOy7AjRoZW6/Jx7AEk4EQSC5l6Ki0atQMY2jjut6p37EVAC3mF+Bfew5xtzQ==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/turbores/-/turbores-1.1.0.tgz", + "integrity": "sha512-iVPm4OkpiKXQYosQM8BQceQx1ZHc0F7IRh72Gdc4aY4rPBomgfQOsWqNK62kADukCSz2Cy41MCeiYf9pIhPE2w==", "license": "MPL-2.0", "funding": { "type": "individual", @@ -12957,7 +12957,7 @@ "version": "1.50.0", "license": "MPL-2.0", "dependencies": { - "turbores": "^1.0.0" + "turbores": "^1.1.0" }, "funding": { "type": "individual", @@ -12972,6 +12972,7 @@ "version": "1.50.0", "license": "MPL-2.0", "dependencies": { + "@mediabunny/prores": "^1.50.0", "node-av": "^6.0.0" }, "funding": { diff --git a/packages/prores/package.json b/packages/prores/package.json index cbed51d..2454beb 100644 --- a/packages/prores/package.json +++ b/packages/prores/package.json @@ -38,7 +38,7 @@ "url": "https://github.com/sponsors/Vanilagy" }, "dependencies": { - "turbores": "^1.0.0" + "turbores": "^1.1.0" }, "peerDependencies": { "mediabunny": "^1.49.0" diff --git a/packages/prores/src/index.ts b/packages/prores/src/index.ts index 40cce24..afd81c8 100644 --- a/packages/prores/src/index.ts +++ b/packages/prores/src/index.ts @@ -125,6 +125,19 @@ class ProresDecoder extends CustomVideoDecoder { fullRange: result.colorRangeFull, }; + let displayWidth = result.visibleWidth; + let displayHeight = result.visibleHeight; + + if (result.pixelAspectRatio.num > result.pixelAspectRatio.den) { + displayWidth = Math.round( + result.visibleWidth * result.pixelAspectRatio.num / result.pixelAspectRatio.den, + ); + } else { + displayHeight = Math.round( + result.visibleHeight * result.pixelAspectRatio.den / result.pixelAspectRatio.num, + ); + } + let sample: VideoSample; if (typeof VideoFrame !== 'undefined') { // Create a VideoFrame directly; this avoids the frame data being copied twice @@ -132,6 +145,8 @@ class ProresDecoder extends CustomVideoDecoder { format: result.pixelFormat as VideoPixelFormat, codedWidth: result.codedWidth, codedHeight: result.codedHeight, + displayWidth, + displayHeight, visibleRect: { x: 0, y: 0, @@ -152,6 +167,8 @@ class ProresDecoder extends CustomVideoDecoder { format: result.pixelFormat, codedWidth: result.codedWidth, codedHeight: result.codedHeight, + displayWidth, + displayHeight, visibleRect: { left: 0, top: 0, diff --git a/packages/server/README.md b/packages/server/README.md index 9d9d3f3..f545c4b 100644 --- a/packages/server/README.md +++ b/packages/server/README.md @@ -12,7 +12,7 @@ By default, Mediabunny requires a browser environment for full access to decoders, encoders, and video processing features. `@mediabunny/server` uses [NodeAV](https://github.com/seydx/node-av) to polyfill this functionality for server-side environments such as Node, Bun, or Deno, enabling the usage of all Mediabunny features on the server. The result is a server-side media processing API that integrates naturally with TypeScript as opposed to the awkwardness and inefficiencies of calling out to the FFmpeg CLI. Features added by this package include: -- Video decoders and encoders for AVC (H.264), HEVC (H.265), VP8, VP9, and AV1. Supports both length-prefixed and Annex B AVC/HEVC as well as transparent video via VP9. +- Video decoders and encoders for AVC (H.264), HEVC (H.265), VP8, VP9, AV1, and ProRes. Supports both length-prefixed and Annex B AVC/HEVC as well as transparent video via VP9 and ProRes. - Audio decoders and encoders for AAC, MP3, Vorbis, Opus, FLAC, AC-3 and E-AC-3. Supports AAC in both AAC and ADTS formats. - Video frame transformation support (resize, rotate, crop) - Automatic hardware acceleration on all platforms (macOS, Linux, Windows) @@ -321,6 +321,8 @@ For encoding, video frames and audio samples are transferred to FFmpeg by conver Whenever possible, `AVFrame`s are never copied over to JavaScript unless explicitly needed. This enables zero-copy decode -> transformation -> encode paths. +ProRes decoding support is provided by [TurboRes](https://github.com/Vanilagy/turbores), as it is often faster than FFmpeg. + ## License `@mediabunny/server` uses the same MPL-2.0 license as Mediabunny. \ No newline at end of file diff --git a/packages/server/package.json b/packages/server/package.json index bfc785e..9e321de 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -34,7 +34,8 @@ "url": "https://github.com/sponsors/Vanilagy" }, "dependencies": { - "node-av": "^6.0.0" + "node-av": "^6.0.0", + "@mediabunny/prores": "^1.50.0" }, "peerDependencies": { "mediabunny": "^1.45.0" diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 818b1dd..530d023 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -22,6 +22,7 @@ import { NodeAvAudioDecoder } from './audio-decoder'; import { NodeAvAudioEncoder } from './audio-encoder'; import { copyVideoSampleToAvFrame, AvFrameVideoSampleResource, transformVideoSample } from './video-sample'; import { copyAudioSampleToAvFrame, AvFrameAudioSampleResource } from './audio-sample'; +import { registerProresDecoder } from '@mediabunny/prores'; const SERVER_LOADED_SYMBOL = Symbol.for('@mediabunny/server loaded'); if ((globalThis as Record)[SERVER_LOADED_SYMBOL]) { @@ -100,6 +101,7 @@ export const registerMediabunnyServer = (options: MediabunnyServerOptions = {}) // Video registerDecoder(NodeAvVideoDecoder); + registerProresDecoder(); // Faster than FFmpeg, so we use it registerEncoder(NodeAvVideoEncoder); // Audio diff --git a/packages/server/src/misc.ts b/packages/server/src/misc.ts index 493f2ce..3c152ae 100644 --- a/packages/server/src/misc.ts +++ b/packages/server/src/misc.ts @@ -16,6 +16,7 @@ export const CODEC_TO_CODEC_ID: Partial> = vp8: NodeAv.AV_CODEC_ID_VP8, vp9: NodeAv.AV_CODEC_ID_VP9, av1: NodeAv.AV_CODEC_ID_AV1, + prores: NodeAv.AV_CODEC_ID_PRORES, aac: NodeAv.AV_CODEC_ID_AAC, opus: NodeAv.AV_CODEC_ID_OPUS, diff --git a/packages/server/src/video-encoder.ts b/packages/server/src/video-encoder.ts index 1152f1b..ed08d33 100644 --- a/packages/server/src/video-encoder.ts +++ b/packages/server/src/video-encoder.ts @@ -38,9 +38,18 @@ import { serializeAvcDecoderConfigurationRecord, serializeHevcDecoderConfigurationRecord, } from '../../../src/codec-data'; -import { extractVideoCodecString } from '../../../src/codec'; +import { extractVideoCodecString, ProresFourCc } from '../../../src/codec'; import { assert, binarySearchLessOrEqual, simplifyRational, toUint8Array } from '../../../src/misc'; +const PRORES_FOURCC_TO_PROFILE: Record = { + apco: NodeAv.AV_PROFILE_PRORES_PROXY, + apcs: NodeAv.AV_PROFILE_PRORES_LT, + apcn: NodeAv.AV_PROFILE_PRORES_STANDARD, + apch: NodeAv.AV_PROFILE_PRORES_HQ, + ap4h: NodeAv.AV_PROFILE_PRORES_4444, + ap4x: NodeAv.AV_PROFILE_PRORES_XQ, +}; + export class NodeAvVideoEncoder extends CustomVideoEncoder { frame!: NodeAv.Frame; packet!: NodeAv.Packet; @@ -62,8 +71,10 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder { }[] = []; static override supports(codec: VideoCodec, config: VideoEncoderConfig): boolean { - return (codec === 'avc' || codec === 'hevc' || codec === 'vp8' || codec === 'vp9' || codec === 'av1') - && config.bitrateMode !== 'quantizer'; + return ( + codec === 'avc' || codec === 'hevc' || codec === 'vp8' || codec === 'vp9' || codec === 'av1' + || codec === 'prores' + ) && config.bitrateMode !== 'quantizer'; } async init(): Promise { @@ -77,13 +88,25 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder { const codecId = CODEC_TO_CODEC_ID[this.codec]; assert(codecId !== undefined); + const getSoftwareCodec = () => { + if (this.codec === 'prores') { + // Prefer prores_ks for ProRes + const proresKs = NodeAv.Codec.findEncoderByName(NodeAv.FF_ENCODER_PRORES_KS); + if (proresKs) { + return proresKs; + } + } + + return NodeAv.Codec.findEncoder(codecId); + }; + let codec: NodeAv.Codec | null = null; if (this.codec === 'vp9' && this.config.alpha === 'keep') { codec = NodeAv.Codec.findEncoderByName(NodeAv.FF_ENCODER_LIBVPX_VP9) ?? NodeAv.Codec.findEncoder(codecId); } else if (this.config.hardwareAcceleration === 'prefer-software') { - codec = NodeAv.Codec.findEncoder(codecId); + codec = getSoftwareCodec(); } else { - codec = (await getHardwareEncoderCodec(codecId)) ?? NodeAv.Codec.findEncoder(codecId); + codec = (await getHardwareEncoderCodec(codecId)) ?? getSoftwareCodec(); } if (!codec) { @@ -108,8 +131,17 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder { pixelFormat = this.avCodec.pixelFormats[0]!; } - if (this.config.alpha === 'keep' && this.avCodec.pixelFormats.includes(NodeAv.AV_PIX_FMT_YUVA420P)) { - pixelFormat = NodeAv.AV_PIX_FMT_YUVA420P; + if (this.config.alpha === 'keep') { + if (this.avCodec.pixelFormats.includes(NodeAv.AV_PIX_FMT_YUVA420P)) { + pixelFormat = NodeAv.AV_PIX_FMT_YUVA420P; + } else { + // Let FFmpeg pick the best alpha-capable format it supports, minimizing data loss versus a + // high-quality YUVA source + pixelFormat = NodeAv.avcodecFindBestPixFmtOfList( + this.avCodec.pixelFormats, + NodeAv.AV_PIX_FMT_YUVA444P12LE, + ); + } } } @@ -173,6 +205,14 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder { } } + if (this.codec === 'prores') { + // Pick the encoder profile from the requested ProRes four-character code + const profile = PRORES_FOURCC_TO_PROFILE[this.config.codec as ProresFourCc]; + assert(profile !== undefined); + + codecContext.setOption('profile', String(profile)); + } + const ret = await codecContext.open2(); NodeAv.FFmpegError.throwIfError(ret, 'Open codec context'); @@ -490,6 +530,22 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder { proresFormat: null, }); } + } else if (this.codec === 'prores') { + if (!this.packetEmitted) { + decoderConfigCodecString = extractVideoCodecString({ + width: this.config.width, + height: this.config.height, + codec: 'prores', + codecDescription: null, + colorSpace: null, + avcType: null, + avcCodecInfo: null, + hevcCodecInfo: null, + vp9CodecInfo: null, + av1CodecInfo: null, + proresFormat: this.config.codec as ProresFourCc, + }); + } } else { throw new Error('Unreachable.'); } diff --git a/src/codec.ts b/src/codec.ts index ff72494..97e2bd5 100644 --- a/src/codec.ts +++ b/src/codec.ts @@ -227,7 +227,23 @@ export const PRORES_FOURCCS = [ ] as const; export type ProresFourCc = typeof PRORES_FOURCCS[number]; -export const buildVideoCodecString = (codec: VideoCodec, width: number, height: number, bitrate: number) => { +// Target data rates of the ProRes profiles at 1920x1080 ~30fps, as published by Apple +const PRORES_PROFILE_TARGET_BITRATES: { fourCc: ProresFourCc; bitrate: number; alpha: boolean }[] = [ + { fourCc: 'apco', bitrate: 45_000_000, alpha: false }, // 422 Proxy + { fourCc: 'apcs', bitrate: 102_000_000, alpha: false }, // 422 LT + { fourCc: 'apcn', bitrate: 147_000_000, alpha: false }, // 422 Standard + { fourCc: 'apch', bitrate: 220_000_000, alpha: false }, // 422 HQ + { fourCc: 'ap4h', bitrate: 330_000_000, alpha: true }, // 4444 + { fourCc: 'ap4x', bitrate: 500_000_000, alpha: true }, // 4444 XQ +]; + +export const buildVideoCodecString = ( + codec: VideoCodec, + width: number, + height: number, + bitrate: number, + alpha: boolean, +) => { if (codec === 'avc') { const profileIndication = 0x64; // High Profile const totalMacroblocks = Math.ceil(width / 16) * Math.ceil(height / 16); @@ -287,7 +303,22 @@ export const buildVideoCodecString = (codec: VideoCodec, width: number, height: return `av01.${profile}.${level}${levelInfo.tier}.${bitDepth}`; } else if (codec === 'prores') { - return 'apch'; + const referencePixels = 1920 * 1080; + const scaleFactor = Math.pow((width * height) / referencePixels, 0.95); + + const candidates = PRORES_PROFILE_TARGET_BITRATES.filter(x => x.alpha === alpha); + + let bestFourCc = candidates[0]!.fourCc; + let smallestDifference = Infinity; + for (const { fourCc, bitrate: targetBitrate } of candidates) { + const difference = Math.abs(targetBitrate * scaleFactor - bitrate); + if (difference < smallestDifference) { + smallestDifference = difference; + bestFourCc = fourCc; + } + } + + return bestFourCc; } else { assertNever(codec); } @@ -700,6 +731,8 @@ export const inferCodecFromCodecString = (codecString: string): MediaCodec | nul return 'vp9'; } else if (codecString.startsWith('av01')) { return 'av1'; + } else if ((PRORES_FOURCCS as readonly string[]).includes(codecString)) { + return 'prores'; } // Audio codecs diff --git a/src/decode.ts b/src/decode.ts index ca1fb8c..62f5c5e 100644 --- a/src/decode.ts +++ b/src/decode.ts @@ -140,7 +140,7 @@ export const canDecodeVideo = async ( ...options, codedWidth: options.codedWidth ?? 1280, codedHeight: options.codedHeight ?? 720, - codec: options.codec ?? buildVideoCodecString(codec, 1280, 720, 1e6), + codec: options.codec ?? buildVideoCodecString(codec, 1280, 720, 1e6, false), }; resolvedOptions.description ??= guessDescriptionForVideo(resolvedOptions); diff --git a/src/encode.ts b/src/encode.ts index b13991b..386ce72 100644 --- a/src/encode.ts +++ b/src/encode.ts @@ -251,8 +251,9 @@ export type VideoEncodingAdditionalOptions = { * What to do with alpha data contained in the video samples. * * - `'discard'` (default): Only the samples' color data is kept; the video is opaque. - * - `'keep'`: The samples' alpha data is also encoded as side data. Make sure to pair this mode with a container - * format that supports transparency (such as WebM or Matroska). + * - `'keep'`: The samples' alpha data is also encoded. Depending on the codec, the alpha may be emitted as packet + * side data or in-band alongside the main packet. For codecs that emit alpha side data, such as VP9, make sure to + * pair this mode with a container format that supports transparency (such as WebM or Matroska). */ alpha?: 'discard' | 'keep'; /** Configures the bitrate mode; defaults to `'variable'`. */ @@ -344,6 +345,7 @@ export const buildVideoEncoderConfig = (options: { options.width, options.height, resolvedBitrate, + options.alpha === 'keep', ), width: options.width, height: options.height, diff --git a/test/node/prores.test.ts b/test/node/prores.test.ts index 5342f15..ccab468 100644 --- a/test/node/prores.test.ts +++ b/test/node/prores.test.ts @@ -137,10 +137,16 @@ test('ProRes decoding', { timeout: 10_000 }, async () => { expect(sample.timestamp).toBe(firstTimestamp); expect(sample.duration).toBeGreaterThan(0); - expect(sample.displayWidth).toBe(await videoTrack.getDisplayWidth()); - expect(sample.displayHeight).toBe(await videoTrack.getDisplayHeight()); + expect(sample.displayWidth).toBe(1920); + expect(sample.displayHeight).toBe(1080); expect(sample.codedWidth).toBe(1920); expect(sample.codedHeight).toBe(1080); // Technically a lie but the field is ill-defined + expect(sample.visibleRect).toEqual({ + left: 0, + top: 0, + width: 1920, + height: 1080, + }); expect(sample.format).toBe('I422P10'); const allocationSize = sample.allocationSize(); diff --git a/test/node/server-extension.test.ts b/test/node/server-extension.test.ts index ea553f3..985d595 100644 --- a/test/node/server-extension.test.ts +++ b/test/node/server-extension.test.ts @@ -15,6 +15,7 @@ import { buildAudioCodecString, buildVideoCodecString, NON_PCM_AUDIO_CODECS, + PRORES_FOURCCS, VideoCodec, } from '../../src/codec.js'; import { EncodedPacket } from '../../src/packet.js'; @@ -30,6 +31,8 @@ import { Output } from '../../src/output.js'; import { Mp4OutputFormat } from '../../src/output-format.js'; import { BufferTarget } from '../../src/target.js'; import { Conversion } from '../../src/conversion.js'; +import { VideoSampleSource } from '../../src/media-source.js'; +import { QUALITY_HIGH } from '../../src/encode.js'; import { AvFrameVideoSampleResource } from '../../packages/server/src/video-sample.js'; import { AvFrameAudioSampleResource } from '../../packages/server/src/audio-sample.js'; import { toAvFrame } from '../../packages/server/src/index.js'; @@ -113,7 +116,7 @@ describe('Video', async () => { encoder.codec = 'avc'; // @ts-expect-error Readonly encoder.config = { - codec: buildVideoCodecString('avc', 1280, 720, 1e6), + codec: buildVideoCodecString('avc', 1280, 720, 1e6, false), width: 1280, height: 720, bitrate: 1e6, @@ -196,7 +199,7 @@ describe('Video', async () => { encoder.codec = 'avc'; // @ts-expect-error Readonly encoder.config = { - codec: buildVideoCodecString('avc', 1280, 720, 1e6), + codec: buildVideoCodecString('avc', 1280, 720, 1e6, false), width: 1280, height: 720, bitrate: 1e6, @@ -471,6 +474,188 @@ describe('Video', async () => { }); }); + test('ProRes encode', async () => { + // ProRes can't be decoded by node-av's wrapper here, so this is encode-only. + const encoder = new NodeAvVideoEncoder(); + // @ts-expect-error Readonly + encoder.codec = 'prores'; + // @ts-expect-error Readonly + encoder.config = { + codec: 'apch', + width: 1280, + height: 720, + bitrate: 1e6, + } satisfies VideoEncoderConfig; + + const packets: EncodedPacket[] = []; + const sampleTimestamps: number[] = []; + + // @ts-expect-error Readonly + encoder.onPacket = (packet: EncodedPacket, meta: EncodedVideoChunkMetadata) => { + expect(packet.duration).toBe(1 / 30); + expect(packet.type).toBe('key'); + expect(String.fromCharCode(...packet.data.slice(4, 8))).toBe('icpf'); + + if (packets.length === 0) { + expect(meta.decoderConfig).toBeDefined(); + expect(meta.decoderConfig!.codec).toBe('apch'); + expect(meta.decoderConfig!.codedWidth).toBe(1280); + expect(meta.decoderConfig!.codedHeight).toBe(720); + expect(meta.decoderConfig!.description).toBeUndefined(); + } + + packets.push(packet); + }; + + await encoder.init(); + + const data = new Uint8Array(1280 * 720 * 4).fill(0xff); // White + for (let i = 0; i < 10; i++) { + using sample = new VideoSample(data, { + format: 'RGBX', + codedWidth: 1280, + codedHeight: 720, + timestamp: i / 30, + duration: 1 / 30, + }); + + sampleTimestamps.push(sample.timestamp); + + await encoder.encode(sample, {}); + } + + await encoder.flush(); + + expect(packets).toHaveLength(10); + expect(packets.map(x => x.timestamp)).toEqual(sampleTimestamps); + + await encoder.close(); + }); + + test('ProRes Proxy encode', async () => { + const encoder = new NodeAvVideoEncoder(); + // @ts-expect-error Readonly + encoder.codec = 'prores'; + // @ts-expect-error Readonly + encoder.config = { + codec: 'apco', // Force ProRes 422 Proxy + width: 1280, + height: 720, + bitrate: 1e6, + } satisfies VideoEncoderConfig; + + const packets: EncodedPacket[] = []; + const sampleTimestamps: number[] = []; + + // @ts-expect-error Readonly + encoder.onPacket = (packet: EncodedPacket, meta: EncodedVideoChunkMetadata) => { + expect(packet.duration).toBe(1 / 30); + expect(packet.type).toBe('key'); + expect(String.fromCharCode(...packet.data.slice(4, 8))).toBe('icpf'); + + if (packets.length === 0) { + expect(meta.decoderConfig).toBeDefined(); + expect(PRORES_FOURCCS).toContain(meta.decoderConfig!.codec); + expect(meta.decoderConfig!.codec).toBe('apco'); + expect(meta.decoderConfig!.codedWidth).toBe(1280); + expect(meta.decoderConfig!.codedHeight).toBe(720); + expect(meta.decoderConfig!.description).toBeUndefined(); + } + + packets.push(packet); + }; + + await encoder.init(); + + const data = new Uint8Array(1280 * 720 * 4).fill(0xff); // White + for (let i = 0; i < 10; i++) { + using sample = new VideoSample(data, { + format: 'RGBX', + codedWidth: 1280, + codedHeight: 720, + timestamp: i / 30, + duration: 1 / 30, + }); + + sampleTimestamps.push(sample.timestamp); + + await encoder.encode(sample, {}); + } + + await encoder.flush(); + + expect(packets).toHaveLength(10); + expect(packets.map(x => x.timestamp)).toEqual(sampleTimestamps); + + await encoder.close(); + }); + + test('Transparent ProRes encode and decode', async () => { + const output = new Output({ + format: new Mp4OutputFormat(), + target: new BufferTarget(), + }); + + const source = new VideoSampleSource({ + codec: 'prores', + bitrate: QUALITY_HIGH, + alpha: 'keep', + }); + output.addVideoTrack(source); + + await output.start(); + + // Solid red at 50% alpha + const data = new Uint8Array(640 * 480 * 4); + for (let i = 0; i < data.byteLength; i += 4) { + data[i] = 0xff; + data[i + 3] = 0x80; + } + + using inputSample = new VideoSample(data, { + format: 'RGBA', + codedWidth: 640, + codedHeight: 480, + timestamp: 0, + duration: 1, + }); + await source.add(inputSample); + + await output.finalize(); + + using input = new Input({ + source: new BufferSource(output.target.buffer!), + formats: ALL_FORMATS, + }); + + const videoTrack = (await input.getPrimaryVideoTrack())!; + expect(await videoTrack.canBeTransparent()).toBe(true); + + const sink = new VideoSampleSink(videoTrack); + using sample = (await sink.getSample(0))!; + + expect(sample.format).toBe('I444AP12'); + + const decodedData = new Uint8Array(sample.allocationSize()); + const layout = await sample.copyTo(decodedData); + + const view = new DataView(decodedData.buffer); + const alphaPlane = layout[3]!; + let minAlpha = Infinity; + let maxAlpha = -Infinity; + for (let y = 0; y < sample.codedHeight; y++) { + for (let x = 0; x < sample.codedWidth; x++) { + const alpha = view.getUint16(alphaPlane.offset + y * alphaPlane.stride + x * 2, true); + minAlpha = Math.min(minAlpha, alpha); + maxAlpha = Math.max(maxAlpha, alpha); + } + } + + expect(maxAlpha).toBeLessThan(4095); + expect(Math.abs(minAlpha - 4095 / 2)).toBeLessThan(32); + expect(Math.abs(maxAlpha - 4095 / 2)).toBeLessThan(32); + }); + test('Non-square pixels encode and decode #1', async () => { await encodeDecodeTest('avc', { displayWidth: 2 * 1280, displayHeight: 720 }, async (packet, meta, i) => { if (i === 0) { @@ -517,7 +702,7 @@ describe('Video', async () => { encoder.codec = codec; // @ts-expect-error Readonly encoder.config = { - codec: buildVideoCodecString(codec, 1280, 720, 1e6), + codec: buildVideoCodecString(codec, 1280, 720, 1e6, false), width: 1280, height: 720, bitrate: 1e6, @@ -616,6 +801,10 @@ describe('Video', async () => { await conversionRoundtrip('av1'); }); + test('ProRes conversion roundtrip', { timeout: 20_000 }, async () => { + await conversionRoundtrip('prores'); + }); + const conversionRoundtrip = async (codec: VideoCodec, duration?: number) => { using input = new Input({ source: new FilePathSource('./test/public/video.mp4'), @@ -878,7 +1067,7 @@ describe('Video', async () => { encoder.codec = 'avc'; // @ts-expect-error Readonly encoder.config = { - codec: buildVideoCodecString('avc', 1920, 1080, 1e6), + codec: buildVideoCodecString('avc', 1920, 1080, 1e6, false), width: 1920, height: 1080, bitrate: 1e6,