Add ProRes encode and decode support to @mediabunny/server

This commit is contained in:
Vanilagy
2026-06-30 21:40:58 +02:00
parent b4ef20cd36
commit 7d89c8de2d
18 changed files with 383 additions and 27 deletions
+1
View File
@@ -144,6 +144,7 @@ export default withMermaid({
{ text: 'VP8', link: '/codec-registry/vp8' }, { text: 'VP8', link: '/codec-registry/vp8' },
{ text: 'VP9', link: '/codec-registry/vp9' }, { text: 'VP9', link: '/codec-registry/vp9' },
{ text: 'AV1', link: '/codec-registry/av1' }, { text: 'AV1', link: '/codec-registry/av1' },
{ text: 'ProRes', link: '/codec-registry/prores' },
], ],
}, },
{ {
+1
View File
@@ -15,6 +15,7 @@ The registry is an extension of the [WebCodecs Codec Registry](https://www.w3.or
- [VP8](./vp8) - [VP8](./vp8)
- [VP9](./vp9) - [VP9](./vp9)
- [AV1](./av1) - [AV1](./av1)
- [ProRes](./prores)
## Audio codecs ## Audio codecs
+44
View File
@@ -0,0 +1,44 @@
---
description: Apple ProRes video codec definition, defining legal codec strings, decoder configs, and packet data formats.
---
<script setup>
import { VPBadge } from 'vitepress/theme'
</script>
<VPBadge type="info" text="Video codec" />
# ProRes codec registration
## Description
The Apple ProRes video codec, as specified in [SMPTE RDD 36](https://pub.smpte.org/doc/rdd36/).
## Codec ID
```ts
'prores'
```
## `EncodedPacket` data
The packet's data must be a `frame()` as defined in [SMPTE RDD 36](https://pub.smpte.org/doc/rdd36/).
## `EncodedPacket` type
Since ProRes is intra-frame-only, every packet is a key frame and its type is therefore always `'key'`.
## `VideoDecoderConfig` codec string
The codec string must be one of the six four-character codes identifying the ProRes profile:
- `'ap4x'` - ProRes 4444 XQ
- `'ap4h'` - ProRes 4444
- `'apch'` - ProRes 422 High Quality
- `'apcn'` - ProRes 422 Standard Definition
- `'apcs'` - ProRes 422 LT
- `'apco'` - ProRes 422 Proxy
## `VideoDecoderConfig` description
`description` is not used for this codec.
+1 -1
View File
@@ -7,7 +7,7 @@ description: Add full video and audio decoder and encoder support for use in ser
By default, Mediabunny requires a browser environment for full access to decoders, encoders, and video processing features. `@mediabunny/server` uses [NodeAV](https://github.com/seydx/node-av) to polyfill this functionality for server-side environments such as Node, Bun, or Deno, enabling the usage of all Mediabunny features on the server. The result is a server-side media processing API that integrates naturally with TypeScript as opposed to the awkwardness and inefficiencies of calling out to the FFmpeg CLI. By default, Mediabunny requires a browser environment for full access to decoders, encoders, and video processing features. `@mediabunny/server` uses [NodeAV](https://github.com/seydx/node-av) to polyfill this functionality for server-side environments such as Node, Bun, or Deno, enabling the usage of all Mediabunny features on the server. The result is a server-side media processing API that integrates naturally with TypeScript as opposed to the awkwardness and inefficiencies of calling out to the FFmpeg CLI.
Features added by this package include: Features added by this package include:
- Video decoders and encoders for AVC (H.264), HEVC (H.265), VP8, VP9, and AV1. Supports both length-prefixed and Annex B AVC/HEVC as well as transparent video via VP9. - Video decoders and encoders for AVC (H.264), HEVC (H.265), VP8, VP9, AV1, and ProRes. Supports both length-prefixed and Annex B AVC/HEVC as well as transparent video via VP9 and ProRes.
- Audio decoders and encoders for AAC, MP3, Vorbis, Opus, FLAC, AC-3 and E-AC-3. Supports AAC in both AAC and ADTS formats. - Audio decoders and encoders for AAC, MP3, Vorbis, Opus, FLAC, AC-3 and E-AC-3. Supports AAC in both AAC and ADTS formats.
- Video frame transformation support (resize, rotate, crop) - Video frame transformation support (resize, rotate, crop)
- Automatic hardware acceleration on all platforms (macOS, Linux, Windows) - Automatic hardware acceleration on all platforms (macOS, Linux, Windows)
+1 -1
View File
@@ -107,7 +107,7 @@ Not all codecs can be used with all containers. The following table specifies th
For HLS, the supported codecs depend on the segment format chosen. For HLS, the supported codecs depend on the segment format chosen.
[^prores]: ProRes is not supported by WebCodecs. To decode it, use the [`@mediabunny/prores`](./extensions/prores) extension package. [^prores]: ProRes is not supported by WebCodecs. To decode it, use the [`@mediabunny/prores`](./extensions/prores) extension package. The [`@mediabunny/server`](./extensions/server) extension package provides both decoding and encoding support for server-side environments.
[^aac]: In some browsers, AAC encoding is not supported by WebCodecs. You can polyfill it with the [`@mediabunny/aac-encoder`](./extensions/aac-encoder) extension package. [^aac]: In some browsers, AAC encoding is not supported by WebCodecs. You can polyfill it with the [`@mediabunny/aac-encoder`](./extensions/aac-encoder) extension package.
[^mp3]: MP3 encoding is not supported by WebCodecs. You can polyfill it with the [`@mediabunny/mp3-encoder`](./extensions/mp3-encoder) extension package. [^mp3]: MP3 encoding is not supported by WebCodecs. You can polyfill it with the [`@mediabunny/mp3-encoder`](./extensions/mp3-encoder) extension package.
[^flac]: FLAC encoding is not supported by WebCodecs. You can polyfill it with the [`@mediabunny/flac-encoder`](./extensions/flac-encoder) extension package. [^flac]: FLAC encoding is not supported by WebCodecs. You can polyfill it with the [`@mediabunny/flac-encoder`](./extensions/flac-encoder) extension package.
+5 -4
View File
@@ -10963,9 +10963,9 @@
"optional": true "optional": true
}, },
"node_modules/turbores": { "node_modules/turbores": {
"version": "1.0.0", "version": "1.1.0",
"resolved": "https://registry.npmjs.org/turbores/-/turbores-1.0.0.tgz", "resolved": "https://registry.npmjs.org/turbores/-/turbores-1.1.0.tgz",
"integrity": "sha512-pSa+DVX7X7LIZnU86gMEvFMR6ioOy7AjRoZW6/Jx7AEk4EQSC5l6Ki0atQMY2jjut6p37EVAC3mF+Bfew5xtzQ==", "integrity": "sha512-iVPm4OkpiKXQYosQM8BQceQx1ZHc0F7IRh72Gdc4aY4rPBomgfQOsWqNK62kADukCSz2Cy41MCeiYf9pIhPE2w==",
"license": "MPL-2.0", "license": "MPL-2.0",
"funding": { "funding": {
"type": "individual", "type": "individual",
@@ -12957,7 +12957,7 @@
"version": "1.50.0", "version": "1.50.0",
"license": "MPL-2.0", "license": "MPL-2.0",
"dependencies": { "dependencies": {
"turbores": "^1.0.0" "turbores": "^1.1.0"
}, },
"funding": { "funding": {
"type": "individual", "type": "individual",
@@ -12972,6 +12972,7 @@
"version": "1.50.0", "version": "1.50.0",
"license": "MPL-2.0", "license": "MPL-2.0",
"dependencies": { "dependencies": {
"@mediabunny/prores": "^1.50.0",
"node-av": "^6.0.0" "node-av": "^6.0.0"
}, },
"funding": { "funding": {
+1 -1
View File
@@ -38,7 +38,7 @@
"url": "https://github.com/sponsors/Vanilagy" "url": "https://github.com/sponsors/Vanilagy"
}, },
"dependencies": { "dependencies": {
"turbores": "^1.0.0" "turbores": "^1.1.0"
}, },
"peerDependencies": { "peerDependencies": {
"mediabunny": "^1.49.0" "mediabunny": "^1.49.0"
+17
View File
@@ -125,6 +125,19 @@ class ProresDecoder extends CustomVideoDecoder {
fullRange: result.colorRangeFull, fullRange: result.colorRangeFull,
}; };
let displayWidth = result.visibleWidth;
let displayHeight = result.visibleHeight;
if (result.pixelAspectRatio.num > result.pixelAspectRatio.den) {
displayWidth = Math.round(
result.visibleWidth * result.pixelAspectRatio.num / result.pixelAspectRatio.den,
);
} else {
displayHeight = Math.round(
result.visibleHeight * result.pixelAspectRatio.den / result.pixelAspectRatio.num,
);
}
let sample: VideoSample; let sample: VideoSample;
if (typeof VideoFrame !== 'undefined') { if (typeof VideoFrame !== 'undefined') {
// Create a VideoFrame directly; this avoids the frame data being copied twice // Create a VideoFrame directly; this avoids the frame data being copied twice
@@ -132,6 +145,8 @@ class ProresDecoder extends CustomVideoDecoder {
format: result.pixelFormat as VideoPixelFormat, format: result.pixelFormat as VideoPixelFormat,
codedWidth: result.codedWidth, codedWidth: result.codedWidth,
codedHeight: result.codedHeight, codedHeight: result.codedHeight,
displayWidth,
displayHeight,
visibleRect: { visibleRect: {
x: 0, x: 0,
y: 0, y: 0,
@@ -152,6 +167,8 @@ class ProresDecoder extends CustomVideoDecoder {
format: result.pixelFormat, format: result.pixelFormat,
codedWidth: result.codedWidth, codedWidth: result.codedWidth,
codedHeight: result.codedHeight, codedHeight: result.codedHeight,
displayWidth,
displayHeight,
visibleRect: { visibleRect: {
left: 0, left: 0,
top: 0, top: 0,
+3 -1
View File
@@ -12,7 +12,7 @@
By default, Mediabunny requires a browser environment for full access to decoders, encoders, and video processing features. `@mediabunny/server` uses [NodeAV](https://github.com/seydx/node-av) to polyfill this functionality for server-side environments such as Node, Bun, or Deno, enabling the usage of all Mediabunny features on the server. The result is a server-side media processing API that integrates naturally with TypeScript as opposed to the awkwardness and inefficiencies of calling out to the FFmpeg CLI. By default, Mediabunny requires a browser environment for full access to decoders, encoders, and video processing features. `@mediabunny/server` uses [NodeAV](https://github.com/seydx/node-av) to polyfill this functionality for server-side environments such as Node, Bun, or Deno, enabling the usage of all Mediabunny features on the server. The result is a server-side media processing API that integrates naturally with TypeScript as opposed to the awkwardness and inefficiencies of calling out to the FFmpeg CLI.
Features added by this package include: Features added by this package include:
- Video decoders and encoders for AVC (H.264), HEVC (H.265), VP8, VP9, and AV1. Supports both length-prefixed and Annex B AVC/HEVC as well as transparent video via VP9. - Video decoders and encoders for AVC (H.264), HEVC (H.265), VP8, VP9, AV1, and ProRes. Supports both length-prefixed and Annex B AVC/HEVC as well as transparent video via VP9 and ProRes.
- Audio decoders and encoders for AAC, MP3, Vorbis, Opus, FLAC, AC-3 and E-AC-3. Supports AAC in both AAC and ADTS formats. - Audio decoders and encoders for AAC, MP3, Vorbis, Opus, FLAC, AC-3 and E-AC-3. Supports AAC in both AAC and ADTS formats.
- Video frame transformation support (resize, rotate, crop) - Video frame transformation support (resize, rotate, crop)
- Automatic hardware acceleration on all platforms (macOS, Linux, Windows) - Automatic hardware acceleration on all platforms (macOS, Linux, Windows)
@@ -321,6 +321,8 @@ For encoding, video frames and audio samples are transferred to FFmpeg by conver
Whenever possible, `AVFrame`s are never copied over to JavaScript unless explicitly needed. This enables zero-copy decode -> transformation -> encode paths. Whenever possible, `AVFrame`s are never copied over to JavaScript unless explicitly needed. This enables zero-copy decode -> transformation -> encode paths.
ProRes decoding support is provided by [TurboRes](https://github.com/Vanilagy/turbores), as it is often faster than FFmpeg.
## License ## License
`@mediabunny/server` uses the same MPL-2.0 license as Mediabunny. `@mediabunny/server` uses the same MPL-2.0 license as Mediabunny.
+2 -1
View File
@@ -34,7 +34,8 @@
"url": "https://github.com/sponsors/Vanilagy" "url": "https://github.com/sponsors/Vanilagy"
}, },
"dependencies": { "dependencies": {
"node-av": "^6.0.0" "node-av": "^6.0.0",
"@mediabunny/prores": "^1.50.0"
}, },
"peerDependencies": { "peerDependencies": {
"mediabunny": "^1.45.0" "mediabunny": "^1.45.0"
+2
View File
@@ -22,6 +22,7 @@ import { NodeAvAudioDecoder } from './audio-decoder';
import { NodeAvAudioEncoder } from './audio-encoder'; import { NodeAvAudioEncoder } from './audio-encoder';
import { copyVideoSampleToAvFrame, AvFrameVideoSampleResource, transformVideoSample } from './video-sample'; import { copyVideoSampleToAvFrame, AvFrameVideoSampleResource, transformVideoSample } from './video-sample';
import { copyAudioSampleToAvFrame, AvFrameAudioSampleResource } from './audio-sample'; import { copyAudioSampleToAvFrame, AvFrameAudioSampleResource } from './audio-sample';
import { registerProresDecoder } from '@mediabunny/prores';
const SERVER_LOADED_SYMBOL = Symbol.for('@mediabunny/server loaded'); const SERVER_LOADED_SYMBOL = Symbol.for('@mediabunny/server loaded');
if ((globalThis as Record<symbol, unknown>)[SERVER_LOADED_SYMBOL]) { if ((globalThis as Record<symbol, unknown>)[SERVER_LOADED_SYMBOL]) {
@@ -100,6 +101,7 @@ export const registerMediabunnyServer = (options: MediabunnyServerOptions = {})
// Video // Video
registerDecoder(NodeAvVideoDecoder); registerDecoder(NodeAvVideoDecoder);
registerProresDecoder(); // Faster than FFmpeg, so we use it
registerEncoder(NodeAvVideoEncoder); registerEncoder(NodeAvVideoEncoder);
// Audio // Audio
+1
View File
@@ -16,6 +16,7 @@ export const CODEC_TO_CODEC_ID: Partial<Record<MediaCodec, NodeAv.AVCodecID>> =
vp8: NodeAv.AV_CODEC_ID_VP8, vp8: NodeAv.AV_CODEC_ID_VP8,
vp9: NodeAv.AV_CODEC_ID_VP9, vp9: NodeAv.AV_CODEC_ID_VP9,
av1: NodeAv.AV_CODEC_ID_AV1, av1: NodeAv.AV_CODEC_ID_AV1,
prores: NodeAv.AV_CODEC_ID_PRORES,
aac: NodeAv.AV_CODEC_ID_AAC, aac: NodeAv.AV_CODEC_ID_AAC,
opus: NodeAv.AV_CODEC_ID_OPUS, opus: NodeAv.AV_CODEC_ID_OPUS,
+63 -7
View File
@@ -38,9 +38,18 @@ import {
serializeAvcDecoderConfigurationRecord, serializeAvcDecoderConfigurationRecord,
serializeHevcDecoderConfigurationRecord, serializeHevcDecoderConfigurationRecord,
} from '../../../src/codec-data'; } from '../../../src/codec-data';
import { extractVideoCodecString } from '../../../src/codec'; import { extractVideoCodecString, ProresFourCc } from '../../../src/codec';
import { assert, binarySearchLessOrEqual, simplifyRational, toUint8Array } from '../../../src/misc'; import { assert, binarySearchLessOrEqual, simplifyRational, toUint8Array } from '../../../src/misc';
const PRORES_FOURCC_TO_PROFILE: Record<ProresFourCc, NodeAv.AVProfile> = {
apco: NodeAv.AV_PROFILE_PRORES_PROXY,
apcs: NodeAv.AV_PROFILE_PRORES_LT,
apcn: NodeAv.AV_PROFILE_PRORES_STANDARD,
apch: NodeAv.AV_PROFILE_PRORES_HQ,
ap4h: NodeAv.AV_PROFILE_PRORES_4444,
ap4x: NodeAv.AV_PROFILE_PRORES_XQ,
};
export class NodeAvVideoEncoder extends CustomVideoEncoder { export class NodeAvVideoEncoder extends CustomVideoEncoder {
frame!: NodeAv.Frame; frame!: NodeAv.Frame;
packet!: NodeAv.Packet; packet!: NodeAv.Packet;
@@ -62,8 +71,10 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
}[] = []; }[] = [];
static override supports(codec: VideoCodec, config: VideoEncoderConfig): boolean { static override supports(codec: VideoCodec, config: VideoEncoderConfig): boolean {
return (codec === 'avc' || codec === 'hevc' || codec === 'vp8' || codec === 'vp9' || codec === 'av1') return (
&& config.bitrateMode !== 'quantizer'; codec === 'avc' || codec === 'hevc' || codec === 'vp8' || codec === 'vp9' || codec === 'av1'
|| codec === 'prores'
) && config.bitrateMode !== 'quantizer';
} }
async init(): Promise<void> { async init(): Promise<void> {
@@ -77,13 +88,25 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
const codecId = CODEC_TO_CODEC_ID[this.codec]; const codecId = CODEC_TO_CODEC_ID[this.codec];
assert(codecId !== undefined); assert(codecId !== undefined);
const getSoftwareCodec = () => {
if (this.codec === 'prores') {
// Prefer prores_ks for ProRes
const proresKs = NodeAv.Codec.findEncoderByName(NodeAv.FF_ENCODER_PRORES_KS);
if (proresKs) {
return proresKs;
}
}
return NodeAv.Codec.findEncoder(codecId);
};
let codec: NodeAv.Codec | null = null; let codec: NodeAv.Codec | null = null;
if (this.codec === 'vp9' && this.config.alpha === 'keep') { if (this.codec === 'vp9' && this.config.alpha === 'keep') {
codec = NodeAv.Codec.findEncoderByName(NodeAv.FF_ENCODER_LIBVPX_VP9) ?? NodeAv.Codec.findEncoder(codecId); codec = NodeAv.Codec.findEncoderByName(NodeAv.FF_ENCODER_LIBVPX_VP9) ?? NodeAv.Codec.findEncoder(codecId);
} else if (this.config.hardwareAcceleration === 'prefer-software') { } else if (this.config.hardwareAcceleration === 'prefer-software') {
codec = NodeAv.Codec.findEncoder(codecId); codec = getSoftwareCodec();
} else { } else {
codec = (await getHardwareEncoderCodec(codecId)) ?? NodeAv.Codec.findEncoder(codecId); codec = (await getHardwareEncoderCodec(codecId)) ?? getSoftwareCodec();
} }
if (!codec) { if (!codec) {
@@ -108,8 +131,17 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
pixelFormat = this.avCodec.pixelFormats[0]!; pixelFormat = this.avCodec.pixelFormats[0]!;
} }
if (this.config.alpha === 'keep' && this.avCodec.pixelFormats.includes(NodeAv.AV_PIX_FMT_YUVA420P)) { if (this.config.alpha === 'keep') {
pixelFormat = NodeAv.AV_PIX_FMT_YUVA420P; if (this.avCodec.pixelFormats.includes(NodeAv.AV_PIX_FMT_YUVA420P)) {
pixelFormat = NodeAv.AV_PIX_FMT_YUVA420P;
} else {
// Let FFmpeg pick the best alpha-capable format it supports, minimizing data loss versus a
// high-quality YUVA source
pixelFormat = NodeAv.avcodecFindBestPixFmtOfList(
this.avCodec.pixelFormats,
NodeAv.AV_PIX_FMT_YUVA444P12LE,
);
}
} }
} }
@@ -173,6 +205,14 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
} }
} }
if (this.codec === 'prores') {
// Pick the encoder profile from the requested ProRes four-character code
const profile = PRORES_FOURCC_TO_PROFILE[this.config.codec as ProresFourCc];
assert(profile !== undefined);
codecContext.setOption('profile', String(profile));
}
const ret = await codecContext.open2(); const ret = await codecContext.open2();
NodeAv.FFmpegError.throwIfError(ret, 'Open codec context'); NodeAv.FFmpegError.throwIfError(ret, 'Open codec context');
@@ -490,6 +530,22 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
proresFormat: null, proresFormat: null,
}); });
} }
} else if (this.codec === 'prores') {
if (!this.packetEmitted) {
decoderConfigCodecString = extractVideoCodecString({
width: this.config.width,
height: this.config.height,
codec: 'prores',
codecDescription: null,
colorSpace: null,
avcType: null,
avcCodecInfo: null,
hevcCodecInfo: null,
vp9CodecInfo: null,
av1CodecInfo: null,
proresFormat: this.config.codec as ProresFourCc,
});
}
} else { } else {
throw new Error('Unreachable.'); throw new Error('Unreachable.');
} }
+35 -2
View File
@@ -227,7 +227,23 @@ export const PRORES_FOURCCS = [
] as const; ] as const;
export type ProresFourCc = typeof PRORES_FOURCCS[number]; export type ProresFourCc = typeof PRORES_FOURCCS[number];
export const buildVideoCodecString = (codec: VideoCodec, width: number, height: number, bitrate: number) => { // Target data rates of the ProRes profiles at 1920x1080 ~30fps, as published by Apple
const PRORES_PROFILE_TARGET_BITRATES: { fourCc: ProresFourCc; bitrate: number; alpha: boolean }[] = [
{ fourCc: 'apco', bitrate: 45_000_000, alpha: false }, // 422 Proxy
{ fourCc: 'apcs', bitrate: 102_000_000, alpha: false }, // 422 LT
{ fourCc: 'apcn', bitrate: 147_000_000, alpha: false }, // 422 Standard
{ fourCc: 'apch', bitrate: 220_000_000, alpha: false }, // 422 HQ
{ fourCc: 'ap4h', bitrate: 330_000_000, alpha: true }, // 4444
{ fourCc: 'ap4x', bitrate: 500_000_000, alpha: true }, // 4444 XQ
];
export const buildVideoCodecString = (
codec: VideoCodec,
width: number,
height: number,
bitrate: number,
alpha: boolean,
) => {
if (codec === 'avc') { if (codec === 'avc') {
const profileIndication = 0x64; // High Profile const profileIndication = 0x64; // High Profile
const totalMacroblocks = Math.ceil(width / 16) * Math.ceil(height / 16); const totalMacroblocks = Math.ceil(width / 16) * Math.ceil(height / 16);
@@ -287,7 +303,22 @@ export const buildVideoCodecString = (codec: VideoCodec, width: number, height:
return `av01.${profile}.${level}${levelInfo.tier}.${bitDepth}`; return `av01.${profile}.${level}${levelInfo.tier}.${bitDepth}`;
} else if (codec === 'prores') { } else if (codec === 'prores') {
return 'apch'; const referencePixels = 1920 * 1080;
const scaleFactor = Math.pow((width * height) / referencePixels, 0.95);
const candidates = PRORES_PROFILE_TARGET_BITRATES.filter(x => x.alpha === alpha);
let bestFourCc = candidates[0]!.fourCc;
let smallestDifference = Infinity;
for (const { fourCc, bitrate: targetBitrate } of candidates) {
const difference = Math.abs(targetBitrate * scaleFactor - bitrate);
if (difference < smallestDifference) {
smallestDifference = difference;
bestFourCc = fourCc;
}
}
return bestFourCc;
} else { } else {
assertNever(codec); assertNever(codec);
} }
@@ -700,6 +731,8 @@ export const inferCodecFromCodecString = (codecString: string): MediaCodec | nul
return 'vp9'; return 'vp9';
} else if (codecString.startsWith('av01')) { } else if (codecString.startsWith('av01')) {
return 'av1'; return 'av1';
} else if ((PRORES_FOURCCS as readonly string[]).includes(codecString)) {
return 'prores';
} }
// Audio codecs // Audio codecs
+1 -1
View File
@@ -140,7 +140,7 @@ export const canDecodeVideo = async (
...options, ...options,
codedWidth: options.codedWidth ?? 1280, codedWidth: options.codedWidth ?? 1280,
codedHeight: options.codedHeight ?? 720, codedHeight: options.codedHeight ?? 720,
codec: options.codec ?? buildVideoCodecString(codec, 1280, 720, 1e6), codec: options.codec ?? buildVideoCodecString(codec, 1280, 720, 1e6, false),
}; };
resolvedOptions.description ??= guessDescriptionForVideo(resolvedOptions); resolvedOptions.description ??= guessDescriptionForVideo(resolvedOptions);
+4 -2
View File
@@ -251,8 +251,9 @@ export type VideoEncodingAdditionalOptions = {
* What to do with alpha data contained in the video samples. * What to do with alpha data contained in the video samples.
* *
* - `'discard'` (default): Only the samples' color data is kept; the video is opaque. * - `'discard'` (default): Only the samples' color data is kept; the video is opaque.
* - `'keep'`: The samples' alpha data is also encoded as side data. Make sure to pair this mode with a container * - `'keep'`: The samples' alpha data is also encoded. Depending on the codec, the alpha may be emitted as packet
* format that supports transparency (such as WebM or Matroska). * side data or in-band alongside the main packet. For codecs that emit alpha side data, such as VP9, make sure to
* pair this mode with a container format that supports transparency (such as WebM or Matroska).
*/ */
alpha?: 'discard' | 'keep'; alpha?: 'discard' | 'keep';
/** Configures the bitrate mode; defaults to `'variable'`. */ /** Configures the bitrate mode; defaults to `'variable'`. */
@@ -344,6 +345,7 @@ export const buildVideoEncoderConfig = (options: {
options.width, options.width,
options.height, options.height,
resolvedBitrate, resolvedBitrate,
options.alpha === 'keep',
), ),
width: options.width, width: options.width,
height: options.height, height: options.height,
+8 -2
View File
@@ -137,10 +137,16 @@ test('ProRes decoding', { timeout: 10_000 }, async () => {
expect(sample.timestamp).toBe(firstTimestamp); expect(sample.timestamp).toBe(firstTimestamp);
expect(sample.duration).toBeGreaterThan(0); expect(sample.duration).toBeGreaterThan(0);
expect(sample.displayWidth).toBe(await videoTrack.getDisplayWidth()); expect(sample.displayWidth).toBe(1920);
expect(sample.displayHeight).toBe(await videoTrack.getDisplayHeight()); expect(sample.displayHeight).toBe(1080);
expect(sample.codedWidth).toBe(1920); expect(sample.codedWidth).toBe(1920);
expect(sample.codedHeight).toBe(1080); // Technically a lie but the field is ill-defined expect(sample.codedHeight).toBe(1080); // Technically a lie but the field is ill-defined
expect(sample.visibleRect).toEqual({
left: 0,
top: 0,
width: 1920,
height: 1080,
});
expect(sample.format).toBe('I422P10'); expect(sample.format).toBe('I422P10');
const allocationSize = sample.allocationSize(); const allocationSize = sample.allocationSize();
+193 -4
View File
@@ -15,6 +15,7 @@ import {
buildAudioCodecString, buildAudioCodecString,
buildVideoCodecString, buildVideoCodecString,
NON_PCM_AUDIO_CODECS, NON_PCM_AUDIO_CODECS,
PRORES_FOURCCS,
VideoCodec, VideoCodec,
} from '../../src/codec.js'; } from '../../src/codec.js';
import { EncodedPacket } from '../../src/packet.js'; import { EncodedPacket } from '../../src/packet.js';
@@ -30,6 +31,8 @@ import { Output } from '../../src/output.js';
import { Mp4OutputFormat } from '../../src/output-format.js'; import { Mp4OutputFormat } from '../../src/output-format.js';
import { BufferTarget } from '../../src/target.js'; import { BufferTarget } from '../../src/target.js';
import { Conversion } from '../../src/conversion.js'; import { Conversion } from '../../src/conversion.js';
import { VideoSampleSource } from '../../src/media-source.js';
import { QUALITY_HIGH } from '../../src/encode.js';
import { AvFrameVideoSampleResource } from '../../packages/server/src/video-sample.js'; import { AvFrameVideoSampleResource } from '../../packages/server/src/video-sample.js';
import { AvFrameAudioSampleResource } from '../../packages/server/src/audio-sample.js'; import { AvFrameAudioSampleResource } from '../../packages/server/src/audio-sample.js';
import { toAvFrame } from '../../packages/server/src/index.js'; import { toAvFrame } from '../../packages/server/src/index.js';
@@ -113,7 +116,7 @@ describe('Video', async () => {
encoder.codec = 'avc'; encoder.codec = 'avc';
// @ts-expect-error Readonly // @ts-expect-error Readonly
encoder.config = { encoder.config = {
codec: buildVideoCodecString('avc', 1280, 720, 1e6), codec: buildVideoCodecString('avc', 1280, 720, 1e6, false),
width: 1280, width: 1280,
height: 720, height: 720,
bitrate: 1e6, bitrate: 1e6,
@@ -196,7 +199,7 @@ describe('Video', async () => {
encoder.codec = 'avc'; encoder.codec = 'avc';
// @ts-expect-error Readonly // @ts-expect-error Readonly
encoder.config = { encoder.config = {
codec: buildVideoCodecString('avc', 1280, 720, 1e6), codec: buildVideoCodecString('avc', 1280, 720, 1e6, false),
width: 1280, width: 1280,
height: 720, height: 720,
bitrate: 1e6, bitrate: 1e6,
@@ -471,6 +474,188 @@ describe('Video', async () => {
}); });
}); });
test('ProRes encode', async () => {
// ProRes can't be decoded by node-av's wrapper here, so this is encode-only.
const encoder = new NodeAvVideoEncoder();
// @ts-expect-error Readonly
encoder.codec = 'prores';
// @ts-expect-error Readonly
encoder.config = {
codec: 'apch',
width: 1280,
height: 720,
bitrate: 1e6,
} satisfies VideoEncoderConfig;
const packets: EncodedPacket[] = [];
const sampleTimestamps: number[] = [];
// @ts-expect-error Readonly
encoder.onPacket = (packet: EncodedPacket, meta: EncodedVideoChunkMetadata) => {
expect(packet.duration).toBe(1 / 30);
expect(packet.type).toBe('key');
expect(String.fromCharCode(...packet.data.slice(4, 8))).toBe('icpf');
if (packets.length === 0) {
expect(meta.decoderConfig).toBeDefined();
expect(meta.decoderConfig!.codec).toBe('apch');
expect(meta.decoderConfig!.codedWidth).toBe(1280);
expect(meta.decoderConfig!.codedHeight).toBe(720);
expect(meta.decoderConfig!.description).toBeUndefined();
}
packets.push(packet);
};
await encoder.init();
const data = new Uint8Array(1280 * 720 * 4).fill(0xff); // White
for (let i = 0; i < 10; i++) {
using sample = new VideoSample(data, {
format: 'RGBX',
codedWidth: 1280,
codedHeight: 720,
timestamp: i / 30,
duration: 1 / 30,
});
sampleTimestamps.push(sample.timestamp);
await encoder.encode(sample, {});
}
await encoder.flush();
expect(packets).toHaveLength(10);
expect(packets.map(x => x.timestamp)).toEqual(sampleTimestamps);
await encoder.close();
});
test('ProRes Proxy encode', async () => {
const encoder = new NodeAvVideoEncoder();
// @ts-expect-error Readonly
encoder.codec = 'prores';
// @ts-expect-error Readonly
encoder.config = {
codec: 'apco', // Force ProRes 422 Proxy
width: 1280,
height: 720,
bitrate: 1e6,
} satisfies VideoEncoderConfig;
const packets: EncodedPacket[] = [];
const sampleTimestamps: number[] = [];
// @ts-expect-error Readonly
encoder.onPacket = (packet: EncodedPacket, meta: EncodedVideoChunkMetadata) => {
expect(packet.duration).toBe(1 / 30);
expect(packet.type).toBe('key');
expect(String.fromCharCode(...packet.data.slice(4, 8))).toBe('icpf');
if (packets.length === 0) {
expect(meta.decoderConfig).toBeDefined();
expect(PRORES_FOURCCS).toContain(meta.decoderConfig!.codec);
expect(meta.decoderConfig!.codec).toBe('apco');
expect(meta.decoderConfig!.codedWidth).toBe(1280);
expect(meta.decoderConfig!.codedHeight).toBe(720);
expect(meta.decoderConfig!.description).toBeUndefined();
}
packets.push(packet);
};
await encoder.init();
const data = new Uint8Array(1280 * 720 * 4).fill(0xff); // White
for (let i = 0; i < 10; i++) {
using sample = new VideoSample(data, {
format: 'RGBX',
codedWidth: 1280,
codedHeight: 720,
timestamp: i / 30,
duration: 1 / 30,
});
sampleTimestamps.push(sample.timestamp);
await encoder.encode(sample, {});
}
await encoder.flush();
expect(packets).toHaveLength(10);
expect(packets.map(x => x.timestamp)).toEqual(sampleTimestamps);
await encoder.close();
});
test('Transparent ProRes encode and decode', async () => {
const output = new Output({
format: new Mp4OutputFormat(),
target: new BufferTarget(),
});
const source = new VideoSampleSource({
codec: 'prores',
bitrate: QUALITY_HIGH,
alpha: 'keep',
});
output.addVideoTrack(source);
await output.start();
// Solid red at 50% alpha
const data = new Uint8Array(640 * 480 * 4);
for (let i = 0; i < data.byteLength; i += 4) {
data[i] = 0xff;
data[i + 3] = 0x80;
}
using inputSample = new VideoSample(data, {
format: 'RGBA',
codedWidth: 640,
codedHeight: 480,
timestamp: 0,
duration: 1,
});
await source.add(inputSample);
await output.finalize();
using input = new Input({
source: new BufferSource(output.target.buffer!),
formats: ALL_FORMATS,
});
const videoTrack = (await input.getPrimaryVideoTrack())!;
expect(await videoTrack.canBeTransparent()).toBe(true);
const sink = new VideoSampleSink(videoTrack);
using sample = (await sink.getSample(0))!;
expect(sample.format).toBe('I444AP12');
const decodedData = new Uint8Array(sample.allocationSize());
const layout = await sample.copyTo(decodedData);
const view = new DataView(decodedData.buffer);
const alphaPlane = layout[3]!;
let minAlpha = Infinity;
let maxAlpha = -Infinity;
for (let y = 0; y < sample.codedHeight; y++) {
for (let x = 0; x < sample.codedWidth; x++) {
const alpha = view.getUint16(alphaPlane.offset + y * alphaPlane.stride + x * 2, true);
minAlpha = Math.min(minAlpha, alpha);
maxAlpha = Math.max(maxAlpha, alpha);
}
}
expect(maxAlpha).toBeLessThan(4095);
expect(Math.abs(minAlpha - 4095 / 2)).toBeLessThan(32);
expect(Math.abs(maxAlpha - 4095 / 2)).toBeLessThan(32);
});
test('Non-square pixels encode and decode #1', async () => { test('Non-square pixels encode and decode #1', async () => {
await encodeDecodeTest('avc', { displayWidth: 2 * 1280, displayHeight: 720 }, async (packet, meta, i) => { await encodeDecodeTest('avc', { displayWidth: 2 * 1280, displayHeight: 720 }, async (packet, meta, i) => {
if (i === 0) { if (i === 0) {
@@ -517,7 +702,7 @@ describe('Video', async () => {
encoder.codec = codec; encoder.codec = codec;
// @ts-expect-error Readonly // @ts-expect-error Readonly
encoder.config = { encoder.config = {
codec: buildVideoCodecString(codec, 1280, 720, 1e6), codec: buildVideoCodecString(codec, 1280, 720, 1e6, false),
width: 1280, width: 1280,
height: 720, height: 720,
bitrate: 1e6, bitrate: 1e6,
@@ -616,6 +801,10 @@ describe('Video', async () => {
await conversionRoundtrip('av1'); await conversionRoundtrip('av1');
}); });
test('ProRes conversion roundtrip', { timeout: 20_000 }, async () => {
await conversionRoundtrip('prores');
});
const conversionRoundtrip = async (codec: VideoCodec, duration?: number) => { const conversionRoundtrip = async (codec: VideoCodec, duration?: number) => {
using input = new Input({ using input = new Input({
source: new FilePathSource('./test/public/video.mp4'), source: new FilePathSource('./test/public/video.mp4'),
@@ -878,7 +1067,7 @@ describe('Video', async () => {
encoder.codec = 'avc'; encoder.codec = 'avc';
// @ts-expect-error Readonly // @ts-expect-error Readonly
encoder.config = { encoder.config = {
codec: buildVideoCodecString('avc', 1920, 1080, 1e6), codec: buildVideoCodecString('avc', 1920, 1080, 1e6, false),
width: 1920, width: 1920,
height: 1080, height: 1080,
bitrate: 1e6, bitrate: 1e6,