arcodange/mediabunny
1
0
Fork 0
mirror of https://github.com/arcodange-org/mediabunny.git synced 2026-09-27 02:43:48 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add ProRes encode and decode support to @mediabunny/server

Browse Source
This commit is contained in:
Vanilagy
2026-06-30 21:40:58 +02:00
parent b4ef20cd36
commit 7d89c8de2d
18 changed files with 383 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/.vitepress/config.mts
+1
View File
@@ -144,6 +144,7 @@ export default withMermaid({
{ text: 'VP8', link: '/codec-registry/vp8' },
{ text: 'VP9', link: '/codec-registry/vp9' },
{ text: 'AV1', link: '/codec-registry/av1' },
{ text: 'ProRes', link: '/codec-registry/prores' },
],
},
{
docs/codec-registry/overview.md
+1
View File
@@ -15,6 +15,7 @@ The registry is an extension of the [WebCodecs Codec Registry](https://www.w3.or
- [VP8](./vp8)
- [VP9](./vp9)
- [AV1](./av1)
- [ProRes](./prores)
## Audio codecs
docs/codec-registry/prores.md
+44
View File
@@ -0,0 +1,44 @@
---
description: Apple ProRes video codec definition, defining legal codec strings, decoder configs, and packet data formats.
---
<script setup>
import { VPBadge } from 'vitepress/theme'
</script>
<VPBadge type="info" text="Video codec" />
# ProRes codec registration
## Description
The Apple ProRes video codec, as specified in [SMPTE RDD 36](https://pub.smpte.org/doc/rdd36/).
## Codec ID
```ts
'prores'
```
## `EncodedPacket` data
The packet's data must be a `frame()` as defined in [SMPTE RDD 36](https://pub.smpte.org/doc/rdd36/).
## `EncodedPacket` type
Since ProRes is intra-frame-only, every packet is a key frame and its type is therefore always `'key'`.
## `VideoDecoderConfig` codec string
The codec string must be one of the six four-character codes identifying the ProRes profile:
- `'ap4x'` - ProRes 4444 XQ
- `'ap4h'` - ProRes 4444
- `'apch'` - ProRes 422 High Quality
- `'apcn'` - ProRes 422 Standard Definition
- `'apcs'` - ProRes 422 LT
- `'apco'` - ProRes 422 Proxy
## `VideoDecoderConfig` description
`description` is not used for this codec.
docs/guide/extensions/server.md
+1 -1
View File
@@ -7,7 +7,7 @@ description: Add full video and audio decoder and encoder support for use in ser
By default, Mediabunny requires a browser environment for full access to decoders, encoders, and video processing features. `@mediabunny/server` uses [NodeAV](https://github.com/seydx/node-av) to polyfill this functionality for server-side environments such as Node, Bun, or Deno, enabling the usage of all Mediabunny features on the server. The result is a server-side media processing API that integrates naturally with TypeScript as opposed to the awkwardness and inefficiencies of calling out to the FFmpeg CLI.
Features added by this package include:
- Video decoders and encoders for AVC (H.264), HEVC (H.265), VP8, VP9, and AV1. Supports both length-prefixed and Annex B AVC/HEVC as well as transparent video via VP9.
- Video decoders and encoders for AVC (H.264), HEVC (H.265), VP8, VP9, AV1, and ProRes. Supports both length-prefixed and Annex B AVC/HEVC as well as transparent video via VP9 and ProRes.
- Audio decoders and encoders for AAC, MP3, Vorbis, Opus, FLAC, AC-3 and E-AC-3. Supports AAC in both AAC and ADTS formats.
- Video frame transformation support (resize, rotate, crop)
- Automatic hardware acceleration on all platforms (macOS, Linux, Windows)
docs/guide/supported-formats-and-codecs.md
+1 -1
View File
@@ -107,7 +107,7 @@ Not all codecs can be used with all containers. The following table specifies th
For HLS, the supported codecs depend on the segment format chosen.
[^prores]: ProRes is not supported by WebCodecs. To decode it, use the [`@mediabunny/prores`](./extensions/prores) extension package.
[^prores]: ProRes is not supported by WebCodecs. To decode it, use the [`@mediabunny/prores`](./extensions/prores) extension package. The [`@mediabunny/server`](./extensions/server) extension package provides both decoding and encoding support for server-side environments.
[^aac]: In some browsers, AAC encoding is not supported by WebCodecs. You can polyfill it with the [`@mediabunny/aac-encoder`](./extensions/aac-encoder) extension package.
[^mp3]: MP3 encoding is not supported by WebCodecs. You can polyfill it with the [`@mediabunny/mp3-encoder`](./extensions/mp3-encoder) extension package.
[^flac]: FLAC encoding is not supported by WebCodecs. You can polyfill it with the [`@mediabunny/flac-encoder`](./extensions/flac-encoder) extension package.
package-lock.json
Generated
+5 -4
View File
@@ -10963,9 +10963,9 @@
"optional": true
},
"node_modules/turbores": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/turbores/-/turbores-1.0.0.tgz",
"integrity": "sha512-pSa+DVX7X7LIZnU86gMEvFMR6ioOy7AjRoZW6/Jx7AEk4EQSC5l6Ki0atQMY2jjut6p37EVAC3mF+Bfew5xtzQ==",
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/turbores/-/turbores-1.1.0.tgz",
"integrity": "sha512-iVPm4OkpiKXQYosQM8BQceQx1ZHc0F7IRh72Gdc4aY4rPBomgfQOsWqNK62kADukCSz2Cy41MCeiYf9pIhPE2w==",
"license": "MPL-2.0",
"funding": {
"type": "individual",
@@ -12957,7 +12957,7 @@
"version": "1.50.0",
"license": "MPL-2.0",
"dependencies": {
"turbores": "^1.0.0"
"turbores": "^1.1.0"
},
"funding": {
"type": "individual",
@@ -12972,6 +12972,7 @@
"version": "1.50.0",
"license": "MPL-2.0",
"dependencies": {
"@mediabunny/prores": "^1.50.0",
"node-av": "^6.0.0"
},
"funding": {
packages/prores/package.json
+1 -1
View File
@@ -38,7 +38,7 @@
"url": "https://github.com/sponsors/Vanilagy"
},
"dependencies": {
"turbores": "^1.0.0"
"turbores": "^1.1.0"
},
"peerDependencies": {
"mediabunny": "^1.49.0"
packages/prores/src/index.ts
+17
View File
@@ -125,6 +125,19 @@ class ProresDecoder extends CustomVideoDecoder {
fullRange: result.colorRangeFull,
};
let displayWidth = result.visibleWidth;
let displayHeight = result.visibleHeight;
if (result.pixelAspectRatio.num > result.pixelAspectRatio.den) {
displayWidth = Math.round(
result.visibleWidth * result.pixelAspectRatio.num / result.pixelAspectRatio.den,
);
} else {
displayHeight = Math.round(
result.visibleHeight * result.pixelAspectRatio.den / result.pixelAspectRatio.num,
);
}
let sample: VideoSample;
if (typeof VideoFrame !== 'undefined') {
// Create a VideoFrame directly; this avoids the frame data being copied twice
@@ -132,6 +145,8 @@ class ProresDecoder extends CustomVideoDecoder {
format: result.pixelFormat as VideoPixelFormat,
codedWidth: result.codedWidth,
codedHeight: result.codedHeight,
displayWidth,
displayHeight,
visibleRect: {
x: 0,
y: 0,
@@ -152,6 +167,8 @@ class ProresDecoder extends CustomVideoDecoder {
format: result.pixelFormat,
codedWidth: result.codedWidth,
codedHeight: result.codedHeight,
displayWidth,
displayHeight,
visibleRect: {
left: 0,
top: 0,
packages/server/README.md
+3 -1
View File
@@ -12,7 +12,7 @@
By default, Mediabunny requires a browser environment for full access to decoders, encoders, and video processing features. `@mediabunny/server` uses [NodeAV](https://github.com/seydx/node-av) to polyfill this functionality for server-side environments such as Node, Bun, or Deno, enabling the usage of all Mediabunny features on the server. The result is a server-side media processing API that integrates naturally with TypeScript as opposed to the awkwardness and inefficiencies of calling out to the FFmpeg CLI.
Features added by this package include:
- Video decoders and encoders for AVC (H.264), HEVC (H.265), VP8, VP9, and AV1. Supports both length-prefixed and Annex B AVC/HEVC as well as transparent video via VP9.
- Video decoders and encoders for AVC (H.264), HEVC (H.265), VP8, VP9, AV1, and ProRes. Supports both length-prefixed and Annex B AVC/HEVC as well as transparent video via VP9 and ProRes.
- Audio decoders and encoders for AAC, MP3, Vorbis, Opus, FLAC, AC-3 and E-AC-3. Supports AAC in both AAC and ADTS formats.
- Video frame transformation support (resize, rotate, crop)
- Automatic hardware acceleration on all platforms (macOS, Linux, Windows)
@@ -321,6 +321,8 @@ For encoding, video frames and audio samples are transferred to FFmpeg by conver
Whenever possible, `AVFrame`s are never copied over to JavaScript unless explicitly needed. This enables zero-copy decode -> transformation -> encode paths.
ProRes decoding support is provided by [TurboRes](https://github.com/Vanilagy/turbores), as it is often faster than FFmpeg.
## License
`@mediabunny/server` uses the same MPL-2.0 license as Mediabunny.
packages/server/package.json
+2 -1
View File
@@ -34,7 +34,8 @@
"url": "https://github.com/sponsors/Vanilagy"
},
"dependencies": {
"node-av": "^6.0.0"
"node-av": "^6.0.0",
"@mediabunny/prores": "^1.50.0"
},
"peerDependencies": {
"mediabunny": "^1.45.0"
packages/server/src/index.ts
+2
View File
@@ -22,6 +22,7 @@ import { NodeAvAudioDecoder } from './audio-decoder';
import { NodeAvAudioEncoder } from './audio-encoder';
import { copyVideoSampleToAvFrame, AvFrameVideoSampleResource, transformVideoSample } from './video-sample';
import { copyAudioSampleToAvFrame, AvFrameAudioSampleResource } from './audio-sample';
import { registerProresDecoder } from '@mediabunny/prores';
const SERVER_LOADED_SYMBOL = Symbol.for('@mediabunny/server loaded');
if ((globalThis as Record<symbol, unknown>)[SERVER_LOADED_SYMBOL]) {
@@ -100,6 +101,7 @@ export const registerMediabunnyServer = (options: MediabunnyServerOptions = {})
// Video
registerDecoder(NodeAvVideoDecoder);
registerProresDecoder(); // Faster than FFmpeg, so we use it
registerEncoder(NodeAvVideoEncoder);
// Audio
packages/server/src/misc.ts
+1
View File
@@ -16,6 +16,7 @@ export const CODEC_TO_CODEC_ID: Partial<Record<MediaCodec, NodeAv.AVCodecID>> =
vp8: NodeAv.AV_CODEC_ID_VP8,
vp9: NodeAv.AV_CODEC_ID_VP9,
av1: NodeAv.AV_CODEC_ID_AV1,
prores: NodeAv.AV_CODEC_ID_PRORES,
aac: NodeAv.AV_CODEC_ID_AAC,
opus: NodeAv.AV_CODEC_ID_OPUS,
packages/server/src/video-encoder.ts
+63 -7
View File
@@ -38,9 +38,18 @@ import {
serializeAvcDecoderConfigurationRecord,
serializeHevcDecoderConfigurationRecord,
} from '../../../src/codec-data';
import { extractVideoCodecString } from '../../../src/codec';
import { extractVideoCodecString, ProresFourCc } from '../../../src/codec';
import { assert, binarySearchLessOrEqual, simplifyRational, toUint8Array } from '../../../src/misc';
const PRORES_FOURCC_TO_PROFILE: Record<ProresFourCc, NodeAv.AVProfile> = {
apco: NodeAv.AV_PROFILE_PRORES_PROXY,
apcs: NodeAv.AV_PROFILE_PRORES_LT,
apcn: NodeAv.AV_PROFILE_PRORES_STANDARD,
apch: NodeAv.AV_PROFILE_PRORES_HQ,
ap4h: NodeAv.AV_PROFILE_PRORES_4444,
ap4x: NodeAv.AV_PROFILE_PRORES_XQ,
};
export class NodeAvVideoEncoder extends CustomVideoEncoder {
frame!: NodeAv.Frame;
packet!: NodeAv.Packet;
@@ -62,8 +71,10 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
}[] = [];
static override supports(codec: VideoCodec, config: VideoEncoderConfig): boolean {
return (codec === 'avc' || codec === 'hevc' || codec === 'vp8' || codec === 'vp9' || codec === 'av1')
&& config.bitrateMode !== 'quantizer';
return (
codec === 'avc' || codec === 'hevc' || codec === 'vp8' || codec === 'vp9' || codec === 'av1'
|| codec === 'prores'
) && config.bitrateMode !== 'quantizer';
}
async init(): Promise<void> {
@@ -77,13 +88,25 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
const codecId = CODEC_TO_CODEC_ID[this.codec];
assert(codecId !== undefined);
const getSoftwareCodec = () => {
if (this.codec === 'prores') {
// Prefer prores_ks for ProRes
const proresKs = NodeAv.Codec.findEncoderByName(NodeAv.FF_ENCODER_PRORES_KS);
if (proresKs) {
return proresKs;
}
}
return NodeAv.Codec.findEncoder(codecId);
};
let codec: NodeAv.Codec | null = null;
if (this.codec === 'vp9' && this.config.alpha === 'keep') {
codec = NodeAv.Codec.findEncoderByName(NodeAv.FF_ENCODER_LIBVPX_VP9) ?? NodeAv.Codec.findEncoder(codecId);
} else if (this.config.hardwareAcceleration === 'prefer-software') {
codec = NodeAv.Codec.findEncoder(codecId);
codec = getSoftwareCodec();
} else {
codec = (await getHardwareEncoderCodec(codecId)) ?? NodeAv.Codec.findEncoder(codecId);
codec = (await getHardwareEncoderCodec(codecId)) ?? getSoftwareCodec();
}
if (!codec) {
@@ -108,8 +131,17 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
pixelFormat = this.avCodec.pixelFormats[0]!;
}
if (this.config.alpha === 'keep' && this.avCodec.pixelFormats.includes(NodeAv.AV_PIX_FMT_YUVA420P)) {
pixelFormat = NodeAv.AV_PIX_FMT_YUVA420P;
if (this.config.alpha === 'keep') {
if (this.avCodec.pixelFormats.includes(NodeAv.AV_PIX_FMT_YUVA420P)) {
pixelFormat = NodeAv.AV_PIX_FMT_YUVA420P;
} else {
// Let FFmpeg pick the best alpha-capable format it supports, minimizing data loss versus a
// high-quality YUVA source
pixelFormat = NodeAv.avcodecFindBestPixFmtOfList(
this.avCodec.pixelFormats,
NodeAv.AV_PIX_FMT_YUVA444P12LE,
);
}
}
}
@@ -173,6 +205,14 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
}
}
if (this.codec === 'prores') {
// Pick the encoder profile from the requested ProRes four-character code
const profile = PRORES_FOURCC_TO_PROFILE[this.config.codec as ProresFourCc];
assert(profile !== undefined);
codecContext.setOption('profile', String(profile));
}
const ret = await codecContext.open2();
NodeAv.FFmpegError.throwIfError(ret, 'Open codec context');
@@ -490,6 +530,22 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
proresFormat: null,
});
}
} else if (this.codec === 'prores') {
if (!this.packetEmitted) {
decoderConfigCodecString = extractVideoCodecString({
width: this.config.width,
height: this.config.height,
codec: 'prores',
codecDescription: null,
colorSpace: null,
avcType: null,
avcCodecInfo: null,
hevcCodecInfo: null,
vp9CodecInfo: null,
av1CodecInfo: null,
proresFormat: this.config.codec as ProresFourCc,
});
}
} else {
throw new Error('Unreachable.');
}
src/codec.ts
+35 -2
View File
@@ -227,7 +227,23 @@ export const PRORES_FOURCCS = [
] as const;
export type ProresFourCc = typeof PRORES_FOURCCS[number];
export const buildVideoCodecString = (codec: VideoCodec, width: number, height: number, bitrate: number) => {
// Target data rates of the ProRes profiles at 1920x1080 ~30fps, as published by Apple
const PRORES_PROFILE_TARGET_BITRATES: { fourCc: ProresFourCc; bitrate: number; alpha: boolean }[] = [
{ fourCc: 'apco', bitrate: 45_000_000, alpha: false }, // 422 Proxy
{ fourCc: 'apcs', bitrate: 102_000_000, alpha: false }, // 422 LT
{ fourCc: 'apcn', bitrate: 147_000_000, alpha: false }, // 422 Standard
{ fourCc: 'apch', bitrate: 220_000_000, alpha: false }, // 422 HQ
{ fourCc: 'ap4h', bitrate: 330_000_000, alpha: true }, // 4444
{ fourCc: 'ap4x', bitrate: 500_000_000, alpha: true }, // 4444 XQ
];
export const buildVideoCodecString = (
codec: VideoCodec,
width: number,
height: number,
bitrate: number,
alpha: boolean,
) => {
if (codec === 'avc') {
const profileIndication = 0x64; // High Profile
const totalMacroblocks = Math.ceil(width / 16) * Math.ceil(height / 16);
@@ -287,7 +303,22 @@ export const buildVideoCodecString = (codec: VideoCodec, width: number, height:
return `av01.${profile}.${level}${levelInfo.tier}.${bitDepth}`;
} else if (codec === 'prores') {
return 'apch';
const referencePixels = 1920 * 1080;
const scaleFactor = Math.pow((width * height) / referencePixels, 0.95);
const candidates = PRORES_PROFILE_TARGET_BITRATES.filter(x => x.alpha === alpha);
let bestFourCc = candidates[0]!.fourCc;
let smallestDifference = Infinity;
for (const { fourCc, bitrate: targetBitrate } of candidates) {
const difference = Math.abs(targetBitrate * scaleFactor - bitrate);
if (difference < smallestDifference) {
smallestDifference = difference;
bestFourCc = fourCc;
}
}
return bestFourCc;
} else {
assertNever(codec);
}
@@ -700,6 +731,8 @@ export const inferCodecFromCodecString = (codecString: string): MediaCodec | nul
return 'vp9';
} else if (codecString.startsWith('av01')) {
return 'av1';
} else if ((PRORES_FOURCCS as readonly string[]).includes(codecString)) {
return 'prores';
}
// Audio codecs
src/decode.ts
+1 -1
View File
@@ -140,7 +140,7 @@ export const canDecodeVideo = async (
...options,
codedWidth: options.codedWidth ?? 1280,
codedHeight: options.codedHeight ?? 720,
codec: options.codec ?? buildVideoCodecString(codec, 1280, 720, 1e6),
codec: options.codec ?? buildVideoCodecString(codec, 1280, 720, 1e6, false),
};
resolvedOptions.description ??= guessDescriptionForVideo(resolvedOptions);
src/encode.ts
+4 -2
View File
@@ -251,8 +251,9 @@ export type VideoEncodingAdditionalOptions = {
* What to do with alpha data contained in the video samples.
*
* - `'discard'` (default): Only the samples' color data is kept; the video is opaque.
* - `'keep'`: The samples' alpha data is also encoded as side data. Make sure to pair this mode with a container
* format that supports transparency (such as WebM or Matroska).
* - `'keep'`: The samples' alpha data is also encoded. Depending on the codec, the alpha may be emitted as packet
* side data or in-band alongside the main packet. For codecs that emit alpha side data, such as VP9, make sure to
* pair this mode with a container format that supports transparency (such as WebM or Matroska).
*/
alpha?: 'discard' | 'keep';
/** Configures the bitrate mode; defaults to `'variable'`. */
@@ -344,6 +345,7 @@ export const buildVideoEncoderConfig = (options: {
options.width,
options.height,
resolvedBitrate,
options.alpha === 'keep',
),
width: options.width,
height: options.height,
test/node/prores.test.ts
+8 -2
View File
@@ -137,10 +137,16 @@ test('ProRes decoding', { timeout: 10_000 }, async () => {
expect(sample.timestamp).toBe(firstTimestamp);
expect(sample.duration).toBeGreaterThan(0);
expect(sample.displayWidth).toBe(await videoTrack.getDisplayWidth());
expect(sample.displayHeight).toBe(await videoTrack.getDisplayHeight());
expect(sample.displayWidth).toBe(1920);
expect(sample.displayHeight).toBe(1080);
expect(sample.codedWidth).toBe(1920);
expect(sample.codedHeight).toBe(1080); // Technically a lie but the field is ill-defined
expect(sample.visibleRect).toEqual({
left: 0,
top: 0,
width: 1920,
height: 1080,
});
expect(sample.format).toBe('I422P10');
const allocationSize = sample.allocationSize();
test/node/server-extension.test.ts
+193 -4
View File
@@ -15,6 +15,7 @@ import {
buildAudioCodecString,
buildVideoCodecString,
NON_PCM_AUDIO_CODECS,
PRORES_FOURCCS,
VideoCodec,
} from '../../src/codec.js';
import { EncodedPacket } from '../../src/packet.js';
@@ -30,6 +31,8 @@ import { Output } from '../../src/output.js';
import { Mp4OutputFormat } from '../../src/output-format.js';
import { BufferTarget } from '../../src/target.js';
import { Conversion } from '../../src/conversion.js';
import { VideoSampleSource } from '../../src/media-source.js';
import { QUALITY_HIGH } from '../../src/encode.js';
import { AvFrameVideoSampleResource } from '../../packages/server/src/video-sample.js';
import { AvFrameAudioSampleResource } from '../../packages/server/src/audio-sample.js';
import { toAvFrame } from '../../packages/server/src/index.js';
@@ -113,7 +116,7 @@ describe('Video', async () => {
encoder.codec = 'avc';
// @ts-expect-error Readonly
encoder.config = {
codec: buildVideoCodecString('avc', 1280, 720, 1e6),
codec: buildVideoCodecString('avc', 1280, 720, 1e6, false),
width: 1280,
height: 720,
bitrate: 1e6,
@@ -196,7 +199,7 @@ describe('Video', async () => {
encoder.codec = 'avc';
// @ts-expect-error Readonly
encoder.config = {
codec: buildVideoCodecString('avc', 1280, 720, 1e6),
codec: buildVideoCodecString('avc', 1280, 720, 1e6, false),
width: 1280,
height: 720,
bitrate: 1e6,
@@ -471,6 +474,188 @@ describe('Video', async () => {
});
});
test('ProRes encode', async () => {
// ProRes can't be decoded by node-av's wrapper here, so this is encode-only.
const encoder = new NodeAvVideoEncoder();
// @ts-expect-error Readonly
encoder.codec = 'prores';
// @ts-expect-error Readonly
encoder.config = {
codec: 'apch',
width: 1280,
height: 720,
bitrate: 1e6,
} satisfies VideoEncoderConfig;
const packets: EncodedPacket[] = [];
const sampleTimestamps: number[] = [];
// @ts-expect-error Readonly
encoder.onPacket = (packet: EncodedPacket, meta: EncodedVideoChunkMetadata) => {
expect(packet.duration).toBe(1 / 30);
expect(packet.type).toBe('key');
expect(String.fromCharCode(...packet.data.slice(4, 8))).toBe('icpf');
if (packets.length === 0) {
expect(meta.decoderConfig).toBeDefined();
expect(meta.decoderConfig!.codec).toBe('apch');
expect(meta.decoderConfig!.codedWidth).toBe(1280);
expect(meta.decoderConfig!.codedHeight).toBe(720);
expect(meta.decoderConfig!.description).toBeUndefined();
}
packets.push(packet);
};
await encoder.init();
const data = new Uint8Array(1280 * 720 * 4).fill(0xff); // White
for (let i = 0; i < 10; i++) {
using sample = new VideoSample(data, {
format: 'RGBX',
codedWidth: 1280,
codedHeight: 720,
timestamp: i / 30,
duration: 1 / 30,
});
sampleTimestamps.push(sample.timestamp);
await encoder.encode(sample, {});
}
await encoder.flush();
expect(packets).toHaveLength(10);
expect(packets.map(x => x.timestamp)).toEqual(sampleTimestamps);
await encoder.close();
});
test('ProRes Proxy encode', async () => {
const encoder = new NodeAvVideoEncoder();
// @ts-expect-error Readonly
encoder.codec = 'prores';
// @ts-expect-error Readonly
encoder.config = {
codec: 'apco', // Force ProRes 422 Proxy
width: 1280,
height: 720,
bitrate: 1e6,
} satisfies VideoEncoderConfig;
const packets: EncodedPacket[] = [];
const sampleTimestamps: number[] = [];
// @ts-expect-error Readonly
encoder.onPacket = (packet: EncodedPacket, meta: EncodedVideoChunkMetadata) => {
expect(packet.duration).toBe(1 / 30);
expect(packet.type).toBe('key');
expect(String.fromCharCode(...packet.data.slice(4, 8))).toBe('icpf');
if (packets.length === 0) {
expect(meta.decoderConfig).toBeDefined();
expect(PRORES_FOURCCS).toContain(meta.decoderConfig!.codec);
expect(meta.decoderConfig!.codec).toBe('apco');
expect(meta.decoderConfig!.codedWidth).toBe(1280);
expect(meta.decoderConfig!.codedHeight).toBe(720);
expect(meta.decoderConfig!.description).toBeUndefined();
}
packets.push(packet);
};
await encoder.init();
const data = new Uint8Array(1280 * 720 * 4).fill(0xff); // White
for (let i = 0; i < 10; i++) {
using sample = new VideoSample(data, {
format: 'RGBX',
codedWidth: 1280,
codedHeight: 720,
timestamp: i / 30,
duration: 1 / 30,
});
sampleTimestamps.push(sample.timestamp);
await encoder.encode(sample, {});
}
await encoder.flush();
expect(packets).toHaveLength(10);
expect(packets.map(x => x.timestamp)).toEqual(sampleTimestamps);
await encoder.close();
});
test('Transparent ProRes encode and decode', async () => {
const output = new Output({
format: new Mp4OutputFormat(),
target: new BufferTarget(),
});
const source = new VideoSampleSource({
codec: 'prores',
bitrate: QUALITY_HIGH,
alpha: 'keep',
});
output.addVideoTrack(source);
await output.start();
// Solid red at 50% alpha
const data = new Uint8Array(640 * 480 * 4);
for (let i = 0; i < data.byteLength; i += 4) {
data[i] = 0xff;
data[i + 3] = 0x80;
}
using inputSample = new VideoSample(data, {
format: 'RGBA',
codedWidth: 640,
codedHeight: 480,
timestamp: 0,
duration: 1,
});
await source.add(inputSample);
await output.finalize();
using input = new Input({
source: new BufferSource(output.target.buffer!),
formats: ALL_FORMATS,
});
const videoTrack = (await input.getPrimaryVideoTrack())!;
expect(await videoTrack.canBeTransparent()).toBe(true);
const sink = new VideoSampleSink(videoTrack);
using sample = (await sink.getSample(0))!;
expect(sample.format).toBe('I444AP12');
const decodedData = new Uint8Array(sample.allocationSize());
const layout = await sample.copyTo(decodedData);
const view = new DataView(decodedData.buffer);
const alphaPlane = layout[3]!;
let minAlpha = Infinity;
let maxAlpha = -Infinity;
for (let y = 0; y < sample.codedHeight; y++) {
for (let x = 0; x < sample.codedWidth; x++) {
const alpha = view.getUint16(alphaPlane.offset + y * alphaPlane.stride + x * 2, true);
minAlpha = Math.min(minAlpha, alpha);
maxAlpha = Math.max(maxAlpha, alpha);
}
}
expect(maxAlpha).toBeLessThan(4095);
expect(Math.abs(minAlpha - 4095 / 2)).toBeLessThan(32);
expect(Math.abs(maxAlpha - 4095 / 2)).toBeLessThan(32);
});
test('Non-square pixels encode and decode #1', async () => {
await encodeDecodeTest('avc', { displayWidth: 2 * 1280, displayHeight: 720 }, async (packet, meta, i) => {
if (i === 0) {
@@ -517,7 +702,7 @@ describe('Video', async () => {
encoder.codec = codec;
// @ts-expect-error Readonly
encoder.config = {
codec: buildVideoCodecString(codec, 1280, 720, 1e6),
codec: buildVideoCodecString(codec, 1280, 720, 1e6, false),
width: 1280,
height: 720,
bitrate: 1e6,
@@ -616,6 +801,10 @@ describe('Video', async () => {
await conversionRoundtrip('av1');
});
test('ProRes conversion roundtrip', { timeout: 20_000 }, async () => {
await conversionRoundtrip('prores');
});
const conversionRoundtrip = async (codec: VideoCodec, duration?: number) => {
using input = new Input({
source: new FilePathSource('./test/public/video.mp4'),
@@ -878,7 +1067,7 @@ describe('Video', async () => {
encoder.codec = 'avc';
// @ts-expect-error Readonly
encoder.config = {
codec: buildVideoCodecString('avc', 1920, 1080, 1e6),
codec: buildVideoCodecString('avc', 1920, 1080, 1e6, false),
width: 1920,
height: 1080,
bitrate: 1e6,