mirror of
https://github.com/arcodange-org/mediabunny.git
synced 2026-10-10 09:13:47 +02:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0480ecbd3a | ||
|
|
9547d36353 | ||
|
|
d8d22aa4eb | ||
|
|
50b281a9f8 | ||
|
|
409961c02b | ||
|
|
d8a6e09a85 | ||
|
|
2c7e8d85ca | ||
|
|
a9441aa9a9 | ||
|
|
403093c514 | ||
|
|
6c6687dfaf | ||
|
|
7d89c8de2d | ||
|
|
b4ef20cd36 | ||
|
|
a4884184e9 | ||
|
|
6f4a40ff4e | ||
|
|
7453336f70 | ||
|
|
bfbcea69b4 | ||
|
|
40e35d6f14 | ||
|
|
8716ddda14 | ||
|
|
8ae0f9d09e | ||
|
|
d5c94a23ab | ||
|
|
af618f014a | ||
|
|
54e74bfa4b | ||
|
|
df76bae503 | ||
|
|
b617c5f720 | ||
|
|
a5df6c7b89 | ||
|
|
1ddeae2e80 | ||
|
|
210d382113 | ||
|
|
a294d943e6 | ||
|
|
9a8ccf6f6c | ||
|
|
f34e4cd369 | ||
|
|
93e6de980d | ||
|
|
e6fe540cbc | ||
|
|
23e6901046 | ||
|
|
143ecea6c0 | ||
|
|
067081d5eb | ||
|
|
f55a2b4764 | ||
|
|
1b1c9a0299 | ||
|
|
2b2e80dc24 | ||
|
|
13d1e280e8 | ||
|
|
62287156d6 | ||
|
|
100852f266 | ||
|
|
f8f08a48e1 | ||
|
|
71dd34fc3c | ||
|
|
487ae3c69a | ||
|
|
0d5bf18864 | ||
|
|
fc67e436e3 | ||
|
|
735a6622a2 | ||
|
|
d51903edfe | ||
|
|
6cdaa76fbb | ||
|
|
8efe7e75db | ||
|
|
a892676453 | ||
|
|
00f020921b | ||
|
|
8071933f15 | ||
|
|
d97611453b | ||
|
|
5e59ae4a1c | ||
|
|
3b148f961a | ||
|
|
95d60805ed | ||
|
|
857956fb07 | ||
|
|
e9fc97e070 | ||
|
|
15db8a27d4 | ||
|
|
637d77b2cf | ||
|
|
b6bb382783 | ||
|
|
7bb9961156 | ||
|
|
776ae43f50 | ||
|
|
e533bab18a | ||
|
|
26f3a576f3 | ||
|
|
1f6457a50c | ||
|
|
b2d00f84d8 | ||
|
|
f6aee7b414 | ||
|
|
0cb4665d8e | ||
|
|
525a03d220 | ||
|
|
808422f4f5 | ||
|
|
e6110a5b9b | ||
|
|
cd2ded8767 | ||
|
|
db91d07780 | ||
|
|
af57c431d3 | ||
|
|
049f24f2cb | ||
|
|
aa509cd62e | ||
|
|
60789841a8 | ||
|
|
b6d3e3269d | ||
|
|
cea3a40902 | ||
|
|
201a1b4066 | ||
|
|
882051daf2 | ||
|
|
91f96458fa | ||
|
|
9d87a3e580 | ||
|
|
040b0747be | ||
|
|
53923e72dc | ||
|
|
46b3ce39fb | ||
|
|
14ae14b565 | ||
|
|
70e3e98b53 | ||
|
|
6b0527b691 | ||
|
|
3e333636ec | ||
|
|
a583b9d168 | ||
|
|
29aaaf64bc | ||
|
|
5c45fd1f63 | ||
|
|
18d64ffb33 | ||
|
|
0dd53ff1f7 | ||
|
|
7232296217 | ||
|
|
30fc407c4c | ||
|
|
52fd8d0e19 | ||
|
|
a3e43987ed | ||
|
|
823ede1668 | ||
|
|
1703e70a16 | ||
|
|
df0932d49e | ||
|
|
64468b0e5b | ||
|
|
903e36430f | ||
|
|
dbb4150718 | ||
|
|
9d1f4a5f80 | ||
|
|
76ac4e7aba | ||
|
|
f0be3ef134 | ||
|
|
77fa426a9a | ||
|
|
b4cbf4c502 | ||
|
|
27bdd33d42 | ||
|
|
5972a5c906 | ||
|
|
da539cf495 | ||
|
|
987ae467ce | ||
|
|
2b5971d1ee | ||
|
|
d735f16ca5 | ||
|
|
9926004401 | ||
|
|
34628390b7 | ||
|
|
1e46df030f | ||
|
|
a6fd297573 | ||
|
|
447dd51e7b | ||
|
|
aebfede89a | ||
|
|
e0a1211140 | ||
|
|
aa8003e679 | ||
|
|
c74b6a44fb | ||
|
|
f21182a7b9 | ||
|
|
fb2dcba707 | ||
|
|
e25525a564 | ||
|
|
f356c05a9c | ||
|
|
4e1428fcef | ||
|
|
ca48f31032 | ||
|
|
b958dd84cc | ||
|
|
f72db4dfa2 | ||
|
|
e9534f57c6 | ||
|
|
31fdd363e3 | ||
|
|
14a55fc476 | ||
|
|
0fc3bb2b1d | ||
|
|
4f670bbe06 | ||
|
|
02f379d9c1 | ||
|
|
588495268c | ||
|
|
230778d887 | ||
|
|
32267f348a | ||
|
|
2aa0fd1433 | ||
|
|
bdc59306ed | ||
|
|
03c1c46f9a | ||
|
|
b4f76fc082 | ||
|
|
d55431dda5 | ||
|
|
6679b6b15d | ||
|
|
ad4dc659a5 | ||
|
|
95fbec86e8 |
@@ -0,0 +1,28 @@
|
||||
name: Build check
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
build-check:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
|
||||
with:
|
||||
node-version: 22
|
||||
cache: 'npm'
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
# Building the docs also means building the core library
|
||||
- name: Build everything
|
||||
run: npm run docs:build
|
||||
@@ -27,7 +27,4 @@ jobs:
|
||||
run: npm run check
|
||||
|
||||
- name: Run ESLint
|
||||
run: npm run lint
|
||||
|
||||
- name: Run build
|
||||
run: npm run build
|
||||
run: npm run lint
|
||||
@@ -60,8 +60,9 @@ jobs:
|
||||
- name: Run tests
|
||||
run: xvfb-run npm test
|
||||
|
||||
- name: Run build
|
||||
run: npm run build
|
||||
# Building the docs also builds the core library
|
||||
- name: Build everything
|
||||
run: npm run docs:build
|
||||
|
||||
- name: Upload build artifacts
|
||||
env:
|
||||
@@ -93,6 +94,11 @@ jobs:
|
||||
packages/flac-encoder/dist/bundles/mediabunny-flac-encoder.mjs
|
||||
packages/flac-encoder/dist/bundles/mediabunny-flac-encoder.min.mjs
|
||||
packages/flac-encoder/dist/mediabunny-flac-encoder.d.ts
|
||||
packages/prores/dist/bundles/mediabunny-prores.js
|
||||
packages/prores/dist/bundles/mediabunny-prores.min.js
|
||||
packages/prores/dist/bundles/mediabunny-prores.mjs
|
||||
packages/prores/dist/bundles/mediabunny-prores.min.mjs
|
||||
packages/prores/dist/mediabunny-prores.d.ts
|
||||
packages/server/dist/bundles/mediabunny-server.cjs
|
||||
packages/server/dist/bundles/mediabunny-server.min.cjs
|
||||
packages/server/dist/bundles/mediabunny-server.mjs
|
||||
|
||||
+2
-1
@@ -10,4 +10,5 @@ packages/mp3-encoder/dist
|
||||
packages/ac3/dist
|
||||
packages/aac-encoder/dist
|
||||
packages/flac-encoder/dist
|
||||
packages/server/dist
|
||||
packages/server/dist
|
||||
packages/prores/dist
|
||||
@@ -121,6 +121,7 @@ export default withMermaid({
|
||||
{ text: 'aac-encoder', link: '/guide/extensions/aac-encoder' },
|
||||
{ text: 'ac3', link: '/guide/extensions/ac3' },
|
||||
{ text: 'flac-encoder', link: '/guide/extensions/flac-encoder' },
|
||||
{ text: 'prores', link: '/guide/extensions/prores' },
|
||||
],
|
||||
},
|
||||
],
|
||||
@@ -143,6 +144,7 @@ export default withMermaid({
|
||||
{ text: 'VP8', link: '/codec-registry/vp8' },
|
||||
{ text: 'VP9', link: '/codec-registry/vp9' },
|
||||
{ text: 'AV1', link: '/codec-registry/av1' },
|
||||
{ text: 'ProRes', link: '/codec-registry/prores' },
|
||||
],
|
||||
},
|
||||
{
|
||||
|
||||
@@ -26,5 +26,6 @@
|
||||
"@mediabunny/mp3-encoder": "Adds MP3 encoder support to Mediabunny.",
|
||||
"@mediabunny/ac3": "Adds AC-3/E-AC-3 decoder and encoder support to Mediabunny.",
|
||||
"@mediabunny/aac-encoder": "Polyfills AAC encoder support to Mediabunny.",
|
||||
"@mediabunny/flac-encoder": "Adds FLAC encoder support to Mediabunny."
|
||||
"@mediabunny/flac-encoder": "Adds FLAC encoder support to Mediabunny.",
|
||||
"@mediabunny/prores": "Adds Apple ProRes decoder support to Mediabunny."
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@ The registry is an extension of the [WebCodecs Codec Registry](https://www.w3.or
|
||||
- [VP8](./vp8)
|
||||
- [VP9](./vp9)
|
||||
- [AV1](./av1)
|
||||
- [ProRes](./prores)
|
||||
|
||||
## Audio codecs
|
||||
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
---
|
||||
description: Apple ProRes video codec definition, defining legal codec strings, decoder configs, and packet data formats.
|
||||
---
|
||||
|
||||
<script setup>
|
||||
import { VPBadge } from 'vitepress/theme'
|
||||
</script>
|
||||
|
||||
<VPBadge type="info" text="Video codec" />
|
||||
|
||||
# ProRes codec registration
|
||||
|
||||
## Description
|
||||
|
||||
The Apple ProRes video codec, as specified in [SMPTE RDD 36](https://pub.smpte.org/doc/rdd36/).
|
||||
|
||||
## Codec ID
|
||||
|
||||
```ts
|
||||
'prores'
|
||||
```
|
||||
|
||||
## `EncodedPacket` data
|
||||
|
||||
The packet's data must be a `frame()` as defined in [SMPTE RDD 36](https://pub.smpte.org/doc/rdd36/).
|
||||
|
||||
## `EncodedPacket` type
|
||||
|
||||
Since ProRes is intra-frame-only, every packet is a key frame and its type is therefore always `'key'`.
|
||||
|
||||
## `VideoDecoderConfig` codec string
|
||||
|
||||
The codec string must be one of the six four-character codes identifying the ProRes profile:
|
||||
|
||||
- `'ap4x'` - ProRes 4444 XQ
|
||||
- `'ap4h'` - ProRes 4444
|
||||
- `'apch'` - ProRes 422 High Quality
|
||||
- `'apcn'` - ProRes 422 Standard Definition
|
||||
- `'apcs'` - ProRes 422 LT
|
||||
- `'apco'` - ProRes 422 Proxy
|
||||
|
||||
## `VideoDecoderConfig` description
|
||||
|
||||
`description` is not used for this codec.
|
||||
@@ -0,0 +1,55 @@
|
||||
---
|
||||
description: The @mediabunny/prores extension provides an extremely fast Apple ProRes decoder for the browser.
|
||||
---
|
||||
|
||||
# @mediabunny/prores
|
||||
|
||||
Browsers have no support for Apple ProRes in their WebCodecs implementations. This extension package provides a decoder for use with Mediabunny, allowing you to decode ProRes directly in the browser at unprecedented speed. It is implemented using Mediabunny's [custom coder API](https://mediabunny.dev/guide/supported-formats-and-codecs#custom-coders) and uses [TurboRes](https://github.com/Vanilagy/turbores), an extremely fast WASM-based ProRes decoder, under the hood.
|
||||
|
||||
<a class="!no-underline inline-flex items-center gap-1.5" :no-icon="true" href="https://github.com/Vanilagy/mediabunny/blob/main/packages/prores/README.md">
|
||||
GitHub page
|
||||
<span class="vpi-arrow-right" />
|
||||
</a>
|
||||
|
||||
## Installation
|
||||
|
||||
This library peer-depends on Mediabunny. Install both using npm:
|
||||
```bash
|
||||
npm install mediabunny @mediabunny/prores
|
||||
```
|
||||
|
||||
Alternatively, directly include them using a script tag:
|
||||
```html
|
||||
<script src="mediabunny.js"></script>
|
||||
<script src="mediabunny-prores.js"></script>
|
||||
```
|
||||
|
||||
This will expose the global objects `Mediabunny` and `MediabunnyProres`. Use `mediabunny-prores.d.ts` to provide types for these globals. You can download the built distribution files from the [releases page](https://github.com/Vanilagy/mediabunny/releases).
|
||||
|
||||
## Setup
|
||||
|
||||
`@mediabunny/prores` can make use of shared-memory multithreading to achieve maximum performance. To enable this in browsers, your website must be cross-origin isolated by setting the following response headers:
|
||||
```
|
||||
Cross-Origin-Opener-Policy: same-origin
|
||||
Cross-Origin-Embedder-Policy: require-corp
|
||||
```
|
||||
|
||||
Alternatively, you can use:
|
||||
```
|
||||
Cross-Origin-Opener-Policy: same-origin
|
||||
Cross-Origin-Embedder-Policy: credentialless
|
||||
```
|
||||
which is generally more permissive but is not supported in Safari (of course).
|
||||
|
||||
---
|
||||
|
||||
If you cannot enable cross-origin isolation, `@mediabunny/prores` will fall back to a slower multithreading algorithm.
|
||||
|
||||
## Usage
|
||||
|
||||
```ts
|
||||
import { registerProresDecoder } from '@mediabunny/prores';
|
||||
|
||||
registerProresDecoder();
|
||||
```
|
||||
That's it - Mediabunny now uses the registered ProRes decoder automatically.
|
||||
@@ -7,7 +7,7 @@ description: Add full video and audio decoder and encoder support for use in ser
|
||||
By default, Mediabunny requires a browser environment for full access to decoders, encoders, and video processing features. `@mediabunny/server` uses [NodeAV](https://github.com/seydx/node-av) to polyfill this functionality for server-side environments such as Node, Bun, or Deno, enabling the usage of all Mediabunny features on the server. The result is a server-side media processing API that integrates naturally with TypeScript as opposed to the awkwardness and inefficiencies of calling out to the FFmpeg CLI.
|
||||
|
||||
Features added by this package include:
|
||||
- Video decoders and encoders for AVC (H.264), HEVC (H.265), VP8, VP9, and AV1. Supports both length-prefixed and Annex B AVC/HEVC as well as transparent video via VP9.
|
||||
- Video decoders and encoders for AVC (H.264), HEVC (H.265), VP8, VP9, AV1, and ProRes. Supports both length-prefixed and Annex B AVC/HEVC as well as transparent video via VP9 and ProRes.
|
||||
- Audio decoders and encoders for AAC, MP3, Vorbis, Opus, FLAC, AC-3 and E-AC-3. Supports AAC in both AAC and ADTS formats.
|
||||
- Video frame transformation support (resize, rotate, crop)
|
||||
- Automatic hardware acceleration on all platforms (macOS, Linux, Windows)
|
||||
|
||||
@@ -40,6 +40,7 @@ Mediabunny ships with built-in decoders and encoders for all audio PCM codecs, m
|
||||
- `'vp8'` - VP8
|
||||
- `'vp9'` - VP9
|
||||
- `'av1'` - AOMedia Video 1 (AV1)
|
||||
- `'prores'` - Apple ProRes [^prores]
|
||||
|
||||
### Audio codecs
|
||||
|
||||
@@ -80,6 +81,7 @@ Not all codecs can be used with all containers. The following table specifies th
|
||||
| `'vp8'` | ✓ | ✓ | ✓ | ✓ | | | | | | |
|
||||
| `'vp9'` | ✓ | ✓ | ✓ | ✓ | | | | | | |
|
||||
| `'av1'` | ✓ | ✓ | ✓ | ✓ | | | | | | |
|
||||
| `'prores'` | ✓ | ✓ | ✓ | | | | | | | |
|
||||
| `'aac'` | ✓ | ✓ | ✓ | | | | | ✓ | | ✓ |
|
||||
| `'opus'` | ✓ | ✓ | ✓ | ✓ | ✓ | | | | | |
|
||||
| `'mp3'` | ✓ | ✓ | ✓ | | | ✓ | | | | ✓ |
|
||||
@@ -105,6 +107,7 @@ Not all codecs can be used with all containers. The following table specifies th
|
||||
|
||||
For HLS, the supported codecs depend on the segment format chosen.
|
||||
|
||||
[^prores]: ProRes is not supported by WebCodecs. To decode it, use the [`@mediabunny/prores`](./extensions/prores) extension package. The [`@mediabunny/server`](./extensions/server) extension package provides both decoding and encoding support for server-side environments.
|
||||
[^aac]: In some browsers, AAC encoding is not supported by WebCodecs. You can polyfill it with the [`@mediabunny/aac-encoder`](./extensions/aac-encoder) extension package.
|
||||
[^mp3]: MP3 encoding is not supported by WebCodecs. You can polyfill it with the [`@mediabunny/mp3-encoder`](./extensions/mp3-encoder) extension package.
|
||||
[^flac]: FLAC encoding is not supported by WebCodecs. You can polyfill it with the [`@mediabunny/flac-encoder`](./extensions/flac-encoder) extension package.
|
||||
@@ -293,10 +296,13 @@ class {
|
||||
codec: AudioCodec;
|
||||
config: AudioEncoderConfig;
|
||||
onPacket: (packet: EncodedPacket, meta?: EncodedAudioChunkMetadata) => unknown;
|
||||
|
||||
// For both:
|
||||
onError: (error: unknown) => void;
|
||||
}
|
||||
```
|
||||
|
||||
`codec` and `config` specify the concrete codec configuration to use, and `onPacket` is a method that your code **must** call for each encoded packet it creates.
|
||||
`codec` and `config` specify the concrete codec configuration to use, and `onPacket` is a method that your code **must** call for each encoded packet it creates. `onError` is a method you can call to surface any out-of-band errors that occur outside of the regular method calls (such as from an asynchronous background task); these errors would otherwise go uncaught.
|
||||
|
||||
You **must** implement the following methods in your custom encoder class:
|
||||
```ts
|
||||
@@ -356,10 +362,13 @@ class {
|
||||
codec: AudioCodec;
|
||||
config: AudioDecoderConfig;
|
||||
onSample: (sample: AudioSample) => unknown;
|
||||
|
||||
// For both:
|
||||
onError: (error: unknown) => void;
|
||||
}
|
||||
```
|
||||
|
||||
`codec` and `config` specify the concrete codec configuration to use, and `onSample` is a method that your code **must** call for each video/audio sample it creates.
|
||||
`codec` and `config` specify the concrete codec configuration to use, and `onSample` is a method that your code **must** call for each video/audio sample it creates. `onError` is a method you can call to surface any out-of-band errors that occur outside of the regular method calls (such as from an asynchronous background task); these errors would otherwise go uncaught.
|
||||
|
||||
You **must** implement the following methods in your custom decoder class:
|
||||
```ts
|
||||
|
||||
@@ -47,6 +47,7 @@ export default tseslint.config(
|
||||
'packages/aac-encoder/build',
|
||||
'packages/flac-encoder/dist',
|
||||
'packages/flac-encoder/build',
|
||||
'packages/prores/dist',
|
||||
'packages/server/dist',
|
||||
'eslint.config.mjs',
|
||||
'docs/.vitepress/cache',
|
||||
|
||||
@@ -68,6 +68,7 @@ const compressFile = async (resource: File | string) => {
|
||||
bitrate: QUALITY_VERY_LOW,
|
||||
},
|
||||
audio: {
|
||||
codec: 'opus',
|
||||
bitrate: QUALITY_VERY_LOW,
|
||||
},
|
||||
});
|
||||
|
||||
@@ -215,7 +215,7 @@ const initMediaPlayer = async (resource: File | string) => {
|
||||
|
||||
if (audioContext.state === 'running') {
|
||||
// Start playback automatically if the audio context permits
|
||||
await play();
|
||||
// await play();
|
||||
}
|
||||
|
||||
loadingElement.style.display = 'none';
|
||||
|
||||
Generated
+55
-9
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "mediabunny",
|
||||
"version": "1.49.0",
|
||||
"version": "1.50.1",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "mediabunny",
|
||||
"version": "1.49.0",
|
||||
"version": "1.50.1",
|
||||
"license": "MPL-2.0",
|
||||
"workspaces": [
|
||||
".",
|
||||
@@ -1461,6 +1461,10 @@
|
||||
"resolved": "packages/mp3-encoder",
|
||||
"link": true
|
||||
},
|
||||
"node_modules/@mediabunny/prores": {
|
||||
"resolved": "packages/prores",
|
||||
"link": true
|
||||
},
|
||||
"node_modules/@mediabunny/server": {
|
||||
"resolved": "packages/server",
|
||||
"link": true
|
||||
@@ -10958,6 +10962,32 @@
|
||||
"license": "0BSD",
|
||||
"optional": true
|
||||
},
|
||||
"node_modules/turbores": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/turbores/-/turbores-1.1.2.tgz",
|
||||
"integrity": "sha512-6k3XH+sG8uOCJEBamKhAXFwIAfLqz7LAQr2mmEx2UBl6zC+4XAGQtp1fGEugnVl2w8Ec5GOUvD+b0tyxUJvmHg==",
|
||||
"license": "MPL-2.0",
|
||||
"funding": {
|
||||
"type": "individual",
|
||||
"url": "https://github.com/sponsors/Vanilagy"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@rollup/rollup-linux-x64-gnu": "4.62.1"
|
||||
}
|
||||
},
|
||||
"node_modules/turbores/node_modules/@rollup/rollup-linux-x64-gnu": {
|
||||
"version": "4.62.1",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.62.1.tgz",
|
||||
"integrity": "sha512-nVQGk/jStQc2V4rrkI+vPD2J+85boKqS4R4nOdPhc3eWw0kyW/b+AYRGoH8qo057XSVqaTx13AliH5qPeLTtgQ==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
]
|
||||
},
|
||||
"node_modules/tweetnacl": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-1.0.3.tgz",
|
||||
@@ -12864,7 +12894,7 @@
|
||||
},
|
||||
"packages/aac-encoder": {
|
||||
"name": "@mediabunny/aac-encoder",
|
||||
"version": "1.49.0",
|
||||
"version": "1.50.1",
|
||||
"license": "MPL-2.0",
|
||||
"devDependencies": {
|
||||
"@types/emscripten": "^1.40.1"
|
||||
@@ -12879,7 +12909,7 @@
|
||||
},
|
||||
"packages/ac3": {
|
||||
"name": "@mediabunny/ac3",
|
||||
"version": "1.49.0",
|
||||
"version": "1.50.1",
|
||||
"license": "MPL-2.0",
|
||||
"devDependencies": {
|
||||
"@types/emscripten": "^1.40.1"
|
||||
@@ -12894,7 +12924,7 @@
|
||||
},
|
||||
"packages/flac-encoder": {
|
||||
"name": "@mediabunny/flac-encoder",
|
||||
"version": "1.49.0",
|
||||
"version": "1.50.1",
|
||||
"license": "MPL-2.0",
|
||||
"devDependencies": {
|
||||
"@types/emscripten": "^1.40.1"
|
||||
@@ -12909,7 +12939,7 @@
|
||||
},
|
||||
"packages/mp3-encoder": {
|
||||
"name": "@mediabunny/mp3-encoder",
|
||||
"version": "1.49.0",
|
||||
"version": "1.50.1",
|
||||
"license": "MPL-2.0",
|
||||
"devDependencies": {
|
||||
"@types/emscripten": "^1.40.1"
|
||||
@@ -12922,11 +12952,27 @@
|
||||
"mediabunny": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"packages/server": {
|
||||
"name": "@mediabunny/server",
|
||||
"version": "1.49.0",
|
||||
"packages/prores": {
|
||||
"name": "@mediabunny/prores",
|
||||
"version": "1.50.1",
|
||||
"license": "MPL-2.0",
|
||||
"dependencies": {
|
||||
"turbores": "^1.1.2"
|
||||
},
|
||||
"funding": {
|
||||
"type": "individual",
|
||||
"url": "https://github.com/sponsors/Vanilagy"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"mediabunny": "^1.49.0"
|
||||
}
|
||||
},
|
||||
"packages/server": {
|
||||
"name": "@mediabunny/server",
|
||||
"version": "1.50.1",
|
||||
"license": "MPL-2.0",
|
||||
"dependencies": {
|
||||
"@mediabunny/prores": "^1.50.0",
|
||||
"node-av": "^6.0.0"
|
||||
},
|
||||
"funding": {
|
||||
|
||||
+2
-2
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "mediabunny",
|
||||
"author": "Vanilagy",
|
||||
"version": "1.49.0",
|
||||
"version": "1.50.1",
|
||||
"description": "Pure TypeScript media toolkit for reading, writing, and converting media files, directly in the browser.",
|
||||
"type": "module",
|
||||
"workspaces": [
|
||||
@@ -51,7 +51,7 @@
|
||||
"docs:dev": "vitepress dev docs",
|
||||
"docs:build": "npm run build && npm run docs:generate && vitepress build docs && npm run examples:build && cp dist/mediabunny.d.ts dist-docs/",
|
||||
"docs:preview": "vitepress preview docs",
|
||||
"docs:generate": "tsx scripts/generate-api-docs.ts src/index.ts packages/mp3-encoder/src/index.ts packages/ac3/src/index.ts packages/aac-encoder/src/index.ts packages/flac-encoder/src/index.ts packages/server/src/index.ts docs/api-config.json",
|
||||
"docs:generate": "tsx scripts/generate-api-docs.ts src/index.ts packages/mp3-encoder/src/index.ts packages/ac3/src/index.ts packages/aac-encoder/src/index.ts packages/flac-encoder/src/index.ts packages/prores/src/index.ts packages/server/src/index.ts docs/api-config.json",
|
||||
"dev": "vite",
|
||||
"examples:build": "vite build",
|
||||
"fix-build-import-paths": "tsx scripts/add-import-extensions.ts",
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "@mediabunny/aac-encoder",
|
||||
"author": "Vanilagy",
|
||||
"version": "1.49.0",
|
||||
"version": "1.50.1",
|
||||
"description": "AAC encoder extension for Mediabunny, based on FFmpeg.",
|
||||
"main": "./dist/bundles/mediabunny-aac-encoder.mjs",
|
||||
"module": "./dist/bundles/mediabunny-aac-encoder.mjs",
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "@mediabunny/ac3",
|
||||
"author": "Vanilagy",
|
||||
"version": "1.49.0",
|
||||
"version": "1.50.1",
|
||||
"description": "AC-3 and E-AC-3 (Dolby Digital) decoder and encoder extension for Mediabunny, based on FFmpeg.",
|
||||
"main": "./dist/bundles/mediabunny-ac3.mjs",
|
||||
"module": "./dist/bundles/mediabunny-ac3.mjs",
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "@mediabunny/flac-encoder",
|
||||
"author": "Vanilagy",
|
||||
"version": "1.49.0",
|
||||
"version": "1.50.1",
|
||||
"description": "FLAC encoder extension for Mediabunny, based on libFLAC.",
|
||||
"main": "./dist/bundles/mediabunny-flac-encoder.mjs",
|
||||
"module": "./dist/bundles/mediabunny-flac-encoder.mjs",
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "@mediabunny/mp3-encoder",
|
||||
"author": "Vanilagy",
|
||||
"version": "1.49.0",
|
||||
"version": "1.50.1",
|
||||
"description": "MP3 encoder extension for Mediabunny, based on LAME.",
|
||||
"main": "./dist/bundles/mediabunny-mp3-encoder.mjs",
|
||||
"module": "./dist/bundles/mediabunny-mp3-encoder.mjs",
|
||||
|
||||
@@ -0,0 +1,373 @@
|
||||
Mozilla Public License Version 2.0
|
||||
==================================
|
||||
|
||||
1. Definitions
|
||||
--------------
|
||||
|
||||
1.1. "Contributor"
|
||||
means each individual or legal entity that creates, contributes to
|
||||
the creation of, or owns Covered Software.
|
||||
|
||||
1.2. "Contributor Version"
|
||||
means the combination of the Contributions of others (if any) used
|
||||
by a Contributor and that particular Contributor's Contribution.
|
||||
|
||||
1.3. "Contribution"
|
||||
means Covered Software of a particular Contributor.
|
||||
|
||||
1.4. "Covered Software"
|
||||
means Source Code Form to which the initial Contributor has attached
|
||||
the notice in Exhibit A, the Executable Form of such Source Code
|
||||
Form, and Modifications of such Source Code Form, in each case
|
||||
including portions thereof.
|
||||
|
||||
1.5. "Incompatible With Secondary Licenses"
|
||||
means
|
||||
|
||||
(a) that the initial Contributor has attached the notice described
|
||||
in Exhibit B to the Covered Software; or
|
||||
|
||||
(b) that the Covered Software was made available under the terms of
|
||||
version 1.1 or earlier of the License, but not also under the
|
||||
terms of a Secondary License.
|
||||
|
||||
1.6. "Executable Form"
|
||||
means any form of the work other than Source Code Form.
|
||||
|
||||
1.7. "Larger Work"
|
||||
means a work that combines Covered Software with other material, in
|
||||
a separate file or files, that is not Covered Software.
|
||||
|
||||
1.8. "License"
|
||||
means this document.
|
||||
|
||||
1.9. "Licensable"
|
||||
means having the right to grant, to the maximum extent possible,
|
||||
whether at the time of the initial grant or subsequently, any and
|
||||
all of the rights conveyed by this License.
|
||||
|
||||
1.10. "Modifications"
|
||||
means any of the following:
|
||||
|
||||
(a) any file in Source Code Form that results from an addition to,
|
||||
deletion from, or modification of the contents of Covered
|
||||
Software; or
|
||||
|
||||
(b) any new file in Source Code Form that contains any Covered
|
||||
Software.
|
||||
|
||||
1.11. "Patent Claims" of a Contributor
|
||||
means any patent claim(s), including without limitation, method,
|
||||
process, and apparatus claims, in any patent Licensable by such
|
||||
Contributor that would be infringed, but for the grant of the
|
||||
License, by the making, using, selling, offering for sale, having
|
||||
made, import, or transfer of either its Contributions or its
|
||||
Contributor Version.
|
||||
|
||||
1.12. "Secondary License"
|
||||
means either the GNU General Public License, Version 2.0, the GNU
|
||||
Lesser General Public License, Version 2.1, the GNU Affero General
|
||||
Public License, Version 3.0, or any later versions of those
|
||||
licenses.
|
||||
|
||||
1.13. "Source Code Form"
|
||||
means the form of the work preferred for making modifications.
|
||||
|
||||
1.14. "You" (or "Your")
|
||||
means an individual or a legal entity exercising rights under this
|
||||
License. For legal entities, "You" includes any entity that
|
||||
controls, is controlled by, or is under common control with You. For
|
||||
purposes of this definition, "control" means (a) the power, direct
|
||||
or indirect, to cause the direction or management of such entity,
|
||||
whether by contract or otherwise, or (b) ownership of more than
|
||||
fifty percent (50%) of the outstanding shares or beneficial
|
||||
ownership of such entity.
|
||||
|
||||
2. License Grants and Conditions
|
||||
--------------------------------
|
||||
|
||||
2.1. Grants
|
||||
|
||||
Each Contributor hereby grants You a world-wide, royalty-free,
|
||||
non-exclusive license:
|
||||
|
||||
(a) under intellectual property rights (other than patent or trademark)
|
||||
Licensable by such Contributor to use, reproduce, make available,
|
||||
modify, display, perform, distribute, and otherwise exploit its
|
||||
Contributions, either on an unmodified basis, with Modifications, or
|
||||
as part of a Larger Work; and
|
||||
|
||||
(b) under Patent Claims of such Contributor to make, use, sell, offer
|
||||
for sale, have made, import, and otherwise transfer either its
|
||||
Contributions or its Contributor Version.
|
||||
|
||||
2.2. Effective Date
|
||||
|
||||
The licenses granted in Section 2.1 with respect to any Contribution
|
||||
become effective for each Contribution on the date the Contributor first
|
||||
distributes such Contribution.
|
||||
|
||||
2.3. Limitations on Grant Scope
|
||||
|
||||
The licenses granted in this Section 2 are the only rights granted under
|
||||
this License. No additional rights or licenses will be implied from the
|
||||
distribution or licensing of Covered Software under this License.
|
||||
Notwithstanding Section 2.1(b) above, no patent license is granted by a
|
||||
Contributor:
|
||||
|
||||
(a) for any code that a Contributor has removed from Covered Software;
|
||||
or
|
||||
|
||||
(b) for infringements caused by: (i) Your and any other third party's
|
||||
modifications of Covered Software, or (ii) the combination of its
|
||||
Contributions with other software (except as part of its Contributor
|
||||
Version); or
|
||||
|
||||
(c) under Patent Claims infringed by Covered Software in the absence of
|
||||
its Contributions.
|
||||
|
||||
This License does not grant any rights in the trademarks, service marks,
|
||||
or logos of any Contributor (except as may be necessary to comply with
|
||||
the notice requirements in Section 3.4).
|
||||
|
||||
2.4. Subsequent Licenses
|
||||
|
||||
No Contributor makes additional grants as a result of Your choice to
|
||||
distribute the Covered Software under a subsequent version of this
|
||||
License (see Section 10.2) or under the terms of a Secondary License (if
|
||||
permitted under the terms of Section 3.3).
|
||||
|
||||
2.5. Representation
|
||||
|
||||
Each Contributor represents that the Contributor believes its
|
||||
Contributions are its original creation(s) or it has sufficient rights
|
||||
to grant the rights to its Contributions conveyed by this License.
|
||||
|
||||
2.6. Fair Use
|
||||
|
||||
This License is not intended to limit any rights You have under
|
||||
applicable copyright doctrines of fair use, fair dealing, or other
|
||||
equivalents.
|
||||
|
||||
2.7. Conditions
|
||||
|
||||
Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted
|
||||
in Section 2.1.
|
||||
|
||||
3. Responsibilities
|
||||
-------------------
|
||||
|
||||
3.1. Distribution of Source Form
|
||||
|
||||
All distribution of Covered Software in Source Code Form, including any
|
||||
Modifications that You create or to which You contribute, must be under
|
||||
the terms of this License. You must inform recipients that the Source
|
||||
Code Form of the Covered Software is governed by the terms of this
|
||||
License, and how they can obtain a copy of this License. You may not
|
||||
attempt to alter or restrict the recipients' rights in the Source Code
|
||||
Form.
|
||||
|
||||
3.2. Distribution of Executable Form
|
||||
|
||||
If You distribute Covered Software in Executable Form then:
|
||||
|
||||
(a) such Covered Software must also be made available in Source Code
|
||||
Form, as described in Section 3.1, and You must inform recipients of
|
||||
the Executable Form how they can obtain a copy of such Source Code
|
||||
Form by reasonable means in a timely manner, at a charge no more
|
||||
than the cost of distribution to the recipient; and
|
||||
|
||||
(b) You may distribute such Executable Form under the terms of this
|
||||
License, or sublicense it under different terms, provided that the
|
||||
license for the Executable Form does not attempt to limit or alter
|
||||
the recipients' rights in the Source Code Form under this License.
|
||||
|
||||
3.3. Distribution of a Larger Work
|
||||
|
||||
You may create and distribute a Larger Work under terms of Your choice,
|
||||
provided that You also comply with the requirements of this License for
|
||||
the Covered Software. If the Larger Work is a combination of Covered
|
||||
Software with a work governed by one or more Secondary Licenses, and the
|
||||
Covered Software is not Incompatible With Secondary Licenses, this
|
||||
License permits You to additionally distribute such Covered Software
|
||||
under the terms of such Secondary License(s), so that the recipient of
|
||||
the Larger Work may, at their option, further distribute the Covered
|
||||
Software under the terms of either this License or such Secondary
|
||||
License(s).
|
||||
|
||||
3.4. Notices
|
||||
|
||||
You may not remove or alter the substance of any license notices
|
||||
(including copyright notices, patent notices, disclaimers of warranty,
|
||||
or limitations of liability) contained within the Source Code Form of
|
||||
the Covered Software, except that You may alter any license notices to
|
||||
the extent required to remedy known factual inaccuracies.
|
||||
|
||||
3.5. Application of Additional Terms
|
||||
|
||||
You may choose to offer, and to charge a fee for, warranty, support,
|
||||
indemnity or liability obligations to one or more recipients of Covered
|
||||
Software. However, You may do so only on Your own behalf, and not on
|
||||
behalf of any Contributor. You must make it absolutely clear that any
|
||||
such warranty, support, indemnity, or liability obligation is offered by
|
||||
You alone, and You hereby agree to indemnify every Contributor for any
|
||||
liability incurred by such Contributor as a result of warranty, support,
|
||||
indemnity or liability terms You offer. You may include additional
|
||||
disclaimers of warranty and limitations of liability specific to any
|
||||
jurisdiction.
|
||||
|
||||
4. Inability to Comply Due to Statute or Regulation
|
||||
---------------------------------------------------
|
||||
|
||||
If it is impossible for You to comply with any of the terms of this
|
||||
License with respect to some or all of the Covered Software due to
|
||||
statute, judicial order, or regulation then You must: (a) comply with
|
||||
the terms of this License to the maximum extent possible; and (b)
|
||||
describe the limitations and the code they affect. Such description must
|
||||
be placed in a text file included with all distributions of the Covered
|
||||
Software under this License. Except to the extent prohibited by statute
|
||||
or regulation, such description must be sufficiently detailed for a
|
||||
recipient of ordinary skill to be able to understand it.
|
||||
|
||||
5. Termination
|
||||
--------------
|
||||
|
||||
5.1. The rights granted under this License will terminate automatically
|
||||
if You fail to comply with any of its terms. However, if You become
|
||||
compliant, then the rights granted under this License from a particular
|
||||
Contributor are reinstated (a) provisionally, unless and until such
|
||||
Contributor explicitly and finally terminates Your grants, and (b) on an
|
||||
ongoing basis, if such Contributor fails to notify You of the
|
||||
non-compliance by some reasonable means prior to 60 days after You have
|
||||
come back into compliance. Moreover, Your grants from a particular
|
||||
Contributor are reinstated on an ongoing basis if such Contributor
|
||||
notifies You of the non-compliance by some reasonable means, this is the
|
||||
first time You have received notice of non-compliance with this License
|
||||
from such Contributor, and You become compliant prior to 30 days after
|
||||
Your receipt of the notice.
|
||||
|
||||
5.2. If You initiate litigation against any entity by asserting a patent
|
||||
infringement claim (excluding declaratory judgment actions,
|
||||
counter-claims, and cross-claims) alleging that a Contributor Version
|
||||
directly or indirectly infringes any patent, then the rights granted to
|
||||
You by any and all Contributors for the Covered Software under Section
|
||||
2.1 of this License shall terminate.
|
||||
|
||||
5.3. In the event of termination under Sections 5.1 or 5.2 above, all
|
||||
end user license agreements (excluding distributors and resellers) which
|
||||
have been validly granted by You or Your distributors under this License
|
||||
prior to termination shall survive termination.
|
||||
|
||||
************************************************************************
|
||||
* *
|
||||
* 6. Disclaimer of Warranty *
|
||||
* ------------------------- *
|
||||
* *
|
||||
* Covered Software is provided under this License on an "as is" *
|
||||
* basis, without warranty of any kind, either expressed, implied, or *
|
||||
* statutory, including, without limitation, warranties that the *
|
||||
* Covered Software is free of defects, merchantable, fit for a *
|
||||
* particular purpose or non-infringing. The entire risk as to the *
|
||||
* quality and performance of the Covered Software is with You. *
|
||||
* Should any Covered Software prove defective in any respect, You *
|
||||
* (not any Contributor) assume the cost of any necessary servicing, *
|
||||
* repair, or correction. This disclaimer of warranty constitutes an *
|
||||
* essential part of this License. No use of any Covered Software is *
|
||||
* authorized under this License except under this disclaimer. *
|
||||
* *
|
||||
************************************************************************
|
||||
|
||||
************************************************************************
|
||||
* *
|
||||
* 7. Limitation of Liability *
|
||||
* -------------------------- *
|
||||
* *
|
||||
* Under no circumstances and under no legal theory, whether tort *
|
||||
* (including negligence), contract, or otherwise, shall any *
|
||||
* Contributor, or anyone who distributes Covered Software as *
|
||||
* permitted above, be liable to You for any direct, indirect, *
|
||||
* special, incidental, or consequential damages of any character *
|
||||
* including, without limitation, damages for lost profits, loss of *
|
||||
* goodwill, work stoppage, computer failure or malfunction, or any *
|
||||
* and all other commercial damages or losses, even if such party *
|
||||
* shall have been informed of the possibility of such damages. This *
|
||||
* limitation of liability shall not apply to liability for death or *
|
||||
* personal injury resulting from such party's negligence to the *
|
||||
* extent applicable law prohibits such limitation. Some *
|
||||
* jurisdictions do not allow the exclusion or limitation of *
|
||||
* incidental or consequential damages, so this exclusion and *
|
||||
* limitation may not apply to You. *
|
||||
* *
|
||||
************************************************************************
|
||||
|
||||
8. Litigation
|
||||
-------------
|
||||
|
||||
Any litigation relating to this License may be brought only in the
|
||||
courts of a jurisdiction where the defendant maintains its principal
|
||||
place of business and such litigation shall be governed by laws of that
|
||||
jurisdiction, without reference to its conflict-of-law provisions.
|
||||
Nothing in this Section shall prevent a party's ability to bring
|
||||
cross-claims or counter-claims.
|
||||
|
||||
9. Miscellaneous
|
||||
----------------
|
||||
|
||||
This License represents the complete agreement concerning the subject
|
||||
matter hereof. If any provision of this License is held to be
|
||||
unenforceable, such provision shall be reformed only to the extent
|
||||
necessary to make it enforceable. Any law or regulation which provides
|
||||
that the language of a contract shall be construed against the drafter
|
||||
shall not be used to construe this License against a Contributor.
|
||||
|
||||
10. Versions of the License
|
||||
---------------------------
|
||||
|
||||
10.1. New Versions
|
||||
|
||||
Mozilla Foundation is the license steward. Except as provided in Section
|
||||
10.3, no one other than the license steward has the right to modify or
|
||||
publish new versions of this License. Each version will be given a
|
||||
distinguishing version number.
|
||||
|
||||
10.2. Effect of New Versions
|
||||
|
||||
You may distribute the Covered Software under the terms of the version
|
||||
of the License under which You originally received the Covered Software,
|
||||
or under the terms of any subsequent version published by the license
|
||||
steward.
|
||||
|
||||
10.3. Modified Versions
|
||||
|
||||
If you create software not governed by this License, and you want to
|
||||
create a new license for such software, you may create and use a
|
||||
modified version of this License if you rename the license and remove
|
||||
any references to the name of the license steward (except to note that
|
||||
such modified license differs from this License).
|
||||
|
||||
10.4. Distributing Source Code Form that is Incompatible With Secondary
|
||||
Licenses
|
||||
|
||||
If You choose to distribute Source Code Form that is Incompatible With
|
||||
Secondary Licenses under the terms of this version of the License, the
|
||||
notice described in Exhibit B of this License must be attached.
|
||||
|
||||
Exhibit A - Source Code Form License Notice
|
||||
-------------------------------------------
|
||||
|
||||
This Source Code Form is subject to the terms of the Mozilla Public
|
||||
License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
If it is not possible or desirable to put the notice in a particular
|
||||
file, then You may include the notice in a location (such as a LICENSE
|
||||
file in a relevant directory) where a recipient would be likely to look
|
||||
for such a notice.
|
||||
|
||||
You may add additional accurate notices of copyright ownership.
|
||||
|
||||
Exhibit B - "Incompatible With Secondary Licenses" Notice
|
||||
---------------------------------------------------------
|
||||
|
||||
This Source Code Form is "Incompatible With Secondary Licenses", as
|
||||
defined by the Mozilla Public License, v. 2.0.
|
||||
@@ -0,0 +1,63 @@
|
||||
# @mediabunny/prores
|
||||
|
||||
[](https://www.npmjs.com/package/@mediabunny/prores)
|
||||
[](https://bundlephobia.com/package/@mediabunny/prores)
|
||||
[](https://www.npmjs.com/package/@mediabunny/prores)
|
||||
[](https://discord.gg/hmpkyYuS4U)
|
||||
|
||||
<div align="center">
|
||||
<img src="../../docs/public/mediabunny-logo.svg" width="180" height="180">
|
||||
</div>
|
||||
|
||||
Browsers have no support for Apple ProRes in their WebCodecs implementations. This extension package provides a decoder for use with [Mediabunny](https://github.com/Vanilagy/mediabunny), allowing you to decode ProRes directly in the browser at unprecedented speed. It is implemented using Mediabunny's [custom coder API](https://mediabunny.dev/guide/supported-formats-and-codecs#custom-coders) and uses [TurboRes](https://github.com/Vanilagy/turbores), an extremely fast WASM-based ProRes decoder, under the hood.
|
||||
|
||||
> This package, like the rest of Mediabunny, is enabled by its [sponsors](https://mediabunny.dev/#sponsors) and their donations. If you've derived value from this package, please consider [leaving a donation](https://github.com/sponsors/Vanilagy)! 💘
|
||||
|
||||
## Installation
|
||||
|
||||
This library peer-depends on Mediabunny. Install both using npm:
|
||||
```bash
|
||||
npm install mediabunny @mediabunny/prores
|
||||
```
|
||||
|
||||
Alternatively, directly include them using a script tag:
|
||||
```html
|
||||
<script src="mediabunny.js"></script>
|
||||
<script src="mediabunny-prores.js"></script>
|
||||
```
|
||||
|
||||
This will expose the global objects `Mediabunny` and `MediabunnyProres`. Use `mediabunny-prores.d.ts` to provide types for these globals. You can download the built distribution files from the [releases page](https://github.com/Vanilagy/mediabunny/releases).
|
||||
|
||||
## Setup
|
||||
|
||||
`@mediabunny/prores` can make use of shared-memory multithreading to achieve maximum performance. To enable this in browsers, your website must be cross-origin isolated by setting the following response headers:
|
||||
```
|
||||
Cross-Origin-Opener-Policy: same-origin
|
||||
Cross-Origin-Embedder-Policy: require-corp
|
||||
```
|
||||
|
||||
Alternatively, you can use:
|
||||
```
|
||||
Cross-Origin-Opener-Policy: same-origin
|
||||
Cross-Origin-Embedder-Policy: credentialless
|
||||
```
|
||||
which is generally more permissive but is not supported in Safari (of course).
|
||||
|
||||
---
|
||||
|
||||
If you cannot enable cross-origin isolation, `@mediabunny/prores` will fall back to a slower multithreading algorithm.
|
||||
|
||||
## Usage
|
||||
|
||||
```ts
|
||||
import { registerProresDecoder } from '@mediabunny/prores';
|
||||
|
||||
registerProresDecoder();
|
||||
```
|
||||
That's it - Mediabunny now uses the registered ProRes decoder automatically.
|
||||
|
||||
For all the ways of using Mediabunny, refer to its [guide](https://mediabunny.dev/guide/introduction).
|
||||
|
||||
## Building and development
|
||||
|
||||
The complete JavaScript package can be built alongside the rest of Mediabunny by running `npm run build` in Mediabunny's root.
|
||||
@@ -0,0 +1,37 @@
|
||||
{
|
||||
"$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json",
|
||||
"mainEntryPointFilePath": "dist/modules/src/index.d.ts",
|
||||
"bundledPackages": [],
|
||||
"compiler": {},
|
||||
"apiReport": {
|
||||
"enabled": false
|
||||
},
|
||||
"docModel": {
|
||||
"enabled": false
|
||||
},
|
||||
"dtsRollup": {
|
||||
"enabled": true,
|
||||
"untrimmedFilePath": "dist/mediabunny-prores.d.ts"
|
||||
},
|
||||
"tsdocMetadata": {
|
||||
"enabled": false
|
||||
},
|
||||
"messages": {
|
||||
"compilerMessageReporting": {
|
||||
"default": {
|
||||
"logLevel": "warning"
|
||||
}
|
||||
},
|
||||
"extractorMessageReporting": {
|
||||
"default": {
|
||||
"logLevel": "warning"
|
||||
}
|
||||
},
|
||||
"tsdocMessageReporting": {
|
||||
"default": {
|
||||
"logLevel": "warning"
|
||||
}
|
||||
}
|
||||
},
|
||||
"newlineKind": "lf"
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
{
|
||||
"name": "@mediabunny/prores",
|
||||
"author": "Vanilagy",
|
||||
"version": "1.50.1",
|
||||
"description": "Apple ProRes decoder extension for Mediabunny, based on TurboRes.",
|
||||
"main": "./dist/bundles/mediabunny-prores.mjs",
|
||||
"module": "./dist/bundles/mediabunny-prores.mjs",
|
||||
"types": "./dist/modules/src/index.d.ts",
|
||||
"exports": {
|
||||
"types": "./dist/modules/src/index.d.ts",
|
||||
"import": "./dist/bundles/mediabunny-prores.mjs",
|
||||
"require": "./dist/bundles/mediabunny-prores.mjs"
|
||||
},
|
||||
"files": [
|
||||
"README.md",
|
||||
"package.json",
|
||||
"LICENSE",
|
||||
"dist",
|
||||
"src"
|
||||
],
|
||||
"sideEffects": false,
|
||||
"license": "MPL-2.0",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git+https://github.com/Vanilagy/mediabunny.git",
|
||||
"directory": "packages/prores"
|
||||
},
|
||||
"bugs": {
|
||||
"url": "https://github.com/Vanilagy/mediabunny/issues"
|
||||
},
|
||||
"homepage": "https://mediabunny.dev/guide/extensions/prores",
|
||||
"funding": {
|
||||
"type": "individual",
|
||||
"url": "https://github.com/sponsors/Vanilagy"
|
||||
},
|
||||
"dependencies": {
|
||||
"turbores": "^1.1.2"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"mediabunny": "^1.49.0"
|
||||
},
|
||||
"keywords": [
|
||||
"prores",
|
||||
"apple-prores",
|
||||
"decoding",
|
||||
"codec",
|
||||
"mediabunny",
|
||||
"turbores",
|
||||
"video",
|
||||
"browser",
|
||||
"wasm",
|
||||
"polyfill"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,256 @@
|
||||
/*!
|
||||
* Copyright (c) 2026-present, Vanilagy and contributors
|
||||
*
|
||||
* This Source Code Form is subject to the terms of the Mozilla Public
|
||||
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
* file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
*/
|
||||
|
||||
import { CustomVideoDecoder, EncodedPacket, Logging, registerDecoder, VideoCodec, VideoSample } from 'mediabunny';
|
||||
import { Decoder, Frame, PixelFormat, PIXEL_FORMATS, FilledFrame } from 'turbores';
|
||||
import {
|
||||
assert,
|
||||
isWebKit,
|
||||
} from '../../../src/misc';
|
||||
import { type ProresFourCc } from '../../../src/codec';
|
||||
|
||||
const PRORES_LOADED_SYMBOL = Symbol.for('@mediabunny/prores loaded');
|
||||
if ((globalThis as Record<symbol, unknown>)[PRORES_LOADED_SYMBOL]) {
|
||||
Logging._error(
|
||||
'[WARNING]\n@mediabunny/prores was loaded twice.'
|
||||
+ ' This will likely cause the decoder not to work correctly.'
|
||||
+ ' Check if multiple dependencies are importing different versions of @mediabunny/prores,'
|
||||
+ ' or if something is being bundled incorrectly.',
|
||||
);
|
||||
}
|
||||
(globalThis as Record<symbol, unknown>)[PRORES_LOADED_SYMBOL] = true;
|
||||
|
||||
class ProresDecoder extends CustomVideoDecoder {
|
||||
private decoder: Decoder | null = null;
|
||||
private framePool: Frame[] = [];
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
static override supports(codec: VideoCodec, config: VideoDecoderConfig): boolean {
|
||||
return codec === 'prores';
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
static _supportedVideoFrameFormats: PixelFormat[] | null = null;
|
||||
|
||||
/** @internal */
|
||||
static _determineSupportedVideoFrameFormats() {
|
||||
const result: PixelFormat[] = [];
|
||||
const data = new Uint8Array(32);
|
||||
|
||||
for (const format of PIXEL_FORMATS) {
|
||||
try {
|
||||
const frame = new VideoFrame(data, {
|
||||
format: format as VideoPixelFormat,
|
||||
codedWidth: 2,
|
||||
codedHeight: 2,
|
||||
timestamp: 0,
|
||||
duration: 0,
|
||||
});
|
||||
frame.close();
|
||||
|
||||
result.push(format);
|
||||
} catch {
|
||||
// Format is not supported
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
async init() {
|
||||
if (typeof VideoFrame !== 'undefined') {
|
||||
// Not all VideoFrame implementations support all pixel formats, therefore let's determine the supported set
|
||||
ProresDecoder._supportedVideoFrameFormats ??= ProresDecoder._determineSupportedVideoFrameFormats();
|
||||
}
|
||||
|
||||
const decoder = await Decoder.create({
|
||||
proresFourCc: this.config.codec as ProresFourCc,
|
||||
useSharedMemory: Decoder.canUseSharedMemory(),
|
||||
allowedOutputFormats: ProresDecoder._supportedVideoFrameFormats ?? undefined,
|
||||
});
|
||||
if (decoder instanceof Error) {
|
||||
throw decoder;
|
||||
}
|
||||
|
||||
this.decoder = decoder;
|
||||
}
|
||||
|
||||
async decode(packet: EncodedPacket) {
|
||||
assert(this.decoder);
|
||||
|
||||
if (this.decoder.useSharedMemory) {
|
||||
await this.runDecode(packet);
|
||||
} else {
|
||||
while (this.decoder.decodeQueueSize >= this.decoder.concurrency) {
|
||||
await this.decoder.dequeued;
|
||||
}
|
||||
|
||||
void this.runDecode(packet)
|
||||
.catch(error => this.onError(error));
|
||||
}
|
||||
}
|
||||
|
||||
private async runDecode(packet: EncodedPacket) {
|
||||
assert(this.decoder);
|
||||
|
||||
let frame: Frame;
|
||||
if (this.framePool.length > 0) {
|
||||
frame = this.framePool.shift()!;
|
||||
} else {
|
||||
frame = new Frame();
|
||||
}
|
||||
|
||||
const result = await this.decoder.decode(packet.data, frame);
|
||||
this.framePool.push(frame);
|
||||
|
||||
if (result instanceof Error) {
|
||||
throw result;
|
||||
}
|
||||
|
||||
if (result.visibleHeight < result.codedHeight && isWebKit()) {
|
||||
// WebKit has (had) a bug with displaying height-trimmed YUV frames, so we must compact the frame data a
|
||||
// little https://bugs.webkit.org/show_bug.cgi?id=317524
|
||||
this.trimCodedHeightToVisibleHeight(result);
|
||||
}
|
||||
|
||||
const colorSpaceInit = {
|
||||
primaries: result.colorPrimariesString as VideoColorPrimaries | undefined,
|
||||
matrix: result.colorMatrixString as VideoMatrixCoefficients | undefined,
|
||||
transfer: result.colorTransferString as VideoTransferCharacteristics | undefined,
|
||||
fullRange: result.colorRangeFull,
|
||||
};
|
||||
|
||||
let displayWidth = result.visibleWidth;
|
||||
let displayHeight = result.visibleHeight;
|
||||
|
||||
if (result.pixelAspectRatio.num > result.pixelAspectRatio.den) {
|
||||
displayWidth = Math.round(
|
||||
result.visibleWidth * result.pixelAspectRatio.num / result.pixelAspectRatio.den,
|
||||
);
|
||||
} else {
|
||||
displayHeight = Math.round(
|
||||
result.visibleHeight * result.pixelAspectRatio.den / result.pixelAspectRatio.num,
|
||||
);
|
||||
}
|
||||
|
||||
let sample: VideoSample;
|
||||
if (typeof VideoFrame !== 'undefined') {
|
||||
// Create a VideoFrame directly; this avoids the frame data being copied twice
|
||||
const frame = new VideoFrame(result.frameData, {
|
||||
format: result.pixelFormat as VideoPixelFormat,
|
||||
codedWidth: result.codedWidth,
|
||||
codedHeight: result.codedHeight,
|
||||
displayWidth,
|
||||
displayHeight,
|
||||
visibleRect: {
|
||||
x: 0,
|
||||
y: 0,
|
||||
width: result.visibleWidth,
|
||||
height: result.visibleHeight,
|
||||
},
|
||||
timestamp: packet.microsecondTimestamp,
|
||||
duration: packet.microsecondDuration,
|
||||
colorSpace: colorSpaceInit,
|
||||
});
|
||||
|
||||
sample = new VideoSample(frame, {
|
||||
timestamp: packet.timestamp,
|
||||
duration: packet.duration,
|
||||
});
|
||||
} else {
|
||||
sample = new VideoSample(result.frameData, {
|
||||
format: result.pixelFormat,
|
||||
codedWidth: result.codedWidth,
|
||||
codedHeight: result.codedHeight,
|
||||
displayWidth,
|
||||
displayHeight,
|
||||
visibleRect: {
|
||||
left: 0,
|
||||
top: 0,
|
||||
width: result.visibleWidth,
|
||||
height: result.visibleHeight,
|
||||
},
|
||||
timestamp: packet.timestamp,
|
||||
duration: packet.duration,
|
||||
colorSpace: colorSpaceInit,
|
||||
});
|
||||
}
|
||||
|
||||
this.onSample(sample);
|
||||
}
|
||||
|
||||
private trimCodedHeightToVisibleHeight(result: FilledFrame) {
|
||||
const bytesPerSample = result.pixelFormat.includes('P') ? 2 : 1;
|
||||
const subWidth = result.pixelFormat.includes('444') ? 1 : 2;
|
||||
const subHeight = result.pixelFormat.includes('420') ? 2 : 1;
|
||||
|
||||
const chromaCodedWidth = result.codedWidth / subWidth;
|
||||
const chromaCodedHeight = result.codedHeight / subHeight;
|
||||
const chromaVisibleHeight = Math.ceil(result.visibleHeight / subHeight);
|
||||
|
||||
const lumaCodedPixels = result.codedWidth * result.codedHeight;
|
||||
const lumaVisiblePixels = result.codedWidth * result.visibleHeight;
|
||||
const chromaCodedPixels = chromaCodedWidth * chromaCodedHeight;
|
||||
const chromaVisiblePixels = chromaCodedWidth * chromaVisibleHeight;
|
||||
|
||||
// U
|
||||
result.frameData.set(
|
||||
result.frameData.subarray(
|
||||
bytesPerSample * lumaCodedPixels,
|
||||
bytesPerSample * (lumaCodedPixels + chromaCodedPixels),
|
||||
),
|
||||
bytesPerSample * lumaVisiblePixels,
|
||||
);
|
||||
// V
|
||||
result.frameData.set(
|
||||
result.frameData.subarray(
|
||||
bytesPerSample * (lumaCodedPixels + chromaCodedPixels),
|
||||
bytesPerSample * (lumaCodedPixels + 2 * chromaCodedPixels),
|
||||
),
|
||||
bytesPerSample * (lumaVisiblePixels + chromaVisiblePixels),
|
||||
);
|
||||
|
||||
result.codedHeight = result.visibleHeight;
|
||||
}
|
||||
|
||||
async flush() {
|
||||
assert(this.decoder);
|
||||
|
||||
while (this.decoder.decodeQueueSize > 0) {
|
||||
await this.decoder.dequeued;
|
||||
}
|
||||
}
|
||||
|
||||
async close() {
|
||||
assert(this.decoder);
|
||||
|
||||
await this.decoder.close();
|
||||
|
||||
for (const frame of this.framePool) {
|
||||
frame.clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let registered = false;
|
||||
|
||||
/**
|
||||
* Registers an Apple ProRes decoder which Mediabunny will then use automatically when applicable. Make sure to call
|
||||
* this function before starting any decoding task.
|
||||
*
|
||||
* @group \@mediabunny/prores
|
||||
* @public
|
||||
*/
|
||||
export const registerProresDecoder = () => {
|
||||
if (registered) {
|
||||
return;
|
||||
}
|
||||
registered = true;
|
||||
|
||||
registerDecoder(ProresDecoder);
|
||||
};
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"extends": "../../tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"outDir": "./dist/modules",
|
||||
"declaration": true,
|
||||
"declarationMap": true,
|
||||
"stripInternal": true,
|
||||
"composite": true,
|
||||
"noEmit": false,
|
||||
"moduleResolution": "nodenext",
|
||||
"module": "nodenext",
|
||||
"allowJs": true,
|
||||
"paths": {
|
||||
"mediabunny": ["../../src/index.ts"],
|
||||
},
|
||||
},
|
||||
"include": [
|
||||
"./src/**/*",
|
||||
],
|
||||
"references": [
|
||||
{ "path": "../../src" }
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"$schema": "https://developer.microsoft.com/json-schemas/tsdoc/v0/tsdoc.schema.json",
|
||||
"extends": ["../../tsdoc.json"]
|
||||
}
|
||||
@@ -12,7 +12,7 @@
|
||||
By default, Mediabunny requires a browser environment for full access to decoders, encoders, and video processing features. `@mediabunny/server` uses [NodeAV](https://github.com/seydx/node-av) to polyfill this functionality for server-side environments such as Node, Bun, or Deno, enabling the usage of all Mediabunny features on the server. The result is a server-side media processing API that integrates naturally with TypeScript as opposed to the awkwardness and inefficiencies of calling out to the FFmpeg CLI.
|
||||
|
||||
Features added by this package include:
|
||||
- Video decoders and encoders for AVC (H.264), HEVC (H.265), VP8, VP9, and AV1. Supports both length-prefixed and Annex B AVC/HEVC as well as transparent video via VP9.
|
||||
- Video decoders and encoders for AVC (H.264), HEVC (H.265), VP8, VP9, AV1, and ProRes. Supports both length-prefixed and Annex B AVC/HEVC as well as transparent video via VP9 and ProRes.
|
||||
- Audio decoders and encoders for AAC, MP3, Vorbis, Opus, FLAC, AC-3 and E-AC-3. Supports AAC in both AAC and ADTS formats.
|
||||
- Video frame transformation support (resize, rotate, crop)
|
||||
- Automatic hardware acceleration on all platforms (macOS, Linux, Windows)
|
||||
@@ -321,6 +321,8 @@ For encoding, video frames and audio samples are transferred to FFmpeg by conver
|
||||
|
||||
Whenever possible, `AVFrame`s are never copied over to JavaScript unless explicitly needed. This enables zero-copy decode -> transformation -> encode paths.
|
||||
|
||||
ProRes decoding support is provided by [TurboRes](https://github.com/Vanilagy/turbores), as it is often faster than FFmpeg.
|
||||
|
||||
## License
|
||||
|
||||
`@mediabunny/server` uses the same MPL-2.0 license as Mediabunny.
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "@mediabunny/server",
|
||||
"author": "Vanilagy",
|
||||
"version": "1.49.0",
|
||||
"version": "1.50.1",
|
||||
"description": "Adds full video and audio decoder and encoder support to Mediabunny for use in server-side environments (Node, Bun, Deno). Based on NodeAV.",
|
||||
"main": "./dist/bundles/mediabunny-server.cjs",
|
||||
"module": "./dist/bundles/mediabunny-server.mjs",
|
||||
@@ -34,7 +34,8 @@
|
||||
"url": "https://github.com/sponsors/Vanilagy"
|
||||
},
|
||||
"dependencies": {
|
||||
"node-av": "^6.0.0"
|
||||
"node-av": "^6.0.0",
|
||||
"@mediabunny/prores": "^1.50.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"mediabunny": "^1.45.0"
|
||||
|
||||
@@ -22,6 +22,7 @@ import { NodeAvAudioDecoder } from './audio-decoder';
|
||||
import { NodeAvAudioEncoder } from './audio-encoder';
|
||||
import { copyVideoSampleToAvFrame, AvFrameVideoSampleResource, transformVideoSample } from './video-sample';
|
||||
import { copyAudioSampleToAvFrame, AvFrameAudioSampleResource } from './audio-sample';
|
||||
import { registerProresDecoder } from '@mediabunny/prores';
|
||||
|
||||
const SERVER_LOADED_SYMBOL = Symbol.for('@mediabunny/server loaded');
|
||||
if ((globalThis as Record<symbol, unknown>)[SERVER_LOADED_SYMBOL]) {
|
||||
@@ -100,6 +101,7 @@ export const registerMediabunnyServer = (options: MediabunnyServerOptions = {})
|
||||
|
||||
// Video
|
||||
registerDecoder(NodeAvVideoDecoder);
|
||||
registerProresDecoder(); // Faster than FFmpeg, so we use it
|
||||
registerEncoder(NodeAvVideoEncoder);
|
||||
|
||||
// Audio
|
||||
|
||||
@@ -16,6 +16,7 @@ export const CODEC_TO_CODEC_ID: Partial<Record<MediaCodec, NodeAv.AVCodecID>> =
|
||||
vp8: NodeAv.AV_CODEC_ID_VP8,
|
||||
vp9: NodeAv.AV_CODEC_ID_VP9,
|
||||
av1: NodeAv.AV_CODEC_ID_AV1,
|
||||
prores: NodeAv.AV_CODEC_ID_PRORES,
|
||||
|
||||
aac: NodeAv.AV_CODEC_ID_AAC,
|
||||
opus: NodeAv.AV_CODEC_ID_OPUS,
|
||||
|
||||
@@ -38,9 +38,18 @@ import {
|
||||
serializeAvcDecoderConfigurationRecord,
|
||||
serializeHevcDecoderConfigurationRecord,
|
||||
} from '../../../src/codec-data';
|
||||
import { extractVideoCodecString } from '../../../src/codec';
|
||||
import { extractVideoCodecString, ProresFourCc } from '../../../src/codec';
|
||||
import { assert, binarySearchLessOrEqual, simplifyRational, toUint8Array } from '../../../src/misc';
|
||||
|
||||
const PRORES_FOURCC_TO_PROFILE: Record<ProresFourCc, NodeAv.AVProfile> = {
|
||||
apco: NodeAv.AV_PROFILE_PRORES_PROXY,
|
||||
apcs: NodeAv.AV_PROFILE_PRORES_LT,
|
||||
apcn: NodeAv.AV_PROFILE_PRORES_STANDARD,
|
||||
apch: NodeAv.AV_PROFILE_PRORES_HQ,
|
||||
ap4h: NodeAv.AV_PROFILE_PRORES_4444,
|
||||
ap4x: NodeAv.AV_PROFILE_PRORES_XQ,
|
||||
};
|
||||
|
||||
export class NodeAvVideoEncoder extends CustomVideoEncoder {
|
||||
frame!: NodeAv.Frame;
|
||||
packet!: NodeAv.Packet;
|
||||
@@ -62,8 +71,10 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
|
||||
}[] = [];
|
||||
|
||||
static override supports(codec: VideoCodec, config: VideoEncoderConfig): boolean {
|
||||
return (codec === 'avc' || codec === 'hevc' || codec === 'vp8' || codec === 'vp9' || codec === 'av1')
|
||||
&& config.bitrateMode !== 'quantizer';
|
||||
return (
|
||||
codec === 'avc' || codec === 'hevc' || codec === 'vp8' || codec === 'vp9' || codec === 'av1'
|
||||
|| codec === 'prores'
|
||||
) && config.bitrateMode !== 'quantizer';
|
||||
}
|
||||
|
||||
async init(): Promise<void> {
|
||||
@@ -77,13 +88,25 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
|
||||
const codecId = CODEC_TO_CODEC_ID[this.codec];
|
||||
assert(codecId !== undefined);
|
||||
|
||||
const getSoftwareCodec = () => {
|
||||
if (this.codec === 'prores') {
|
||||
// Prefer prores_ks for ProRes
|
||||
const proresKs = NodeAv.Codec.findEncoderByName(NodeAv.FF_ENCODER_PRORES_KS);
|
||||
if (proresKs) {
|
||||
return proresKs;
|
||||
}
|
||||
}
|
||||
|
||||
return NodeAv.Codec.findEncoder(codecId);
|
||||
};
|
||||
|
||||
let codec: NodeAv.Codec | null = null;
|
||||
if (this.codec === 'vp9' && this.config.alpha === 'keep') {
|
||||
codec = NodeAv.Codec.findEncoderByName(NodeAv.FF_ENCODER_LIBVPX_VP9) ?? NodeAv.Codec.findEncoder(codecId);
|
||||
} else if (this.config.hardwareAcceleration === 'prefer-software') {
|
||||
codec = NodeAv.Codec.findEncoder(codecId);
|
||||
codec = getSoftwareCodec();
|
||||
} else {
|
||||
codec = (await getHardwareEncoderCodec(codecId)) ?? NodeAv.Codec.findEncoder(codecId);
|
||||
codec = (await getHardwareEncoderCodec(codecId)) ?? getSoftwareCodec();
|
||||
}
|
||||
|
||||
if (!codec) {
|
||||
@@ -108,8 +131,17 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
|
||||
pixelFormat = this.avCodec.pixelFormats[0]!;
|
||||
}
|
||||
|
||||
if (this.config.alpha === 'keep' && this.avCodec.pixelFormats.includes(NodeAv.AV_PIX_FMT_YUVA420P)) {
|
||||
pixelFormat = NodeAv.AV_PIX_FMT_YUVA420P;
|
||||
if (this.config.alpha === 'keep') {
|
||||
if (this.avCodec.pixelFormats.includes(NodeAv.AV_PIX_FMT_YUVA420P)) {
|
||||
pixelFormat = NodeAv.AV_PIX_FMT_YUVA420P;
|
||||
} else {
|
||||
// Let FFmpeg pick the best alpha-capable format it supports, minimizing data loss versus a
|
||||
// high-quality YUVA source
|
||||
pixelFormat = NodeAv.avcodecFindBestPixFmtOfList(
|
||||
this.avCodec.pixelFormats,
|
||||
NodeAv.AV_PIX_FMT_YUVA444P12LE,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -173,6 +205,14 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
|
||||
}
|
||||
}
|
||||
|
||||
if (this.codec === 'prores') {
|
||||
// Pick the encoder profile from the requested ProRes four-character code
|
||||
const profile = PRORES_FOURCC_TO_PROFILE[this.config.codec as ProresFourCc];
|
||||
assert(profile !== undefined);
|
||||
|
||||
codecContext.setOption('profile', String(profile));
|
||||
}
|
||||
|
||||
const ret = await codecContext.open2();
|
||||
NodeAv.FFmpegError.throwIfError(ret, 'Open codec context');
|
||||
|
||||
@@ -375,6 +415,7 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
|
||||
hevcCodecInfo: null,
|
||||
vp9CodecInfo: null,
|
||||
av1CodecInfo: null,
|
||||
proresFormat: null,
|
||||
});
|
||||
|
||||
if (!expectsAnnexB) {
|
||||
@@ -450,6 +491,7 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
|
||||
hevcCodecInfo: null,
|
||||
vp9CodecInfo: null,
|
||||
av1CodecInfo: null,
|
||||
proresFormat: null,
|
||||
});
|
||||
}
|
||||
} else if (this.codec === 'vp9') {
|
||||
@@ -467,6 +509,7 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
|
||||
hevcCodecInfo: null,
|
||||
vp9CodecInfo,
|
||||
av1CodecInfo: null,
|
||||
proresFormat: null,
|
||||
});
|
||||
}
|
||||
} else if (this.codec === 'av1') {
|
||||
@@ -484,6 +527,23 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
|
||||
hevcCodecInfo: null,
|
||||
vp9CodecInfo: null,
|
||||
av1CodecInfo,
|
||||
proresFormat: null,
|
||||
});
|
||||
}
|
||||
} else if (this.codec === 'prores') {
|
||||
if (!this.packetEmitted) {
|
||||
decoderConfigCodecString = extractVideoCodecString({
|
||||
width: this.config.width,
|
||||
height: this.config.height,
|
||||
codec: 'prores',
|
||||
codecDescription: null,
|
||||
colorSpace: null,
|
||||
avcType: null,
|
||||
avcCodecInfo: null,
|
||||
hevcCodecInfo: null,
|
||||
vp9CodecInfo: null,
|
||||
av1CodecInfo: null,
|
||||
proresFormat: this.config.codec as ProresFourCc,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
|
||||
@@ -9,6 +9,7 @@ rm -rf packages/mp3-encoder/dist
|
||||
rm -rf packages/ac3/dist
|
||||
rm -rf packages/aac-encoder/dist
|
||||
rm -rf packages/flac-encoder/dist
|
||||
rm -rf packages/prores/dist
|
||||
rm -rf packages/server/dist
|
||||
|
||||
# Ensure license headers on all source files
|
||||
@@ -20,6 +21,7 @@ tsc -p packages/mp3-encoder
|
||||
tsc -p packages/ac3
|
||||
tsc -p packages/aac-encoder
|
||||
tsc -p packages/flac-encoder
|
||||
tsc -p packages/prores
|
||||
tsc -p packages/server
|
||||
|
||||
# Generate the root again, now with internals properly stripped
|
||||
@@ -39,6 +41,7 @@ api-extractor run -c packages/mp3-encoder/api-extractor.json
|
||||
api-extractor run -c packages/ac3/api-extractor.json
|
||||
api-extractor run -c packages/aac-encoder/api-extractor.json
|
||||
api-extractor run -c packages/flac-encoder/api-extractor.json
|
||||
api-extractor run -c packages/prores/api-extractor.json
|
||||
api-extractor run -c packages/server/api-extractor.json
|
||||
|
||||
# Checks that all symbols are documented
|
||||
@@ -47,6 +50,7 @@ tsx scripts/check-docblocks.ts packages/mp3-encoder/dist/mediabunny-mp3-encoder.
|
||||
tsx scripts/check-docblocks.ts packages/ac3/dist/mediabunny-ac3.d.ts
|
||||
tsx scripts/check-docblocks.ts packages/aac-encoder/dist/mediabunny-aac-encoder.d.ts
|
||||
tsx scripts/check-docblocks.ts packages/flac-encoder/dist/mediabunny-flac-encoder.d.ts
|
||||
tsx scripts/check-docblocks.ts packages/prores/dist/mediabunny-prores.d.ts
|
||||
tsx scripts/check-docblocks.ts packages/server/dist/mediabunny-server.d.ts
|
||||
|
||||
# Checks that API docs are generatable
|
||||
@@ -58,4 +62,5 @@ echo 'export as namespace MediabunnyMp3Encoder;' >> packages/mp3-encoder/dist/me
|
||||
echo 'export as namespace MediabunnyAc3;' >> packages/ac3/dist/mediabunny-ac3.d.ts
|
||||
echo 'export as namespace MediabunnyAacEncoder;' >> packages/aac-encoder/dist/mediabunny-aac-encoder.d.ts
|
||||
echo 'export as namespace MediabunnyFlacEncoder;' >> packages/flac-encoder/dist/mediabunny-flac-encoder.d.ts
|
||||
echo 'export as namespace MediabunnyProres;' >> packages/prores/dist/mediabunny-prores.d.ts
|
||||
echo 'export as namespace MediabunnyServer;' >> packages/server/dist/mediabunny-server.d.ts
|
||||
|
||||
@@ -223,6 +223,24 @@ const flacEncoderVariants = await createVariants(
|
||||
},
|
||||
);
|
||||
|
||||
const proresVariants = await createVariants(
|
||||
'packages/prores/src/index.ts',
|
||||
'MediabunnyProres',
|
||||
'packages/prores/dist/bundles/mediabunny-prores',
|
||||
'js', // The bundles are purely for the browser, not for Node (due to the peer dependecy)
|
||||
{
|
||||
plugins: [
|
||||
PluginExternalGlobal.externalGlobalPlugin({
|
||||
mediabunny: 'Mediabunny',
|
||||
}),
|
||||
],
|
||||
},
|
||||
{
|
||||
external: ['mediabunny'],
|
||||
platform: 'node', // To retain the Node imports
|
||||
},
|
||||
);
|
||||
|
||||
const serverVariants = await createVariants(
|
||||
'packages/server/src/index.ts',
|
||||
'MediabunnyServer',
|
||||
@@ -246,6 +264,7 @@ const contexts = [
|
||||
...ac3Variants,
|
||||
...aacEncoderVariants,
|
||||
...flacEncoderVariants,
|
||||
...proresVariants,
|
||||
...serverVariants,
|
||||
];
|
||||
|
||||
|
||||
@@ -16,6 +16,9 @@ tsc -p packages/aac-encoder
|
||||
rm -rf packages/flac-encoder/dist/modules
|
||||
tsc -p packages/flac-encoder
|
||||
|
||||
rm -rf packages/prores/dist/modules
|
||||
tsc -p packages/prores
|
||||
|
||||
rm -rf packages/server/dist/modules
|
||||
tsc -p packages/server
|
||||
|
||||
|
||||
@@ -40,6 +40,7 @@ checkDirectory(path.join(__dirname, '..', 'packages', 'mp3-encoder', 'src'));
|
||||
checkDirectory(path.join(__dirname, '..', 'packages', 'ac3', 'src'));
|
||||
checkDirectory(path.join(__dirname, '..', 'packages', 'flac-encoder', 'src'));
|
||||
checkDirectory(path.join(__dirname, '..', 'packages', 'aac-encoder', 'src'));
|
||||
checkDirectory(path.join(__dirname, '..', 'packages', 'prores', 'src'));
|
||||
checkDirectory(path.join(__dirname, '..', 'packages', 'server', 'src'));
|
||||
|
||||
if (missingFiles.length > 0) {
|
||||
|
||||
@@ -134,8 +134,18 @@ const generateDocs = (entryFiles: string[], apiConfigFile: string, dry = false)
|
||||
if (aliasedDeclaration) {
|
||||
const sourceFile = aliasedDeclaration.getSourceFile();
|
||||
const moduleSymbol = typeChecker.getSymbolAtLocation(sourceFile);
|
||||
if (moduleSymbol) {
|
||||
// If the aliased declaration lives in a module we've already visited (e.g. a
|
||||
// same-file `export type { Foo }` re-export of a local declaration), recursing
|
||||
// won't reach it, so add the alias symbol directly. Otherwise follow the reexport.
|
||||
if (moduleSymbol && !visited.has(moduleSymbol)) {
|
||||
symbols.push(...getAllExportedSymbols(moduleSymbol, visited));
|
||||
} else {
|
||||
// Push the aliased symbol (not the alias) so downstream sees the real
|
||||
// declaration and its JSDoc rather than the empty ExportSpecifier.
|
||||
const hasPublicTag = ts.getJSDocTags(aliasedDeclaration).some(tag => tag.tagName.text === 'public');
|
||||
if (hasPublicTag) {
|
||||
symbols.push(aliasedSymbol);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2456,6 +2456,10 @@ export const determineVideoPacketType = (
|
||||
return null;
|
||||
};
|
||||
|
||||
case 'prores': {
|
||||
return 'key';
|
||||
};
|
||||
|
||||
default: {
|
||||
assertNever(codec);
|
||||
assert(false);
|
||||
|
||||
+76
-5
@@ -18,6 +18,7 @@ import {
|
||||
MATRIX_COEFFICIENTS_MAP,
|
||||
TRANSFER_CHARACTERISTICS_MAP,
|
||||
assert,
|
||||
assertNever,
|
||||
base64ToBytes,
|
||||
bytesToHexString,
|
||||
isAllowSharedBufferSource,
|
||||
@@ -38,6 +39,7 @@ export const VIDEO_CODECS = [
|
||||
'vp9',
|
||||
'av1',
|
||||
'vp8',
|
||||
'prores',
|
||||
] as const;
|
||||
/**
|
||||
* List of known PCM (uncompressed) audio codecs, ordered by encoding preference.
|
||||
@@ -215,7 +217,33 @@ const AV1_LEVEL_TABLE = [
|
||||
const VP9_DEFAULT_SUFFIX = '.01.01.01.01.00';
|
||||
const AV1_DEFAULT_SUFFIX = '.0.110.01.01.01.0';
|
||||
|
||||
export const buildVideoCodecString = (codec: VideoCodec, width: number, height: number, bitrate: number) => {
|
||||
export const PRORES_FOURCCS = [
|
||||
'ap4x', // ProRes 4444 XQ
|
||||
'ap4h', // ProRes 4444
|
||||
'apch', // ProRes 422 High Quality
|
||||
'apcn', // ProRes 422 Standard Definition
|
||||
'apcs', // ProRes 422 LT
|
||||
'apco', // ProRes 422 Proxy
|
||||
] as const;
|
||||
export type ProresFourCc = typeof PRORES_FOURCCS[number];
|
||||
|
||||
// Target data rates of the ProRes profiles at 1920x1080 ~30fps, as published by Apple
|
||||
const PRORES_PROFILE_TARGET_BITRATES: { fourCc: ProresFourCc; bitrate: number; alpha: boolean }[] = [
|
||||
{ fourCc: 'apco', bitrate: 45_000_000, alpha: false }, // 422 Proxy
|
||||
{ fourCc: 'apcs', bitrate: 102_000_000, alpha: false }, // 422 LT
|
||||
{ fourCc: 'apcn', bitrate: 147_000_000, alpha: false }, // 422 Standard
|
||||
{ fourCc: 'apch', bitrate: 220_000_000, alpha: false }, // 422 HQ
|
||||
{ fourCc: 'ap4h', bitrate: 330_000_000, alpha: true }, // 4444
|
||||
{ fourCc: 'ap4x', bitrate: 500_000_000, alpha: true }, // 4444 XQ
|
||||
];
|
||||
|
||||
export const buildVideoCodecString = (
|
||||
codec: VideoCodec,
|
||||
width: number,
|
||||
height: number,
|
||||
bitrate: number,
|
||||
alpha: boolean,
|
||||
) => {
|
||||
if (codec === 'avc') {
|
||||
const profileIndication = 0x64; // High Profile
|
||||
const totalMacroblocks = Math.ceil(width / 16) * Math.ceil(height / 16);
|
||||
@@ -274,10 +302,28 @@ export const buildVideoCodecString = (codec: VideoCodec, width: number, height:
|
||||
const bitDepth = '08'; // 8-bit
|
||||
|
||||
return `av01.${profile}.${level}${levelInfo.tier}.${bitDepth}`;
|
||||
} else if (codec === 'prores') {
|
||||
const referencePixels = 1920 * 1080;
|
||||
const scaleFactor = Math.pow((width * height) / referencePixels, 0.95);
|
||||
|
||||
const candidates = PRORES_PROFILE_TARGET_BITRATES.filter(x => x.alpha === alpha);
|
||||
|
||||
let bestFourCc = candidates[0]!.fourCc;
|
||||
let smallestDifference = Infinity;
|
||||
for (const { fourCc, bitrate: targetBitrate } of candidates) {
|
||||
const difference = Math.abs(targetBitrate * scaleFactor - bitrate);
|
||||
if (difference < smallestDifference) {
|
||||
smallestDifference = difference;
|
||||
bestFourCc = fourCc;
|
||||
}
|
||||
}
|
||||
|
||||
return bestFourCc;
|
||||
} else {
|
||||
assertNever(codec);
|
||||
}
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/restrict-template-expressions
|
||||
throw new TypeError(`Unhandled codec '${codec}'.`);
|
||||
throw new TypeError(`Unhandled codec '${String(codec)}'.`);
|
||||
};
|
||||
|
||||
export const generateVp9CodecConfigurationFromCodecString = (codecString: string) => {
|
||||
@@ -345,8 +391,18 @@ export const extractVideoCodecString = (trackInfo: {
|
||||
hevcCodecInfo: HevcDecoderConfigurationRecord | null;
|
||||
vp9CodecInfo: Vp9CodecInfo | null;
|
||||
av1CodecInfo: Av1CodecInfo | null;
|
||||
proresFormat: ProresFourCc | null;
|
||||
}) => {
|
||||
const { codec, codecDescription, colorSpace, avcCodecInfo, hevcCodecInfo, vp9CodecInfo, av1CodecInfo } = trackInfo;
|
||||
const {
|
||||
codec,
|
||||
codecDescription,
|
||||
colorSpace,
|
||||
avcCodecInfo,
|
||||
hevcCodecInfo,
|
||||
vp9CodecInfo,
|
||||
av1CodecInfo,
|
||||
proresFormat,
|
||||
} = trackInfo;
|
||||
|
||||
if (codec === 'avc') {
|
||||
assert(trackInfo.avcType !== null);
|
||||
@@ -504,6 +560,10 @@ export const extractVideoCodecString = (trackInfo: {
|
||||
}
|
||||
|
||||
return string;
|
||||
} else if (codec === 'prores') {
|
||||
return proresFormat ?? 'apch';
|
||||
} else if (codec !== null) {
|
||||
assertNever(codec);
|
||||
}
|
||||
|
||||
throw new TypeError(`Unhandled codec '${codec}'.`);
|
||||
@@ -671,6 +731,8 @@ export const inferCodecFromCodecString = (codecString: string): MediaCodec | nul
|
||||
return 'vp9';
|
||||
} else if (codecString.startsWith('av01')) {
|
||||
return 'av1';
|
||||
} else if ((PRORES_FOURCCS as readonly string[]).includes(codecString)) {
|
||||
return 'prores';
|
||||
}
|
||||
|
||||
// Audio codecs
|
||||
@@ -746,7 +808,7 @@ export const getAudioEncoderConfigExtension = (codec: AudioCodec) => {
|
||||
return {};
|
||||
};
|
||||
|
||||
const VALID_VIDEO_CODEC_STRING_PREFIXES = ['avc1', 'avc3', 'hev1', 'hvc1', 'vp8', 'vp09', 'av01'];
|
||||
const VALID_VIDEO_CODEC_STRING_PREFIXES = ['avc1', 'avc3', 'hev1', 'hvc1', 'vp8', 'vp09', 'av01', ...PRORES_FOURCCS];
|
||||
const AVC_CODEC_STRING_REGEX = /^(avc1|avc3)\.[0-9a-fA-F]{6}$/;
|
||||
const HEVC_CODEC_STRING_REGEX = /^(hev1|hvc1)\.(?:[ABC]?\d+)\.[0-9a-fA-F]{1,8}\.[LH]\d+(?:\.[0-9a-fA-F]{1,2}){0,6}$/;
|
||||
const VP9_CODEC_STRING_REGEX = /^vp09(?:\.\d{2}){3}(?:(?:\.\d{2}){5})?$/;
|
||||
@@ -913,6 +975,15 @@ export const validateVideoChunkMetadata = (metadata: EncodedVideoChunkMetadata |
|
||||
+ ' specified in Section "Codecs Parameter String" of https://aomediacodec.github.io/av1-isobmff/.',
|
||||
);
|
||||
}
|
||||
} else if (PRORES_FOURCCS.some(x => metadata.decoderConfig!.codec.startsWith(x))) {
|
||||
// ProRes-specific validation
|
||||
|
||||
if (!PRORES_FOURCCS.some(x => metadata.decoderConfig!.codec === x)) {
|
||||
throw new TypeError(
|
||||
`Video chunk metadata decoder configuration codec string for ProRes must be one of the valid ProRes`
|
||||
+ ` four-character codes: ${PRORES_FOURCCS.join(', ')}.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
+33
-17
@@ -54,6 +54,7 @@ import {
|
||||
AudioSample,
|
||||
clampCropRectangle,
|
||||
CropRectangle,
|
||||
getBytesPerSample,
|
||||
validateCropRectangle,
|
||||
VideoSample,
|
||||
VideoSampleResource,
|
||||
@@ -1328,9 +1329,13 @@ export class Conversion {
|
||||
firstSample.close();
|
||||
await tempOutput.finalize();
|
||||
} catch (error) {
|
||||
Logging._info('Error when probing encoder support. Falling back to rerender path.', error);
|
||||
needsRerender = true;
|
||||
Logging._warn(
|
||||
'An error occurred when probing encoder support. Falling back to rerender path.', error,
|
||||
);
|
||||
void tempOutput.cancel();
|
||||
|
||||
needsRerender = true;
|
||||
encodingConfig.transform.force = true;
|
||||
}
|
||||
} else {
|
||||
await tempOutput.cancel();
|
||||
@@ -1442,7 +1447,7 @@ export class Conversion {
|
||||
let sampleRate = trackOptions.sampleRate ?? originalSampleRate;
|
||||
|
||||
const needsTrimming = firstTimestamp < this._startTimestamp;
|
||||
const needsPadding = firstTimestamp > this._startTimestamp && !this.output.format.supportsTimestampedMediaData;
|
||||
let needsPadding = firstTimestamp > this._startTimestamp && !this.output.format.supportsTimestampedMediaData;
|
||||
|
||||
let audioCodecs = this.output.format.getSupportedAudioCodecs();
|
||||
if (
|
||||
@@ -1589,20 +1594,6 @@ export class Conversion {
|
||||
this._trackPromises.push((async () => {
|
||||
await this._started;
|
||||
|
||||
if (needsPadding) {
|
||||
const paddingLength = firstTimestamp - this._startTimestamp;
|
||||
const paddingLengthSamples = Math.round(paddingLength * originalSampleRate);
|
||||
|
||||
const silentSample = new AudioSample({
|
||||
data: new Float32Array(paddingLengthSamples * originalNumberOfChannels),
|
||||
format: 'f32-planar',
|
||||
numberOfChannels: originalNumberOfChannels,
|
||||
sampleRate: originalSampleRate,
|
||||
timestamp: 0,
|
||||
});
|
||||
await this._registerAudioSample(silentSample, source, outputTrackId, () => lastSampleTimestamp);
|
||||
}
|
||||
|
||||
const sink = new AudioSampleSink(track);
|
||||
for await (let sample of sink.samples(this._startTimestamp, this._endTimestamp)) {
|
||||
if (this._canceled) {
|
||||
@@ -1610,6 +1601,31 @@ export class Conversion {
|
||||
return;
|
||||
}
|
||||
|
||||
if (needsPadding) {
|
||||
// Add one padding sample at the beginning
|
||||
const paddingLength = firstTimestamp - this._startTimestamp;
|
||||
const paddingLengthSamples = Math.round(paddingLength * originalSampleRate);
|
||||
|
||||
const bytesPerSample = getBytesPerSample(sample.format);
|
||||
const data = new Uint8Array(bytesPerSample * paddingLengthSamples * originalNumberOfChannels);
|
||||
if (sample.format === 'u8' || sample.format === 'u8-planar') {
|
||||
data.fill(2 ** 7); // Fill it with the silent value
|
||||
}
|
||||
|
||||
const silentSample = new AudioSample({
|
||||
data,
|
||||
// Use the same format the decoder is spitting out. This avoids feeding changing sample
|
||||
// formats to the audio encoder.
|
||||
format: sample.format,
|
||||
numberOfChannels: originalNumberOfChannels,
|
||||
sampleRate: originalSampleRate,
|
||||
timestamp: 0,
|
||||
});
|
||||
await this._registerAudioSample(silentSample, source, outputTrackId, () => lastSampleTimestamp);
|
||||
|
||||
needsPadding = false;
|
||||
}
|
||||
|
||||
let startFrame = 0;
|
||||
let endFrame = sample.numberOfFrames;
|
||||
|
||||
|
||||
@@ -27,6 +27,8 @@ export abstract class CustomVideoDecoder {
|
||||
readonly config!: VideoDecoderConfig;
|
||||
/** The callback to call when a decoded VideoSample is available. */
|
||||
readonly onSample!: (sample: VideoSample) => unknown;
|
||||
/** The callback to call to surface out-of-band errors that can't be surfaced through the main methods. */
|
||||
readonly onError!: (error: unknown) => undefined;
|
||||
|
||||
/** Returns true if and only if the decoder can decode the given codec configuration. */
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
@@ -57,6 +59,8 @@ export abstract class CustomAudioDecoder {
|
||||
readonly config!: AudioDecoderConfig;
|
||||
/** The callback to call when a decoded AudioSample is available. */
|
||||
readonly onSample!: (sample: AudioSample) => unknown;
|
||||
/** The callback to call to surface out-of-band errors that can't be surfaced through the main methods. */
|
||||
readonly onError!: (error: unknown) => undefined;
|
||||
|
||||
/** Returns true if and only if the decoder can decode the given codec configuration. */
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
@@ -87,6 +91,8 @@ export abstract class CustomVideoEncoder {
|
||||
readonly config!: VideoEncoderConfig;
|
||||
/** The callback to call when an EncodedPacket is available. */
|
||||
readonly onPacket!: (packet: EncodedPacket, meta?: EncodedVideoChunkMetadata) => unknown;
|
||||
/** The callback to call to surface out-of-band errors that can't be surfaced through the main methods. */
|
||||
readonly onError!: (error: unknown) => undefined;
|
||||
|
||||
/** Returns true if and only if the encoder can encode the given codec configuration. */
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
@@ -117,6 +123,8 @@ export abstract class CustomAudioEncoder {
|
||||
readonly config!: AudioEncoderConfig;
|
||||
/** The callback to call when an EncodedPacket is available. */
|
||||
readonly onPacket!: (packet: EncodedPacket, meta?: EncodedAudioChunkMetadata) => unknown;
|
||||
/** The callback to call to surface out-of-band errors that can't be surfaced through the main methods. */
|
||||
readonly onError!: (error: unknown) => undefined;
|
||||
|
||||
/** Returns true if and only if the encoder can encode the given codec configuration. */
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
|
||||
+1
-1
@@ -140,7 +140,7 @@ export const canDecodeVideo = async (
|
||||
...options,
|
||||
codedWidth: options.codedWidth ?? 1280,
|
||||
codedHeight: options.codedHeight ?? 720,
|
||||
codec: options.codec ?? buildVideoCodecString(codec, 1280, 720, 1e6),
|
||||
codec: options.codec ?? buildVideoCodecString(codec, 1280, 720, 1e6, false),
|
||||
};
|
||||
resolvedOptions.description ??= guessDescriptionForVideo(resolvedOptions);
|
||||
|
||||
|
||||
+10
-8
@@ -251,8 +251,9 @@ export type VideoEncodingAdditionalOptions = {
|
||||
* What to do with alpha data contained in the video samples.
|
||||
*
|
||||
* - `'discard'` (default): Only the samples' color data is kept; the video is opaque.
|
||||
* - `'keep'`: The samples' alpha data is also encoded as side data. Make sure to pair this mode with a container
|
||||
* format that supports transparency (such as WebM or Matroska).
|
||||
* - `'keep'`: The samples' alpha data is also encoded. Depending on the codec, the alpha may be emitted as packet
|
||||
* side data or in-band alongside the main packet. For codecs that emit alpha side data, such as VP9, make sure to
|
||||
* pair this mode with a container format that supports transparency (such as WebM or Matroska).
|
||||
*/
|
||||
alpha?: 'discard' | 'keep';
|
||||
/** Configures the bitrate mode; defaults to `'variable'`. */
|
||||
@@ -344,6 +345,7 @@ export const buildVideoEncoderConfig = (options: {
|
||||
options.width,
|
||||
options.height,
|
||||
resolvedBitrate,
|
||||
options.alpha === 'keep',
|
||||
),
|
||||
width: options.width,
|
||||
height: options.height,
|
||||
@@ -549,20 +551,20 @@ export class Quality {
|
||||
/** @internal */
|
||||
_toVideoBitrate(codec: VideoCodec, width: number, height: number) {
|
||||
const pixels = width * height;
|
||||
const referencePixels = 1920 * 1080;
|
||||
const referenceBitrate = 3_000_000;
|
||||
const scaleFactor = Math.pow(pixels / referencePixels, 0.95); // Slight non-linear scaling
|
||||
const baseBitrate = referenceBitrate * scaleFactor;
|
||||
|
||||
const codecEfficiencyFactors = {
|
||||
const codecEfficiencyFactors: Record<VideoCodec, number> = {
|
||||
avc: 1.0, // H.264/AVC (baseline)
|
||||
hevc: 0.6, // H.265/HEVC (~40% more efficient than AVC)
|
||||
vp9: 0.6, // Similar to HEVC
|
||||
av1: 0.4, // ~60% more efficient than AVC
|
||||
vp8: 1.2, // Slightly less efficient than AVC
|
||||
prores: 220_000_000 / referenceBitrate, // Apple ProRes white paper claims 220 Mbps for 1080p 422 HQ @30Hz
|
||||
};
|
||||
|
||||
const referencePixels = 1920 * 1080;
|
||||
const referenceBitrate = 3000000;
|
||||
const scaleFactor = Math.pow(pixels / referencePixels, 0.95); // Slight non-linear scaling
|
||||
const baseBitrate = referenceBitrate * scaleFactor;
|
||||
|
||||
const codecAdjustedBitrate = baseBitrate * codecEfficiencyFactors[codec];
|
||||
const finalBitrate = codecAdjustedBitrate * this._factor;
|
||||
|
||||
|
||||
@@ -722,7 +722,7 @@ export const videoSampleDescription = (
|
||||
u16(0x0018), // Depth
|
||||
i16(0xffff), // Pre-defined
|
||||
], [
|
||||
VIDEO_CODEC_TO_CONFIGURATION_BOX[trackData.track.source._codec](trackData),
|
||||
VIDEO_CODEC_TO_CONFIGURATION_BOX[trackData.track.source._codec]?.(trackData) ?? null,
|
||||
pasp(trackData),
|
||||
colorSpaceIsComplete(trackData.info.decoderConfig.colorSpace) ? colr(trackData) : null,
|
||||
]);
|
||||
@@ -1780,15 +1780,20 @@ const videoCodecToBoxName = (codec: VideoCodec, fullCodecString: string) => {
|
||||
case 'vp8': return 'vp08';
|
||||
case 'vp9': return 'vp09';
|
||||
case 'av1': return 'av01';
|
||||
case 'prores': return fullCodecString;
|
||||
}
|
||||
};
|
||||
|
||||
const VIDEO_CODEC_TO_CONFIGURATION_BOX: Record<VideoCodec, (trackData: IsobmffVideoTrackData) => Box | null> = {
|
||||
const VIDEO_CODEC_TO_CONFIGURATION_BOX: Record<
|
||||
VideoCodec,
|
||||
((trackData: IsobmffVideoTrackData) => Box | null) | null
|
||||
> = {
|
||||
avc: avcC,
|
||||
hevc: hvcC,
|
||||
vp8: vpcC,
|
||||
vp9: vpcC,
|
||||
av1: av1C,
|
||||
prores: null,
|
||||
};
|
||||
|
||||
const audioCodecToBoxName = (codec: AudioCodec, isQuickTime: boolean): string => {
|
||||
|
||||
@@ -18,6 +18,8 @@ import {
|
||||
parsePcmCodec,
|
||||
PCM_AUDIO_CODECS,
|
||||
PcmAudioCodec,
|
||||
PRORES_FOURCCS,
|
||||
ProresFourCc,
|
||||
VideoCodec,
|
||||
} from '../codec';
|
||||
import {
|
||||
@@ -147,6 +149,7 @@ type InternalTrack = {
|
||||
hevcCodecInfo: HevcDecoderConfigurationRecord | null;
|
||||
vp9CodecInfo: Vp9CodecInfo | null;
|
||||
av1CodecInfo: Av1CodecInfo | null;
|
||||
proresFormat: ProresFourCc | null;
|
||||
};
|
||||
} | {
|
||||
info: {
|
||||
@@ -1008,6 +1011,7 @@ export class IsobmffDemuxer extends Demuxer {
|
||||
hevcCodecInfo: null,
|
||||
vp9CodecInfo: null,
|
||||
av1CodecInfo: null,
|
||||
proresFormat: null,
|
||||
};
|
||||
} else if (handlerType === 'soun') {
|
||||
track.info = {
|
||||
@@ -1093,6 +1097,9 @@ export class IsobmffDemuxer extends Demuxer {
|
||||
track.info.codec = 'vp9';
|
||||
} else if (codecName === 'av01') {
|
||||
track.info.codec = 'av1';
|
||||
} else if ((PRORES_FOURCCS as readonly string[]).includes(lowercaseBoxName)) {
|
||||
track.info.codec = 'prores';
|
||||
track.info.proresFormat = lowercaseBoxName as ProresFourCc;
|
||||
} else if (codecName === null) {
|
||||
Logging._warn(`Unknown encrypted video codec due to missing frma box.`);
|
||||
} else {
|
||||
@@ -3341,7 +3348,10 @@ class IsobmffVideoTrackBacking extends IsobmffTrackBacking implements InputVideo
|
||||
}
|
||||
|
||||
async canBeTransparent() {
|
||||
return false;
|
||||
return this.internalTrack.info.codec === 'prores' && (
|
||||
this.internalTrack.info.proresFormat === 'ap4h'
|
||||
|| this.internalTrack.info.proresFormat === 'ap4x'
|
||||
);
|
||||
}
|
||||
|
||||
async getDecoderConfig(): Promise<VideoDecoderConfig | null> {
|
||||
|
||||
@@ -741,6 +741,7 @@ export const CODEC_STRING_MAP: Partial<Record<MediaCodec, string>> = {
|
||||
'vp8': 'V_VP8',
|
||||
'vp9': 'V_VP9',
|
||||
'av1': 'V_AV1',
|
||||
'prores': 'V_PRORES',
|
||||
|
||||
'aac': 'A_AAC',
|
||||
'mp3': 'A_MPEG/L3',
|
||||
|
||||
@@ -20,6 +20,8 @@ import {
|
||||
extractVideoCodecString,
|
||||
MediaCodec,
|
||||
OPUS_SAMPLE_RATE,
|
||||
PRORES_FOURCCS,
|
||||
ProresFourCc,
|
||||
VideoCodec,
|
||||
} from '../codec';
|
||||
import { Demuxer } from '../demuxer';
|
||||
@@ -43,6 +45,7 @@ import {
|
||||
normalizeRotation,
|
||||
Rotation,
|
||||
roundIfAlmostInteger,
|
||||
textDecoder,
|
||||
TRANSFER_CHARACTERISTICS_MAP_INVERSE,
|
||||
UNDETERMINED_LANGUAGE,
|
||||
} from '../misc';
|
||||
@@ -214,6 +217,7 @@ type InternalTrack = {
|
||||
codecDescription: Uint8Array | null;
|
||||
colorSpace: VideoColorSpaceInit | null;
|
||||
alphaMode: boolean;
|
||||
proresFormat: ProresFourCc | null;
|
||||
}
|
||||
| {
|
||||
type: 'audio';
|
||||
@@ -1077,6 +1081,18 @@ export class MatroskaDemuxer extends Demuxer {
|
||||
this.currentTrack.info.codec = 'vp9';
|
||||
} else if (codecIdWithoutSuffix === CODEC_STRING_MAP.av1) {
|
||||
this.currentTrack.info.codec = 'av1';
|
||||
} else if (codecIdWithoutSuffix === CODEC_STRING_MAP.prores) {
|
||||
const format = this.currentTrack.codecPrivate
|
||||
? textDecoder.decode(this.currentTrack.codecPrivate)
|
||||
: '';
|
||||
|
||||
if ((PRORES_FOURCCS as readonly string[]).includes(format)) {
|
||||
this.currentTrack.info.codec = 'prores';
|
||||
this.currentTrack.info.proresFormat = format as ProresFourCc;
|
||||
} else {
|
||||
// Either an invalid string or ProRes RAW, which we don't support yet (it's a
|
||||
// different codec).
|
||||
}
|
||||
}
|
||||
|
||||
const videoTrack = this.currentTrack as InternalVideoTrack;
|
||||
@@ -1170,6 +1186,7 @@ export class MatroskaDemuxer extends Demuxer {
|
||||
codecDescription: null,
|
||||
colorSpace: null,
|
||||
alphaMode: false,
|
||||
proresFormat: null,
|
||||
};
|
||||
} else if (type === 2) {
|
||||
this.currentTrack.info = {
|
||||
@@ -2435,7 +2452,12 @@ class MatroskaVideoTrackBacking extends MatroskaTrackBacking implements InputVid
|
||||
}
|
||||
|
||||
async canBeTransparent() {
|
||||
return this.internalTrack.info.alphaMode;
|
||||
return this.internalTrack.info.alphaMode || (
|
||||
this.internalTrack.info.codec === 'prores' && (
|
||||
this.internalTrack.info.proresFormat === 'ap4h'
|
||||
|| this.internalTrack.info.proresFormat === 'ap4x'
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
async getDecoderConfig(): Promise<VideoDecoderConfig | null> {
|
||||
@@ -2477,6 +2499,7 @@ class MatroskaVideoTrackBacking extends MatroskaTrackBacking implements InputVid
|
||||
av1CodecInfo: this.internalTrack.info.codec === 'av1' && firstPacket
|
||||
? extractAv1CodecInfoFromPacket(firstPacket.data)
|
||||
: null,
|
||||
proresFormat: this.internalTrack.info.proresFormat,
|
||||
}),
|
||||
codedWidth: this.internalTrack.info.width,
|
||||
codedHeight: this.internalTrack.info.height,
|
||||
|
||||
@@ -87,6 +87,7 @@ type InternalMediaChunk = {
|
||||
type MatroskaTrackData = {
|
||||
chunkQueue: InternalMediaChunk[];
|
||||
lastWrittenMsTimestamp: number | null;
|
||||
codecPrivate: AllowSharedBufferSource | null;
|
||||
closed: boolean;
|
||||
} & ({
|
||||
track: OutputVideoTrack;
|
||||
@@ -334,6 +335,9 @@ export class MatroskaMuxer extends Muxer {
|
||||
{ id: EBMLId.FlagLacing, data: 0 },
|
||||
{ id: EBMLId.Language, data: trackData.track.metadata.languageCode ?? UNDETERMINED_LANGUAGE },
|
||||
{ id: EBMLId.CodecID, data: codecId },
|
||||
trackData.codecPrivate
|
||||
? { id: EBMLId.CodecPrivate, data: toUint8Array(trackData.codecPrivate) }
|
||||
: null,
|
||||
{ id: EBMLId.CodecDelay, data: 0 },
|
||||
{ id: EBMLId.SeekPreRoll, data: seekPreRollNs },
|
||||
trackData.track.metadata.name !== undefined
|
||||
@@ -350,12 +354,6 @@ export class MatroskaMuxer extends Muxer {
|
||||
const { frameRate, rotation } = trackData.track.metadata;
|
||||
|
||||
const elements: EBMLElement['data'] = [
|
||||
(trackData.info.decoderConfig.description
|
||||
? {
|
||||
id: EBMLId.CodecPrivate,
|
||||
data: toUint8Array(trackData.info.decoderConfig.description),
|
||||
}
|
||||
: null),
|
||||
(frameRate
|
||||
? {
|
||||
id: EBMLId.DefaultDuration,
|
||||
@@ -432,12 +430,6 @@ export class MatroskaMuxer extends Muxer {
|
||||
: null;
|
||||
|
||||
return [
|
||||
(trackData.info.decoderConfig.description
|
||||
? {
|
||||
id: EBMLId.CodecPrivate,
|
||||
data: toUint8Array(trackData.info.decoderConfig.description),
|
||||
}
|
||||
: null),
|
||||
{ id: EBMLId.Audio, data: [
|
||||
{ id: EBMLId.SamplingFrequency, data: new EBMLFloat32(trackData.info.sampleRate) },
|
||||
{ id: EBMLId.Channels, data: trackData.info.numberOfChannels },
|
||||
@@ -446,10 +438,9 @@ export class MatroskaMuxer extends Muxer {
|
||||
];
|
||||
}
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
private subtitleSpecificTrackInfo(trackData: MatroskaSubtitleTrackData) {
|
||||
return [
|
||||
{ id: EBMLId.CodecPrivate, data: textEncoder.encode(trackData.info.config.description) },
|
||||
];
|
||||
return [];
|
||||
}
|
||||
|
||||
private maybeCreateTags() {
|
||||
@@ -768,28 +759,26 @@ export class MatroskaMuxer extends Muxer {
|
||||
},
|
||||
chunkQueue: [],
|
||||
lastWrittenMsTimestamp: null,
|
||||
codecPrivate: meta.decoderConfig.description ?? null,
|
||||
closed: false,
|
||||
};
|
||||
|
||||
if (track.source._codec === 'vp9') {
|
||||
// https://www.webmproject.org/docs/container specifies that VP9 "SHOULD" make use of the CodecPrivate
|
||||
// field. Since WebCodecs makes no use of the description field for VP9, we need to derive it ourselves:
|
||||
newTrackData.info.decoderConfig = {
|
||||
...newTrackData.info.decoderConfig,
|
||||
description: new Uint8Array(
|
||||
generateVp9CodecConfigurationFromCodecString(newTrackData.info.decoderConfig.codec),
|
||||
),
|
||||
};
|
||||
newTrackData.codecPrivate = new Uint8Array(
|
||||
generateVp9CodecConfigurationFromCodecString(newTrackData.info.decoderConfig.codec),
|
||||
);
|
||||
} else if (track.source._codec === 'av1') {
|
||||
// Per https://github.com/ietf-wg-cellar/matroska-specification/blob/master/codec/av1.md, AV1 requires
|
||||
// CodecPrivate to be set, but WebCodecs makes no use of the description field for AV1. Thus, let's derive
|
||||
// it ourselves:
|
||||
newTrackData.info.decoderConfig = {
|
||||
...newTrackData.info.decoderConfig,
|
||||
description: new Uint8Array(
|
||||
generateAv1CodecConfigurationFromCodecString(newTrackData.info.decoderConfig.codec),
|
||||
),
|
||||
};
|
||||
newTrackData.codecPrivate = new Uint8Array(
|
||||
generateAv1CodecConfigurationFromCodecString(newTrackData.info.decoderConfig.codec),
|
||||
);
|
||||
} else if (track.source._codec === 'prores') {
|
||||
// "The Private Data contains the FourCC as found in MP4 movies"
|
||||
newTrackData.codecPrivate = textEncoder.encode(meta.decoderConfig.codec);
|
||||
}
|
||||
|
||||
this.trackDatas.push(newTrackData);
|
||||
@@ -855,6 +844,7 @@ export class MatroskaMuxer extends Muxer {
|
||||
},
|
||||
chunkQueue: [],
|
||||
lastWrittenMsTimestamp: null,
|
||||
codecPrivate: decoderConfig.description ?? null,
|
||||
closed: false,
|
||||
};
|
||||
|
||||
@@ -887,6 +877,7 @@ export class MatroskaMuxer extends Muxer {
|
||||
},
|
||||
chunkQueue: [],
|
||||
lastWrittenMsTimestamp: null,
|
||||
codecPrivate: textEncoder.encode(meta.config.description),
|
||||
closed: false,
|
||||
};
|
||||
|
||||
|
||||
+172
-308
@@ -28,6 +28,7 @@ import {
|
||||
assert,
|
||||
assertNever,
|
||||
CallSerializer,
|
||||
clamp,
|
||||
getInt24,
|
||||
getUint24,
|
||||
insertSorted,
|
||||
@@ -38,6 +39,7 @@ import {
|
||||
last,
|
||||
mapAsyncGenerator,
|
||||
promiseWithResolvers,
|
||||
removeItem,
|
||||
Rotation,
|
||||
toAsyncIterator,
|
||||
toDataView,
|
||||
@@ -54,7 +56,6 @@ import {
|
||||
VideoSample,
|
||||
VideoSamplePixelFormat,
|
||||
} from './sample';
|
||||
import { Logging } from './logging';
|
||||
|
||||
/**
|
||||
* Additional options for controlling packet retrieval.
|
||||
@@ -333,7 +334,8 @@ export class EncodedPacketSink {
|
||||
// This stores errors that are "out of band" in the sense that they didn't occur in the normal flow of this
|
||||
// method but instead in a different context. This error should not go unnoticed and must be bubbled up to
|
||||
// the consumer.
|
||||
let outOfBandError = null as Error | null;
|
||||
let outOfBandError = null as unknown;
|
||||
let hasOutOfBandError = false;
|
||||
|
||||
const timestamps: number[] = [];
|
||||
// The queue should always be big enough to hold 1 second worth of packets
|
||||
@@ -364,9 +366,10 @@ export class EncodedPacketSink {
|
||||
|
||||
ended = true;
|
||||
onQueueNotEmpty();
|
||||
})().catch((error: Error) => {
|
||||
if (!outOfBandError) {
|
||||
})().catch((error) => {
|
||||
if (!hasOutOfBandError) {
|
||||
outOfBandError = error;
|
||||
hasOutOfBandError = true;
|
||||
onQueueNotEmpty();
|
||||
}
|
||||
});
|
||||
@@ -380,7 +383,7 @@ export class EncodedPacketSink {
|
||||
throw new InputDisposedError();
|
||||
} else if (terminated) {
|
||||
return { value: undefined, done: true };
|
||||
} else if (outOfBandError) {
|
||||
} else if (hasOutOfBandError) {
|
||||
throw outOfBandError;
|
||||
} else if (packetQueue.length > 0) {
|
||||
const value = packetQueue.shift()!;
|
||||
@@ -423,7 +426,7 @@ abstract class DecoderWrapper<
|
||||
> {
|
||||
constructor(
|
||||
public onSample: (sample: MediaSample) => unknown,
|
||||
public onError: (error: Error) => unknown,
|
||||
public onError: (error: unknown) => unknown,
|
||||
) {}
|
||||
|
||||
abstract getDecodeQueueSize(): number;
|
||||
@@ -446,7 +449,7 @@ export abstract class BaseMediaSampleSink<
|
||||
/** @internal */
|
||||
abstract _createDecoder(
|
||||
onSample: (sample: MediaSample) => unknown,
|
||||
onError: (error: Error) => unknown
|
||||
onError: (error: unknown) => unknown
|
||||
): Promise<DecoderWrapper<MediaSample>>;
|
||||
/** @internal */
|
||||
abstract _createPacketSink(): EncodedPacketSink;
|
||||
@@ -472,7 +475,8 @@ export abstract class BaseMediaSampleSink<
|
||||
// This stores errors that are "out of band" in the sense that they didn't occur in the normal flow of this
|
||||
// method but instead in a different context. This error should not go unnoticed and must be bubbled up to
|
||||
// the consumer.
|
||||
let outOfBandError = null as Error | null;
|
||||
let outOfBandError = null as unknown;
|
||||
let hasOutOfBandError = false;
|
||||
|
||||
const packetRetrievalOptions: PacketRetrievalOptions = {
|
||||
...options,
|
||||
@@ -518,8 +522,9 @@ export abstract class BaseMediaSampleSink<
|
||||
({ promise: queueNotEmpty, resolve: onQueueNotEmpty } = promiseWithResolvers());
|
||||
}
|
||||
}, (error) => {
|
||||
if (!outOfBandError) {
|
||||
if (!hasOutOfBandError) {
|
||||
outOfBandError = error;
|
||||
hasOutOfBandError = true;
|
||||
onQueueNotEmpty();
|
||||
}
|
||||
});
|
||||
@@ -572,9 +577,10 @@ export abstract class BaseMediaSampleSink<
|
||||
|
||||
decoderIsFlushed = true;
|
||||
onQueueNotEmpty(); // To unstuck the generator
|
||||
})().catch((error: Error) => {
|
||||
if (!outOfBandError) {
|
||||
})().catch((error) => {
|
||||
if (!hasOutOfBandError) {
|
||||
outOfBandError = error;
|
||||
hasOutOfBandError = true;
|
||||
onQueueNotEmpty();
|
||||
}
|
||||
});
|
||||
@@ -595,7 +601,7 @@ export abstract class BaseMediaSampleSink<
|
||||
throw new InputDisposedError();
|
||||
} else if (terminated) {
|
||||
return { value: undefined, done: true };
|
||||
} else if (outOfBandError) {
|
||||
} else if (hasOutOfBandError) {
|
||||
closeSamples();
|
||||
throw outOfBandError;
|
||||
} else if (sampleQueue.length > 0) {
|
||||
@@ -645,7 +651,8 @@ export abstract class BaseMediaSampleSink<
|
||||
// This stores errors that are "out of band" in the sense that they didn't occur in the normal flow of this
|
||||
// method but instead in a different context. This error should not go unnoticed and must be bubbled up to
|
||||
// the consumer.
|
||||
let outOfBandError = null as Error | null;
|
||||
let outOfBandError = null as unknown;
|
||||
let hasOutOfBandError = false;
|
||||
|
||||
const pushToQueue = (sample: MediaSample | null) => {
|
||||
sampleQueue.push(sample);
|
||||
@@ -687,8 +694,9 @@ export abstract class BaseMediaSampleSink<
|
||||
sample.close();
|
||||
}
|
||||
}, (error) => {
|
||||
if (!outOfBandError) {
|
||||
if (!hasOutOfBandError) {
|
||||
outOfBandError = error;
|
||||
hasOutOfBandError = true;
|
||||
onQueueNotEmpty();
|
||||
}
|
||||
});
|
||||
@@ -792,9 +800,10 @@ export abstract class BaseMediaSampleSink<
|
||||
|
||||
decoderIsFlushed = true;
|
||||
onQueueNotEmpty(); // To unstuck the generator
|
||||
})().catch((error: Error) => {
|
||||
if (!outOfBandError) {
|
||||
})().catch((error) => {
|
||||
if (!hasOutOfBandError) {
|
||||
outOfBandError = error;
|
||||
hasOutOfBandError = true;
|
||||
onQueueNotEmpty();
|
||||
}
|
||||
});
|
||||
@@ -814,7 +823,7 @@ export abstract class BaseMediaSampleSink<
|
||||
throw new InputDisposedError();
|
||||
} else if (terminated) {
|
||||
return { value: undefined, done: true };
|
||||
} else if (outOfBandError) {
|
||||
} else if (hasOutOfBandError) {
|
||||
closeSamples();
|
||||
throw outOfBandError;
|
||||
} else if (sampleQueue.length > 0) {
|
||||
@@ -878,11 +887,12 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
|
||||
nullAlphaFrameQueue: number[] = [];
|
||||
currentAlphaPacketIndex = 0;
|
||||
alphaRaslSkipped = false; // For HEVC stuff
|
||||
frameHandlerSerializer = new CallSerializer();
|
||||
finalSamples: { sample: VideoSample | null }[] = [];
|
||||
mergeAlphaPromises: Promise<void>[] = [];
|
||||
|
||||
constructor(
|
||||
onSample: (sample: VideoSample) => unknown,
|
||||
onError: (error: Error) => unknown,
|
||||
onError: (error: unknown) => unknown,
|
||||
public codec: VideoCodec,
|
||||
public decoderConfig: VideoDecoderConfig,
|
||||
public rotation: Rotation,
|
||||
@@ -906,21 +916,25 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
|
||||
|
||||
this.finalizeAndEmitSample(sample);
|
||||
};
|
||||
// @ts-expect-error It's technically readonly
|
||||
this.customDecoder.onError = (error) => {
|
||||
onError(error);
|
||||
};
|
||||
|
||||
void this.customDecoderCallSerializer.call(() => this.customDecoder!.init());
|
||||
void this.customDecoderCallSerializer
|
||||
.call(() => this.customDecoder!.init())
|
||||
.catch(error => onError(error));
|
||||
} else {
|
||||
const colorHandler = (frame: VideoFrame) => {
|
||||
this.frameHandlerSerializer.call(async () => {
|
||||
if (this.alphaQueue.length > 0) {
|
||||
// Even when no alpha data is present (most of the time), there will be nulls in this queue
|
||||
const alphaFrame = this.alphaQueue.shift();
|
||||
assert(alphaFrame !== undefined);
|
||||
if (this.alphaQueue.length > 0) {
|
||||
// Even when no alpha data is present (most of the time), there will be nulls in this queue
|
||||
const alphaFrame = this.alphaQueue.shift();
|
||||
assert(alphaFrame !== undefined);
|
||||
|
||||
await this.mergeAlpha(frame, alphaFrame);
|
||||
} else {
|
||||
this.colorQueue.push(frame);
|
||||
}
|
||||
}).catch((error: Error) => this.onError(error));
|
||||
void this.mergeAlpha(frame, alphaFrame);
|
||||
} else {
|
||||
this.colorQueue.push(frame);
|
||||
}
|
||||
};
|
||||
|
||||
if (codec === 'avc' && this.decoderConfig.description && isChromium()) {
|
||||
@@ -946,7 +960,7 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
|
||||
try {
|
||||
colorHandler(frame);
|
||||
} catch (error) {
|
||||
this.onError(error as Error);
|
||||
this.onError(error);
|
||||
}
|
||||
},
|
||||
error: (error) => {
|
||||
@@ -984,7 +998,8 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
|
||||
this.customDecoderQueueSize++;
|
||||
void this.customDecoderCallSerializer
|
||||
.call(() => this.customDecoder!.decode(packet))
|
||||
.then(() => this.customDecoderQueueSize--);
|
||||
.catch(error => this.onError(error))
|
||||
.finally(() => this.customDecoderQueueSize--);
|
||||
} else {
|
||||
assert(this.decoder);
|
||||
|
||||
@@ -1045,36 +1060,34 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
|
||||
// Check if we need to set up the alpha decoder
|
||||
if (!this.alphaDecoder) {
|
||||
const alphaHandler = (frame: VideoFrame) => {
|
||||
this.frameHandlerSerializer.call(async () => {
|
||||
if (this.colorQueue.length > 0) {
|
||||
const colorFrame = this.colorQueue.shift();
|
||||
assert(colorFrame !== undefined);
|
||||
|
||||
void this.mergeAlpha(colorFrame, frame);
|
||||
} else {
|
||||
this.alphaQueue.push(frame);
|
||||
}
|
||||
|
||||
// Check if any null frames have been queued for this point
|
||||
this.decodedAlphaChunkCount++;
|
||||
while (
|
||||
this.nullAlphaFrameQueue.length > 0
|
||||
&& this.nullAlphaFrameQueue[0] === this.decodedAlphaChunkCount
|
||||
) {
|
||||
this.nullAlphaFrameQueue.shift();
|
||||
|
||||
if (this.colorQueue.length > 0) {
|
||||
const colorFrame = this.colorQueue.shift();
|
||||
assert(colorFrame !== undefined);
|
||||
|
||||
await this.mergeAlpha(colorFrame, frame);
|
||||
void this.mergeAlpha(colorFrame, null);
|
||||
} else {
|
||||
this.alphaQueue.push(frame);
|
||||
this.alphaQueue.push(null);
|
||||
}
|
||||
}
|
||||
|
||||
// Check if any null frames have been queued for this point
|
||||
this.decodedAlphaChunkCount++;
|
||||
while (
|
||||
this.nullAlphaFrameQueue.length > 0
|
||||
&& this.nullAlphaFrameQueue[0] === this.decodedAlphaChunkCount
|
||||
) {
|
||||
this.nullAlphaFrameQueue.shift();
|
||||
|
||||
if (this.colorQueue.length > 0) {
|
||||
const colorFrame = this.colorQueue.shift();
|
||||
assert(colorFrame !== undefined);
|
||||
|
||||
await this.mergeAlpha(colorFrame, null);
|
||||
} else {
|
||||
this.alphaQueue.push(null);
|
||||
}
|
||||
}
|
||||
|
||||
this.alphaDecoderQueueSize--;
|
||||
}).catch((error: Error) => this.onError(error));
|
||||
this.alphaDecoderQueueSize--;
|
||||
};
|
||||
|
||||
const stack = new Error('Decoding error').stack;
|
||||
@@ -1084,7 +1097,7 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
|
||||
try {
|
||||
alphaHandler(frame);
|
||||
} catch (error) {
|
||||
this.onError(error as Error);
|
||||
this.onError(error);
|
||||
}
|
||||
},
|
||||
error: (error) => {
|
||||
@@ -1196,21 +1209,38 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
|
||||
}
|
||||
|
||||
async mergeAlpha(color: VideoFrame, alpha: VideoFrame | null) {
|
||||
if (!alpha) {
|
||||
// Nothing needs to be merged
|
||||
const finalSample = new VideoSample(color);
|
||||
this.sampleHandler(finalSample);
|
||||
const resolver = promiseWithResolvers();
|
||||
this.mergeAlphaPromises.push(resolver.promise);
|
||||
|
||||
return;
|
||||
// Alpha merging is concurrent but the samples must still be emitted in the same order in which the merging
|
||||
// began. Therefore, serialize the results in an array.
|
||||
const result: { sample: VideoSample | null } = { sample: null };
|
||||
this.finalSamples.push(result);
|
||||
|
||||
try {
|
||||
if (!alpha) {
|
||||
// Nothing needs to be merged
|
||||
result.sample = new VideoSample(color);
|
||||
} else {
|
||||
assert(this.merger);
|
||||
|
||||
// The merger takes ownership of the frames, so no need to close them ourselves
|
||||
const finalFrame = await this.merger.merge(color, alpha);
|
||||
result.sample = new VideoSample(finalFrame);
|
||||
}
|
||||
|
||||
// Emit any leading samples that are ready, preserving input order
|
||||
while (this.finalSamples.length > 0 && this.finalSamples[0]!.sample !== null) {
|
||||
const next = this.finalSamples.shift()!;
|
||||
this.sampleHandler(next.sample!);
|
||||
}
|
||||
} catch (error) {
|
||||
removeItem(this.finalSamples, result);
|
||||
this.onError(error);
|
||||
} finally {
|
||||
removeItem(this.mergeAlphaPromises, resolver.promise);
|
||||
resolver.resolve();
|
||||
}
|
||||
|
||||
assert(this.merger);
|
||||
|
||||
// The merger takes ownership of the frames, so no need to close them ourselves
|
||||
const finalFrame = await this.merger.update(color, alpha);
|
||||
|
||||
const finalSample = new VideoSample(finalFrame);
|
||||
this.sampleHandler(finalSample);
|
||||
}
|
||||
|
||||
async flush() {
|
||||
@@ -1222,7 +1252,7 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
|
||||
this.decoder.flush(),
|
||||
this.alphaDecoder?.flush(),
|
||||
]);
|
||||
await this.frameHandlerSerializer.currentPromise;
|
||||
await Promise.all(this.mergeAlphaPromises);
|
||||
|
||||
this.colorQueue.forEach(x => x.close());
|
||||
this.colorQueue.length = 0;
|
||||
@@ -1272,246 +1302,73 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
|
||||
}
|
||||
}
|
||||
|
||||
let mergerGpuUnavailable = false;
|
||||
let mergerWorkerUrl: string | null = null;
|
||||
|
||||
/** Utility class that merges together color and alpha information using simple WebGL 2 shaders. */
|
||||
/** Utility class that merges together color and alpha information on the CPU in a pool of workers. */
|
||||
export class ColorAlphaMerger {
|
||||
static forceCpu = true;
|
||||
|
||||
canvas: OffscreenCanvas | HTMLCanvasElement | null = null;
|
||||
private gl: WebGL2RenderingContext | null = null;
|
||||
private program: WebGLProgram | null = null;
|
||||
private vao: WebGLVertexArrayObject | null = null;
|
||||
private colorTexture: WebGLTexture | null = null;
|
||||
private alphaTexture: WebGLTexture | null = null;
|
||||
|
||||
private worker: Worker | null = null;
|
||||
private workers: Worker[] = [];
|
||||
private nextWorkerIndex = 0;
|
||||
private pendingRequests = new Map<number, ReturnType<typeof promiseWithResolvers<VideoFrame>>>();
|
||||
private nextRequestId = 0;
|
||||
|
||||
constructor() {
|
||||
const canMakeCanvas = typeof OffscreenCanvas !== 'undefined'
|
||||
// eslint-disable-next-line @typescript-eslint/no-deprecated
|
||||
|| (typeof document !== 'undefined' && typeof document.createElement === 'function');
|
||||
|
||||
if (!ColorAlphaMerger.forceCpu && canMakeCanvas && !mergerGpuUnavailable) {
|
||||
// Try the GPU path. If anything goes wrong, we silently fall back to the CPU path.
|
||||
try {
|
||||
// Canvas will be resized later
|
||||
if (typeof OffscreenCanvas !== 'undefined') {
|
||||
// Prefer OffscreenCanvas for Worker environments
|
||||
this.canvas = new OffscreenCanvas(300, 150);
|
||||
} else {
|
||||
this.canvas = document.createElement('canvas');
|
||||
}
|
||||
|
||||
const gl = this.canvas.getContext('webgl2', {
|
||||
premultipliedAlpha: false,
|
||||
}) as unknown as WebGL2RenderingContext | null; // Casting because of some TypeScript weirdness
|
||||
if (!gl) {
|
||||
throw new Error('Couldn\'t acquire WebGL 2 context.');
|
||||
}
|
||||
|
||||
this.gl = gl;
|
||||
this.program = this.createProgram();
|
||||
this.vao = this.createVAO();
|
||||
this.colorTexture = this.createTexture();
|
||||
this.alphaTexture = this.createTexture();
|
||||
|
||||
this.gl.useProgram(this.program);
|
||||
this.gl.uniform1i(this.gl.getUniformLocation(this.program, 'u_colorTexture'), 0);
|
||||
this.gl.uniform1i(this.gl.getUniformLocation(this.program, 'u_alphaTexture'), 1);
|
||||
} catch (error) {
|
||||
this.gl = null;
|
||||
this.canvas = null;
|
||||
mergerGpuUnavailable = true;
|
||||
Logging._warn('Falling back to CPU for color/alpha merging.', error);
|
||||
merge(color: VideoFrame, alpha: VideoFrame): Promise<VideoFrame> {
|
||||
if (this.workers.length === 0) {
|
||||
if (!mergerWorkerUrl) {
|
||||
const blob = new Blob(
|
||||
[`(${colorAlphaMergerWorkerCode.toString()})()`],
|
||||
{ type: 'application/javascript' },
|
||||
);
|
||||
mergerWorkerUrl = URL.createObjectURL(blob);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async update(color: VideoFrame, alpha: VideoFrame): Promise<VideoFrame> {
|
||||
if (this.gl) {
|
||||
return this.updateGpu(color, alpha);
|
||||
} else {
|
||||
return this.updateCpu(color, alpha);
|
||||
}
|
||||
}
|
||||
const poolSize = clamp(navigator.hardwareConcurrency, 1, 4);
|
||||
for (let i = 0; i < poolSize; i++) {
|
||||
const worker = new Worker(mergerWorkerUrl);
|
||||
|
||||
private createProgram(): WebGLProgram {
|
||||
assert(this.gl);
|
||||
worker.addEventListener('message', (event: MessageEvent<ColorAlphaMergerWorkerResponse>) => {
|
||||
const data = event.data;
|
||||
const pending = this.pendingRequests.get(data.id);
|
||||
if (!pending) {
|
||||
return;
|
||||
}
|
||||
this.pendingRequests.delete(data.id);
|
||||
|
||||
const vertexShader = this.createShader(this.gl.VERTEX_SHADER, `#version 300 es
|
||||
in vec2 a_position;
|
||||
in vec2 a_texCoord;
|
||||
out vec2 v_texCoord;
|
||||
|
||||
void main() {
|
||||
gl_Position = vec4(a_position, 0.0, 1.0);
|
||||
v_texCoord = a_texCoord;
|
||||
if ('error' in data) {
|
||||
pending.reject(new Error(data.error));
|
||||
} else {
|
||||
pending.resolve(data.frame);
|
||||
}
|
||||
});
|
||||
|
||||
worker.addEventListener('error', (event) => {
|
||||
const error = new Error(event.message || 'Color/alpha merge worker error.');
|
||||
for (const pending of this.pendingRequests.values()) {
|
||||
pending.reject(error);
|
||||
}
|
||||
this.pendingRequests.clear();
|
||||
});
|
||||
|
||||
this.workers.push(worker);
|
||||
}
|
||||
`);
|
||||
|
||||
const fragmentShader = this.createShader(this.gl.FRAGMENT_SHADER, `#version 300 es
|
||||
precision highp float;
|
||||
|
||||
uniform sampler2D u_colorTexture;
|
||||
uniform sampler2D u_alphaTexture;
|
||||
in vec2 v_texCoord;
|
||||
out vec4 fragColor;
|
||||
|
||||
void main() {
|
||||
vec3 color = texture(u_colorTexture, v_texCoord).rgb;
|
||||
float alpha = texture(u_alphaTexture, v_texCoord).r;
|
||||
fragColor = vec4(color, alpha);
|
||||
}
|
||||
`);
|
||||
|
||||
const program = this.gl.createProgram();
|
||||
this.gl.attachShader(program, vertexShader);
|
||||
this.gl.attachShader(program, fragmentShader);
|
||||
this.gl.linkProgram(program);
|
||||
|
||||
return program;
|
||||
}
|
||||
|
||||
private createShader(type: number, source: string): WebGLShader {
|
||||
assert(this.gl);
|
||||
|
||||
const shader = this.gl.createShader(type)!;
|
||||
this.gl.shaderSource(shader, source);
|
||||
this.gl.compileShader(shader);
|
||||
return shader;
|
||||
}
|
||||
|
||||
private createVAO(): WebGLVertexArrayObject {
|
||||
assert(this.gl);
|
||||
assert(this.program);
|
||||
|
||||
const vao = this.gl.createVertexArray();
|
||||
this.gl.bindVertexArray(vao);
|
||||
|
||||
const vertices = new Float32Array([
|
||||
-1, -1, 0, 1,
|
||||
1, -1, 1, 1,
|
||||
-1, 1, 0, 0,
|
||||
1, 1, 1, 0,
|
||||
]);
|
||||
|
||||
const buffer = this.gl.createBuffer();
|
||||
this.gl.bindBuffer(this.gl.ARRAY_BUFFER, buffer);
|
||||
this.gl.bufferData(this.gl.ARRAY_BUFFER, vertices, this.gl.STATIC_DRAW);
|
||||
|
||||
const positionLocation = this.gl.getAttribLocation(this.program, 'a_position');
|
||||
const texCoordLocation = this.gl.getAttribLocation(this.program, 'a_texCoord');
|
||||
|
||||
this.gl.enableVertexAttribArray(positionLocation);
|
||||
this.gl.vertexAttribPointer(positionLocation, 2, this.gl.FLOAT, false, 16, 0);
|
||||
|
||||
this.gl.enableVertexAttribArray(texCoordLocation);
|
||||
this.gl.vertexAttribPointer(texCoordLocation, 2, this.gl.FLOAT, false, 16, 8);
|
||||
|
||||
return vao;
|
||||
}
|
||||
|
||||
private createTexture(): WebGLTexture {
|
||||
assert(this.gl);
|
||||
|
||||
const texture = this.gl.createTexture();
|
||||
|
||||
this.gl.bindTexture(this.gl.TEXTURE_2D, texture);
|
||||
this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_S, this.gl.CLAMP_TO_EDGE);
|
||||
this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_T, this.gl.CLAMP_TO_EDGE);
|
||||
this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MIN_FILTER, this.gl.LINEAR);
|
||||
this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MAG_FILTER, this.gl.LINEAR);
|
||||
|
||||
return texture;
|
||||
}
|
||||
|
||||
private updateGpu(color: VideoFrame, alpha: VideoFrame): VideoFrame {
|
||||
assert(this.gl);
|
||||
assert(this.canvas);
|
||||
|
||||
if (color.displayWidth !== this.canvas.width || color.displayHeight !== this.canvas.height) {
|
||||
this.canvas.width = color.displayWidth;
|
||||
this.canvas.height = color.displayHeight;
|
||||
}
|
||||
|
||||
this.gl.activeTexture(this.gl.TEXTURE0);
|
||||
this.gl.bindTexture(this.gl.TEXTURE_2D, this.colorTexture);
|
||||
this.gl.texImage2D(this.gl.TEXTURE_2D, 0, this.gl.RGBA, this.gl.RGBA, this.gl.UNSIGNED_BYTE, color);
|
||||
|
||||
this.gl.activeTexture(this.gl.TEXTURE1);
|
||||
this.gl.bindTexture(this.gl.TEXTURE_2D, this.alphaTexture);
|
||||
this.gl.texImage2D(this.gl.TEXTURE_2D, 0, this.gl.RGBA, this.gl.RGBA, this.gl.UNSIGNED_BYTE, alpha);
|
||||
|
||||
this.gl.viewport(0, 0, this.canvas.width, this.canvas.height);
|
||||
this.gl.clear(this.gl.COLOR_BUFFER_BIT);
|
||||
|
||||
this.gl.bindVertexArray(this.vao);
|
||||
this.gl.drawArrays(this.gl.TRIANGLE_STRIP, 0, 4);
|
||||
|
||||
const finalFrame = new VideoFrame(this.canvas, {
|
||||
timestamp: color.timestamp,
|
||||
duration: color.duration ?? undefined,
|
||||
});
|
||||
|
||||
color.close();
|
||||
alpha.close();
|
||||
|
||||
return finalFrame;
|
||||
}
|
||||
|
||||
private updateCpu(color: VideoFrame, alpha: VideoFrame): Promise<VideoFrame> {
|
||||
if (!this.worker) {
|
||||
const blob = new Blob(
|
||||
[`(${colorAlphaMergerWorkerCode.toString()})()`],
|
||||
{ type: 'application/javascript' },
|
||||
);
|
||||
const url = URL.createObjectURL(blob);
|
||||
this.worker = new Worker(url);
|
||||
URL.revokeObjectURL(url);
|
||||
|
||||
this.worker.addEventListener('message', (event: MessageEvent<ColorAlphaMergerWorkerResponse>) => {
|
||||
const data = event.data;
|
||||
const pending = this.pendingRequests.get(data.id);
|
||||
if (!pending) {
|
||||
return;
|
||||
}
|
||||
this.pendingRequests.delete(data.id);
|
||||
|
||||
if ('error' in data) {
|
||||
pending.reject(new Error(data.error));
|
||||
} else {
|
||||
pending.resolve(data.frame);
|
||||
}
|
||||
});
|
||||
|
||||
this.worker.addEventListener('error', (event) => {
|
||||
const error = new Error(event.message || 'Color/alpha merge worker error.');
|
||||
for (const pending of this.pendingRequests.values()) {
|
||||
pending.reject(error);
|
||||
}
|
||||
this.pendingRequests.clear();
|
||||
});
|
||||
}
|
||||
|
||||
const id = this.nextRequestId++;
|
||||
const pending = promiseWithResolvers<VideoFrame>();
|
||||
this.pendingRequests.set(id, pending);
|
||||
|
||||
this.worker.postMessage({ id, color, alpha }, { transfer: [color, alpha] });
|
||||
// Hand the job to the next worker in round-robin fashion
|
||||
const worker = this.workers[this.nextWorkerIndex]!;
|
||||
this.nextWorkerIndex = (this.nextWorkerIndex + 1) % this.workers.length;
|
||||
worker.postMessage({ id, color, alpha }, { transfer: [color, alpha] });
|
||||
|
||||
return pending.promise;
|
||||
}
|
||||
|
||||
close() {
|
||||
this.gl?.getExtension('WEBGL_lose_context')?.loseContext();
|
||||
this.gl = null;
|
||||
this.canvas = null;
|
||||
|
||||
this.worker?.terminate();
|
||||
this.worker = null;
|
||||
for (const worker of this.workers) {
|
||||
worker.terminate();
|
||||
}
|
||||
this.workers.length = 0;
|
||||
|
||||
const error = new Error('Color/alpha merger closed.');
|
||||
for (const pending of this.pendingRequests.values()) {
|
||||
@@ -1574,19 +1431,16 @@ const colorAlphaMergerWorkerCode = () => {
|
||||
);
|
||||
}
|
||||
|
||||
const width = color.codedWidth;
|
||||
const height = color.codedHeight;
|
||||
|
||||
if (format === 'RGBX' || format === 'RGBA' || format === 'BGRX' || format === 'BGRA') {
|
||||
return await mergeInterleavedRgba(color, alpha, width, height, format);
|
||||
return await mergeInterleavedRgba(color, alpha, format);
|
||||
} else if (
|
||||
format === 'I420' || format === 'I420P10' || format === 'I420P12'
|
||||
|| format === 'I422' || format === 'I422P10' || format === 'I422P12'
|
||||
|| format === 'I444' || format === 'I444P10' || format === 'I444P12'
|
||||
) {
|
||||
return await mergePlanarYuv(color, alpha, width, height, format);
|
||||
return await mergePlanarYuv(color, alpha, format);
|
||||
} else if (format === 'NV12') {
|
||||
return await mergeNv12(color, alpha, width, height);
|
||||
return await mergeNv12(color, alpha);
|
||||
}
|
||||
|
||||
throw new Error(`CPU color/alpha merging does not support format '${format}'.`);
|
||||
@@ -1595,10 +1449,11 @@ const colorAlphaMergerWorkerCode = () => {
|
||||
const mergeInterleavedRgba = async (
|
||||
color: VideoFrame,
|
||||
alpha: VideoFrame,
|
||||
width: number,
|
||||
height: number,
|
||||
format: 'RGBX' | 'RGBA' | 'BGRX' | 'BGRA',
|
||||
): Promise<VideoFrame> => {
|
||||
const width = color.visibleRect?.width ?? color.codedWidth;
|
||||
const height = color.visibleRect?.height ?? color.codedHeight;
|
||||
|
||||
const pixelCount = width * height;
|
||||
const output = new Uint8Array(pixelCount * 4);
|
||||
|
||||
@@ -1627,13 +1482,14 @@ const colorAlphaMergerWorkerCode = () => {
|
||||
const mergePlanarYuv = async (
|
||||
color: VideoFrame,
|
||||
alpha: VideoFrame,
|
||||
width: number,
|
||||
height: number,
|
||||
format:
|
||||
| 'I420' | 'I420P10' | 'I420P12'
|
||||
| 'I422' | 'I422P10' | 'I422P12'
|
||||
| 'I444' | 'I444P10' | 'I444P12',
|
||||
): Promise<VideoFrame> => {
|
||||
const width = color.visibleRect?.width ?? color.codedWidth;
|
||||
const height = color.visibleRect?.height ?? color.codedHeight;
|
||||
|
||||
const is10 = format.includes('P10');
|
||||
const is12 = format.includes('P12');
|
||||
const bytesPerSample = (is10 || is12) ? 2 : 1;
|
||||
@@ -1684,9 +1540,10 @@ const colorAlphaMergerWorkerCode = () => {
|
||||
const mergeNv12 = async (
|
||||
color: VideoFrame,
|
||||
alpha: VideoFrame,
|
||||
width: number,
|
||||
height: number,
|
||||
): Promise<VideoFrame> => {
|
||||
const width = color.visibleRect?.width ?? color.codedWidth;
|
||||
const height = color.visibleRect?.height ?? color.codedHeight;
|
||||
|
||||
const ySize = width * height;
|
||||
const chromaW = Math.ceil(width / 2);
|
||||
const chromaH = Math.ceil(height / 2);
|
||||
@@ -1813,7 +1670,7 @@ export class VideoSampleSink extends BaseMediaSampleSink<VideoSample> {
|
||||
/** @internal */
|
||||
async _createDecoder(
|
||||
onSample: (sample: VideoSample) => unknown,
|
||||
onError: (error: Error) => unknown,
|
||||
onError: (error: unknown) => unknown,
|
||||
) {
|
||||
if (!(await this._track.canDecode())) {
|
||||
throw new Error(
|
||||
@@ -2215,7 +2072,7 @@ class AudioDecoderWrapper extends DecoderWrapper<AudioSample> {
|
||||
|
||||
constructor(
|
||||
onSample: (sample: AudioSample) => unknown,
|
||||
onError: (error: Error) => unknown,
|
||||
onError: (error: unknown) => unknown,
|
||||
codec: AudioCodec,
|
||||
decoderConfig: AudioDecoderConfig,
|
||||
) {
|
||||
@@ -2271,8 +2128,14 @@ class AudioDecoderWrapper extends DecoderWrapper<AudioSample> {
|
||||
|
||||
sampleHandler(sample);
|
||||
};
|
||||
// @ts-expect-error It's technically readonly
|
||||
this.customDecoder.onError = (error) => {
|
||||
onError(error);
|
||||
};
|
||||
|
||||
void this.customDecoderCallSerializer.call(() => this.customDecoder!.init());
|
||||
void this.customDecoderCallSerializer
|
||||
.call(() => this.customDecoder!.init())
|
||||
.catch(error => onError(error));
|
||||
} else {
|
||||
const stack = new Error('Decoding error').stack;
|
||||
|
||||
@@ -2281,7 +2144,7 @@ class AudioDecoderWrapper extends DecoderWrapper<AudioSample> {
|
||||
try {
|
||||
sampleHandler(new AudioSample(data));
|
||||
} catch (error) {
|
||||
this.onError(error as Error);
|
||||
this.onError(error);
|
||||
}
|
||||
},
|
||||
error: (error) => {
|
||||
@@ -2307,7 +2170,8 @@ class AudioDecoderWrapper extends DecoderWrapper<AudioSample> {
|
||||
this.customDecoderQueueSize++;
|
||||
void this.customDecoderCallSerializer
|
||||
.call(() => this.customDecoder!.decode(packet))
|
||||
.then(() => this.customDecoderQueueSize--);
|
||||
.catch(error => this.onError(error))
|
||||
.finally(() => this.customDecoderQueueSize--);
|
||||
} else {
|
||||
assert(this.decoder);
|
||||
|
||||
@@ -2357,7 +2221,7 @@ class PcmAudioDecoderWrapper extends DecoderWrapper<AudioSample> {
|
||||
|
||||
constructor(
|
||||
onSample: (sample: AudioSample) => unknown,
|
||||
onError: (error: Error) => unknown,
|
||||
onError: (error: unknown) => unknown,
|
||||
public decoderConfig: AudioDecoderConfig,
|
||||
) {
|
||||
super(onSample, onError);
|
||||
@@ -2547,7 +2411,7 @@ export class AudioSampleSink extends BaseMediaSampleSink<AudioSample> {
|
||||
/** @internal */
|
||||
async _createDecoder(
|
||||
onSample: (sample: AudioSample) => unknown,
|
||||
onError: (error: Error) => unknown,
|
||||
onError: (error: unknown) => unknown,
|
||||
) {
|
||||
if (!(await this._track.canDecode())) {
|
||||
throw new Error(
|
||||
|
||||
+84
-352
@@ -36,6 +36,7 @@ import {
|
||||
setUint24,
|
||||
toUint8Array,
|
||||
UnthrottledTimerHandle,
|
||||
wait,
|
||||
} from './misc';
|
||||
import { Muxer } from './muxer';
|
||||
import { SubtitleParser } from './subtitles';
|
||||
@@ -265,10 +266,18 @@ class VideoEncoderWrapper {
|
||||
* However, we want to surface these errors to the user within the normal control flow, so they don't go uncaught.
|
||||
* So, we keep track of the encoder error and throw it as soon as we get the chance.
|
||||
*/
|
||||
private error: Error | null = null;
|
||||
private closed = false;
|
||||
private error: unknown = null;
|
||||
private errorSet = false;
|
||||
|
||||
private setError(error: unknown) {
|
||||
if (!this.errorSet) {
|
||||
this.error = error;
|
||||
this.errorSet = true;
|
||||
}
|
||||
}
|
||||
|
||||
private lastMuxerPromise: Promise<void> = Promise.resolve();
|
||||
private closed = false;
|
||||
|
||||
constructor(private source: VideoSource, private encodingConfig: VideoEncodingConfig) {}
|
||||
|
||||
@@ -492,9 +501,9 @@ class VideoEncoderWrapper {
|
||||
|
||||
const promise = this.customEncoderCallSerializer
|
||||
.call(() => this.customEncoder!.encode(clonedSample, finalEncodeOptions))
|
||||
.then(() => this.customEncoderQueueSize--)
|
||||
.catch((error: Error) => this.error ??= error)
|
||||
.catch((error: unknown) => this.setError(error))
|
||||
.finally(() => {
|
||||
this.customEncoderQueueSize--;
|
||||
clonedSample.close();
|
||||
});
|
||||
|
||||
@@ -551,15 +560,12 @@ class VideoEncoderWrapper {
|
||||
this.encoder.encode(videoFrame, finalEncodeOptions);
|
||||
videoFrame.close();
|
||||
} else {
|
||||
const width = videoFrame.displayWidth;
|
||||
const height = videoFrame.displayHeight;
|
||||
|
||||
if (!this.splitter) {
|
||||
this.splitter = new ColorAlphaSplitter(width, height);
|
||||
this.splitter = new ColorAlphaSplitter();
|
||||
}
|
||||
|
||||
// The splitter takes ownership, so no need to close the frames ourselves
|
||||
const { colorFrame, alphaFrame } = await this.splitter.update(videoFrame);
|
||||
const { colorFrame, alphaFrame } = await this.splitter.split(videoFrame);
|
||||
|
||||
this.alphaFrameQueue.push(alphaFrame);
|
||||
this.encoder.encode(colorFrame, finalEncodeOptions);
|
||||
@@ -641,9 +647,13 @@ class VideoEncoderWrapper {
|
||||
this.lastMuxerPromise
|
||||
= this.muxer!.addEncodedVideoPacket(this.source._connectedTrack!, packet, meta)
|
||||
.catch((error) => {
|
||||
this.error ??= error;
|
||||
this.setError(error);
|
||||
});
|
||||
};
|
||||
// @ts-expect-error It's technically readonly
|
||||
this.customEncoder.onError = (error) => {
|
||||
this.setError(error);
|
||||
};
|
||||
|
||||
await this.customEncoder.init();
|
||||
} else {
|
||||
@@ -746,7 +756,7 @@ class VideoEncoderWrapper {
|
||||
this.lastMuxerPromise
|
||||
= this.muxer!.addEncodedVideoPacket(this.source._connectedTrack!, packet, meta)
|
||||
.catch((error) => {
|
||||
this.error ??= error;
|
||||
this.setError(error);
|
||||
});
|
||||
|
||||
this.emittedEncoderPackets++;
|
||||
@@ -791,7 +801,7 @@ class VideoEncoderWrapper {
|
||||
},
|
||||
error: (error) => {
|
||||
error.stack = stack; // Provide a more useful stack trace, the default one sucks
|
||||
this.error ??= error;
|
||||
this.setError(error);
|
||||
},
|
||||
});
|
||||
this.encoder.configure(encoderConfig);
|
||||
@@ -827,7 +837,7 @@ class VideoEncoderWrapper {
|
||||
},
|
||||
error: (error) => {
|
||||
error.stack = stack; // Provide a more useful stack trace
|
||||
this.error ??= error;
|
||||
this.setError(error);
|
||||
},
|
||||
});
|
||||
this.alphaEncoder.configure(encoderConfig);
|
||||
@@ -869,6 +879,9 @@ class VideoEncoderWrapper {
|
||||
// These are wired in series, therefore they must also be flushed in series
|
||||
await this.encoder.flush();
|
||||
await this.alphaEncoder?.flush();
|
||||
|
||||
// Workaround for https://issues.chromium.org/issues/529852980 to give it time for errors to surface
|
||||
await wait(25);
|
||||
}
|
||||
|
||||
if (this.encoder.state !== 'closed') {
|
||||
@@ -899,27 +912,16 @@ class VideoEncoderWrapper {
|
||||
}
|
||||
|
||||
checkForEncoderError() {
|
||||
if (this.error) {
|
||||
if (this.errorSet) {
|
||||
throw this.error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let splitterGpuUnavailable = false;
|
||||
let splitterWorkerUrl: string | null = null;
|
||||
|
||||
/** Utility class for splitting a composite frame into separate color and alpha components. */
|
||||
/** Utility class for splitting a composite frame into separate color and alpha parts on the CPU in a worker. */
|
||||
export class ColorAlphaSplitter {
|
||||
static forceCpu = true;
|
||||
|
||||
canvas: OffscreenCanvas | HTMLCanvasElement | null = null;
|
||||
|
||||
private gl: WebGL2RenderingContext | null = null;
|
||||
private colorProgram: WebGLProgram | null = null;
|
||||
private alphaProgram: WebGLProgram | null = null;
|
||||
private vao: WebGLVertexArrayObject | null = null;
|
||||
private sourceTexture: WebGLTexture | null = null;
|
||||
private alphaResolutionLocation: WebGLUniformLocation | null = null;
|
||||
|
||||
private worker: Worker | null = null;
|
||||
private pendingRequests = new Map<
|
||||
number,
|
||||
@@ -928,308 +930,17 @@ export class ColorAlphaSplitter {
|
||||
|
||||
private nextRequestId = 0;
|
||||
|
||||
constructor(initialWidth: number, initialHeight: number) {
|
||||
const canMakeCanvas = typeof OffscreenCanvas !== 'undefined'
|
||||
// eslint-disable-next-line @typescript-eslint/no-deprecated
|
||||
|| (typeof document !== 'undefined' && typeof document.createElement === 'function');
|
||||
|
||||
if (!ColorAlphaSplitter.forceCpu && canMakeCanvas && !splitterGpuUnavailable) {
|
||||
// Try the GPU path. If anything goes wrong, we silently fall back to the CPU path.
|
||||
try {
|
||||
if (typeof OffscreenCanvas !== 'undefined') {
|
||||
this.canvas = new OffscreenCanvas(initialWidth, initialHeight);
|
||||
} else {
|
||||
this.canvas = document.createElement('canvas');
|
||||
this.canvas.width = initialWidth;
|
||||
this.canvas.height = initialHeight;
|
||||
}
|
||||
|
||||
const gl = this.canvas.getContext('webgl2', {
|
||||
alpha: true, // Needed due to the YUV thing we do for alpha
|
||||
}) as unknown as WebGL2RenderingContext | null; // Casting because of some TypeScript weirdness
|
||||
if (!gl) {
|
||||
throw new Error('Couldn\'t acquire WebGL 2 context.');
|
||||
}
|
||||
|
||||
this.gl = gl;
|
||||
|
||||
this.colorProgram = this.createColorProgram();
|
||||
this.alphaProgram = this.createAlphaProgram();
|
||||
this.vao = this.createVAO();
|
||||
this.sourceTexture = this.createTexture();
|
||||
|
||||
this.alphaResolutionLocation = this.gl.getUniformLocation(this.alphaProgram, 'u_resolution')!;
|
||||
|
||||
this.gl.useProgram(this.colorProgram);
|
||||
this.gl.uniform1i(this.gl.getUniformLocation(this.colorProgram, 'u_sourceTexture'), 0);
|
||||
|
||||
this.gl.useProgram(this.alphaProgram);
|
||||
this.gl.uniform1i(this.gl.getUniformLocation(this.alphaProgram, 'u_sourceTexture'), 0);
|
||||
} catch (error) {
|
||||
this.gl = null;
|
||||
this.canvas = null;
|
||||
splitterGpuUnavailable = true;
|
||||
Logging._warn('Falling back to CPU for color/alpha splitting.', error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async update(sourceFrame: VideoFrame) {
|
||||
if (this.gl) {
|
||||
return this.updateGpu(sourceFrame);
|
||||
} else {
|
||||
return this.updateCpu(sourceFrame);
|
||||
}
|
||||
}
|
||||
|
||||
private updateGpu(sourceFrame: VideoFrame) {
|
||||
assert(this.gl);
|
||||
assert(this.canvas);
|
||||
|
||||
if (sourceFrame.displayWidth !== this.canvas.width || sourceFrame.displayHeight !== this.canvas.height) {
|
||||
this.canvas.width = sourceFrame.displayWidth;
|
||||
this.canvas.height = sourceFrame.displayHeight;
|
||||
}
|
||||
|
||||
this.gl.activeTexture(this.gl.TEXTURE0);
|
||||
this.gl.bindTexture(this.gl.TEXTURE_2D, this.sourceTexture);
|
||||
this.gl.texImage2D(this.gl.TEXTURE_2D, 0, this.gl.RGBA, this.gl.RGBA, this.gl.UNSIGNED_BYTE, sourceFrame);
|
||||
|
||||
const colorFrame = this.runColorProgram(sourceFrame);
|
||||
const alphaFrame = this.runAlphaProgram(sourceFrame);
|
||||
|
||||
sourceFrame.close();
|
||||
|
||||
return { colorFrame, alphaFrame };
|
||||
}
|
||||
|
||||
private createVertexShader(): WebGLShader {
|
||||
assert(this.gl);
|
||||
|
||||
return this.createShader(this.gl.VERTEX_SHADER, `#version 300 es
|
||||
in vec2 a_position;
|
||||
in vec2 a_texCoord;
|
||||
out vec2 v_texCoord;
|
||||
|
||||
void main() {
|
||||
gl_Position = vec4(a_position, 0.0, 1.0);
|
||||
v_texCoord = a_texCoord;
|
||||
}
|
||||
`);
|
||||
}
|
||||
|
||||
private createColorProgram(): WebGLProgram {
|
||||
assert(this.gl);
|
||||
|
||||
const vertexShader = this.createVertexShader();
|
||||
|
||||
// This shader is simple, simply copy the color information while setting alpha to 1
|
||||
const fragmentShader = this.createShader(this.gl.FRAGMENT_SHADER, `#version 300 es
|
||||
precision highp float;
|
||||
|
||||
uniform sampler2D u_sourceTexture;
|
||||
in vec2 v_texCoord;
|
||||
out vec4 fragColor;
|
||||
|
||||
void main() {
|
||||
vec4 source = texture(u_sourceTexture, v_texCoord);
|
||||
fragColor = vec4(source.rgb, 1.0);
|
||||
}
|
||||
`);
|
||||
|
||||
const program = this.gl.createProgram();
|
||||
this.gl.attachShader(program, vertexShader);
|
||||
this.gl.attachShader(program, fragmentShader);
|
||||
this.gl.linkProgram(program);
|
||||
|
||||
return program;
|
||||
}
|
||||
|
||||
private createAlphaProgram(): WebGLProgram {
|
||||
assert(this.gl);
|
||||
|
||||
const vertexShader = this.createVertexShader();
|
||||
|
||||
// This shader's more complex. The main reason is that this shader writes data in I420 (yuv420) pixel format
|
||||
// instead of regular RGBA. In other words, we use the shader to write out I420 data into an RGBA canvas, which
|
||||
// we then later read out with JavaScript. The reason being that browsers weirdly encode canvases and mess up
|
||||
// the color spaces, and the only way to have full control over the color space is by outputting YUV data
|
||||
// directly (avoiding the RGB conversion). Doing this conversion in JS is painfully slow, so let's utlize the
|
||||
// GPU since we're already calling it anyway.
|
||||
const fragmentShader = this.createShader(this.gl.FRAGMENT_SHADER, `#version 300 es
|
||||
precision highp float;
|
||||
|
||||
uniform sampler2D u_sourceTexture;
|
||||
uniform vec2 u_resolution; // The width and height of the canvas
|
||||
in vec2 v_texCoord;
|
||||
out vec4 fragColor;
|
||||
|
||||
// This function determines the value for a single byte in the YUV stream
|
||||
float getByteValue(float byteOffset) {
|
||||
float width = u_resolution.x;
|
||||
float height = u_resolution.y;
|
||||
|
||||
float yPlaneSize = width * height;
|
||||
|
||||
if (byteOffset < yPlaneSize) {
|
||||
// This byte is in the luma plane. Find the corresponding pixel coordinates to sample from
|
||||
float y = floor(byteOffset / width);
|
||||
float x = mod(byteOffset, width);
|
||||
|
||||
// Add 0.5 to sample the center of the texel
|
||||
vec2 sampleCoord = (vec2(x, y) + 0.5) / u_resolution;
|
||||
|
||||
// The luma value is the alpha from the source texture
|
||||
return texture(u_sourceTexture, sampleCoord).a;
|
||||
} else {
|
||||
// Write a fixed value for chroma and beyond
|
||||
return 128.0 / 255.0;
|
||||
}
|
||||
}
|
||||
|
||||
void main() {
|
||||
// Each fragment writes 4 bytes (R, G, B, A)
|
||||
float pixelIndex = floor(gl_FragCoord.y) * u_resolution.x + floor(gl_FragCoord.x);
|
||||
float baseByteOffset = pixelIndex * 4.0;
|
||||
|
||||
vec4 result;
|
||||
for (int i = 0; i < 4; i++) {
|
||||
float currentByteOffset = baseByteOffset + float(i);
|
||||
result[i] = getByteValue(currentByteOffset);
|
||||
}
|
||||
|
||||
fragColor = result;
|
||||
}
|
||||
`);
|
||||
|
||||
const program = this.gl.createProgram();
|
||||
this.gl.attachShader(program, vertexShader);
|
||||
this.gl.attachShader(program, fragmentShader);
|
||||
this.gl.linkProgram(program);
|
||||
|
||||
return program;
|
||||
}
|
||||
|
||||
private createShader(type: number, source: string): WebGLShader {
|
||||
assert(this.gl);
|
||||
|
||||
const shader = this.gl.createShader(type)!;
|
||||
this.gl.shaderSource(shader, source);
|
||||
this.gl.compileShader(shader);
|
||||
if (!this.gl.getShaderParameter(shader, this.gl.COMPILE_STATUS)) {
|
||||
Logging._error('Shader compile error:', this.gl.getShaderInfoLog(shader));
|
||||
}
|
||||
return shader;
|
||||
}
|
||||
|
||||
private createVAO(): WebGLVertexArrayObject {
|
||||
assert(this.gl);
|
||||
assert(this.colorProgram);
|
||||
|
||||
const vao = this.gl.createVertexArray();
|
||||
this.gl.bindVertexArray(vao);
|
||||
|
||||
const vertices = new Float32Array([
|
||||
-1, -1, 0, 1,
|
||||
1, -1, 1, 1,
|
||||
-1, 1, 0, 0,
|
||||
1, 1, 1, 0,
|
||||
]);
|
||||
|
||||
const buffer = this.gl.createBuffer();
|
||||
this.gl.bindBuffer(this.gl.ARRAY_BUFFER, buffer);
|
||||
this.gl.bufferData(this.gl.ARRAY_BUFFER, vertices, this.gl.STATIC_DRAW);
|
||||
|
||||
const positionLocation = this.gl.getAttribLocation(this.colorProgram, 'a_position');
|
||||
const texCoordLocation = this.gl.getAttribLocation(this.colorProgram, 'a_texCoord');
|
||||
|
||||
this.gl.enableVertexAttribArray(positionLocation);
|
||||
this.gl.vertexAttribPointer(positionLocation, 2, this.gl.FLOAT, false, 16, 0);
|
||||
|
||||
this.gl.enableVertexAttribArray(texCoordLocation);
|
||||
this.gl.vertexAttribPointer(texCoordLocation, 2, this.gl.FLOAT, false, 16, 8);
|
||||
|
||||
return vao;
|
||||
}
|
||||
|
||||
private createTexture(): WebGLTexture {
|
||||
assert(this.gl);
|
||||
|
||||
const texture = this.gl.createTexture();
|
||||
|
||||
this.gl.bindTexture(this.gl.TEXTURE_2D, texture);
|
||||
this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_S, this.gl.CLAMP_TO_EDGE);
|
||||
this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_T, this.gl.CLAMP_TO_EDGE);
|
||||
this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MIN_FILTER, this.gl.LINEAR);
|
||||
this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MAG_FILTER, this.gl.LINEAR);
|
||||
|
||||
return texture;
|
||||
}
|
||||
|
||||
private runColorProgram(sourceFrame: VideoFrame) {
|
||||
assert(this.gl);
|
||||
assert(this.canvas);
|
||||
|
||||
this.gl.useProgram(this.colorProgram);
|
||||
this.gl.viewport(0, 0, this.canvas.width, this.canvas.height);
|
||||
this.gl.clear(this.gl.COLOR_BUFFER_BIT);
|
||||
this.gl.bindVertexArray(this.vao);
|
||||
this.gl.drawArrays(this.gl.TRIANGLE_STRIP, 0, 4);
|
||||
|
||||
return new VideoFrame(this.canvas, {
|
||||
timestamp: sourceFrame.timestamp,
|
||||
duration: sourceFrame.duration ?? undefined,
|
||||
alpha: 'discard',
|
||||
});
|
||||
}
|
||||
|
||||
private runAlphaProgram(sourceFrame: VideoFrame) {
|
||||
assert(this.gl);
|
||||
assert(this.canvas);
|
||||
|
||||
this.gl.useProgram(this.alphaProgram);
|
||||
this.gl.uniform2f(this.alphaResolutionLocation, this.canvas.width, this.canvas.height);
|
||||
|
||||
this.gl.viewport(0, 0, this.canvas.width, this.canvas.height);
|
||||
this.gl.clear(this.gl.COLOR_BUFFER_BIT);
|
||||
this.gl.bindVertexArray(this.vao);
|
||||
this.gl.drawArrays(this.gl.TRIANGLE_STRIP, 0, 4);
|
||||
|
||||
const { width, height } = this.canvas;
|
||||
|
||||
const chromaSamples = Math.ceil(width / 2) * Math.ceil(height / 2);
|
||||
const yuvSize = width * height + chromaSamples * 2;
|
||||
const requiredHeight = Math.ceil(yuvSize / (width * 4));
|
||||
|
||||
let yuv = new Uint8Array(4 * width * requiredHeight);
|
||||
this.gl.readPixels(0, 0, width, requiredHeight, this.gl.RGBA, this.gl.UNSIGNED_BYTE, yuv);
|
||||
yuv = yuv.subarray(0, yuvSize);
|
||||
|
||||
assert(yuv[width * height] === 128); // Where chroma data starts
|
||||
assert(yuv[yuv.length - 1] === 128); // Assert the YUV data has been fully written
|
||||
|
||||
// Defining this separately because TypeScript doesn't know `transfer` and I can't be bothered to do declaration
|
||||
// merging right now
|
||||
const init = {
|
||||
format: 'I420' as const,
|
||||
codedWidth: width,
|
||||
codedHeight: height,
|
||||
timestamp: sourceFrame.timestamp,
|
||||
duration: sourceFrame.duration ?? undefined,
|
||||
transfer: [yuv.buffer],
|
||||
};
|
||||
return new VideoFrame(yuv, init);
|
||||
}
|
||||
|
||||
private updateCpu(sourceFrame: VideoFrame): Promise<{ colorFrame: VideoFrame; alphaFrame: VideoFrame }> {
|
||||
split(sourceFrame: VideoFrame): Promise<{ colorFrame: VideoFrame; alphaFrame: VideoFrame }> {
|
||||
if (!this.worker) {
|
||||
const blob = new Blob(
|
||||
[`(${colorAlphaSplitterWorkerCode.toString()})()`],
|
||||
{ type: 'application/javascript' },
|
||||
);
|
||||
const url = URL.createObjectURL(blob);
|
||||
this.worker = new Worker(url);
|
||||
URL.revokeObjectURL(url);
|
||||
if (!splitterWorkerUrl) {
|
||||
const blob = new Blob(
|
||||
[`(${colorAlphaSplitterWorkerCode.toString()})()`],
|
||||
{ type: 'application/javascript' },
|
||||
);
|
||||
splitterWorkerUrl = URL.createObjectURL(blob);
|
||||
}
|
||||
|
||||
this.worker = new Worker(splitterWorkerUrl);
|
||||
|
||||
this.worker.addEventListener('message', (event: MessageEvent<ColorAlphaSplitterWorkerResponse>) => {
|
||||
const data = event.data;
|
||||
@@ -1259,16 +970,14 @@ export class ColorAlphaSplitter {
|
||||
const pending = promiseWithResolvers<{ colorFrame: VideoFrame; alphaFrame: VideoFrame }>();
|
||||
this.pendingRequests.set(id, pending);
|
||||
this.worker.postMessage({ id, sourceFrame }, { transfer: [sourceFrame] });
|
||||
|
||||
return pending.promise;
|
||||
}
|
||||
|
||||
close() {
|
||||
this.gl?.getExtension('WEBGL_lose_context')?.loseContext();
|
||||
this.gl = null;
|
||||
this.canvas = null;
|
||||
|
||||
this.worker?.terminate();
|
||||
this.worker = null;
|
||||
|
||||
const error = new Error('Color/alpha splitter closed.');
|
||||
for (const pending of this.pendingRequests.values()) {
|
||||
pending.reject(error);
|
||||
@@ -1312,8 +1021,6 @@ const colorAlphaSplitterWorkerCode = () => {
|
||||
throw new Error('CPU color/alpha splitting requires a known VideoFrame format.');
|
||||
}
|
||||
|
||||
const width = sourceFrame.codedWidth;
|
||||
const height = sourceFrame.codedHeight;
|
||||
const sourceSize = sourceFrame.allocationSize();
|
||||
|
||||
if (!cpuSourceBuffer || cpuSourceBuffer.byteLength !== sourceSize) {
|
||||
@@ -1322,13 +1029,13 @@ const colorAlphaSplitterWorkerCode = () => {
|
||||
await sourceFrame.copyTo(cpuSourceBuffer);
|
||||
|
||||
if (format === 'RGBA' || format === 'BGRA') {
|
||||
return splitInterleavedRgba(cpuSourceBuffer, width, height, format, sourceFrame);
|
||||
return splitInterleavedRgba(cpuSourceBuffer, format, sourceFrame);
|
||||
} else if (
|
||||
format === 'I420A' || format === 'I420AP10' || format === 'I420AP12'
|
||||
|| format === 'I422A' || format === 'I422AP10' || format === 'I422AP12'
|
||||
|| format === 'I444A' || format === 'I444AP10' || format === 'I444AP12'
|
||||
) {
|
||||
return splitPlanarYuvA(cpuSourceBuffer, width, height, format, sourceFrame);
|
||||
return splitPlanarYuvA(cpuSourceBuffer, format, sourceFrame);
|
||||
}
|
||||
|
||||
throw new Error(`CPU color/alpha splitting does not support format '${format}'.`);
|
||||
@@ -1336,11 +1043,12 @@ const colorAlphaSplitterWorkerCode = () => {
|
||||
|
||||
const splitInterleavedRgba = (
|
||||
source: Uint8Array,
|
||||
width: number,
|
||||
height: number,
|
||||
format: 'RGBA' | 'BGRA',
|
||||
sourceFrame: VideoFrame,
|
||||
) => {
|
||||
const width = sourceFrame.visibleRect?.width ?? sourceFrame.codedWidth;
|
||||
const height = sourceFrame.visibleRect?.height ?? sourceFrame.codedHeight;
|
||||
|
||||
const pixelCount = width * height;
|
||||
const chromaW = Math.ceil(width / 2);
|
||||
const chromaH = Math.ceil(height / 2);
|
||||
@@ -1376,14 +1084,15 @@ const colorAlphaSplitterWorkerCode = () => {
|
||||
|
||||
const splitPlanarYuvA = (
|
||||
source: Uint8Array,
|
||||
width: number,
|
||||
height: number,
|
||||
format:
|
||||
| 'I420A' | 'I420AP10' | 'I420AP12'
|
||||
| 'I422A' | 'I422AP10' | 'I422AP12'
|
||||
| 'I444A' | 'I444AP10' | 'I444AP12',
|
||||
sourceFrame: VideoFrame,
|
||||
) => {
|
||||
const width = sourceFrame.visibleRect?.width ?? sourceFrame.codedWidth;
|
||||
const height = sourceFrame.visibleRect?.height ?? sourceFrame.codedHeight;
|
||||
|
||||
const is10 = format.includes('P10');
|
||||
const is12 = format.includes('P12');
|
||||
const bytesPerSample = (is10 || is12) ? 2 : 1;
|
||||
@@ -2043,7 +1752,16 @@ class AudioEncoderWrapper {
|
||||
* However, we want to surface these errors to the user within the normal control flow, so they don't go uncaught.
|
||||
* So, we keep track of the encoder error and throw it as soon as we get the chance.
|
||||
*/
|
||||
private error: Error | null = null;
|
||||
private error: unknown = null;
|
||||
private errorSet = false;
|
||||
|
||||
private setError(error: unknown) {
|
||||
if (!this.errorSet) {
|
||||
this.error = error;
|
||||
this.errorSet = true;
|
||||
}
|
||||
}
|
||||
|
||||
private lastMuxerPromise: Promise<void> = Promise.resolve();
|
||||
private closed = false;
|
||||
|
||||
@@ -2222,9 +1940,9 @@ class AudioEncoderWrapper {
|
||||
|
||||
const promise = this.customEncoderCallSerializer
|
||||
.call(() => this.customEncoder!.encode(clonedSample))
|
||||
.then(() => this.customEncoderQueueSize--)
|
||||
.catch((error: Error) => this.error ??= error)
|
||||
.catch((error: unknown) => this.setError(error))
|
||||
.finally(() => {
|
||||
this.customEncoderQueueSize--;
|
||||
clonedSample.close();
|
||||
});
|
||||
|
||||
@@ -2365,9 +2083,13 @@ class AudioEncoderWrapper {
|
||||
this.lastMuxerPromise
|
||||
= this.muxer!.addEncodedAudioPacket(this.source._connectedTrack!, packet, meta)
|
||||
.catch((error) => {
|
||||
this.error ??= error;
|
||||
this.setError(error);
|
||||
});
|
||||
};
|
||||
// @ts-expect-error It's technically readonly
|
||||
this.customEncoder.onError = (error) => {
|
||||
this.setError(error);
|
||||
};
|
||||
|
||||
await this.customEncoder.init();
|
||||
} else if ((PCM_AUDIO_CODECS as readonly string[]).includes(this.encodingConfig.codec)) {
|
||||
@@ -2431,12 +2153,12 @@ class AudioEncoderWrapper {
|
||||
this.lastMuxerPromise
|
||||
= this.muxer!.addEncodedAudioPacket(this.source._connectedTrack!, packet, meta)
|
||||
.catch((error) => {
|
||||
this.error ??= error;
|
||||
this.setError(error);
|
||||
});
|
||||
},
|
||||
error: (error) => {
|
||||
error.stack = stack; // Provide a more useful stack trace
|
||||
this.error ??= error;
|
||||
this.setError(error);
|
||||
},
|
||||
});
|
||||
this.encoder.configure(encoderConfig);
|
||||
@@ -2587,7 +2309,7 @@ class AudioEncoderWrapper {
|
||||
}
|
||||
|
||||
checkForEncoderError() {
|
||||
if (this.error) {
|
||||
if (this.errorSet) {
|
||||
throw this.error;
|
||||
}
|
||||
}
|
||||
@@ -2959,7 +2681,7 @@ type MediaStreamTrackProcessorWorkerMessage = {
|
||||
} | {
|
||||
type: 'error';
|
||||
trackId: number;
|
||||
error: Error;
|
||||
error: unknown;
|
||||
};
|
||||
|
||||
type MediaStreamTrackProcessorControllerMessage = {
|
||||
@@ -3018,7 +2740,7 @@ const mediaStreamTrackProcessorWorkerCode = () => {
|
||||
|
||||
processor.readable.pipeTo(consumer, {
|
||||
signal: abortController.signal,
|
||||
}).catch((error: Error) => {
|
||||
}).catch((error: unknown) => {
|
||||
// Handle AbortError silently
|
||||
if (error instanceof DOMException && error.name === 'AbortError') return;
|
||||
|
||||
@@ -3138,7 +2860,9 @@ export class TextSubtitleSource extends SubtitleSource {
|
||||
/** @internal */
|
||||
private _parser: SubtitleParser;
|
||||
/** @internal */
|
||||
private _error: Error | null = null;
|
||||
private _error: unknown = null;
|
||||
/** @internal */
|
||||
private _errorSet = false;
|
||||
/** @internal */
|
||||
private _lastMuxerPromise: Promise<void> = Promise.resolve();
|
||||
|
||||
@@ -3152,7 +2876,7 @@ export class TextSubtitleSource extends SubtitleSource {
|
||||
this._lastMuxerPromise
|
||||
= this._connectedTrack!.output._muxer.addSubtitleCue(this._connectedTrack!, cue, metadata)
|
||||
.catch((error) => {
|
||||
this._error ??= error;
|
||||
this._setError(error);
|
||||
});
|
||||
},
|
||||
});
|
||||
@@ -3178,9 +2902,17 @@ export class TextSubtitleSource extends SubtitleSource {
|
||||
return this._lastMuxerPromise; // Allow the writer to apply backpressure
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
private _setError(error: unknown) {
|
||||
if (!this._errorSet) {
|
||||
this._error = error;
|
||||
this._errorSet = true;
|
||||
}
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
_checkForError() {
|
||||
if (this._error) {
|
||||
if (this._errorSet) {
|
||||
throw this._error;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -589,6 +589,7 @@ export class MpegTsDemuxer extends Demuxer {
|
||||
hevcCodecInfo: elementaryStream.info.hevcCodecInfo,
|
||||
vp9CodecInfo: null,
|
||||
av1CodecInfo: null,
|
||||
proresFormat: null,
|
||||
}),
|
||||
codedWidth: elementaryStream.info.width,
|
||||
codedHeight: elementaryStream.info.height,
|
||||
|
||||
+21
-10
@@ -423,16 +423,13 @@ export class VideoSample implements Disposable {
|
||||
);
|
||||
}
|
||||
|
||||
this._data = init._doNotCopy
|
||||
? toUint8Array(data)
|
||||
: toUint8Array(data).slice(); // Copy it
|
||||
this._layout = init.layout ?? createDefaultPlaneLayout(init.format, init.codedWidth!, init.codedHeight!);
|
||||
|
||||
this.format = init.format;
|
||||
this.rotation = init.rotation ?? 0;
|
||||
this.timestamp = init.timestamp!;
|
||||
this.duration = init.duration ?? 0;
|
||||
|
||||
const layout = init.layout ?? createDefaultPlaneLayout(init.format, init.codedWidth!, init.codedHeight!);
|
||||
|
||||
let colorSpaceInit = init.colorSpace ?? null;
|
||||
if (colorSpaceInit === null) {
|
||||
if (
|
||||
@@ -457,8 +454,6 @@ export class VideoSample implements Disposable {
|
||||
}
|
||||
}
|
||||
|
||||
this.colorSpace = new VideoSampleColorSpace(colorSpaceInit);
|
||||
|
||||
this.visibleRect = {
|
||||
left: init.visibleRect?.left ?? 0,
|
||||
top: init.visibleRect?.top ?? 0,
|
||||
@@ -473,6 +468,17 @@ export class VideoSample implements Disposable {
|
||||
this.squarePixelWidth = this.visibleRect.width;
|
||||
this.squarePixelHeight = this.visibleRect.height;
|
||||
}
|
||||
|
||||
// As an optimization, one could check if VideoFrame is defined and if it is, create a VideoFrame here from
|
||||
// the data. Since VideoFrames are typically needed anyway, doing it this way would avoid an additional
|
||||
// copy of the frame data. But due to https://issues.chromium.org/issues/529413114, this is currently
|
||||
// not done.
|
||||
|
||||
this._data = init._doNotCopy
|
||||
? toUint8Array(data)
|
||||
: toUint8Array(data).slice(); // Copy it
|
||||
this._layout = layout;
|
||||
this.colorSpace = new VideoSampleColorSpace(colorSpaceInit);
|
||||
} else if (typeof VideoFrame !== 'undefined' && data instanceof VideoFrame) {
|
||||
if (init?.rotation !== undefined && ![0, 90, 180, 270].includes(init.rotation)) {
|
||||
throw new TypeError('init.rotation, when provided, must be 0, 90, 180, or 270.');
|
||||
@@ -1036,6 +1042,7 @@ export class VideoSample implements Disposable {
|
||||
timestamp: this.microsecondTimestamp,
|
||||
duration: this.microsecondDuration,
|
||||
colorSpace: this.colorSpace,
|
||||
visibleRect: this.visibleRect,
|
||||
displayWidth: this.squarePixelWidth, // Not display* since we're not passing rotation
|
||||
displayHeight: this.squarePixelHeight,
|
||||
});
|
||||
@@ -1045,13 +1052,17 @@ export class VideoSample implements Disposable {
|
||||
duration: this.microsecondDuration || undefined, // Drag 0 duration to undefined, glitches some codecs
|
||||
});
|
||||
} else if (this._data instanceof Uint8Array) {
|
||||
assert(this._layout);
|
||||
|
||||
return new VideoFrame(this._data, {
|
||||
format: this.format! as VideoPixelFormat,
|
||||
codedWidth: this.codedWidth,
|
||||
codedHeight: this.codedHeight,
|
||||
codedWidth: this.codedWidth, // This is technically wrong! codedWidth is a lie technically. But, since
|
||||
codedHeight: this.codedHeight, // we pass the layout (which contains the true coded width), we're good.
|
||||
layout: this._layout,
|
||||
timestamp: this.microsecondTimestamp,
|
||||
duration: this.microsecondDuration || undefined,
|
||||
colorSpace: this.colorSpace,
|
||||
visibleRect: this.visibleRect,
|
||||
displayWidth: this.squarePixelWidth, // Not display* since we're not passing rotation
|
||||
displayHeight: this.squarePixelHeight,
|
||||
});
|
||||
@@ -3006,7 +3017,7 @@ export class AudioSample implements Disposable {
|
||||
}
|
||||
}
|
||||
|
||||
const getBytesPerSample = (format: AudioSampleFormat): number => {
|
||||
export const getBytesPerSample = (format: AudioSampleFormat): number => {
|
||||
switch (format) {
|
||||
case 'u8':
|
||||
case 'u8-planar':
|
||||
|
||||
@@ -2,11 +2,11 @@ import { expect, test } from 'vitest';
|
||||
import { Input } from '../../src/input.js';
|
||||
import { BufferSource, UrlSource } from '../../src/source.js';
|
||||
import { ALL_FORMATS } from '../../src/input-format.js';
|
||||
import { CanvasSink, ColorAlphaMerger, EncodedPacketSink, VideoSampleSink } from '../../src/media-sink.js';
|
||||
import { CanvasSink, EncodedPacketSink, VideoSampleSink } from '../../src/media-sink.js';
|
||||
import { Output } from '../../src/output.js';
|
||||
import { WebMOutputFormat } from '../../src/output-format.js';
|
||||
import { BufferTarget } from '../../src/target.js';
|
||||
import { CanvasSource, ColorAlphaSplitter, VideoSampleSource } from '../../src/media-source.js';
|
||||
import { CanvasSource, VideoSampleSource } from '../../src/media-source.js';
|
||||
import { canEncodeVideo, QUALITY_HIGH } from '../../src/encode.js';
|
||||
import { VideoSample } from '../../src/sample.js';
|
||||
import { Conversion } from '../../src/conversion.js';
|
||||
@@ -39,15 +39,6 @@ test('Can decode transparent video', async () => {
|
||||
await decodeTransparentVideoTest();
|
||||
});
|
||||
|
||||
test('Can decode transparent video, forced CPU path', async () => {
|
||||
try {
|
||||
ColorAlphaMerger.forceCpu = true;
|
||||
await decodeTransparentVideoTest();
|
||||
} finally {
|
||||
ColorAlphaMerger.forceCpu = false;
|
||||
}
|
||||
});
|
||||
|
||||
test('Can decode faulty transparent video and behaves gracefully', async () => {
|
||||
using input = new Input({
|
||||
source: new UrlSource('/transparency-faulty.webm'),
|
||||
@@ -192,15 +183,6 @@ test('Can encode transparent video', async () => {
|
||||
await encodeTransparentVideoTest();
|
||||
});
|
||||
|
||||
test('Can encode transparent video, forced CPU path', async () => {
|
||||
try {
|
||||
ColorAlphaSplitter.forceCpu = true;
|
||||
await encodeTransparentVideoTest();
|
||||
} finally {
|
||||
ColorAlphaSplitter.forceCpu = false;
|
||||
}
|
||||
});
|
||||
|
||||
test('Can encode video with alternating transparency', async () => {
|
||||
// This test is already brutal when it comes to finding async race conditions, so run it thrice to be MORE brutal
|
||||
for (let j = 0; j < 3; j++) {
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
import { expect, test } from 'vitest';
|
||||
import { registerProresDecoder } from '@mediabunny/prores';
|
||||
import { Input } from '../../src/input.js';
|
||||
import { BufferSource, UrlSource } from '../../src/source.js';
|
||||
import { ALL_FORMATS } from '../../src/input-format.js';
|
||||
import { Output } from '../../src/output.js';
|
||||
import { MkvOutputFormat, MovOutputFormat } from '../../src/output-format.js';
|
||||
import { BufferTarget } from '../../src/target.js';
|
||||
import { Conversion } from '../../src/conversion.js';
|
||||
import { VideoSampleSink } from '../../src/media-sink.js';
|
||||
import { assert } from '../../src/misc.js';
|
||||
|
||||
const SAMPLE_URL = 'https://pub-1ee78aacb848486482b20a72b55b3121.r2.dev/turbores-sample.mov';
|
||||
|
||||
test.concurrent('ProRes MOV file reading', async () => {
|
||||
using input = new Input({
|
||||
source: new UrlSource(SAMPLE_URL),
|
||||
formats: ALL_FORMATS,
|
||||
});
|
||||
|
||||
const videoTrack = (await input.getPrimaryVideoTrack())!;
|
||||
expect(await videoTrack.getCodec()).toBe('prores');
|
||||
expect(await videoTrack.getCodedWidth()).toBe(1920);
|
||||
expect(await videoTrack.getCodedHeight()).toBe(1080);
|
||||
|
||||
const decoderConfig = (await videoTrack.getDecoderConfig())!;
|
||||
expect(decoderConfig.codec).toBe('apch');
|
||||
expect(decoderConfig.description).toBeUndefined();
|
||||
});
|
||||
|
||||
test.concurrent('ProRes transmuxing into MOV', { timeout: 10_000 }, async () => {
|
||||
using input = new Input({
|
||||
source: new UrlSource(SAMPLE_URL),
|
||||
formats: ALL_FORMATS,
|
||||
});
|
||||
|
||||
const output = new Output({
|
||||
format: new MovOutputFormat(),
|
||||
target: new BufferTarget(),
|
||||
});
|
||||
|
||||
const conversion = await Conversion.init({
|
||||
input,
|
||||
output,
|
||||
audio: {
|
||||
discard: true,
|
||||
},
|
||||
trim: {
|
||||
end: 0.5,
|
||||
},
|
||||
});
|
||||
await conversion.execute();
|
||||
|
||||
using newInput = new Input({
|
||||
source: new BufferSource(output.target.buffer!),
|
||||
formats: ALL_FORMATS,
|
||||
});
|
||||
|
||||
const videoTrack = (await newInput.getPrimaryVideoTrack())!;
|
||||
expect(await videoTrack.getCodec()).toBe('prores');
|
||||
expect((await videoTrack.computePacketStats()).packetCount).toBe(15);
|
||||
|
||||
const decoderConfig = (await videoTrack.getDecoderConfig())!;
|
||||
expect(decoderConfig.codec).toBe('apch');
|
||||
expect(decoderConfig.description).toBeUndefined();
|
||||
});
|
||||
|
||||
test.concurrent('ProRes transmuxing into MKV', { timeout: 10_000 }, async () => {
|
||||
using input = new Input({
|
||||
source: new UrlSource(SAMPLE_URL),
|
||||
formats: ALL_FORMATS,
|
||||
});
|
||||
|
||||
const output = new Output({
|
||||
format: new MkvOutputFormat(),
|
||||
target: new BufferTarget(),
|
||||
});
|
||||
|
||||
const conversion = await Conversion.init({
|
||||
input,
|
||||
output,
|
||||
audio: {
|
||||
discard: true,
|
||||
},
|
||||
trim: {
|
||||
end: 0.5,
|
||||
},
|
||||
});
|
||||
await conversion.execute();
|
||||
|
||||
using newInput = new Input({
|
||||
source: new BufferSource(output.target.buffer!),
|
||||
formats: ALL_FORMATS,
|
||||
});
|
||||
|
||||
const videoTrack = (await newInput.getPrimaryVideoTrack())!;
|
||||
expect(await videoTrack.getCodec()).toBe('prores');
|
||||
expect((await videoTrack.computePacketStats()).packetCount).toBe(15);
|
||||
|
||||
const decoderConfig = (await videoTrack.getDecoderConfig())!;
|
||||
expect(decoderConfig.codec).toBe('apch');
|
||||
expect(decoderConfig.description).toBeUndefined();
|
||||
});
|
||||
|
||||
test('Custom coder registration', { timeout: 10_000 }, async () => {
|
||||
using input = new Input({
|
||||
source: new UrlSource(SAMPLE_URL),
|
||||
formats: ALL_FORMATS,
|
||||
});
|
||||
|
||||
const videoTrack = (await input.getPrimaryVideoTrack())!;
|
||||
expect(await videoTrack.getCodec()).toBe('prores');
|
||||
|
||||
// Without a registered decoder, there's no way to decode ProRes in this environment
|
||||
expect(await videoTrack.canDecode()).toBe(false);
|
||||
|
||||
registerProresDecoder();
|
||||
|
||||
expect(await videoTrack.canDecode()).toBe(true);
|
||||
});
|
||||
|
||||
test('ProRes decoding', { timeout: 10_000 }, async () => {
|
||||
registerProresDecoder();
|
||||
|
||||
using input = new Input({
|
||||
source: new UrlSource(SAMPLE_URL),
|
||||
formats: ALL_FORMATS,
|
||||
});
|
||||
|
||||
const videoTrack = (await input.getPrimaryVideoTrack())!;
|
||||
const firstTimestamp = await videoTrack.getFirstTimestamp();
|
||||
|
||||
const sink = new VideoSampleSink(videoTrack);
|
||||
using sample = await sink.getSample(firstTimestamp);
|
||||
assert(sample);
|
||||
|
||||
expect(sample.timestamp).toBe(firstTimestamp);
|
||||
expect(sample.duration).toBeGreaterThan(0);
|
||||
|
||||
expect(sample.displayWidth).toBe(1920);
|
||||
expect(sample.displayHeight).toBe(1080);
|
||||
expect(sample.codedWidth).toBe(1920);
|
||||
expect(sample.codedHeight).toBe(1080); // Technically a lie but the field is ill-defined
|
||||
expect(sample.visibleRect).toEqual({
|
||||
left: 0,
|
||||
top: 0,
|
||||
width: 1920,
|
||||
height: 1080,
|
||||
});
|
||||
expect(sample.format).toBe('I422P10');
|
||||
|
||||
const allocationSize = sample.allocationSize();
|
||||
expect(allocationSize).toBeGreaterThan(0);
|
||||
|
||||
const pixels = new Uint8Array(allocationSize);
|
||||
await sample.copyTo(pixels);
|
||||
expect(pixels.some(byte => byte !== 0)).toBe(true);
|
||||
});
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
buildAudioCodecString,
|
||||
buildVideoCodecString,
|
||||
NON_PCM_AUDIO_CODECS,
|
||||
PRORES_FOURCCS,
|
||||
VideoCodec,
|
||||
} from '../../src/codec.js';
|
||||
import { EncodedPacket } from '../../src/packet.js';
|
||||
@@ -30,6 +31,8 @@ import { Output } from '../../src/output.js';
|
||||
import { Mp4OutputFormat } from '../../src/output-format.js';
|
||||
import { BufferTarget } from '../../src/target.js';
|
||||
import { Conversion } from '../../src/conversion.js';
|
||||
import { VideoSampleSource } from '../../src/media-source.js';
|
||||
import { QUALITY_HIGH } from '../../src/encode.js';
|
||||
import { AvFrameVideoSampleResource } from '../../packages/server/src/video-sample.js';
|
||||
import { AvFrameAudioSampleResource } from '../../packages/server/src/audio-sample.js';
|
||||
import { toAvFrame } from '../../packages/server/src/index.js';
|
||||
@@ -113,7 +116,7 @@ describe('Video', async () => {
|
||||
encoder.codec = 'avc';
|
||||
// @ts-expect-error Readonly
|
||||
encoder.config = {
|
||||
codec: buildVideoCodecString('avc', 1280, 720, 1e6),
|
||||
codec: buildVideoCodecString('avc', 1280, 720, 1e6, false),
|
||||
width: 1280,
|
||||
height: 720,
|
||||
bitrate: 1e6,
|
||||
@@ -196,7 +199,7 @@ describe('Video', async () => {
|
||||
encoder.codec = 'avc';
|
||||
// @ts-expect-error Readonly
|
||||
encoder.config = {
|
||||
codec: buildVideoCodecString('avc', 1280, 720, 1e6),
|
||||
codec: buildVideoCodecString('avc', 1280, 720, 1e6, false),
|
||||
width: 1280,
|
||||
height: 720,
|
||||
bitrate: 1e6,
|
||||
@@ -471,6 +474,188 @@ describe('Video', async () => {
|
||||
});
|
||||
});
|
||||
|
||||
test('ProRes encode', async () => {
|
||||
// ProRes can't be decoded by node-av's wrapper here, so this is encode-only.
|
||||
const encoder = new NodeAvVideoEncoder();
|
||||
// @ts-expect-error Readonly
|
||||
encoder.codec = 'prores';
|
||||
// @ts-expect-error Readonly
|
||||
encoder.config = {
|
||||
codec: 'apch',
|
||||
width: 1280,
|
||||
height: 720,
|
||||
bitrate: 1e6,
|
||||
} satisfies VideoEncoderConfig;
|
||||
|
||||
const packets: EncodedPacket[] = [];
|
||||
const sampleTimestamps: number[] = [];
|
||||
|
||||
// @ts-expect-error Readonly
|
||||
encoder.onPacket = (packet: EncodedPacket, meta: EncodedVideoChunkMetadata) => {
|
||||
expect(packet.duration).toBe(1 / 30);
|
||||
expect(packet.type).toBe('key');
|
||||
expect(String.fromCharCode(...packet.data.slice(4, 8))).toBe('icpf');
|
||||
|
||||
if (packets.length === 0) {
|
||||
expect(meta.decoderConfig).toBeDefined();
|
||||
expect(meta.decoderConfig!.codec).toBe('apch');
|
||||
expect(meta.decoderConfig!.codedWidth).toBe(1280);
|
||||
expect(meta.decoderConfig!.codedHeight).toBe(720);
|
||||
expect(meta.decoderConfig!.description).toBeUndefined();
|
||||
}
|
||||
|
||||
packets.push(packet);
|
||||
};
|
||||
|
||||
await encoder.init();
|
||||
|
||||
const data = new Uint8Array(1280 * 720 * 4).fill(0xff); // White
|
||||
for (let i = 0; i < 10; i++) {
|
||||
using sample = new VideoSample(data, {
|
||||
format: 'RGBX',
|
||||
codedWidth: 1280,
|
||||
codedHeight: 720,
|
||||
timestamp: i / 30,
|
||||
duration: 1 / 30,
|
||||
});
|
||||
|
||||
sampleTimestamps.push(sample.timestamp);
|
||||
|
||||
await encoder.encode(sample, {});
|
||||
}
|
||||
|
||||
await encoder.flush();
|
||||
|
||||
expect(packets).toHaveLength(10);
|
||||
expect(packets.map(x => x.timestamp)).toEqual(sampleTimestamps);
|
||||
|
||||
await encoder.close();
|
||||
});
|
||||
|
||||
test('ProRes Proxy encode', async () => {
|
||||
const encoder = new NodeAvVideoEncoder();
|
||||
// @ts-expect-error Readonly
|
||||
encoder.codec = 'prores';
|
||||
// @ts-expect-error Readonly
|
||||
encoder.config = {
|
||||
codec: 'apco', // Force ProRes 422 Proxy
|
||||
width: 1280,
|
||||
height: 720,
|
||||
bitrate: 1e6,
|
||||
} satisfies VideoEncoderConfig;
|
||||
|
||||
const packets: EncodedPacket[] = [];
|
||||
const sampleTimestamps: number[] = [];
|
||||
|
||||
// @ts-expect-error Readonly
|
||||
encoder.onPacket = (packet: EncodedPacket, meta: EncodedVideoChunkMetadata) => {
|
||||
expect(packet.duration).toBe(1 / 30);
|
||||
expect(packet.type).toBe('key');
|
||||
expect(String.fromCharCode(...packet.data.slice(4, 8))).toBe('icpf');
|
||||
|
||||
if (packets.length === 0) {
|
||||
expect(meta.decoderConfig).toBeDefined();
|
||||
expect(PRORES_FOURCCS).toContain(meta.decoderConfig!.codec);
|
||||
expect(meta.decoderConfig!.codec).toBe('apco');
|
||||
expect(meta.decoderConfig!.codedWidth).toBe(1280);
|
||||
expect(meta.decoderConfig!.codedHeight).toBe(720);
|
||||
expect(meta.decoderConfig!.description).toBeUndefined();
|
||||
}
|
||||
|
||||
packets.push(packet);
|
||||
};
|
||||
|
||||
await encoder.init();
|
||||
|
||||
const data = new Uint8Array(1280 * 720 * 4).fill(0xff); // White
|
||||
for (let i = 0; i < 10; i++) {
|
||||
using sample = new VideoSample(data, {
|
||||
format: 'RGBX',
|
||||
codedWidth: 1280,
|
||||
codedHeight: 720,
|
||||
timestamp: i / 30,
|
||||
duration: 1 / 30,
|
||||
});
|
||||
|
||||
sampleTimestamps.push(sample.timestamp);
|
||||
|
||||
await encoder.encode(sample, {});
|
||||
}
|
||||
|
||||
await encoder.flush();
|
||||
|
||||
expect(packets).toHaveLength(10);
|
||||
expect(packets.map(x => x.timestamp)).toEqual(sampleTimestamps);
|
||||
|
||||
await encoder.close();
|
||||
});
|
||||
|
||||
test('Transparent ProRes encode and decode', async () => {
|
||||
const output = new Output({
|
||||
format: new Mp4OutputFormat(),
|
||||
target: new BufferTarget(),
|
||||
});
|
||||
|
||||
const source = new VideoSampleSource({
|
||||
codec: 'prores',
|
||||
bitrate: QUALITY_HIGH,
|
||||
alpha: 'keep',
|
||||
});
|
||||
output.addVideoTrack(source);
|
||||
|
||||
await output.start();
|
||||
|
||||
// Solid red at 50% alpha
|
||||
const data = new Uint8Array(640 * 480 * 4);
|
||||
for (let i = 0; i < data.byteLength; i += 4) {
|
||||
data[i] = 0xff;
|
||||
data[i + 3] = 0x80;
|
||||
}
|
||||
|
||||
using inputSample = new VideoSample(data, {
|
||||
format: 'RGBA',
|
||||
codedWidth: 640,
|
||||
codedHeight: 480,
|
||||
timestamp: 0,
|
||||
duration: 1,
|
||||
});
|
||||
await source.add(inputSample);
|
||||
|
||||
await output.finalize();
|
||||
|
||||
using input = new Input({
|
||||
source: new BufferSource(output.target.buffer!),
|
||||
formats: ALL_FORMATS,
|
||||
});
|
||||
|
||||
const videoTrack = (await input.getPrimaryVideoTrack())!;
|
||||
expect(await videoTrack.canBeTransparent()).toBe(true);
|
||||
|
||||
const sink = new VideoSampleSink(videoTrack);
|
||||
using sample = (await sink.getSample(0))!;
|
||||
|
||||
expect(sample.format).toBe('I444AP12');
|
||||
|
||||
const decodedData = new Uint8Array(sample.allocationSize());
|
||||
const layout = await sample.copyTo(decodedData);
|
||||
|
||||
const view = new DataView(decodedData.buffer);
|
||||
const alphaPlane = layout[3]!;
|
||||
let minAlpha = Infinity;
|
||||
let maxAlpha = -Infinity;
|
||||
for (let y = 0; y < sample.codedHeight; y++) {
|
||||
for (let x = 0; x < sample.codedWidth; x++) {
|
||||
const alpha = view.getUint16(alphaPlane.offset + y * alphaPlane.stride + x * 2, true);
|
||||
minAlpha = Math.min(minAlpha, alpha);
|
||||
maxAlpha = Math.max(maxAlpha, alpha);
|
||||
}
|
||||
}
|
||||
|
||||
expect(maxAlpha).toBeLessThan(4095);
|
||||
expect(Math.abs(minAlpha - 4095 / 2)).toBeLessThan(32);
|
||||
expect(Math.abs(maxAlpha - 4095 / 2)).toBeLessThan(32);
|
||||
});
|
||||
|
||||
test('Non-square pixels encode and decode #1', async () => {
|
||||
await encodeDecodeTest('avc', { displayWidth: 2 * 1280, displayHeight: 720 }, async (packet, meta, i) => {
|
||||
if (i === 0) {
|
||||
@@ -517,7 +702,7 @@ describe('Video', async () => {
|
||||
encoder.codec = codec;
|
||||
// @ts-expect-error Readonly
|
||||
encoder.config = {
|
||||
codec: buildVideoCodecString(codec, 1280, 720, 1e6),
|
||||
codec: buildVideoCodecString(codec, 1280, 720, 1e6, false),
|
||||
width: 1280,
|
||||
height: 720,
|
||||
bitrate: 1e6,
|
||||
@@ -616,6 +801,10 @@ describe('Video', async () => {
|
||||
await conversionRoundtrip('av1');
|
||||
});
|
||||
|
||||
test('ProRes conversion roundtrip', { timeout: 30_000 }, async () => {
|
||||
await conversionRoundtrip('prores');
|
||||
});
|
||||
|
||||
const conversionRoundtrip = async (codec: VideoCodec, duration?: number) => {
|
||||
using input = new Input({
|
||||
source: new FilePathSource('./test/public/video.mp4'),
|
||||
@@ -872,13 +1061,13 @@ describe('Video', async () => {
|
||||
});
|
||||
|
||||
// https://github.com/Vanilagy/mediabunny/issues/392
|
||||
test('Memory doesn\'t leak', async () => {
|
||||
test('Memory doesn\'t leak', { timeout: 10_000 }, async () => {
|
||||
const encoder = new NodeAvVideoEncoder();
|
||||
// @ts-expect-error Readonly
|
||||
encoder.codec = 'avc';
|
||||
// @ts-expect-error Readonly
|
||||
encoder.config = {
|
||||
codec: buildVideoCodecString('avc', 1920, 1080, 1e6),
|
||||
codec: buildVideoCodecString('avc', 1920, 1080, 1e6, false),
|
||||
width: 1920,
|
||||
height: 1080,
|
||||
bitrate: 1e6,
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
"@mediabunny/ac3": ["./packages/ac3/src/index.ts"],
|
||||
"@mediabunny/aac-encoder": ["./packages/aac-encoder/src/index.ts"],
|
||||
"@mediabunny/flac-encoder": ["./packages/flac-encoder/src/index.ts"],
|
||||
"@mediabunny/prores": ["./packages/prores/src/index.ts"],
|
||||
},
|
||||
"types": ["vite/client"]
|
||||
},
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
"@mediabunny/aac-encoder": ["./packages/aac-encoder/src/index.ts"],
|
||||
"@mediabunny/flac-encoder": ["./packages/flac-encoder/src/index.ts"],
|
||||
"@mediabunny/mp3-encoder": ["./packages/mp3-encoder/src/index.ts"],
|
||||
"@mediabunny/prores": ["./packages/prores/src/index.ts"],
|
||||
"@mediabunny/server": ["./packages/server/src/index.ts"],
|
||||
},
|
||||
},
|
||||
@@ -22,6 +23,7 @@
|
||||
{ "path": "./packages/aac-encoder" },
|
||||
{ "path": "./packages/flac-encoder" },
|
||||
{ "path": "./packages/mp3-encoder" },
|
||||
{ "path": "./packages/prores" },
|
||||
{ "path": "./packages/server" },
|
||||
]
|
||||
}
|
||||
|
||||
@@ -25,6 +25,8 @@ export default defineConfig({
|
||||
path.resolve(__dirname, './packages/aac-encoder/dist/bundles/mediabunny-aac-encoder.mjs'),
|
||||
'@mediabunny/flac-encoder':
|
||||
path.resolve(__dirname, './packages/flac-encoder/dist/bundles/mediabunny-flac-encoder.mjs'),
|
||||
'@mediabunny/prores':
|
||||
path.resolve(__dirname, './packages/prores/dist/bundles/mediabunny-prores.mjs'),
|
||||
},
|
||||
},
|
||||
plugins: [
|
||||
@@ -33,6 +35,10 @@ export default defineConfig({
|
||||
server: {
|
||||
hmr: false,
|
||||
allowedHosts: true,
|
||||
headers: {
|
||||
'Cross-Origin-Opener-Policy': 'same-origin',
|
||||
'Cross-Origin-Embedder-Policy': 'require-corp',
|
||||
},
|
||||
},
|
||||
build: {
|
||||
outDir: 'dist-docs', // Build them directly into the docs build folder
|
||||
|
||||
+3
-1
@@ -1,7 +1,7 @@
|
||||
/// <reference types="@vitest/browser/providers/webdriverio" />
|
||||
|
||||
import { defineConfig } from 'vitest/config';
|
||||
import path from 'path';
|
||||
import path from 'node:path';
|
||||
|
||||
export default defineConfig({
|
||||
resolve: {
|
||||
@@ -14,6 +14,8 @@ export default defineConfig({
|
||||
path.resolve(__dirname, './packages/flac-encoder/dist/bundles/mediabunny-flac-encoder.mjs'),
|
||||
'@mediabunny/mp3-encoder':
|
||||
path.resolve(__dirname, './packages/mp3-encoder/dist/bundles/mediabunny-mp3-encoder.mjs'),
|
||||
'@mediabunny/prores':
|
||||
path.resolve(__dirname, './packages/prores/dist/bundles/mediabunny-prores.mjs'),
|
||||
'@mediabunny/server':
|
||||
path.resolve(__dirname, './packages/server/src/index.ts'),
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user