diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e06b340..c10309d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -93,6 +93,11 @@ jobs: packages/flac-encoder/dist/bundles/mediabunny-flac-encoder.mjs packages/flac-encoder/dist/bundles/mediabunny-flac-encoder.min.mjs packages/flac-encoder/dist/mediabunny-flac-encoder.d.ts + packages/prores/dist/bundles/mediabunny-prores.js + packages/prores/dist/bundles/mediabunny-prores.min.js + packages/prores/dist/bundles/mediabunny-prores.mjs + packages/prores/dist/bundles/mediabunny-prores.min.mjs + packages/prores/dist/mediabunny-prores.d.ts packages/server/dist/bundles/mediabunny-server.cjs packages/server/dist/bundles/mediabunny-server.min.cjs packages/server/dist/bundles/mediabunny-server.mjs diff --git a/.gitignore b/.gitignore index 10904ca..30c66a0 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,5 @@ packages/mp3-encoder/dist packages/ac3/dist packages/aac-encoder/dist packages/flac-encoder/dist -packages/server/dist \ No newline at end of file +packages/server/dist +packages/prores/dist \ No newline at end of file diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 7b779a6..d9d6b7b 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -121,6 +121,7 @@ export default withMermaid({ { text: 'aac-encoder', link: '/guide/extensions/aac-encoder' }, { text: 'ac3', link: '/guide/extensions/ac3' }, { text: 'flac-encoder', link: '/guide/extensions/flac-encoder' }, + { text: 'prores', link: '/guide/extensions/prores' }, ], }, ], diff --git a/docs/api-config.json b/docs/api-config.json index 376e45e..b56656e 100644 --- a/docs/api-config.json +++ b/docs/api-config.json @@ -26,5 +26,6 @@ "@mediabunny/mp3-encoder": "Adds MP3 encoder support to Mediabunny.", "@mediabunny/ac3": "Adds AC-3/E-AC-3 decoder and encoder support to Mediabunny.", "@mediabunny/aac-encoder": "Polyfills AAC encoder support to Mediabunny.", - "@mediabunny/flac-encoder": "Adds FLAC encoder support to Mediabunny." + "@mediabunny/flac-encoder": "Adds FLAC encoder support to Mediabunny.", + "@mediabunny/prores": "Adds Apple ProRes decoder support to Mediabunny." } diff --git a/docs/guide/extensions/prores.md b/docs/guide/extensions/prores.md new file mode 100644 index 0000000..c1bf689 --- /dev/null +++ b/docs/guide/extensions/prores.md @@ -0,0 +1,36 @@ +--- +description: The @mediabunny/prores extension provides an extremely fast Apple ProRes decoder for the browser. +--- + +# @mediabunny/prores + +Browsers have no support for Apple ProRes in their WebCodecs implementations. This extension package provides a decoder for use with Mediabunny, allowing you to decode ProRes directly in the browser at unprecedented speed. It is implemented using Mediabunny's [custom coder API](https://mediabunny.dev/guide/supported-formats-and-codecs#custom-coders) and uses [TurboRes](https://github.com/Vanilagy/turbores), an extremely fast WASM-based ProRes decoder, under the hood. + + + GitHub page + + + +## Installation + +This library peer-depends on Mediabunny. Install both using npm: +```bash +npm install mediabunny @mediabunny/prores +``` + +Alternatively, directly include them using a script tag: +```html + + +``` + +This will expose the global objects `Mediabunny` and `MediabunnyProres`. Use `mediabunny-prores.d.ts` to provide types for these globals. You can download the built distribution files from the [releases page](https://github.com/Vanilagy/mediabunny/releases). + +## Usage + +```ts +import { registerProresDecoder } from '@mediabunny/prores'; + +registerProresDecoder(); +``` +That's it - Mediabunny now uses the registered ProRes decoder automatically. diff --git a/docs/guide/supported-formats-and-codecs.md b/docs/guide/supported-formats-and-codecs.md index 1ce556d..780f78b 100644 --- a/docs/guide/supported-formats-and-codecs.md +++ b/docs/guide/supported-formats-and-codecs.md @@ -40,6 +40,7 @@ Mediabunny ships with built-in decoders and encoders for all audio PCM codecs, m - `'vp8'` - VP8 - `'vp9'` - VP9 - `'av1'` - AOMedia Video 1 (AV1) +- `'prores'` - Apple ProRes [^prores] ### Audio codecs @@ -80,6 +81,7 @@ Not all codecs can be used with all containers. The following table specifies th | `'vp8'` | ✓ | ✓ | ✓ | ✓ | | | | | | | | `'vp9'` | ✓ | ✓ | ✓ | ✓ | | | | | | | | `'av1'` | ✓ | ✓ | ✓ | ✓ | | | | | | | +| `'prores'` | ✓ | ✓ | ✓ | | | | | | | | | `'aac'` | ✓ | ✓ | ✓ | | | | | ✓ | | ✓ | | `'opus'` | ✓ | ✓ | ✓ | ✓ | ✓ | | | | | | | `'mp3'` | ✓ | ✓ | ✓ | | | ✓ | | | | ✓ | @@ -105,6 +107,7 @@ Not all codecs can be used with all containers. The following table specifies th For HLS, the supported codecs depend on the segment format chosen. +[^prores]: ProRes is not supported by WebCodecs. To decode it, use the [`@mediabunny/prores`](./extensions/prores) extension package. [^aac]: In some browsers, AAC encoding is not supported by WebCodecs. You can polyfill it with the [`@mediabunny/aac-encoder`](./extensions/aac-encoder) extension package. [^mp3]: MP3 encoding is not supported by WebCodecs. You can polyfill it with the [`@mediabunny/mp3-encoder`](./extensions/mp3-encoder) extension package. [^flac]: FLAC encoding is not supported by WebCodecs. You can polyfill it with the [`@mediabunny/flac-encoder`](./extensions/flac-encoder) extension package. @@ -293,10 +296,13 @@ class { codec: AudioCodec; config: AudioEncoderConfig; onPacket: (packet: EncodedPacket, meta?: EncodedAudioChunkMetadata) => unknown; + + // For both: + onError: (error: unknown) => void; } ``` -`codec` and `config` specify the concrete codec configuration to use, and `onPacket` is a method that your code **must** call for each encoded packet it creates. +`codec` and `config` specify the concrete codec configuration to use, and `onPacket` is a method that your code **must** call for each encoded packet it creates. `onError` is a method you can call to surface any out-of-band errors that occur outside of the regular method calls (such as from an asynchronous background task); these errors would otherwise go uncaught. You **must** implement the following methods in your custom encoder class: ```ts @@ -356,10 +362,13 @@ class { codec: AudioCodec; config: AudioDecoderConfig; onSample: (sample: AudioSample) => unknown; + + // For both: + onError: (error: unknown) => void; } ``` -`codec` and `config` specify the concrete codec configuration to use, and `onSample` is a method that your code **must** call for each video/audio sample it creates. +`codec` and `config` specify the concrete codec configuration to use, and `onSample` is a method that your code **must** call for each video/audio sample it creates. `onError` is a method you can call to surface any out-of-band errors that occur outside of the regular method calls (such as from an asynchronous background task); these errors would otherwise go uncaught. You **must** implement the following methods in your custom decoder class: ```ts diff --git a/eslint.config.mjs b/eslint.config.mjs index b0330c2..6a9caaf 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -47,6 +47,7 @@ export default tseslint.config( 'packages/aac-encoder/build', 'packages/flac-encoder/dist', 'packages/flac-encoder/build', + 'packages/prores/dist', 'packages/server/dist', 'eslint.config.mjs', 'docs/.vitepress/cache', diff --git a/examples/file-compression/file-compression.ts b/examples/file-compression/file-compression.ts index 7c68c05..d4613fa 100644 --- a/examples/file-compression/file-compression.ts +++ b/examples/file-compression/file-compression.ts @@ -68,6 +68,7 @@ const compressFile = async (resource: File | string) => { bitrate: QUALITY_VERY_LOW, }, audio: { + codec: 'opus', bitrate: QUALITY_VERY_LOW, }, }); diff --git a/examples/media-player/media-player.ts b/examples/media-player/media-player.ts index 7d22cdf..30f6040 100644 --- a/examples/media-player/media-player.ts +++ b/examples/media-player/media-player.ts @@ -4,7 +4,6 @@ import { BlobSource, CanvasSink, Input, - registerDecoder, UrlSource, WrappedAudioBuffer, WrappedCanvas, @@ -75,9 +74,6 @@ let volumeMuted = false; /** === INIT LOGIC === */ -import { ProResDecoder } from '../../packages/prores-decoder/src/index.js'; -registerDecoder(ProResDecoder); - const initMediaPlayer = async (resource: File | string) => { try { // First, dispose any ongoing playback: diff --git a/package-lock.json b/package-lock.json index 4fa0b13..7f15975 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1461,6 +1461,10 @@ "resolved": "packages/mp3-encoder", "link": true }, + "node_modules/@mediabunny/prores": { + "resolved": "packages/prores", + "link": true + }, "node_modules/@mediabunny/server": { "resolved": "packages/server", "link": true @@ -12922,6 +12926,21 @@ "mediabunny": "^1.0.0" } }, + "packages/prores": { + "name": "@mediabunny/prores", + "version": "1.25.8", + "license": "MPL-2.0", + "devDependencies": { + "@types/emscripten": "^1.40.1" + }, + "funding": { + "type": "individual", + "url": "https://github.com/sponsors/Vanilagy" + }, + "peerDependencies": { + "mediabunny": "^1.0.0" + } + }, "packages/server": { "name": "@mediabunny/server", "version": "1.49.0", diff --git a/package.json b/package.json index 8e96992..eaf8658 100644 --- a/package.json +++ b/package.json @@ -51,7 +51,7 @@ "docs:dev": "vitepress dev docs", "docs:build": "npm run build && npm run docs:generate && vitepress build docs && npm run examples:build && cp dist/mediabunny.d.ts dist-docs/", "docs:preview": "vitepress preview docs", - "docs:generate": "tsx scripts/generate-api-docs.ts src/index.ts packages/mp3-encoder/src/index.ts packages/ac3/src/index.ts packages/aac-encoder/src/index.ts packages/flac-encoder/src/index.ts packages/server/src/index.ts docs/api-config.json", + "docs:generate": "tsx scripts/generate-api-docs.ts src/index.ts packages/mp3-encoder/src/index.ts packages/ac3/src/index.ts packages/aac-encoder/src/index.ts packages/flac-encoder/src/index.ts packages/prores/src/index.ts packages/server/src/index.ts docs/api-config.json", "dev": "vite", "examples:build": "vite build", "fix-build-import-paths": "tsx scripts/add-import-extensions.ts", diff --git a/packages/prores-decoder/README.md b/packages/prores-decoder/README.md deleted file mode 100644 index 275a5e8..0000000 --- a/packages/prores-decoder/README.md +++ /dev/null @@ -1,46 +0,0 @@ -```bash -emmake make distclean - -emconfigure ./configure \ - --target-os=none \ - --arch=x86_32 \ - --enable-cross-compile \ - --disable-asm \ - --disable-x86asm \ - --disable-inline-asm \ - --disable-stripping \ - --disable-programs \ - --disable-doc \ - --disable-debug \ - --disable-all \ - --disable-everything \ - --disable-pthreads \ - --enable-avcodec \ - --enable-decoder=prores \ - --cc="emcc" \ - --cxx=em++ \ - --ar=emar \ - --ranlib=emranlib \ - --extra-cflags="-DNDEBUG -O3 -msimd128" - -emmake make -``` - -```bash -export FFMPEG_PATH=/path/to/ffmpeg - -emcc src/bridge.c \ - $FFMPEG_PATH/libavcodec/libavcodec.a \ - $FFMPEG_PATH/libavutil/libavutil.a \ - -I$FFMPEG_PATH \ - -s MODULARIZE=1 \ - -s EXPORT_ES6=1 \ - -s SINGLE_FILE=1 \ - -s ALLOW_MEMORY_GROWTH=1 \ - -s ENVIRONMENT=web,worker \ - -s EXPORTED_RUNTIME_METHODS=cwrap,HEAPU8 \ - -s EXPORTED_FUNCTIONS=_malloc,_free \ - -msimd128 \ - -O3 \ - -o build/prores.js -``` \ No newline at end of file diff --git a/packages/prores-decoder/build/prores.js b/packages/prores-decoder/build/prores.js deleted file mode 100644 index ea5911b..0000000 Binary files a/packages/prores-decoder/build/prores.js and /dev/null differ diff --git a/packages/prores-decoder/src/bridge.c b/packages/prores-decoder/src/bridge.c deleted file mode 100644 index e0976f0..0000000 --- a/packages/prores-decoder/src/bridge.c +++ /dev/null @@ -1,182 +0,0 @@ -#include -#include -#include "libavcodec/avcodec.h" -#include "libavutil/pixdesc.h" -#include "libavutil/imgutils.h" - -typedef struct { - AVCodecContext *codec_ctx; - AVPacket *packet; - AVFrame *frame; - uint8_t *buffer; - int buffer_size; -} DecoderContext; - -EMSCRIPTEN_KEEPALIVE -DecoderContext *init_decoder() { - const AVCodec *codec = avcodec_find_decoder(AV_CODEC_ID_PRORES); - if (!codec) return NULL; - - AVCodecContext *codec_ctx = avcodec_alloc_context3(codec); - if (!codec_ctx) return NULL; - - if (avcodec_open2(codec_ctx, codec, NULL) < 0) { - avcodec_free_context(&codec_ctx); - return NULL; - } - - AVPacket *packet = av_packet_alloc(); - if (!packet) { - avcodec_free_context(&codec_ctx); - return NULL; - } - - AVFrame *frame = av_frame_alloc(); - if (!frame) { - av_packet_free(&packet); - avcodec_free_context(&codec_ctx); - return NULL; - } - - DecoderContext *ctx = malloc(sizeof(DecoderContext)); - if (!ctx) { - av_frame_free(&frame); - av_packet_free(&packet); - avcodec_free_context(&codec_ctx); - return NULL; - } - - ctx->codec_ctx = codec_ctx; - ctx->packet = packet; - ctx->frame = frame; - ctx->buffer = NULL; - ctx->buffer_size = 0; - - return ctx; -} - -EMSCRIPTEN_KEEPALIVE -uint8_t *configure_packet(DecoderContext *ctx, int size) { - if (av_new_packet(ctx->packet, size) < 0) { - return NULL; - } - - ctx->packet->pts = 0; - ctx->packet->dts = AV_NOPTS_VALUE; - ctx->packet->time_base.num = 1; - ctx->packet->time_base.den = 1; - ctx->packet->flags = AV_PKT_FLAG_KEY; - - return ctx->packet->data; -} - -EMSCRIPTEN_KEEPALIVE -int decode_packet(DecoderContext *ctx) { - double start = emscripten_get_now(); - - int ret = avcodec_send_packet(ctx->codec_ctx, ctx->packet); - if (ret < 0) { - return ret; - } - - double after_send = emscripten_get_now(); - - ret = avcodec_receive_frame(ctx->codec_ctx, ctx->frame); - if (ret < 0) { - return ret; - } - - double after_receive = emscripten_get_now(); - - // Calculate required buffer size - int required_size = av_image_get_buffer_size( - ctx->frame->format, - ctx->frame->width, - ctx->frame->height, - 1 - ); - - if (required_size < 0) { - return required_size; - } - - // Reallocate buffer if needed - if (ctx->buffer_size < required_size) { - free(ctx->buffer); - ctx->buffer = malloc(required_size); - if (!ctx->buffer) { - ctx->buffer_size = 0; - return AVERROR(ENOMEM); - } - ctx->buffer_size = required_size; - } - - double after_alloc = emscripten_get_now(); - - // Copy frame data to contiguous buffer - ret = av_image_copy_to_buffer( - ctx->buffer, - ctx->buffer_size, - (const uint8_t * const *)ctx->frame->data, - ctx->frame->linesize, - ctx->frame->format, - ctx->frame->width, - ctx->frame->height, - 1 - ); - - if (ret < 0) { - return ret; - } - - double after_copy = emscripten_get_now(); - - printf("Timing: send=%.2fms receive=%.2fms alloc=%.2fms copy=%.2fms total=%.2fms\n", - after_send - start, - after_receive - after_send, - after_alloc - after_receive, - after_copy - after_alloc, - after_copy - start); - - return 0; -} - -EMSCRIPTEN_KEEPALIVE -int get_frame_width(DecoderContext *ctx) { - return ctx->frame->width; -} - -EMSCRIPTEN_KEEPALIVE -int get_frame_height(DecoderContext *ctx) { - return ctx->frame->height; -} - -EMSCRIPTEN_KEEPALIVE -int get_frame_format(DecoderContext *ctx) { - return ctx->frame->format; -} - -EMSCRIPTEN_KEEPALIVE -int get_frame_num_planes(DecoderContext *ctx) { - return av_pix_fmt_count_planes(ctx->frame->format); -} - -EMSCRIPTEN_KEEPALIVE -int get_frame_linesize(DecoderContext *ctx, int n) { - return ctx->frame->linesize[n]; -} - -EMSCRIPTEN_KEEPALIVE -uint8_t *get_frame_data(DecoderContext *ctx, int n) { - return ctx->frame->data[n]; -} - -EMSCRIPTEN_KEEPALIVE -uint8_t *get_frame_data_ptr(DecoderContext *ctx) { - return ctx->buffer; -} - -EMSCRIPTEN_KEEPALIVE -int get_frame_data_size(DecoderContext *ctx) { - return ctx->buffer_size; -} \ No newline at end of file diff --git a/packages/prores-decoder/src/index.ts b/packages/prores-decoder/src/index.ts deleted file mode 100644 index e200812..0000000 --- a/packages/prores-decoder/src/index.ts +++ /dev/null @@ -1,90 +0,0 @@ -import { CustomVideoDecoder, EncodedPacket, MaybePromise, VideoCodec } from 'mediabunny'; -import createModule from '../build/prores'; -import { VideoSample } from 'mediabunny'; - -type ExtendedEmscriptenModule = EmscriptenModule & { - cwrap: typeof cwrap; -}; - -export class ProResDecoder extends CustomVideoDecoder { - // eslint-disable-next-line @typescript-eslint/no-unused-vars - static override supports(codec: VideoCodec, config: VideoDecoderConfig): boolean { - return codec === 'prores'; - } - - module!: ExtendedEmscriptenModule; - initDecoder!: () => number; - configurePacket!: (ctx: number, size: number) => number; - decodePacket!: (ctx: number) => number; - getFrameWidth!: (ctx: number) => number; - getFrameHeight!: (ctx: number) => number; - getFrameFormat!: (ctx: number) => number; - getFrameNumPlanes!: (ctx: number) => number; - getFrameLinesize!: (ctx: number, n: number) => number; - getFrameData!: (ctx: number, n: number) => number; - getFrameDataPtr!: (ctx: number) => number; - getFrameDataSize!: (ctx: number) => number; - - decoderContextPtr!: number; - - override async init() { - console.log('THIS IS BEING CALLED'); - - this.module = (await createModule()) as ExtendedEmscriptenModule; - - // Set up the functions - this.initDecoder = this.module.cwrap('init_decoder', 'number', []); - this.configurePacket = this.module.cwrap('configure_packet', 'number', ['number', 'number']); - this.decodePacket = this.module.cwrap('decode_packet', 'number', ['number']); - this.getFrameWidth = this.module.cwrap('get_frame_width', 'number', ['number']); - this.getFrameHeight = this.module.cwrap('get_frame_height', 'number', ['number']); - this.getFrameFormat = this.module.cwrap('get_frame_format', 'number', ['number']); - this.getFrameNumPlanes = this.module.cwrap('get_frame_num_planes', 'number', ['number']); - this.getFrameLinesize = this.module.cwrap('get_frame_linesize', 'number', ['number', 'number']); - this.getFrameData = this.module.cwrap('get_frame_data', 'number', ['number', 'number']); - this.getFrameDataPtr = this.module.cwrap('get_frame_data_ptr', 'number', ['number']); - this.getFrameDataSize = this.module.cwrap('get_frame_data_size', 'number', ['number']); - - this.decoderContextPtr = this.initDecoder(); - } - - override decode(packet: EncodedPacket) { - const dataPtr = this.configurePacket(this.decoderContextPtr, packet.byteLength); - if (dataPtr === 0) { - throw new Error('todo'); - } - - this.module.HEAPU8.set(packet.data, dataPtr); - - console.time(); - const ret = this.decodePacket(this.decoderContextPtr); - console.timeEnd(); - - if (ret === 0) { - const width = this.getFrameWidth(this.decoderContextPtr); - const height = this.getFrameHeight(this.decoderContextPtr); - const format = this.getFrameFormat(this.decoderContextPtr); - const dataPtr = this.getFrameDataPtr(this.decoderContextPtr); - const dataSize = this.getFrameDataSize(this.decoderContextPtr); - - const data = this.module.HEAPU8.subarray(dataPtr, dataPtr + dataSize); - - const sample = new VideoSample(data, { - format: 'I422P10' as VideoPixelFormat, - codedWidth: width, - codedHeight: height, - timestamp: packet.timestamp, - duration: packet.duration, - }); - this.onSample(sample); - } - } - - override flush(): MaybePromise { - // nada - } - - override close(): MaybePromise { - // nada - } -} diff --git a/packages/prores/LICENSE b/packages/prores/LICENSE new file mode 100644 index 0000000..d0a1fa1 --- /dev/null +++ b/packages/prores/LICENSE @@ -0,0 +1,373 @@ +Mozilla Public License Version 2.0 +================================== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at https://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. diff --git a/packages/prores/README.md b/packages/prores/README.md new file mode 100644 index 0000000..ee71c31 --- /dev/null +++ b/packages/prores/README.md @@ -0,0 +1,44 @@ +# @mediabunny/prores + +[![](https://img.shields.io/npm/v/@mediabunny/prores)](https://www.npmjs.com/package/@mediabunny/prores) +[![](https://img.shields.io/bundlephobia/minzip/@mediabunny/prores)](https://bundlephobia.com/package/@mediabunny/prores) +[![](https://img.shields.io/npm/dm/@mediabunny/prores)](https://www.npmjs.com/package/@mediabunny/prores) +[![](https://img.shields.io/discord/1390044844285497344?logo=discord&label=Discord)](https://discord.gg/hmpkyYuS4U) + +
+ +
+ +Browsers have no support for Apple ProRes in their WebCodecs implementations. This extension package provides a decoder for use with [Mediabunny](https://github.com/Vanilagy/mediabunny), allowing you to decode ProRes directly in the browser at unprecedented speed. It is implemented using Mediabunny's [custom coder API](https://mediabunny.dev/guide/supported-formats-and-codecs#custom-coders) and uses [TurboRes](https://github.com/Vanilagy/turbores), an extremely fast WASM-based ProRes decoder, under the hood. + +> This package, like the rest of Mediabunny, is enabled by its [sponsors](https://mediabunny.dev/#sponsors) and their donations. If you've derived value from this package, please consider [leaving a donation](https://github.com/sponsors/Vanilagy)! 💘 + +## Installation + +This library peer-depends on Mediabunny. Install both using npm: +```bash +npm install mediabunny @mediabunny/prores +``` + +Alternatively, directly include them using a script tag: +```html + + +``` + +This will expose the global objects `Mediabunny` and `MediabunnyProres`. Use `mediabunny-prores.d.ts` to provide types for these globals. You can download the built distribution files from the [releases page](https://github.com/Vanilagy/mediabunny/releases). + +## Usage + +```ts +import { registerProresDecoder } from '@mediabunny/prores'; + +registerProresDecoder(); +``` +That's it - Mediabunny now uses the registered ProRes decoder automatically. + +For all the ways of using Mediabunny, refer to its [guide](https://mediabunny.dev/guide/introduction). + +## Building and development + +The complete JavaScript package can be built alongside the rest of Mediabunny by running `npm run build` in Mediabunny's root. diff --git a/packages/prores/api-extractor.json b/packages/prores/api-extractor.json new file mode 100644 index 0000000..32669c1 --- /dev/null +++ b/packages/prores/api-extractor.json @@ -0,0 +1,37 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", + "mainEntryPointFilePath": "dist/modules/src/index.d.ts", + "bundledPackages": [], + "compiler": {}, + "apiReport": { + "enabled": false + }, + "docModel": { + "enabled": false + }, + "dtsRollup": { + "enabled": true, + "untrimmedFilePath": "dist/mediabunny-prores.d.ts" + }, + "tsdocMetadata": { + "enabled": false + }, + "messages": { + "compilerMessageReporting": { + "default": { + "logLevel": "warning" + } + }, + "extractorMessageReporting": { + "default": { + "logLevel": "warning" + } + }, + "tsdocMessageReporting": { + "default": { + "logLevel": "warning" + } + } + }, + "newlineKind": "lf" +} diff --git a/packages/prores-decoder/package.json b/packages/prores/package.json similarity index 50% rename from packages/prores-decoder/package.json rename to packages/prores/package.json index 1a4e350..1c7ef75 100644 --- a/packages/prores-decoder/package.json +++ b/packages/prores/package.json @@ -1,15 +1,15 @@ { - "name": "@mediabunny/prores-decoder", + "name": "@mediabunny/prores", "author": "Vanilagy", - "version": "1.25.8", - "description": "MP3 encoder extension for Mediabunny, based on LAME.", - "main": "./dist/bundles/mediabunny-mp3-encoder.mjs", - "module": "./dist/bundles/mediabunny-mp3-encoder.mjs", + "version": "1.49.0", + "description": "Apple ProRes decoder extension for Mediabunny, based on TurboRes.", + "main": "./dist/bundles/mediabunny-prores.mjs", + "module": "./dist/bundles/mediabunny-prores.mjs", "types": "./dist/modules/src/index.d.ts", "exports": { "types": "./dist/modules/src/index.d.ts", - "import": "./dist/bundles/mediabunny-mp3-encoder.mjs", - "require": "./dist/bundles/mediabunny-mp3-encoder.mjs" + "import": "./dist/bundles/mediabunny-prores.mjs", + "require": "./dist/bundles/mediabunny-prores.mjs" }, "files": [ "README.md", @@ -19,32 +19,38 @@ "src" ], "sideEffects": false, + "browser": { + "node:worker_threads": false, + "node:os": false + }, "license": "MPL-2.0", "repository": { "type": "git", "url": "git+https://github.com/Vanilagy/mediabunny.git", - "directory": "packages/mp3-encoder" + "directory": "packages/prores" }, "bugs": { "url": "https://github.com/Vanilagy/mediabunny/issues" }, - "homepage": "https://mediabunny.dev/guide/extensions/mp3-encoder", + "homepage": "https://mediabunny.dev/guide/extensions/prores", "funding": { "type": "individual", "url": "https://github.com/sponsors/Vanilagy" }, - "peerDependencies": { - "mediabunny": "^1.0.0" + "dependencies": { + "turbores": "^1.0.0" }, - "devDependencies": { - "@types/emscripten": "^1.40.1" + "peerDependencies": { + "mediabunny": "^1.49.0" }, "keywords": [ - "mp3", - "encoding", + "prores", + "apple-prores", + "decoding", "codec", "mediabunny", - "lame", + "turbores", + "video", "browser", "wasm", "polyfill" diff --git a/packages/prores/src/index.ts b/packages/prores/src/index.ts new file mode 100644 index 0000000..381ec9d --- /dev/null +++ b/packages/prores/src/index.ts @@ -0,0 +1,212 @@ +/*! + * Copyright (c) 2026-present, Vanilagy and contributors + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +import { CustomVideoDecoder, EncodedPacket, Logging, registerDecoder, VideoCodec, VideoSample } from 'mediabunny'; +import { Decoder, Frame, PixelFormat, PIXEL_FORMATS, FilledFrame } from 'turbores'; +import { + assert, + isWebKit, +} from '../../../src/misc'; +import { type ProresFourCc } from '../../../src/codec'; + +const PRORES_LOADED_SYMBOL = Symbol.for('@mediabunny/prores loaded'); +if ((globalThis as Record)[PRORES_LOADED_SYMBOL]) { + Logging._error( + '[WARNING]\n@mediabunny/prores was loaded twice.' + + ' This will likely cause the decoder not to work correctly.' + + ' Check if multiple dependencies are importing different versions of @mediabunny/prores,' + + ' or if something is being bundled incorrectly.', + ); +} +(globalThis as Record)[PRORES_LOADED_SYMBOL] = true; + +class ProresDecoder extends CustomVideoDecoder { + private decoder: Decoder | null = null; + private framePool: Frame[] = []; + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + static override supports(codec: VideoCodec, config: VideoDecoderConfig): boolean { + return codec === 'prores'; + } + + /** @internal */ + static _supportedVideoFrameFormats: PixelFormat[] | null = null; + + /** @internal */ + static _determineSupportedVideoFrameFormats() { + const result: PixelFormat[] = []; + const data = new Uint8Array(32); + + for (const format of PIXEL_FORMATS) { + try { + const frame = new VideoFrame(data, { + format: format as VideoPixelFormat, + codedWidth: 2, + codedHeight: 2, + timestamp: 0, + duration: 0, + }); + frame.close(); + + result.push(format); + } catch { + // Format is not supported + } + } + + return result; + } + + async init() { + if (typeof VideoFrame !== 'undefined') { + // Not all VideoFrame implementations support all pixel formats, therefore let's determine the supported set + ProresDecoder._supportedVideoFrameFormats ??= ProresDecoder._determineSupportedVideoFrameFormats(); + } + + const decoder = await Decoder.create({ + proresFourCc: this.config.codec as ProresFourCc, + useSharedMemory: Decoder.canUseSharedMemory(), + allowedOutputFormats: ProresDecoder._supportedVideoFrameFormats ?? undefined, + }); + if (decoder instanceof Error) { + throw decoder; + } + + this.decoder = decoder; + } + + async decode(packet: EncodedPacket) { + assert(this.decoder); + + if (this.decoder.useSharedMemory) { + await this.runDecode(packet); + } else { + while (this.decoder.decodeQueueSize >= this.decoder.concurrency) { + await this.decoder.dequeued; + } + + void this.runDecode(packet) + .catch(error => this.onError(error)); + } + } + + private async runDecode(packet: EncodedPacket) { + assert(this.decoder); + + let frame: Frame; + if (this.framePool.length > 0) { + frame = this.framePool.shift()!; + } else { + frame = new Frame(); + } + + const result = await this.decoder.decode(packet.data, frame); + this.framePool.push(frame); + + if (result instanceof Error) { + throw result; + } + + if (result.visibleHeight < result.codedHeight && isWebKit()) { + // WebKit has (had) a bug with displaying height-trimmed YUV frames, so we must compact the frame data a + // little https://bugs.webkit.org/show_bug.cgi?id=317524 + this.trimCodedHeightToVisibleHeight(result); + } + + const sample = new VideoSample(result.frameData, { + format: result.pixelFormat, + codedWidth: result.codedWidth, + codedHeight: result.codedHeight, + visibleRect: { + left: 0, + top: 0, + width: result.visibleWidth, + height: result.visibleHeight, + }, + timestamp: packet.timestamp, + duration: packet.duration, + colorSpace: { + primaries: result.colorPrimariesString as VideoColorPrimaries | undefined, + matrix: result.colorMatrixString as VideoMatrixCoefficients | undefined, + transfer: result.colorTransferString as VideoTransferCharacteristics | undefined, + fullRange: result.colorRangeFull, + }, + }); + this.onSample(sample); + } + + private trimCodedHeightToVisibleHeight(result: FilledFrame) { + const bytesPerSample = result.pixelFormat.includes('P') ? 2 : 1; + const subWidth = result.pixelFormat.includes('444') ? 1 : 2; + const subHeight = result.pixelFormat.includes('420') ? 2 : 1; + + const chromaCodedWidth = result.codedWidth / subWidth; + const chromaCodedHeight = result.codedHeight / subHeight; + const chromaVisibleHeight = Math.ceil(result.visibleHeight / subHeight); + + const lumaCodedPixels = result.codedWidth * result.codedHeight; + const lumaVisiblePixels = result.codedWidth * result.visibleHeight; + const chromaCodedPixels = chromaCodedWidth * chromaCodedHeight; + const chromaVisiblePixels = chromaCodedWidth * chromaVisibleHeight; + + // U + result.frameData.set( + result.frameData.subarray( + bytesPerSample * lumaCodedPixels, + bytesPerSample * (lumaCodedPixels + chromaCodedPixels), + ), + bytesPerSample * lumaVisiblePixels, + ); + // V + result.frameData.set( + result.frameData.subarray( + bytesPerSample * (lumaCodedPixels + chromaCodedPixels), + bytesPerSample * (lumaCodedPixels + 2 * chromaCodedPixels), + ), + bytesPerSample * (lumaVisiblePixels + chromaVisiblePixels), + ); + + result.codedHeight = result.visibleHeight; + } + + async flush() { + assert(this.decoder); + + while (this.decoder.decodeQueueSize > 0) { + await this.decoder.dequeued; + } + } + + async close() { + assert(this.decoder); + + await this.decoder.close(); + + for (const frame of this.framePool) { + frame.clear(); + } + } +} + +let registered = false; + +/** + * Registers an Apple ProRes decoder which Mediabunny will then use automatically when applicable. Make sure to call + * this function before starting any decoding task. + * + * @group \@mediabunny/prores + * @public + */ +export const registerProresDecoder = () => { + if (registered) { + return; + } + registered = true; + + registerDecoder(ProresDecoder); +}; diff --git a/packages/prores-decoder/src/tsconfig.json b/packages/prores/tsconfig.json similarity index 63% rename from packages/prores-decoder/src/tsconfig.json rename to packages/prores/tsconfig.json index 425e6bc..6a0d092 100644 --- a/packages/prores-decoder/src/tsconfig.json +++ b/packages/prores/tsconfig.json @@ -1,7 +1,7 @@ { - "extends": "../../../tsconfig.json", + "extends": "../../tsconfig.json", "compilerOptions": { - "outDir": "../dist/modules", + "outDir": "./dist/modules", "declaration": true, "declarationMap": true, "stripInternal": true, @@ -11,13 +11,13 @@ "module": "nodenext", "allowJs": true, "paths": { - "mediabunny": ["../../../src/index.ts"], + "mediabunny": ["../../src/index.ts"], }, }, "include": [ - "**/*", + "./src/**/*", ], "references": [ - { "path": "../../../src" } + { "path": "../../src" } ] } \ No newline at end of file diff --git a/packages/prores/tsdoc.json b/packages/prores/tsdoc.json new file mode 100644 index 0000000..51d29d3 --- /dev/null +++ b/packages/prores/tsdoc.json @@ -0,0 +1,4 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/tsdoc/v0/tsdoc.schema.json", + "extends": ["../../tsdoc.json"] +} diff --git a/packages/server/src/video-encoder.ts b/packages/server/src/video-encoder.ts index 7c7c4f3..1152f1b 100644 --- a/packages/server/src/video-encoder.ts +++ b/packages/server/src/video-encoder.ts @@ -375,6 +375,7 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder { hevcCodecInfo: null, vp9CodecInfo: null, av1CodecInfo: null, + proresFormat: null, }); if (!expectsAnnexB) { @@ -450,6 +451,7 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder { hevcCodecInfo: null, vp9CodecInfo: null, av1CodecInfo: null, + proresFormat: null, }); } } else if (this.codec === 'vp9') { @@ -467,6 +469,7 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder { hevcCodecInfo: null, vp9CodecInfo, av1CodecInfo: null, + proresFormat: null, }); } } else if (this.codec === 'av1') { @@ -484,6 +487,7 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder { hevcCodecInfo: null, vp9CodecInfo: null, av1CodecInfo, + proresFormat: null, }); } } else { diff --git a/scripts/build.sh b/scripts/build.sh index f33e33a..0ffad0a 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -9,6 +9,7 @@ rm -rf packages/mp3-encoder/dist rm -rf packages/ac3/dist rm -rf packages/aac-encoder/dist rm -rf packages/flac-encoder/dist +rm -rf packages/prores/dist rm -rf packages/server/dist # Ensure license headers on all source files @@ -20,6 +21,7 @@ tsc -p packages/mp3-encoder tsc -p packages/ac3 tsc -p packages/aac-encoder tsc -p packages/flac-encoder +tsc -p packages/prores tsc -p packages/server # Generate the root again, now with internals properly stripped @@ -39,6 +41,7 @@ api-extractor run -c packages/mp3-encoder/api-extractor.json api-extractor run -c packages/ac3/api-extractor.json api-extractor run -c packages/aac-encoder/api-extractor.json api-extractor run -c packages/flac-encoder/api-extractor.json +api-extractor run -c packages/prores/api-extractor.json api-extractor run -c packages/server/api-extractor.json # Checks that all symbols are documented @@ -47,6 +50,7 @@ tsx scripts/check-docblocks.ts packages/mp3-encoder/dist/mediabunny-mp3-encoder. tsx scripts/check-docblocks.ts packages/ac3/dist/mediabunny-ac3.d.ts tsx scripts/check-docblocks.ts packages/aac-encoder/dist/mediabunny-aac-encoder.d.ts tsx scripts/check-docblocks.ts packages/flac-encoder/dist/mediabunny-flac-encoder.d.ts +tsx scripts/check-docblocks.ts packages/prores/dist/mediabunny-prores.d.ts tsx scripts/check-docblocks.ts packages/server/dist/mediabunny-server.d.ts # Checks that API docs are generatable @@ -58,4 +62,5 @@ echo 'export as namespace MediabunnyMp3Encoder;' >> packages/mp3-encoder/dist/me echo 'export as namespace MediabunnyAc3;' >> packages/ac3/dist/mediabunny-ac3.d.ts echo 'export as namespace MediabunnyAacEncoder;' >> packages/aac-encoder/dist/mediabunny-aac-encoder.d.ts echo 'export as namespace MediabunnyFlacEncoder;' >> packages/flac-encoder/dist/mediabunny-flac-encoder.d.ts +echo 'export as namespace MediabunnyProres;' >> packages/prores/dist/mediabunny-prores.d.ts echo 'export as namespace MediabunnyServer;' >> packages/server/dist/mediabunny-server.d.ts diff --git a/scripts/bundle.ts b/scripts/bundle.ts index 0d965eb..f9df856 100644 --- a/scripts/bundle.ts +++ b/scripts/bundle.ts @@ -223,6 +223,24 @@ const flacEncoderVariants = await createVariants( }, ); +const proresVariants = await createVariants( + 'packages/prores/src/index.ts', + 'MediabunnyProres', + 'packages/prores/dist/bundles/mediabunny-prores', + 'js', // The bundles are purely for the browser, not for Node (due to the peer dependecy) + { + plugins: [ + PluginExternalGlobal.externalGlobalPlugin({ + mediabunny: 'Mediabunny', + }), + ], + }, + { + external: ['mediabunny'], + platform: 'node', // To retain the Node imports + }, +); + const serverVariants = await createVariants( 'packages/server/src/index.ts', 'MediabunnyServer', @@ -246,6 +264,7 @@ const contexts = [ ...ac3Variants, ...aacEncoderVariants, ...flacEncoderVariants, + ...proresVariants, ...serverVariants, ]; diff --git a/scripts/check.sh b/scripts/check.sh index bb28006..ed2d185 100755 --- a/scripts/check.sh +++ b/scripts/check.sh @@ -16,6 +16,9 @@ tsc -p packages/aac-encoder rm -rf packages/flac-encoder/dist/modules tsc -p packages/flac-encoder +rm -rf packages/prores/dist/modules +tsc -p packages/prores + rm -rf packages/server/dist/modules tsc -p packages/server diff --git a/scripts/ensure-license-headers.ts b/scripts/ensure-license-headers.ts index 91921dc..44b2dbe 100644 --- a/scripts/ensure-license-headers.ts +++ b/scripts/ensure-license-headers.ts @@ -40,6 +40,7 @@ checkDirectory(path.join(__dirname, '..', 'packages', 'mp3-encoder', 'src')); checkDirectory(path.join(__dirname, '..', 'packages', 'ac3', 'src')); checkDirectory(path.join(__dirname, '..', 'packages', 'flac-encoder', 'src')); checkDirectory(path.join(__dirname, '..', 'packages', 'aac-encoder', 'src')); +checkDirectory(path.join(__dirname, '..', 'packages', 'prores', 'src')); checkDirectory(path.join(__dirname, '..', 'packages', 'server', 'src')); if (missingFiles.length > 0) { diff --git a/src/codec.ts b/src/codec.ts index 8e5bc55..ff72494 100644 --- a/src/codec.ts +++ b/src/codec.ts @@ -224,7 +224,8 @@ export const PRORES_FOURCCS = [ 'apcn', // ProRes 422 Standard Definition 'apcs', // ProRes 422 LT 'apco', // ProRes 422 Proxy -]; +] as const; +export type ProresFourCc = typeof PRORES_FOURCCS[number]; export const buildVideoCodecString = (codec: VideoCodec, width: number, height: number, bitrate: number) => { if (codec === 'avc') { @@ -359,7 +360,7 @@ export const extractVideoCodecString = (trackInfo: { hevcCodecInfo: HevcDecoderConfigurationRecord | null; vp9CodecInfo: Vp9CodecInfo | null; av1CodecInfo: Av1CodecInfo | null; - proresFormat: string | null; + proresFormat: ProresFourCc | null; }) => { const { codec, @@ -946,7 +947,7 @@ export const validateVideoChunkMetadata = (metadata: EncodedVideoChunkMetadata | if (!PRORES_FOURCCS.some(x => metadata.decoderConfig!.codec === x)) { throw new TypeError( - 'Video chunk metadata decoder configuration codec string for ProRes must be one of the valid ProRes' + `Video chunk metadata decoder configuration codec string for ProRes must be one of the valid ProRes` + ` four-character codes: ${PRORES_FOURCCS.join(', ')}.`, ); } diff --git a/src/custom-coder.ts b/src/custom-coder.ts index 0152876..4488a39 100644 --- a/src/custom-coder.ts +++ b/src/custom-coder.ts @@ -27,6 +27,8 @@ export abstract class CustomVideoDecoder { readonly config!: VideoDecoderConfig; /** The callback to call when a decoded VideoSample is available. */ readonly onSample!: (sample: VideoSample) => unknown; + /** The callback to call to surface out-of-band errors that can't be surfaced through the main methods. */ + readonly onError!: (error: unknown) => undefined; /** Returns true if and only if the decoder can decode the given codec configuration. */ // eslint-disable-next-line @typescript-eslint/no-unused-vars @@ -57,6 +59,8 @@ export abstract class CustomAudioDecoder { readonly config!: AudioDecoderConfig; /** The callback to call when a decoded AudioSample is available. */ readonly onSample!: (sample: AudioSample) => unknown; + /** The callback to call to surface out-of-band errors that can't be surfaced through the main methods. */ + readonly onError!: (error: unknown) => undefined; /** Returns true if and only if the decoder can decode the given codec configuration. */ // eslint-disable-next-line @typescript-eslint/no-unused-vars @@ -87,6 +91,8 @@ export abstract class CustomVideoEncoder { readonly config!: VideoEncoderConfig; /** The callback to call when an EncodedPacket is available. */ readonly onPacket!: (packet: EncodedPacket, meta?: EncodedVideoChunkMetadata) => unknown; + /** The callback to call to surface out-of-band errors that can't be surfaced through the main methods. */ + readonly onError!: (error: unknown) => undefined; /** Returns true if and only if the encoder can encode the given codec configuration. */ // eslint-disable-next-line @typescript-eslint/no-unused-vars @@ -117,6 +123,8 @@ export abstract class CustomAudioEncoder { readonly config!: AudioEncoderConfig; /** The callback to call when an EncodedPacket is available. */ readonly onPacket!: (packet: EncodedPacket, meta?: EncodedAudioChunkMetadata) => unknown; + /** The callback to call to surface out-of-band errors that can't be surfaced through the main methods. */ + readonly onError!: (error: unknown) => undefined; /** Returns true if and only if the encoder can encode the given codec configuration. */ // eslint-disable-next-line @typescript-eslint/no-unused-vars diff --git a/src/encode.ts b/src/encode.ts index 4f52495..b13991b 100644 --- a/src/encode.ts +++ b/src/encode.ts @@ -549,20 +549,20 @@ export class Quality { /** @internal */ _toVideoBitrate(codec: VideoCodec, width: number, height: number) { const pixels = width * height; + const referencePixels = 1920 * 1080; + const referenceBitrate = 3_000_000; + const scaleFactor = Math.pow(pixels / referencePixels, 0.95); // Slight non-linear scaling + const baseBitrate = referenceBitrate * scaleFactor; - const codecEfficiencyFactors = { + const codecEfficiencyFactors: Record = { avc: 1.0, // H.264/AVC (baseline) hevc: 0.6, // H.265/HEVC (~40% more efficient than AVC) vp9: 0.6, // Similar to HEVC av1: 0.4, // ~60% more efficient than AVC vp8: 1.2, // Slightly less efficient than AVC + prores: 220_000_000 / referenceBitrate, // Apple ProRes white paper claims 220 Mbps for 1080p 422 HQ @30Hz }; - const referencePixels = 1920 * 1080; - const referenceBitrate = 3000000; - const scaleFactor = Math.pow(pixels / referencePixels, 0.95); // Slight non-linear scaling - const baseBitrate = referenceBitrate * scaleFactor; - const codecAdjustedBitrate = baseBitrate * codecEfficiencyFactors[codec]; const finalBitrate = codecAdjustedBitrate * this._factor; diff --git a/src/isobmff/isobmff-demuxer.ts b/src/isobmff/isobmff-demuxer.ts index a6f5a87..2d96f9d 100644 --- a/src/isobmff/isobmff-demuxer.ts +++ b/src/isobmff/isobmff-demuxer.ts @@ -19,6 +19,7 @@ import { PCM_AUDIO_CODECS, PcmAudioCodec, PRORES_FOURCCS, + ProresFourCc, VideoCodec, } from '../codec'; import { @@ -148,7 +149,7 @@ type InternalTrack = { hevcCodecInfo: HevcDecoderConfigurationRecord | null; vp9CodecInfo: Vp9CodecInfo | null; av1CodecInfo: Av1CodecInfo | null; - proresFormat: string | null; + proresFormat: ProresFourCc | null; }; } | { info: { @@ -1096,9 +1097,9 @@ export class IsobmffDemuxer extends Demuxer { track.info.codec = 'vp9'; } else if (codecName === 'av01') { track.info.codec = 'av1'; - } else if (PRORES_FOURCCS.includes(lowercaseBoxName)) { + } else if ((PRORES_FOURCCS as readonly string[]).includes(lowercaseBoxName)) { track.info.codec = 'prores'; - track.info.proresFormat = lowercaseBoxName; + track.info.proresFormat = lowercaseBoxName as ProresFourCc; } else if (codecName === null) { Logging._warn(`Unknown encrypted video codec due to missing frma box.`); } else { @@ -3347,7 +3348,10 @@ class IsobmffVideoTrackBacking extends IsobmffTrackBacking implements InputVideo } async canBeTransparent() { - return false; + return this.internalTrack.info.codec === 'prores' && ( + this.internalTrack.info.proresFormat === 'ap4h' + || this.internalTrack.info.proresFormat === 'ap4x' + ); } async getDecoderConfig(): Promise { diff --git a/src/matroska/matroska-demuxer.ts b/src/matroska/matroska-demuxer.ts index 9741da9..27ce186 100644 --- a/src/matroska/matroska-demuxer.ts +++ b/src/matroska/matroska-demuxer.ts @@ -21,6 +21,7 @@ import { MediaCodec, OPUS_SAMPLE_RATE, PRORES_FOURCCS, + ProresFourCc, VideoCodec, } from '../codec'; import { Demuxer } from '../demuxer'; @@ -216,6 +217,7 @@ type InternalTrack = { codecDescription: Uint8Array | null; colorSpace: VideoColorSpaceInit | null; alphaMode: boolean; + proresFormat: ProresFourCc | null; } | { type: 'audio'; @@ -1084,10 +1086,12 @@ export class MatroskaDemuxer extends Demuxer { ? textDecoder.decode(this.currentTrack.codecPrivate) : ''; - if (PRORES_FOURCCS.includes(format)) { + if ((PRORES_FOURCCS as readonly string[]).includes(format)) { this.currentTrack.info.codec = 'prores'; + this.currentTrack.info.proresFormat = format as ProresFourCc; } else { - // We don't support ProRes RAW yet + // Either an invalid string or ProRes RAW, which we don't support yet (it's a + // different codec). } } @@ -1182,6 +1186,7 @@ export class MatroskaDemuxer extends Demuxer { codecDescription: null, colorSpace: null, alphaMode: false, + proresFormat: null, }; } else if (type === 2) { this.currentTrack.info = { @@ -2447,7 +2452,12 @@ class MatroskaVideoTrackBacking extends MatroskaTrackBacking implements InputVid } async canBeTransparent() { - return this.internalTrack.info.alphaMode; + return this.internalTrack.info.alphaMode || ( + this.internalTrack.info.codec === 'prores' && ( + this.internalTrack.info.proresFormat === 'ap4h' + || this.internalTrack.info.proresFormat === 'ap4x' + ) + ); } async getDecoderConfig(): Promise { @@ -2489,9 +2499,7 @@ class MatroskaVideoTrackBacking extends MatroskaTrackBacking implements InputVid av1CodecInfo: this.internalTrack.info.codec === 'av1' && firstPacket ? extractAv1CodecInfoFromPacket(firstPacket.data) : null, - proresFormat: this.internalTrack.info.codec === 'prores' && this.internalTrack.codecPrivate - ? textDecoder.decode(this.internalTrack.codecPrivate) - : null, + proresFormat: this.internalTrack.info.proresFormat, }), codedWidth: this.internalTrack.info.width, codedHeight: this.internalTrack.info.height, diff --git a/src/media-sink.ts b/src/media-sink.ts index 72466ea..3871c81 100644 --- a/src/media-sink.ts +++ b/src/media-sink.ts @@ -333,7 +333,8 @@ export class EncodedPacketSink { // This stores errors that are "out of band" in the sense that they didn't occur in the normal flow of this // method but instead in a different context. This error should not go unnoticed and must be bubbled up to // the consumer. - let outOfBandError = null as Error | null; + let outOfBandError = null as unknown; + let hasOutOfBandError = false; const timestamps: number[] = []; // The queue should always be big enough to hold 1 second worth of packets @@ -364,9 +365,10 @@ export class EncodedPacketSink { ended = true; onQueueNotEmpty(); - })().catch((error: Error) => { - if (!outOfBandError) { + })().catch((error) => { + if (!hasOutOfBandError) { outOfBandError = error; + hasOutOfBandError = true; onQueueNotEmpty(); } }); @@ -380,7 +382,7 @@ export class EncodedPacketSink { throw new InputDisposedError(); } else if (terminated) { return { value: undefined, done: true }; - } else if (outOfBandError) { + } else if (hasOutOfBandError) { throw outOfBandError; } else if (packetQueue.length > 0) { const value = packetQueue.shift()!; @@ -423,7 +425,7 @@ abstract class DecoderWrapper< > { constructor( public onSample: (sample: MediaSample) => unknown, - public onError: (error: Error) => unknown, + public onError: (error: unknown) => unknown, ) {} abstract getDecodeQueueSize(): number; @@ -446,7 +448,7 @@ export abstract class BaseMediaSampleSink< /** @internal */ abstract _createDecoder( onSample: (sample: MediaSample) => unknown, - onError: (error: Error) => unknown + onError: (error: unknown) => unknown ): Promise>; /** @internal */ abstract _createPacketSink(): EncodedPacketSink; @@ -472,7 +474,8 @@ export abstract class BaseMediaSampleSink< // This stores errors that are "out of band" in the sense that they didn't occur in the normal flow of this // method but instead in a different context. This error should not go unnoticed and must be bubbled up to // the consumer. - let outOfBandError = null as Error | null; + let outOfBandError = null as unknown; + let hasOutOfBandError = false; const packetRetrievalOptions: PacketRetrievalOptions = { ...options, @@ -518,8 +521,9 @@ export abstract class BaseMediaSampleSink< ({ promise: queueNotEmpty, resolve: onQueueNotEmpty } = promiseWithResolvers()); } }, (error) => { - if (!outOfBandError) { + if (!hasOutOfBandError) { outOfBandError = error; + hasOutOfBandError = true; onQueueNotEmpty(); } }); @@ -572,9 +576,10 @@ export abstract class BaseMediaSampleSink< decoderIsFlushed = true; onQueueNotEmpty(); // To unstuck the generator - })().catch((error: Error) => { - if (!outOfBandError) { + })().catch((error) => { + if (!hasOutOfBandError) { outOfBandError = error; + hasOutOfBandError = true; onQueueNotEmpty(); } }); @@ -595,7 +600,7 @@ export abstract class BaseMediaSampleSink< throw new InputDisposedError(); } else if (terminated) { return { value: undefined, done: true }; - } else if (outOfBandError) { + } else if (hasOutOfBandError) { closeSamples(); throw outOfBandError; } else if (sampleQueue.length > 0) { @@ -645,7 +650,8 @@ export abstract class BaseMediaSampleSink< // This stores errors that are "out of band" in the sense that they didn't occur in the normal flow of this // method but instead in a different context. This error should not go unnoticed and must be bubbled up to // the consumer. - let outOfBandError = null as Error | null; + let outOfBandError = null as unknown; + let hasOutOfBandError = false; const pushToQueue = (sample: MediaSample | null) => { sampleQueue.push(sample); @@ -687,8 +693,9 @@ export abstract class BaseMediaSampleSink< sample.close(); } }, (error) => { - if (!outOfBandError) { + if (!hasOutOfBandError) { outOfBandError = error; + hasOutOfBandError = true; onQueueNotEmpty(); } }); @@ -792,9 +799,10 @@ export abstract class BaseMediaSampleSink< decoderIsFlushed = true; onQueueNotEmpty(); // To unstuck the generator - })().catch((error: Error) => { - if (!outOfBandError) { + })().catch((error) => { + if (!hasOutOfBandError) { outOfBandError = error; + hasOutOfBandError = true; onQueueNotEmpty(); } }); @@ -814,7 +822,7 @@ export abstract class BaseMediaSampleSink< throw new InputDisposedError(); } else if (terminated) { return { value: undefined, done: true }; - } else if (outOfBandError) { + } else if (hasOutOfBandError) { closeSamples(); throw outOfBandError; } else if (sampleQueue.length > 0) { @@ -882,7 +890,7 @@ class VideoDecoderWrapper extends DecoderWrapper { constructor( onSample: (sample: VideoSample) => unknown, - onError: (error: Error) => unknown, + onError: (error: unknown) => unknown, public codec: VideoCodec, public decoderConfig: VideoDecoderConfig, public rotation: Rotation, @@ -906,8 +914,14 @@ class VideoDecoderWrapper extends DecoderWrapper { this.finalizeAndEmitSample(sample); }; + // @ts-expect-error It's technically readonly + this.customDecoder.onError = (error) => { + onError(error); + }; - void this.customDecoderCallSerializer.call(() => this.customDecoder!.init()); + void this.customDecoderCallSerializer + .call(() => this.customDecoder!.init()) + .catch(error => onError(error)); } else { const colorHandler = (frame: VideoFrame) => { this.frameHandlerSerializer.call(async () => { @@ -920,7 +934,7 @@ class VideoDecoderWrapper extends DecoderWrapper { } else { this.colorQueue.push(frame); } - }).catch((error: Error) => this.onError(error)); + }).catch(error => this.onError(error)); }; if (codec === 'avc' && this.decoderConfig.description && isChromium()) { @@ -946,7 +960,7 @@ class VideoDecoderWrapper extends DecoderWrapper { try { colorHandler(frame); } catch (error) { - this.onError(error as Error); + this.onError(error); } }, error: (error) => { @@ -984,7 +998,8 @@ class VideoDecoderWrapper extends DecoderWrapper { this.customDecoderQueueSize++; void this.customDecoderCallSerializer .call(() => this.customDecoder!.decode(packet)) - .then(() => this.customDecoderQueueSize--); + .catch(error => this.onError(error)) + .finally(() => this.customDecoderQueueSize--); } else { assert(this.decoder); @@ -1074,7 +1089,7 @@ class VideoDecoderWrapper extends DecoderWrapper { } this.alphaDecoderQueueSize--; - }).catch((error: Error) => this.onError(error)); + }).catch(error => this.onError(error)); }; const stack = new Error('Decoding error').stack; @@ -1084,7 +1099,7 @@ class VideoDecoderWrapper extends DecoderWrapper { try { alphaHandler(frame); } catch (error) { - this.onError(error as Error); + this.onError(error); } }, error: (error) => { @@ -1813,7 +1828,7 @@ export class VideoSampleSink extends BaseMediaSampleSink { /** @internal */ async _createDecoder( onSample: (sample: VideoSample) => unknown, - onError: (error: Error) => unknown, + onError: (error: unknown) => unknown, ) { if (!(await this._track.canDecode())) { throw new Error( @@ -2215,7 +2230,7 @@ class AudioDecoderWrapper extends DecoderWrapper { constructor( onSample: (sample: AudioSample) => unknown, - onError: (error: Error) => unknown, + onError: (error: unknown) => unknown, codec: AudioCodec, decoderConfig: AudioDecoderConfig, ) { @@ -2271,8 +2286,14 @@ class AudioDecoderWrapper extends DecoderWrapper { sampleHandler(sample); }; + // @ts-expect-error It's technically readonly + this.customDecoder.onError = (error) => { + onError(error); + }; - void this.customDecoderCallSerializer.call(() => this.customDecoder!.init()); + void this.customDecoderCallSerializer + .call(() => this.customDecoder!.init()) + .catch(error => onError(error)); } else { const stack = new Error('Decoding error').stack; @@ -2281,7 +2302,7 @@ class AudioDecoderWrapper extends DecoderWrapper { try { sampleHandler(new AudioSample(data)); } catch (error) { - this.onError(error as Error); + this.onError(error); } }, error: (error) => { @@ -2307,7 +2328,8 @@ class AudioDecoderWrapper extends DecoderWrapper { this.customDecoderQueueSize++; void this.customDecoderCallSerializer .call(() => this.customDecoder!.decode(packet)) - .then(() => this.customDecoderQueueSize--); + .catch(error => this.onError(error)) + .finally(() => this.customDecoderQueueSize--); } else { assert(this.decoder); @@ -2357,7 +2379,7 @@ class PcmAudioDecoderWrapper extends DecoderWrapper { constructor( onSample: (sample: AudioSample) => unknown, - onError: (error: Error) => unknown, + onError: (error: unknown) => unknown, public decoderConfig: AudioDecoderConfig, ) { super(onSample, onError); @@ -2547,7 +2569,7 @@ export class AudioSampleSink extends BaseMediaSampleSink { /** @internal */ async _createDecoder( onSample: (sample: AudioSample) => unknown, - onError: (error: Error) => unknown, + onError: (error: unknown) => unknown, ) { if (!(await this._track.canDecode())) { throw new Error( diff --git a/src/media-source.ts b/src/media-source.ts index bb7ade9..dfe92f4 100644 --- a/src/media-source.ts +++ b/src/media-source.ts @@ -265,10 +265,18 @@ class VideoEncoderWrapper { * However, we want to surface these errors to the user within the normal control flow, so they don't go uncaught. * So, we keep track of the encoder error and throw it as soon as we get the chance. */ - private error: Error | null = null; - private closed = false; + private error: unknown = null; + private errorSet = false; + + private setError(error: unknown) { + if (!this.errorSet) { + this.error = error; + this.errorSet = true; + } + } private lastMuxerPromise: Promise = Promise.resolve(); + private closed = false; constructor(private source: VideoSource, private encodingConfig: VideoEncodingConfig) {} @@ -492,9 +500,9 @@ class VideoEncoderWrapper { const promise = this.customEncoderCallSerializer .call(() => this.customEncoder!.encode(clonedSample, finalEncodeOptions)) - .then(() => this.customEncoderQueueSize--) - .catch((error: Error) => this.error ??= error) + .catch((error: unknown) => this.setError(error)) .finally(() => { + this.customEncoderQueueSize--; clonedSample.close(); }); @@ -641,9 +649,13 @@ class VideoEncoderWrapper { this.lastMuxerPromise = this.muxer!.addEncodedVideoPacket(this.source._connectedTrack!, packet, meta) .catch((error) => { - this.error ??= error; + this.setError(error); }); }; + // @ts-expect-error It's technically readonly + this.customEncoder.onError = (error) => { + this.setError(error); + }; await this.customEncoder.init(); } else { @@ -746,7 +758,7 @@ class VideoEncoderWrapper { this.lastMuxerPromise = this.muxer!.addEncodedVideoPacket(this.source._connectedTrack!, packet, meta) .catch((error) => { - this.error ??= error; + this.setError(error); }); this.emittedEncoderPackets++; @@ -791,7 +803,7 @@ class VideoEncoderWrapper { }, error: (error) => { error.stack = stack; // Provide a more useful stack trace, the default one sucks - this.error ??= error; + this.setError(error); }, }); this.encoder.configure(encoderConfig); @@ -827,7 +839,7 @@ class VideoEncoderWrapper { }, error: (error) => { error.stack = stack; // Provide a more useful stack trace - this.error ??= error; + this.setError(error); }, }); this.alphaEncoder.configure(encoderConfig); @@ -899,7 +911,7 @@ class VideoEncoderWrapper { } checkForEncoderError() { - if (this.error) { + if (this.errorSet) { throw this.error; } } @@ -2043,7 +2055,16 @@ class AudioEncoderWrapper { * However, we want to surface these errors to the user within the normal control flow, so they don't go uncaught. * So, we keep track of the encoder error and throw it as soon as we get the chance. */ - private error: Error | null = null; + private error: unknown = null; + private errorSet = false; + + private setError(error: unknown) { + if (!this.errorSet) { + this.error = error; + this.errorSet = true; + } + } + private lastMuxerPromise: Promise = Promise.resolve(); private closed = false; @@ -2222,9 +2243,9 @@ class AudioEncoderWrapper { const promise = this.customEncoderCallSerializer .call(() => this.customEncoder!.encode(clonedSample)) - .then(() => this.customEncoderQueueSize--) - .catch((error: Error) => this.error ??= error) + .catch((error: unknown) => this.setError(error)) .finally(() => { + this.customEncoderQueueSize--; clonedSample.close(); }); @@ -2365,9 +2386,13 @@ class AudioEncoderWrapper { this.lastMuxerPromise = this.muxer!.addEncodedAudioPacket(this.source._connectedTrack!, packet, meta) .catch((error) => { - this.error ??= error; + this.setError(error); }); }; + // @ts-expect-error It's technically readonly + this.customEncoder.onError = (error) => { + this.setError(error); + }; await this.customEncoder.init(); } else if ((PCM_AUDIO_CODECS as readonly string[]).includes(this.encodingConfig.codec)) { @@ -2431,12 +2456,12 @@ class AudioEncoderWrapper { this.lastMuxerPromise = this.muxer!.addEncodedAudioPacket(this.source._connectedTrack!, packet, meta) .catch((error) => { - this.error ??= error; + this.setError(error); }); }, error: (error) => { error.stack = stack; // Provide a more useful stack trace - this.error ??= error; + this.setError(error); }, }); this.encoder.configure(encoderConfig); @@ -2587,7 +2612,7 @@ class AudioEncoderWrapper { } checkForEncoderError() { - if (this.error) { + if (this.errorSet) { throw this.error; } } @@ -2959,7 +2984,7 @@ type MediaStreamTrackProcessorWorkerMessage = { } | { type: 'error'; trackId: number; - error: Error; + error: unknown; }; type MediaStreamTrackProcessorControllerMessage = { @@ -3018,7 +3043,7 @@ const mediaStreamTrackProcessorWorkerCode = () => { processor.readable.pipeTo(consumer, { signal: abortController.signal, - }).catch((error: Error) => { + }).catch((error: unknown) => { // Handle AbortError silently if (error instanceof DOMException && error.name === 'AbortError') return; @@ -3138,7 +3163,9 @@ export class TextSubtitleSource extends SubtitleSource { /** @internal */ private _parser: SubtitleParser; /** @internal */ - private _error: Error | null = null; + private _error: unknown = null; + /** @internal */ + private _errorSet = false; /** @internal */ private _lastMuxerPromise: Promise = Promise.resolve(); @@ -3152,7 +3179,7 @@ export class TextSubtitleSource extends SubtitleSource { this._lastMuxerPromise = this._connectedTrack!.output._muxer.addSubtitleCue(this._connectedTrack!, cue, metadata) .catch((error) => { - this._error ??= error; + this._setError(error); }); }, }); @@ -3178,9 +3205,17 @@ export class TextSubtitleSource extends SubtitleSource { return this._lastMuxerPromise; // Allow the writer to apply backpressure } + /** @internal */ + private _setError(error: unknown) { + if (!this._errorSet) { + this._error = error; + this._errorSet = true; + } + } + /** @internal */ _checkForError() { - if (this._error) { + if (this._errorSet) { throw this._error; } } diff --git a/src/mpeg-ts/mpeg-ts-demuxer.ts b/src/mpeg-ts/mpeg-ts-demuxer.ts index f6c281f..f00e2a8 100644 --- a/src/mpeg-ts/mpeg-ts-demuxer.ts +++ b/src/mpeg-ts/mpeg-ts-demuxer.ts @@ -589,6 +589,7 @@ export class MpegTsDemuxer extends Demuxer { hevcCodecInfo: elementaryStream.info.hevcCodecInfo, vp9CodecInfo: null, av1CodecInfo: null, + proresFormat: null, }), codedWidth: elementaryStream.info.width, codedHeight: elementaryStream.info.height, diff --git a/src/sample.ts b/src/sample.ts index 19cd62b..5ea232a 100644 --- a/src/sample.ts +++ b/src/sample.ts @@ -423,16 +423,13 @@ export class VideoSample implements Disposable { ); } - this._data = init._doNotCopy - ? toUint8Array(data) - : toUint8Array(data).slice(); // Copy it - this._layout = init.layout ?? createDefaultPlaneLayout(init.format, init.codedWidth!, init.codedHeight!); - this.format = init.format; this.rotation = init.rotation ?? 0; this.timestamp = init.timestamp!; this.duration = init.duration ?? 0; + const layout = init.layout ?? createDefaultPlaneLayout(init.format, init.codedWidth!, init.codedHeight!); + let colorSpaceInit = init.colorSpace ?? null; if (colorSpaceInit === null) { if ( @@ -457,8 +454,6 @@ export class VideoSample implements Disposable { } } - this.colorSpace = new VideoSampleColorSpace(colorSpaceInit); - this.visibleRect = { left: init.visibleRect?.left ?? 0, top: init.visibleRect?.top ?? 0, @@ -473,6 +468,48 @@ export class VideoSample implements Disposable { this.squarePixelWidth = this.visibleRect.width; this.squarePixelHeight = this.visibleRect.height; } + + // If VideoFrame is available, route through it directly instead of holding onto the buffer. Going + // buffer -> VideoSample -> VideoFrame would copy the data twice (once here, once in toVideoFrame); + // building the VideoFrame now means it's only ever copied once. + if (typeof VideoFrame !== 'undefined' && !init._doNotCopy) { + let videoFrame: VideoFrame | null = null; + + try { + videoFrame = new VideoFrame(toUint8Array(data), { + format: init.format as VideoPixelFormat, + codedWidth: init.codedWidth!, + codedHeight: init.codedHeight!, + layout, + colorSpace: colorSpaceInit, + visibleRect: { + x: this.visibleRect.left, + y: this.visibleRect.top, + width: this.visibleRect.width, + height: this.visibleRect.height, + }, + displayWidth: this.squarePixelWidth, + displayHeight: this.squarePixelHeight, + timestamp: Math.trunc(init.timestamp! * SECOND_TO_MICROSECOND_FACTOR), + // Drag 0 to undefined + duration: Math.trunc((init.duration ?? 0) * SECOND_TO_MICROSECOND_FACTOR) || undefined, + }); + } catch { + // Ignore the error and move on like it didn't happen. We don't want errors caused by the local + // VideoFrame implementation to prevent us from creating the VideoSample. Errors, for example, could + // be caused by an unsupported pixel format. + } + + if (videoFrame) { + return new VideoSample(videoFrame, init); + } + } + + this._data = init._doNotCopy + ? toUint8Array(data) + : toUint8Array(data).slice(); // Copy it + this._layout = layout; + this.colorSpace = new VideoSampleColorSpace(colorSpaceInit); } else if (typeof VideoFrame !== 'undefined' && data instanceof VideoFrame) { if (init?.rotation !== undefined && ![0, 90, 180, 270].includes(init.rotation)) { throw new TypeError('init.rotation, when provided, must be 0, 90, 180, or 270.'); @@ -1036,6 +1073,7 @@ export class VideoSample implements Disposable { timestamp: this.microsecondTimestamp, duration: this.microsecondDuration, colorSpace: this.colorSpace, + visibleRect: this.visibleRect, displayWidth: this.squarePixelWidth, // Not display* since we're not passing rotation displayHeight: this.squarePixelHeight, }); @@ -1045,13 +1083,17 @@ export class VideoSample implements Disposable { duration: this.microsecondDuration || undefined, // Drag 0 duration to undefined, glitches some codecs }); } else if (this._data instanceof Uint8Array) { + assert(this._layout); + return new VideoFrame(this._data, { format: this.format! as VideoPixelFormat, - codedWidth: this.codedWidth, - codedHeight: this.codedHeight, + codedWidth: this.codedWidth, // This is technically wrong! codedWidth is a lie technically. But, since + codedHeight: this.codedHeight, // we pass the layout (which contains the true coded width), we're good. + layout: this._layout, timestamp: this.microsecondTimestamp, duration: this.microsecondDuration || undefined, colorSpace: this.colorSpace, + visibleRect: this.visibleRect, displayWidth: this.squarePixelWidth, // Not display* since we're not passing rotation displayHeight: this.squarePixelHeight, }); diff --git a/test/node/prores.test.ts b/test/node/prores.test.ts index 3fd9444..5342f15 100644 --- a/test/node/prores.test.ts +++ b/test/node/prores.test.ts @@ -1,4 +1,5 @@ import { expect, test } from 'vitest'; +import { registerProresDecoder } from '@mediabunny/prores'; import { Input } from '../../src/input.js'; import { BufferSource, UrlSource } from '../../src/source.js'; import { ALL_FORMATS } from '../../src/input-format.js'; @@ -7,28 +8,29 @@ import { MkvOutputFormat, MovOutputFormat } from '../../src/output-format.js'; import { BufferTarget } from '../../src/target.js'; import { Conversion } from '../../src/conversion.js'; import { VideoSampleSink } from '../../src/media-sink.js'; -import { CustomVideoDecoder, registerDecoder } from '../../src/custom-coder.js'; -import { ProResDecoder } from '../../packages/prores-decoder/src/index.js'; +import { assert } from '../../src/misc.js'; + +const SAMPLE_URL = 'https://pub-1ee78aacb848486482b20a72b55b3121.r2.dev/turbores-sample.mov'; test.concurrent('ProRes MOV file reading', async () => { using input = new Input({ - source: new UrlSource('https://pub-cf9fcfcb5c0a44e9b1bb5ff890e041ae.r2.dev/IMG_0158-prores-log.MOV'), + source: new UrlSource(SAMPLE_URL), formats: ALL_FORMATS, }); const videoTrack = (await input.getPrimaryVideoTrack())!; - expect(videoTrack.codec).toBe('prores'); - expect(videoTrack.codedWidth).toBe(1920); - expect(videoTrack.codedHeight).toBe(1080); + expect(await videoTrack.getCodec()).toBe('prores'); + expect(await videoTrack.getCodedWidth()).toBe(1920); + expect(await videoTrack.getCodedHeight()).toBe(1080); const decoderConfig = (await videoTrack.getDecoderConfig())!; expect(decoderConfig.codec).toBe('apch'); expect(decoderConfig.description).toBeUndefined(); }); -test.concurrent('ProRes transmuxing into MOV', { timeout: 60_000 }, async () => { +test.concurrent('ProRes transmuxing into MOV', { timeout: 10_000 }, async () => { using input = new Input({ - source: new UrlSource('https://pub-cf9fcfcb5c0a44e9b1bb5ff890e041ae.r2.dev/IMG_0158-prores-log.MOV'), + source: new UrlSource(SAMPLE_URL), formats: ALL_FORMATS, }); @@ -55,7 +57,7 @@ test.concurrent('ProRes transmuxing into MOV', { timeout: 60_000 }, async () => }); const videoTrack = (await newInput.getPrimaryVideoTrack())!; - expect(videoTrack.codec).toBe('prores'); + expect(await videoTrack.getCodec()).toBe('prores'); expect((await videoTrack.computePacketStats()).packetCount).toBe(15); const decoderConfig = (await videoTrack.getDecoderConfig())!; @@ -63,9 +65,9 @@ test.concurrent('ProRes transmuxing into MOV', { timeout: 60_000 }, async () => expect(decoderConfig.description).toBeUndefined(); }); -test.concurrent('ProRes transmuxing into MKV', { timeout: 60_000 }, async () => { +test.concurrent('ProRes transmuxing into MKV', { timeout: 10_000 }, async () => { using input = new Input({ - source: new UrlSource('https://pub-cf9fcfcb5c0a44e9b1bb5ff890e041ae.r2.dev/IMG_0158-prores-log.MOV'), + source: new UrlSource(SAMPLE_URL), formats: ALL_FORMATS, }); @@ -77,9 +79,6 @@ test.concurrent('ProRes transmuxing into MKV', { timeout: 60_000 }, async () => const conversion = await Conversion.init({ input, output, - video: { - rotate: 270, // To undo the source rotation metadata so the copy path is taken - }, audio: { discard: true, }, @@ -95,7 +94,7 @@ test.concurrent('ProRes transmuxing into MKV', { timeout: 60_000 }, async () => }); const videoTrack = (await newInput.getPrimaryVideoTrack())!; - expect(videoTrack.codec).toBe('prores'); + expect(await videoTrack.getCodec()).toBe('prores'); expect((await videoTrack.computePacketStats()).packetCount).toBe(15); const decoderConfig = (await videoTrack.getDecoderConfig())!; @@ -103,17 +102,51 @@ test.concurrent('ProRes transmuxing into MKV', { timeout: 60_000 }, async () => expect(decoderConfig.description).toBeUndefined(); }); -test.concurrent.only('ProRes sample ahh', async () => { - registerDecoder(ProResDecoder); - +test('Custom coder registration', { timeout: 10_000 }, async () => { using input = new Input({ - source: new UrlSource('https://pub-cf9fcfcb5c0a44e9b1bb5ff890e041ae.r2.dev/IMG_0158-prores-log.MOV'), + source: new UrlSource(SAMPLE_URL), formats: ALL_FORMATS, }); const videoTrack = (await input.getPrimaryVideoTrack())!; - const sink = new VideoSampleSink(videoTrack); - const sample = await sink.getSample(0); + expect(await videoTrack.getCodec()).toBe('prores'); - console.log(sample); + // Without a registered decoder, there's no way to decode ProRes in this environment + expect(await videoTrack.canDecode()).toBe(false); + + registerProresDecoder(); + + expect(await videoTrack.canDecode()).toBe(true); +}); + +test('ProRes decoding', { timeout: 10_000 }, async () => { + registerProresDecoder(); + + using input = new Input({ + source: new UrlSource(SAMPLE_URL), + formats: ALL_FORMATS, + }); + + const videoTrack = (await input.getPrimaryVideoTrack())!; + const firstTimestamp = await videoTrack.getFirstTimestamp(); + + const sink = new VideoSampleSink(videoTrack); + using sample = await sink.getSample(firstTimestamp); + assert(sample); + + expect(sample.timestamp).toBe(firstTimestamp); + expect(sample.duration).toBeGreaterThan(0); + + expect(sample.displayWidth).toBe(await videoTrack.getDisplayWidth()); + expect(sample.displayHeight).toBe(await videoTrack.getDisplayHeight()); + expect(sample.codedWidth).toBe(1920); + expect(sample.codedHeight).toBe(1080); // Technically a lie but the field is ill-defined + expect(sample.format).toBe('I422P10'); + + const allocationSize = sample.allocationSize(); + expect(allocationSize).toBeGreaterThan(0); + + const pixels = new Uint8Array(allocationSize); + await sample.copyTo(pixels); + expect(pixels.some(byte => byte !== 0)).toBe(true); }); diff --git a/todo.txt b/todo.txt deleted file mode 100644 index 65834c2..0000000 --- a/todo.txt +++ /dev/null @@ -1,2 +0,0 @@ -- different codec strings -- separate out prores and prores raw (?) ffmpeg does this, so probably good reasons. Matroska does not. \ No newline at end of file diff --git a/tsconfig.vite.json b/tsconfig.vite.json index ad7a720..b8dad8d 100644 --- a/tsconfig.vite.json +++ b/tsconfig.vite.json @@ -10,6 +10,7 @@ "@mediabunny/ac3": ["./packages/ac3/src/index.ts"], "@mediabunny/aac-encoder": ["./packages/aac-encoder/src/index.ts"], "@mediabunny/flac-encoder": ["./packages/flac-encoder/src/index.ts"], + "@mediabunny/prores": ["./packages/prores/src/index.ts"], }, "types": ["vite/client"] }, diff --git a/tsconfig.vitest.json b/tsconfig.vitest.json index d665820..9ef6c27 100644 --- a/tsconfig.vitest.json +++ b/tsconfig.vitest.json @@ -11,6 +11,7 @@ "@mediabunny/aac-encoder": ["./packages/aac-encoder/src/index.ts"], "@mediabunny/flac-encoder": ["./packages/flac-encoder/src/index.ts"], "@mediabunny/mp3-encoder": ["./packages/mp3-encoder/src/index.ts"], + "@mediabunny/prores": ["./packages/prores/src/index.ts"], "@mediabunny/server": ["./packages/server/src/index.ts"], }, }, @@ -22,6 +23,7 @@ { "path": "./packages/aac-encoder" }, { "path": "./packages/flac-encoder" }, { "path": "./packages/mp3-encoder" }, + { "path": "./packages/prores" }, { "path": "./packages/server" }, ] } diff --git a/vite.config.ts b/vite.config.ts index f1c5335..df28f91 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -25,6 +25,8 @@ export default defineConfig({ path.resolve(__dirname, './packages/aac-encoder/dist/bundles/mediabunny-aac-encoder.mjs'), '@mediabunny/flac-encoder': path.resolve(__dirname, './packages/flac-encoder/dist/bundles/mediabunny-flac-encoder.mjs'), + '@mediabunny/prores': + path.resolve(__dirname, './packages/prores/dist/bundles/mediabunny-prores.mjs'), }, }, plugins: [ @@ -33,6 +35,10 @@ export default defineConfig({ server: { hmr: false, allowedHosts: true, + headers: { + 'Cross-Origin-Opener-Policy': 'same-origin', + 'Cross-Origin-Embedder-Policy': 'require-corp', + }, }, build: { outDir: 'dist-docs', // Build them directly into the docs build folder diff --git a/vitest.config.ts b/vitest.config.ts index 811875f..5963bbb 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -14,6 +14,8 @@ export default defineConfig({ path.resolve(__dirname, './packages/flac-encoder/dist/bundles/mediabunny-flac-encoder.mjs'), '@mediabunny/mp3-encoder': path.resolve(__dirname, './packages/mp3-encoder/dist/bundles/mediabunny-mp3-encoder.mjs'), + '@mediabunny/prores': + path.resolve(__dirname, './packages/prores/dist/bundles/mediabunny-prores.mjs'), '@mediabunny/server': path.resolve(__dirname, './packages/server/src/index.ts'), },