diff --git a/examples/media-player/media-player.ts b/examples/media-player/media-player.ts index 868183c..f7c0080 100644 --- a/examples/media-player/media-player.ts +++ b/examples/media-player/media-player.ts @@ -4,6 +4,7 @@ import { BlobSource, CanvasSink, Input, + registerDecoder, UrlSource, WrappedAudioBuffer, WrappedCanvas, @@ -69,6 +70,9 @@ let volumeMuted = false; /** === INIT LOGIC === */ +import { ProResDecoder } from '../../packages/prores-decoder/src/index.js'; +registerDecoder(ProResDecoder); + const initMediaPlayer = async (resource: File | string) => { try { // First, dispose any ongoing playback: @@ -189,7 +193,7 @@ const initMediaPlayer = async (resource: File | string) => { if (audioContext.state === 'running') { // Start playback automatically if the audio context permits - await play(); + // await play(); } loadingElement.style.display = 'none'; diff --git a/package-lock.json b/package-lock.json index 1e7c162..2c60249 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1417,6 +1417,10 @@ "resolved": "packages/mp3-encoder", "link": true }, + "node_modules/@mediabunny/prores-decoder": { + "resolved": "packages/prores-decoder", + "link": true + }, "node_modules/@mermaid-js/mermaid-mindmap": { "version": "9.3.0", "resolved": "https://registry.npmjs.org/@mermaid-js/mermaid-mindmap/-/mermaid-mindmap-9.3.0.tgz", @@ -12077,6 +12081,21 @@ "peerDependencies": { "mediabunny": "^1.0.0" } + }, + "packages/prores-decoder": { + "name": "@mediabunny/prores-decoder", + "version": "1.25.8", + "license": "MPL-2.0", + "devDependencies": { + "@types/emscripten": "^1.40.1" + }, + "funding": { + "type": "individual", + "url": "https://github.com/sponsors/Vanilagy" + }, + "peerDependencies": { + "mediabunny": "^1.0.0" + } } } } diff --git a/packages/prores-decoder/README.md b/packages/prores-decoder/README.md new file mode 100644 index 0000000..275a5e8 --- /dev/null +++ b/packages/prores-decoder/README.md @@ -0,0 +1,46 @@ +```bash +emmake make distclean + +emconfigure ./configure \ + --target-os=none \ + --arch=x86_32 \ + --enable-cross-compile \ + --disable-asm \ + --disable-x86asm \ + --disable-inline-asm \ + --disable-stripping \ + --disable-programs \ + --disable-doc \ + --disable-debug \ + --disable-all \ + --disable-everything \ + --disable-pthreads \ + --enable-avcodec \ + --enable-decoder=prores \ + --cc="emcc" \ + --cxx=em++ \ + --ar=emar \ + --ranlib=emranlib \ + --extra-cflags="-DNDEBUG -O3 -msimd128" + +emmake make +``` + +```bash +export FFMPEG_PATH=/path/to/ffmpeg + +emcc src/bridge.c \ + $FFMPEG_PATH/libavcodec/libavcodec.a \ + $FFMPEG_PATH/libavutil/libavutil.a \ + -I$FFMPEG_PATH \ + -s MODULARIZE=1 \ + -s EXPORT_ES6=1 \ + -s SINGLE_FILE=1 \ + -s ALLOW_MEMORY_GROWTH=1 \ + -s ENVIRONMENT=web,worker \ + -s EXPORTED_RUNTIME_METHODS=cwrap,HEAPU8 \ + -s EXPORTED_FUNCTIONS=_malloc,_free \ + -msimd128 \ + -O3 \ + -o build/prores.js +``` \ No newline at end of file diff --git a/packages/prores-decoder/build/prores.js b/packages/prores-decoder/build/prores.js new file mode 100644 index 0000000..ea5911b Binary files /dev/null and b/packages/prores-decoder/build/prores.js differ diff --git a/packages/prores-decoder/package.json b/packages/prores-decoder/package.json new file mode 100644 index 0000000..1a4e350 --- /dev/null +++ b/packages/prores-decoder/package.json @@ -0,0 +1,52 @@ +{ + "name": "@mediabunny/prores-decoder", + "author": "Vanilagy", + "version": "1.25.8", + "description": "MP3 encoder extension for Mediabunny, based on LAME.", + "main": "./dist/bundles/mediabunny-mp3-encoder.mjs", + "module": "./dist/bundles/mediabunny-mp3-encoder.mjs", + "types": "./dist/modules/src/index.d.ts", + "exports": { + "types": "./dist/modules/src/index.d.ts", + "import": "./dist/bundles/mediabunny-mp3-encoder.mjs", + "require": "./dist/bundles/mediabunny-mp3-encoder.mjs" + }, + "files": [ + "README.md", + "package.json", + "LICENSE", + "dist", + "src" + ], + "sideEffects": false, + "license": "MPL-2.0", + "repository": { + "type": "git", + "url": "git+https://github.com/Vanilagy/mediabunny.git", + "directory": "packages/mp3-encoder" + }, + "bugs": { + "url": "https://github.com/Vanilagy/mediabunny/issues" + }, + "homepage": "https://mediabunny.dev/guide/extensions/mp3-encoder", + "funding": { + "type": "individual", + "url": "https://github.com/sponsors/Vanilagy" + }, + "peerDependencies": { + "mediabunny": "^1.0.0" + }, + "devDependencies": { + "@types/emscripten": "^1.40.1" + }, + "keywords": [ + "mp3", + "encoding", + "codec", + "mediabunny", + "lame", + "browser", + "wasm", + "polyfill" + ] +} diff --git a/packages/prores-decoder/src/bridge.c b/packages/prores-decoder/src/bridge.c new file mode 100644 index 0000000..e0976f0 --- /dev/null +++ b/packages/prores-decoder/src/bridge.c @@ -0,0 +1,182 @@ +#include +#include +#include "libavcodec/avcodec.h" +#include "libavutil/pixdesc.h" +#include "libavutil/imgutils.h" + +typedef struct { + AVCodecContext *codec_ctx; + AVPacket *packet; + AVFrame *frame; + uint8_t *buffer; + int buffer_size; +} DecoderContext; + +EMSCRIPTEN_KEEPALIVE +DecoderContext *init_decoder() { + const AVCodec *codec = avcodec_find_decoder(AV_CODEC_ID_PRORES); + if (!codec) return NULL; + + AVCodecContext *codec_ctx = avcodec_alloc_context3(codec); + if (!codec_ctx) return NULL; + + if (avcodec_open2(codec_ctx, codec, NULL) < 0) { + avcodec_free_context(&codec_ctx); + return NULL; + } + + AVPacket *packet = av_packet_alloc(); + if (!packet) { + avcodec_free_context(&codec_ctx); + return NULL; + } + + AVFrame *frame = av_frame_alloc(); + if (!frame) { + av_packet_free(&packet); + avcodec_free_context(&codec_ctx); + return NULL; + } + + DecoderContext *ctx = malloc(sizeof(DecoderContext)); + if (!ctx) { + av_frame_free(&frame); + av_packet_free(&packet); + avcodec_free_context(&codec_ctx); + return NULL; + } + + ctx->codec_ctx = codec_ctx; + ctx->packet = packet; + ctx->frame = frame; + ctx->buffer = NULL; + ctx->buffer_size = 0; + + return ctx; +} + +EMSCRIPTEN_KEEPALIVE +uint8_t *configure_packet(DecoderContext *ctx, int size) { + if (av_new_packet(ctx->packet, size) < 0) { + return NULL; + } + + ctx->packet->pts = 0; + ctx->packet->dts = AV_NOPTS_VALUE; + ctx->packet->time_base.num = 1; + ctx->packet->time_base.den = 1; + ctx->packet->flags = AV_PKT_FLAG_KEY; + + return ctx->packet->data; +} + +EMSCRIPTEN_KEEPALIVE +int decode_packet(DecoderContext *ctx) { + double start = emscripten_get_now(); + + int ret = avcodec_send_packet(ctx->codec_ctx, ctx->packet); + if (ret < 0) { + return ret; + } + + double after_send = emscripten_get_now(); + + ret = avcodec_receive_frame(ctx->codec_ctx, ctx->frame); + if (ret < 0) { + return ret; + } + + double after_receive = emscripten_get_now(); + + // Calculate required buffer size + int required_size = av_image_get_buffer_size( + ctx->frame->format, + ctx->frame->width, + ctx->frame->height, + 1 + ); + + if (required_size < 0) { + return required_size; + } + + // Reallocate buffer if needed + if (ctx->buffer_size < required_size) { + free(ctx->buffer); + ctx->buffer = malloc(required_size); + if (!ctx->buffer) { + ctx->buffer_size = 0; + return AVERROR(ENOMEM); + } + ctx->buffer_size = required_size; + } + + double after_alloc = emscripten_get_now(); + + // Copy frame data to contiguous buffer + ret = av_image_copy_to_buffer( + ctx->buffer, + ctx->buffer_size, + (const uint8_t * const *)ctx->frame->data, + ctx->frame->linesize, + ctx->frame->format, + ctx->frame->width, + ctx->frame->height, + 1 + ); + + if (ret < 0) { + return ret; + } + + double after_copy = emscripten_get_now(); + + printf("Timing: send=%.2fms receive=%.2fms alloc=%.2fms copy=%.2fms total=%.2fms\n", + after_send - start, + after_receive - after_send, + after_alloc - after_receive, + after_copy - after_alloc, + after_copy - start); + + return 0; +} + +EMSCRIPTEN_KEEPALIVE +int get_frame_width(DecoderContext *ctx) { + return ctx->frame->width; +} + +EMSCRIPTEN_KEEPALIVE +int get_frame_height(DecoderContext *ctx) { + return ctx->frame->height; +} + +EMSCRIPTEN_KEEPALIVE +int get_frame_format(DecoderContext *ctx) { + return ctx->frame->format; +} + +EMSCRIPTEN_KEEPALIVE +int get_frame_num_planes(DecoderContext *ctx) { + return av_pix_fmt_count_planes(ctx->frame->format); +} + +EMSCRIPTEN_KEEPALIVE +int get_frame_linesize(DecoderContext *ctx, int n) { + return ctx->frame->linesize[n]; +} + +EMSCRIPTEN_KEEPALIVE +uint8_t *get_frame_data(DecoderContext *ctx, int n) { + return ctx->frame->data[n]; +} + +EMSCRIPTEN_KEEPALIVE +uint8_t *get_frame_data_ptr(DecoderContext *ctx) { + return ctx->buffer; +} + +EMSCRIPTEN_KEEPALIVE +int get_frame_data_size(DecoderContext *ctx) { + return ctx->buffer_size; +} \ No newline at end of file diff --git a/packages/prores-decoder/src/index.ts b/packages/prores-decoder/src/index.ts new file mode 100644 index 0000000..e200812 --- /dev/null +++ b/packages/prores-decoder/src/index.ts @@ -0,0 +1,90 @@ +import { CustomVideoDecoder, EncodedPacket, MaybePromise, VideoCodec } from 'mediabunny'; +import createModule from '../build/prores'; +import { VideoSample } from 'mediabunny'; + +type ExtendedEmscriptenModule = EmscriptenModule & { + cwrap: typeof cwrap; +}; + +export class ProResDecoder extends CustomVideoDecoder { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + static override supports(codec: VideoCodec, config: VideoDecoderConfig): boolean { + return codec === 'prores'; + } + + module!: ExtendedEmscriptenModule; + initDecoder!: () => number; + configurePacket!: (ctx: number, size: number) => number; + decodePacket!: (ctx: number) => number; + getFrameWidth!: (ctx: number) => number; + getFrameHeight!: (ctx: number) => number; + getFrameFormat!: (ctx: number) => number; + getFrameNumPlanes!: (ctx: number) => number; + getFrameLinesize!: (ctx: number, n: number) => number; + getFrameData!: (ctx: number, n: number) => number; + getFrameDataPtr!: (ctx: number) => number; + getFrameDataSize!: (ctx: number) => number; + + decoderContextPtr!: number; + + override async init() { + console.log('THIS IS BEING CALLED'); + + this.module = (await createModule()) as ExtendedEmscriptenModule; + + // Set up the functions + this.initDecoder = this.module.cwrap('init_decoder', 'number', []); + this.configurePacket = this.module.cwrap('configure_packet', 'number', ['number', 'number']); + this.decodePacket = this.module.cwrap('decode_packet', 'number', ['number']); + this.getFrameWidth = this.module.cwrap('get_frame_width', 'number', ['number']); + this.getFrameHeight = this.module.cwrap('get_frame_height', 'number', ['number']); + this.getFrameFormat = this.module.cwrap('get_frame_format', 'number', ['number']); + this.getFrameNumPlanes = this.module.cwrap('get_frame_num_planes', 'number', ['number']); + this.getFrameLinesize = this.module.cwrap('get_frame_linesize', 'number', ['number', 'number']); + this.getFrameData = this.module.cwrap('get_frame_data', 'number', ['number', 'number']); + this.getFrameDataPtr = this.module.cwrap('get_frame_data_ptr', 'number', ['number']); + this.getFrameDataSize = this.module.cwrap('get_frame_data_size', 'number', ['number']); + + this.decoderContextPtr = this.initDecoder(); + } + + override decode(packet: EncodedPacket) { + const dataPtr = this.configurePacket(this.decoderContextPtr, packet.byteLength); + if (dataPtr === 0) { + throw new Error('todo'); + } + + this.module.HEAPU8.set(packet.data, dataPtr); + + console.time(); + const ret = this.decodePacket(this.decoderContextPtr); + console.timeEnd(); + + if (ret === 0) { + const width = this.getFrameWidth(this.decoderContextPtr); + const height = this.getFrameHeight(this.decoderContextPtr); + const format = this.getFrameFormat(this.decoderContextPtr); + const dataPtr = this.getFrameDataPtr(this.decoderContextPtr); + const dataSize = this.getFrameDataSize(this.decoderContextPtr); + + const data = this.module.HEAPU8.subarray(dataPtr, dataPtr + dataSize); + + const sample = new VideoSample(data, { + format: 'I422P10' as VideoPixelFormat, + codedWidth: width, + codedHeight: height, + timestamp: packet.timestamp, + duration: packet.duration, + }); + this.onSample(sample); + } + } + + override flush(): MaybePromise { + // nada + } + + override close(): MaybePromise { + // nada + } +} diff --git a/packages/prores-decoder/src/tsconfig.json b/packages/prores-decoder/src/tsconfig.json new file mode 100644 index 0000000..425e6bc --- /dev/null +++ b/packages/prores-decoder/src/tsconfig.json @@ -0,0 +1,23 @@ +{ + "extends": "../../../tsconfig.json", + "compilerOptions": { + "outDir": "../dist/modules", + "declaration": true, + "declarationMap": true, + "stripInternal": true, + "composite": true, + "noEmit": false, + "moduleResolution": "nodenext", + "module": "nodenext", + "allowJs": true, + "paths": { + "mediabunny": ["../../../src/index.ts"], + }, + }, + "include": [ + "**/*", + ], + "references": [ + { "path": "../../../src" } + ] +} \ No newline at end of file diff --git a/src/sample.ts b/src/sample.ts index fae15bf..25da586 100644 --- a/src/sample.ts +++ b/src/sample.ts @@ -805,6 +805,52 @@ export class VideoSample implements Disposable { } } +/** + * Describes the color space of a video frame. + * + * Corresponds to the WebCodecs VideoColorSpace API. + */ +export class VideoColorSpace { + /** + * The color primaries standard used. + */ + readonly primaries: VideoColorPrimaries | null; + /** + * The transfer characteristics used. + */ + readonly transfer: VideoTransferCharacteristics | null; + /** + * The color matrix coefficients used. + */ + readonly matrix: VideoMatrixCoefficients | null; + /** + * Whether the color values use the full range or limited range. + */ + readonly fullRange: boolean | null; + + /** + * Creates a new VideoColorSpace. + */ + constructor(init?: VideoColorSpaceInit) { + this.primaries = init?.primaries ?? null; + this.transfer = init?.transfer ?? null; + this.matrix = init?.matrix ?? null; + this.fullRange = init?.fullRange ?? null; + } + + /** + * Serializes the color space to a JSON object. + */ + toJSON() { + return { + primaries: this.primaries, + transfer: this.transfer, + matrix: this.matrix, + fullRange: this.fullRange, + }; + } +} + const isVideoFrame = (x: unknown): x is VideoFrame => { return typeof VideoFrame !== 'undefined' && x instanceof VideoFrame; }; diff --git a/test/node/prores.test.ts b/test/node/prores.test.ts index 642208c..3fd9444 100644 --- a/test/node/prores.test.ts +++ b/test/node/prores.test.ts @@ -6,6 +6,9 @@ import { Output } from '../../src/output.js'; import { MkvOutputFormat, MovOutputFormat } from '../../src/output-format.js'; import { BufferTarget } from '../../src/target.js'; import { Conversion } from '../../src/conversion.js'; +import { VideoSampleSink } from '../../src/media-sink.js'; +import { CustomVideoDecoder, registerDecoder } from '../../src/custom-coder.js'; +import { ProResDecoder } from '../../packages/prores-decoder/src/index.js'; test.concurrent('ProRes MOV file reading', async () => { using input = new Input({ @@ -99,3 +102,18 @@ test.concurrent('ProRes transmuxing into MKV', { timeout: 60_000 }, async () => expect(decoderConfig.codec).toBe('apch'); expect(decoderConfig.description).toBeUndefined(); }); + +test.concurrent.only('ProRes sample ahh', async () => { + registerDecoder(ProResDecoder); + + using input = new Input({ + source: new UrlSource('https://pub-cf9fcfcb5c0a44e9b1bb5ff890e041ae.r2.dev/IMG_0158-prores-log.MOV'), + formats: ALL_FORMATS, + }); + + const videoTrack = (await input.getPrimaryVideoTrack())!; + const sink = new VideoSampleSink(videoTrack); + const sample = await sink.getSample(0); + + console.log(sample); +}); diff --git a/tsconfig.vitest.json b/tsconfig.vitest.json index f9a4276..d217181 100644 --- a/tsconfig.vitest.json +++ b/tsconfig.vitest.json @@ -7,5 +7,5 @@ "noEmit": false }, "include": ["vitest.config.ts", "./test/**/*"], - "references": [{ "path": "./src" }] + "references": [{ "path": "./src" }, { "path": "./packages/prores-decoder/src" }] } diff --git a/vitest.config.ts b/vitest.config.ts index 33a53df..2b0ee00 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -1,11 +1,18 @@ /// import { defineConfig } from 'vitest/config'; +import path from 'node:path'; export default defineConfig({ + resolve: { + alias: { + mediabunny: path.resolve(__dirname, './src/index.ts'), + }, + }, test: { projects: [ { + extends: true, test: { name: 'node', root: 'test', @@ -14,6 +21,7 @@ export default defineConfig({ }, }, { + extends: true, test: { name: 'browser', root: 'test',