More work

examples/media-player/media-player.ts

@@ -4,6 +4,7 @@ import {
 	BlobSource,
 	CanvasSink,
 	Input,
+	registerDecoder,
 	UrlSource,
 	WrappedAudioBuffer,
 	WrappedCanvas,
@@ -69,6 +70,9 @@ let volumeMuted = false;
 
 /** === INIT LOGIC === */
 
+import { ProResDecoder } from '../../packages/prores-decoder/src/index.js';
+registerDecoder(ProResDecoder);
+
 const initMediaPlayer = async (resource: File | string) => {
 	try {
 		// First, dispose any ongoing playback:
@@ -189,7 +193,7 @@ const initMediaPlayer = async (resource: File | string) => {
 
 		if (audioContext.state === 'running') {
 			// Start playback automatically if the audio context permits
-			await play();
+			// await play();
 		}
 
 		loadingElement.style.display = 'none';

package-lock.json

@@ -1417,6 +1417,10 @@
       "resolved": "packages/mp3-encoder",
       "link": true
     },
+    "node_modules/@mediabunny/prores-decoder": {
+      "resolved": "packages/prores-decoder",
+      "link": true
+    },
     "node_modules/@mermaid-js/mermaid-mindmap": {
       "version": "9.3.0",
       "resolved": "https://registry.npmjs.org/@mermaid-js/mermaid-mindmap/-/mermaid-mindmap-9.3.0.tgz",
@@ -12077,6 +12081,21 @@
       "peerDependencies": {
         "mediabunny": "^1.0.0"
       }
+    },
+    "packages/prores-decoder": {
+      "name": "@mediabunny/prores-decoder",
+      "version": "1.25.8",
+      "license": "MPL-2.0",
+      "devDependencies": {
+        "@types/emscripten": "^1.40.1"
+      },
+      "funding": {
+        "type": "individual",
+        "url": "https://github.com/sponsors/Vanilagy"
+      },
+      "peerDependencies": {
+        "mediabunny": "^1.0.0"
+      }
     }
   }
 }

packages/prores-decoder/README.md

@@ -0,0 +1,46 @@
+```bash
+emmake make distclean
+
+emconfigure ./configure \
+    --target-os=none \
+    --arch=x86_32 \
+    --enable-cross-compile \
+    --disable-asm \
+    --disable-x86asm \
+    --disable-inline-asm \
+    --disable-stripping \
+    --disable-programs \
+    --disable-doc \
+    --disable-debug \
+    --disable-all \
+    --disable-everything \
+    --disable-pthreads \
+    --enable-avcodec \
+    --enable-decoder=prores \
+    --cc="emcc" \
+    --cxx=em++ \
+    --ar=emar \
+    --ranlib=emranlib \
+    --extra-cflags="-DNDEBUG -O3 -msimd128"
+
+emmake make
+```
+
+```bash
+export FFMPEG_PATH=/path/to/ffmpeg
+
+emcc src/bridge.c \
+    $FFMPEG_PATH/libavcodec/libavcodec.a \
+    $FFMPEG_PATH/libavutil/libavutil.a \
+    -I$FFMPEG_PATH \
+    -s MODULARIZE=1 \
+    -s EXPORT_ES6=1 \
+    -s SINGLE_FILE=1 \
+    -s ALLOW_MEMORY_GROWTH=1 \
+    -s ENVIRONMENT=web,worker \
+    -s EXPORTED_RUNTIME_METHODS=cwrap,HEAPU8 \
+    -s EXPORTED_FUNCTIONS=_malloc,_free \
+    -msimd128 \
+    -O3 \
+    -o build/prores.js
+```

packages/prores-decoder/package.json

@@ -0,0 +1,52 @@
+{
+  "name": "@mediabunny/prores-decoder",
+  "author": "Vanilagy",
+  "version": "1.25.8",
+  "description": "MP3 encoder extension for Mediabunny, based on LAME.",
+  "main": "./dist/bundles/mediabunny-mp3-encoder.mjs",
+  "module": "./dist/bundles/mediabunny-mp3-encoder.mjs",
+  "types": "./dist/modules/src/index.d.ts",
+  "exports": {
+    "types": "./dist/modules/src/index.d.ts",
+    "import": "./dist/bundles/mediabunny-mp3-encoder.mjs",
+    "require": "./dist/bundles/mediabunny-mp3-encoder.mjs"
+  },
+  "files": [
+    "README.md",
+    "package.json",
+    "LICENSE",
+    "dist",
+    "src"
+  ],
+  "sideEffects": false,
+  "license": "MPL-2.0",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Vanilagy/mediabunny.git",
+    "directory": "packages/mp3-encoder"
+  },
+  "bugs": {
+    "url": "https://github.com/Vanilagy/mediabunny/issues"
+  },
+  "homepage": "https://mediabunny.dev/guide/extensions/mp3-encoder",
+  "funding": {
+    "type": "individual",
+    "url": "https://github.com/sponsors/Vanilagy"
+  },
+  "peerDependencies": {
+    "mediabunny": "^1.0.0"
+  },
+  "devDependencies": {
+    "@types/emscripten": "^1.40.1"
+  },
+  "keywords": [
+    "mp3",
+    "encoding",
+    "codec",
+    "mediabunny",
+    "lame",
+    "browser",
+    "wasm",
+    "polyfill"
+  ]
+}

packages/prores-decoder/src/bridge.c

@@ -0,0 +1,182 @@
+#include <emscripten.h>
+#include <stdio.h>
+#include "libavcodec/avcodec.h"
+#include "libavutil/pixdesc.h"
+#include "libavutil/imgutils.h"
+
+typedef struct {
+	AVCodecContext *codec_ctx;
+	AVPacket *packet;
+	AVFrame *frame;
+	uint8_t *buffer;
+	int buffer_size;
+} DecoderContext;
+
+EMSCRIPTEN_KEEPALIVE
+DecoderContext *init_decoder() {
+	const AVCodec *codec = avcodec_find_decoder(AV_CODEC_ID_PRORES);
+	if (!codec) return NULL;
+
+	AVCodecContext *codec_ctx = avcodec_alloc_context3(codec);
+	if (!codec_ctx) return NULL;
+
+	if (avcodec_open2(codec_ctx, codec, NULL) < 0) {
+		avcodec_free_context(&codec_ctx);
+		return NULL;
+	}
+
+	AVPacket *packet = av_packet_alloc();
+	if (!packet) {
+		avcodec_free_context(&codec_ctx);
+		return NULL;
+	}
+
+	AVFrame *frame = av_frame_alloc();
+	if (!frame) {
+		av_packet_free(&packet);
+		avcodec_free_context(&codec_ctx);
+		return NULL;
+	}
+
+	DecoderContext *ctx = malloc(sizeof(DecoderContext));
+	if (!ctx) {
+		av_frame_free(&frame);
+		av_packet_free(&packet);
+		avcodec_free_context(&codec_ctx);
+		return NULL;
+	}
+
+	ctx->codec_ctx = codec_ctx;
+	ctx->packet = packet;
+	ctx->frame = frame;
+	ctx->buffer = NULL;
+	ctx->buffer_size = 0;
+
+	return ctx;
+}
+
+EMSCRIPTEN_KEEPALIVE
+uint8_t *configure_packet(DecoderContext *ctx, int size) {
+	if (av_new_packet(ctx->packet, size) < 0) {
+		return NULL;
+	}
+
+	ctx->packet->pts = 0;
+	ctx->packet->dts = AV_NOPTS_VALUE;
+	ctx->packet->time_base.num = 1;
+	ctx->packet->time_base.den = 1;
+	ctx->packet->flags = AV_PKT_FLAG_KEY;
+
+	return ctx->packet->data;
+}
+
+EMSCRIPTEN_KEEPALIVE
+int decode_packet(DecoderContext *ctx) {
+	double start = emscripten_get_now();
+
+	int ret = avcodec_send_packet(ctx->codec_ctx, ctx->packet);
+	if (ret < 0) {
+		return ret;
+	}
+
+	double after_send = emscripten_get_now();
+
+	ret = avcodec_receive_frame(ctx->codec_ctx, ctx->frame);
+	if (ret < 0) {
+		return ret;
+	}
+
+	double after_receive = emscripten_get_now();
+
+	// Calculate required buffer size
+	int required_size = av_image_get_buffer_size(
+		ctx->frame->format,
+		ctx->frame->width,
+		ctx->frame->height,
+		1
+	);
+
+	if (required_size < 0) {
+		return required_size;
+	}
+
+	// Reallocate buffer if needed
+	if (ctx->buffer_size < required_size) {
+		free(ctx->buffer);
+		ctx->buffer = malloc(required_size);
+		if (!ctx->buffer) {
+			ctx->buffer_size = 0;
+			return AVERROR(ENOMEM);
+		}
+		ctx->buffer_size = required_size;
+	}
+
+	double after_alloc = emscripten_get_now();
+
+	// Copy frame data to contiguous buffer
+	ret = av_image_copy_to_buffer(
+		ctx->buffer,
+		ctx->buffer_size,
+		(const uint8_t * const *)ctx->frame->data,
+		ctx->frame->linesize,
+		ctx->frame->format,
+		ctx->frame->width,
+		ctx->frame->height,
+		1
+	);
+
+	if (ret < 0) {
+		return ret;
+	}
+
+	double after_copy = emscripten_get_now();
+
+	printf("Timing: send=%.2fms receive=%.2fms alloc=%.2fms copy=%.2fms total=%.2fms\n",
+		after_send - start,
+		after_receive - after_send,
+		after_alloc - after_receive,
+		after_copy - after_alloc,
+		after_copy - start);
+
+	return 0;
+}
+
+EMSCRIPTEN_KEEPALIVE
+int get_frame_width(DecoderContext *ctx) {
+	return ctx->frame->width;
+}
+
+EMSCRIPTEN_KEEPALIVE
+int get_frame_height(DecoderContext *ctx) {
+	return ctx->frame->height;
+}
+
+EMSCRIPTEN_KEEPALIVE
+int get_frame_format(DecoderContext *ctx) {
+	return ctx->frame->format;
+}
+
+EMSCRIPTEN_KEEPALIVE
+int get_frame_num_planes(DecoderContext *ctx) {
+	return av_pix_fmt_count_planes(ctx->frame->format);
+}
+
+EMSCRIPTEN_KEEPALIVE
+int get_frame_linesize(DecoderContext *ctx, int n) {
+	return ctx->frame->linesize[n];
+}
+
+EMSCRIPTEN_KEEPALIVE
+uint8_t *get_frame_data(DecoderContext *ctx, int n) {
+	return ctx->frame->data[n];
+}
+
+EMSCRIPTEN_KEEPALIVE
+uint8_t *get_frame_data_ptr(DecoderContext *ctx) {
+	return ctx->buffer;
+}
+
+EMSCRIPTEN_KEEPALIVE
+int get_frame_data_size(DecoderContext *ctx) {
+	return ctx->buffer_size;
+}

packages/prores-decoder/src/index.ts

@@ -0,0 +1,90 @@
+import { CustomVideoDecoder, EncodedPacket, MaybePromise, VideoCodec } from 'mediabunny';
+import createModule from '../build/prores';
+import { VideoSample } from 'mediabunny';
+
+type ExtendedEmscriptenModule = EmscriptenModule & {
+	cwrap: typeof cwrap;
+};
+
+export class ProResDecoder extends CustomVideoDecoder {
+	// eslint-disable-next-line @typescript-eslint/no-unused-vars
+	static override supports(codec: VideoCodec, config: VideoDecoderConfig): boolean {
+		return codec === 'prores';
+	}
+
+	module!: ExtendedEmscriptenModule;
+	initDecoder!: () => number;
+	configurePacket!: (ctx: number, size: number) => number;
+	decodePacket!: (ctx: number) => number;
+	getFrameWidth!: (ctx: number) => number;
+	getFrameHeight!: (ctx: number) => number;
+	getFrameFormat!: (ctx: number) => number;
+	getFrameNumPlanes!: (ctx: number) => number;
+	getFrameLinesize!: (ctx: number, n: number) => number;
+	getFrameData!: (ctx: number, n: number) => number;
+	getFrameDataPtr!: (ctx: number) => number;
+	getFrameDataSize!: (ctx: number) => number;
+
+	decoderContextPtr!: number;
+
+	override async init() {
+		console.log('THIS IS BEING CALLED');
+
+		this.module = (await createModule()) as ExtendedEmscriptenModule;
+
+		// Set up the functions
+		this.initDecoder = this.module.cwrap('init_decoder', 'number', []);
+		this.configurePacket = this.module.cwrap('configure_packet', 'number', ['number', 'number']);
+		this.decodePacket = this.module.cwrap('decode_packet', 'number', ['number']);
+		this.getFrameWidth = this.module.cwrap('get_frame_width', 'number', ['number']);
+		this.getFrameHeight = this.module.cwrap('get_frame_height', 'number', ['number']);
+		this.getFrameFormat = this.module.cwrap('get_frame_format', 'number', ['number']);
+		this.getFrameNumPlanes = this.module.cwrap('get_frame_num_planes', 'number', ['number']);
+		this.getFrameLinesize = this.module.cwrap('get_frame_linesize', 'number', ['number', 'number']);
+		this.getFrameData = this.module.cwrap('get_frame_data', 'number', ['number', 'number']);
+		this.getFrameDataPtr = this.module.cwrap('get_frame_data_ptr', 'number', ['number']);
+		this.getFrameDataSize = this.module.cwrap('get_frame_data_size', 'number', ['number']);
+
+		this.decoderContextPtr = this.initDecoder();
+	}
+
+	override decode(packet: EncodedPacket) {
+		const dataPtr = this.configurePacket(this.decoderContextPtr, packet.byteLength);
+		if (dataPtr === 0) {
+			throw new Error('todo');
+		}
+
+		this.module.HEAPU8.set(packet.data, dataPtr);
+
+		console.time();
+		const ret = this.decodePacket(this.decoderContextPtr);
+		console.timeEnd();
+
+		if (ret === 0) {
+			const width = this.getFrameWidth(this.decoderContextPtr);
+			const height = this.getFrameHeight(this.decoderContextPtr);
+			const format = this.getFrameFormat(this.decoderContextPtr);
+			const dataPtr = this.getFrameDataPtr(this.decoderContextPtr);
+			const dataSize = this.getFrameDataSize(this.decoderContextPtr);
+
+			const data = this.module.HEAPU8.subarray(dataPtr, dataPtr + dataSize);
+
+			const sample = new VideoSample(data, {
+				format: 'I422P10' as VideoPixelFormat,
+				codedWidth: width,
+				codedHeight: height,
+				timestamp: packet.timestamp,
+				duration: packet.duration,
+			});
+			this.onSample(sample);
+		}
+	}
+
+	override flush(): MaybePromise<void> {
+		// nada
+	}
+
+	override close(): MaybePromise<void> {
+		// nada
+	}
+}

packages/prores-decoder/src/tsconfig.json

@@ -0,0 +1,23 @@
+{
+	"extends": "../../../tsconfig.json",
+	"compilerOptions": {
+		"outDir": "../dist/modules",
+		"declaration": true,
+		"declarationMap": true,
+		"stripInternal": true,
+		"composite": true,
+		"noEmit": false,
+		"moduleResolution": "nodenext",
+		"module": "nodenext",
+		"allowJs": true,
+		"paths": {
+			"mediabunny": ["../../../src/index.ts"],
+		},
+	},
+	"include": [
+		"**/*",
+	],
+	"references": [
+        { "path": "../../../src" }
+    ]
+}

src/sample.ts

@@ -805,6 +805,52 @@ export class VideoSample implements Disposable {
 	}
 }
 
+/**
+ * Describes the color space of a video frame.
+ *
+ * Corresponds to the WebCodecs VideoColorSpace API.
+ */
+export class VideoColorSpace {
+	/**
+	 * The color primaries standard used.
+	 */
+	readonly primaries: VideoColorPrimaries | null;
+	/**
+	 * The transfer characteristics used.
+	 */
+	readonly transfer: VideoTransferCharacteristics | null;
+	/**
+	 * The color matrix coefficients used.
+	 */
+	readonly matrix: VideoMatrixCoefficients | null;
+	/**
+	 * Whether the color values use the full range or limited range.
+	 */
+	readonly fullRange: boolean | null;
+
+	/**
+	 * Creates a new VideoColorSpace.
+	 */
+	constructor(init?: VideoColorSpaceInit) {
+		this.primaries = init?.primaries ?? null;
+		this.transfer = init?.transfer ?? null;
+		this.matrix = init?.matrix ?? null;
+		this.fullRange = init?.fullRange ?? null;
+	}
+
+	/**
+	 * Serializes the color space to a JSON object.
+	 */
+	toJSON() {
+		return {
+			primaries: this.primaries,
+			transfer: this.transfer,
+			matrix: this.matrix,
+			fullRange: this.fullRange,
+		};
+	}
+}
+
 const isVideoFrame = (x: unknown): x is VideoFrame => {
 	return typeof VideoFrame !== 'undefined' && x instanceof VideoFrame;
 };

test/node/prores.test.ts

@@ -6,6 +6,9 @@ import { Output } from '../../src/output.js';
 import { MkvOutputFormat, MovOutputFormat } from '../../src/output-format.js';
 import { BufferTarget } from '../../src/target.js';
 import { Conversion } from '../../src/conversion.js';
+import { VideoSampleSink } from '../../src/media-sink.js';
+import { CustomVideoDecoder, registerDecoder } from '../../src/custom-coder.js';
+import { ProResDecoder } from '../../packages/prores-decoder/src/index.js';
 
 test.concurrent('ProRes MOV file reading', async () => {
 	using input = new Input({
@@ -99,3 +102,18 @@ test.concurrent('ProRes transmuxing into MKV', { timeout: 60_000 }, async () =>
 	expect(decoderConfig.codec).toBe('apch');
 	expect(decoderConfig.description).toBeUndefined();
 });
+
+test.concurrent.only('ProRes sample ahh', async () => {
+	registerDecoder(ProResDecoder);
+
+	using input = new Input({
+		source: new UrlSource('https://pub-cf9fcfcb5c0a44e9b1bb5ff890e041ae.r2.dev/IMG_0158-prores-log.MOV'),
+		formats: ALL_FORMATS,
+	});
+
+	const videoTrack = (await input.getPrimaryVideoTrack())!;
+	const sink = new VideoSampleSink(videoTrack);
+	const sample = await sink.getSample(0);
+
+	console.log(sample);
+});

tsconfig.vitest.json

@@ -7,5 +7,5 @@
 		"noEmit": false
 	},
 	"include": ["vitest.config.ts", "./test/**/*"],
-	"references": [{ "path": "./src" }]
+	"references": [{ "path": "./src" }, { "path": "./packages/prores-decoder/src" }]
 }

vitest.config.ts

@@ -1,11 +1,18 @@
 /// <reference types="@vitest/browser/providers/webdriverio" />
 
 import { defineConfig } from 'vitest/config';
+import path from 'node:path';
 
 export default defineConfig({
+	resolve: {
+		alias: {
+			mediabunny: path.resolve(__dirname, './src/index.ts'),
+		},
+	},
 	test: {
 		projects: [
 			{
+				extends: true,
 				test: {
 					name: 'node',
 					root: 'test',
@@ -14,6 +21,7 @@ export default defineConfig({
 				},
 			},
 			{
+				extends: true,
 				test: {
 					name: 'browser',
 					root: 'test',