Fix test runner in release script

* Add support for reading transparent Matroska and implement alpha side data & decode

* Few fixes

* Implement alpha encoding

* Gracefully handle inability to acquire WebGL context, properly clean up WebGL contexts

* Improve touch device detection

* Test test

* Test test #2

* Test test 3

* Test test 4

* Test test 5

* Test test 6

* Test test 7

* Test test 8

* Test test 9

* Test test 10

* Test test 11

* Test test 12

* Test test 13

* Test test 14

* Test test 15

* Test test 16

* Test test 17

* Test test 18

* Test test 19

* Test test 20

* Fix type errors, improve MetadataTags docs

* Do a bunch of docs work

* Test test?

* "Unexpected only modifier 🤓"

* Add InputVideoTrack.canBeTransparent()

* Some clean-up

* Adjust CI to be less spammy in PRs

* Fix broken license headers

.github/workflows/lint.yml

@@ -3,6 +3,7 @@ name: Lint
 on:
   push:
   pull_request:
+    types: [opened, reopened]
 
 jobs:
   lint:
@@ -21,11 +22,11 @@ jobs:
       - name: Install dependencies
         run: npm ci
 
-      - name: Run build
-        run: npm run build
-
       - name: Run TypeScript
         run: npm run check
 
       - name: Run ESLint
         run: npm run lint
+
+      - name: Run build
+        run: npm run build

.github/workflows/release.yml

@@ -47,7 +47,7 @@ jobs:
         run: npm ci
 
       - name: Run tests
-        run: npm run test
+        run: xvfb-run npm test
 
       - name: Run build
         run: npm run build

.github/workflows/test.yml

@@ -3,6 +3,7 @@ name: Test
 on:
   push:
   pull_request:
+    types: [opened, reopened]
 
 jobs:
   test:
@@ -22,4 +23,4 @@ jobs:
         run: npm ci
 
       - name: Run tests
-        run: npm run test
+        run: xvfb-run npm test

.gitignore

@@ -4,5 +4,6 @@ node_modules
 .DS_Store
 /docs/.vitepress/cache
 /docs/api
+*.tsbuildinfo
 
 packages/mp3-encoder/dist

.vscode/settings.json

@@ -2,5 +2,6 @@
 	"editor.defaultFormatter": "dbaeumer.vscode-eslint",
 	"editor.codeActionsOnSave": {
 		"source.fixAll.eslint": "explicit"
-	}
+	},
+	"typescript.tsdk": "node_modules/typescript/lib"
 }

dev/demux.html

@@ -14,16 +14,29 @@
 			source: new Mediabunny.BlobSource(file),
 		});
 
-		window.doDispose = () => input.dispose();
+		const audioTrack = await input.getPrimaryAudioTrack();
+		
+		const sink = new Mediabunny.EncodedPacketSink(audioTrack);
+		for await (const packet of sink.packets()) {
+			console.log(packet)
+		}
 
+		/*
 		const videoTrack = await input.getPrimaryVideoTrack();
+		const sink = new Mediabunny.VideoSampleSink(videoTrack);
+
+		const sample = await sink.getSample(0);
+
+		console.log(sample);
+		*/
+
+		/*
 		const sink = new Mediabunny.EncodedPacketSink(videoTrack);
 
-		for await (const sample of sink.packets()) {
-			console.log(sample.timestamp)
-			//sample.close();
-			await new Promise(resolve => setTimeout(resolve, 0))
+		for await (const packet of sink.packets()) {
+			console.log(packet.timestamp, packet.type, await videoTrack.determinePacketType(packet));
 		}
+		*/
 
 		/*
 		console.log(await input.computeDuration());

dev/mux.html

@@ -34,8 +34,8 @@
 	}
 
 	const canvas = document.createElement('canvas');
-	canvas.width = 1280;
-	canvas.height = 720;
+	canvas.width = 640;
+	canvas.height = 480;
 	const context = canvas.getContext('2d');
 
 	let format = new Mediabunny.MkvOutputFormat({ streamable: false });
@@ -47,6 +47,7 @@
 	format = new Mediabunny.MkvOutputFormat();
 	format = new Mediabunny.MovOutputFormat();
 	format = new Mediabunny.Mp4OutputFormat({ fastStart: 'reserve' });
+	format = new Mediabunny.WebMOutputFormat();
 	let target = new Mediabunny.BufferTarget();
 
 	/*
@@ -120,6 +121,7 @@
 		codec: 'vp9',
 		//fullCodecString: 'avc1.42001f',
 		bitrate: 1e6,
+		alpha: 'keep',
 		onEncoderConfig: console.log,
 	});
 	let audioSource = new Mediabunny.AudioBufferSource({
@@ -191,12 +193,17 @@ Testing... <00:17.350>One... <00:18.125>Two...
 
 	//subtitleSource.add(simpleWebvttFile);
 	//subtitleSource.close();
+	
+	const p = document.createElement('p');
+	document.body.append(p);
 
 	for (let i = 0; i < 100; i++) {
 		context.fillStyle = ['red', 'green', 'blue', 'yellow'][i % 4];
 		context.fillRect(canvas.width * Math.random(), canvas.height * Math.random(), canvas.width * Math.random(), canvas.height * Math.random());
 
 		await videoSource.add(i / 10, 1 / 10);
+
+		p.textContent = i;
 	}
 
 	let audioContext = new AudioContext();

docs/guide/introduction.md

@@ -12,7 +12,7 @@ Here's a long list of stuff this library does:
 - Converting media files
 - Hardware-accelerated decoding & encoding (via the WebCodecs API)
 - Support for multiple video, audio and subtitle tracks
-- Read & write support for many container formats (.mp4, .mov, .webm, .mkv, .mp3, .wav, .ogg, .aac, .flac), including variations such as MP4 with Fast Start, fragmented MP4, or streamable Matroska
+- Read & write support for many container formats (.mp4, .mov, .webm, .mkv, .mp3, .wav, .ogg, .aac, .flac), including variations such as MP4 with Fast Start, fragmented MP4, streamable Matroska, transparent WebM, etc.
 - Support for 25 different codecs
 - Lazy, optimized, on-demand file reading
 - Input and output streaming, arbitrary file size support

docs/guide/media-sources.md

@@ -47,6 +47,7 @@ All video sources that handle encoding internally require you to specify a `Vide
 type VideoEncodingConfig = {
 	codec: VideoCodec;
 	bitrate: number | Quality;
+	alpha?: 'discard' | 'keep';
 	bitrateMode?: 'constant' | 'variable';
 	latencyMode?: 'quality' | 'realtime';
 	keyFrameInterval?: number;
@@ -67,6 +68,9 @@ type VideoEncodingConfig = {
 ```
 - `codec`: The [video codec](./supported-formats-and-codecs#video-codecs) used for encoding.
 - `bitrate`: The target number of bits per second. Alternatively, this can be a [subjective quality](#subjective-qualities).
+- `alpha`:  What to do with alpha data contained in the video samples.
+	- `'discard'` (default): Only the samples' color data is kept; the video is opaque.
+	- `'keep'`: The samples' alpha data is also encoded as side data. Make sure to pair this mode with a container format that supports transparency (such as WebM or Matroska).
 - `bitrateMode`: Can be used to control constant vs. variable bitrate.
 - `latencyMode`: The latency mode as specified by the WebCodecs API. Browsers default to `quality`. Media stream-driven video sources will automatically use the `realtime` setting.
 - `keyFrameInterval`: The maximum interval in seconds between two adjacent key frames. Defaults to 5 seconds. More frequent key frames improve seeking behavior but increase file size. When using multiple video tracks, this value should be set to the same value for all tracks.

docs/guide/quick-start.md

@@ -397,6 +397,42 @@ await output.finalize();
 - This is basically [`MediaRecorder`](https://developer.mozilla.org/en-US/docs/Web/API/MediaRecorder), but less sucky.
 :::
 
+## Creating transparent video
+
+```ts
+import {
+	Output,
+	WebMOutputFormat,
+	BufferTarget,
+	CanvasSource,
+	QUALITY_MEDIUM,
+} from 'mediabunny';
+
+const output = new Output({
+	// Use a format that supports transparency:
+	format: new WebMOutputFormat(),
+	target: new BufferTarget(),
+});
+
+const canvas = new OffscreenCanvas(1280, 720);
+const context = canvas.getContext('2d', { alpha: true })!;
+
+const source = new CanvasSource(canvas, {
+	codec: 'vp9',
+	quality: QUALITY_MEDIUM,
+	alpha: 'keep', // => Also encode alpha data
+});
+output.addVideoTrack(source);
+
+await output.start();
+
+// Add data...
+await source.add(0, 1 / 30);
+// ...
+
+await output.finalize();
+```
+
 ## Check encoding support
 
 ```ts

examples/media-player/media-player.ts

@@ -35,7 +35,7 @@ const fullscreenButton = document.querySelector('#fullscreen-button') as HTMLBut
 const errorElement = document.querySelector('#error-element') as HTMLDivElement;
 const warningElement = document.querySelector('#warning-element') as HTMLDivElement;
 
-const context = canvas.getContext('2d', { alpha: false, desynchronized: true })!;
+const context = canvas.getContext('2d')!;
 
 let audioContext: AudioContext | null = null;
 let gainNode: GainNode | null = null;
@@ -149,11 +149,18 @@ const initMediaPlayer = async (resource: File | string) => {
 		gainNode.connect(audioContext.destination);
 		updateVolume();
 
+		const videoCanBeTransparent = videoTrack
+			? await videoTrack.canBeTransparent()
+			: false;
+
+		playerContainer.style.background = videoCanBeTransparent ? 'transparent' : '';
+
 		// For video, let's use a CanvasSink as it handles rotation and closing video samples for us.
 		// Pool size of 2: We'll only ever have the current and the next frame around, so we only need two canvases.
 		videoSink = videoTrack && new CanvasSink(videoTrack, {
 			poolSize: 2,
 			fit: 'contain', // In case the video changes dimensions over time
+			alpha: videoCanBeTransparent,
 		});
 		// For audio, we'll use an AudioBufferSink to directly retrieve AudioBuffers compatible with the Web Audio API
 		audioSink = audioTrack && new AudioBufferSink(audioTrack);
@@ -226,6 +233,7 @@ const startVideoIterator = async () => {
 
 	if (firstFrame) {
 		// Draw the first frame
+		context.clearRect(0, 0, canvas.width, canvas.height);
 		context.drawImage(firstFrame.canvas, 0, 0);
 	}
 };
@@ -242,6 +250,7 @@ const render = (requestFrame = true) => {
 
 		// Check if the current playback time has caught up to the next frame
 		if (nextFrame && nextFrame.timestamp <= playbackTime) {
+			context.clearRect(0, 0, canvas.width, canvas.height);
 			context.drawImage(nextFrame.canvas, 0, 0);
 			nextFrame = null;
 
@@ -281,6 +290,7 @@ const updateNextFrame = async () => {
 		const playbackTime = getPlaybackTime();
 		if (newNextFrame.timestamp <= playbackTime) {
 			// Draw it immediately
+			context.clearRect(0, 0, canvas.width, canvas.height);
 			context.drawImage(newNextFrame.canvas, 0, 0);
 		} else {
 			// Save it for later
@@ -594,9 +604,7 @@ fullscreenButton.addEventListener('click', () => {
 
 // I'm sorry for this
 const isTouchDevice = () => {
-	return (('ontouchstart' in window)
-		|| (navigator.maxTouchPoints > 0)
-		|| ('msMaxTouchPoints' in navigator && (navigator.msMaxTouchPoints as number) > 0));
+	return 'ontouchstart' in window;
 };
 
 playerContainer.addEventListener('click', () => {

examples/metadata-extraction/metadata-extraction.ts

@@ -55,6 +55,7 @@ const extractMetadata = (resource: File | string) => {
 						'Coded width': `${track.codedWidth} pixels`,
 						'Coded height': `${track.codedHeight} pixels`,
 						'Rotation': `${track.rotation}° clockwise`,
+						'Transparency': track.canBeTransparent(),
 					}
 				: track.isAudioTrack()
 					? {

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "mediabunny",
-  "version": "1.19.1",
+  "version": "1.20.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "mediabunny",
-      "version": "1.19.1",
+      "version": "1.20.0",
       "license": "MPL-2.0",
       "workspaces": [
         "packages/*"
@@ -7749,9 +7749,9 @@
       }
     },
     "node_modules/mediabunny": {
-      "version": "1.19.0",
-      "resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.19.0.tgz",
-      "integrity": "sha512-tqQqaqY/G8U+oa6NsgffxsBJnoYl8A6bMtHWVnvjyp1FAeBH7f5oZKQpRVGWMpRMd9EK852sFvOOq/dOmb+Mvg==",
+      "version": "1.19.1",
+      "resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.19.1.tgz",
+      "integrity": "sha512-nyaxskRhj7w634RZ/D0+xbc//nclDVBM1eG9hrrPL/lEG4PW1pBSRHsgPRXQX/PvUXNfJTYQsTl0SYFCrch9PA==",
       "license": "MPL-2.0",
       "peer": true,
       "workspaces": [
@@ -12242,7 +12242,7 @@
     },
     "packages/mp3-encoder": {
       "name": "@mediabunny/mp3-encoder",
-      "version": "1.19.1",
+      "version": "1.20.0",
       "license": "MPL-2.0",
       "devDependencies": {
         "@types/emscripten": "^1.40.1"

package.json

@@ -1,7 +1,7 @@
 {
   "name": "mediabunny",
   "author": "Vanilagy",
-  "version": "1.19.1",
+  "version": "1.20.0",
   "description": "Pure TypeScript media toolkit for reading, writing, and converting media files, directly in the browser.",
   "type": "module",
   "workspaces": [
@@ -32,10 +32,10 @@
     "build": "./build.sh",
     "watch": "tsx scripts/bundle.ts --watch",
     "lint": "eslint .",
-    "test-node": "cd test && vitest node --run",
-    "test-browser": "cd test && vitest browser --run --browser",
-    "test": "npm run test-node && npm run test-browser",
-    "check": "tsc -p src --noEmit && tsc -p packages/mp3-encoder/src --noEmit && tsc -p scripts --noEmit && tsc -p tsconfig.vite.json --noEmit && rm tsconfig.vite.tsbuildinfo",
+    "test": "npx vitest --run",
+    "test-node": "npm run test node/",
+    "test-browser": "npm run test browser/",
+    "check": "rm -rf dist/modules && tsc -p src && tsc -p packages/mp3-encoder/src --noEmit && tsc -p tsconfig.vitest.json --noEmit && tsc -p scripts --noEmit && tsc -p tsconfig.vite.json --noEmit",
     "check-docblocks": "tsx scripts/check-docblocks.ts dist/mediabunny.d.ts",
     "docs:dev": "vitepress dev docs",
     "docs:build": "npm run build && npm run docs:generate && vitepress build docs && npm run examples:build && cp dist/mediabunny.d.ts dist-docs/",

packages/mp3-encoder/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@mediabunny/mp3-encoder",
   "author": "Vanilagy",
-  "version": "1.19.1",
+  "version": "1.20.0",
   "description": "MP3 encoder extension for Mediabunny, based on LAME.",
   "main": "./dist/bundles/mediabunny-mp3-encoder.mjs",
   "module": "./dist/bundles/mediabunny-mp3-encoder.mjs",

scripts/check-docblocks.ts

@@ -207,11 +207,11 @@ const main = () => {
 			console.log(`✅ All symbols in ${filePath} have meaningful docblocks.`);
 		} else {
 			console.log(
-				`❌ Found ${result.missingDocblocks.length} symbols with insufficient docblocks in ${filePath}:`,
+				`❌ Found ${result.missingDocblocks.length} symbols with insufficient docblocks:`,
 			);
 
 			result.missingDocblocks.forEach((item) => {
-				console.log(`  - ${item.kind} '${item.name}' at line ${item.line}: ${item.reason}`);
+				console.log(`  - ${item.kind} '${item.name}' at ${filePath}:${item.line}: ${item.reason}`);
 			});
 
 			process.exit(1);

src/codec-data.ts

@@ -6,8 +6,7 @@
  * file, You can obtain one at https://mozilla.org/MPL/2.0/.
  */
 
-import { VP9_LEVEL_TABLE } from './codec';
-import { InputVideoTrack } from './input-track';
+import { VideoCodec, VP9_LEVEL_TABLE } from './codec';
 import {
 	assert,
 	assertNever,
@@ -24,7 +23,7 @@ import {
 	toDataView,
 	toUint8Array,
 } from './misc';
-import { EncodedPacket, PacketType } from './packet';
+import { PacketType } from './packet';
 import { MetadataTags } from './tags';
 
 // References for AVC/HEVC code:
@@ -1479,28 +1478,21 @@ export const parseModesFromVorbisSetupPacket = (setupHeader: Uint8Array) => {
 };
 
 /** Determines a packet's type (key or delta) by digging into the packet bitstream. */
-export const determineVideoPacketType = async (
-	videoTrack: InputVideoTrack,
-	packet: EncodedPacket,
-): Promise<PacketType | null> => {
-	assert(videoTrack.codec);
-
-	switch (videoTrack.codec) {
+export const determineVideoPacketType = (
+	codec: VideoCodec,
+	decoderConfig: VideoDecoderConfig,
+	packetData: Uint8Array,
+): PacketType | null => {
+	switch (codec) {
 		case 'avc': {
-			const decoderConfig = await videoTrack.getDecoderConfig();
-			assert(decoderConfig);
-
-			const nalUnits = extractAvcNalUnits(packet.data, decoderConfig);
+			const nalUnits = extractAvcNalUnits(packetData, decoderConfig);
 			const isKeyframe = nalUnits.some(x => extractNalUnitTypeForAvc(x) === AvcNalUnitType.IDR);
 
 			return isKeyframe ? 'key' : 'delta';
 		};
 
 		case 'hevc': {
-			const decoderConfig = await videoTrack.getDecoderConfig();
-			assert(decoderConfig);
-
-			const nalUnits = extractHevcNalUnits(packet.data, decoderConfig);
+			const nalUnits = extractHevcNalUnits(packetData, decoderConfig);
 			const isKeyframe = nalUnits.some((x) => {
 				const type = extractNalUnitTypeForHevc(x);
 				return HevcNalUnitType.BLA_W_LP <= type && type <= HevcNalUnitType.RSV_IRAP_VCL23;
@@ -1511,12 +1503,12 @@ export const determineVideoPacketType = async (
 
 		case 'vp8': {
 			// VP8, once again, by far the easiest to deal with.
-			const frameType = packet.data[0]! & 0b1;
+			const frameType = packetData[0]! & 0b1;
 			return frameType === 0 ? 'key' : 'delta';
 		};
 
 		case 'vp9': {
-			const bitstream = new Bitstream(packet.data);
+			const bitstream = new Bitstream(packetData);
 
 			if (bitstream.readBits(2) !== 2) {
 				return null;
@@ -1543,7 +1535,7 @@ export const determineVideoPacketType = async (
 		case 'av1': {
 			let reducedStillPictureHeader = false;
 
-			for (const { type, data } of iterateAv1PacketObus(packet.data)) {
+			for (const { type, data } of iterateAv1PacketObus(packetData)) {
 				if (type === 1) { // OBU_SEQUENCE_HEADER
 					const bitstream = new Bitstream(data);
 
@@ -1573,7 +1565,7 @@ export const determineVideoPacketType = async (
 		};
 
 		default: {
-			assertNever(videoTrack.codec);
+			assertNever(codec);
 			assert(false);
 		};
 	}

src/encode.ts

@@ -107,6 +107,14 @@ export const validateVideoEncodingConfig = (config: VideoEncodingConfig) => {
  * @public
  */
 export type VideoEncodingAdditionalOptions = {
+	/**
+	 * What to do with alpha data contained in the video samples.
+	 *
+	 * - `'discard'` (default): Only the samples' color data is kept; the video is opaque.
+	 * - `'keep'`: The samples' alpha data is also encoded as side data. Make sure to pair this mode with a container
+	 * format that supports transparency (such as WebM or Matroska).
+	 */
+	alpha?: 'discard' | 'keep';
 	/** Configures the bitrate mode. */
 	bitrateMode?: 'constant' | 'variable';
 	/** The latency mode used by the encoder; controls the performance-quality tradeoff. */
@@ -136,6 +144,9 @@ export const validateVideoEncodingAdditionalOptions = (codec: VideoCodec, option
 	if (!options || typeof options !== 'object') {
 		throw new TypeError('Encoding options must be an object.');
 	}
+	if (options.alpha !== undefined && !['discard', 'keep'].includes(options.alpha)) {
+		throw new TypeError('options.alpha, when provided, must be \'discard\' or \'keep\'.');
+	}
 	if (options.bitrateMode !== undefined && !['constant', 'variable'].includes(options.bitrateMode)) {
 		throw new TypeError('bitrateMode, when provided, must be \'constant\' or \'variable\'.');
 	}
@@ -189,7 +200,8 @@ export const buildVideoEncoderConfig = (options: {
 		height: options.height,
 		bitrate: resolvedBitrate,
 		bitrateMode: options.bitrateMode,
-		framerate: options.framerate, // this.source._connectedTrack?.metadata.frameRate,
+		alpha: options.alpha ?? 'discard',
+		framerate: options.framerate,
 		latencyMode: options.latencyMode,
 		hardwareAcceleration: options.hardwareAcceleration,
 		scalabilityMode: options.scalabilityMode,
@@ -506,6 +518,7 @@ export const canEncodeVideo = async (
 		bitrate,
 		framerate: undefined,
 		...restOptions,
+		alpha: 'discard', // Since we handle alpha ourselves
 	});
 
 	const support = await VideoEncoder.isConfigSupported(encoderConfig);

src/index.ts

@@ -157,6 +157,7 @@ export {
 } from './input-track';
 export {
 	EncodedPacket,
+	EncodedPacketSideData,
 	PacketType,
 } from './packet';
 export {

src/input-track.ts

@@ -192,6 +192,7 @@ export interface InputVideoTrackBacking extends InputTrackBacking {
 	getCodedHeight(): number;
 	getRotation(): Rotation;
 	getColorSpace(): Promise<VideoColorSpaceInit>;
+	canBeTransparent(): Promise<boolean>;
 	getDecoderConfig(): Promise<VideoDecoderConfig | null>;
 }
 
@@ -260,6 +261,11 @@ export class InputVideoTrack extends InputTrack {
 			|| (colorSpace.matrix as string) === 'bt2020-ncl';
 	}
 
+	/** Checks if this track may contain transparent samples with alpha data. */
+	canBeTransparent() {
+		return this._backing.canBeTransparent();
+	}
+
 	/**
 	 * Returns the [decoder configuration](https://www.w3.org/TR/webcodecs/#video-decoder-config) for decoding the
 	 * track's packets using a [`VideoDecoder`](https://developer.mozilla.org/en-US/docs/Web/API/VideoDecoder). Returns
@@ -312,7 +318,10 @@ export class InputVideoTrack extends InputTrack {
 			return null;
 		}
 
-		return determineVideoPacketType(this, packet);
+		const decoderConfig = await this.getDecoderConfig();
+		assert(decoderConfig);
+
+		return determineVideoPacketType(this.codec, decoderConfig, packet.data);
 	}
 }
 

src/input.ts

@@ -8,10 +8,12 @@
 
 import { Demuxer } from './demuxer';
 import { InputFormat } from './input-format';
-import { assert } from './misc';
+import { assert, polyfillSymbolDispose } from './misc';
 import { Reader } from './reader';
 import { Source } from './source';
 
+polyfillSymbolDispose();
+
 /**
  * The options for creating an Input object.
  * @group Input files & tracks
@@ -24,10 +26,6 @@ export type InputOptions<S extends Source = Source> = {
 	source: S;
 };
 
-// https://www.typescriptlang.org/docs/handbook/release-notes/typescript-5-2.html
-// @ts-expect-error Readonly
-Symbol.dispose ??= Symbol('Symbol.dispose');
-
 /**
  * Represents an input media file. This is the root object from which all media read operations start.
  * @group Input files & tracks

src/isobmff/isobmff-demuxer.ts

@@ -2966,6 +2966,10 @@ class IsobmffVideoTrackBacking extends IsobmffTrackBacking implements InputVideo
 		};
 	}
 
+	async canBeTransparent() {
+		return false;
+	}
+
 	async getDecoderConfig(): Promise<VideoDecoderConfig | null> {
 		if (!this.internalTrack.info.codec) {
 			return null;

src/matroska/ebml.ts

@@ -100,6 +100,7 @@ export enum EBMLId {
 	Video = 0xe0,
 	PixelWidth = 0xb0,
 	PixelHeight = 0xba,
+	AlphaMode = 0x53c0,
 	Audio = 0xe1,
 	SamplingFrequency = 0xb5,
 	Channels = 0x9f,

src/matroska/matroska-demuxer.ts

@@ -50,7 +50,7 @@ import {
 	TRANSFER_CHARACTERISTICS_MAP_INVERSE,
 	UNDETERMINED_LANGUAGE,
 } from '../misc';
-import { EncodedPacket, PLACEHOLDER_DATA } from '../packet';
+import { EncodedPacket, EncodedPacketSideData, PLACEHOLDER_DATA } from '../packet';
 import {
 	assertDefinedSize,
 	CODEC_STRING_MAP,
@@ -143,6 +143,7 @@ type ClusterBlock = {
 	data: Uint8Array;
 	lacing: BlockLacing;
 	decoded: boolean;
+	mainAdditional: Uint8Array | null;
 };
 
 type CuePoint = {
@@ -204,6 +205,7 @@ type InternalTrack = {
 			codec: VideoCodec | null;
 			codecDescription: Uint8Array | null;
 			colorSpace: VideoColorSpaceInit | null;
+			alphaMode: boolean;
 		}
 		| {
 			type: 'audio';
@@ -236,6 +238,11 @@ export class MatroskaDemuxer extends Demuxer {
 	currentTrack: InternalTrack | null = null;
 	currentCluster: Cluster | null = null;
 	currentBlock: ClusterBlock | null = null;
+	currentBlockAdditional: {
+		addId: number;
+		data: Uint8Array | null;
+	} | null = null;
+
 	currentCueTime: number | null = null;
 	currentDecodingInstruction: DecodingInstruction | null = null;
 	currentTagTargetIsMovie: boolean = true;
@@ -845,6 +852,7 @@ export class MatroskaDemuxer extends Demuxer {
 					data: frameData,
 					lacing: BlockLacing.None,
 					decoded: true,
+					mainAdditional: originalBlock.mainAdditional,
 				});
 			}
 
@@ -1110,6 +1118,7 @@ export class MatroskaDemuxer extends Demuxer {
 						codec: null,
 						codecDescription: null,
 						colorSpace: null,
+						alphaMode: false,
 					};
 				} else if (type === 2) {
 					this.currentTrack.info = {
@@ -1214,6 +1223,12 @@ export class MatroskaDemuxer extends Demuxer {
 				this.currentTrack.info.height = readUnsignedInt(slice, size);
 			}; break;
 
+			case EBMLId.AlphaMode: {
+				if (this.currentTrack?.info?.type !== 'video') break;
+
+				this.currentTrack.info.alphaMode = readUnsignedInt(slice, size) === 1;
+			}; break;
+
 			case EBMLId.Colour: {
 				if (this.currentTrack?.info?.type !== 'video') break;
 
@@ -1365,6 +1380,7 @@ export class MatroskaDemuxer extends Demuxer {
 					data: blockData,
 					lacing,
 					decoded: !hasDecodingInstructions,
+					mainAdditional: null,
 				});
 			}; break;
 
@@ -1407,10 +1423,43 @@ export class MatroskaDemuxer extends Demuxer {
 					data: blockData,
 					lacing,
 					decoded: !hasDecodingInstructions,
+					mainAdditional: null,
 				};
 				trackData.blocks.push(this.currentBlock);
 			}; break;
 
+			case EBMLId.BlockAdditions: {
+				this.readContiguousElements(slice.slice(dataStartPos, size));
+			}; break;
+
+			case EBMLId.BlockMore: {
+				if (!this.currentBlock) break;
+
+				this.currentBlockAdditional = {
+					addId: 1,
+					data: null,
+				};
+
+				this.readContiguousElements(slice.slice(dataStartPos, size));
+
+				if (this.currentBlockAdditional.data && this.currentBlockAdditional.addId === 1) {
+					this.currentBlock.mainAdditional = this.currentBlockAdditional.data;
+				}
+				this.currentBlockAdditional = null;
+			}; break;
+
+			case EBMLId.BlockAdditional: {
+				if (!this.currentBlockAdditional) break;
+
+				this.currentBlockAdditional.data = readBytes(slice, size);
+			}; break;
+
+			case EBMLId.BlockAddID: {
+				if (!this.currentBlockAdditional) break;
+
+				this.currentBlockAdditional.addId = readUnsignedInt(slice, size);
+			}; break;
+
 			case EBMLId.BlockDuration: {
 				if (!this.currentBlock) break;
 
@@ -2004,6 +2053,13 @@ abstract class MatroskaTrackBacking implements InputTrackBacking {
 		const data = options.metadataOnly ? PLACEHOLDER_DATA : block.data;
 		const timestamp = block.timestamp / this.internalTrack.segment.timestampFactor;
 		const duration = block.duration / this.internalTrack.segment.timestampFactor;
+
+		const sideData: EncodedPacketSideData = {};
+		if (block.mainAdditional && this.internalTrack.info?.type === 'video' && this.internalTrack.info.alphaMode) {
+			sideData.alpha = options.metadataOnly ? PLACEHOLDER_DATA : block.mainAdditional;
+			sideData.alphaByteLength = block.mainAdditional.byteLength;
+		}
+
 		const packet = new EncodedPacket(
 			data,
 			block.isKeyFrame ? 'key' : 'delta',
@@ -2011,6 +2067,7 @@ abstract class MatroskaTrackBacking implements InputTrackBacking {
 			duration,
 			cluster.dataStartPos + blockIndex,
 			block.data.byteLength,
+			sideData,
 		);
 
 		this.packetToClusterLocation.set(packet, { cluster, blockIndex });
@@ -2319,6 +2376,10 @@ class MatroskaVideoTrackBacking extends MatroskaTrackBacking implements InputVid
 		};
 	}
 
+	async canBeTransparent() {
+		return this.internalTrack.info.alphaMode;
+	}
+
 	async getDecoderConfig(): Promise<VideoDecoderConfig | null> {
 		if (!this.internalTrack.info.codec) {
 			return null;

src/matroska/matroska-muxer.ts

@@ -89,6 +89,7 @@ type MatroskaTrackData = {
 		width: number;
 		height: number;
 		decoderConfig: VideoDecoderConfig;
+		alphaMode: boolean;
 	};
 } | {
 	track: OutputAudioTrack;
@@ -343,6 +344,7 @@ export class MatroskaMuxer extends Muxer {
 		const videoElement: EBMLElement = { id: EBMLId.Video, data: [
 			{ id: EBMLId.PixelWidth, data: trackData.info.width },
 			{ id: EBMLId.PixelHeight, data: trackData.info.height },
+			trackData.info.alphaMode ? { id: EBMLId.AlphaMode, data: 1 } : null,
 			(colorSpaceIsComplete(colorSpace)
 				? {
 						id: EBMLId.Colour,
@@ -695,7 +697,7 @@ export class MatroskaMuxer extends Muxer {
 		});
 	}
 
-	private getVideoTrackData(track: OutputVideoTrack, meta?: EncodedVideoChunkMetadata) {
+	private getVideoTrackData(track: OutputVideoTrack, packet: EncodedPacket, meta?: EncodedVideoChunkMetadata) {
 		const existingTrackData = this.trackDatas.find(x => x.track === track);
 		if (existingTrackData) {
 			return existingTrackData as MatroskaVideoTrackData;
@@ -715,6 +717,7 @@ export class MatroskaMuxer extends Muxer {
 				width: meta.decoderConfig.codedWidth,
 				height: meta.decoderConfig.codedHeight,
 				decoderConfig: meta.decoderConfig,
+				alphaMode: !!packet.sideData.alpha, // The first packet determines if this track has alpha or not
 			},
 			chunkQueue: [],
 			lastWrittenMsTimestamp: null,
@@ -819,7 +822,7 @@ export class MatroskaMuxer extends Muxer {
 		const release = await this.mutex.acquire();
 
 		try {
-			const trackData = this.getVideoTrackData(track, meta);
+			const trackData = this.getVideoTrackData(track, packet, meta);
 
 			const isKeyFrame = packet.type === 'key';
 			let timestamp = this.validateAndNormalizeTimestamp(trackData.track, packet.timestamp, isKeyFrame);
@@ -831,7 +834,11 @@ export class MatroskaMuxer extends Muxer {
 				duration = roundToMultiple(duration, 1 / track.metadata.frameRate);
 			}
 
-			const videoChunk = this.createInternalChunk(packet.data, timestamp, duration, packet.type);
+			const additions = trackData.info.alphaMode
+				? packet.sideData.alpha ?? null
+				: null;
+
+			const videoChunk = this.createInternalChunk(packet.data, timestamp, duration, packet.type, additions);
 			if (track.source._codec === 'vp9') this.fixVP9ColorSpace(trackData, videoChunk);
 
 			trackData.chunkQueue.push(videoChunk);
@@ -1086,8 +1093,8 @@ export class MatroskaMuxer extends Muxer {
 				chunk.additions
 					? { id: EBMLId.BlockAdditions, data: [
 							{ id: EBMLId.BlockMore, data: [
+								{ id: EBMLId.BlockAddID, data: 1 }, // Some players expect BlockAddID to come first
 								{ id: EBMLId.BlockAdditional, data: chunk.additions },
-								{ id: EBMLId.BlockAddID, data: 1 },
 							] },
 						] }
 					: null,

src/media-sink.ts

@@ -7,7 +7,12 @@
  */
 
 import { parsePcmCodec, PCM_AUDIO_CODECS, PcmAudioCodec, VideoCodec, AudioCodec } from './codec';
-import { extractHevcNalUnits, extractNalUnitTypeForHevc, HevcNalUnitType } from './codec-data';
+import {
+	determineVideoPacketType,
+	extractHevcNalUnits,
+	extractNalUnitTypeForHevc,
+	HevcNalUnitType,
+} from './codec-data';
 import { CustomVideoDecoder, customVideoDecoders, CustomAudioDecoder, customAudioDecoders } from './custom-coder';
 import { InputDisposedError } from './input';
 import { InputAudioTrack, InputTrack, InputVideoTrack } from './input-track';
@@ -365,7 +370,7 @@ abstract class DecoderWrapper<
 > {
 	constructor(
 		public onSample: (sample: MediaSample) => unknown,
-		public onError: (error: DOMException) => unknown,
+		public onError: (error: Error) => unknown,
 	) {}
 
 	abstract getDecodeQueueSize(): number;
@@ -388,7 +393,7 @@ export abstract class BaseMediaSampleSink<
 	/** @internal */
 	abstract _createDecoder(
 		onSample: (sample: MediaSample) => unknown,
-		onError: (error: DOMException) => unknown
+		onError: (error: Error) => unknown
 	): Promise<DecoderWrapper<MediaSample>>;
 	/** @internal */
 	abstract _createPacketSink(): EncodedPacketSink;
@@ -811,9 +816,23 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
 	currentPacketIndex = 0;
 	raslSkipped = false; // For HEVC stuff
 
+	// Alpha stuff
+	alphaDecoder: VideoDecoder | null = null;
+	alphaHadKeyframe = false;
+	colorQueue: VideoFrame[] = [];
+	alphaQueue: (VideoFrame | null)[] = [];
+	merger: ColorAlphaMerger | null = null;
+	mergerCreationFailed = false;
+	decodedAlphaChunkCount = 0;
+	alphaDecoderQueueSize = 0;
+	/** Each value is the number of decoded alpha chunks at which a null alpha frame should be added. */
+	nullAlphaFrameQueue: number[] = [];
+	currentAlphaPacketIndex = 0;
+	alphaRaslSkipped = false; // For HEVC stuff
+
 	constructor(
 		onSample: (sample: VideoSample) => unknown,
-		onError: (error: DOMException) => unknown,
+		onError: (error: Error) => unknown,
 		public codec: VideoCodec,
 		public decoderConfig: VideoDecoderConfig,
 		public rotation: Rotation,
@@ -840,79 +859,48 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
 
 			void this.customDecoderCallSerializer.call(() => this.customDecoder!.init());
 		} else {
-			// Specific handler for the WebCodecs VideoDecoder to iron out browser differences
-			const sampleHandler = (sample: VideoSample) => {
-				if (isSafari()) {
-					// For correct B-frame handling, we don't just hand over the frames directly but instead add them to
-					// a queue, because we want to ensure frames are emitted in presentation order. We flush the queue
-					// each time we receive a frame with a timestamp larger than the highest we've seen so far, as we
-					// can sure that is not a B-frame. Typically, WebCodecs automatically guarantees that frames are
-					// emitted in presentation order, but Safari doesn't always follow this rule.
-					if (this.sampleQueue.length > 0 && (sample.timestamp >= last(this.sampleQueue)!.timestamp)) {
-						for (const sample of this.sampleQueue) {
-							this.finalizeAndEmitSample(sample);
-						}
+			const colorHandler = (frame: VideoFrame) => {
+				if (this.alphaQueue.length > 0) {
+					// Even when no alpha data is present (most of the time), there will be nulls in this queue
+					const alphaFrame = this.alphaQueue.shift();
+					assert(alphaFrame !== undefined);
 
-						this.sampleQueue.length = 0;
-					}
-
-					insertSorted(this.sampleQueue, sample, x => x.timestamp);
+					this.mergeAlpha(frame, alphaFrame);
 				} else {
-					// Assign it the next earliest timestamp from the input. We do this because browsers, by spec, are
-					// required to emit decoded frames in presentation order *while* retaining the timestamp of their
-					// originating EncodedVideoChunk. For files with B-frames but no out-of-order timestamps (like a
-					// missing ctts box, for example), this causes a mismatch. We therefore fix the timestamps and
-					// ensure they are sorted by doing this.
-					const timestamp = this.inputTimestamps.shift();
-
-					// There's no way we'd have more decoded frames than encoded packets we passed in. Actually, the
-					// correspondence should be 1:1.
-					assert(timestamp !== undefined);
-
-					sample.setTimestamp(timestamp);
-					this.finalizeAndEmitSample(sample);
+					this.colorQueue.push(frame);
 				}
 			};
 
 			this.decoder = new VideoDecoder({
-				output: frame => sampleHandler(new VideoSample(frame)),
+				output: (frame) => {
+					try {
+						colorHandler(frame);
+					} catch (error) {
+						this.onError(error as Error);
+					}
+				},
 				error: onError,
 			});
 			this.decoder.configure(decoderConfig);
 		}
 	}
 
-	finalizeAndEmitSample(sample: VideoSample) {
-		// Round the timestamps to the time resolution
-		sample.setTimestamp(Math.round(sample.timestamp * this.timeResolution) / this.timeResolution);
-		sample.setDuration(Math.round(sample.duration * this.timeResolution) / this.timeResolution);
-		sample.setRotation(this.rotation);
-
-		this.onSample(sample);
-	}
-
 	getDecodeQueueSize() {
 		if (this.customDecoder) {
 			return this.customDecoderQueueSize;
 		} else {
 			assert(this.decoder);
-			return this.decoder.decodeQueueSize;
+
+			return Math.max(
+				this.decoder.decodeQueueSize,
+				this.alphaDecoder?.decodeQueueSize ?? 0,
+			);
 		}
 	}
 
 	decode(packet: EncodedPacket) {
 		if (this.codec === 'hevc' && this.currentPacketIndex > 0 && !this.raslSkipped) {
-			// If we're using HEVC, we need to make sure to skip any RASL slices that follow a non-IDR key frame such as
-			// CRA_NUT. This is because RASL slices cannot be decoded without data before the CRA_NUT. Browsers behave
-			// differently here: Chromium drops the packets, Safari throws a decoder error. Either way, it's not good
-			// and causes bugs upstream. So, let's take the dropping into our own hands.
-			const nalUnits = extractHevcNalUnits(packet.data, this.decoderConfig);
-			const hasRaslPicture = nalUnits.some((x) => {
-				const type = extractNalUnitTypeForHevc(x);
-				return type === HevcNalUnitType.RASL_N || type === HevcNalUnitType.RASL_R;
-			});
-
-			if (hasRaslPicture) {
+			if (this.hasHevcRaslPicture(packet.data)) {
 				return; // Drop
 			}
 
@@ -934,15 +922,218 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
 			}
 
 			this.decoder.decode(packet.toEncodedVideoChunk());
+			this.decodeAlphaData(packet);
 		}
 	}
 
+	decodeAlphaData(packet: EncodedPacket) {
+		if (!packet.sideData.alpha || this.mergerCreationFailed) {
+			// No alpha side data in the packet, most common case
+			this.pushNullAlphaFrame();
+			return;
+		}
+
+		if (!this.merger) {
+			try {
+				this.merger = new ColorAlphaMerger();
+			} catch (error) {
+				console.error('Due to an error, only color data will be decoded.', error);
+
+				this.mergerCreationFailed = true;
+				this.decodeAlphaData(packet); // Go again
+
+				return;
+			}
+		}
+
+		// Check if we need to set up the alpha decoder
+		if (!this.alphaDecoder) {
+			const alphaHandler = (frame: VideoFrame) => {
+				this.alphaDecoderQueueSize--;
+
+				if (this.colorQueue.length > 0) {
+					const colorFrame = this.colorQueue.shift();
+					assert(colorFrame !== undefined);
+
+					this.mergeAlpha(colorFrame, frame);
+				} else {
+					this.alphaQueue.push(frame);
+				}
+
+				// Check if any null frames have been queued for this point
+				this.decodedAlphaChunkCount++;
+				while (
+					this.nullAlphaFrameQueue.length > 0
+					&& this.nullAlphaFrameQueue[0] === this.decodedAlphaChunkCount
+				) {
+					this.nullAlphaFrameQueue.shift();
+
+					if (this.colorQueue.length > 0) {
+						const colorFrame = this.colorQueue.shift();
+						assert(colorFrame !== undefined);
+
+						this.mergeAlpha(colorFrame, null);
+					} else {
+						this.alphaQueue.push(null);
+					}
+				}
+			};
+
+			this.alphaDecoder = new VideoDecoder({
+				output: (frame) => {
+					try {
+						alphaHandler(frame);
+					} catch (error) {
+						this.onError(error as Error);
+					}
+				},
+				error: this.onError,
+			});
+			this.alphaDecoder.configure(this.decoderConfig);
+		}
+
+		const type = determineVideoPacketType(this.codec, this.decoderConfig, packet.sideData.alpha);
+
+		// Alpha packets might follow a different key frame rhythm than the main packets. Therefore, before we start
+		// decoding, we must first find a packet that's actually a key frame. Until then, we treat the image as opaque.
+		if (!this.alphaHadKeyframe) {
+			this.alphaHadKeyframe = type === 'key';
+		}
+
+		if (this.alphaHadKeyframe) {
+			// Same RASL skipping logic as for color, unlikely to be hit (since who uses HEVC with separate alpha??) but
+			// here for symmetry.
+			if (this.codec === 'hevc' && this.currentAlphaPacketIndex > 0 && !this.alphaRaslSkipped) {
+				if (this.hasHevcRaslPicture(packet.sideData.alpha)) {
+					this.pushNullAlphaFrame();
+					return;
+				}
+
+				this.alphaRaslSkipped = true;
+			}
+
+			this.currentAlphaPacketIndex++;
+			this.alphaDecoder.decode(packet.alphaToEncodedVideoChunk(type ?? packet.type));
+			this.alphaDecoderQueueSize++;
+		} else {
+			this.pushNullAlphaFrame();
+		}
+	}
+
+	pushNullAlphaFrame() {
+		if (this.alphaDecoderQueueSize === 0) {
+			// Easy
+			this.alphaQueue.push(null);
+		} else {
+			// There are still alpha chunks being decoded, so pushing `null` immediately would result in out-of-order
+			// data and be incorrect. Instead, we need to enqueue a "null frame" for when the current decoder workload
+			// has finished.
+			this.nullAlphaFrameQueue.push(this.decodedAlphaChunkCount + this.alphaDecoderQueueSize);
+		}
+	}
+
+	/**
+	 * If we're using HEVC, we need to make sure to skip any RASL slices that follow a non-IDR key frame such as
+	 * CRA_NUT. This is because RASL slices cannot be decoded without data before the CRA_NUT. Browsers behave
+	 * differently here: Chromium drops the packets, Safari throws a decoder error. Either way, it's not good
+	 * and causes bugs upstream. So, let's take the dropping into our own hands.
+	 */
+	hasHevcRaslPicture(packetData: Uint8Array) {
+		const nalUnits = extractHevcNalUnits(packetData, this.decoderConfig);
+		return nalUnits.some((x) => {
+			const type = extractNalUnitTypeForHevc(x);
+			return type === HevcNalUnitType.RASL_N || type === HevcNalUnitType.RASL_R;
+		});
+	}
+
+	/** Handler for the WebCodecs VideoDecoder for ironing out browser differences. */
+	sampleHandler(sample: VideoSample) {
+		if (isSafari()) {
+			// For correct B-frame handling, we don't just hand over the frames directly but instead add them to
+			// a queue, because we want to ensure frames are emitted in presentation order. We flush the queue
+			// each time we receive a frame with a timestamp larger than the highest we've seen so far, as we
+			// can sure that is not a B-frame. Typically, WebCodecs automatically guarantees that frames are
+			// emitted in presentation order, but Safari doesn't always follow this rule.
+			if (this.sampleQueue.length > 0 && (sample.timestamp >= last(this.sampleQueue)!.timestamp)) {
+				for (const sample of this.sampleQueue) {
+					this.finalizeAndEmitSample(sample);
+				}
+
+				this.sampleQueue.length = 0;
+			}
+
+			insertSorted(this.sampleQueue, sample, x => x.timestamp);
+		} else {
+			// Assign it the next earliest timestamp from the input. We do this because browsers, by spec, are
+			// required to emit decoded frames in presentation order *while* retaining the timestamp of their
+			// originating EncodedVideoChunk. For files with B-frames but no out-of-order timestamps (like a
+			// missing ctts box, for example), this causes a mismatch. We therefore fix the timestamps and
+			// ensure they are sorted by doing this.
+			const timestamp = this.inputTimestamps.shift();
+
+			// There's no way we'd have more decoded frames than encoded packets we passed in. Actually, the
+			// correspondence should be 1:1.
+			assert(timestamp !== undefined);
+
+			sample.setTimestamp(timestamp);
+			this.finalizeAndEmitSample(sample);
+		}
+	}
+
+	finalizeAndEmitSample(sample: VideoSample) {
+		// Round the timestamps to the time resolution
+		sample.setTimestamp(Math.round(sample.timestamp * this.timeResolution) / this.timeResolution);
+		sample.setDuration(Math.round(sample.duration * this.timeResolution) / this.timeResolution);
+		sample.setRotation(this.rotation);
+
+		this.onSample(sample);
+	}
+
+	mergeAlpha(color: VideoFrame, alpha: VideoFrame | null) {
+		if (!alpha) {
+			// Nothing needs to be merged
+			const finalSample = new VideoSample(color);
+			this.sampleHandler(finalSample);
+
+			return;
+		}
+
+		assert(this.merger);
+
+		this.merger.update(color, alpha);
+		color.close();
+		alpha.close();
+
+		const finalFrame = new VideoFrame(this.merger.canvas, {
+			timestamp: color.timestamp,
+			duration: color.duration ?? undefined,
+		});
+
+		const finalSample = new VideoSample(finalFrame);
+		this.sampleHandler(finalSample);
+	}
+
 	async flush() {
 		if (this.customDecoder) {
 			await this.customDecoderCallSerializer.call(() => this.customDecoder!.flush());
 		} else {
 			assert(this.decoder);
-			await this.decoder.flush();
+			await Promise.all([
+				this.decoder.flush(),
+				this.alphaDecoder?.flush(),
+			]);
+
+			this.colorQueue.forEach(x => x.close());
+			this.colorQueue.length = 0;
+			this.alphaQueue.forEach(x => x?.close());
+			this.alphaQueue.length = 0;
+
+			this.alphaHadKeyframe = false;
+			this.decodedAlphaChunkCount = 0;
+			this.alphaDecoderQueueSize = 0;
+			this.nullAlphaFrameQueue.length = 0;
+			this.currentAlphaPacketIndex = 0;
+			this.alphaRaslSkipped = false;
 		}
 
 		if (isSafari()) {
@@ -963,6 +1154,14 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
 		} else {
 			assert(this.decoder);
 			this.decoder.close();
+			this.alphaDecoder?.close();
+
+			this.colorQueue.forEach(x => x.close());
+			this.colorQueue.length = 0;
+			this.alphaQueue.forEach(x => x?.close());
+			this.alphaQueue.length = 0;
+
+			this.merger?.close();
 		}
 
 		for (const sample of this.sampleQueue) {
@@ -972,6 +1171,150 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
 	}
 }
 
+/** Utility class that merges together color and alpha information using simple WebGL 2 shaders. */
+class ColorAlphaMerger {
+	canvas: OffscreenCanvas | HTMLCanvasElement;
+	private gl: WebGL2RenderingContext;
+	private program: WebGLProgram;
+	private vao: WebGLVertexArrayObject;
+	private colorTexture: WebGLTexture;
+	private alphaTexture: WebGLTexture;
+
+	constructor() {
+		// Canvas will be resized later
+		if (typeof OffscreenCanvas !== 'undefined') {
+			// Prefer OffscreenCanvas for Worker environments
+			this.canvas = new OffscreenCanvas(300, 150);
+		} else {
+			this.canvas = document.createElement('canvas');
+		}
+
+		const gl = this.canvas.getContext('webgl2', {
+			premultipliedAlpha: false,
+		}) as unknown as WebGL2RenderingContext | null; // Casting because of some TypeScript weirdness
+		if (!gl) {
+			throw new Error('Couldn\'t acquire WebGL 2 context.');
+		}
+
+		this.gl = gl;
+		this.program = this.createProgram();
+		this.vao = this.createVAO();
+		this.colorTexture = this.createTexture();
+		this.alphaTexture = this.createTexture();
+
+		this.gl.useProgram(this.program);
+		this.gl.uniform1i(this.gl.getUniformLocation(this.program, 'u_colorTexture'), 0);
+		this.gl.uniform1i(this.gl.getUniformLocation(this.program, 'u_alphaTexture'), 1);
+	}
+
+	private createProgram(): WebGLProgram {
+		const vertexShader = this.createShader(this.gl.VERTEX_SHADER, `#version 300 es
+			in vec2 a_position;
+			in vec2 a_texCoord;
+			out vec2 v_texCoord;
+			
+			void main() {
+				gl_Position = vec4(a_position, 0.0, 1.0);
+				v_texCoord = a_texCoord;
+			}
+		`);
+
+		const fragmentShader = this.createShader(this.gl.FRAGMENT_SHADER, `#version 300 es
+			precision highp float;
+			
+			uniform sampler2D u_colorTexture;
+			uniform sampler2D u_alphaTexture;
+			in vec2 v_texCoord;
+			out vec4 fragColor;
+			
+			void main() {
+				vec3 color = texture(u_colorTexture, v_texCoord).rgb;
+				float alpha = texture(u_alphaTexture, v_texCoord).r;
+				fragColor = vec4(color, alpha);
+			}
+		`);
+
+		const program = this.gl.createProgram();
+		this.gl.attachShader(program, vertexShader);
+		this.gl.attachShader(program, fragmentShader);
+		this.gl.linkProgram(program);
+
+		return program;
+	}
+
+	private createShader(type: number, source: string): WebGLShader {
+		const shader = this.gl.createShader(type)!;
+		this.gl.shaderSource(shader, source);
+		this.gl.compileShader(shader);
+		return shader;
+	}
+
+	private createVAO(): WebGLVertexArrayObject {
+		const vao = this.gl.createVertexArray();
+		this.gl.bindVertexArray(vao);
+
+		const vertices = new Float32Array([
+			-1, -1, 0, 1,
+			1, -1, 1, 1,
+			-1, 1, 0, 0,
+			1, 1, 1, 0,
+		]);
+
+		const buffer = this.gl.createBuffer();
+		this.gl.bindBuffer(this.gl.ARRAY_BUFFER, buffer);
+		this.gl.bufferData(this.gl.ARRAY_BUFFER, vertices, this.gl.STATIC_DRAW);
+
+		const positionLocation = this.gl.getAttribLocation(this.program, 'a_position');
+		const texCoordLocation = this.gl.getAttribLocation(this.program, 'a_texCoord');
+
+		this.gl.enableVertexAttribArray(positionLocation);
+		this.gl.vertexAttribPointer(positionLocation, 2, this.gl.FLOAT, false, 16, 0);
+
+		this.gl.enableVertexAttribArray(texCoordLocation);
+		this.gl.vertexAttribPointer(texCoordLocation, 2, this.gl.FLOAT, false, 16, 8);
+
+		return vao;
+	}
+
+	private createTexture(): WebGLTexture {
+		const texture = this.gl.createTexture();
+
+		this.gl.bindTexture(this.gl.TEXTURE_2D, texture);
+		this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_S, this.gl.CLAMP_TO_EDGE);
+		this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_T, this.gl.CLAMP_TO_EDGE);
+		this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MIN_FILTER, this.gl.LINEAR);
+		this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MAG_FILTER, this.gl.LINEAR);
+
+		return texture;
+	}
+
+	update(color: VideoFrame, alpha: VideoFrame): void {
+		if (color.displayWidth !== this.canvas.width || color.displayHeight !== this.canvas.height) {
+			this.canvas.width = color.displayWidth;
+			this.canvas.height = color.displayHeight;
+		}
+
+		this.gl.activeTexture(this.gl.TEXTURE0);
+		this.gl.bindTexture(this.gl.TEXTURE_2D, this.colorTexture);
+		this.gl.texImage2D(this.gl.TEXTURE_2D, 0, this.gl.RGBA, this.gl.RGBA, this.gl.UNSIGNED_BYTE, color);
+
+		this.gl.activeTexture(this.gl.TEXTURE1);
+		this.gl.bindTexture(this.gl.TEXTURE_2D, this.alphaTexture);
+		this.gl.texImage2D(this.gl.TEXTURE_2D, 0, this.gl.RGBA, this.gl.RGBA, this.gl.UNSIGNED_BYTE, alpha);
+
+		this.gl.viewport(0, 0, this.canvas.width, this.canvas.height);
+		this.gl.clear(this.gl.COLOR_BUFFER_BIT);
+
+		this.gl.bindVertexArray(this.vao);
+		this.gl.drawArrays(this.gl.TRIANGLE_STRIP, 0, 4);
+	}
+
+	close() {
+		this.gl.getExtension('WEBGL_lose_context')?.loseContext();
+		this.gl = null as unknown as WebGL2RenderingContext;
+	}
+}
+
 /**
  * A sink that retrieves decoded video samples (video frames) from a video track.
  * @group Media sinks
@@ -995,7 +1338,7 @@ export class VideoSampleSink extends BaseMediaSampleSink<VideoSample> {
 	/** @internal */
 	async _createDecoder(
 		onSample: (sample: VideoSample) => unknown,
-		onError: (error: DOMException) => unknown,
+		onError: (error: Error) => unknown,
 	) {
 		if (!(await this._track.canDecode())) {
 			throw new Error(
@@ -1078,6 +1421,11 @@ export type WrappedCanvas = {
  * @public
  */
 export type CanvasSinkOptions = {
+	/**
+	 * Whether the output canvases should have transparency instead of a black background. Defaults to `false`. Set
+	 * this to `true` when using this sink to read transparent videos.
+	 */
+	alpha?: boolean;
 	/**
 	 * The width of the output canvas in pixels, defaulting to the display width of the video track. If height is not
 	 * set, it will be deduced automatically based on aspect ratio.
@@ -1130,6 +1478,8 @@ export class CanvasSink {
 	/** @internal */
 	_videoTrack: InputVideoTrack;
 	/** @internal */
+	_alpha: boolean;
+	/** @internal */
 	_width: number;
 	/** @internal */
 	_height: number;
@@ -1154,6 +1504,9 @@ export class CanvasSink {
 		if (options && typeof options !== 'object') {
 			throw new TypeError('options must be an object.');
 		}
+		if (options.alpha !== undefined && typeof options.alpha !== 'boolean') {
+			throw new TypeError('options.alpha, when provided, must be a boolean.');
+		}
 		if (options.width !== undefined && (!Number.isInteger(options.width) || options.width <= 0)) {
 			throw new TypeError('options.width, when defined, must be a positive integer.');
 		}
@@ -1214,6 +1567,7 @@ export class CanvasSink {
 		}
 
 		this._videoTrack = videoTrack;
+		this._alpha = options.alpha ?? false;
 		this._width = width;
 		this._height = height;
 		this._rotation = rotation;
@@ -1250,14 +1604,14 @@ export class CanvasSink {
 		}
 
 		const context = canvas.getContext('2d', {
-			alpha: isFirefox(), // Firefox has VideoFrame glitches with opaque canvases
+			alpha: this._alpha || isFirefox(), // Firefox has VideoFrame glitches with opaque canvases
 		}) as CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;
 		assert(context);
 
 		context.resetTransform();
 
 		if (!canvasIsNew) {
-			if (isFirefox()) {
+			if (!this._alpha && isFirefox()) {
 				context.fillStyle = 'black';
 				context.fillRect(0, 0, this._width, this._height);
 			} else {
@@ -1338,7 +1692,7 @@ class AudioDecoderWrapper extends DecoderWrapper<AudioSample> {
 
 	constructor(
 		onSample: (sample: AudioSample) => unknown,
-		onError: (error: DOMException) => unknown,
+		onError: (error: Error) => unknown,
 		codec: AudioCodec,
 		decoderConfig: AudioDecoderConfig,
 	) {
@@ -1390,7 +1744,13 @@ class AudioDecoderWrapper extends DecoderWrapper<AudioSample> {
 			void this.customDecoderCallSerializer.call(() => this.customDecoder!.init());
 		} else {
 			this.decoder = new AudioDecoder({
-				output: data => sampleHandler(new AudioSample(data)),
+				output: (data) => {
+					try {
+						sampleHandler(new AudioSample(data));
+					} catch (error) {
+						this.onError(error as Error);
+					}
+				},
 				error: onError,
 			});
 			this.decoder.configure(decoderConfig);
@@ -1455,7 +1815,7 @@ class PcmAudioDecoderWrapper extends DecoderWrapper<AudioSample> {
 
 	constructor(
 		onSample: (sample: AudioSample) => unknown,
-		onError: (error: DOMException) => unknown,
+		onError: (error: Error) => unknown,
 		public decoderConfig: AudioDecoderConfig,
 	) {
 		super(onSample, onError);
@@ -1645,7 +2005,7 @@ export class AudioSampleSink extends BaseMediaSampleSink<AudioSample> {
 	/** @internal */
 	async _createDecoder(
 		onSample: (sample: AudioSample) => unknown,
-		onError: (error: DOMException) => unknown,
+		onError: (error: Error) => unknown,
 	) {
 		if (!(await this._track.canDecode())) {
 			throw new Error(

src/media-source.ts

@@ -37,7 +37,7 @@ import {
 	customVideoEncoders,
 	customAudioEncoders,
 } from './custom-coder';
-import { EncodedPacket } from './packet';
+import { EncodedPacket, EncodedPacketSideData } from './packet';
 import { AudioSample, VideoSample } from './sample';
 import {
 	AudioEncodingConfig,
@@ -213,12 +213,19 @@ class VideoEncoderWrapper {
 	private customEncoderCallSerializer = new CallSerializer();
 	private customEncoderQueueSize = 0;
 
+	// Alpha stuff
+	private alphaEncoder: VideoEncoder | null = null;
+	private splitter: ColorAlphaSplitter | null = null;
+	private splitterCreationFailed = false;
+	private alphaFrameQueue: (VideoFrame | null)[] = [];
+
 	/**
 	 * Encoders typically throw their errors "out of band", meaning asynchronously in some other execution context.
 	 * However, we want to surface these errors to the user within the normal control flow, so they don't go uncaught.
 	 * So, we keep track of the encoder error and throw it as soon as we get the chance.
 	 */
-	private encoderError: Error | null = null;
+	private error: Error | null = null;
+	private errorNeedsNewStack = true;
 
 	constructor(private source: VideoSource, private encodingConfig: VideoEncodingConfig) {}
 
@@ -329,7 +336,7 @@ class VideoEncoderWrapper {
 				const promise = this.customEncoderCallSerializer
 					.call(() => this.customEncoder!.encode(clonedSample, finalEncodeOptions))
 					.then(() => this.customEncoderQueueSize--)
-					.catch((error: Error) => this.encoderError ??= error)
+					.catch((error: Error) => this.error ??= error)
 					.finally(() => {
 						clonedSample.close();
 						// `videoSample` gets closed in the finally block at the end of the method
@@ -340,9 +347,49 @@ class VideoEncoderWrapper {
 				}
 			} else {
 				assert(this.encoder);
+
 				const videoFrame = videoSample.toVideoFrame();
-				this.encoder.encode(videoFrame, finalEncodeOptions);
-				videoFrame.close();
+
+				if (!this.alphaEncoder) {
+					// No alpha encoder, simple case
+					this.encoder.encode(videoFrame, finalEncodeOptions);
+					videoFrame.close();
+				} else {
+					// We're expected to encode alpha as well
+					const frameDefinitelyHasNoAlpha = !!videoFrame.format && !videoFrame.format.includes('A');
+
+					if (frameDefinitelyHasNoAlpha || this.splitterCreationFailed) {
+						this.alphaFrameQueue.push(null);
+						this.encoder.encode(videoFrame, finalEncodeOptions);
+						videoFrame.close();
+					} else {
+						const width = videoFrame.displayWidth;
+						const height = videoFrame.displayHeight;
+
+						if (!this.splitter) {
+							try {
+								this.splitter = new ColorAlphaSplitter(width, height);
+							} catch (error) {
+								console.error('Due to an error, only color data will be encoded.', error);
+
+								this.splitterCreationFailed = true;
+								this.alphaFrameQueue.push(null);
+								this.encoder.encode(videoFrame, finalEncodeOptions);
+								videoFrame.close();
+							}
+						}
+
+						if (this.splitter) {
+							const colorFrame = this.splitter.extractColor(videoFrame);
+							const alphaFrame = this.splitter.extractAlpha(videoFrame);
+
+							this.alphaFrameQueue.push(alphaFrame);
+							this.encoder.encode(colorFrame, finalEncodeOptions);
+							colorFrame.close();
+							videoFrame.close();
+						}
+					}
+				}
 
 				if (shouldClose) {
 					videoSample.close();
@@ -364,10 +411,6 @@ class VideoEncoderWrapper {
 	}
 
 	private ensureEncoder(videoSample: VideoSample) {
-		if (this.encoder) {
-			return;
-		}
-
 		const encoderError = new Error();
 		this.ensureEncoderPromise = (async () => {
 			const encoderConfig = buildVideoEncoderConfig({
@@ -400,7 +443,11 @@ class VideoEncoderWrapper {
 					}
 
 					this.encodingConfig.onEncodedPacket?.(packet, meta);
-					void this.muxer!.addEncodedVideoPacket(this.source._connectedTrack!, packet, meta);
+					void this.muxer!.addEncodedVideoPacket(this.source._connectedTrack!, packet, meta)
+						.catch((error) => {
+							this.error ??= error;
+							this.errorNeedsNewStack = false;
+						});
 				};
 
 				await this.customEncoder.init();
@@ -409,6 +456,15 @@ class VideoEncoderWrapper {
 					throw new Error('VideoEncoder is not supported by this browser.');
 				}
 
+				encoderConfig.alpha = 'discard'; // Since we handle alpha ourselves
+
+				if (this.encodingConfig.alpha === 'keep') {
+					// Encoding alpha requires using two parallel encoders, so we need to make sure they stay in sync
+					// and that neither of them drops frames. Setting latencyMode to 'quality' achieves this, because
+					// "User Agents MUST not drop frames to achieve the target bitrate and/or framerate."
+					encoderConfig.latencyMode = 'quality';
+				}
+
 				const hasOddDimension = encoderConfig.width % 2 === 1 || encoderConfig.height % 2 === 1;
 				if (
 					hasOddDimension
@@ -432,19 +488,116 @@ class VideoEncoderWrapper {
 					);
 				}
 
+				/** Queue of color chunks waiting for their alpha counterpart. */
+				const colorChunkQueue: {
+					chunk: EncodedVideoChunk;
+					meta: EncodedVideoChunkMetadata | undefined;
+				}[] = [];
+				/** Each value is the number of encoded alpha chunks at which a null alpha chunk should be added. */
+				const nullAlphaChunkQueue: number[] = [];
+				let encodedAlphaChunkCount = 0;
+				let alphaEncoderQueue = 0;
+
+				const addPacket = (
+					colorChunk: EncodedVideoChunk,
+					alphaChunk: EncodedVideoChunk | null,
+					meta: EncodedVideoChunkMetadata | undefined,
+				) => {
+					const sideData: EncodedPacketSideData = {};
+
+					if (alphaChunk) {
+						const alphaData = new Uint8Array(alphaChunk.byteLength);
+						alphaChunk.copyTo(alphaData);
+
+						sideData.alpha = alphaData;
+					}
+
+					const packet = EncodedPacket.fromEncodedChunk(colorChunk, sideData);
+
+					this.encodingConfig.onEncodedPacket?.(packet, meta);
+					void this.muxer!.addEncodedVideoPacket(this.source._connectedTrack!, packet, meta)
+						.catch((error) => {
+							this.error ??= error;
+							this.errorNeedsNewStack = false;
+						});
+				};
+
 				this.encoder = new VideoEncoder({
 					output: (chunk, meta) => {
-						const packet = EncodedPacket.fromEncodedChunk(chunk);
+						if (!this.alphaEncoder) {
+							// We're done
+							addPacket(chunk, null, meta);
+							return;
+						}
 
-						this.encodingConfig.onEncodedPacket?.(packet, meta);
-						void this.muxer!.addEncodedVideoPacket(this.source._connectedTrack!, packet, meta);
+						const alphaFrame = this.alphaFrameQueue.shift();
+						assert(alphaFrame !== undefined);
+
+						if (alphaFrame) {
+							this.alphaEncoder.encode(alphaFrame, {
+								// Crucial: The alpha frame is forced to be a key frame whenever the color frame
+								// also is. Without this, playback can glitch and even crash in some browsers.
+								// This is the reason why the two encoders are wired in series and not in parallel.
+								keyFrame: chunk.type === 'key',
+							});
+							alphaEncoderQueue++;
+							alphaFrame.close();
+							colorChunkQueue.push({ chunk, meta });
+						} else {
+							// There was no alpha component for this frame
+							if (alphaEncoderQueue === 0) {
+								// No pending alpha encodes either, so we're done
+								addPacket(chunk, null, meta);
+							} else {
+								// There are still alpha encodes pending, so we can't add the packet immediately since
+								// we'd end up with out-of-order packets. Instead, let's queue a null alpha chunk to be
+								// added in the future, after the current encoder workload has completed:
+								nullAlphaChunkQueue.push(encodedAlphaChunkCount + alphaEncoderQueue);
+								colorChunkQueue.push({ chunk, meta });
+							}
+						}
 					},
 					error: (error) => {
 						error.stack = encoderError.stack; // Provide a more useful stack trace
-						this.encoderError ??= error;
+						this.error ??= error;
 					},
 				});
 				this.encoder.configure(encoderConfig);
+
+				if (this.encodingConfig.alpha === 'keep') {
+					// We need to encode alpha as well, which we do with a separate encoder
+					this.alphaEncoder = new VideoEncoder({
+						// We ignore the alpha chunk's metadata
+						// eslint-disable-next-line @typescript-eslint/no-unused-vars
+						output: (chunk, meta) => {
+							alphaEncoderQueue--;
+
+							// There has to be a color chunk because the encoders are wired in series
+							const colorChunk = colorChunkQueue.shift();
+							assert(colorChunk !== undefined);
+
+							addPacket(colorChunk.chunk, chunk, colorChunk.meta);
+
+							// See if there are any null alpha chunks queued up
+							encodedAlphaChunkCount++;
+							while (
+								nullAlphaChunkQueue.length > 0
+								&& nullAlphaChunkQueue[0] === encodedAlphaChunkCount
+							) {
+								nullAlphaChunkQueue.shift();
+								const colorChunk = colorChunkQueue.shift();
+								assert(colorChunk !== undefined);
+
+								addPacket(colorChunk.chunk, null, colorChunk.meta);
+							}
+						},
+						error: (error) => {
+							error.stack = encoderError.stack; // Provide a more useful stack trace
+							this.error ??= error;
+						},
+					});
+					this.alphaEncoder.configure(encoderConfig);
+				}
 			}
 
 			assert(this.source._connectedTrack);
@@ -465,12 +618,21 @@ class VideoEncoderWrapper {
 			await this.customEncoderCallSerializer.call(() => this.customEncoder!.close());
 		} else if (this.encoder) {
 			if (!forceClose) {
+				// These are wired in series, therefore they must also be flushed in series
 				await this.encoder.flush();
+				await this.alphaEncoder?.flush();
 			}
 
 			if (this.encoder.state !== 'closed') {
 				this.encoder.close();
 			}
+			if (this.alphaEncoder && this.alphaEncoder.state !== 'closed') {
+				this.alphaEncoder.close();
+			}
+
+			this.alphaFrameQueue.forEach(x => x?.close());
+
+			this.splitter?.close();
 		}
 
 		if (!forceClose) this.checkForEncoderError();
@@ -480,18 +642,293 @@ class VideoEncoderWrapper {
 		if (this.customEncoder) {
 			return this.customEncoderQueueSize;
 		} else {
+			// Because the color and alpha encoders are wired in series, there's no need to also include the alpha
+			// encoder's queue size here
 			return this.encoder?.encodeQueueSize ?? 0;
 		}
 	}
 
 	checkForEncoderError() {
-		if (this.encoderError) {
-			this.encoderError.stack = new Error().stack; // Provide an even more useful stack trace
-			throw this.encoderError;
+		if (this.error) {
+			if (this.errorNeedsNewStack) {
+				this.error.stack = new Error().stack; // Provide an even more useful stack trace
+			}
+
+			throw this.error;
 		}
 	}
 }
 
+/** Utility class for splitting a composite frame into separate color and alpha components. */
+class ColorAlphaSplitter {
+	canvas: OffscreenCanvas | HTMLCanvasElement;
+
+	private gl: WebGL2RenderingContext;
+	private colorProgram: WebGLProgram;
+	private alphaProgram: WebGLProgram;
+	private vao: WebGLVertexArrayObject;
+	private sourceTexture: WebGLTexture;
+	private lastFrame: VideoFrame | null = null;
+	private alphaResolutionLocation: WebGLUniformLocation;
+
+	constructor(initialWidth: number, initialHeight: number) {
+		if (typeof OffscreenCanvas !== 'undefined') {
+			this.canvas = new OffscreenCanvas(initialWidth, initialHeight);
+		} else {
+			this.canvas = document.createElement('canvas');
+			this.canvas.width = initialWidth;
+			this.canvas.height = initialHeight;
+		}
+
+		const gl = this.canvas.getContext('webgl2', {
+			alpha: true, // Needed due to the YUV thing we do for alpha
+		}) as unknown as WebGL2RenderingContext | null; // Casting because of some TypeScript weirdness
+		if (!gl) {
+			throw new Error('Couldn\'t acquire WebGL 2 context.');
+		}
+
+		this.gl = gl;
+
+		this.colorProgram = this.createColorProgram();
+		this.alphaProgram = this.createAlphaProgram();
+		this.vao = this.createVAO();
+		this.sourceTexture = this.createTexture();
+
+		this.alphaResolutionLocation = this.gl.getUniformLocation(this.alphaProgram, 'u_resolution')!;
+
+		this.gl.useProgram(this.colorProgram);
+		this.gl.uniform1i(this.gl.getUniformLocation(this.colorProgram, 'u_sourceTexture'), 0);
+
+		this.gl.useProgram(this.alphaProgram);
+		this.gl.uniform1i(this.gl.getUniformLocation(this.alphaProgram, 'u_sourceTexture'), 0);
+	}
+
+	private createVertexShader(): WebGLShader {
+		return this.createShader(this.gl.VERTEX_SHADER, `#version 300 es
+			in vec2 a_position;
+			in vec2 a_texCoord;
+			out vec2 v_texCoord;
+			
+			void main() {
+				gl_Position = vec4(a_position, 0.0, 1.0);
+				v_texCoord = a_texCoord;
+			}
+		`);
+	}
+
+	private createColorProgram(): WebGLProgram {
+		const vertexShader = this.createVertexShader();
+
+		// This shader is simple, simply copy the color information while setting alpha to 1
+		const fragmentShader = this.createShader(this.gl.FRAGMENT_SHADER, `#version 300 es
+			precision highp float;
+			
+			uniform sampler2D u_sourceTexture;
+			in vec2 v_texCoord;
+			out vec4 fragColor;
+			
+			void main() {
+				vec4 source = texture(u_sourceTexture, v_texCoord);
+				fragColor = vec4(source.rgb, 1.0);
+			}
+		`);
+
+		const program = this.gl.createProgram();
+		this.gl.attachShader(program, vertexShader);
+		this.gl.attachShader(program, fragmentShader);
+		this.gl.linkProgram(program);
+
+		return program;
+	}
+
+	private createAlphaProgram(): WebGLProgram {
+		const vertexShader = this.createVertexShader();
+
+		// This shader's more complex. The main reason is that this shader writes data in I420 (yuv420) pixel format
+		// instead of regular RGBA. In other words, we use the shader to write out I420 data into an RGBA canvas, which
+		// we then later read out with JavaScript. The reason being that browsers weirdly encode canvases and mess up
+		// the color spaces, and the only way to have full control over the color space is by outputting YUV data
+		// directly (avoiding the RGB conversion). Doing this conversion in JS is painfully slow, so let's utlize the
+		// GPU since we're already calling it anyway.
+		const fragmentShader = this.createShader(this.gl.FRAGMENT_SHADER, `#version 300 es
+			precision highp float;
+			
+			uniform sampler2D u_sourceTexture;
+			uniform vec2 u_resolution; // The width and height of the canvas
+			in vec2 v_texCoord;
+			out vec4 fragColor;
+
+			// This function determines the value for a single byte in the YUV stream
+			float getByteValue(float byteOffset) {
+				float width = u_resolution.x;
+				float height = u_resolution.y;
+
+				float yPlaneSize = width * height;
+
+				if (byteOffset < yPlaneSize) {
+					// This byte is in the luma plane. Find the corresponding pixel coordinates to sample from
+					float y = floor(byteOffset / width);
+					float x = mod(byteOffset, width);
+					
+					// Add 0.5 to sample the center of the texel
+					vec2 sampleCoord = (vec2(x, y) + 0.5) / u_resolution;
+					
+					// The luma value is the alpha from the source texture
+					return texture(u_sourceTexture, sampleCoord).a;
+				} else {
+					// Write a fixed value for chroma and beyond
+					return 128.0 / 255.0;
+				}
+			}
+			
+			void main() {
+				// Each fragment writes 4 bytes (R, G, B, A)
+				float pixelIndex = floor(gl_FragCoord.y) * u_resolution.x + floor(gl_FragCoord.x);
+				float baseByteOffset = pixelIndex * 4.0;
+
+				vec4 result;
+				for (int i = 0; i < 4; i++) {
+					float currentByteOffset = baseByteOffset + float(i);
+					result[i] = getByteValue(currentByteOffset);
+				}
+				
+				fragColor = result;
+			}
+		`);
+
+		const program = this.gl.createProgram();
+		this.gl.attachShader(program, vertexShader);
+		this.gl.attachShader(program, fragmentShader);
+		this.gl.linkProgram(program);
+
+		return program;
+	}
+
+	private createShader(type: number, source: string): WebGLShader {
+		const shader = this.gl.createShader(type)!;
+		this.gl.shaderSource(shader, source);
+		this.gl.compileShader(shader);
+		if (!this.gl.getShaderParameter(shader, this.gl.COMPILE_STATUS)) {
+			console.error('Shader compile error:', this.gl.getShaderInfoLog(shader));
+		}
+		return shader;
+	}
+
+	private createVAO(): WebGLVertexArrayObject {
+		const vao = this.gl.createVertexArray();
+		this.gl.bindVertexArray(vao);
+
+		const vertices = new Float32Array([
+			-1, -1, 0, 1,
+			1, -1, 1, 1,
+			-1, 1, 0, 0,
+			1, 1, 1, 0,
+		]);
+
+		const buffer = this.gl.createBuffer();
+		this.gl.bindBuffer(this.gl.ARRAY_BUFFER, buffer);
+		this.gl.bufferData(this.gl.ARRAY_BUFFER, vertices, this.gl.STATIC_DRAW);
+
+		const positionLocation = this.gl.getAttribLocation(this.colorProgram, 'a_position');
+		const texCoordLocation = this.gl.getAttribLocation(this.colorProgram, 'a_texCoord');
+
+		this.gl.enableVertexAttribArray(positionLocation);
+		this.gl.vertexAttribPointer(positionLocation, 2, this.gl.FLOAT, false, 16, 0);
+
+		this.gl.enableVertexAttribArray(texCoordLocation);
+		this.gl.vertexAttribPointer(texCoordLocation, 2, this.gl.FLOAT, false, 16, 8);
+
+		return vao;
+	}
+
+	private createTexture(): WebGLTexture {
+		const texture = this.gl.createTexture();
+
+		this.gl.bindTexture(this.gl.TEXTURE_2D, texture);
+		this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_S, this.gl.CLAMP_TO_EDGE);
+		this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_T, this.gl.CLAMP_TO_EDGE);
+		this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MIN_FILTER, this.gl.LINEAR);
+		this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MAG_FILTER, this.gl.LINEAR);
+
+		return texture;
+	}
+
+	private updateTexture(sourceFrame: VideoFrame): void {
+		if (this.lastFrame === sourceFrame) {
+			return;
+		}
+
+		if (sourceFrame.displayWidth !== this.canvas.width || sourceFrame.displayHeight !== this.canvas.height) {
+			this.canvas.width = sourceFrame.displayWidth;
+			this.canvas.height = sourceFrame.displayHeight;
+		}
+
+		this.gl.activeTexture(this.gl.TEXTURE0);
+		this.gl.bindTexture(this.gl.TEXTURE_2D, this.sourceTexture);
+		this.gl.texImage2D(this.gl.TEXTURE_2D, 0, this.gl.RGBA, this.gl.RGBA, this.gl.UNSIGNED_BYTE, sourceFrame);
+
+		this.lastFrame = sourceFrame;
+	}
+
+	extractColor(sourceFrame: VideoFrame) {
+		this.updateTexture(sourceFrame);
+
+		this.gl.useProgram(this.colorProgram);
+		this.gl.viewport(0, 0, this.canvas.width, this.canvas.height);
+		this.gl.clear(this.gl.COLOR_BUFFER_BIT);
+		this.gl.bindVertexArray(this.vao);
+		this.gl.drawArrays(this.gl.TRIANGLE_STRIP, 0, 4);
+
+		return new VideoFrame(this.canvas, {
+			timestamp: sourceFrame.timestamp,
+			duration: sourceFrame.duration ?? undefined,
+			alpha: 'discard',
+		});
+	}
+
+	extractAlpha(sourceFrame: VideoFrame) {
+		this.updateTexture(sourceFrame);
+
+		this.gl.useProgram(this.alphaProgram);
+		this.gl.uniform2f(this.alphaResolutionLocation, this.canvas.width, this.canvas.height);
+
+		this.gl.viewport(0, 0, this.canvas.width, this.canvas.height);
+		this.gl.clear(this.gl.COLOR_BUFFER_BIT);
+		this.gl.bindVertexArray(this.vao);
+		this.gl.drawArrays(this.gl.TRIANGLE_STRIP, 0, 4);
+
+		const { width, height } = this.canvas;
+
+		const chromaSamples = Math.ceil(width / 2) * Math.ceil(height / 2);
+		const yuvSize = width * height + chromaSamples * 2;
+		const requiredHeight = Math.ceil(yuvSize / (width * 4));
+
+		let yuv = new Uint8Array(4 * width * requiredHeight);
+		this.gl.readPixels(0, 0, width, requiredHeight, this.gl.RGBA, this.gl.UNSIGNED_BYTE, yuv);
+		yuv = yuv.subarray(0, yuvSize);
+
+		assert(yuv[width * height] === 128); // Where chroma data starts
+		assert(yuv[yuv.length - 1] === 128); // Assert the YUV data has been fully written
+
+		// Defining this separately because TypeScript doesn't know `transfer` and I can't be bothered to do declaration
+		// merging right now
+		const init = {
+			format: 'I420' as const,
+			codedWidth: width,
+			codedHeight: height,
+			timestamp: sourceFrame.timestamp,
+			duration: sourceFrame.duration ?? undefined,
+			transfer: [yuv.buffer],
+		};
+		return new VideoFrame(yuv, init);
+	}
+
+	close() {
+		this.gl.getExtension('WEBGL_lose_context')?.loseContext();
+		this.gl = null as unknown as WebGL2RenderingContext;
+	}
+}
+
 /**
  * This source can be used to add raw, unencoded video samples (frames) to an output video track. These frames will
  * automatically be encoded and then piped into the output.
@@ -858,7 +1295,8 @@ class AudioEncoderWrapper {
 	 * However, we want to surface these errors to the user within the normal control flow, so they don't go uncaught.
 	 * So, we keep track of the encoder error and throw it as soon as we get the chance.
 	 */
-	private encoderError: Error | null = null;
+	private error: Error | null = null;
+	private errorNeedsNewStack = true;
 
 	constructor(private source: AudioSource, private encodingConfig: AudioEncodingConfig) {}
 
@@ -909,7 +1347,7 @@ class AudioEncoderWrapper {
 				const promise = this.customEncoderCallSerializer
 					.call(() => this.customEncoder!.encode(clonedSample))
 					.then(() => this.customEncoderQueueSize--)
-					.catch((error: Error) => this.encoderError ??= error)
+					.catch((error: Error) => this.error ??= error)
 					.finally(() => {
 						clonedSample.close();
 						// `audioSample` gets closed in the finally block at the end of the method
@@ -1018,10 +1456,6 @@ class AudioEncoderWrapper {
 	}
 
 	private ensureEncoder(audioSample: AudioSample) {
-		if (this.encoderInitialized) {
-			return;
-		}
-
 		const encoderError = new Error();
 		this.ensureEncoderPromise = (async () => {
 			const { numberOfChannels, sampleRate } = audioSample;
@@ -1055,7 +1489,11 @@ class AudioEncoderWrapper {
 					}
 
 					this.encodingConfig.onEncodedPacket?.(packet, meta);
-					void this.muxer!.addEncodedAudioPacket(this.source._connectedTrack!, packet, meta);
+					void this.muxer!.addEncodedAudioPacket(this.source._connectedTrack!, packet, meta)
+						.catch((error) => {
+							this.error ??= error;
+							this.errorNeedsNewStack = false;
+						});
 				};
 
 				await this.customEncoder.init();
@@ -1080,11 +1518,15 @@ class AudioEncoderWrapper {
 						const packet = EncodedPacket.fromEncodedChunk(chunk);
 
 						this.encodingConfig.onEncodedPacket?.(packet, meta);
-						void this.muxer!.addEncodedAudioPacket(this.source._connectedTrack!, packet, meta);
+						void this.muxer!.addEncodedAudioPacket(this.source._connectedTrack!, packet, meta)
+							.catch((error) => {
+								this.error ??= error;
+								this.errorNeedsNewStack = false;
+							});
 					},
 					error: (error) => {
 						error.stack = encoderError.stack; // Provide a more useful stack trace
-						this.encoderError ??= error;
+						this.error ??= error;
 					},
 				});
 				this.encoder.configure(encoderConfig);
@@ -1223,9 +1665,12 @@ class AudioEncoderWrapper {
 	}
 
 	checkForEncoderError() {
-		if (this.encoderError) {
-			this.encoderError.stack = new Error().stack; // Provide an even more useful stack trace
-			throw this.encoderError;
+		if (this.error) {
+			if (this.errorNeedsNewStack) {
+				this.error.stack = new Error().stack; // Provide an even more useful stack trace
+			}
+
+			throw this.error;
 		}
 	}
 }
@@ -1679,6 +2124,8 @@ export abstract class SubtitleSource extends MediaSource {
 export class TextSubtitleSource extends SubtitleSource {
 	/** @internal */
 	private _parser: SubtitleParser;
+	/** @internal */
+	private _error: Error | null = null;
 
 	/** Creates a new {@link TextSubtitleSource} where added text chunks are in the specified `codec`. */
 	constructor(codec: SubtitleCodec) {
@@ -1686,8 +2133,12 @@ export class TextSubtitleSource extends SubtitleSource {
 
 		this._parser = new SubtitleParser({
 			codec,
-			output: (cue, metadata) =>
-				this._connectedTrack?.output._muxer.addSubtitleCue(this._connectedTrack, cue, metadata),
+			output: (cue, metadata) => {
+				void this._connectedTrack?.output._muxer.addSubtitleCue(this._connectedTrack, cue, metadata)
+					.catch((error) => {
+						this._error ??= error;
+					});
+			},
 		});
 	}
 
@@ -1703,9 +2154,25 @@ export class TextSubtitleSource extends SubtitleSource {
 			throw new TypeError('text must be a string.');
 		}
 
+		this._checkForError();
+
 		this._ensureValidAdd();
 		this._parser.parse(text);
 
 		return this._connectedTrack!.output._muxer.mutex.currentPromise;
 	}
+
+	/** @internal */
+	_checkForError() {
+		if (this._error) {
+			throw this._error;
+		}
+	}
+
+	/** @internal */
+	override async _flushAndClose(forceClose: boolean) {
+		if (!forceClose) {
+			this._checkForError();
+		}
+	}
 }

src/misc.ts

@@ -155,7 +155,7 @@ export const writeBits = (bytes: Uint8Array, start: number, end: number, value:
 };
 
 export const toUint8Array = (source: AllowSharedBufferSource): Uint8Array => {
-	if (source instanceof Uint8Array) {
+	if (source.constructor === Uint8Array) { // We want a true Uint8Array, not something that extends it like Buffer
 		return source;
 	} else if (source instanceof ArrayBuffer) {
 		return new Uint8Array(source);
@@ -165,7 +165,7 @@ export const toUint8Array = (source: AllowSharedBufferSource): Uint8Array => {
 };
 
 export const toDataView = (source: AllowSharedBufferSource) => {
-	if (source instanceof DataView) {
+	if (source.constructor === DataView) {
 		return source;
 	} else if (source instanceof ArrayBuffer) {
 		return new DataView(source);
@@ -780,3 +780,9 @@ export const uint8ArraysAreEqual = (a: Uint8Array, b: Uint8Array) => {
 
 	return true;
 };
+
+export const polyfillSymbolDispose = () => {
+	// https://www.typescriptlang.org/docs/handbook/release-notes/typescript-5-2.html
+	// @ts-expect-error Readonly
+	Symbol.dispose ??= Symbol('Symbol.dispose');
+};

src/output-format.ts

@@ -403,6 +403,10 @@ export type MkvOutputFormatOptions = {
 
 /**
  * Matroska file format.
+ *
+ * Supports writing transparent video. For a video track to be marked as transparent, the first packet added must
+ * contain alpha side data.
+ *
  * @group Output formats
  * @public
  */
@@ -490,6 +494,10 @@ export type WebMOutputFormatOptions = MkvOutputFormatOptions;
 
 /**
  * WebM file format, based on Matroska.
+ *
+ * Supports writing transparent video. For a video track to be marked as transparent, the first packet added must
+ * contain alpha side data.
+ *
  * @group Output formats
  * @public
  */

src/packet.ts

@@ -18,6 +18,24 @@ export const PLACEHOLDER_DATA = new Uint8Array(0);
  */
 export type PacketType = 'key' | 'delta';
 
+/**
+ * Holds additional data accompanying an {@link EncodedPacket}.
+ * @group Packets
+ * @public
+ */
+export type EncodedPacketSideData = {
+	/**
+	 * An encoded alpha frame, encoded with the same codec as the packet. Typically used for transparent videos, where
+	 * the alpha information is stored separately from the color information.
+	 */
+	alpha?: Uint8Array;
+	/**
+	 * The actual byte length of the alpha data. This field is useful for metadata-only packets where the
+	 * `alpha` field contains no bytes.
+	 */
+	alphaByteLength?: number;
+};
+
 /**
  * Represents an encoded chunk of media. Mainly used as an expressive wrapper around WebCodecs API's
  * [`EncodedVideoChunk`](https://developer.mozilla.org/en-US/docs/Web/API/EncodedVideoChunk) and
@@ -33,6 +51,9 @@ export class EncodedPacket {
 	 */
 	readonly byteLength: number;
 
+	/** Additional data carried with this packet. */
+	readonly sideData: EncodedPacketSideData;
+
 	/** Creates a new {@link EncodedPacket} from raw bytes and timing information. */
 	constructor(
 		/** The encoded data of this packet. */
@@ -54,6 +75,7 @@ export class EncodedPacket {
 		 */
 		public readonly sequenceNumber = -1,
 		byteLength?: number,
+		sideData?: EncodedPacketSideData,
 	) {
 		if (data === PLACEHOLDER_DATA && byteLength === undefined) {
 			throw new Error(
@@ -83,8 +105,25 @@ export class EncodedPacket {
 		if (!Number.isInteger(byteLength) || byteLength < 0) {
 			throw new TypeError('byteLength must be a non-negative integer.');
 		}
+		if (sideData !== undefined && (typeof sideData !== 'object' || !sideData)) {
+			throw new TypeError('sideData, when provided, must be an object.');
+		}
+		if (sideData?.alpha !== undefined && !(sideData.alpha instanceof Uint8Array)) {
+			throw new TypeError('sideData.alpha, when provided, must be a Uint8Array.');
+		}
+		if (
+			sideData?.alphaByteLength !== undefined
+			&& (!Number.isInteger(sideData.alphaByteLength) || sideData.alphaByteLength < 0)
+		) {
+			throw new TypeError('sideData.alphaByteLength, when provided, must be a non-negative integer.');
+		}
 
 		this.byteLength = byteLength;
+		this.sideData = sideData ?? {};
+
+		if (this.sideData.alpha && this.sideData.alphaByteLength === undefined) {
+			this.sideData.alphaByteLength = this.sideData.alpha.byteLength;
+		}
 	}
 
 	/** If this packet is a metadata-only packet. Metadata-only packets don't contain their packet data. */
@@ -102,7 +141,9 @@ export class EncodedPacket {
 		return Math.trunc(SECOND_TO_MICROSECOND_FACTOR * this.duration);
 	}
 
-	/** Converts this packet to an EncodedVideoChunk for use with the WebCodecs API. */
+	/** Converts this packet to an
+	 * [`EncodedVideoChunk`](https://developer.mozilla.org/en-US/docs/Web/API/EncodedVideoChunk) for use with the
+	 * WebCodecs API. */
 	toEncodedVideoChunk() {
 		if (this.isMetadataOnly) {
 			throw new TypeError('Metadata-only packets cannot be converted to a video chunk.');
@@ -119,7 +160,33 @@ export class EncodedPacket {
 		});
 	}
 
-	/** Converts this packet to an EncodedAudioChunk for use with the WebCodecs API. */
+	/**
+	 * Converts this packet to an
+	 * [`EncodedVideoChunk`](https://developer.mozilla.org/en-US/docs/Web/API/EncodedVideoChunk) for use with the
+	 * WebCodecs API, using the alpha side data instead of the color data. Throws if no alpha side data is defined.
+	 */
+	alphaToEncodedVideoChunk(type = this.type) {
+		if (!this.sideData.alpha) {
+			throw new TypeError('This packet does not contain alpha side data.');
+		}
+		if (this.isMetadataOnly) {
+			throw new TypeError('Metadata-only packets cannot be converted to a video chunk.');
+		}
+		if (typeof EncodedVideoChunk === 'undefined') {
+			throw new Error('Your browser does not support EncodedVideoChunk.');
+		}
+
+		return new EncodedVideoChunk({
+			data: this.sideData.alpha,
+			type,
+			timestamp: this.microsecondTimestamp,
+			duration: this.microsecondDuration,
+		});
+	}
+
+	/** Converts this packet to an
+	 * [`EncodedAudioChunk`](https://developer.mozilla.org/en-US/docs/Web/API/EncodedAudioChunk) for use with the
+	 * WebCodecs API. */
 	toEncodedAudioChunk() {
 		if (this.isMetadataOnly) {
 			throw new TypeError('Metadata-only packets cannot be converted to an audio chunk.');
@@ -137,10 +204,15 @@ export class EncodedPacket {
 	}
 
 	/**
-	 * Creates an EncodedPacket from an EncodedVideoChunk or EncodedAudioChunk. This method is useful for converting
-	 * chunks from the WebCodecs API to EncodedPackets.
+	 * Creates an {@link EncodedPacket} from an
+	 * [`EncodedVideoChunk`](https://developer.mozilla.org/en-US/docs/Web/API/EncodedVideoChunk) or
+	 * [`EncodedAudioChunk`](https://developer.mozilla.org/en-US/docs/Web/API/EncodedAudioChunk). This method is useful
+	 * for converting chunks from the WebCodecs API to `EncodedPacket` instances.
 	 */
-	static fromEncodedChunk(chunk: EncodedVideoChunk | EncodedAudioChunk): EncodedPacket {
+	static fromEncodedChunk(
+		chunk: EncodedVideoChunk | EncodedAudioChunk,
+		sideData?: EncodedPacketSideData,
+	): EncodedPacket {
 		if (!(chunk instanceof EncodedVideoChunk || chunk instanceof EncodedAudioChunk)) {
 			throw new TypeError('chunk must be an EncodedVideoChunk or EncodedAudioChunk.');
 		}
@@ -153,6 +225,9 @@ export class EncodedPacket {
 			chunk.type as PacketType,
 			chunk.timestamp / 1e6,
 			(chunk.duration ?? 0) / 1e6,
+			undefined,
+			undefined,
+			sideData,
 		);
 	}
 

src/reader.ts

@@ -149,16 +149,32 @@ export class FileSlice {
 	}
 }
 
+const checkIsInRange = (slice: FileSlice, bytesToRead: number) => {
+	if (slice.filePos < slice.start || slice.filePos + bytesToRead > slice.end) {
+		throw new RangeError(
+			`Tried reading [${slice.filePos}, ${slice.filePos + bytesToRead}), but slice is`
+			+ ` [${slice.start}, ${slice.end}).`,
+		);
+	}
+};
+
 export const readBytes = (slice: FileSlice, length: number) => {
+	checkIsInRange(slice, length);
+
 	const bytes = slice.bytes.subarray(slice.bufferPos, slice.bufferPos + length);
 	slice.bufferPos += length;
 
 	return bytes;
 };
 
-export const readU8 = (slice: FileSlice) => slice.view.getUint8(slice.bufferPos++);
+export const readU8 = (slice: FileSlice) => {
+	checkIsInRange(slice, 1);
+	return slice.view.getUint8(slice.bufferPos++);
+};
 
 export const readU16 = (slice: FileSlice, littleEndian: boolean) => {
+	checkIsInRange(slice, 2);
+
 	const value = slice.view.getUint16(slice.bufferPos, littleEndian);
 	slice.bufferPos += 2;
 
@@ -166,6 +182,8 @@ export const readU16 = (slice: FileSlice, littleEndian: boolean) => {
 };
 
 export const readU16Be = (slice: FileSlice) => {
+	checkIsInRange(slice, 2);
+
 	const value = slice.view.getUint16(slice.bufferPos, false);
 	slice.bufferPos += 2;
 
@@ -173,6 +191,8 @@ export const readU16Be = (slice: FileSlice) => {
 };
 
 export const readU24Be = (slice: FileSlice) => {
+	checkIsInRange(slice, 3);
+
 	const value = getUint24(slice.view, slice.bufferPos, false);
 	slice.bufferPos += 3;
 
@@ -180,6 +200,8 @@ export const readU24Be = (slice: FileSlice) => {
 };
 
 export const readI16Be = (slice: FileSlice) => {
+	checkIsInRange(slice, 2);
+
 	const value = slice.view.getInt16(slice.bufferPos, false);
 	slice.bufferPos += 2;
 
@@ -187,6 +209,8 @@ export const readI16Be = (slice: FileSlice) => {
 };
 
 export const readU32 = (slice: FileSlice, littleEndian: boolean) => {
+	checkIsInRange(slice, 4);
+
 	const value = slice.view.getUint32(slice.bufferPos, littleEndian);
 	slice.bufferPos += 4;
 
@@ -194,6 +218,8 @@ export const readU32 = (slice: FileSlice, littleEndian: boolean) => {
 };
 
 export const readU32Be = (slice: FileSlice) => {
+	checkIsInRange(slice, 4);
+
 	const value = slice.view.getUint32(slice.bufferPos, false);
 	slice.bufferPos += 4;
 
@@ -201,6 +227,8 @@ export const readU32Be = (slice: FileSlice) => {
 };
 
 export const readU32Le = (slice: FileSlice) => {
+	checkIsInRange(slice, 4);
+
 	const value = slice.view.getUint32(slice.bufferPos, true);
 	slice.bufferPos += 4;
 
@@ -208,6 +236,8 @@ export const readU32Le = (slice: FileSlice) => {
 };
 
 export const readI32Be = (slice: FileSlice) => {
+	checkIsInRange(slice, 4);
+
 	const value = slice.view.getInt32(slice.bufferPos, false);
 	slice.bufferPos += 4;
 
@@ -215,6 +245,8 @@ export const readI32Be = (slice: FileSlice) => {
 };
 
 export const readI32Le = (slice: FileSlice) => {
+	checkIsInRange(slice, 4);
+
 	const value = slice.view.getInt32(slice.bufferPos, true);
 	slice.bufferPos += 4;
 
@@ -255,6 +287,8 @@ export const readI64Le = (slice: FileSlice) => {
 };
 
 export const readF32Be = (slice: FileSlice) => {
+	checkIsInRange(slice, 4);
+
 	const value = slice.view.getFloat32(slice.bufferPos, false);
 	slice.bufferPos += 4;
 
@@ -262,6 +296,8 @@ export const readF32Be = (slice: FileSlice) => {
 };
 
 export const readF64Be = (slice: FileSlice) => {
+	checkIsInRange(slice, 8);
+
 	const value = slice.view.getFloat64(slice.bufferPos, false);
 	slice.bufferPos += 8;
 
@@ -269,9 +305,7 @@ export const readF64Be = (slice: FileSlice) => {
 };
 
 export const readAscii = (slice: FileSlice, length: number) => {
-	if (slice.bufferPos + length > slice.bytes.length) {
-		throw new RangeError('Reading past end of slice.');
-	}
+	checkIsInRange(slice, length);
 
 	let str = '';
 

src/sample.ts

@@ -16,8 +16,11 @@ import {
 	toUint8Array,
 	SetRequired,
 	isFirefox,
+	polyfillSymbolDispose,
 } from './misc';
 
+polyfillSymbolDispose();
+
 /**
  * Metadata used for VideoSample initialization.
  * @group Samples
@@ -49,7 +52,7 @@ export type VideoSampleInit = {
  * @group Samples
  * @public
  */
-export class VideoSample {
+export class VideoSample implements Disposable {
 	/** @internal */
 	_data!: VideoFrame | OffscreenCanvas | Uint8Array | null;
 	/** @internal */
@@ -96,6 +99,14 @@ export class VideoSample {
 		return Math.trunc(SECOND_TO_MICROSECOND_FACTOR * this.duration);
 	}
 
+	/**
+	 * Whether this sample uses a pixel format that can hold transparency data. Note that this doesn't necessarily mean
+	 * that the sample is transparent.
+	 */
+	get hasAlpha() {
+		return this.format && this.format.includes('A');
+	}
+
 	/**
 	 * Creates a new {@link VideoSample} from a
 	 * [`VideoFrame`](https://developer.mozilla.org/en-US/docs/Web/API/VideoFrame). This is essentially a near zero-cost
@@ -728,6 +739,11 @@ export class VideoSample {
 		// eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion
 		(this.duration as number) = newDuration;
 	}
+
+	/** Calls `.close()`. */
+	[Symbol.dispose]() {
+		this.close();
+	}
 }
 
 const isVideoFrame = (x: unknown): x is VideoFrame => {
@@ -832,7 +848,7 @@ export type AudioSampleCopyToOptions = {
  * @group Samples
  * @public
  */
-export class AudioSample {
+export class AudioSample implements Disposable {
 	/** @internal */
 	_data: AudioData | Uint8Array;
 	/** @internal */
@@ -1268,6 +1284,11 @@ export class AudioSample {
 		(this.timestamp as number) = newTimestamp;
 	}
 
+	/** Calls `.close()`. */
+	[Symbol.dispose]() {
+		this.close();
+	}
+
 	/** @internal */
 	static* _fromAudioBuffer(audioBuffer: AudioBuffer, timestamp: number) {
 		if (!(audioBuffer instanceof AudioBuffer)) {

src/source.ts

@@ -764,6 +764,8 @@ export class StreamSource extends Source {
 			if (data instanceof Promise) data = await data;
 
 			if (data instanceof Uint8Array) {
+				data = toUint8Array(data); // Normalize things like Node.js Buffer to Uint8Array
+
 				if (data.length !== originalTargetPos - worker.currentPos) {
 					// Yes, we're that strict
 					throw new Error(
@@ -798,8 +800,10 @@ export class StreamSource extends Source {
 						throw new TypeError('ReadableStream returned by options.read must yield Uint8Array chunks.');
 					}
 
-					this.onread?.(worker.currentPos, worker.currentPos + value.length);
-					this._orchestrator.supplyWorkerData(worker, value);
+					const data = toUint8Array(value); // Normalize things like Node.js Buffer to Uint8Array
+
+					this.onread?.(worker.currentPos, worker.currentPos + data.length);
+					this._orchestrator.supplyWorkerData(worker, data);
 				}
 			} else {
 				throw new TypeError('options.read must return or resolve to a Uint8Array or a ReadableStream.');

src/tags.ts

@@ -12,7 +12,7 @@
  * directly read or write the underlying metadata tags (which differ by format).
  *
  * - For MP4/QuickTime files, the metadata refers to the data in `'moov'`-level `'udta'` and `'meta'` atoms.
- * - For Matroska files, the metadata refers to the Tags and Attachments elements whose target is 50 (MOVIE).
+ * - For WebM/Matroska files, the metadata refers to the Tags and Attachments elements whose target is 50 (MOVIE).
  * - For MP3 files, the metadata refers to the ID3v2 or ID3v1 tags.
  * - For Ogg files, there is no global metadata so instead, the metadata refers to the combined metadata of all tracks,
  * in Vorbis-style comment headers.
@@ -65,9 +65,9 @@ export type MetadataTags = {
 	 * is also used, then the keys reflect the keys specified there (such as `'com.apple.quicktime.version'`).
 	 * Additionally, any atoms within the `'udta'` atom are dumped into here, however with unknown internal format
 	 * (`Uint8Array`).
-	 * - Matroska: `SimpleTag` elements whose target is 50 (MOVIE), either containing string or `Uint8Array` values.
-	 * Additionally, all attached files (such as font files) are included here, where the key corresponds to the FileUID
-	 * and the value is an {@link AttachedFile}.
+	 * - WebM/Matroska: `SimpleTag` elements whose target is 50 (MOVIE), either containing string or `Uint8Array`
+	 * values. Additionally, all attached files (such as font files) are included here, where the key corresponds to
+	 * the FileUID and the value is an {@link AttachedFile}.
 	 * - MP3: The ID3v2 tags, or a single `'TAG'` key with the contents of the ID3v1 tag.
 	 * - Ogg: The key-value string pairs from the Vorbis-style comment header (see RFC 7845, Section 5.2).
 	 * Additionally, the `'vendor'` key refers to the vendor string within this header.

src/wave/wave-demuxer.ts

@@ -102,8 +102,12 @@ export class WaveDemuxer extends Demuxer {
 				} else if (chunkId === 'ds64') {
 					// File and data chunk sizes are defined in here instead
 
-					const riffChunkSize = readU64(slice, littleEndian);
-					dataChunkSize = readU64(slice, littleEndian);
+					let ds64Slice = this.reader.requestSlice(startPos, chunkSize);
+					if (ds64Slice instanceof Promise) ds64Slice = await ds64Slice;
+					if (!ds64Slice) break;
+
+					const riffChunkSize = readU64(ds64Slice, littleEndian);
+					dataChunkSize = readU64(ds64Slice, littleEndian);
 
 					totalFileSize = Math.min(riffChunkSize + 8, this.reader.fileSize ?? Infinity);
 				} else if (chunkId === 'LIST') {

test/browser/transparency.test.ts

@@ -0,0 +1,271 @@
+import { expect, test } from 'vitest';
+import { Input } from '../../src/input.js';
+import { BufferSource, UrlSource } from '../../src/source.js';
+import { ALL_FORMATS } from '../../src/input-format.js';
+import { CanvasSink, EncodedPacketSink, VideoSampleSink } from '../../src/media-sink.js';
+import { Output } from '../../src/output.js';
+import { WebMOutputFormat } from '../../src/output-format.js';
+import { BufferTarget } from '../../src/target.js';
+import { CanvasSource, VideoSampleSource } from '../../src/media-source.js';
+import { canEncodeVideo, QUALITY_HIGH } from '../../src/encode.js';
+import { VideoSample } from '../../src/sample.js';
+
+test('Can decode transparent video', async () => {
+	using input = new Input({
+		source: new UrlSource('/transparency.webm'),
+		formats: ALL_FORMATS,
+	});
+
+	const videoTrack = (await input.getPrimaryVideoTrack())!;
+	expect(await videoTrack.canBeTransparent()).toBe(true);
+
+	const sink = new VideoSampleSink(videoTrack);
+	const sample = (await sink.getSample(0.5))!;
+
+	expect(sample.format).toContain('A'); // Probably RGBA
+	expect(sample.hasAlpha).toBe(true);
+
+	const canvas = new OffscreenCanvas(sample.displayWidth, sample.displayHeight);
+	const context = canvas.getContext('2d')!;
+
+	sample.draw(context, 0, 0);
+
+	const imageData = context.getImageData(0, 0, canvas.width, canvas.height);
+	expect(imageData.data[3]).toBeLessThan(255); // Check that there's actually transparent pixels
+});
+
+test('Can decode faulty transparent video and behaves gracefully', async () => {
+	using input = new Input({
+		source: new UrlSource('/transparency-faulty.webm'),
+		formats: ALL_FORMATS,
+	});
+
+	const videoTrack = (await input.getPrimaryVideoTrack())!;
+	const packetSink = new EncodedPacketSink(videoTrack);
+	const secondKeyPacket = (await packetSink.getNextKeyPacket((await packetSink.getFirstPacket())!))!;
+
+	const sink = new VideoSampleSink(videoTrack);
+
+	const startSample = (await sink.getSample(await videoTrack.getFirstTimestamp()))!;
+	expect(startSample.format).toContain('A');
+
+	const secondSample = (await sink.getSample(secondKeyPacket.timestamp))!;
+	expect(secondSample.format).not.toContain('A'); // There was no alpha key frame for this one
+	expect(secondSample.hasAlpha).toBe(false);
+});
+
+test('Can extract transparent frames via CanvasSink', async () => {
+	using input = new Input({
+		source: new UrlSource('/transparency.webm'),
+		formats: ALL_FORMATS,
+	});
+
+	const videoTrack = (await input.getPrimaryVideoTrack())!;
+	const sink = new CanvasSink(videoTrack, { alpha: true });
+	const wrappedCanvas = (await sink.getCanvas(await videoTrack.getFirstTimestamp()))!;
+
+	const canvas = new OffscreenCanvas(wrappedCanvas.canvas.width, wrappedCanvas.canvas.height);
+	const context = canvas.getContext('2d')!;
+	context.drawImage(wrappedCanvas.canvas, 0, 0);
+
+	let imageData = context.getImageData(0, 0, canvas.width, canvas.height);
+	expect(imageData.data[3]).toBeLessThan(255); // Check that there's actually transparent pixels
+
+	const opaqueSink = new CanvasSink(videoTrack); // Default is alpha: false
+	const opaqueWrappedCanvas = (await opaqueSink.getCanvas(await videoTrack.getFirstTimestamp()))!;
+
+	context.drawImage(opaqueWrappedCanvas.canvas, 0, 0);
+
+	imageData = context.getImageData(0, 0, canvas.width, canvas.height);
+	expect(imageData.data[3]).toBe(255);
+});
+
+test('Can encode transparent video', async () => {
+	const output = new Output({
+		format: new WebMOutputFormat(),
+		target: new BufferTarget(),
+	});
+
+	const canvas = new OffscreenCanvas(1280, 720);
+	const context = canvas.getContext('2d')!;
+
+	const source = new CanvasSource(canvas, {
+		codec: 'vp9',
+		bitrate: QUALITY_HIGH,
+		alpha: 'keep',
+	});
+	output.addVideoTrack(source);
+
+	await output.start();
+
+	context.fillStyle = '#ff0000';
+	context.fillRect(200, 200, 200, 200);
+	await source.add(0, 1);
+
+	context.fillStyle = '#00ff00';
+	context.fillRect(300, 300, 200, 200);
+	await source.add(1, 1);
+
+	context.fillStyle = '#0000ff';
+	context.fillRect(400, 400, 200, 200);
+	await source.add(2, 1);
+
+	await output.finalize();
+
+	const blob = new Blob([output.target.buffer!], {
+		type: output.format.mimeType,
+	});
+	const url = URL.createObjectURL(blob);
+
+	const video = document.createElement('video');
+	video.src = url;
+	video.muted = true;
+	void video.play();
+
+	await new Promise(resolve => video.addEventListener('loadeddata', resolve));
+
+	// Let the video play for a little bit to prevent flake
+	while (video.currentTime < 0.1) {
+		await new Promise(resolve => setTimeout(resolve, 0));
+	}
+
+	expect(video.videoWidth).toBe(1280);
+	expect(video.videoHeight).toBe(720);
+
+	const probeCanvas = new OffscreenCanvas(1280, 720);
+	const probeContext = probeCanvas.getContext('2d')!;
+
+	probeContext.drawImage(video, 0, 0);
+
+	let imageData = probeContext.getImageData(0, 0, probeCanvas.width, probeCanvas.height);
+	expect(imageData.data[3]).lessThanOrEqual(2); // Transparent (within error)
+
+	const pos = { x: 300, y: 300 }; // Dead center in the red square
+	const index = (pos.x + pos.y * probeCanvas.width) * 4;
+
+	// Red (within error)
+	expect(imageData.data[index + 0]).greaterThanOrEqual(253);
+	expect(imageData.data[index + 1]).lessThanOrEqual(2);
+	expect(imageData.data[index + 2]).lessThanOrEqual(2);
+
+	expect(imageData.data[index + 3]).greaterThanOrEqual(253); // Opaque (within error)
+
+	// Let's also check it's read correctly by Mediabunny
+	using input = new Input({
+		source: new BufferSource(output.target.buffer!),
+		formats: ALL_FORMATS,
+	});
+
+	const videoTrack = (await input.getPrimaryVideoTrack())!;
+	expect(await videoTrack.canBeTransparent()).toBe(true);
+
+	const sink = new VideoSampleSink(videoTrack);
+
+	const firstSample = (await sink.getSample(0))!;
+	expect(firstSample.format).toContain('A');
+
+	probeContext.clearRect(0, 0, probeCanvas.width, probeCanvas.height);
+	firstSample.draw(probeContext, 0, 0);
+
+	imageData = probeContext.getImageData(0, 0, probeCanvas.width, probeCanvas.height);
+	expect(imageData.data[3]).lessThanOrEqual(2); // Transparent (within error)
+});
+
+test('Can encode video with alternating transparency', async () => {
+	const output = new Output({
+		format: new WebMOutputFormat(),
+		target: new BufferTarget(),
+	});
+
+	const canvas1 = new OffscreenCanvas(640, 480);
+	const context1 = canvas1.getContext('2d', { alpha: true })!;
+	context1.fillStyle = '#ff000080';
+	context1.fillRect(0, 0, canvas1.width, canvas1.height);
+
+	const canvas2 = new OffscreenCanvas(640, 480);
+	const context2 = canvas2.getContext('2d', { alpha: false })!;
+	context2.fillStyle = '#0000ff';
+	context2.fillRect(0, 0, canvas2.width, canvas2.height);
+
+	const source = new VideoSampleSource({
+		codec: 'vp9',
+		bitrate: QUALITY_HIGH,
+		alpha: 'keep',
+	});
+	output.addVideoTrack(source);
+
+	await output.start();
+
+	for (let i = 0; i < 64; i++) {
+		const sample = new VideoSample(new Uint8Array(640 * 480 * 4), {
+			format: i % 2 ? 'RGBX' : 'RGBA',
+			codedWidth: 640,
+			codedHeight: 480,
+			timestamp: i,
+			duration: 1,
+		});
+		await source.add(sample);
+	}
+
+	await output.finalize();
+
+	using input = new Input({
+		source: new BufferSource(output.target.buffer!),
+		formats: ALL_FORMATS,
+	});
+
+	const videoTrack = (await input.getPrimaryVideoTrack())!;
+	const packetSink = new EncodedPacketSink(videoTrack);
+
+	let i = 0;
+	for await (const packet of packetSink.packets()) {
+		if (i % 2) {
+			expect(packet.sideData.alpha).toBeUndefined();
+		} else {
+			expect(packet.sideData.alpha).toBeDefined();
+		}
+
+		i++;
+	}
+
+	const sampleSink = new VideoSampleSink(videoTrack);
+
+	i = 0;
+	for await (const sample of sampleSink.samples()) {
+		if (i % 2) {
+			expect(sample.format).not.toContain('A');
+		} else {
+			expect(sample.format).toContain('A');
+		}
+
+		i++;
+	}
+});
+
+test('Can encode transparent video with odd dimensions', async () => {
+	const output = new Output({
+		format: new WebMOutputFormat(),
+		target: new BufferTarget(),
+	});
+
+	const canvas = new OffscreenCanvas(641, 479);
+	const context = canvas.getContext('2d', { alpha: true })!;
+	context.fillStyle = '#ff000080';
+	context.fillRect(0, 0, canvas.width, canvas.height);
+
+	const source = new CanvasSource(canvas, {
+		codec: 'vp9',
+		bitrate: QUALITY_HIGH,
+		alpha: 'keep',
+	});
+	output.addVideoTrack(source);
+
+	await output.start();
+	await source.add(0, 1);
+	await output.finalize();
+});
+
+test('Positive encodability check with alpha', async () => {
+	const result = await canEncodeVideo('vp9', { alpha: 'keep' });
+	expect(result).toBe(true);
+});

test/tsconfig.json

@@ -1,13 +0,0 @@
-{
-	"extends": "../tsconfig.json",
-	"compilerOptions": {
-		"noEmit": true,
-		"moduleResolution": "nodenext",
-		"module": "NodeNext",
-		"declaration": true,
-		"declarationMap": true,
-		"stripInternal": true
-	},
-	"include": ["**/*"],
-	"references": [{ "path": "../src" }]
-}

test/vitest.config.ts

@@ -1,11 +0,0 @@
-import { defineConfig } from 'vitest/config';
-
-export default defineConfig({
-	test: {
-		browser: {
-			provider: 'webdriverio',
-			instances: [{ browser: 'chrome' }],
-			headless: true,
-		},
-	},
-});

tsconfig.json

@@ -11,5 +11,8 @@
 		"allowJs": true,
 		"noEmit": true,
 	},
-	"references": [{ "path": "./tsconfig.vite.json" }]
+	"references": [
+		{ "path": "./tsconfig.vite.json" },
+		{ "path": "./tsconfig.vitest.json" }
+	]
 }

tsconfig.vitest.json

@@ -0,0 +1,11 @@
+{
+	"extends": "./tsconfig.json",
+	"compilerOptions": {
+		"moduleResolution": "nodenext",
+		"module": "NodeNext",
+		"composite": true,
+		"noEmit": false
+	},
+	"include": ["vitest.config.ts", "./test/**/*"],
+	"references": [{ "path": "./src" }]
+}

vitest.config.ts

@@ -0,0 +1,34 @@
+/// <reference types="@vitest/browser/providers/webdriverio" />
+
+import { defineConfig } from 'vitest/config';
+
+export default defineConfig({
+	test: {
+		projects: [
+			{
+				test: {
+					name: 'node',
+					root: 'test',
+					include: ['node/**/*.test.ts'],
+					environment: 'node',
+				},
+			},
+			{
+				test: {
+					name: 'browser',
+					root: 'test',
+					include: ['browser/**/*.test.ts'],
+					browser: {
+						enabled: true,
+						provider: 'webdriverio',
+						instances: [{
+							browser: 'chrome',
+						}],
+						headless: false, // A bunch of features need the head
+						screenshotFailures: false,
+					},
+				},
+			},
+		],
+	},
+});