Transparent video read/write support (#145)

* Add support for reading transparent Matroska and implement alpha side data & decode * Few fixes * Implement alpha encoding * Gracefully handle inability to acquire WebGL context, properly clean up WebGL contexts * Improve touch device detection * Test test * Test test #2 * Test test 3 * Test test 4 * Test test 5 * Test test 6 * Test test 7 * Test test 8 * Test test 9 * Test test 10 * Test test 11 * Test test 12 * Test test 13 * Test test 14 * Test test 15 * Test test 16 * Test test 17 * Test test 18 * Test test 19 * Test test 20 * Fix type errors, improve MetadataTags docs * Do a bunch of docs work * Test test? * "Unexpected only modifier 🤓" * Add InputVideoTrack.canBeTransparent() * Some clean-up * Adjust CI to be less spammy in PRs * Fix broken license headers
2026-10-02 13:23:51 +02:00 · 2025-09-24 21:20:14 +02:00
parent 0878dd2ca9
commit fa4e064345
34 changed files with 1537 additions and 181 deletions
@@ -100,6 +100,7 @@ export enum EBMLId {
 	Video = 0xe0,
 	PixelWidth = 0xb0,
 	PixelHeight = 0xba,
+	AlphaMode = 0x53c0,
 	Audio = 0xe1,
 	SamplingFrequency = 0xb5,
 	Channels = 0x9f,
@@ -50,7 +50,7 @@ import {
 	TRANSFER_CHARACTERISTICS_MAP_INVERSE,
 	UNDETERMINED_LANGUAGE,
 } from '../misc';
-import { EncodedPacket, PLACEHOLDER_DATA } from '../packet';
+import { EncodedPacket, EncodedPacketSideData, PLACEHOLDER_DATA } from '../packet';
 import {
 	assertDefinedSize,
 	CODEC_STRING_MAP,
@@ -143,6 +143,7 @@ type ClusterBlock = {
 	data: Uint8Array;
 	lacing: BlockLacing;
 	decoded: boolean;
+	mainAdditional: Uint8Array | null;
 };

 type CuePoint = {
@@ -204,6 +205,7 @@ type InternalTrack = {
 			codec: VideoCodec | null;
 			codecDescription: Uint8Array | null;
 			colorSpace: VideoColorSpaceInit | null;
+			alphaMode: boolean;
 		}
 		| {
 			type: 'audio';
@@ -236,6 +238,11 @@ export class MatroskaDemuxer extends Demuxer {
 	currentTrack: InternalTrack | null = null;
 	currentCluster: Cluster | null = null;
 	currentBlock: ClusterBlock | null = null;
+	currentBlockAdditional: {
+		addId: number;
+		data: Uint8Array | null;
+	} | null = null;
+
 	currentCueTime: number | null = null;
 	currentDecodingInstruction: DecodingInstruction | null = null;
 	currentTagTargetIsMovie: boolean = true;
@@ -845,6 +852,7 @@ export class MatroskaDemuxer extends Demuxer {
 					data: frameData,
 					lacing: BlockLacing.None,
 					decoded: true,
+					mainAdditional: originalBlock.mainAdditional,
 				});
 			}

@@ -1110,6 +1118,7 @@ export class MatroskaDemuxer extends Demuxer {
 						codec: null,
 						codecDescription: null,
 						colorSpace: null,
+						alphaMode: false,
 					};
 				} else if (type === 2) {
 					this.currentTrack.info = {
@@ -1214,6 +1223,12 @@ export class MatroskaDemuxer extends Demuxer {
 				this.currentTrack.info.height = readUnsignedInt(slice, size);
 			}; break;

+			case EBMLId.AlphaMode: {
+				if (this.currentTrack?.info?.type !== 'video') break;
+
+				this.currentTrack.info.alphaMode = readUnsignedInt(slice, size) === 1;
+			}; break;
+
 			case EBMLId.Colour: {
 				if (this.currentTrack?.info?.type !== 'video') break;

@@ -1365,6 +1380,7 @@ export class MatroskaDemuxer extends Demuxer {
 					data: blockData,
 					lacing,
 					decoded: !hasDecodingInstructions,
+					mainAdditional: null,
 				});
 			}; break;

@@ -1407,10 +1423,43 @@ export class MatroskaDemuxer extends Demuxer {
 					data: blockData,
 					lacing,
 					decoded: !hasDecodingInstructions,
+					mainAdditional: null,
 				};
 				trackData.blocks.push(this.currentBlock);
 			}; break;

+			case EBMLId.BlockAdditions: {
+				this.readContiguousElements(slice.slice(dataStartPos, size));
+			}; break;
+
+			case EBMLId.BlockMore: {
+				if (!this.currentBlock) break;
+
+				this.currentBlockAdditional = {
+					addId: 1,
+					data: null,
+				};
+
+				this.readContiguousElements(slice.slice(dataStartPos, size));
+
+				if (this.currentBlockAdditional.data && this.currentBlockAdditional.addId === 1) {
+					this.currentBlock.mainAdditional = this.currentBlockAdditional.data;
+				}
+				this.currentBlockAdditional = null;
+			}; break;
+
+			case EBMLId.BlockAdditional: {
+				if (!this.currentBlockAdditional) break;
+
+				this.currentBlockAdditional.data = readBytes(slice, size);
+			}; break;
+
+			case EBMLId.BlockAddID: {
+				if (!this.currentBlockAdditional) break;
+
+				this.currentBlockAdditional.addId = readUnsignedInt(slice, size);
+			}; break;
+
 			case EBMLId.BlockDuration: {
 				if (!this.currentBlock) break;

@@ -2004,6 +2053,13 @@ abstract class MatroskaTrackBacking implements InputTrackBacking {
 		const data = options.metadataOnly ? PLACEHOLDER_DATA : block.data;
 		const timestamp = block.timestamp / this.internalTrack.segment.timestampFactor;
 		const duration = block.duration / this.internalTrack.segment.timestampFactor;
+
+		const sideData: EncodedPacketSideData = {};
+		if (block.mainAdditional && this.internalTrack.info?.type === 'video' && this.internalTrack.info.alphaMode) {
+			sideData.alpha = options.metadataOnly ? PLACEHOLDER_DATA : block.mainAdditional;
+			sideData.alphaByteLength = block.mainAdditional.byteLength;
+		}
+
 		const packet = new EncodedPacket(
 			data,
 			block.isKeyFrame ? 'key' : 'delta',
@@ -2011,6 +2067,7 @@ abstract class MatroskaTrackBacking implements InputTrackBacking {
 			duration,
 			cluster.dataStartPos + blockIndex,
 			block.data.byteLength,
+			sideData,
 		);

 		this.packetToClusterLocation.set(packet, { cluster, blockIndex });
@@ -2319,6 +2376,10 @@ class MatroskaVideoTrackBacking extends MatroskaTrackBacking implements InputVid
 		};
 	}

+	async canBeTransparent() {
+		return this.internalTrack.info.alphaMode;
+	}
+
 	async getDecoderConfig(): Promise<VideoDecoderConfig | null> {
 		if (!this.internalTrack.info.codec) {
 			return null;
@@ -89,6 +89,7 @@ type MatroskaTrackData = {
 		width: number;
 		height: number;
 		decoderConfig: VideoDecoderConfig;
+		alphaMode: boolean;
 	};
 } | {
 	track: OutputAudioTrack;
@@ -343,6 +344,7 @@ export class MatroskaMuxer extends Muxer {
 		const videoElement: EBMLElement = { id: EBMLId.Video, data: [
 			{ id: EBMLId.PixelWidth, data: trackData.info.width },
 			{ id: EBMLId.PixelHeight, data: trackData.info.height },
+			trackData.info.alphaMode ? { id: EBMLId.AlphaMode, data: 1 } : null,
 			(colorSpaceIsComplete(colorSpace)
 				? {
 						id: EBMLId.Colour,
@@ -695,7 +697,7 @@ export class MatroskaMuxer extends Muxer {
 		});
 	}

-	private getVideoTrackData(track: OutputVideoTrack, meta?: EncodedVideoChunkMetadata) {
+	private getVideoTrackData(track: OutputVideoTrack, packet: EncodedPacket, meta?: EncodedVideoChunkMetadata) {
 		const existingTrackData = this.trackDatas.find(x => x.track === track);
 		if (existingTrackData) {
 			return existingTrackData as MatroskaVideoTrackData;
@@ -715,6 +717,7 @@ export class MatroskaMuxer extends Muxer {
 				width: meta.decoderConfig.codedWidth,
 				height: meta.decoderConfig.codedHeight,
 				decoderConfig: meta.decoderConfig,
+				alphaMode: !!packet.sideData.alpha, // The first packet determines if this track has alpha or not
 			},
 			chunkQueue: [],
 			lastWrittenMsTimestamp: null,
@@ -819,7 +822,7 @@ export class MatroskaMuxer extends Muxer {
 		const release = await this.mutex.acquire();

 		try {
-			const trackData = this.getVideoTrackData(track, meta);
+			const trackData = this.getVideoTrackData(track, packet, meta);

 			const isKeyFrame = packet.type === 'key';
 			let timestamp = this.validateAndNormalizeTimestamp(trackData.track, packet.timestamp, isKeyFrame);
@@ -831,7 +834,11 @@ export class MatroskaMuxer extends Muxer {
 				duration = roundToMultiple(duration, 1 / track.metadata.frameRate);
 			}

-			const videoChunk = this.createInternalChunk(packet.data, timestamp, duration, packet.type);
+			const additions = trackData.info.alphaMode
+				? packet.sideData.alpha ?? null
+				: null;
+
+			const videoChunk = this.createInternalChunk(packet.data, timestamp, duration, packet.type, additions);
 			if (track.source._codec === 'vp9') this.fixVP9ColorSpace(trackData, videoChunk);

 			trackData.chunkQueue.push(videoChunk);
@@ -1086,8 +1093,8 @@ export class MatroskaMuxer extends Muxer {
 				chunk.additions
 					? { id: EBMLId.BlockAdditions, data: [
 							{ id: EBMLId.BlockMore, data: [
+								{ id: EBMLId.BlockAddID, data: 1 }, // Some players expect BlockAddID to come first
 								{ id: EBMLId.BlockAdditional, data: chunk.additions },
-								{ id: EBMLId.BlockAddID, data: 1 },
 							] },
 						] }
 					: null,