Fall back to color space information from bitstream, fill missing color space fields when passing it to Chromium's VideoDecoder, fix AV1 Sequence Header OBU parsing, fix vpcC box generation, still mux partial color space info into container (fixes #474, closes #475)

dev/demux.html

@@ -17,14 +17,11 @@
 			source: new Mediabunny.BlobSource(file),
 		});
 
-		const track = await input.getPrimaryAudioTrack();
-		const sink = new Mediabunny.AudioSampleSink(track);
-
-		for await (const sample of sink.samples()) {
-			console.log(sample);
-			sample.close()
-		}
+		const track = await input.getPrimaryVideoTrack();
+		const sink = new Mediabunny.EncodedPacketSink(track);
 
+		const packet = await sink.getFirstPacket();
+		console.log(packet);
 		/*
 
 		for await (const packet of packetSink.packets()) {

src/codec-data.ts

@@ -1868,6 +1868,12 @@ export const extractVp9CodecInfoFromPacket = (
 	};
 };
 
+export const vp9CodecInfoHasColorInfo = (info: Vp9CodecInfo) => {
+	return info.colourPrimaries !== 2
+		|| info.transferCharacteristics !== 2
+		|| info.matrixCoefficients !== 2;
+};
+
 export type Av1CodecInfo = {
 	profile: number;
 	level: number;
@@ -1877,6 +1883,10 @@ export type Av1CodecInfo = {
 	chromaSubsamplingX: number;
 	chromaSubsamplingY: number;
 	chromaSamplePosition: number;
+	videoFullRangeFlag: number;
+	colourPrimaries: number;
+	transferCharacteristics: number;
+	matrixCoefficients: number;
 };
 
 /** Iterates over all OBUs in an AV1 packet bitstream. */
@@ -1891,7 +1901,8 @@ export const iterateAv1PacketObus = function* (packet: Uint8Array) {
 		for (let i = 0; i < 8; i++) {
 			const byte = bitstream.readAlignedByte();
 
-			value |= ((byte & 0x7f) << (i * 7));
+			// No bit shift since the value can get big
+			value += (byte & 0x7f) * 2 ** (i * 7);
 
 			if (!(byte & 0x80)) {
 				break;
@@ -1904,7 +1915,7 @@ export const iterateAv1PacketObus = function* (packet: Uint8Array) {
 		}
 
 		// Spec requirement
-		if (value >= 2 ** 32 - 1) {
+		if (value > 2 ** 32 - 1) {
 			return null;
 		}
 
@@ -1978,43 +1989,43 @@ export const extractAv1CodecInfoFromPacket = (
 		if (reducedStillPictureHeader) {
 			seqLevel = bitstream.readBits(5);
 		} else {
-			// Parse timing_info_present_flag
 			const timingInfoPresentFlag = bitstream.readBits(1);
 
+			let decoderModelInfoPresentFlag = 0;
+
 			if (timingInfoPresentFlag) {
-				// Skip timing info (num_units_in_display_tick, time_scale, equal_picture_interval)
 				bitstream.skipBits(32); // num_units_in_display_tick
 				bitstream.skipBits(32); // time_scale
 				const equalPictureInterval = bitstream.readBits(1);
 
 				if (equalPictureInterval) {
-					// Skip num_ticks_per_picture_minus_1 (uvlc)
-					// Since this is variable length, we'd need to implement uvlc reading
-					// For now, we'll return null as this is rare
-					return null;
+					// num_ticks_per_picture_minus_1 is uvlc-coded, so we need to seek past it manually
+					let leadingZeros = 0;
+					while (leadingZeros < 32 && !bitstream.readBits(1)) {
+						leadingZeros++;
+					}
+
+					if (leadingZeros < 32) {
+						bitstream.skipBits(leadingZeros);
+					}
+				}
+
+				decoderModelInfoPresentFlag = bitstream.readBits(1);
+
+				if (decoderModelInfoPresentFlag) {
+					bufferDelayLengthMinus1 = bitstream.readBits(5);
+					bitstream.skipBits(32); // num_units_in_decoding_tick
+					bitstream.skipBits(5); // buffer_removal_time_length_minus_1
+					bitstream.skipBits(5); // frame_presentation_time_length_minus_1
 				}
 			}
 
-			// Parse decoder_model_info_present_flag
-			const decoderModelInfoPresentFlag = bitstream.readBits(1);
-
-			if (decoderModelInfoPresentFlag) {
-				// Store buffer_delay_length_minus_1 instead of just skipping
-				bufferDelayLengthMinus1 = bitstream.readBits(5);
-				bitstream.skipBits(32); // num_units_in_decoding_tick
-				bitstream.skipBits(5); // buffer_removal_time_length_minus_1
-				bitstream.skipBits(5); // frame_presentation_time_length_minus_1
-			}
-
-			// Parse operating_points_cnt_minus_1
+			const initialDisplayDelayPresentFlag = bitstream.readBits(1);
 			const operatingPointsCntMinus1 = bitstream.readBits(5);
 
-			// For each operating point
 			for (let i = 0; i <= operatingPointsCntMinus1; i++) {
-				// operating_point_idc[i]
-				bitstream.skipBits(12);
+				bitstream.skipBits(12); // operating_point_idc[i]
 
-				// seq_level_idx[i]
 				const seqLevelIdx = bitstream.readBits(5);
 
 				if (i === 0) {
@@ -2022,7 +2033,6 @@ export const extractAv1CodecInfoFromPacket = (
 				}
 
 				if (seqLevelIdx > 7) {
-					// seq_tier[i]
 					const seqTierTemp = bitstream.readBits(1);
 					if (i === 0) {
 						seqTier = seqTierTemp;
@@ -2030,7 +2040,6 @@ export const extractAv1CodecInfoFromPacket = (
 				}
 
 				if (decoderModelInfoPresentFlag) {
-					// decoder_model_present_for_this_op[i]
 					const decoderModelPresentForThisOp = bitstream.readBits(1);
 
 					if (decoderModelPresentForThisOp) {
@@ -2041,12 +2050,12 @@ export const extractAv1CodecInfoFromPacket = (
 					}
 				}
 
-				// initial_display_delay_present_flag
-				const initialDisplayDelayPresentFlag = bitstream.readBits(1);
-
 				if (initialDisplayDelayPresentFlag) {
-					// initial_display_delay_minus_1[i]
-					bitstream.skipBits(4);
+					const initialDisplayDelayPresentForThisOp = bitstream.readBits(1);
+
+					if (initialDisplayDelayPresentForThisOp) {
+						bitstream.skipBits(4); // initial_display_delay_minus_1[i]
+					}
 				}
 			}
 		}
@@ -2129,11 +2138,32 @@ export const extractAv1CodecInfoFromPacket = (
 			monochrome = bitstream.readBits(1);
 		}
 
+		let colourPrimaries = 2; // CP_UNSPECIFIED
+		let transferCharacteristics = 2; // TC_UNSPECIFIED
+		let matrixCoefficients = 2; // MC_UNSPECIFIED
+
+		const colorDescriptionPresentFlag = bitstream.readBits(1);
+		if (colorDescriptionPresentFlag) {
+			colourPrimaries = bitstream.readBits(8);
+			transferCharacteristics = bitstream.readBits(8);
+			matrixCoefficients = bitstream.readBits(8);
+		}
+
+		let videoFullRangeFlag = 0;
 		let chromaSubsamplingX = 1;
 		let chromaSubsamplingY = 1;
-		let chromaSamplePosition = 0;
+		let chromaSamplePosition = 0; // CSP_UNKNOWN
+
+		if (monochrome) {
+			videoFullRangeFlag = bitstream.readBits(1);
+		} else if (colourPrimaries === 1 && transferCharacteristics === 13 && matrixCoefficients === 0) {
+			// sRGB with an identity matrix, which is always full range 4:4:4
+			videoFullRangeFlag = 1;
+			chromaSubsamplingX = 0;
+			chromaSubsamplingY = 0;
+		} else {
+			videoFullRangeFlag = bitstream.readBits(1);
 
-		if (!monochrome) {
 			if (seqProfile === 0) {
 				chromaSubsamplingX = 1;
 				chromaSubsamplingY = 1;
@@ -2143,9 +2173,10 @@ export const extractAv1CodecInfoFromPacket = (
 			} else {
 				if (bitDepth === 12) {
 					chromaSubsamplingX = bitstream.readBits(1);
-					if (chromaSubsamplingX) {
-						chromaSubsamplingY = bitstream.readBits(1);
-					}
+					chromaSubsamplingY = chromaSubsamplingX ? bitstream.readBits(1) : 0;
+				} else {
+					chromaSubsamplingX = 1;
+					chromaSubsamplingY = 0;
 				}
 			}
 
@@ -2163,12 +2194,56 @@ export const extractAv1CodecInfoFromPacket = (
 			chromaSubsamplingX,
 			chromaSubsamplingY,
 			chromaSamplePosition,
+			videoFullRangeFlag,
+			colourPrimaries,
+			transferCharacteristics,
+			matrixCoefficients,
 		};
 	}
 
 	return null;
 };
 
+export const av1CodecInfoHasColorInfo = (info: Av1CodecInfo) => {
+	return info.colourPrimaries !== 2
+		|| info.transferCharacteristics !== 2
+		|| info.matrixCoefficients !== 2;
+};
+
+export type ProresCodecInfo = {
+	colourPrimaries: number;
+	transferCharacteristics: number;
+	matrixCoefficients: number;
+	fullRange: boolean;
+};
+
+export const extractProresCodecInfoFromPacket = (packet: Uint8Array): ProresCodecInfo | null => {
+	// https://wiki.multimedia.cx/index.php/Apple_ProRes
+
+	const frameHeaderStart = 8;
+	if (packet.length < frameHeaderStart + 28) {
+		return null;
+	}
+
+	const view = toDataView(packet);
+
+	if (view.getUint32(4) !== 0x69637066) { // 'icpf'
+		return null;
+	}
+
+	const headerSize = view.getUint16(frameHeaderStart);
+	if (headerSize < 28) {
+		return null;
+	}
+
+	return {
+		fullRange: false, // ProRes is always limited range
+		colourPrimaries: view.getUint8(frameHeaderStart + 14),
+		transferCharacteristics: view.getUint8(frameHeaderStart + 15),
+		matrixCoefficients: view.getUint8(frameHeaderStart + 16),
+	};
+};
+
 export const parseOpusIdentificationHeader = (bytes: Uint8Array) => {
 	const view = toDataView(bytes);
 

src/codec.ts

@@ -10,13 +10,22 @@ import { parseAacAudioSpecificConfig } from '../shared/aac-misc';
 import {
 	Av1CodecInfo,
 	AvcDecoderConfigurationRecord,
+	deserializeAvcDecoderConfigurationRecord,
+	deserializeHevcDecoderConfigurationRecord,
 	HevcDecoderConfigurationRecord,
+	HevcNalUnitType,
+	parseAvcSps,
+	parseHevcSps,
+	ProresCodecInfo,
 	Vp9CodecInfo,
 } from './codec-data';
 import {
 	COLOR_PRIMARIES_MAP,
+	COLOR_PRIMARIES_MAP_INVERSE,
 	MATRIX_COEFFICIENTS_MAP,
+	MATRIX_COEFFICIENTS_MAP_INVERSE,
 	TRANSFER_CHARACTERISTICS_MAP,
+	TRANSFER_CHARACTERISTICS_MAP_INVERSE,
 	assert,
 	assertNever,
 	base64ToBytes,
@@ -370,7 +379,7 @@ export const generateAv1CodecConfigurationFromCodecString = (codecString: string
 	const tier = levelAndTier.slice(-1) === 'H' ? 1 : 0;
 	const bitDepth = Number(parts[3]);
 	const highBitDepth = bitDepth === 8 ? 0 : 1;
-	const twelveBit = 0;
+	const twelveBit = bitDepth === 12 ? 1 : 0;
 	const monochrome = parts[4] ? Number(parts[4]) : 0;
 	const chromaSubsamplingX = parts[5] ? Number(parts[5][0]) : 1;
 	const chromaSubsamplingY = parts[5] ? Number(parts[5][1]) : 1;
@@ -578,6 +587,132 @@ export const extractVideoCodecString = (trackInfo: {
 	throw new TypeError(`Unhandled codec '${codec}'.`);
 };
 
+export const extractColorSpace = (info: {
+	codec: VideoCodec | null;
+	codecDescription: Uint8Array | null;
+	avcCodecInfo: AvcDecoderConfigurationRecord | null;
+	hevcCodecInfo: HevcDecoderConfigurationRecord | null;
+	vp9CodecInfo: Vp9CodecInfo | null;
+	av1CodecInfo: Av1CodecInfo | null;
+	proresCodecInfo: ProresCodecInfo | null;
+}): VideoColorSpaceInit => {
+	switch (info.codec) {
+		case 'avc': {
+			let spsData = info.avcCodecInfo?.sequenceParameterSets[0];
+			if (!spsData && info.codecDescription) {
+				spsData = deserializeAvcDecoderConfigurationRecord(info.codecDescription)?.sequenceParameterSets[0];
+			}
+
+			if (spsData) {
+				const spsInfo = parseAvcSps(spsData);
+				if (spsInfo) {
+					return {
+						primaries:
+							COLOR_PRIMARIES_MAP_INVERSE[spsInfo.colourPrimaries] as
+							VideoColorPrimaries | undefined,
+						transfer:
+							TRANSFER_CHARACTERISTICS_MAP_INVERSE[spsInfo.transferCharacteristics] as
+							VideoTransferCharacteristics | undefined,
+						matrix:
+							MATRIX_COEFFICIENTS_MAP_INVERSE[spsInfo.matrixCoefficients] as
+							VideoMatrixCoefficients | undefined,
+						fullRange: !!spsInfo.fullRangeFlag,
+					};
+				}
+			}
+		}; break;
+
+		case 'hevc': {
+			let spsData = info.hevcCodecInfo?.arrays
+				.find(x => x.nalUnitType === HevcNalUnitType.SPS_NUT)?.nalUnits[0];
+			if (!spsData && info.codecDescription) {
+				spsData = deserializeHevcDecoderConfigurationRecord(info.codecDescription)?.arrays
+					.find(x => x.nalUnitType === HevcNalUnitType.SPS_NUT)?.nalUnits[0];
+			}
+
+			if (spsData) {
+				const spsInfo = parseHevcSps(spsData);
+				if (spsInfo) {
+					return {
+						primaries:
+							COLOR_PRIMARIES_MAP_INVERSE[spsInfo.colourPrimaries] as
+							VideoColorPrimaries | undefined,
+						transfer:
+							TRANSFER_CHARACTERISTICS_MAP_INVERSE[spsInfo.transferCharacteristics] as
+							VideoTransferCharacteristics | undefined,
+						matrix:
+							MATRIX_COEFFICIENTS_MAP_INVERSE[spsInfo.matrixCoefficients] as
+							VideoMatrixCoefficients | undefined,
+						fullRange: !!spsInfo.fullRangeFlag,
+					};
+				}
+			}
+		}; break;
+
+		case 'vp8': {
+			// The situation is fucky; do nothing for now.
+		}; break;
+
+		case 'vp9': {
+			if (info.vp9CodecInfo) {
+				return {
+					primaries:
+						COLOR_PRIMARIES_MAP_INVERSE[info.vp9CodecInfo.colourPrimaries] as
+						VideoColorPrimaries | undefined,
+					transfer:
+						TRANSFER_CHARACTERISTICS_MAP_INVERSE[info.vp9CodecInfo.transferCharacteristics] as
+						VideoTransferCharacteristics | undefined,
+					matrix:
+						MATRIX_COEFFICIENTS_MAP_INVERSE[info.vp9CodecInfo.matrixCoefficients] as
+						VideoMatrixCoefficients | undefined,
+					fullRange: !!info.vp9CodecInfo.videoFullRangeFlag,
+				};
+			}
+		}; break;
+
+		case 'av1': {
+			if (info.av1CodecInfo) {
+				return {
+					primaries:
+						COLOR_PRIMARIES_MAP_INVERSE[info.av1CodecInfo.colourPrimaries] as
+						VideoColorPrimaries | undefined,
+					transfer:
+						TRANSFER_CHARACTERISTICS_MAP_INVERSE[info.av1CodecInfo.transferCharacteristics] as
+						VideoTransferCharacteristics | undefined,
+					matrix:
+						MATRIX_COEFFICIENTS_MAP_INVERSE[info.av1CodecInfo.matrixCoefficients] as
+						VideoMatrixCoefficients | undefined,
+					fullRange: !!info.av1CodecInfo.videoFullRangeFlag,
+				};
+			}
+		}; break;
+
+		case 'prores': {
+			if (info.proresCodecInfo) {
+				return {
+					primaries:
+						COLOR_PRIMARIES_MAP_INVERSE[info.proresCodecInfo.colourPrimaries] as
+						VideoColorPrimaries | undefined,
+					transfer:
+						TRANSFER_CHARACTERISTICS_MAP_INVERSE[info.proresCodecInfo.transferCharacteristics] as
+						VideoTransferCharacteristics | undefined,
+					matrix:
+						MATRIX_COEFFICIENTS_MAP_INVERSE[info.proresCodecInfo.matrixCoefficients] as
+						VideoMatrixCoefficients | undefined,
+					fullRange: info.proresCodecInfo.fullRange,
+				};
+			}
+		}; break;
+	}
+
+	return {
+		primaries: undefined,
+		transfer: undefined,
+		matrix: undefined,
+		fullRange: undefined,
+	};
+};
+
 export const buildAudioCodecString = (codec: AudioCodec, numberOfChannels: number, sampleRate: number) => {
 	if (codec === 'aac') {
 		// If stereo or higher channels and lower sample rate, likely using HE-AAC v2 with PS

src/isobmff/isobmff-boxes.ts

@@ -16,7 +16,7 @@ import {
 	COLOR_PRIMARIES_MAP,
 	TRANSFER_CHARACTERISTICS_MAP,
 	MATRIX_COEFFICIENTS_MAP,
-	colorSpaceIsComplete,
+	colorSpaceIsEmpty,
 	UNDETERMINED_LANGUAGE,
 	assertNever,
 	keyValueIterator,
@@ -750,7 +750,9 @@ export const videoSampleDescription = (
 ], [
 	VIDEO_CODEC_TO_CONFIGURATION_BOX[trackData.track.source._codec]?.(trackData) ?? null,
 	pasp(trackData),
-	colorSpaceIsComplete(trackData.info.decoderConfig.colorSpace) ? colr(trackData) : null,
+	colorSpaceIsEmpty(trackData.info.decoderConfig.colorSpace)
+		? null
+		: colr(trackData),
 ]);
 
 /** Pixel Aspect Ratio Box: Specifies pixel width:height spacing for non-square pixels. */
@@ -766,15 +768,28 @@ export const pasp = (trackData: IsobmffVideoTrackData) => {
 };
 
 /** Colour Information Box: Specifies the color space of the video. */
-export const colr = (trackData: IsobmffVideoTrackData) => box('colr', [
-	ascii(trackData.muxer.isQuickTime ? 'nclc' : 'nclx'), // Colour type
-	u16(COLOR_PRIMARIES_MAP[trackData.info.decoderConfig.colorSpace!.primaries!]), // Colour primaries
-	u16(TRANSFER_CHARACTERISTICS_MAP[trackData.info.decoderConfig.colorSpace!.transfer!]), // Transfer characteristics
-	u16(MATRIX_COEFFICIENTS_MAP[trackData.info.decoderConfig.colorSpace!.matrix!]), // Matrix coefficients
-	trackData.muxer.isQuickTime
-		? [] // Doesn't have it
-		: u8((trackData.info.decoderConfig.colorSpace!.fullRange ? 1 : 0) << 7), // Full range flag
-]);
+export const colr = (trackData: IsobmffVideoTrackData) => {
+	const colorSpace = trackData.info.decoderConfig.colorSpace;
+
+	return box('colr', [
+		// Colour type
+		ascii(trackData.muxer.isQuickTime ? 'nclc' : 'nclx'),
+
+		// Colour primaries
+		u16(colorSpace?.primaries != null ? COLOR_PRIMARIES_MAP[colorSpace.primaries] : 2),
+
+		// Transfer characteristics
+		u16(colorSpace?.transfer != null ? TRANSFER_CHARACTERISTICS_MAP[colorSpace.transfer] : 2),
+
+		// Matrix coefficients
+		u16(colorSpace?.matrix != null ? MATRIX_COEFFICIENTS_MAP[colorSpace.matrix] : 2),
+
+		// Full range flag
+		trackData.muxer.isQuickTime
+			? [] // Doesn't have it
+			: u8((colorSpace?.fullRange ? 1 : 0) << 7),
+	]);
+};
 
 /** AVC Configuration Box: Provides additional information to the decoder. */
 export const avcC = (trackData: IsobmffVideoTrackData) => trackData.info.decoderConfig && box('avcC', [
@@ -811,17 +826,17 @@ export const vpcC = (trackData: IsobmffVideoTrackData) => {
 		? Number(parts[5])
 		: decoderConfig.colorSpace?.primaries
 			? COLOR_PRIMARIES_MAP[decoderConfig.colorSpace.primaries]
-			: 2; // Default to undetermined
+			: 1;
 	const transferCharacteristics = parts[6]
 		? Number(parts[6])
 		: decoderConfig.colorSpace?.transfer
 			? TRANSFER_CHARACTERISTICS_MAP[decoderConfig.colorSpace.transfer]
-			: 2;
+			: 1;
 	const matrixCoefficients = parts[7]
 		? Number(parts[7])
 		: decoderConfig.colorSpace?.matrix
 			? MATRIX_COEFFICIENTS_MAP[decoderConfig.colorSpace.matrix]
-			: 2;
+			: 1;
 
 	return fullBox('vpcC', 1, 0, [
 		u8(profile), // Profile

src/isobmff/isobmff-demuxer.ts

@@ -14,6 +14,7 @@ import {
 	DTS_FOURCCS,
 	DtsFourCc,
 	extractAudioCodecString,
+	extractColorSpace,
 	extractVideoCodecString,
 	MediaCodec,
 	OPUS_SAMPLE_RATE,
@@ -26,12 +27,16 @@ import {
 } from '../codec';
 import {
 	Av1CodecInfo,
+	av1CodecInfoHasColorInfo,
 	AvcDecoderConfigurationRecord,
 	extractAv1CodecInfoFromPacket,
+	extractProresCodecInfoFromPacket,
 	extractVp9CodecInfoFromPacket,
 	FlacBlockType,
 	HevcDecoderConfigurationRecord,
+	ProresCodecInfo,
 	Vp9CodecInfo,
+	vp9CodecInfoHasColorInfo,
 	parseEac3Config,
 	getEac3SampleRate,
 	getEac3ChannelCount,
@@ -72,6 +77,8 @@ import {
 	roundIfAlmostInteger,
 	hexStringToBytes,
 	HEX_STRING_REGEX,
+	EMPTY_COLOR_SPACE,
+	colorSpaceIsComplete,
 } from '../misc';
 import { EncodedPacket, PLACEHOLDER_DATA } from '../packet';
 import { buildIsobmffMimeType, parsePsshBoxContents, psshBoxesAreEqual, PsshBox } from './isobmff-misc';
@@ -151,12 +158,13 @@ type InternalTrack = {
 		squarePixelHeight: number;
 		codec: VideoCodec | null;
 		codecDescription: Uint8Array | null;
-		colorSpace: VideoColorSpaceInit | null;
+		colorSpace: VideoColorSpaceInit;
 		avcType: 1 | 3 | null;
 		avcCodecInfo: AvcDecoderConfigurationRecord | null;
 		hevcCodecInfo: HevcDecoderConfigurationRecord | null;
 		vp9CodecInfo: Vp9CodecInfo | null;
 		av1CodecInfo: Av1CodecInfo | null;
+		proresCodecInfo: ProresCodecInfo | null;
 		proresFormat: ProresFourCc | null;
 	};
 } | {
@@ -1027,12 +1035,13 @@ export class IsobmffDemuxer extends Demuxer {
 						squarePixelHeight: -1,
 						codec: null,
 						codecDescription: null,
-						colorSpace: null,
+						colorSpace: { ...EMPTY_COLOR_SPACE },
 						avcType: null,
 						avcCodecInfo: null,
 						hevcCodecInfo: null,
 						vp9CodecInfo: null,
 						av1CodecInfo: null,
+						proresCodecInfo: null,
 						proresFormat: null,
 					};
 				} else if (handlerType === 'soun') {
@@ -1472,6 +1481,12 @@ export class IsobmffDemuxer extends Demuxer {
 				// Logic from https://aomediacodec.github.io/av1-spec/av1-spec.pdf
 				const bitDepth = profile === 2 && highBitDepth ? (twelveBit ? 12 : 10) : (highBitDepth ? 10 : 8);
 
+				slice.skip(1); // Reserved bits + initial presentation delay
+
+				// Parse config OBUs if there are any
+				const configObus = readBytes(slice, boxInfo.contentSize - 4);
+				const configObuInfo = extractAv1CodecInfoFromPacket(configObus);
+
 				track.info.av1CodecInfo = {
 					profile,
 					level,
@@ -1481,6 +1496,10 @@ export class IsobmffDemuxer extends Demuxer {
 					chromaSubsamplingX,
 					chromaSubsamplingY,
 					chromaSamplePosition,
+					videoFullRangeFlag: configObuInfo?.videoFullRangeFlag ?? 0,
+					colourPrimaries: configObuInfo?.colourPrimaries ?? 2,
+					transferCharacteristics: configObuInfo?.transferCharacteristics ?? 2,
+					matrixCoefficients: configObuInfo?.matrixCoefficients ?? 2,
 				};
 			}; break;
 
@@ -3410,11 +3429,16 @@ class IsobmffVideoTrackBacking extends IsobmffTrackBacking implements InputVideo
 	}
 
 	async getColorSpace(): Promise<VideoColorSpaceInit> {
+		const decoderConfig = await this.getDecoderConfig();
+		if (!decoderConfig) {
+			return this.internalTrack.info.colorSpace;
+		}
+
 		return {
-			primaries: this.internalTrack.info.colorSpace?.primaries,
-			transfer: this.internalTrack.info.colorSpace?.transfer,
-			matrix: this.internalTrack.info.colorSpace?.matrix,
-			fullRange: this.internalTrack.info.colorSpace?.fullRange,
+			primaries: decoderConfig.colorSpace?.primaries,
+			transfer: decoderConfig.colorSpace?.transfer,
+			matrix: decoderConfig.colorSpace?.matrix,
+			fullRange: decoderConfig.colorSpace?.fullRange,
 		};
 	}
 
@@ -3439,12 +3463,56 @@ class IsobmffVideoTrackBacking extends IsobmffTrackBacking implements InputVideo
 				const firstPacket = await this.getFirstPacket({});
 				this.internalTrack.info.hevcCodecInfo
 					= firstPacket && extractHevcDecoderConfigurationRecord(firstPacket.data);
-			} else if (this.internalTrack.info.codec === 'vp9' && !this.internalTrack.info.vp9CodecInfo) {
+			} else if (
+				this.internalTrack.info.codec === 'vp9'
+				&& (
+					!this.internalTrack.info.vp9CodecInfo
+					// The codec info extracted from vpcC may claim the color space is "undefined"
+					|| !vp9CodecInfoHasColorInfo(this.internalTrack.info.vp9CodecInfo)
+				)
+			) {
 				const firstPacket = await this.getFirstPacket({});
-				this.internalTrack.info.vp9CodecInfo = firstPacket && extractVp9CodecInfoFromPacket(firstPacket.data);
-			} else if (this.internalTrack.info.codec === 'av1' && !this.internalTrack.info.av1CodecInfo) {
+				const packetInfo = firstPacket && extractVp9CodecInfoFromPacket(firstPacket.data);
+
+				if (packetInfo) {
+					this.internalTrack.info.vp9CodecInfo = {
+						...(this.internalTrack.info.vp9CodecInfo ?? packetInfo),
+						videoFullRangeFlag: packetInfo.videoFullRangeFlag,
+						colourPrimaries: packetInfo.colourPrimaries,
+						transferCharacteristics: packetInfo.transferCharacteristics,
+						matrixCoefficients: packetInfo.matrixCoefficients,
+					};
+				}
+			} else if (
+				this.internalTrack.info.codec === 'av1'
+				&& (
+					!this.internalTrack.info.av1CodecInfo
+					// The codec info extracted from av1C may not contain color space information
+					|| !av1CodecInfoHasColorInfo(
+						this.internalTrack.info.av1CodecInfo,
+					)
+				)
+			) {
 				const firstPacket = await this.getFirstPacket({});
-				this.internalTrack.info.av1CodecInfo = firstPacket && extractAv1CodecInfoFromPacket(firstPacket.data);
+				const packetInfo = firstPacket && extractAv1CodecInfoFromPacket(firstPacket.data);
+
+				if (packetInfo) {
+					this.internalTrack.info.av1CodecInfo = packetInfo;
+				}
+			} else if (this.internalTrack.info.codec === 'prores' && !this.internalTrack.info.proresCodecInfo) {
+				const firstPacket = await this.getFirstPacket({});
+				this.internalTrack.info.proresCodecInfo
+					= firstPacket && extractProresCodecInfoFromPacket(firstPacket.data);
+			}
+
+			if (!colorSpaceIsComplete(this.internalTrack.info.colorSpace)) {
+				const colorSpace = extractColorSpace(this.internalTrack.info);
+
+				// Fill the missing values
+				this.internalTrack.info.colorSpace.primaries ??= colorSpace.primaries;
+				this.internalTrack.info.colorSpace.transfer ??= colorSpace.transfer;
+				this.internalTrack.info.colorSpace.matrix ??= colorSpace.matrix;
+				this.internalTrack.info.colorSpace.fullRange ??= colorSpace.fullRange;
 			}
 
 			const config: VideoDecoderConfig = {
@@ -3452,7 +3520,7 @@ class IsobmffVideoTrackBacking extends IsobmffTrackBacking implements InputVideo
 				codedWidth: this.internalTrack.info.width,
 				codedHeight: this.internalTrack.info.height,
 				description: this.internalTrack.info.codecDescription ?? undefined,
-				colorSpace: this.internalTrack.info.colorSpace ?? undefined,
+				colorSpace: this.internalTrack.info.colorSpace,
 			};
 
 			if (

src/matroska/matroska-demuxer.ts

@@ -12,6 +12,7 @@ import {
 	extractDtsFourCcFromPacket,
 	extractAvcDecoderConfigurationRecord,
 	extractHevcDecoderConfigurationRecord,
+	extractProresCodecInfoFromPacket,
 	extractVp9CodecInfoFromPacket,
 } from '../codec-data';
 import {
@@ -19,6 +20,7 @@ import {
 	AudioCodec,
 	DtsFourCc,
 	extractAudioCodecString,
+	extractColorSpace,
 	extractVideoCodecString,
 	MediaCodec,
 	OPUS_SAMPLE_RATE,
@@ -40,6 +42,8 @@ import {
 	assert,
 	binarySearchLessOrEqual,
 	COLOR_PRIMARIES_MAP_INVERSE,
+	colorSpaceIsComplete,
+	EMPTY_COLOR_SPACE,
 	findLastIndex,
 	isIso639Dash2LanguageCode,
 	isThenable,
@@ -220,7 +224,7 @@ type InternalTrack = {
 			rotation: Rotation;
 			codec: VideoCodec | null;
 			codecDescription: Uint8Array | null;
-			colorSpace: VideoColorSpaceInit | null;
+			colorSpace: VideoColorSpaceInit;
 			alphaMode: boolean;
 			proresFormat: ProresFourCc | null;
 		}
@@ -1199,7 +1203,7 @@ export class MatroskaDemuxer extends Demuxer {
 						rotation: 0,
 						codec: null,
 						codecDescription: null,
-						colorSpace: null,
+						colorSpace: { ...EMPTY_COLOR_SPACE },
 						alphaMode: false,
 						proresFormat: null,
 					};
@@ -1363,37 +1367,39 @@ export class MatroskaDemuxer extends Demuxer {
 			case EBMLId.Colour: {
 				if (this.currentTrack?.info?.type !== 'video') break;
 
-				this.currentTrack.info.colorSpace = {};
 				this.readContiguousElements(slice.slice(dataStartPos, size));
 			}; break;
 
 			case EBMLId.MatrixCoefficients: {
-				if (this.currentTrack?.info?.type !== 'video' || !this.currentTrack.info.colorSpace) break;
+				if (this.currentTrack?.info?.type !== 'video') break;
 
 				const matrixCoefficients = readUnsignedInt(slice, size);
-				const mapped = MATRIX_COEFFICIENTS_MAP_INVERSE[matrixCoefficients] ?? null;
+				const mapped = MATRIX_COEFFICIENTS_MAP_INVERSE[matrixCoefficients];
 				this.currentTrack.info.colorSpace.matrix = mapped as VideoColorSpaceInit['matrix'];
 			}; break;
 
 			case EBMLId.Range: {
-				if (this.currentTrack?.info?.type !== 'video' || !this.currentTrack.info.colorSpace) break;
+				if (this.currentTrack?.info?.type !== 'video') break;
 
-				this.currentTrack.info.colorSpace.fullRange = readUnsignedInt(slice, size) === 2;
+				const range = readUnsignedInt(slice, size);
+				this.currentTrack.info.colorSpace.fullRange = range === 1 || range === 2
+					? range === 2
+					: undefined;
 			}; break;
 
 			case EBMLId.TransferCharacteristics: {
-				if (this.currentTrack?.info?.type !== 'video' || !this.currentTrack.info.colorSpace) break;
+				if (this.currentTrack?.info?.type !== 'video') break;
 
 				const transferCharacteristics = readUnsignedInt(slice, size);
-				const mapped = TRANSFER_CHARACTERISTICS_MAP_INVERSE[transferCharacteristics] ?? null;
+				const mapped = TRANSFER_CHARACTERISTICS_MAP_INVERSE[transferCharacteristics];
 				this.currentTrack.info.colorSpace.transfer = mapped as VideoColorSpaceInit['transfer'];
 			}; break;
 
 			case EBMLId.Primaries: {
-				if (this.currentTrack?.info?.type !== 'video' || !this.currentTrack.info.colorSpace) break;
+				if (this.currentTrack?.info?.type !== 'video') break;
 
 				const primaries = readUnsignedInt(slice, size);
-				const mapped = COLOR_PRIMARIES_MAP_INVERSE[primaries] ?? null;
+				const mapped = COLOR_PRIMARIES_MAP_INVERSE[primaries];
 				this.currentTrack.info.colorSpace.primaries = mapped as VideoColorSpaceInit['primaries'];
 			}; break;
 
@@ -2490,11 +2496,16 @@ class MatroskaVideoTrackBacking extends MatroskaTrackBacking implements InputVid
 	}
 
 	async getColorSpace(): Promise<VideoColorSpaceInit> {
+		const decoderConfig = await this.getDecoderConfig();
+		if (!decoderConfig) {
+			return this.internalTrack.info.colorSpace;
+		}
+
 		return {
-			primaries: this.internalTrack.info.colorSpace?.primaries,
-			transfer: this.internalTrack.info.colorSpace?.transfer,
-			matrix: this.internalTrack.info.colorSpace?.matrix,
-			fullRange: this.internalTrack.info.colorSpace?.fullRange,
+			primaries: decoderConfig.colorSpace?.primaries,
+			transfer: decoderConfig.colorSpace?.transfer,
+			matrix: decoderConfig.colorSpace?.matrix,
+			fullRange: decoderConfig.colorSpace?.fullRange,
 		};
 	}
 
@@ -2517,6 +2528,7 @@ class MatroskaVideoTrackBacking extends MatroskaTrackBacking implements InputVid
 			const needsPacketForAdditionalInfo
 				= this.internalTrack.info.codec === 'vp9'
 					|| this.internalTrack.info.codec === 'av1'
+					|| this.internalTrack.info.codec === 'prores'
 					// Packets are in Annex B format:
 					|| (this.internalTrack.info.codec === 'avc' && !this.internalTrack.info.codecDescription)
 					// Packets are in Annex B format:
@@ -2526,32 +2538,47 @@ class MatroskaVideoTrackBacking extends MatroskaTrackBacking implements InputVid
 				firstPacket = await this.getFirstPacket({});
 			}
 
+			const codecInfo = {
+				width: this.internalTrack.info.width,
+				height: this.internalTrack.info.height,
+				codec: this.internalTrack.info.codec,
+				codecDescription: this.internalTrack.info.codecDescription,
+				colorSpace: this.internalTrack.info.colorSpace,
+				avcType: 1 as const, // We don't know better (or do we?) so just assume 'avc1'
+				avcCodecInfo: this.internalTrack.info.codec === 'avc' && firstPacket
+					? extractAvcDecoderConfigurationRecord(firstPacket.data)
+					: null,
+				hevcCodecInfo: this.internalTrack.info.codec === 'hevc' && firstPacket
+					? extractHevcDecoderConfigurationRecord(firstPacket.data)
+					: null,
+				vp9CodecInfo: this.internalTrack.info.codec === 'vp9' && firstPacket
+					? extractVp9CodecInfoFromPacket(firstPacket.data)
+					: null,
+				av1CodecInfo: this.internalTrack.info.codec === 'av1' && firstPacket
+					? extractAv1CodecInfoFromPacket(firstPacket.data)
+					: null,
+				proresCodecInfo: this.internalTrack.info.codec === 'prores' && firstPacket
+					? extractProresCodecInfoFromPacket(firstPacket.data)
+					: null,
+				proresFormat: this.internalTrack.info.proresFormat,
+			};
+
+			if (!colorSpaceIsComplete(this.internalTrack.info.colorSpace)) {
+				const colorSpace = extractColorSpace(codecInfo);
+
+				// Fill the missing values
+				this.internalTrack.info.colorSpace.primaries ??= colorSpace.primaries;
+				this.internalTrack.info.colorSpace.transfer ??= colorSpace.transfer;
+				this.internalTrack.info.colorSpace.matrix ??= colorSpace.matrix;
+				this.internalTrack.info.colorSpace.fullRange ??= colorSpace.fullRange;
+			}
+
 			const config: VideoDecoderConfig = {
-				codec: extractVideoCodecString({
-					width: this.internalTrack.info.width,
-					height: this.internalTrack.info.height,
-					codec: this.internalTrack.info.codec,
-					codecDescription: this.internalTrack.info.codecDescription,
-					colorSpace: this.internalTrack.info.colorSpace,
-					avcType: 1, // We don't know better (or do we?) so just assume 'avc1'
-					avcCodecInfo: this.internalTrack.info.codec === 'avc' && firstPacket
-						? extractAvcDecoderConfigurationRecord(firstPacket.data)
-						: null,
-					hevcCodecInfo: this.internalTrack.info.codec === 'hevc' && firstPacket
-						? extractHevcDecoderConfigurationRecord(firstPacket.data)
-						: null,
-					vp9CodecInfo: this.internalTrack.info.codec === 'vp9' && firstPacket
-						? extractVp9CodecInfoFromPacket(firstPacket.data)
-						: null,
-					av1CodecInfo: this.internalTrack.info.codec === 'av1' && firstPacket
-						? extractAv1CodecInfoFromPacket(firstPacket.data)
-						: null,
-					proresFormat: this.internalTrack.info.proresFormat,
-				}),
+				codec: extractVideoCodecString(codecInfo),
 				codedWidth: this.internalTrack.info.width,
 				codedHeight: this.internalTrack.info.height,
 				description: this.internalTrack.info.codecDescription ?? undefined,
-				colorSpace: this.internalTrack.info.colorSpace ?? undefined,
+				colorSpace: this.internalTrack.info.colorSpace,
 			};
 
 			if (

src/matroska/matroska-muxer.ts

@@ -14,7 +14,7 @@ import {
 	UNDETERMINED_LANGUAGE,
 	assert,
 	assertNever,
-	colorSpaceIsComplete,
+	colorSpaceIsEmpty,
 	imageMimeTypeToExtension,
 	keyValueIterator,
 	normalizeRotation,
@@ -405,29 +405,31 @@ export class MatroskaMuxer extends Muxer {
 			(hasNonSquarePixelAspectRatio ? { id: EBMLId.DisplayHeight, data: trackData.info.aspectRatio!.den } : null),
 			(hasNonSquarePixelAspectRatio ? { id: EBMLId.DisplayUnit, data: 3 } : null), // 3 = display aspect ratio
 			trackData.info.alphaMode ? { id: EBMLId.AlphaMode, data: 1 } : null,
-			(colorSpaceIsComplete(colorSpace)
-				? {
+			(colorSpaceIsEmpty(colorSpace)
+				? null
+				: {
 						id: EBMLId.Colour,
 						data: [
 							{
 								id: EBMLId.MatrixCoefficients,
-								data: MATRIX_COEFFICIENTS_MAP[colorSpace.matrix],
+								data: colorSpace?.matrix != null ? MATRIX_COEFFICIENTS_MAP[colorSpace.matrix] : 2,
 							},
 							{
 								id: EBMLId.TransferCharacteristics,
-								data: TRANSFER_CHARACTERISTICS_MAP[colorSpace.transfer],
+								data: colorSpace?.transfer != null
+									? TRANSFER_CHARACTERISTICS_MAP[colorSpace.transfer]
+									: 2,
 							},
 							{
 								id: EBMLId.Primaries,
-								data: COLOR_PRIMARIES_MAP[colorSpace.primaries],
+								data: colorSpace?.primaries != null ? COLOR_PRIMARIES_MAP[colorSpace.primaries] : 2,
 							},
 							{
 								id: EBMLId.Range,
-								data: colorSpace.fullRange ? 2 : 1,
+								data: colorSpace?.fullRange != null ? (colorSpace.fullRange ? 2 : 1) : 0,
 							},
 						],
-					}
-				: null),
+					}),
 			(flippedRotation
 				? {
 						id: EBMLId.Projection,

src/media-sink.ts

@@ -29,6 +29,7 @@ import {
 	assertNever,
 	CallSerializer,
 	clamp,
+	colorSpaceIsComplete,
 	getInt24,
 	getUint24,
 	insertSorted,
@@ -944,20 +945,42 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
 				}
 			};
 
-			if (codec === 'avc' && this.decoderConfig.description && isChromium()) {
-				// Chromium has/had a bug with playing interlaced AVC (https://issues.chromium.org/issues/456919096)
-				// which can be worked around by requesting that software decoding be used. So, here we peek into the
-				// AVC description, if present, and switch to software decoding if we find interlaced content.
-				const record = deserializeAvcDecoderConfigurationRecord(toUint8Array(this.decoderConfig.description));
-				if (record && record.sequenceParameterSets.length > 0) {
-					const sps = parseAvcSps(record.sequenceParameterSets[0]!);
-					if (sps && sps.frameMbsOnlyFlag === 0) {
-						this.decoderConfig = {
-							...this.decoderConfig,
-							hardwareAcceleration: 'prefer-software',
-						};
+			if (isChromium()) {
+				if (codec === 'avc' && this.decoderConfig.description) {
+					// Chromium has/had a bug with playing interlaced AVC (https://issues.chromium.org/issues/456919096)
+					// which can be worked around by requesting that software decoding be used. So, here we peek into
+					// the AVC description, if present, and switch to software decoding if we find interlaced content.
+					const record = deserializeAvcDecoderConfigurationRecord(
+						toUint8Array(this.decoderConfig.description),
+					);
+					if (record && record.sequenceParameterSets.length > 0) {
+						const sps = parseAvcSps(record.sequenceParameterSets[0]!);
+						if (sps && sps.frameMbsOnlyFlag === 0) {
+							this.decoderConfig = {
+								...this.decoderConfig,
+								hardwareAcceleration: 'prefer-software',
+							};
+						}
 					}
 				}
+
+				if (!colorSpaceIsComplete(this.decoderConfig.colorSpace)) {
+					// Found via https://github.com/remotion-dev/remotion/issues/10841.
+					// If the color space is incomplete (which is often that it's just partially filled), Chromium has
+					// some nasty logic where it doesn't pass that information along to the GPU at all. The result is
+					// that information is genuinely lost, like the color matrix for example. Chromium has other code
+					// paths where it just fills the missing values with a hardcoded default, so we do the exact same
+					// thing here, with the same hardcoded defaults:
+					this.decoderConfig = {
+						...this.decoderConfig,
+						colorSpace: {
+							primaries: this.decoderConfig.colorSpace?.primaries ?? 'bt709',
+							matrix: this.decoderConfig.colorSpace?.matrix ?? 'bt709',
+							transfer: this.decoderConfig.colorSpace?.transfer ?? 'bt709',
+							fullRange: this.decoderConfig.colorSpace?.fullRange ?? false,
+						},
+					};
+				}
 			}
 
 			const stack = new Error('Decoding error').stack;

src/misc.ts

@@ -162,6 +162,25 @@ export const colorSpaceIsComplete = (
 	);
 };
 
+export const colorSpaceIsEmpty = (colorSpace: VideoColorSpaceInit | undefined) => {
+	return (
+		!colorSpace
+		|| (
+			colorSpace.primaries == null
+			&& colorSpace.transfer == null
+			&& colorSpace.matrix == null
+			&& colorSpace.fullRange == null
+		)
+	);
+};
+
+export const EMPTY_COLOR_SPACE: VideoColorSpaceInit = {
+	primaries: undefined,
+	transfer: undefined,
+	matrix: undefined,
+	fullRange: undefined,
+};
+
 export const isAllowSharedBufferSource = (x: unknown) => {
 	return (
 		x instanceof ArrayBuffer

test/browser/color-space.test.ts

@@ -0,0 +1,222 @@
+import { expect, test } from 'vitest';
+import { VideoCodec } from '../../src/codec.js';
+import { ALL_FORMATS } from '../../src/input-format.js';
+import { Input } from '../../src/input.js';
+import { EncodedPacketSink } from '../../src/media-sink.js';
+import { EncodedVideoPacketSource } from '../../src/media-source.js';
+import { assert, colorSpaceIsComplete } from '../../src/misc.js';
+import { Output } from '../../src/output.js';
+import { MkvOutputFormat, Mp4OutputFormat } from '../../src/output-format.js';
+import { EncodedPacket } from '../../src/packet.js';
+import { BufferSource, Source, UrlSource } from '../../src/source.js';
+import { BufferTarget } from '../../src/target.js';
+
+test('Color space extraction, AVC in MP4', async () => {
+	const source = await readPackets('/video.mp4');
+	const buffer = await remuxWithoutColorSpace(new Mp4OutputFormat(), source);
+
+	await expectColorSpace(new BufferSource(buffer), {
+		primaries: undefined,
+		transfer: undefined,
+		matrix: 'bt470bg',
+		fullRange: true,
+	});
+});
+
+test('Color space extraction, HEVC in MP4', async () => {
+	const source = await readPackets('/video-h265.mp4');
+	const buffer = await remuxWithoutColorSpace(new Mp4OutputFormat(), source);
+
+	await expectColorSpace(new BufferSource(buffer), {
+		primaries: undefined,
+		transfer: undefined,
+		matrix: 'bt470bg',
+		fullRange: true,
+	});
+});
+
+test('Color space extraction, VP9 in MP4', async () => {
+	const source = await encodePackets('vp9', 'vp09.00.10.08');
+	const buffer = await remuxWithoutColorSpace(new Mp4OutputFormat(), source);
+
+	await expectCompleteColorSpace(new BufferSource(buffer));
+});
+
+test('Color space extraction, AV1 in MP4', async () => {
+	const source = await encodePackets('av1', 'av01.0.04M.08');
+	const buffer = await remuxWithoutColorSpace(new Mp4OutputFormat(), source);
+
+	await expectCompleteColorSpace(new BufferSource(buffer));
+});
+
+test('Color space extraction, AVC in Matroska', async () => {
+	const source = await readPackets('/video.mp4');
+	const buffer = await remuxWithoutColorSpace(new MkvOutputFormat(), source);
+
+	await expectColorSpace(new BufferSource(buffer), {
+		primaries: undefined,
+		transfer: undefined,
+		matrix: 'bt470bg',
+		fullRange: true,
+	});
+});
+
+test('Color space extraction, HEVC in Matroska', async () => {
+	const source = await readPackets('/video-h265.mp4');
+	const buffer = await remuxWithoutColorSpace(new MkvOutputFormat(), source);
+
+	await expectColorSpace(new BufferSource(buffer), {
+		// The SPS only signals the matrix and the range, leaving the rest unspecified
+		primaries: undefined,
+		transfer: undefined,
+		matrix: 'bt470bg',
+		fullRange: true,
+	});
+});
+
+test('Color space extraction, VP9 in Matroska', async () => {
+	const source = await encodePackets('vp9', 'vp09.00.10.08');
+	const buffer = await remuxWithoutColorSpace(new MkvOutputFormat(), source);
+
+	await expectCompleteColorSpace(new BufferSource(buffer));
+});
+
+test('Color space extraction, AV1 in Matroska', async () => {
+	const source = await encodePackets('av1', 'av01.0.04M.08');
+	const buffer = await remuxWithoutColorSpace(new MkvOutputFormat(), source);
+
+	await expectCompleteColorSpace(new BufferSource(buffer));
+});
+
+const readPackets = async (path: string) => {
+	using input = new Input({
+		source: new UrlSource(path),
+		formats: ALL_FORMATS,
+	});
+
+	const track = await input.getPrimaryVideoTrack();
+	assert(track);
+
+	const codec = await track.getCodec();
+	assert(codec);
+
+	const decoderConfig = await track.getDecoderConfig();
+	assert(decoderConfig);
+
+	const sink = new EncodedPacketSink(track);
+	const packets: EncodedPacket[] = [];
+
+	for await (const packet of sink.packets()) {
+		packets.push(packet);
+
+		if (packets.length === 10) {
+			break;
+		}
+	}
+
+	return { codec, decoderConfig, packets };
+};
+
+const encodePackets = async (codec: VideoCodec, codecString: string) => {
+	const width = 320;
+	const height = 240;
+
+	const canvas = new OffscreenCanvas(width, height);
+	const context = canvas.getContext('2d')!;
+
+	let decoderConfig: VideoDecoderConfig | null = null;
+	const packets: EncodedPacket[] = [];
+
+	// We manually go through WebCodecs here so we can intercept the decoder config
+	const encoder = new VideoEncoder({
+		output: (chunk, metadata) => {
+			decoderConfig ??= metadata?.decoderConfig ?? null;
+			packets.push(EncodedPacket.fromEncodedChunk(chunk));
+		},
+		error: (error) => {
+			throw error;
+		},
+	});
+	encoder.configure({ codec: codecString, width, height });
+
+	for (let i = 0; i < 10; i++) {
+		context.fillStyle = `hsl(${36 * i}, 100%, 50%)`;
+		context.fillRect(0, 0, width, height);
+		context.fillStyle = 'black';
+		context.fillRect(20 * i, 10 * i, 80, 80);
+
+		const frame = new VideoFrame(canvas, { timestamp: i * 1e6 / 30, duration: 1e6 / 30 });
+		encoder.encode(frame, { keyFrame: i === 0 });
+		frame.close();
+	}
+
+	await encoder.flush();
+	encoder.close();
+
+	assert(decoderConfig);
+
+	return { codec, decoderConfig, packets };
+};
+
+const remuxWithoutColorSpace = async (
+	format: Mp4OutputFormat | MkvOutputFormat,
+	source: { codec: VideoCodec; decoderConfig: VideoDecoderConfig; packets: EncodedPacket[] },
+) => {
+	const output = new Output({
+		format,
+		target: new BufferTarget(),
+	});
+
+	const videoSource = new EncodedVideoPacketSource(source.codec);
+	output.addVideoTrack(videoSource);
+
+	await output.start();
+
+	const decoderConfig = { ...source.decoderConfig };
+	delete decoderConfig.colorSpace;
+
+	let isFirstPacket = true;
+	for (const packet of source.packets) {
+		await videoSource.add(packet, isFirstPacket ? { decoderConfig } : undefined);
+		isFirstPacket = false;
+	}
+
+	await output.finalize();
+
+	const buffer = output.target.buffer;
+	assert(buffer);
+
+	return buffer;
+};
+
+const expectColorSpace = async (source: Source, expected: VideoColorSpaceInit) => {
+	using input = new Input({
+		source,
+		formats: ALL_FORMATS,
+	});
+
+	const track = await input.getPrimaryVideoTrack();
+	assert(track);
+
+	const decoderConfig = await track.getDecoderConfig();
+	assert(decoderConfig);
+
+	expect(await track.getColorSpace()).toEqual(expected);
+	expect({ ...decoderConfig.colorSpace }).toEqual(expected);
+};
+
+const expectCompleteColorSpace = async (source: Source) => {
+	using input = new Input({
+		source,
+		formats: ALL_FORMATS,
+	});
+
+	const track = await input.getPrimaryVideoTrack();
+	assert(track);
+
+	const decoderConfig = await track.getDecoderConfig();
+	assert(decoderConfig);
+
+	expect(colorSpaceIsComplete(await track.getColorSpace())).toBe(true);
+	expect(colorSpaceIsComplete(decoderConfig.colorSpace)).toBe(true);
+};