Force software decoding in Chromium for interlaced AVC

src/codec-data.ts

@@ -229,7 +229,7 @@ const extractNalUnitTypeForAvc = (data: Uint8Array) => {
 };
 
 /** Builds an AvcDecoderConfigurationRecord from an AVC packet in Annex B format. */
-export const extractAvcDecoderConfigurationRecord = (packetData: Uint8Array) => {
+export const extractAvcDecoderConfigurationRecord = (packetData: Uint8Array): AvcDecoderConfigurationRecord | null => {
 	try {
 		const nalUnits = findNalUnitsInAnnexB(packetData);
 
@@ -247,60 +247,27 @@ export const extractAvcDecoderConfigurationRecord = (packetData: Uint8Array) =>
 
 		// Let's get the first SPS for profile and level information
 		const spsData = spsUnits[0]!;
-		const bitstream = new Bitstream(removeEmulationPreventionBytes(spsData));
+		const spsInfo = parseAvcSps(spsData);
+		assert(spsInfo !== null);
 
-		bitstream.skipBits(1); // forbidden_zero_bit
+		const hasExtendedData = spsInfo.profileIdc === 100
-		bitstream.skipBits(2); // nal_ref_idc
+			|| spsInfo.profileIdc === 110
-		const nal_unit_type = bitstream.readBits(5);
+			|| spsInfo.profileIdc === 122
+			|| spsInfo.profileIdc === 144;
 
-		if (nal_unit_type !== 7) { // SPS NAL unit type is 7
+		return {
-			console.error('Invalid SPS NAL unit type');
-			return null;
-		}
-
-		const profile_idc = bitstream.readAlignedByte();
-		const constraint_flags = bitstream.readAlignedByte();
-		const level_idc = bitstream.readAlignedByte();
-
-		const record: AvcDecoderConfigurationRecord = {
 			configurationVersion: 1,
-			avcProfileIndication: profile_idc,
+			avcProfileIndication: spsInfo.profileIdc,
-			profileCompatibility: constraint_flags,
+			profileCompatibility: spsInfo.constraintFlags,
-			avcLevelIndication: level_idc,
+			avcLevelIndication: spsInfo.levelIdc,
 			lengthSizeMinusOne: 3, // Typically 4 bytes for length field
 			sequenceParameterSets: spsUnits,
 			pictureParameterSets: ppsUnits,
-			chromaFormat: null,
+			chromaFormat: hasExtendedData ? spsInfo.chromaFormatIdc : null,
-			bitDepthLumaMinus8: null,
+			bitDepthLumaMinus8: hasExtendedData ? spsInfo.bitDepthLumaMinus8 : null,
-			bitDepthChromaMinus8: null,
+			bitDepthChromaMinus8: hasExtendedData ? spsInfo.bitDepthChromaMinus8 : null,
-			sequenceParameterSetExt: null,
+			sequenceParameterSetExt: hasExtendedData ? spsExtUnits : null,
 		};
-
-		if (
-			profile_idc === 100
-			|| profile_idc === 110
-			|| profile_idc === 122
-			|| profile_idc === 144
-		) {
-			readExpGolomb(bitstream); // seq_parameter_set_id
-
-			const chroma_format_idc = readExpGolomb(bitstream);
-
-			if (chroma_format_idc === 3) {
-				bitstream.skipBits(1); // separate_colour_plane_flag
-			}
-
-			const bit_depth_luma_minus8 = readExpGolomb(bitstream);
-
-			const bit_depth_chroma_minus8 = readExpGolomb(bitstream);
-
-			record.chromaFormat = chroma_format_idc;
-			record.bitDepthLumaMinus8 = bit_depth_luma_minus8;
-			record.bitDepthChromaMinus8 = bit_depth_chroma_minus8;
-			record.sequenceParameterSetExt = spsExtUnits;
-		}
-
-		return record;
 	} catch (error) {
 		console.error('Error building AVC Decoder Configuration Record:', error);
 		return null;
@@ -377,6 +344,206 @@ export const serializeAvcDecoderConfigurationRecord = (record: AvcDecoderConfigu
 	return new Uint8Array(bytes);
 };
 
+/** Deserializes an AvcDecoderConfigurationRecord from the format specified in Section 5.3.3.1 of ISO 14496-15. */
+export const deserializeAvcDecoderConfigurationRecord = (data: Uint8Array): AvcDecoderConfigurationRecord | null => {
+	try {
+		const view = toDataView(data);
+		let offset = 0;
+
+		// Read header
+		const configurationVersion = view.getUint8(offset++);
+		const avcProfileIndication = view.getUint8(offset++);
+		const profileCompatibility = view.getUint8(offset++);
+		const avcLevelIndication = view.getUint8(offset++);
+		const lengthSizeMinusOne = view.getUint8(offset++) & 0x03;
+
+		const numOfSequenceParameterSets = view.getUint8(offset++) & 0x1F;
+
+		// Read SPS
+		const sequenceParameterSets: Uint8Array[] = [];
+		for (let i = 0; i < numOfSequenceParameterSets; i++) {
+			const length = view.getUint16(offset, false);
+			offset += 2;
+
+			sequenceParameterSets.push(data.subarray(offset, offset + length));
+			offset += length;
+		}
+
+		const numOfPictureParameterSets = view.getUint8(offset++);
+
+		// Read PPS
+		const pictureParameterSets: Uint8Array[] = [];
+		for (let i = 0; i < numOfPictureParameterSets; i++) {
+			const length = view.getUint16(offset, false);
+			offset += 2;
+
+			pictureParameterSets.push(data.subarray(offset, offset + length));
+			offset += length;
+		}
+
+		const record: AvcDecoderConfigurationRecord = {
+			configurationVersion,
+			avcProfileIndication,
+			profileCompatibility,
+			avcLevelIndication,
+			lengthSizeMinusOne,
+			sequenceParameterSets,
+			pictureParameterSets,
+			chromaFormat: null,
+			bitDepthLumaMinus8: null,
+			bitDepthChromaMinus8: null,
+			sequenceParameterSetExt: null,
+		};
+
+		// Check if there are extended profile fields
+		if (
+			(
+				avcProfileIndication === 100
+				|| avcProfileIndication === 110
+				|| avcProfileIndication === 122
+				|| avcProfileIndication === 144
+			)
+			&& offset + 4 <= data.length
+		) {
+			const chromaFormat = view.getUint8(offset++) & 0x03;
+			const bitDepthLumaMinus8 = view.getUint8(offset++) & 0x07;
+			const bitDepthChromaMinus8 = view.getUint8(offset++) & 0x07;
+			const numOfSequenceParameterSetExt = view.getUint8(offset++);
+
+			record.chromaFormat = chromaFormat;
+			record.bitDepthLumaMinus8 = bitDepthLumaMinus8;
+			record.bitDepthChromaMinus8 = bitDepthChromaMinus8;
+
+			// Read SPS Ext
+			const sequenceParameterSetExt: Uint8Array[] = [];
+			for (let i = 0; i < numOfSequenceParameterSetExt; i++) {
+				const length = view.getUint16(offset, false);
+				offset += 2;
+
+				sequenceParameterSetExt.push(data.subarray(offset, offset + length));
+				offset += length;
+			}
+
+			record.sequenceParameterSetExt = sequenceParameterSetExt;
+		}
+
+		return record;
+	} catch (error) {
+		console.error('Error deserializing AVC Decoder Configuration Record:', error);
+		return null;
+	}
+};
+
+export type AvcSpsInfo = {
+	profileIdc: number;
+	constraintFlags: number;
+	levelIdc: number;
+	frameMbsOnlyFlag: number;
+	chromaFormatIdc: number | null;
+	bitDepthLumaMinus8: number | null;
+	bitDepthChromaMinus8: number | null;
+};
+
+/** Parses an AVC SPS (Sequence Parameter Set) to extract basic information. */
+export const parseAvcSps = (sps: Uint8Array): AvcSpsInfo | null => {
+	try {
+		const bitstream = new Bitstream(removeEmulationPreventionBytes(sps));
+
+		bitstream.skipBits(1); // forbidden_zero_bit
+		bitstream.skipBits(2); // nal_ref_idc
+		const nalUnitType = bitstream.readBits(5);
+
+		if (nalUnitType !== 7) { // SPS NAL unit type is 7
+			return null;
+		}
+
+		const profileIdc = bitstream.readAlignedByte();
+		const constraintFlags = bitstream.readAlignedByte();
+		const levelIdc = bitstream.readAlignedByte();
+
+		readExpGolomb(bitstream); // seq_parameter_set_id
+
+		let chromaFormatIdc: number | null = null;
+		let bitDepthLumaMinus8: number | null = null;
+		let bitDepthChromaMinus8: number | null = null;
+
+		// Handle high profile chroma_format_idc
+		if (
+			profileIdc === 100
+			|| profileIdc === 110
+			|| profileIdc === 122
+			|| profileIdc === 244
+			|| profileIdc === 44
+			|| profileIdc === 83
+			|| profileIdc === 86
+			|| profileIdc === 118
+			|| profileIdc === 128
+		) {
+			chromaFormatIdc = readExpGolomb(bitstream);
+			if (chromaFormatIdc === 3) {
+				bitstream.skipBits(1); // separate_colour_plane_flag
+			}
+			bitDepthLumaMinus8 = readExpGolomb(bitstream);
+			bitDepthChromaMinus8 = readExpGolomb(bitstream);
+			bitstream.skipBits(1); // qpprime_y_zero_transform_bypass_flag
+			const seqScalingMatrixPresentFlag = bitstream.readBits(1);
+			if (seqScalingMatrixPresentFlag) {
+				for (let i = 0; i < (chromaFormatIdc !== 3 ? 8 : 12); i++) {
+					const seqScalingListPresentFlag = bitstream.readBits(1);
+					if (seqScalingListPresentFlag) {
+						const sizeOfScalingList = i < 6 ? 16 : 64;
+						let lastScale = 8;
+						let nextScale = 8;
+						for (let j = 0; j < sizeOfScalingList; j++) {
+							if (nextScale !== 0) {
+								const deltaScale = readSignedExpGolomb(bitstream);
+								nextScale = (lastScale + deltaScale + 256) % 256;
+							}
+							lastScale = nextScale === 0 ? lastScale : nextScale;
+						}
+					}
+				}
+			}
+		}
+
+		readExpGolomb(bitstream); // log2_max_frame_num_minus4
+
+		const picOrderCntType = readExpGolomb(bitstream);
+		if (picOrderCntType === 0) {
+			readExpGolomb(bitstream); // log2_max_pic_order_cnt_lsb_minus4
+		} else if (picOrderCntType === 1) {
+			bitstream.skipBits(1); // delta_pic_order_always_zero_flag
+			readSignedExpGolomb(bitstream); // offset_for_non_ref_pic
+			readSignedExpGolomb(bitstream); // offset_for_top_to_bottom_field
+			const numRefFramesInPicOrderCntCycle = readExpGolomb(bitstream);
+			for (let i = 0; i < numRefFramesInPicOrderCntCycle; i++) {
+				readSignedExpGolomb(bitstream); // offset_for_ref_frame[i]
+			}
+		}
+
+		readExpGolomb(bitstream); // max_num_ref_frames
+		bitstream.skipBits(1); // gaps_in_frame_num_value_allowed_flag
+
+		readExpGolomb(bitstream); // pic_width_in_mbs_minus1
+		readExpGolomb(bitstream); // pic_height_in_map_units_minus1
+
+		const frameMbsOnlyFlag = bitstream.readBits(1);
+
+		return {
+			profileIdc,
+			constraintFlags,
+			levelIdc,
+			frameMbsOnlyFlag,
+			chromaFormatIdc,
+			bitDepthLumaMinus8,
+			bitDepthChromaMinus8,
+		};
+	} catch (error) {
+		console.error('Error parsing AVC SPS:', error);
+		return null;
+	}
+};
+
 // Data specified in ISO 14496-15
 export type HevcDecoderConfigurationRecord = {
 	configurationVersion: number;

src/media-sink.ts

@@ -8,10 +8,12 @@
 
 import { parsePcmCodec, PCM_AUDIO_CODECS, PcmAudioCodec, VideoCodec, AudioCodec } from './codec';
 import {
+	deserializeAvcDecoderConfigurationRecord,
 	determineVideoPacketType,
 	extractHevcNalUnits,
 	extractNalUnitTypeForHevc,
 	HevcNalUnitType,
+	parseAvcSps,
 } from './codec-data';
 import { CustomVideoDecoder, customVideoDecoders, CustomAudioDecoder, customAudioDecoders } from './custom-coder';
 import { InputDisposedError } from './input';
@@ -24,6 +26,7 @@ import {
 	getInt24,
 	getUint24,
 	insertSorted,
+	isChromium,
 	isFirefox,
 	isNumber,
 	isWebKit,
@@ -33,6 +36,7 @@ import {
 	Rotation,
 	toAsyncIterator,
 	toDataView,
+	toUint8Array,
 	validateAnyIterable,
 } from './misc';
 import { EncodedPacket } from './packet';
@@ -872,6 +876,22 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
 				}
 			};
 
+			if (codec === 'avc' && this.decoderConfig.description && isChromium()) {
+				// Chromium has/had a bug with playing interlaced AVC (https://issues.chromium.org/issues/456919096)
+				// which can be worked around by requesting that software decoding be used. So, here we peek into the
+				// AVC description, if present, and switch to software decoding if we find interlaced content.
+				const record = deserializeAvcDecoderConfigurationRecord(toUint8Array(this.decoderConfig.description));
+				if (record && record.sequenceParameterSets.length > 0) {
+					const sps = parseAvcSps(record.sequenceParameterSets[0]!);
+					if (sps && sps.frameMbsOnlyFlag === 0) {
+						this.decoderConfig = {
+							...this.decoderConfig,
+							hardwareAcceleration: 'prefer-software',
+						};
+					}
+				}
+			}
+
 			this.decoder = new VideoDecoder({
 				output: (frame) => {
 					try {
@@ -882,7 +902,7 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
 				},
 				error: onError,
 			});
-			this.decoder.configure(decoderConfig);
+			this.decoder.configure(this.decoderConfig);
 		}
 	}
 

src/misc.ts

@@ -665,10 +665,7 @@ export const isWebKit = () => {
 	}
 
 	// This even returns true for WebKit-wrapping browsers such as Chrome on iOS
-	const result = !!(typeof navigator !== 'undefined' && navigator.vendor?.match(/apple/i));
+	return isWebKitCache = !!(typeof navigator !== 'undefined' && navigator.vendor?.match(/apple/i));
-
-	isWebKitCache = result;
-	return result;
 };
 
 let isFirefoxCache: boolean | null = null;
@@ -680,6 +677,15 @@ export const isFirefox = () => {
 	return isFirefoxCache = typeof navigator !== 'undefined' && navigator.userAgent?.includes('Firefox');
 };
 
+let isChromiumCache: boolean | null = null;
+export const isChromium = () => {
+	if (isChromiumCache !== null) {
+		return isChromiumCache;
+	}
+
+	return isChromiumCache = !!(typeof navigator !== 'undefined' && navigator.vendor?.includes('Google Inc'));
+};
+
 /**
  * T or a promise that resolves to T.
  * @group Miscellaneous