Fix incomplete AV1 sequence header OBU parsing, fix missing saample close() calls in audio resampling path, add error logging when open samples are GC'd, bump patch

src/codec-data.ts

@@ -590,9 +590,7 @@ export const extractNalUnitTypeForHevc = (data: Uint8Array) => {
 };
 
 /** Builds a HevcDecoderConfigurationRecord from an HEVC packet in Annex B format. */
-export const extractHevcDecoderConfigurationRecord = (
-	packetData: Uint8Array,
-) => {
+export const extractHevcDecoderConfigurationRecord = (packetData: Uint8Array) => {
 	try {
 		const nalUnits = findNalUnitsInAnnexB(packetData);
 
@@ -1456,6 +1454,69 @@ export const extractAv1CodecInfoFromPacket = (
 			}
 		}
 
+		// Frame size
+		const frameWidthBitsMinus1 = bitstream.readBits(4);
+		const frameHeightBitsMinus1 = bitstream.readBits(4);
+		const n1 = frameWidthBitsMinus1 + 1;
+		bitstream.skipBits(n1); // max_frame_width_minus_1
+		const n2 = frameHeightBitsMinus1 + 1;
+		bitstream.skipBits(n2); // max_frame_height_minus_1
+
+		// Frame IDs
+		let frameIdNumbersPresentFlag = 0;
+		if (reducedStillPictureHeader) {
+			frameIdNumbersPresentFlag = 0;
+		} else {
+			frameIdNumbersPresentFlag = bitstream.readBits(1);
+		}
+
+		if (frameIdNumbersPresentFlag) {
+			bitstream.skipBits(4); // delta_frame_id_length_minus_2
+			bitstream.skipBits(3); // additional_frame_id_length_minus_1
+		}
+
+		bitstream.skipBits(1); // use_128x128_superblock
+		bitstream.skipBits(1); // enable_filter_intra
+		bitstream.skipBits(1); // enable_intra_edge_filter
+
+		if (!reducedStillPictureHeader) {
+			bitstream.skipBits(1); // enable_interintra_compound
+			bitstream.skipBits(1); // enable_masked_compound
+			bitstream.skipBits(1); // enable_warped_motion
+			bitstream.skipBits(1); // enable_dual_filter
+			const enableOrderHint = bitstream.readBits(1);
+
+			if (enableOrderHint) {
+				bitstream.skipBits(1); // enable_jnt_comp
+				bitstream.skipBits(1); // enable_ref_frame_mvs
+			}
+
+			const seqChooseScreenContentTools = bitstream.readBits(1);
+			let seqForceScreenContentTools = 0;
+
+			if (seqChooseScreenContentTools) {
+				seqForceScreenContentTools = 2; // SELECT_SCREEN_CONTENT_TOOLS
+			} else {
+				seqForceScreenContentTools = bitstream.readBits(1);
+			}
+
+			if (seqForceScreenContentTools > 0) {
+				const seqChooseIntegerMv = bitstream.readBits(1);
+				if (!seqChooseIntegerMv) {
+					bitstream.skipBits(1); // seq_force_integer_mv
+				}
+			}
+
+			if (enableOrderHint) {
+				bitstream.skipBits(3); // order_hint_bits_minus_1
+			}
+		}
+
+		bitstream.skipBits(1); // enable_superres
+		bitstream.skipBits(1); // enable_cdef
+		bitstream.skipBits(1); // enable_restoration
+
+		// color_config()
 		const highBitdepth = bitstream.readBits(1);
 
 		let bitDepth = 8;

src/conversion.ts

@@ -1509,7 +1509,10 @@ export class Conversion {
 				targetSampleRate,
 				startTime: this._startTimestamp,
 				endTime: this._endTimestamp,
-				onSample: sample => this._registerAudioSample(track, trackOptions, source, sample),
+				onSample: async (sample) => {
+					await this._registerAudioSample(track, trackOptions, source, sample);
+					sample.close();
+				},
 			});
 
 			const sink = new AudioSampleSink(track);
@@ -1521,6 +1524,7 @@ export class Conversion {
 				}
 
 				await resampler.add(sample);
+				sample.close();
 			}
 
 			await resampler.finalize();

src/sample.ts

@@ -21,6 +21,54 @@ import {
 
 polyfillSymbolDispose();
 
+type FinalizationRegistryValue = {
+	type: 'video';
+	data: VideoFrame | OffscreenCanvas | Uint8Array;
+} | {
+	type: 'audio';
+	data: AudioData | Uint8Array;
+};
+
+// Let's manually handle logging the garbage collection errors that are typically logged by the browser. This way, they
+// also kick for audio samples (which is normally not the case), making sure any incorrect code is quickly caught.
+let lastVideoGcErrorLog = -Infinity;
+let lastAudioGcErrorLog = -Infinity;
+let finalizationRegistry: FinalizationRegistry<FinalizationRegistryValue> | null = null;
+if (typeof FinalizationRegistry !== 'undefined') {
+	finalizationRegistry = new FinalizationRegistry<FinalizationRegistryValue>((value) => {
+		const now = Date.now();
+
+		if (value.type === 'video') {
+			if (now - lastVideoGcErrorLog >= 1000) {
+				// This error is annoying but oh so important
+				console.error(
+					`A VideoSample was garbage collected without first being closed. For proper resource management,`
+					+ ` make sure to call close() on all your VideoSamples as soon as you're done using them.`,
+				);
+
+				lastVideoGcErrorLog = now;
+			}
+
+			if (typeof VideoFrame !== 'undefined' && value.data instanceof VideoFrame) {
+				value.data.close(); // Prevent the browser error since we're logging our own
+			}
+		} else {
+			if (now - lastAudioGcErrorLog >= 1000) {
+				console.error(
+					`An AudioSample was garbage collected without first being closed. For proper resource management,`
+					+ ` make sure to call close() on all your AudioSamples as soon as you're done using them.`,
+				);
+
+				lastAudioGcErrorLog = now;
+			}
+
+			if (typeof AudioData !== 'undefined' && value.data instanceof AudioData) {
+				value.data.close();
+			}
+		}
+	});
+}
+
 /**
  * Metadata used for VideoSample initialization.
  * @group Samples
@@ -133,7 +181,11 @@ export class VideoSample implements Disposable {
 		data: VideoFrame | CanvasImageSource | AllowSharedBufferSource,
 		init?: VideoSampleInit,
 	) {
-		if (data instanceof ArrayBuffer || ArrayBuffer.isView(data)) {
+		if (
+			data instanceof ArrayBuffer
+			|| (typeof SharedArrayBuffer !== 'undefined' && data instanceof SharedArrayBuffer)
+			|| ArrayBuffer.isView(data)
+		) {
 			if (!init || typeof init !== 'object') {
 				throw new TypeError('init must be an object.');
 			}
@@ -265,6 +317,8 @@ export class VideoSample implements Disposable {
 		} else {
 			throw new TypeError('Invalid data type: Must be a BufferSource or CanvasImageSource.');
 		}
+
+		finalizationRegistry?.register(this, { type: 'video', data: this._data }, this);
 	}
 
 	/** Clones this video sample. */
@@ -313,6 +367,8 @@ export class VideoSample implements Disposable {
 			return;
 		}
 
+		finalizationRegistry?.unregister(this);
+
 		if (isVideoFrame(this._data)) {
 			this._data.close();
 		} else {
@@ -853,8 +909,8 @@ export type AudioSampleCopyToOptions = {
  */
 export class AudioSample implements Disposable {
 	/** @internal */
-	_data: AudioData | Uint8Array;
 	/** @internal */
+	_data: AudioData | Uint8Array;
 	_closed: boolean = false;
 
 	/**
@@ -955,6 +1011,8 @@ export class AudioSample implements Disposable {
 
 			this._data = dataBuffer;
 		}
+
+		finalizationRegistry?.register(this, { type: 'audio', data: this._data }, this);
 	}
 
 	/** Returns the number of bytes required to hold the audio sample's data as specified by the given options. */
@@ -1116,9 +1174,8 @@ export class AudioSample implements Disposable {
 				}
 			}
 		} else {
-			// Branch for Uint8Array data (non-AudioData)
 			const uint8Data = this._data;
-			const srcView = new DataView(uint8Data.buffer, uint8Data.byteOffset, uint8Data.byteLength);
+			const srcView = toDataView(uint8Data);
 
 			const srcFormat = this.format;
 			const readFn = getReadFunction(srcFormat);
@@ -1188,6 +1245,8 @@ export class AudioSample implements Disposable {
 			return;
 		}
 
+		finalizationRegistry?.unregister(this);
+
 		if (isAudioData(this._data)) {
 			this._data.close();
 		} else {