From 03b9c6c9ae076f839c73d7709c210b799584839e Mon Sep 17 00:00:00 2001
From: Vanilagy <1696106+Vanilagy@users.noreply.github.com>
Date: Fri, 11 Apr 2025 19:19:49 +0200
Subject: [PATCH] Adjust media sink max queue size logic to prevent playback
 stalls

---
 src/media-sink.ts | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/src/media-sink.ts b/src/media-sink.ts
index e9c2c51..0cc4b52 100644
--- a/src/media-sink.ts
+++ b/src/media-sink.ts
@@ -264,7 +264,6 @@ export abstract class BaseMediaSampleSink<
 		validateTimestamp(startTimestamp);
 		validateTimestamp(endTimestamp);
 
-		const MAX_QUEUE_SIZE = 8;
 		const sampleQueue: MediaSample[] = [];
 		let firstSampleQueued = false;
 		let lastSample: MediaSample | null = null;
@@ -355,7 +354,8 @@ export abstract class BaseMediaSampleSink<
 			await packets.next(); // Skip the start packet as we already have it
 
 			while (currentPacket && !ended) {
-				if (sampleQueue.length + decoder.getDecodeQueueSize() > MAX_QUEUE_SIZE) {
+				const maxQueueSize = computeMaxQueueSize(sampleQueue.length);
+				if (sampleQueue.length + decoder.getDecodeQueueSize() > maxQueueSize) {
 					({ promise: queueDequeue, resolve: onQueueDequeue } = promiseWithResolvers());
 					await queueDequeue;
 					continue;
@@ -438,7 +438,6 @@ export abstract class BaseMediaSampleSink<
 		const timestampIterator = toAsyncIterator(timestamps);
 		const timestampsOfInterest: number[] = [];
 
-		const MAX_QUEUE_SIZE = 8;
 		const sampleQueue: (MediaSample | null)[] = [];
 		let { promise: queueNotEmpty, resolve: onQueueNotEmpty } = promiseWithResolvers();
 		let { promise: queueDequeue, resolve: onQueueDequeue } = promiseWithResolvers();
@@ -499,7 +498,8 @@ export abstract class BaseMediaSampleSink<
 			for await (const timestamp of timestampIterator) {
 				validateTimestamp(timestamp);
 
-				while (sampleQueue.length + decoder.getDecodeQueueSize() > MAX_QUEUE_SIZE && !terminated) {
+				const maxQueueSize = computeMaxQueueSize(sampleQueue.length);
+				while (sampleQueue.length + decoder.getDecodeQueueSize() > maxQueueSize && !terminated) {
 					({ promise: queueDequeue, resolve: onQueueDequeue } = promiseWithResolvers());
 					await queueDequeue;
 				}
@@ -618,6 +618,13 @@ export abstract class BaseMediaSampleSink<
 	}
 }
 
+const computeMaxQueueSize = (decodedSampleQueueSize: number) => {
+	// If we have decoded samples lying around, limit the total queue size to a small value (decoded samples can use up
+	// a lot of memory). If not, we're fine with a much bigger queue of encoded packets waiting to be decoded. In fact,
+	// some decoders only start flushing out decoded chunks when the packet queue is large enough.
+	return decodedSampleQueueSize === 0 ? 40 : 8;
+};
+
 class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
 	decoder: VideoDecoder | null = null;