Add frame rate-based sampling for MediaStreamVideoTrackSource, reducing the probability of files with wildly irregular FPS; add unthrottled timer primitives

dev/live.html

@@ -17,9 +17,9 @@
 
     const button = document.querySelector('button');
     button.addEventListener('click', async () => {
-        const stream = await navigator.mediaDevices.getUserMedia({ video: true, audio: true });
+        const stream = await navigator.mediaDevices.getDisplayMedia({ video: true, audio: false });
         const videoTrack = stream.getVideoTracks()[0];
-        const audioTrack = stream.getAudioTracks()[0];
+        const audioTrack = null;//stream.getAudioTracks()[0];
 
         const output = new Mediabunny.Output({
             target: new Mediabunny.BufferTarget(),
@@ -30,7 +30,8 @@
         if (videoTrack) {
             videoSource = new Mediabunny.MediaStreamVideoTrackSource(videoTrack, {
                 codec: 'avc',
-                bitrate: Mediabunny.QUALITY_MEDIUM
+                bitrate: Mediabunny.QUALITY_MEDIUM,
+				sizeChangeBehavior: 'passThrough',
             });
 
             videoSource.errorPromise.catch((d) => console.log("Hello?????", d));
@@ -45,12 +46,13 @@
 
             audioSource.errorPromise.catch((d) => console.log("Hello!!???", d));
 
-            output.addAudioTrack(audioSource);
+            //output.addAudioTrack(audioSource);
         }
 
         await output.start();
 
 		setTimeout(() => {
+			return;
 			videoSource?.pause();
 			audioSource?.pause();
 

src/index.ts

@@ -64,6 +64,7 @@ export {
 	EncodedAudioPacketSource,
 	EncodedVideoPacketSource,
 	MediaStreamAudioTrackSource,
+	MediaStreamVideoTrackSourceOptions,
 	MediaStreamVideoTrackSource,
 	TextSubtitleSource,
 	VideoSampleSource,

src/media-source.ts

@@ -24,12 +24,15 @@ import {
 	assertNever,
 	CallSerializer,
 	clamp,
+	clearIntervalUnthrottled,
 	isFirefox,
 	last,
 	promiseWithResolvers,
 	setInt24,
+	setIntervalUnthrottled,
 	setUint24,
 	toUint8Array,
+	UnthrottledTimerHandle,
 } from './misc';
 import { Muxer } from './muxer';
 import { SubtitleParser } from './subtitles';
@@ -1026,6 +1029,21 @@ export class CanvasSource extends VideoSource {
 	}
 }
 
+/**
+ * Options for MediaStreamVideoTrackSource.
+ * @group Media sources
+ * @public
+ */
+export type MediaStreamVideoTrackSourceOptions = {
+	/**
+	 * The frame rate at which the underlying video track is sampled. Defaults to the frame rate specified in the
+	 * track's [`MediaTrackSettings`](https://developer.mozilla.org/en-US/docs/Web/API/MediaTrackSettings). Set to
+	 * `null` to only add a frame whenever the underlying track pushes one - this minimizes frame count but can
+	 * lead to wildly irregular FPS.
+	 */
+	frameRate?: number | null;
+};
+
 /**
  * Video source that encodes the frames of a
  * [`MediaStreamVideoTrack`](https://developer.mozilla.org/en-US/docs/Web/API/MediaStreamTrack) and pipes them into the
@@ -1036,6 +1054,8 @@ export class CanvasSource extends VideoSource {
  * @public
  */
 export class MediaStreamVideoTrackSource extends VideoSource {
+	/** @internal */
+	private _options: MediaStreamVideoTrackSourceOptions;
 	/** @internal */
 	private _encoder: VideoEncoderWrapper;
 	/** @internal */
@@ -1053,9 +1073,9 @@ export class MediaStreamVideoTrackSource extends VideoSource {
 	/** @internal */
 	private _paused = false;
 	/** @internal */
-	private _lastSampleTimestamp: number | null = null;
+	private _lastVideoFrame: VideoFrame | null = null;
 	/** @internal */
-	private _pauseOffset = 0;
+	private _timerHandle: UnthrottledTimerHandle | null = null;
 
 	/** A promise that rejects upon any error within this source. This promise never resolves. */
 	get errorPromise() {
@@ -1073,11 +1093,21 @@ export class MediaStreamVideoTrackSource extends VideoSource {
 	 * [`MediaStreamVideoTrack`](https://developer.mozilla.org/en-US/docs/Web/API/MediaStreamTrack), which will pull
 	 * video samples from the stream in real time and encode them according to {@link VideoEncodingConfig}.
 	 */
-	constructor(track: MediaStreamVideoTrack, encodingConfig: VideoEncodingConfig) {
+	constructor(
+		track: MediaStreamVideoTrack,
+		encodingConfig: VideoEncodingConfig,
+		options: MediaStreamVideoTrackSourceOptions = {},
+	) {
 		if (!(track instanceof MediaStreamTrack) || track.kind !== 'video') {
 			throw new TypeError('track must be a video MediaStreamTrack.');
 		}
 		validateVideoEncodingConfig(encodingConfig);
+		if (typeof options !== 'object' || !options) {
+			throw new TypeError('options must be an object.');
+		}
+		if (options.frameRate != null && (typeof options.frameRate !== 'number' || options.frameRate <= 0)) {
+			throw new TypeError('options.frameRate, when provided, must be either a positive number or null.');
+		}
 
 		encodingConfig = {
 			...encodingConfig,
@@ -1085,6 +1115,8 @@ export class MediaStreamVideoTrackSource extends VideoSource {
 		};
 
 		super(encodingConfig.codec);
+
+		this._options = options;
 		this._encoder = new VideoEncoderWrapper(this, encodingConfig);
 		this._track = track;
 	}
@@ -1098,32 +1130,89 @@ export class MediaStreamVideoTrackSource extends VideoSource {
 			);
 		}
 
+		const frameRate = this._options.frameRate !== undefined
+			? this._options.frameRate
+			: (this._track.getSettings().frameRate ?? null);
+
 		this._abortController = new AbortController();
 
 		let firstVideoFrameTimestamp: number | null = null;
+		let lastFrameTime: number | null = null;
+		let frameCount = 0;
 		let errored = false;
+		let lastSampleTimestamp: number | null = null;
+		let pauseOffset = 0;
+
+		const tick = () => {
+			assert(frameRate !== null);
+
+			if (!this._lastVideoFrame) {
+				return;
+			}
+
+			assert(lastFrameTime !== null);
+			assert(firstVideoFrameTimestamp !== null);
+
+			const now = performance.now();
+
+			// Add as many frames as warranted by the elapsed time.
+			// > instead of >= intentionally because tick() is called before the _lastVideoFrame is changed
+			while (now - lastFrameTime > 1000 / frameRate) {
+				lastFrameTime += 1000 / frameRate;
+				const timestamp = firstVideoFrameTimestamp + frameCount / frameRate;
+				const clone = new VideoFrame(this._lastVideoFrame, {
+					timestamp: 1e6 * timestamp,
+					duration: 1e6 / frameRate,
+				});
+
+				addVideoFrame(clone, now);
+			}
+		};
+
+		if (frameRate !== null) {
+			this._timerHandle = setIntervalUnthrottled(tick, 4); // Run it at 250 Hz
+		}
 
 		const onVideoFrame = (videoFrame: VideoFrame) => {
+			if (frameRate === null) {
+				addVideoFrame(videoFrame);
+			} else {
+				const now = performance.now();
+
+				if (!this._lastVideoFrame) {
+					addVideoFrame(videoFrame.clone(), now);
+					lastFrameTime = now;
+					this._lastVideoFrame = videoFrame;
+				} else {
+					tick();
+					this._lastVideoFrame?.close();
+					this._lastVideoFrame = videoFrame;
+				}
+			}
+		};
+
+		const addVideoFrame = (videoFrame: VideoFrame, now = performance.now()) => {
 			if (errored) {
 				videoFrame.close();
 				return;
 			}
 
+			frameCount++;
 			const currentTimestamp = videoFrame.timestamp / 1e6;
 
 			if (this._paused) {
 				const frameSeen = firstVideoFrameTimestamp !== null;
 				if (frameSeen) {
-					if (this._lastSampleTimestamp !== null) {
+					if (lastSampleTimestamp !== null) {
 						// In addition to dropping this frame, let's also keep track of the time we have lost due to the
 						// pause. Doing it like this instead of simply keeping track of the paused time is better since
 						// it retains the frame rate of the underlying source.
-						const timeDelta = currentTimestamp - this._lastSampleTimestamp;
+						const timeDelta = currentTimestamp - lastSampleTimestamp;
 						// We modify this field instead of _timestampOffset since we still might have data in flight
 						// in the encoder, with which we don't want to mess.
-						this._pauseOffset -= timeDelta;
+						pauseOffset -= timeDelta;
 					}
-					this._lastSampleTimestamp = currentTimestamp;
+					lastSampleTimestamp = currentTimestamp;
 				}
 
 				videoFrame.close();
@@ -1135,24 +1224,24 @@ export class MediaStreamVideoTrackSource extends VideoSource {
 
 				const muxer = this._connectedTrack!.output._muxer;
 				if (muxer.firstMediaStreamTimestamp === null) {
-					muxer.firstMediaStreamTimestamp = performance.now() / 1000;
+					muxer.firstMediaStreamTimestamp = now / 1000;
 					this._timestampOffset = -firstVideoFrameTimestamp;
 				} else {
-					this._timestampOffset = (performance.now() / 1000 - muxer.firstMediaStreamTimestamp)
+					this._timestampOffset = (now / 1000 - muxer.firstMediaStreamTimestamp)
 						- firstVideoFrameTimestamp;
 				}
 			}
 
-			this._lastSampleTimestamp = currentTimestamp;
+			lastSampleTimestamp = currentTimestamp;
 
-			if (this._encoder.getQueueSize() >= 4) {
+			if (this._encoder.getQueueSize() >= 8) {
 				// Drop frames if the encoder is overloaded
 				videoFrame.close();
 				return;
 			}
 
 			const sample = new VideoSample(videoFrame, {
-				timestamp: currentTimestamp + this._pauseOffset,
+				timestamp: currentTimestamp + pauseOffset,
 			});
 
 			void this._encoder.add(sample, true)
@@ -1235,6 +1324,11 @@ export class MediaStreamVideoTrackSource extends VideoSource {
 			this._abortController = null;
 		}
 
+		if (this._timerHandle) {
+			clearIntervalUnthrottled(this._timerHandle);
+		}
+		this._lastVideoFrame?.close();
+
 		if (this._workerTrackId !== null) {
 			assert(this._workerListener);
 
@@ -1899,10 +1993,6 @@ export class MediaStreamAudioTrackSource extends AudioSource {
 	private _errorPromiseAccessed = false;
 	/** @internal */
 	private _paused = false;
-	/** @internal */
-	private _lastSampleTimestamp: number | null = null;
-	/** @internal */
-	private _pauseOffset = 0;
 
 	/** A promise that rejects upon any error within this source. This promise never resolves. */
 	get errorPromise() {
@@ -1934,7 +2024,7 @@ export class MediaStreamAudioTrackSource extends AudioSource {
 	override async _start() {
 		if (!this._errorPromiseAccessed) {
 			console.warn(
-				'Make sure not to ignore the `errorPromise` field on MediaStreamVideoTrackSource, so that any internal'
+				'Make sure not to ignore the `errorPromise` field on MediaStreamAudioTrackSource, so that any internal'
 				+ ' errors get bubbled up properly.',
 			);
 		}
@@ -1943,6 +2033,8 @@ export class MediaStreamAudioTrackSource extends AudioSource {
 
 		let firstAudioDataTimestamp: number | null = null;
 		let errored = false;
+		let lastSampleTimestamp: number | null = null;
+		let pauseOffset = 0;
 
 		const onAudioSample = (audioSample: AudioSample) => {
 			if (errored) {
@@ -1955,16 +2047,16 @@ export class MediaStreamAudioTrackSource extends AudioSource {
 			if (this._paused) {
 				const dataSeen = firstAudioDataTimestamp !== null;
 				if (dataSeen) {
-					if (this._lastSampleTimestamp !== null) {
+					if (lastSampleTimestamp !== null) {
 						// In addition to dropping this sample, let's also keep track of the time we have lost due to
 						// the pause. Doing it like this instead of simply keeping track of the paused time is better
 						// since it retains the sample rate of the underlying source.
-						const timeDelta = currentTimestamp - this._lastSampleTimestamp;
+						const timeDelta = currentTimestamp - lastSampleTimestamp;
 						// We modify this field instead of _timestampOffset since we still might have data in flight
 						// in the encoder, with which we don't want to mess.
-						this._pauseOffset -= timeDelta;
+						pauseOffset -= timeDelta;
 					}
-					this._lastSampleTimestamp = currentTimestamp;
+					lastSampleTimestamp = currentTimestamp;
 				}
 
 				audioSample.close();
@@ -1984,15 +2076,15 @@ export class MediaStreamAudioTrackSource extends AudioSource {
 				}
 			}
 
-			this._lastSampleTimestamp = currentTimestamp;
+			lastSampleTimestamp = currentTimestamp;
 
-			if (this._encoder.getQueueSize() >= 4) {
+			if (this._encoder.getQueueSize() >= 8) {
 				// Drop data if the encoder is overloaded
 				audioSample.close();
 				return;
 			}
 
-			audioSample.setTimestamp(currentTimestamp + this._pauseOffset);
+			audioSample.setTimestamp(currentTimestamp + pauseOffset);
 
 			void this._encoder.add(audioSample, true)
 				.catch((error) => {

src/misc.ts

@@ -839,3 +839,173 @@ export const validateRectangle = (rect: Rectangle, propertyPath: string) => {
 		throw new TypeError(`${propertyPath}.height must be a non-negative integer.`);
 	}
 };
+
+export type UnthrottledTimerHandle = {
+	id: ReturnType<typeof setTimeout> | number;
+};
+
+type UnthrottledTimerMessage =
+	| { type: 'set-timeout'; timerId: number; delay: number }
+	| { type: 'set-interval'; timerId: number; delay: number }
+	| { type: 'clear-timeout'; timerId: number }
+	| { type: 'clear-interval'; timerId: number };
+
+type UnthrottledTimerEvent = { type: 'fire'; timerId: number };
+
+let unthrottledTimerWorker: Worker | undefined;
+let nextUnthrottledTimerId = 1;
+const unthrottledTimeoutCallbacks = new Map<number, () => void>();
+const unthrottledIntervalCallbacks = new Map<number, () => void>();
+
+const shouldUseNativeTimers = () => {
+	return typeof window === 'undefined';
+};
+
+const unthrottledTimerWorkerMain = () => {
+	const timeoutHandles = new Map<number, ReturnType<typeof setTimeout>>();
+	const intervalHandles = new Map<number, ReturnType<typeof setInterval>>();
+
+	self.onmessage = (event: MessageEvent<UnthrottledTimerMessage>) => {
+		const message = event.data;
+
+		switch (message.type) {
+			case 'set-timeout': {
+				const handle = setTimeout(() => {
+					timeoutHandles.delete(message.timerId);
+					self.postMessage({ type: 'fire', timerId: message.timerId });
+				}, message.delay);
+
+				timeoutHandles.set(message.timerId, handle);
+			}; break;
+
+			case 'set-interval': {
+				const handle = setInterval(() => {
+					self.postMessage({ type: 'fire', timerId: message.timerId });
+				}, message.delay);
+
+				intervalHandles.set(message.timerId, handle);
+			}; break;
+
+			case 'clear-timeout': {
+				const handle = timeoutHandles.get(message.timerId);
+				if (handle !== undefined) {
+					clearTimeout(handle);
+					timeoutHandles.delete(message.timerId);
+				}
+			}; break;
+
+			case 'clear-interval': {
+				const handle = intervalHandles.get(message.timerId);
+				if (handle !== undefined) {
+					clearInterval(handle);
+					intervalHandles.delete(message.timerId);
+				}
+			}; break;
+		}
+	};
+};
+
+const getUnthrottledTimerWorker = () => {
+	if (unthrottledTimerWorker) {
+		return unthrottledTimerWorker;
+	}
+
+	const workerSource = `(${unthrottledTimerWorkerMain.toString()})();`;
+	const workerURL = URL.createObjectURL(new Blob([workerSource], { type: 'text/javascript' }));
+	unthrottledTimerWorker = new Worker(workerURL);
+	URL.revokeObjectURL(workerURL);
+
+	unthrottledTimerWorker.onmessage = (event: MessageEvent<UnthrottledTimerEvent>) => {
+		const message = event.data;
+
+		const timeoutCallback = unthrottledTimeoutCallbacks.get(message.timerId);
+		if (timeoutCallback) {
+			unthrottledTimeoutCallbacks.delete(message.timerId);
+			timeoutCallback();
+			return;
+		}
+
+		const intervalCallback = unthrottledIntervalCallbacks.get(message.timerId);
+		if (intervalCallback) {
+			intervalCallback();
+		}
+	};
+
+	return unthrottledTimerWorker;
+};
+
+export const setTimeoutUnthrottled = (
+	// eslint-disable-next-line @typescript-eslint/no-unsafe-function-type
+	callback: Function,
+	delay: number,
+): UnthrottledTimerHandle => {
+	if (shouldUseNativeTimers()) {
+		return { id: setTimeout(callback, delay) };
+	}
+
+	const timerId = nextUnthrottledTimerId++;
+	unthrottledTimeoutCallbacks.set(timerId, () => {
+		(callback as () => void)();
+	});
+
+	getUnthrottledTimerWorker().postMessage({
+		type: 'set-timeout',
+		timerId,
+		delay,
+	} satisfies UnthrottledTimerMessage);
+
+	return { id: timerId };
+};
+
+export const clearTimeoutUnthrottled = (timer: UnthrottledTimerHandle) => {
+	if (shouldUseNativeTimers()) {
+		clearTimeout(timer.id);
+		return;
+	}
+
+	assert(typeof timer.id === 'number');
+	unthrottledTimeoutCallbacks.delete(timer.id);
+
+	getUnthrottledTimerWorker().postMessage({
+		type: 'clear-timeout',
+		timerId: timer.id,
+	} satisfies UnthrottledTimerMessage);
+};
+
+export const setIntervalUnthrottled = (
+	// eslint-disable-next-line @typescript-eslint/no-unsafe-function-type
+	callback: Function,
+	delay: number,
+): UnthrottledTimerHandle => {
+	if (shouldUseNativeTimers()) {
+		return { id: setInterval(callback, delay) };
+	}
+
+	const timerId = nextUnthrottledTimerId++;
+	unthrottledIntervalCallbacks.set(timerId, () => {
+		(callback as () => void)();
+	});
+
+	getUnthrottledTimerWorker().postMessage({
+		type: 'set-interval',
+		timerId,
+		delay,
+	} satisfies UnthrottledTimerMessage);
+
+	return { id: timerId };
+};
+
+export const clearIntervalUnthrottled = (timer: UnthrottledTimerHandle) => {
+	if (shouldUseNativeTimers()) {
+		clearInterval(timer.id);
+		return;
+	}
+
+	assert(typeof timer.id === 'number');
+	unthrottledIntervalCallbacks.delete(timer.id);
+
+	getUnthrottledTimerWorker().postMessage({
+		type: 'clear-interval',
+		timerId: timer.id,
+	} satisfies UnthrottledTimerMessage);
+};