Add pause and resume methods to MediaStreamTrack sources (closes #241)

dev/live.html

@@ -25,29 +25,41 @@
             target: new Mediabunny.BufferTarget(),
             format: new Mediabunny.Mp4OutputFormat(),
         });
+		let videoSource = null;
+		let audioSource = null;
         if (videoTrack) {
-            const source = new Mediabunny.MediaStreamVideoTrackSource(videoTrack, {
+            videoSource = new Mediabunny.MediaStreamVideoTrackSource(videoTrack, {
                 codec: 'avc',
                 bitrate: Mediabunny.QUALITY_MEDIUM
             });
 
-            source.errorPromise.catch((d) => console.log("Hello?????", d));
+            videoSource.errorPromise.catch((d) => console.log("Hello?????", d));
 
-            output.addVideoTrack(source);
+            output.addVideoTrack(videoSource);
         }
         if (audioTrack) {
-            const source = new Mediabunny.MediaStreamAudioTrackSource(audioTrack, {
+            audioSource = new Mediabunny.MediaStreamAudioTrackSource(audioTrack, {
                 codec: 'mp3',
                 bitrate: Mediabunny.QUALITY_MEDIUM
             });
 
-            source.errorPromise.catch((d) => console.log("Hello!!???", d));
+            audioSource.errorPromise.catch((d) => console.log("Hello!!???", d));
 
-            output.addAudioTrack(source);
+            output.addAudioTrack(audioSource);
         }
 
         await output.start();
 
+		setTimeout(() => {
+			videoSource?.pause();
+			audioSource?.pause();
+
+			setTimeout(() => {
+				videoSource?.resume();
+				audioSource?.resume();
+			}, 1000);
+		}, 1000);
+
         await new Promise(resolve => setTimeout(resolve, 5000));
 
         await output.finalize();

docs/guide/media-sources.md

@@ -187,6 +187,16 @@ videoTrackSource.errorPromise.catch((error) => ...);
 
 This source requires no additional method calls; data will automatically be captured and piped to the output file as soon as `start()` is called on the `Output`. Make sure to `stop()` on `videoTrack` after finalizing the `Output` if you don't need the user's media anymore.
 
+If you want to temporarily stop capturing video frames from this source, you can use the `pause()` and `resume()` methods:
+```ts
+videoTrackSource.pause();
+
+// Later:
+videoTrackSource.resume();
+```
+
+While paused, video frames emitted by the stream will be ignored. When resumed, video frames are let through again, offset in timestamp such that the result plays back continuously with no gap in playback. Note that pausing does *not* stop the underlying media stream.
+
 ::: info
 If this source is the only MediaStreamTrack source in the `Output`, then the first video sample added by it starts at timestamp 0. If there are multiple, then the earliest media sample across all tracks starts at timestamp 0, and all tracks will be perfectly synchronized with each other.
 :::
@@ -344,6 +354,16 @@ audioTrackSource.errorPromise.catch((error) => ...);
 
 This source requires no additional method calls; data will automatically be captured and piped to the output file as soon as `start()` is called on the `Output`. Make sure to `stop()` on `audioTrack` after finalizing the `Output` if you don't need the user's media anymore.
 
+If you want to temporarily stop capturing audio data from this source, you can use the `pause()` and `resume()` methods:
+```ts
+audioTrackSource.pause();
+
+// Later:
+audioTrackSource.resume();
+```
+
+While paused, audio data emitted by the stream will be ignored. When resumed, audio data are let through again, offset in timestamp such that the result plays back continuously with no gap in playback. Note that pausing does *not* stop the underlying media stream.
+
 ::: info
 If this source is the only MediaStreamTrack source in the `Output`, then the first audio sample added by it starts at timestamp 0. If there are multiple, then the earliest media sample across all tracks starts at timestamp 0, and all tracks will be perfectly synchronized with each other.
 :::

src/media-source.ts

@@ -1055,6 +1055,12 @@ export class MediaStreamVideoTrackSource extends VideoSource {
 	private _promiseWithResolvers = promiseWithResolvers();
 	/** @internal */
 	private _errorPromiseAccessed = false;
+	/** @internal */
+	private _paused = false;
+	/** @internal */
+	private _lastSampleTimestamp: number | null = null;
+	/** @internal */
+	private _pauseOffset = 0;
 
 	/** A promise that rejects upon any error within this source. This promise never resolves. */
 	get errorPromise() {
@@ -1062,6 +1068,11 @@ export class MediaStreamVideoTrackSource extends VideoSource {
 		return this._promiseWithResolvers.promise;
 	}
 
+	/** Whether this source is currently paused as a result of calling `.pause()`. */
+	get paused() {
+		return this._paused;
+	}
+
 	/**
 	 * Creates a new {@link MediaStreamVideoTrackSource} from a
 	 * [`MediaStreamVideoTrack`](https://developer.mozilla.org/en-US/docs/Web/API/MediaStreamTrack), which will pull
@@ -1103,8 +1114,29 @@ export class MediaStreamVideoTrackSource extends VideoSource {
 				return;
 			}
 
+			const currentTimestamp = videoFrame.timestamp / 1e6;
+
+			if (this._paused) {
+				const frameSeen = firstVideoFrameTimestamp !== null;
+				if (frameSeen) {
+					if (this._lastSampleTimestamp !== null) {
+						// In addition to dropping this frame, let's also keep track of the time we have lost due to the
+						// pause. Doing it like this instead of simply keeping track of the paused time is better since
+						// it retains the frame rate of the underlying source.
+						const timeDelta = currentTimestamp - this._lastSampleTimestamp;
+						// We modify this field instead of _timestampOffset since we still might have data in flight
+						// in the encoder, with which we don't want to mess.
+						this._pauseOffset -= timeDelta;
+					}
+					this._lastSampleTimestamp = currentTimestamp;
+				}
+
+				videoFrame.close();
+				return;
+			}
+
 			if (firstVideoFrameTimestamp === null) {
-				firstVideoFrameTimestamp = videoFrame.timestamp / 1e6;
+				firstVideoFrameTimestamp = currentTimestamp;
 
 				const muxer = this._connectedTrack!.output._muxer;
 				if (muxer.firstMediaStreamTimestamp === null) {
@@ -1116,13 +1148,19 @@ export class MediaStreamVideoTrackSource extends VideoSource {
 				}
 			}
 
+			this._lastSampleTimestamp = currentTimestamp;
+
 			if (this._encoder.getQueueSize() >= 4) {
 				// Drop frames if the encoder is overloaded
 				videoFrame.close();
 				return;
 			}
 
-			void this._encoder.add(new VideoSample(videoFrame), true)
+			const sample = new VideoSample(videoFrame, {
+				timestamp: currentTimestamp + this._pauseOffset,
+			});
+
+			void this._encoder.add(sample, true)
 				.catch((error) => {
 					errored = true;
 
@@ -1182,6 +1220,19 @@ export class MediaStreamVideoTrackSource extends VideoSource {
 		}
 	}
 
+	/**
+	 * Pauses the capture of video frames - any video frames emitted by the underlying media stream will be ignored
+	 * while paused. This does *not* close the underlying `MediaStreamVideoTrack`, it just ignores its output.
+	 */
+	pause() {
+		this._paused = true;
+	}
+
+	/** Resumes the capture of video frames after being paused. */
+	resume() {
+		this._paused = false;
+	}
+
 	/** @internal */
 	override async _flushAndClose(forceClose: boolean) {
 		if (this._abortController) {
@@ -1857,6 +1908,12 @@ export class MediaStreamAudioTrackSource extends AudioSource {
 	private _promiseWithResolvers = promiseWithResolvers();
 	/** @internal */
 	private _errorPromiseAccessed = false;
+	/** @internal */
+	private _paused = false;
+	/** @internal */
+	private _lastSampleTimestamp: number | null = null;
+	/** @internal */
+	private _pauseOffset = 0;
 
 	/** A promise that rejects upon any error within this source. This promise never resolves. */
 	get errorPromise() {
@@ -1864,6 +1921,11 @@ export class MediaStreamAudioTrackSource extends AudioSource {
 		return this._promiseWithResolvers.promise;
 	}
 
+	/** Whether this source is currently paused as a result of calling `.pause()`. */
+	get paused() {
+		return this._paused;
+	}
+
 	/**
 	 * Creates a new {@link MediaStreamAudioTrackSource} from a `MediaStreamAudioTrack`, which will pull audio samples
 	 * from the stream in real time and encode them according to {@link AudioEncodingConfig}.
@@ -1899,6 +1961,27 @@ export class MediaStreamAudioTrackSource extends AudioSource {
 				return;
 			}
 
+			const currentTimestamp = audioSample.timestamp;
+
+			if (this._paused) {
+				const dataSeen = firstAudioDataTimestamp !== null;
+				if (dataSeen) {
+					if (this._lastSampleTimestamp !== null) {
+						// In addition to dropping this sample, let's also keep track of the time we have lost due to
+						// the pause. Doing it like this instead of simply keeping track of the paused time is better
+						// since it retains the sample rate of the underlying source.
+						const timeDelta = currentTimestamp - this._lastSampleTimestamp;
+						// We modify this field instead of _timestampOffset since we still might have data in flight
+						// in the encoder, with which we don't want to mess.
+						this._pauseOffset -= timeDelta;
+					}
+					this._lastSampleTimestamp = currentTimestamp;
+				}
+
+				audioSample.close();
+				return;
+			}
+
 			if (firstAudioDataTimestamp === null) {
 				firstAudioDataTimestamp = audioSample.timestamp;
 
@@ -1912,12 +1995,16 @@ export class MediaStreamAudioTrackSource extends AudioSource {
 				}
 			}
 
+			this._lastSampleTimestamp = currentTimestamp;
+
 			if (this._encoder.getQueueSize() >= 4) {
 				// Drop data if the encoder is overloaded
 				audioSample.close();
 				return;
 			}
 
+			audioSample.setTimestamp(currentTimestamp + this._pauseOffset);
+
 			void this._encoder.add(audioSample, true)
 				.catch((error) => {
 					errored = true;
@@ -1973,6 +2060,19 @@ export class MediaStreamAudioTrackSource extends AudioSource {
 		}
 	}
 
+	/**
+	 * Pauses the capture of audio data - any audio data emitted by the underlying media stream will be ignored
+	 * while paused. This does *not* close the underlying `MediaStreamAudioTrack`, it just ignores its output.
+	 */
+	pause() {
+		this._paused = true;
+	}
+
+	/** Resumes the capture of audio data after being paused. */
+	resume() {
+		this._paused = false;
+	}
+
 	/** @internal */
 	override async _flushAndClose(forceClose: boolean) {
 		if (this._abortController) {