diff --git a/dev/live.html b/dev/live.html
index 8eef45b..bdd2fc2 100644
--- a/dev/live.html
+++ b/dev/live.html
@@ -17,9 +17,9 @@
const button = document.querySelector('button');
button.addEventListener('click', async () => {
- const stream = await navigator.mediaDevices.getUserMedia({ video: true, audio: true });
+ const stream = await navigator.mediaDevices.getDisplayMedia({ video: true, audio: false });
const videoTrack = stream.getVideoTracks()[0];
- const audioTrack = stream.getAudioTracks()[0];
+ const audioTrack = null;//stream.getAudioTracks()[0];
const output = new Mediabunny.Output({
target: new Mediabunny.BufferTarget(),
@@ -30,7 +30,8 @@
if (videoTrack) {
videoSource = new Mediabunny.MediaStreamVideoTrackSource(videoTrack, {
codec: 'avc',
- bitrate: Mediabunny.QUALITY_MEDIUM
+ bitrate: Mediabunny.QUALITY_MEDIUM,
+ sizeChangeBehavior: 'passThrough',
});
videoSource.errorPromise.catch((d) => console.log("Hello?????", d));
@@ -45,12 +46,13 @@
audioSource.errorPromise.catch((d) => console.log("Hello!!???", d));
- output.addAudioTrack(audioSource);
+ //output.addAudioTrack(audioSource);
}
await output.start();
setTimeout(() => {
+ return;
videoSource?.pause();
audioSource?.pause();
diff --git a/src/index.ts b/src/index.ts
index b83f7bb..0126657 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -64,6 +64,7 @@ export {
EncodedAudioPacketSource,
EncodedVideoPacketSource,
MediaStreamAudioTrackSource,
+ MediaStreamVideoTrackSourceOptions,
MediaStreamVideoTrackSource,
TextSubtitleSource,
VideoSampleSource,
diff --git a/src/media-source.ts b/src/media-source.ts
index 3580a14..c3b4604 100644
--- a/src/media-source.ts
+++ b/src/media-source.ts
@@ -24,12 +24,15 @@ import {
assertNever,
CallSerializer,
clamp,
+ clearIntervalUnthrottled,
isFirefox,
last,
promiseWithResolvers,
setInt24,
+ setIntervalUnthrottled,
setUint24,
toUint8Array,
+ UnthrottledTimerHandle,
} from './misc';
import { Muxer } from './muxer';
import { SubtitleParser } from './subtitles';
@@ -1026,6 +1029,21 @@ export class CanvasSource extends VideoSource {
}
}
+/**
+ * Options for MediaStreamVideoTrackSource.
+ * @group Media sources
+ * @public
+ */
+export type MediaStreamVideoTrackSourceOptions = {
+ /**
+ * The frame rate at which the underlying video track is sampled. Defaults to the frame rate specified in the
+ * track's [`MediaTrackSettings`](https://developer.mozilla.org/en-US/docs/Web/API/MediaTrackSettings). Set to
+ * `null` to only add a frame whenever the underlying track pushes one - this minimizes frame count but can
+ * lead to wildly irregular FPS.
+ */
+ frameRate?: number | null;
+};
+
/**
* Video source that encodes the frames of a
* [`MediaStreamVideoTrack`](https://developer.mozilla.org/en-US/docs/Web/API/MediaStreamTrack) and pipes them into the
@@ -1036,6 +1054,8 @@ export class CanvasSource extends VideoSource {
* @public
*/
export class MediaStreamVideoTrackSource extends VideoSource {
+ /** @internal */
+ private _options: MediaStreamVideoTrackSourceOptions;
/** @internal */
private _encoder: VideoEncoderWrapper;
/** @internal */
@@ -1053,9 +1073,9 @@ export class MediaStreamVideoTrackSource extends VideoSource {
/** @internal */
private _paused = false;
/** @internal */
- private _lastSampleTimestamp: number | null = null;
+ private _lastVideoFrame: VideoFrame | null = null;
/** @internal */
- private _pauseOffset = 0;
+ private _timerHandle: UnthrottledTimerHandle | null = null;
/** A promise that rejects upon any error within this source. This promise never resolves. */
get errorPromise() {
@@ -1073,11 +1093,21 @@ export class MediaStreamVideoTrackSource extends VideoSource {
* [`MediaStreamVideoTrack`](https://developer.mozilla.org/en-US/docs/Web/API/MediaStreamTrack), which will pull
* video samples from the stream in real time and encode them according to {@link VideoEncodingConfig}.
*/
- constructor(track: MediaStreamVideoTrack, encodingConfig: VideoEncodingConfig) {
+ constructor(
+ track: MediaStreamVideoTrack,
+ encodingConfig: VideoEncodingConfig,
+ options: MediaStreamVideoTrackSourceOptions = {},
+ ) {
if (!(track instanceof MediaStreamTrack) || track.kind !== 'video') {
throw new TypeError('track must be a video MediaStreamTrack.');
}
validateVideoEncodingConfig(encodingConfig);
+ if (typeof options !== 'object' || !options) {
+ throw new TypeError('options must be an object.');
+ }
+ if (options.frameRate != null && (typeof options.frameRate !== 'number' || options.frameRate <= 0)) {
+ throw new TypeError('options.frameRate, when provided, must be either a positive number or null.');
+ }
encodingConfig = {
...encodingConfig,
@@ -1085,6 +1115,8 @@ export class MediaStreamVideoTrackSource extends VideoSource {
};
super(encodingConfig.codec);
+
+ this._options = options;
this._encoder = new VideoEncoderWrapper(this, encodingConfig);
this._track = track;
}
@@ -1098,32 +1130,89 @@ export class MediaStreamVideoTrackSource extends VideoSource {
);
}
+ const frameRate = this._options.frameRate !== undefined
+ ? this._options.frameRate
+ : (this._track.getSettings().frameRate ?? null);
+
this._abortController = new AbortController();
let firstVideoFrameTimestamp: number | null = null;
+ let lastFrameTime: number | null = null;
+ let frameCount = 0;
let errored = false;
+ let lastSampleTimestamp: number | null = null;
+ let pauseOffset = 0;
+
+ const tick = () => {
+ assert(frameRate !== null);
+
+ if (!this._lastVideoFrame) {
+ return;
+ }
+
+ assert(lastFrameTime !== null);
+ assert(firstVideoFrameTimestamp !== null);
+
+ const now = performance.now();
+
+ // Add as many frames as warranted by the elapsed time.
+ // > instead of >= intentionally because tick() is called before the _lastVideoFrame is changed
+ while (now - lastFrameTime > 1000 / frameRate) {
+ lastFrameTime += 1000 / frameRate;
+ const timestamp = firstVideoFrameTimestamp + frameCount / frameRate;
+ const clone = new VideoFrame(this._lastVideoFrame, {
+ timestamp: 1e6 * timestamp,
+ duration: 1e6 / frameRate,
+ });
+
+ addVideoFrame(clone, now);
+ }
+ };
+
+ if (frameRate !== null) {
+ this._timerHandle = setIntervalUnthrottled(tick, 4); // Run it at 250 Hz
+ }
const onVideoFrame = (videoFrame: VideoFrame) => {
+ if (frameRate === null) {
+ addVideoFrame(videoFrame);
+ } else {
+ const now = performance.now();
+
+ if (!this._lastVideoFrame) {
+ addVideoFrame(videoFrame.clone(), now);
+ lastFrameTime = now;
+ this._lastVideoFrame = videoFrame;
+ } else {
+ tick();
+ this._lastVideoFrame?.close();
+ this._lastVideoFrame = videoFrame;
+ }
+ }
+ };
+
+ const addVideoFrame = (videoFrame: VideoFrame, now = performance.now()) => {
if (errored) {
videoFrame.close();
return;
}
+ frameCount++;
const currentTimestamp = videoFrame.timestamp / 1e6;
if (this._paused) {
const frameSeen = firstVideoFrameTimestamp !== null;
if (frameSeen) {
- if (this._lastSampleTimestamp !== null) {
+ if (lastSampleTimestamp !== null) {
// In addition to dropping this frame, let's also keep track of the time we have lost due to the
// pause. Doing it like this instead of simply keeping track of the paused time is better since
// it retains the frame rate of the underlying source.
- const timeDelta = currentTimestamp - this._lastSampleTimestamp;
+ const timeDelta = currentTimestamp - lastSampleTimestamp;
// We modify this field instead of _timestampOffset since we still might have data in flight
// in the encoder, with which we don't want to mess.
- this._pauseOffset -= timeDelta;
+ pauseOffset -= timeDelta;
}
- this._lastSampleTimestamp = currentTimestamp;
+ lastSampleTimestamp = currentTimestamp;
}
videoFrame.close();
@@ -1135,24 +1224,24 @@ export class MediaStreamVideoTrackSource extends VideoSource {
const muxer = this._connectedTrack!.output._muxer;
if (muxer.firstMediaStreamTimestamp === null) {
- muxer.firstMediaStreamTimestamp = performance.now() / 1000;
+ muxer.firstMediaStreamTimestamp = now / 1000;
this._timestampOffset = -firstVideoFrameTimestamp;
} else {
- this._timestampOffset = (performance.now() / 1000 - muxer.firstMediaStreamTimestamp)
+ this._timestampOffset = (now / 1000 - muxer.firstMediaStreamTimestamp)
- firstVideoFrameTimestamp;
}
}
- this._lastSampleTimestamp = currentTimestamp;
+ lastSampleTimestamp = currentTimestamp;
- if (this._encoder.getQueueSize() >= 4) {
+ if (this._encoder.getQueueSize() >= 8) {
// Drop frames if the encoder is overloaded
videoFrame.close();
return;
}
const sample = new VideoSample(videoFrame, {
- timestamp: currentTimestamp + this._pauseOffset,
+ timestamp: currentTimestamp + pauseOffset,
});
void this._encoder.add(sample, true)
@@ -1235,6 +1324,11 @@ export class MediaStreamVideoTrackSource extends VideoSource {
this._abortController = null;
}
+ if (this._timerHandle) {
+ clearIntervalUnthrottled(this._timerHandle);
+ }
+ this._lastVideoFrame?.close();
+
if (this._workerTrackId !== null) {
assert(this._workerListener);
@@ -1899,10 +1993,6 @@ export class MediaStreamAudioTrackSource extends AudioSource {
private _errorPromiseAccessed = false;
/** @internal */
private _paused = false;
- /** @internal */
- private _lastSampleTimestamp: number | null = null;
- /** @internal */
- private _pauseOffset = 0;
/** A promise that rejects upon any error within this source. This promise never resolves. */
get errorPromise() {
@@ -1934,7 +2024,7 @@ export class MediaStreamAudioTrackSource extends AudioSource {
override async _start() {
if (!this._errorPromiseAccessed) {
console.warn(
- 'Make sure not to ignore the `errorPromise` field on MediaStreamVideoTrackSource, so that any internal'
+ 'Make sure not to ignore the `errorPromise` field on MediaStreamAudioTrackSource, so that any internal'
+ ' errors get bubbled up properly.',
);
}
@@ -1943,6 +2033,8 @@ export class MediaStreamAudioTrackSource extends AudioSource {
let firstAudioDataTimestamp: number | null = null;
let errored = false;
+ let lastSampleTimestamp: number | null = null;
+ let pauseOffset = 0;
const onAudioSample = (audioSample: AudioSample) => {
if (errored) {
@@ -1955,16 +2047,16 @@ export class MediaStreamAudioTrackSource extends AudioSource {
if (this._paused) {
const dataSeen = firstAudioDataTimestamp !== null;
if (dataSeen) {
- if (this._lastSampleTimestamp !== null) {
+ if (lastSampleTimestamp !== null) {
// In addition to dropping this sample, let's also keep track of the time we have lost due to
// the pause. Doing it like this instead of simply keeping track of the paused time is better
// since it retains the sample rate of the underlying source.
- const timeDelta = currentTimestamp - this._lastSampleTimestamp;
+ const timeDelta = currentTimestamp - lastSampleTimestamp;
// We modify this field instead of _timestampOffset since we still might have data in flight
// in the encoder, with which we don't want to mess.
- this._pauseOffset -= timeDelta;
+ pauseOffset -= timeDelta;
}
- this._lastSampleTimestamp = currentTimestamp;
+ lastSampleTimestamp = currentTimestamp;
}
audioSample.close();
@@ -1984,15 +2076,15 @@ export class MediaStreamAudioTrackSource extends AudioSource {
}
}
- this._lastSampleTimestamp = currentTimestamp;
+ lastSampleTimestamp = currentTimestamp;
- if (this._encoder.getQueueSize() >= 4) {
+ if (this._encoder.getQueueSize() >= 8) {
// Drop data if the encoder is overloaded
audioSample.close();
return;
}
- audioSample.setTimestamp(currentTimestamp + this._pauseOffset);
+ audioSample.setTimestamp(currentTimestamp + pauseOffset);
void this._encoder.add(audioSample, true)
.catch((error) => {
diff --git a/src/misc.ts b/src/misc.ts
index b963ae7..982949b 100644
--- a/src/misc.ts
+++ b/src/misc.ts
@@ -839,3 +839,173 @@ export const validateRectangle = (rect: Rectangle, propertyPath: string) => {
throw new TypeError(`${propertyPath}.height must be a non-negative integer.`);
}
};
+
+export type UnthrottledTimerHandle = {
+ id: ReturnType | number;
+};
+
+type UnthrottledTimerMessage =
+ | { type: 'set-timeout'; timerId: number; delay: number }
+ | { type: 'set-interval'; timerId: number; delay: number }
+ | { type: 'clear-timeout'; timerId: number }
+ | { type: 'clear-interval'; timerId: number };
+
+type UnthrottledTimerEvent = { type: 'fire'; timerId: number };
+
+let unthrottledTimerWorker: Worker | undefined;
+let nextUnthrottledTimerId = 1;
+const unthrottledTimeoutCallbacks = new Map void>();
+const unthrottledIntervalCallbacks = new Map void>();
+
+const shouldUseNativeTimers = () => {
+ return typeof window === 'undefined';
+};
+
+const unthrottledTimerWorkerMain = () => {
+ const timeoutHandles = new Map>();
+ const intervalHandles = new Map>();
+
+ self.onmessage = (event: MessageEvent) => {
+ const message = event.data;
+
+ switch (message.type) {
+ case 'set-timeout': {
+ const handle = setTimeout(() => {
+ timeoutHandles.delete(message.timerId);
+ self.postMessage({ type: 'fire', timerId: message.timerId });
+ }, message.delay);
+
+ timeoutHandles.set(message.timerId, handle);
+ }; break;
+
+ case 'set-interval': {
+ const handle = setInterval(() => {
+ self.postMessage({ type: 'fire', timerId: message.timerId });
+ }, message.delay);
+
+ intervalHandles.set(message.timerId, handle);
+ }; break;
+
+ case 'clear-timeout': {
+ const handle = timeoutHandles.get(message.timerId);
+ if (handle !== undefined) {
+ clearTimeout(handle);
+ timeoutHandles.delete(message.timerId);
+ }
+ }; break;
+
+ case 'clear-interval': {
+ const handle = intervalHandles.get(message.timerId);
+ if (handle !== undefined) {
+ clearInterval(handle);
+ intervalHandles.delete(message.timerId);
+ }
+ }; break;
+ }
+ };
+};
+
+const getUnthrottledTimerWorker = () => {
+ if (unthrottledTimerWorker) {
+ return unthrottledTimerWorker;
+ }
+
+ const workerSource = `(${unthrottledTimerWorkerMain.toString()})();`;
+ const workerURL = URL.createObjectURL(new Blob([workerSource], { type: 'text/javascript' }));
+ unthrottledTimerWorker = new Worker(workerURL);
+ URL.revokeObjectURL(workerURL);
+
+ unthrottledTimerWorker.onmessage = (event: MessageEvent) => {
+ const message = event.data;
+
+ const timeoutCallback = unthrottledTimeoutCallbacks.get(message.timerId);
+ if (timeoutCallback) {
+ unthrottledTimeoutCallbacks.delete(message.timerId);
+ timeoutCallback();
+ return;
+ }
+
+ const intervalCallback = unthrottledIntervalCallbacks.get(message.timerId);
+ if (intervalCallback) {
+ intervalCallback();
+ }
+ };
+
+ return unthrottledTimerWorker;
+};
+
+export const setTimeoutUnthrottled = (
+ // eslint-disable-next-line @typescript-eslint/no-unsafe-function-type
+ callback: Function,
+ delay: number,
+): UnthrottledTimerHandle => {
+ if (shouldUseNativeTimers()) {
+ return { id: setTimeout(callback, delay) };
+ }
+
+ const timerId = nextUnthrottledTimerId++;
+ unthrottledTimeoutCallbacks.set(timerId, () => {
+ (callback as () => void)();
+ });
+
+ getUnthrottledTimerWorker().postMessage({
+ type: 'set-timeout',
+ timerId,
+ delay,
+ } satisfies UnthrottledTimerMessage);
+
+ return { id: timerId };
+};
+
+export const clearTimeoutUnthrottled = (timer: UnthrottledTimerHandle) => {
+ if (shouldUseNativeTimers()) {
+ clearTimeout(timer.id);
+ return;
+ }
+
+ assert(typeof timer.id === 'number');
+ unthrottledTimeoutCallbacks.delete(timer.id);
+
+ getUnthrottledTimerWorker().postMessage({
+ type: 'clear-timeout',
+ timerId: timer.id,
+ } satisfies UnthrottledTimerMessage);
+};
+
+export const setIntervalUnthrottled = (
+ // eslint-disable-next-line @typescript-eslint/no-unsafe-function-type
+ callback: Function,
+ delay: number,
+): UnthrottledTimerHandle => {
+ if (shouldUseNativeTimers()) {
+ return { id: setInterval(callback, delay) };
+ }
+
+ const timerId = nextUnthrottledTimerId++;
+ unthrottledIntervalCallbacks.set(timerId, () => {
+ (callback as () => void)();
+ });
+
+ getUnthrottledTimerWorker().postMessage({
+ type: 'set-interval',
+ timerId,
+ delay,
+ } satisfies UnthrottledTimerMessage);
+
+ return { id: timerId };
+};
+
+export const clearIntervalUnthrottled = (timer: UnthrottledTimerHandle) => {
+ if (shouldUseNativeTimers()) {
+ clearInterval(timer.id);
+ return;
+ }
+
+ assert(typeof timer.id === 'number');
+ unthrottledIntervalCallbacks.delete(timer.id);
+
+ getUnthrottledTimerWorker().postMessage({
+ type: 'clear-interval',
+ timerId: timer.id,
+ } satisfies UnthrottledTimerMessage);
+};