diff --git a/dev/live.html b/dev/live.html index 8eef45b..bdd2fc2 100644 --- a/dev/live.html +++ b/dev/live.html @@ -17,9 +17,9 @@ const button = document.querySelector('button'); button.addEventListener('click', async () => { - const stream = await navigator.mediaDevices.getUserMedia({ video: true, audio: true }); + const stream = await navigator.mediaDevices.getDisplayMedia({ video: true, audio: false }); const videoTrack = stream.getVideoTracks()[0]; - const audioTrack = stream.getAudioTracks()[0]; + const audioTrack = null;//stream.getAudioTracks()[0]; const output = new Mediabunny.Output({ target: new Mediabunny.BufferTarget(), @@ -30,7 +30,8 @@ if (videoTrack) { videoSource = new Mediabunny.MediaStreamVideoTrackSource(videoTrack, { codec: 'avc', - bitrate: Mediabunny.QUALITY_MEDIUM + bitrate: Mediabunny.QUALITY_MEDIUM, + sizeChangeBehavior: 'passThrough', }); videoSource.errorPromise.catch((d) => console.log("Hello?????", d)); @@ -45,12 +46,13 @@ audioSource.errorPromise.catch((d) => console.log("Hello!!???", d)); - output.addAudioTrack(audioSource); + //output.addAudioTrack(audioSource); } await output.start(); setTimeout(() => { + return; videoSource?.pause(); audioSource?.pause(); diff --git a/src/index.ts b/src/index.ts index b83f7bb..0126657 100644 --- a/src/index.ts +++ b/src/index.ts @@ -64,6 +64,7 @@ export { EncodedAudioPacketSource, EncodedVideoPacketSource, MediaStreamAudioTrackSource, + MediaStreamVideoTrackSourceOptions, MediaStreamVideoTrackSource, TextSubtitleSource, VideoSampleSource, diff --git a/src/media-source.ts b/src/media-source.ts index 3580a14..c3b4604 100644 --- a/src/media-source.ts +++ b/src/media-source.ts @@ -24,12 +24,15 @@ import { assertNever, CallSerializer, clamp, + clearIntervalUnthrottled, isFirefox, last, promiseWithResolvers, setInt24, + setIntervalUnthrottled, setUint24, toUint8Array, + UnthrottledTimerHandle, } from './misc'; import { Muxer } from './muxer'; import { SubtitleParser } from './subtitles'; @@ -1026,6 +1029,21 @@ export class CanvasSource extends VideoSource { } } +/** + * Options for MediaStreamVideoTrackSource. + * @group Media sources + * @public + */ +export type MediaStreamVideoTrackSourceOptions = { + /** + * The frame rate at which the underlying video track is sampled. Defaults to the frame rate specified in the + * track's [`MediaTrackSettings`](https://developer.mozilla.org/en-US/docs/Web/API/MediaTrackSettings). Set to + * `null` to only add a frame whenever the underlying track pushes one - this minimizes frame count but can + * lead to wildly irregular FPS. + */ + frameRate?: number | null; +}; + /** * Video source that encodes the frames of a * [`MediaStreamVideoTrack`](https://developer.mozilla.org/en-US/docs/Web/API/MediaStreamTrack) and pipes them into the @@ -1036,6 +1054,8 @@ export class CanvasSource extends VideoSource { * @public */ export class MediaStreamVideoTrackSource extends VideoSource { + /** @internal */ + private _options: MediaStreamVideoTrackSourceOptions; /** @internal */ private _encoder: VideoEncoderWrapper; /** @internal */ @@ -1053,9 +1073,9 @@ export class MediaStreamVideoTrackSource extends VideoSource { /** @internal */ private _paused = false; /** @internal */ - private _lastSampleTimestamp: number | null = null; + private _lastVideoFrame: VideoFrame | null = null; /** @internal */ - private _pauseOffset = 0; + private _timerHandle: UnthrottledTimerHandle | null = null; /** A promise that rejects upon any error within this source. This promise never resolves. */ get errorPromise() { @@ -1073,11 +1093,21 @@ export class MediaStreamVideoTrackSource extends VideoSource { * [`MediaStreamVideoTrack`](https://developer.mozilla.org/en-US/docs/Web/API/MediaStreamTrack), which will pull * video samples from the stream in real time and encode them according to {@link VideoEncodingConfig}. */ - constructor(track: MediaStreamVideoTrack, encodingConfig: VideoEncodingConfig) { + constructor( + track: MediaStreamVideoTrack, + encodingConfig: VideoEncodingConfig, + options: MediaStreamVideoTrackSourceOptions = {}, + ) { if (!(track instanceof MediaStreamTrack) || track.kind !== 'video') { throw new TypeError('track must be a video MediaStreamTrack.'); } validateVideoEncodingConfig(encodingConfig); + if (typeof options !== 'object' || !options) { + throw new TypeError('options must be an object.'); + } + if (options.frameRate != null && (typeof options.frameRate !== 'number' || options.frameRate <= 0)) { + throw new TypeError('options.frameRate, when provided, must be either a positive number or null.'); + } encodingConfig = { ...encodingConfig, @@ -1085,6 +1115,8 @@ export class MediaStreamVideoTrackSource extends VideoSource { }; super(encodingConfig.codec); + + this._options = options; this._encoder = new VideoEncoderWrapper(this, encodingConfig); this._track = track; } @@ -1098,32 +1130,89 @@ export class MediaStreamVideoTrackSource extends VideoSource { ); } + const frameRate = this._options.frameRate !== undefined + ? this._options.frameRate + : (this._track.getSettings().frameRate ?? null); + this._abortController = new AbortController(); let firstVideoFrameTimestamp: number | null = null; + let lastFrameTime: number | null = null; + let frameCount = 0; let errored = false; + let lastSampleTimestamp: number | null = null; + let pauseOffset = 0; + + const tick = () => { + assert(frameRate !== null); + + if (!this._lastVideoFrame) { + return; + } + + assert(lastFrameTime !== null); + assert(firstVideoFrameTimestamp !== null); + + const now = performance.now(); + + // Add as many frames as warranted by the elapsed time. + // > instead of >= intentionally because tick() is called before the _lastVideoFrame is changed + while (now - lastFrameTime > 1000 / frameRate) { + lastFrameTime += 1000 / frameRate; + const timestamp = firstVideoFrameTimestamp + frameCount / frameRate; + const clone = new VideoFrame(this._lastVideoFrame, { + timestamp: 1e6 * timestamp, + duration: 1e6 / frameRate, + }); + + addVideoFrame(clone, now); + } + }; + + if (frameRate !== null) { + this._timerHandle = setIntervalUnthrottled(tick, 4); // Run it at 250 Hz + } const onVideoFrame = (videoFrame: VideoFrame) => { + if (frameRate === null) { + addVideoFrame(videoFrame); + } else { + const now = performance.now(); + + if (!this._lastVideoFrame) { + addVideoFrame(videoFrame.clone(), now); + lastFrameTime = now; + this._lastVideoFrame = videoFrame; + } else { + tick(); + this._lastVideoFrame?.close(); + this._lastVideoFrame = videoFrame; + } + } + }; + + const addVideoFrame = (videoFrame: VideoFrame, now = performance.now()) => { if (errored) { videoFrame.close(); return; } + frameCount++; const currentTimestamp = videoFrame.timestamp / 1e6; if (this._paused) { const frameSeen = firstVideoFrameTimestamp !== null; if (frameSeen) { - if (this._lastSampleTimestamp !== null) { + if (lastSampleTimestamp !== null) { // In addition to dropping this frame, let's also keep track of the time we have lost due to the // pause. Doing it like this instead of simply keeping track of the paused time is better since // it retains the frame rate of the underlying source. - const timeDelta = currentTimestamp - this._lastSampleTimestamp; + const timeDelta = currentTimestamp - lastSampleTimestamp; // We modify this field instead of _timestampOffset since we still might have data in flight // in the encoder, with which we don't want to mess. - this._pauseOffset -= timeDelta; + pauseOffset -= timeDelta; } - this._lastSampleTimestamp = currentTimestamp; + lastSampleTimestamp = currentTimestamp; } videoFrame.close(); @@ -1135,24 +1224,24 @@ export class MediaStreamVideoTrackSource extends VideoSource { const muxer = this._connectedTrack!.output._muxer; if (muxer.firstMediaStreamTimestamp === null) { - muxer.firstMediaStreamTimestamp = performance.now() / 1000; + muxer.firstMediaStreamTimestamp = now / 1000; this._timestampOffset = -firstVideoFrameTimestamp; } else { - this._timestampOffset = (performance.now() / 1000 - muxer.firstMediaStreamTimestamp) + this._timestampOffset = (now / 1000 - muxer.firstMediaStreamTimestamp) - firstVideoFrameTimestamp; } } - this._lastSampleTimestamp = currentTimestamp; + lastSampleTimestamp = currentTimestamp; - if (this._encoder.getQueueSize() >= 4) { + if (this._encoder.getQueueSize() >= 8) { // Drop frames if the encoder is overloaded videoFrame.close(); return; } const sample = new VideoSample(videoFrame, { - timestamp: currentTimestamp + this._pauseOffset, + timestamp: currentTimestamp + pauseOffset, }); void this._encoder.add(sample, true) @@ -1235,6 +1324,11 @@ export class MediaStreamVideoTrackSource extends VideoSource { this._abortController = null; } + if (this._timerHandle) { + clearIntervalUnthrottled(this._timerHandle); + } + this._lastVideoFrame?.close(); + if (this._workerTrackId !== null) { assert(this._workerListener); @@ -1899,10 +1993,6 @@ export class MediaStreamAudioTrackSource extends AudioSource { private _errorPromiseAccessed = false; /** @internal */ private _paused = false; - /** @internal */ - private _lastSampleTimestamp: number | null = null; - /** @internal */ - private _pauseOffset = 0; /** A promise that rejects upon any error within this source. This promise never resolves. */ get errorPromise() { @@ -1934,7 +2024,7 @@ export class MediaStreamAudioTrackSource extends AudioSource { override async _start() { if (!this._errorPromiseAccessed) { console.warn( - 'Make sure not to ignore the `errorPromise` field on MediaStreamVideoTrackSource, so that any internal' + 'Make sure not to ignore the `errorPromise` field on MediaStreamAudioTrackSource, so that any internal' + ' errors get bubbled up properly.', ); } @@ -1943,6 +2033,8 @@ export class MediaStreamAudioTrackSource extends AudioSource { let firstAudioDataTimestamp: number | null = null; let errored = false; + let lastSampleTimestamp: number | null = null; + let pauseOffset = 0; const onAudioSample = (audioSample: AudioSample) => { if (errored) { @@ -1955,16 +2047,16 @@ export class MediaStreamAudioTrackSource extends AudioSource { if (this._paused) { const dataSeen = firstAudioDataTimestamp !== null; if (dataSeen) { - if (this._lastSampleTimestamp !== null) { + if (lastSampleTimestamp !== null) { // In addition to dropping this sample, let's also keep track of the time we have lost due to // the pause. Doing it like this instead of simply keeping track of the paused time is better // since it retains the sample rate of the underlying source. - const timeDelta = currentTimestamp - this._lastSampleTimestamp; + const timeDelta = currentTimestamp - lastSampleTimestamp; // We modify this field instead of _timestampOffset since we still might have data in flight // in the encoder, with which we don't want to mess. - this._pauseOffset -= timeDelta; + pauseOffset -= timeDelta; } - this._lastSampleTimestamp = currentTimestamp; + lastSampleTimestamp = currentTimestamp; } audioSample.close(); @@ -1984,15 +2076,15 @@ export class MediaStreamAudioTrackSource extends AudioSource { } } - this._lastSampleTimestamp = currentTimestamp; + lastSampleTimestamp = currentTimestamp; - if (this._encoder.getQueueSize() >= 4) { + if (this._encoder.getQueueSize() >= 8) { // Drop data if the encoder is overloaded audioSample.close(); return; } - audioSample.setTimestamp(currentTimestamp + this._pauseOffset); + audioSample.setTimestamp(currentTimestamp + pauseOffset); void this._encoder.add(audioSample, true) .catch((error) => { diff --git a/src/misc.ts b/src/misc.ts index b963ae7..982949b 100644 --- a/src/misc.ts +++ b/src/misc.ts @@ -839,3 +839,173 @@ export const validateRectangle = (rect: Rectangle, propertyPath: string) => { throw new TypeError(`${propertyPath}.height must be a non-negative integer.`); } }; + +export type UnthrottledTimerHandle = { + id: ReturnType | number; +}; + +type UnthrottledTimerMessage = + | { type: 'set-timeout'; timerId: number; delay: number } + | { type: 'set-interval'; timerId: number; delay: number } + | { type: 'clear-timeout'; timerId: number } + | { type: 'clear-interval'; timerId: number }; + +type UnthrottledTimerEvent = { type: 'fire'; timerId: number }; + +let unthrottledTimerWorker: Worker | undefined; +let nextUnthrottledTimerId = 1; +const unthrottledTimeoutCallbacks = new Map void>(); +const unthrottledIntervalCallbacks = new Map void>(); + +const shouldUseNativeTimers = () => { + return typeof window === 'undefined'; +}; + +const unthrottledTimerWorkerMain = () => { + const timeoutHandles = new Map>(); + const intervalHandles = new Map>(); + + self.onmessage = (event: MessageEvent) => { + const message = event.data; + + switch (message.type) { + case 'set-timeout': { + const handle = setTimeout(() => { + timeoutHandles.delete(message.timerId); + self.postMessage({ type: 'fire', timerId: message.timerId }); + }, message.delay); + + timeoutHandles.set(message.timerId, handle); + }; break; + + case 'set-interval': { + const handle = setInterval(() => { + self.postMessage({ type: 'fire', timerId: message.timerId }); + }, message.delay); + + intervalHandles.set(message.timerId, handle); + }; break; + + case 'clear-timeout': { + const handle = timeoutHandles.get(message.timerId); + if (handle !== undefined) { + clearTimeout(handle); + timeoutHandles.delete(message.timerId); + } + }; break; + + case 'clear-interval': { + const handle = intervalHandles.get(message.timerId); + if (handle !== undefined) { + clearInterval(handle); + intervalHandles.delete(message.timerId); + } + }; break; + } + }; +}; + +const getUnthrottledTimerWorker = () => { + if (unthrottledTimerWorker) { + return unthrottledTimerWorker; + } + + const workerSource = `(${unthrottledTimerWorkerMain.toString()})();`; + const workerURL = URL.createObjectURL(new Blob([workerSource], { type: 'text/javascript' })); + unthrottledTimerWorker = new Worker(workerURL); + URL.revokeObjectURL(workerURL); + + unthrottledTimerWorker.onmessage = (event: MessageEvent) => { + const message = event.data; + + const timeoutCallback = unthrottledTimeoutCallbacks.get(message.timerId); + if (timeoutCallback) { + unthrottledTimeoutCallbacks.delete(message.timerId); + timeoutCallback(); + return; + } + + const intervalCallback = unthrottledIntervalCallbacks.get(message.timerId); + if (intervalCallback) { + intervalCallback(); + } + }; + + return unthrottledTimerWorker; +}; + +export const setTimeoutUnthrottled = ( + // eslint-disable-next-line @typescript-eslint/no-unsafe-function-type + callback: Function, + delay: number, +): UnthrottledTimerHandle => { + if (shouldUseNativeTimers()) { + return { id: setTimeout(callback, delay) }; + } + + const timerId = nextUnthrottledTimerId++; + unthrottledTimeoutCallbacks.set(timerId, () => { + (callback as () => void)(); + }); + + getUnthrottledTimerWorker().postMessage({ + type: 'set-timeout', + timerId, + delay, + } satisfies UnthrottledTimerMessage); + + return { id: timerId }; +}; + +export const clearTimeoutUnthrottled = (timer: UnthrottledTimerHandle) => { + if (shouldUseNativeTimers()) { + clearTimeout(timer.id); + return; + } + + assert(typeof timer.id === 'number'); + unthrottledTimeoutCallbacks.delete(timer.id); + + getUnthrottledTimerWorker().postMessage({ + type: 'clear-timeout', + timerId: timer.id, + } satisfies UnthrottledTimerMessage); +}; + +export const setIntervalUnthrottled = ( + // eslint-disable-next-line @typescript-eslint/no-unsafe-function-type + callback: Function, + delay: number, +): UnthrottledTimerHandle => { + if (shouldUseNativeTimers()) { + return { id: setInterval(callback, delay) }; + } + + const timerId = nextUnthrottledTimerId++; + unthrottledIntervalCallbacks.set(timerId, () => { + (callback as () => void)(); + }); + + getUnthrottledTimerWorker().postMessage({ + type: 'set-interval', + timerId, + delay, + } satisfies UnthrottledTimerMessage); + + return { id: timerId }; +}; + +export const clearIntervalUnthrottled = (timer: UnthrottledTimerHandle) => { + if (shouldUseNativeTimers()) { + clearInterval(timer.id); + return; + } + + assert(typeof timer.id === 'number'); + unthrottledIntervalCallbacks.delete(timer.id); + + getUnthrottledTimerWorker().postMessage({ + type: 'clear-interval', + timerId: timer.id, + } satisfies UnthrottledTimerMessage); +};