diff --git a/dev/convert.html b/dev/convert.html index 96c013a..9f18bfd 100644 --- a/dev/convert.html +++ b/dev/convert.html @@ -26,7 +26,7 @@ chunked: true, chunkSize: 2**20 }); - const outputFormat = new Mediabunny.Mp4OutputFormat({}); + const outputFormat = new Mediabunny.Mp4OutputFormat({fastStart: 'fragmented'}); const button = document.createElement('button'); button.textContent = 'Cancel'; diff --git a/dev/live.html b/dev/live.html index 8eef45b..bdd2fc2 100644 --- a/dev/live.html +++ b/dev/live.html @@ -17,9 +17,9 @@ const button = document.querySelector('button'); button.addEventListener('click', async () => { - const stream = await navigator.mediaDevices.getUserMedia({ video: true, audio: true }); + const stream = await navigator.mediaDevices.getDisplayMedia({ video: true, audio: false }); const videoTrack = stream.getVideoTracks()[0]; - const audioTrack = stream.getAudioTracks()[0]; + const audioTrack = null;//stream.getAudioTracks()[0]; const output = new Mediabunny.Output({ target: new Mediabunny.BufferTarget(), @@ -30,7 +30,8 @@ if (videoTrack) { videoSource = new Mediabunny.MediaStreamVideoTrackSource(videoTrack, { codec: 'avc', - bitrate: Mediabunny.QUALITY_MEDIUM + bitrate: Mediabunny.QUALITY_MEDIUM, + sizeChangeBehavior: 'passThrough', }); videoSource.errorPromise.catch((d) => console.log("Hello?????", d)); @@ -45,12 +46,13 @@ audioSource.errorPromise.catch((d) => console.log("Hello!!???", d)); - output.addAudioTrack(audioSource); + //output.addAudioTrack(audioSource); } await output.start(); setTimeout(() => { + return; videoSource?.pause(); audioSource?.pause(); diff --git a/docs/guide/extensions/flac-encoder.md b/docs/guide/extensions/flac-encoder.md index c2f2ee9..34ae4d7 100644 --- a/docs/guide/extensions/flac-encoder.md +++ b/docs/guide/extensions/flac-encoder.md @@ -1,6 +1,6 @@ # @mediabunny/flac-encoder -No browser currently supports FLAC encoding in their WebCodecs implementations. This extension package provides a reliable FLAC encoder for use with Mediabunny. It is implemented using Mediabunny's [custom coder API](../supported-formats-and-codecs#custom-coders) and uses a fast, size-optimized WASM build of [libFLAC](https://github.com/xiph/flac) (the reference FLAC encoder) under the hood. +No browser currently supports FLAC encoding in their WebCodecs implementations. This extension package provides a reliable FLAC encoder for use with Mediabunny. It is implemented using Mediabunny's [custom coder API](../supported-formats-and-codecs#custom-coders) and uses a fast, size-optimized WASM build of [libFLAC](https://github.com/xiph/flac) under the hood. GitHub page diff --git a/docs/guide/supported-formats-and-codecs.md b/docs/guide/supported-formats-and-codecs.md index 1e2f5c3..553facd 100644 --- a/docs/guide/supported-formats-and-codecs.md +++ b/docs/guide/supported-formats-and-codecs.md @@ -37,13 +37,13 @@ Mediabunny ships with built-in decoders and encoders for all audio PCM codecs, m ### Audio codecs -- `'aac'` - Advanced Audio Coding (AAC) [^1] +- `'aac'` - Advanced Audio Coding (AAC) [^aac] - `'opus'` - Opus -- `'mp3'` - MP3 [^2] +- `'mp3'` - MP3 [^mp3] - `'vorbis'` - Vorbis -- `'flac'` - Free Lossless Audio Codec (FLAC) -- `'ac3'` - Dolby Digital (AC-3) [^3] -- `'eac3'` - Dolby Digital Plus (E-AC-3) [^3] +- `'flac'` - Free Lossless Audio Codec (FLAC) [^flac] +- `'ac3'` - Dolby Digital (AC-3) [^ac3] +- `'eac3'` - Dolby Digital Plus (E-AC-3) [^ac3] - `'pcm-u8'` - 8-bit unsigned PCM - `'pcm-s8'` - 8-bit signed PCM - `'pcm-s16'` - 16-bit little-endian signed PCM @@ -67,7 +67,7 @@ Mediabunny ships with built-in decoders and encoders for all audio PCM codecs, m Not all codecs can be used with all containers. The following table specifies the supported codec-container combinations: -| | .mp4 | .mov | .mkv | .webm[^4] | .ogg | .mp3 | .wav | .aac | .flac | .ts | +| | .mp4 | .mov | .mkv | .webm[^webm] | .ogg | .mp3 | .wav | .aac | .flac | .ts | |:--------------:|:--------:|:-----:|:-----:|:---------:|:-----:|:-----:|:-----:|:-----:|:-----:|:-----:| | `'avc'` | ✓ | ✓ | ✓ | | | | | | | ✓ | | `'hevc'` | ✓ | ✓ | ✓ | | | | | | | ✓ | @@ -95,13 +95,14 @@ Not all codecs can be used with all containers. The following table specifies th | `'pcm-f64be'` | ✓ | ✓ | | | | | | | | | | `'ulaw'` | | ✓ | | | | | ✓ | | | | | `'alaw'` | | ✓ | | | | | ✓ | | | | -| `'webvtt'`[^5] | (✓) | | (✓) | (✓) | | | | | | | +| `'webvtt'`[^webvtt] | (✓) | | (✓) | (✓) | | | | | | | -[^1]: In some browsers, AAC encoding is not supported by WebCodecs. You can polyfill it with the [`@mediabunny/aac-encoder`](./extensions/aac-encoder) extension package, or provide your own [custom coder](#custom-coders). -[^2]: MP3 encoding is not supported by WebCodecs. You can polyfill it with the [`@mediabunny/mp3-encoder`](./extensions/mp3-encoder) extension package, or provide your own [custom coder](#custom-coders). -[^3]: AC-3 and E-AC-3 are not natively supported by WebCodecs. To encode or decode these codecs, you can use the [`@mediabunny/ac3`](./extensions/ac3) extension package, or provide your own [custom coder](#custom-coders). -[^4]: WebM only supports a small subset of the codecs supported by Matroska. However, this library can technically read all codecs from a WebM that are supported by Matroska. -[^5]: WebVTT can only be written, not read. +[^aac]: In some browsers, AAC encoding is not supported by WebCodecs. You can polyfill it with the [`@mediabunny/aac-encoder`](./extensions/aac-encoder) extension package. +[^mp3]: MP3 encoding is not supported by WebCodecs. You can polyfill it with the [`@mediabunny/mp3-encoder`](./extensions/mp3-encoder) extension package. +[^flac]: FLAC encoding is not supported by WebCodecs. You can polyfill it with the [`@mediabunny/flac-encoder`](./extensions/flac-encoder) extension package. +[^ac3]: AC-3 and E-AC-3 are not natively supported by WebCodecs. To encode or decode these codecs, you can use the [`@mediabunny/ac3`](./extensions/ac3) extension package. +[^webm]: WebM only supports a small subset of the codecs supported by Matroska. However, this library can technically read all codecs from a WebM that are supported by Matroska. +[^webvtt]: WebVTT can only be written, not read. ## Querying codec encodability diff --git a/docs/index.md b/docs/index.md index a98654a..d71da59 100644 --- a/docs/index.md +++ b/docs/index.md @@ -112,7 +112,6 @@ const sponsors = { { image: 'https://avatars.githubusercontent.com/u/82552321', name: 'Polotno', url: 'https://github.com/polotno-project' }, { image: 'https://avatars.githubusercontent.com/u/489051', name: 'Roman Rädle', url: 'https://github.com/raedle' }, { image: 'https://avatars.githubusercontent.com/u/197597', name: 'Christopher Chedeau', url: 'https://github.com/vjeux' }, - { image: 'https://avatars.githubusercontent.com/u/84167135', name: 'Memenome', url: 'https://github.com/memenome' }, { image: 'https://avatars.githubusercontent.com/u/5913254', name: 'Brandon McConnell', url: 'https://github.com/brandonmcconnell' }, { image: 'https://avatars.githubusercontent.com/u/9549394', name: 'studnitz', url: 'https://github.com/studnitz' }, { image: 'https://avatars.githubusercontent.com/u/4714175', name: 'Phoomparin Mano', url: 'https://github.com/heypoom' }, diff --git a/package-lock.json b/package-lock.json index 8b2ab7c..3c36243 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "mediabunny", - "version": "1.37.0", + "version": "1.38.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "mediabunny", - "version": "1.37.0", + "version": "1.38.0", "license": "MPL-2.0", "workspaces": [ "packages/*" @@ -7751,9 +7751,9 @@ } }, "node_modules/mediabunny": { - "version": "1.36.0", - "resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.36.0.tgz", - "integrity": "sha512-+RsDJ9O0JcrLenlgT0jzUA4+78UQhPDHnUax7QnfwOL4JMAitzp+6aV3WKUYaplb7wapvxpn/0k/S2UkAbAHWQ==", + "version": "1.37.0", + "resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.37.0.tgz", + "integrity": "sha512-eV7M9IJ29pr/8RNL1sYtIxNbdMfDMN1hMwMaOFfNLhwuKKGSC+eKwiJFpdVjEJ3zrMA4LGerF4Hps0SENFSAlg==", "license": "MPL-2.0", "peer": true, "workspaces": [ @@ -12077,7 +12077,7 @@ }, "packages/aac-encoder": { "name": "@mediabunny/aac-encoder", - "version": "1.37.0", + "version": "1.38.0", "license": "MPL-2.0", "devDependencies": { "@types/emscripten": "^1.40.1" @@ -12092,7 +12092,7 @@ }, "packages/ac3": { "name": "@mediabunny/ac3", - "version": "1.37.0", + "version": "1.38.0", "license": "MPL-2.0", "devDependencies": { "@types/emscripten": "^1.40.1" @@ -12107,7 +12107,7 @@ }, "packages/flac-encoder": { "name": "@mediabunny/flac-encoder", - "version": "1.37.0", + "version": "1.38.0", "license": "MPL-2.0", "devDependencies": { "@types/emscripten": "^1.40.1" @@ -12122,7 +12122,7 @@ }, "packages/mp3-encoder": { "name": "@mediabunny/mp3-encoder", - "version": "1.37.0", + "version": "1.38.0", "license": "MPL-2.0", "devDependencies": { "@types/emscripten": "^1.40.1" diff --git a/package.json b/package.json index 7e5624a..669a648 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "mediabunny", "author": "Vanilagy", - "version": "1.37.0", + "version": "1.38.0", "description": "Pure TypeScript media toolkit for reading, writing, and converting media files, directly in the browser.", "type": "module", "workspaces": [ diff --git a/packages/aac-encoder/package.json b/packages/aac-encoder/package.json index 344ee78..786a388 100644 --- a/packages/aac-encoder/package.json +++ b/packages/aac-encoder/package.json @@ -1,7 +1,7 @@ { "name": "@mediabunny/aac-encoder", "author": "Vanilagy", - "version": "1.37.0", + "version": "1.38.0", "description": "AAC encoder extension for Mediabunny, based on FFmpeg.", "main": "./dist/bundles/mediabunny-aac-encoder.mjs", "module": "./dist/bundles/mediabunny-aac-encoder.mjs", diff --git a/packages/ac3/package.json b/packages/ac3/package.json index 6e7b27a..de7d2d4 100644 --- a/packages/ac3/package.json +++ b/packages/ac3/package.json @@ -1,7 +1,7 @@ { "name": "@mediabunny/ac3", "author": "Vanilagy", - "version": "1.37.0", + "version": "1.38.0", "description": "AC-3 and E-AC-3 (Dolby Digital) decoder and encoder extension for Mediabunny, based on FFmpeg.", "main": "./dist/bundles/mediabunny-ac3.mjs", "module": "./dist/bundles/mediabunny-ac3.mjs", diff --git a/packages/flac-encoder/package.json b/packages/flac-encoder/package.json index b25bc7b..ef0109d 100644 --- a/packages/flac-encoder/package.json +++ b/packages/flac-encoder/package.json @@ -1,7 +1,7 @@ { "name": "@mediabunny/flac-encoder", "author": "Vanilagy", - "version": "1.37.0", + "version": "1.38.0", "description": "FLAC encoder extension for Mediabunny, based on libFLAC.", "main": "./dist/bundles/mediabunny-flac-encoder.mjs", "module": "./dist/bundles/mediabunny-flac-encoder.mjs", diff --git a/packages/mp3-encoder/package.json b/packages/mp3-encoder/package.json index c1fbad3..6128c54 100644 --- a/packages/mp3-encoder/package.json +++ b/packages/mp3-encoder/package.json @@ -1,7 +1,7 @@ { "name": "@mediabunny/mp3-encoder", "author": "Vanilagy", - "version": "1.37.0", + "version": "1.38.0", "description": "MP3 encoder extension for Mediabunny, based on LAME.", "main": "./dist/bundles/mediabunny-mp3-encoder.mjs", "module": "./dist/bundles/mediabunny-mp3-encoder.mjs", diff --git a/src/index.ts b/src/index.ts index 88d617b..b8e9fda 100644 --- a/src/index.ts +++ b/src/index.ts @@ -64,6 +64,7 @@ export { EncodedAudioPacketSource, EncodedVideoPacketSource, MediaStreamAudioTrackSource, + MediaStreamVideoTrackSourceOptions, MediaStreamVideoTrackSource, TextSubtitleSource, VideoSampleSource, diff --git a/src/isobmff/isobmff-muxer.ts b/src/isobmff/isobmff-muxer.ts index 4256f53..b7a6158 100644 --- a/src/isobmff/isobmff-muxer.ts +++ b/src/isobmff/isobmff-muxer.ts @@ -1245,11 +1245,13 @@ export class IsobmffMuxer extends Muxer { override async onTrackClose(track: OutputTrack) { const release = await this.mutex.acquire(); - if (track.type === 'subtitle' && track.source._codec === 'webvtt') { - const trackData = this.trackDatas.find(x => x.track === track) as IsobmffSubtitleTrackData; - if (trackData) { + const trackData = this.trackDatas.find(x => x.track === track); + if (trackData) { + if (trackData.type === 'subtitle' && track.source._codec === 'webvtt') { await this.processWebVTTCues(trackData, Infinity); } + + this.processTimestamps(trackData); } if (this.allTracksAreKnown()) { @@ -1274,19 +1276,15 @@ export class IsobmffMuxer extends Muxer { if (trackData.type === 'subtitle' && trackData.track.source._codec === 'webvtt') { await this.processWebVTTCues(trackData, Infinity); } + + this.processTimestamps(trackData); } if (this.isFragmented) { await this.interleaveSamples(true); - - for (const trackData of this.trackDatas) { - this.processTimestamps(trackData); - } - await this.finalizeFragment(false); // Don't flush the last fragment as we will flush it with the mfra box } else { for (const trackData of this.trackDatas) { - this.processTimestamps(trackData); await this.finalizeCurrentChunk(trackData); } } diff --git a/src/media-source.ts b/src/media-source.ts index 3580a14..c3b4604 100644 --- a/src/media-source.ts +++ b/src/media-source.ts @@ -24,12 +24,15 @@ import { assertNever, CallSerializer, clamp, + clearIntervalUnthrottled, isFirefox, last, promiseWithResolvers, setInt24, + setIntervalUnthrottled, setUint24, toUint8Array, + UnthrottledTimerHandle, } from './misc'; import { Muxer } from './muxer'; import { SubtitleParser } from './subtitles'; @@ -1026,6 +1029,21 @@ export class CanvasSource extends VideoSource { } } +/** + * Options for MediaStreamVideoTrackSource. + * @group Media sources + * @public + */ +export type MediaStreamVideoTrackSourceOptions = { + /** + * The frame rate at which the underlying video track is sampled. Defaults to the frame rate specified in the + * track's [`MediaTrackSettings`](https://developer.mozilla.org/en-US/docs/Web/API/MediaTrackSettings). Set to + * `null` to only add a frame whenever the underlying track pushes one - this minimizes frame count but can + * lead to wildly irregular FPS. + */ + frameRate?: number | null; +}; + /** * Video source that encodes the frames of a * [`MediaStreamVideoTrack`](https://developer.mozilla.org/en-US/docs/Web/API/MediaStreamTrack) and pipes them into the @@ -1036,6 +1054,8 @@ export class CanvasSource extends VideoSource { * @public */ export class MediaStreamVideoTrackSource extends VideoSource { + /** @internal */ + private _options: MediaStreamVideoTrackSourceOptions; /** @internal */ private _encoder: VideoEncoderWrapper; /** @internal */ @@ -1053,9 +1073,9 @@ export class MediaStreamVideoTrackSource extends VideoSource { /** @internal */ private _paused = false; /** @internal */ - private _lastSampleTimestamp: number | null = null; + private _lastVideoFrame: VideoFrame | null = null; /** @internal */ - private _pauseOffset = 0; + private _timerHandle: UnthrottledTimerHandle | null = null; /** A promise that rejects upon any error within this source. This promise never resolves. */ get errorPromise() { @@ -1073,11 +1093,21 @@ export class MediaStreamVideoTrackSource extends VideoSource { * [`MediaStreamVideoTrack`](https://developer.mozilla.org/en-US/docs/Web/API/MediaStreamTrack), which will pull * video samples from the stream in real time and encode them according to {@link VideoEncodingConfig}. */ - constructor(track: MediaStreamVideoTrack, encodingConfig: VideoEncodingConfig) { + constructor( + track: MediaStreamVideoTrack, + encodingConfig: VideoEncodingConfig, + options: MediaStreamVideoTrackSourceOptions = {}, + ) { if (!(track instanceof MediaStreamTrack) || track.kind !== 'video') { throw new TypeError('track must be a video MediaStreamTrack.'); } validateVideoEncodingConfig(encodingConfig); + if (typeof options !== 'object' || !options) { + throw new TypeError('options must be an object.'); + } + if (options.frameRate != null && (typeof options.frameRate !== 'number' || options.frameRate <= 0)) { + throw new TypeError('options.frameRate, when provided, must be either a positive number or null.'); + } encodingConfig = { ...encodingConfig, @@ -1085,6 +1115,8 @@ export class MediaStreamVideoTrackSource extends VideoSource { }; super(encodingConfig.codec); + + this._options = options; this._encoder = new VideoEncoderWrapper(this, encodingConfig); this._track = track; } @@ -1098,32 +1130,89 @@ export class MediaStreamVideoTrackSource extends VideoSource { ); } + const frameRate = this._options.frameRate !== undefined + ? this._options.frameRate + : (this._track.getSettings().frameRate ?? null); + this._abortController = new AbortController(); let firstVideoFrameTimestamp: number | null = null; + let lastFrameTime: number | null = null; + let frameCount = 0; let errored = false; + let lastSampleTimestamp: number | null = null; + let pauseOffset = 0; + + const tick = () => { + assert(frameRate !== null); + + if (!this._lastVideoFrame) { + return; + } + + assert(lastFrameTime !== null); + assert(firstVideoFrameTimestamp !== null); + + const now = performance.now(); + + // Add as many frames as warranted by the elapsed time. + // > instead of >= intentionally because tick() is called before the _lastVideoFrame is changed + while (now - lastFrameTime > 1000 / frameRate) { + lastFrameTime += 1000 / frameRate; + const timestamp = firstVideoFrameTimestamp + frameCount / frameRate; + const clone = new VideoFrame(this._lastVideoFrame, { + timestamp: 1e6 * timestamp, + duration: 1e6 / frameRate, + }); + + addVideoFrame(clone, now); + } + }; + + if (frameRate !== null) { + this._timerHandle = setIntervalUnthrottled(tick, 4); // Run it at 250 Hz + } const onVideoFrame = (videoFrame: VideoFrame) => { + if (frameRate === null) { + addVideoFrame(videoFrame); + } else { + const now = performance.now(); + + if (!this._lastVideoFrame) { + addVideoFrame(videoFrame.clone(), now); + lastFrameTime = now; + this._lastVideoFrame = videoFrame; + } else { + tick(); + this._lastVideoFrame?.close(); + this._lastVideoFrame = videoFrame; + } + } + }; + + const addVideoFrame = (videoFrame: VideoFrame, now = performance.now()) => { if (errored) { videoFrame.close(); return; } + frameCount++; const currentTimestamp = videoFrame.timestamp / 1e6; if (this._paused) { const frameSeen = firstVideoFrameTimestamp !== null; if (frameSeen) { - if (this._lastSampleTimestamp !== null) { + if (lastSampleTimestamp !== null) { // In addition to dropping this frame, let's also keep track of the time we have lost due to the // pause. Doing it like this instead of simply keeping track of the paused time is better since // it retains the frame rate of the underlying source. - const timeDelta = currentTimestamp - this._lastSampleTimestamp; + const timeDelta = currentTimestamp - lastSampleTimestamp; // We modify this field instead of _timestampOffset since we still might have data in flight // in the encoder, with which we don't want to mess. - this._pauseOffset -= timeDelta; + pauseOffset -= timeDelta; } - this._lastSampleTimestamp = currentTimestamp; + lastSampleTimestamp = currentTimestamp; } videoFrame.close(); @@ -1135,24 +1224,24 @@ export class MediaStreamVideoTrackSource extends VideoSource { const muxer = this._connectedTrack!.output._muxer; if (muxer.firstMediaStreamTimestamp === null) { - muxer.firstMediaStreamTimestamp = performance.now() / 1000; + muxer.firstMediaStreamTimestamp = now / 1000; this._timestampOffset = -firstVideoFrameTimestamp; } else { - this._timestampOffset = (performance.now() / 1000 - muxer.firstMediaStreamTimestamp) + this._timestampOffset = (now / 1000 - muxer.firstMediaStreamTimestamp) - firstVideoFrameTimestamp; } } - this._lastSampleTimestamp = currentTimestamp; + lastSampleTimestamp = currentTimestamp; - if (this._encoder.getQueueSize() >= 4) { + if (this._encoder.getQueueSize() >= 8) { // Drop frames if the encoder is overloaded videoFrame.close(); return; } const sample = new VideoSample(videoFrame, { - timestamp: currentTimestamp + this._pauseOffset, + timestamp: currentTimestamp + pauseOffset, }); void this._encoder.add(sample, true) @@ -1235,6 +1324,11 @@ export class MediaStreamVideoTrackSource extends VideoSource { this._abortController = null; } + if (this._timerHandle) { + clearIntervalUnthrottled(this._timerHandle); + } + this._lastVideoFrame?.close(); + if (this._workerTrackId !== null) { assert(this._workerListener); @@ -1899,10 +1993,6 @@ export class MediaStreamAudioTrackSource extends AudioSource { private _errorPromiseAccessed = false; /** @internal */ private _paused = false; - /** @internal */ - private _lastSampleTimestamp: number | null = null; - /** @internal */ - private _pauseOffset = 0; /** A promise that rejects upon any error within this source. This promise never resolves. */ get errorPromise() { @@ -1934,7 +2024,7 @@ export class MediaStreamAudioTrackSource extends AudioSource { override async _start() { if (!this._errorPromiseAccessed) { console.warn( - 'Make sure not to ignore the `errorPromise` field on MediaStreamVideoTrackSource, so that any internal' + 'Make sure not to ignore the `errorPromise` field on MediaStreamAudioTrackSource, so that any internal' + ' errors get bubbled up properly.', ); } @@ -1943,6 +2033,8 @@ export class MediaStreamAudioTrackSource extends AudioSource { let firstAudioDataTimestamp: number | null = null; let errored = false; + let lastSampleTimestamp: number | null = null; + let pauseOffset = 0; const onAudioSample = (audioSample: AudioSample) => { if (errored) { @@ -1955,16 +2047,16 @@ export class MediaStreamAudioTrackSource extends AudioSource { if (this._paused) { const dataSeen = firstAudioDataTimestamp !== null; if (dataSeen) { - if (this._lastSampleTimestamp !== null) { + if (lastSampleTimestamp !== null) { // In addition to dropping this sample, let's also keep track of the time we have lost due to // the pause. Doing it like this instead of simply keeping track of the paused time is better // since it retains the sample rate of the underlying source. - const timeDelta = currentTimestamp - this._lastSampleTimestamp; + const timeDelta = currentTimestamp - lastSampleTimestamp; // We modify this field instead of _timestampOffset since we still might have data in flight // in the encoder, with which we don't want to mess. - this._pauseOffset -= timeDelta; + pauseOffset -= timeDelta; } - this._lastSampleTimestamp = currentTimestamp; + lastSampleTimestamp = currentTimestamp; } audioSample.close(); @@ -1984,15 +2076,15 @@ export class MediaStreamAudioTrackSource extends AudioSource { } } - this._lastSampleTimestamp = currentTimestamp; + lastSampleTimestamp = currentTimestamp; - if (this._encoder.getQueueSize() >= 4) { + if (this._encoder.getQueueSize() >= 8) { // Drop data if the encoder is overloaded audioSample.close(); return; } - audioSample.setTimestamp(currentTimestamp + this._pauseOffset); + audioSample.setTimestamp(currentTimestamp + pauseOffset); void this._encoder.add(audioSample, true) .catch((error) => { diff --git a/src/misc.ts b/src/misc.ts index b370988..d8b1b1b 100644 --- a/src/misc.ts +++ b/src/misc.ts @@ -951,3 +951,173 @@ export const prefer = (value: boolean) => { export type NonFunctionKeys = { [K in keyof T]-?: T[K] extends ((...args: never[]) => unknown) ? never : K }[keyof T]; + +export type UnthrottledTimerHandle = { + id: ReturnType | number; +}; + +type UnthrottledTimerMessage = + | { type: 'set-timeout'; timerId: number; delay: number } + | { type: 'set-interval'; timerId: number; delay: number } + | { type: 'clear-timeout'; timerId: number } + | { type: 'clear-interval'; timerId: number }; + +type UnthrottledTimerEvent = { type: 'fire'; timerId: number }; + +let unthrottledTimerWorker: Worker | undefined; +let nextUnthrottledTimerId = 1; +const unthrottledTimeoutCallbacks = new Map void>(); +const unthrottledIntervalCallbacks = new Map void>(); + +const shouldUseNativeTimers = () => { + return typeof window === 'undefined'; +}; + +const unthrottledTimerWorkerMain = () => { + const timeoutHandles = new Map>(); + const intervalHandles = new Map>(); + + self.onmessage = (event: MessageEvent) => { + const message = event.data; + + switch (message.type) { + case 'set-timeout': { + const handle = setTimeout(() => { + timeoutHandles.delete(message.timerId); + self.postMessage({ type: 'fire', timerId: message.timerId }); + }, message.delay); + + timeoutHandles.set(message.timerId, handle); + }; break; + + case 'set-interval': { + const handle = setInterval(() => { + self.postMessage({ type: 'fire', timerId: message.timerId }); + }, message.delay); + + intervalHandles.set(message.timerId, handle); + }; break; + + case 'clear-timeout': { + const handle = timeoutHandles.get(message.timerId); + if (handle !== undefined) { + clearTimeout(handle); + timeoutHandles.delete(message.timerId); + } + }; break; + + case 'clear-interval': { + const handle = intervalHandles.get(message.timerId); + if (handle !== undefined) { + clearInterval(handle); + intervalHandles.delete(message.timerId); + } + }; break; + } + }; +}; + +const getUnthrottledTimerWorker = () => { + if (unthrottledTimerWorker) { + return unthrottledTimerWorker; + } + + const workerSource = `(${unthrottledTimerWorkerMain.toString()})();`; + const workerURL = URL.createObjectURL(new Blob([workerSource], { type: 'text/javascript' })); + unthrottledTimerWorker = new Worker(workerURL); + URL.revokeObjectURL(workerURL); + + unthrottledTimerWorker.onmessage = (event: MessageEvent) => { + const message = event.data; + + const timeoutCallback = unthrottledTimeoutCallbacks.get(message.timerId); + if (timeoutCallback) { + unthrottledTimeoutCallbacks.delete(message.timerId); + timeoutCallback(); + return; + } + + const intervalCallback = unthrottledIntervalCallbacks.get(message.timerId); + if (intervalCallback) { + intervalCallback(); + } + }; + + return unthrottledTimerWorker; +}; + +export const setTimeoutUnthrottled = ( + // eslint-disable-next-line @typescript-eslint/no-unsafe-function-type + callback: Function, + delay: number, +): UnthrottledTimerHandle => { + if (shouldUseNativeTimers()) { + return { id: setTimeout(callback, delay) }; + } + + const timerId = nextUnthrottledTimerId++; + unthrottledTimeoutCallbacks.set(timerId, () => { + (callback as () => void)(); + }); + + getUnthrottledTimerWorker().postMessage({ + type: 'set-timeout', + timerId, + delay, + } satisfies UnthrottledTimerMessage); + + return { id: timerId }; +}; + +export const clearTimeoutUnthrottled = (timer: UnthrottledTimerHandle) => { + if (shouldUseNativeTimers()) { + clearTimeout(timer.id); + return; + } + + assert(typeof timer.id === 'number'); + unthrottledTimeoutCallbacks.delete(timer.id); + + getUnthrottledTimerWorker().postMessage({ + type: 'clear-timeout', + timerId: timer.id, + } satisfies UnthrottledTimerMessage); +}; + +export const setIntervalUnthrottled = ( + // eslint-disable-next-line @typescript-eslint/no-unsafe-function-type + callback: Function, + delay: number, +): UnthrottledTimerHandle => { + if (shouldUseNativeTimers()) { + return { id: setInterval(callback, delay) }; + } + + const timerId = nextUnthrottledTimerId++; + unthrottledIntervalCallbacks.set(timerId, () => { + (callback as () => void)(); + }); + + getUnthrottledTimerWorker().postMessage({ + type: 'set-interval', + timerId, + delay, + } satisfies UnthrottledTimerMessage); + + return { id: timerId }; +}; + +export const clearIntervalUnthrottled = (timer: UnthrottledTimerHandle) => { + if (shouldUseNativeTimers()) { + clearInterval(timer.id); + return; + } + + assert(typeof timer.id === 'number'); + unthrottledIntervalCallbacks.delete(timer.id); + + getUnthrottledTimerWorker().postMessage({ + type: 'clear-interval', + timerId: timer.id, + } satisfies UnthrottledTimerMessage); +}; diff --git a/test/node/isobmff-muxer.test.ts b/test/node/isobmff-muxer.test.ts index 55cee50..83d1d9e 100644 --- a/test/node/isobmff-muxer.test.ts +++ b/test/node/isobmff-muxer.test.ts @@ -61,3 +61,46 @@ test('ISOBMFF muxer internally converts ADTS to AAC', async () => { expect(count).toBe(4557); }); + +test('Fragmented fMP4 with video+audio preserves B-frame CTS', async () => { + using input = new Input({ + source: new FilePathSource(path.join(__dirname, '../public/video.mp4')), + formats: ALL_FORMATS, + }); + + const videoTrack = await input.getPrimaryVideoTrack(); + const audioTrack = await input.getPrimaryAudioTrack(); + assert(videoTrack); + assert(audioTrack); + + const originalVideoSink = new EncodedPacketSink(videoTrack); + const originalTimestamps: number[] = []; + for await (const packet of originalVideoSink.packets()) { + originalTimestamps.push(packet.timestamp); + } + + const output = new Output({ + format: new Mp4OutputFormat({ fastStart: 'fragmented' }), + target: new BufferTarget(), + }); + + const conversion = await Conversion.init({ input, output, showWarnings: false }); + await conversion.execute(); + + using outputAsInput = new Input({ + source: new BufferSource(output.target.buffer!), + formats: ALL_FORMATS, + }); + + const outputVideoTrack = await outputAsInput.getPrimaryVideoTrack(); + assert(outputVideoTrack); + + const videoSink = new EncodedPacketSink(outputVideoTrack); + + const timestamps: number[] = []; + for await (const packet of videoSink.packets()) { + timestamps.push(packet.timestamp); + } + + expect(timestamps).toEqual(originalTimestamps); +});