diff --git a/dev/demux.html b/dev/demux.html index 580bc38..3afb290 100644 --- a/dev/demux.html +++ b/dev/demux.html @@ -18,13 +18,7 @@ }); const track = await input.getPrimaryVideoTrack(); - const packetSink = new Mediabunny.EncodedPacketSink(track); - - const first = await packetSink.getFirstPacket({ verifyKeyPackets: true }); - const second = await packetSink.getNextPacket(first, { verifyKeyPackets: true }); - const third = await packetSink.getNextPacket(second, { verifyKeyPackets: true }); - const fourth = await packetSink.getNextPacket(third, { verifyKeyPackets: true }); - console.log(first, second, third, fourth); + console.log(await track.computeFrameRateMetrics()); /* diff --git a/docs/guide/quick-start.md b/docs/guide/quick-start.md index b947878..d70fbfe 100644 --- a/docs/guide/quick-start.md +++ b/docs/guide/quick-start.md @@ -27,8 +27,8 @@ if (videoTrack) { await videoTrack.getRotation(); // in degrees clockwise // Estimate frame rate (FPS) - const packetStats = await videoTrack.computePacketStats(100); - const averageFrameRate = packetStats.averagePacketRate; + const frameRateMetrics = await videoTrack.computeFrameRateMetrics(); + const frameRate = frameRateMetrics.bestGuessFrameRate; } // Extract audio metadata diff --git a/docs/guide/reading-media-files.md b/docs/guide/reading-media-files.md index 61c7393..b07f1ab 100644 --- a/docs/guide/reading-media-files.md +++ b/docs/guide/reading-media-files.md @@ -241,7 +241,7 @@ Intuitively, this is the maximum possible "frame rate" of the track (assuming th $$ \frac{k}{x},\quad k \in \mathbb{Z} $$ ::: info -This field only gives an upper bound on a track's frame rate. To get a track's actual frame rate based on its samples, compute its [packet statistics](#packet-statistics). +This field only gives an upper bound on a track's frame rate. To determine a video track's actual frame rate based on its frames, compute its [frame rate metrics](#frame-rate-metrics). ::: Some tracks (especially live tracks) have timestamps which are relative to the Unix epoch (Jan 1 1970, midnight UTC). In other words, their timestamps *are* Unix timestamps. This allows you to map the media data to a definitive point in wall-clock time. To see if this is the case, use: @@ -281,7 +281,7 @@ This means the video track has a total of 14315 frames, a frame rate of exactly ```ts await track.computePacketStats(50); ``` -This will only look at the first ~50 packets and then return the result. This is great for quickly getting an estimate of frame rate and bitrate, without having to scan through the entire file. For videos with a constant frame rate, this will also always return the correct frame rate. +This will only look at the first ~50 packets and then return the result. This is great for quickly getting an estimate of bitrate, without having to scan through the entire file. To determine a video track's frame rate, prefer using [frame rate metrics](#frame-rate-metrics) instead. ### Video track metadata @@ -310,11 +310,7 @@ await videoTrack.getRotation(); // => 0 | 90 | 180 | 270 await videoTrack.getPixelAspectRatio(); // => { num: number, den: number } ``` -To compute a video track's average frame rate (FPS), use [`computePacketStats`](#packet-statistics): -```ts -const stats = await videoTrack.computePacketStats(100); -const frameRate = stats.averagePacketRate; // Approximate, but often exact -``` +To determine a video track's frame rate (FPS), compute its [frame rate metrics](#frame-rate-metrics). You can retrieve the track's decoder configuration, which is a `VideoDecoderConfig` from the WebCodecs API for usage within `VideoDecoder`: ```ts @@ -350,6 +346,21 @@ await videoTrack.hasHighDynamicRange(); // => boolean ``` This method compares with the available color space metadata. If it resolves to `true`, then the video is HDR; if it resolves to `false`, the video may or may not be HDR. +#### Frame rate metrics + +You can compute metrics about a video track's frame rate (FPS): +```ts +const metrics = await videoTrack.computeFrameRateMetrics(); // => FrameRateMetrics +metrics.bestGuessFrameRate; // => number +``` + +Frame rate is never determined from file metadata (which is unreliable) but is always deduced directly from the actual frame timestamps. See [`FrameRateMetrics`](../api/FrameRateMetrics) for more. + +By default, this method probes the first 256 packets of the track, which is enough for a reliable estimate. You can control this number using the `targetPacketCount` option, but note that increasing it makes the call more expensive: +```ts +await videoTrack.computeFrameRateMetrics({ targetPacketCount: 1024 }); +``` + ### Audio track metadata In addition to the [common track metadata](#common-track-metadata), audio tracks have additional metadata you can query: diff --git a/docs/index.md b/docs/index.md index 61ed0f9..6bdbc80 100644 --- a/docs/index.md +++ b/docs/index.md @@ -123,6 +123,7 @@ const sponsors = { { image: 'https://avatars.githubusercontent.com/u/67356781', name: 'Ahmed Rowaihi', url: 'https://github.com/ahmedrowaihi' }, { image: 'https://avatars.githubusercontent.com/u/4714175', name: 'Phoomparin Mano', url: 'https://github.com/heypoom' }, { image: 'https://avatars.githubusercontent.com/u/504909', name: 'Hirbod', url: 'https://github.com/hirbod' }, + { image: 'https://avatars.githubusercontent.com/u/15199031', name: 'Marc Rousavy', url: 'https://github.com/mrousavy' }, { image: 'https://avatars.githubusercontent.com/u/2565549', name: 'MatthewNorton', url: 'https://github.com/MatthewNorton' }, { image: 'https://avatars.githubusercontent.com/u/272247752', name: 'Motionik', url: 'https://github.com/Motionik' }, { image: 'https://avatars.githubusercontent.com/u/2698271', name: 'Matthew Gardner', url: 'https://github.com/spheric' }, diff --git a/package-lock.json b/package-lock.json index 1d7a236..e8a6e4e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "mediabunny", - "version": "1.53.1", + "version": "1.54.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "mediabunny", - "version": "1.53.1", + "version": "1.54.0", "license": "MPL-2.0", "workspaces": [ ".", @@ -12954,7 +12954,7 @@ }, "packages/aac-encoder": { "name": "@mediabunny/aac-encoder", - "version": "1.53.1", + "version": "1.54.0", "license": "MPL-2.0", "devDependencies": { "@types/emscripten": "^1.40.1" @@ -12969,7 +12969,7 @@ }, "packages/ac3": { "name": "@mediabunny/ac3", - "version": "1.53.1", + "version": "1.54.0", "license": "MPL-2.0", "devDependencies": { "@types/emscripten": "^1.40.1" @@ -12984,7 +12984,7 @@ }, "packages/flac-encoder": { "name": "@mediabunny/flac-encoder", - "version": "1.53.1", + "version": "1.54.0", "license": "MPL-2.0", "devDependencies": { "@types/emscripten": "^1.40.1" @@ -12999,7 +12999,7 @@ }, "packages/mp3-encoder": { "name": "@mediabunny/mp3-encoder", - "version": "1.53.1", + "version": "1.54.0", "license": "MPL-2.0", "devDependencies": { "@types/emscripten": "^1.40.1" @@ -13014,7 +13014,7 @@ }, "packages/prores": { "name": "@mediabunny/prores", - "version": "1.53.1", + "version": "1.54.0", "license": "MPL-2.0", "dependencies": { "turbores": "^1.2.2" @@ -13029,10 +13029,10 @@ }, "packages/server": { "name": "@mediabunny/server", - "version": "1.53.1", + "version": "1.54.0", "license": "MPL-2.0", "dependencies": { - "@mediabunny/prores": "^1.53.1", + "@mediabunny/prores": "^1.54.0", "node-av": "^6.0.0" }, "funding": { diff --git a/package.json b/package.json index 363543c..b3ea4ae 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "mediabunny", "author": "Vanilagy", - "version": "1.53.1", + "version": "1.54.0", "description": "Pure TypeScript media toolkit for reading, writing, and converting media files, directly in the browser.", "type": "module", "workspaces": [ diff --git a/packages/aac-encoder/package.json b/packages/aac-encoder/package.json index 9d73d10..0efade3 100644 --- a/packages/aac-encoder/package.json +++ b/packages/aac-encoder/package.json @@ -1,7 +1,7 @@ { "name": "@mediabunny/aac-encoder", "author": "Vanilagy", - "version": "1.53.1", + "version": "1.54.0", "description": "AAC encoder extension for Mediabunny, based on FFmpeg.", "main": "./dist/bundles/mediabunny-aac-encoder.mjs", "module": "./dist/bundles/mediabunny-aac-encoder.mjs", diff --git a/packages/ac3/package.json b/packages/ac3/package.json index 55c4958..eb7be18 100644 --- a/packages/ac3/package.json +++ b/packages/ac3/package.json @@ -1,7 +1,7 @@ { "name": "@mediabunny/ac3", "author": "Vanilagy", - "version": "1.53.1", + "version": "1.54.0", "description": "AC-3 and E-AC-3 (Dolby Digital) decoder and encoder extension for Mediabunny, based on FFmpeg.", "main": "./dist/bundles/mediabunny-ac3.mjs", "module": "./dist/bundles/mediabunny-ac3.mjs", diff --git a/packages/flac-encoder/package.json b/packages/flac-encoder/package.json index 7c228ab..9e0f60f 100644 --- a/packages/flac-encoder/package.json +++ b/packages/flac-encoder/package.json @@ -1,7 +1,7 @@ { "name": "@mediabunny/flac-encoder", "author": "Vanilagy", - "version": "1.53.1", + "version": "1.54.0", "description": "FLAC encoder extension for Mediabunny, based on libFLAC.", "main": "./dist/bundles/mediabunny-flac-encoder.mjs", "module": "./dist/bundles/mediabunny-flac-encoder.mjs", diff --git a/packages/mp3-encoder/package.json b/packages/mp3-encoder/package.json index 67f4f3c..22fab64 100644 --- a/packages/mp3-encoder/package.json +++ b/packages/mp3-encoder/package.json @@ -1,7 +1,7 @@ { "name": "@mediabunny/mp3-encoder", "author": "Vanilagy", - "version": "1.53.1", + "version": "1.54.0", "description": "MP3 encoder extension for Mediabunny, based on LAME.", "main": "./dist/bundles/mediabunny-mp3-encoder.mjs", "module": "./dist/bundles/mediabunny-mp3-encoder.mjs", diff --git a/packages/prores/package.json b/packages/prores/package.json index 5c2a8ce..a9686a5 100644 --- a/packages/prores/package.json +++ b/packages/prores/package.json @@ -1,7 +1,7 @@ { "name": "@mediabunny/prores", "author": "Vanilagy", - "version": "1.53.1", + "version": "1.54.0", "description": "Apple ProRes decoder extension for Mediabunny, based on TurboRes.", "main": "./dist/bundles/mediabunny-prores.mjs", "module": "./dist/bundles/mediabunny-prores.mjs", diff --git a/packages/server/package.json b/packages/server/package.json index 69e237b..ff27d68 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@mediabunny/server", "author": "Vanilagy", - "version": "1.53.1", + "version": "1.54.0", "description": "Adds full video and audio decoder and encoder support to Mediabunny for use in server-side environments (Node, Bun, Deno). Based on NodeAV.", "main": "./dist/bundles/mediabunny-server.cjs", "module": "./dist/bundles/mediabunny-server.mjs", @@ -35,7 +35,7 @@ }, "dependencies": { "node-av": "^6.0.0", - "@mediabunny/prores": "^1.53.1" + "@mediabunny/prores": "^1.54.0" }, "peerDependencies": { "mediabunny": "^1.45.0" diff --git a/src/index.ts b/src/index.ts index ff9e5f9..ebb36a5 100644 --- a/src/index.ts +++ b/src/index.ts @@ -251,6 +251,8 @@ export { InputVideoTrack, InputAudioTrack, type InputTrackQuery, + type FrameRateMetrics, + type FrameRateMetricsOptions, type PacketStats, asc, desc, diff --git a/src/input-track.ts b/src/input-track.ts index b488495..2a6400d 100644 --- a/src/input-track.ts +++ b/src/input-track.ts @@ -32,6 +32,75 @@ export type PacketStats = { averageBitrate: number; }; +/** + * Computed metrics about the frame rate of a video track. + * @group Input files & tracks + * @public + */ +export type FrameRateMetrics = { + /** + * The true, underlying frame rate of the video that produced the frame timestamps. This value is heuristically + * determined by examining the time differences between frames, excluding outliers, and then fitting them to + * fractions. This algorithm is stable even when frames are dropped, as long as all frames lie roughly on one + * uniform lattice. + * + * If this field is not `null`, Mediabunny is extremely confident in having found the actual frame rate of + * the video. + * + * This field is `null` for videos where no consistent underlying frame rate was detected, indicating a variable + * frame-rate (VFR) video. + */ + underlyingFrameRate: number | null; + /** + * Mediabunny's best guess for the video's actual intended frame rate. + * + * This value is determined heuristically. If `underlyingFrameRate` exists, this field will be equal to it. If it + * doesn't, Mediabunny will check if `medianFrameRate` is close to a "common" frame rate and if so, snap to it. + * Otherwise, this field simply falls back to `medianFrameRate`. + */ + bestGuessFrameRate: number; + /** + * The minimum frame rate of the video at any given point, based on the largest distance between two + * consecutive frames. + */ + minFrameRate: number; + /** + * The maximum frame rate of the video at any given point, based on the smallest distance between two + * consecutive frames. + */ + maxFrameRate: number; + /** + * The average frame rate across the duration of the probed packets. This value represents the average frames per + * second; it is not the average of frame rates across the video. + */ + averageFrameRate: number; + /** + * The median frame rate, as determined by the distance between any two consecutive frames. For variable frame-rate + * (VFR) videos, this field gives you a good idea of the _de facto_ frame rate of the video. + */ + medianFrameRate: number; + /** Is `true` only when the underlying video is CFR (constant frame-rate) and no frames are skipped. */ + frameRateIsConstant: boolean; + /** + * The number of packets that were probed/inspected to estimate the frame rate. Probe at least 256 packets for a + * reliable estimation of frame rate. + */ + probedPacketCount: number; +}; + +/** + * Options for controlling how a video track's {@link FrameRateMetrics} are computed. + * @group Input files & tracks + * @public + */ +export type FrameRateMetricsOptions = { + /** + * The target number of packets to inspect to compute the frame rate. Defaults to `256`. Pass `Infinity` to scan + * the entire file. + */ + targetPacketCount?: number; +}; + export interface InputTrackBacking { getType(): TrackType; getId(): number; @@ -788,6 +857,152 @@ export class InputVideoTrack extends InputTrack { return determineVideoPacketType(codec, decoderConfig, packet.data); } + + /** + * Computes frame rate metrics for this video track, i.e. estimates the video's frame rate. Frame rate is never + * determined from file metadata (which is unreliable) but is always deduced directly from the actual frame + * timestamps. + */ + async computeFrameRateMetrics(options: FrameRateMetricsOptions = {}): Promise { + if (!options || typeof options !== 'object') { + throw new TypeError('options must be an object.'); + } + if ( + options.targetPacketCount !== undefined + && (!Number.isFinite(options.targetPacketCount) || options.targetPacketCount < 0) + ) { + throw new TypeError('options.targetPacketCount must be a non-negative number.'); + } + + const timeResolution = await this.getTimeResolution(); + const targetPacketCount = options.targetPacketCount ?? 256; + + const sink = new EncodedPacketSink(this); + const timestamps: number[] = []; + let maxTimestamp = -Infinity; + + let probedPacketCount = 0; + + for await (const packet of sink.packets(undefined, undefined, { metadataOnly: true })) { + if ( + timestamps.length >= targetPacketCount + // Needed for out-of-presentation-order packets. + && packet.timestamp >= maxTimestamp + ) { + break; + } + + timestamps.push(packet.timestamp); + maxTimestamp = Math.max(maxTimestamp, packet.timestamp); + probedPacketCount++; + } + + const ticks = new Float64Array(timestamps.length); + + for (let i = 0; i < timestamps.length; i++) { + ticks[i] = Math.round(timestamps[i]! * timeResolution); + } + + ticks.sort(); + + // Deduplicate in-place; only ticks[0..n) remains active. + let n = 1; + + for (let i = 1; i < ticks.length; i++) { + if (ticks[i] !== ticks[n - 1]) { + ticks[n++] = ticks[i]!; + } + } + + if (n < 2) { + return { + underlyingFrameRate: null, + bestGuessFrameRate: timeResolution, + minFrameRate: timeResolution, + maxFrameRate: timeResolution, + averageFrameRate: timeResolution, + medianFrameRate: timeResolution, + frameRateIsConstant: true, + probedPacketCount, + }; + } + + const activeTicks = ticks.subarray(0, n); + const underlyingFrameRate = findUnderlyingFrameRate(activeTicks, timeResolution); + + // If an underlying frame rate exists, differences are expressed in inferred frame intervals. Otherwise they + // remain raw timestamp ticks. + const unitRate = underlyingFrameRate ?? timeResolution; + const ticksPerFrame = underlyingFrameRate !== null + ? timeResolution / underlyingFrameRate + : null; + + const histogram = new Map(); + + let minDifference = Infinity; + let maxDifference = -Infinity; + let totalDifference = 0; + + for (let i = 1; i < n; i++) { + const tickDifference = activeTicks[i]! - activeTicks[i - 1]!; + + const difference = ticksPerFrame !== null + ? Math.max(1, Math.round(tickDifference / ticksPerFrame)) + : tickDifference; + + histogram.set(difference, (histogram.get(difference) ?? 0) + 1); + + minDifference = Math.min(minDifference, difference); + maxDifference = Math.max(maxDifference, difference); + totalDifference += difference; + } + + const differenceCount = n - 1; + + // Get the two middle differences so the even-sized case has a true median. + const sortedDifferences = [...histogram.keys()].sort((a, b) => a - b); + const middleA = (differenceCount - 1) >> 1; + const middleB = differenceCount >> 1; + + let medianDifferenceA = 0; + let medianDifferenceB = 0; + let cumulativeCount = 0; + + for (const difference of sortedDifferences) { + cumulativeCount += histogram.get(difference)!; + + if (medianDifferenceA === 0 && cumulativeCount > middleA) { + medianDifferenceA = difference; + } + + if (cumulativeCount > middleB) { + medianDifferenceB = difference; + break; + } + } + + // Median is defined over instantaneous frame rates, not durations. + const medianFrameRate = ( + unitRate / medianDifferenceA + + unitRate / medianDifferenceB + ) / 2; + + return { + underlyingFrameRate, + bestGuessFrameRate: underlyingFrameRate !== null + ? underlyingFrameRate + : getBestGuessFrameRate(medianFrameRate), + minFrameRate: unitRate / maxDifference, + maxFrameRate: unitRate / minDifference, + averageFrameRate: unitRate * differenceCount / totalDifference, + medianFrameRate, + frameRateIsConstant: + underlyingFrameRate !== null + && minDifference === 1 + && maxDifference === 1, + probedPacketCount, + }; + } } export interface InputAudioTrackBacking extends InputTrackBacking { @@ -1139,3 +1354,267 @@ export const queryInputTracks = async ( }) .map(x => x.track); }; + +/** + * Estimates the underlying CFR-like frame rate from timestamp deltas. + * + * Each delta is modeled as an integer multiple of one frame period, allowing dropped frames and a small number of + * malformed timestamps. + */ +export const findUnderlyingFrameRate = ( + ticks: Float64Array, + resolution: number, +): number | null => { + const MAX_DENOMINATOR = 1_000_000; + const MIN_INLIER_RATIO = 0.98; + const DELTA_TOLERANCE = 1 + 1e-9; + const MAX_EFFECTIVE_FRAME_SPAN = 1000; + + const KNOWN_FRAME_RATES = [ + 12, + 15, + 20, + 24000 / 1001, + 24, + 25, + 30000 / 1001, + 30, + 48, + 50, + 60000 / 1001, + 60, + 100, + 120000 / 1001, + 120, + 144, + 240, + ]; + + if (ticks.length < 2) { + return null; + } + + const gaps = new Float64Array(ticks.length - 1); + + for (let i = 1; i < ticks.length; i++) { + const gap = ticks[i]! - ticks[i - 1]!; + + if (!(gap > 0)) { + return null; + } + + gaps[i - 1] = gap; + } + + // Start near the low end so dropped frames don't inflate the estimate, without letting one anomalously short gap + // determine it. + const sortedGaps = gaps.slice(); + sortedGaps.sort(); + + let period = sortedGaps[Math.floor(sortedGaps.length * 0.05)]!; + + // Repeatedly infer how many frame periods each gap spans and refine the underlying period from the locally + // consistent gaps. + for (let iteration = 0; iteration < 6; iteration++) { + let totalTicks = 0; + let totalFrames = 0; + + for (const gap of gaps) { + const multiple = Math.max(1, Math.round(gap / period)); + + if (Math.abs(gap - multiple * period) >= DELTA_TOLERANCE) { + continue; + } + + totalTicks += gap; + totalFrames += multiple; + } + + if (totalFrames === 0) { + return null; + } + + const refinedPeriod = totalTicks / totalFrames; + + if ( + Math.abs(refinedPeriod - period) + <= 1e-12 * Math.max(1, period) + ) { + period = refinedPeriod; + break; + } + + period = refinedPeriod; + } + + let inlierCount = 0; + let totalTicks = 0; + let totalFrames = 0; + + for (const gap of gaps) { + const multiple = Math.max(1, Math.round(gap / period)); + + if (Math.abs(gap - multiple * period) >= DELTA_TOLERANCE) { + continue; + } + + inlierCount++; + totalTicks += gap; + totalFrames += multiple; + } + + if (inlierCount / gaps.length < MIN_INLIER_RATIO) { + return null; + } + + period = totalTicks / totalFrames; + + // Don't let arbitrarily long files force arbitrarily precise rational reconstruction from microscopic clock drift. + const uncertainty = 1 / Math.min( + totalFrames, + MAX_EFFECTIVE_FRAME_SPAN, + ); + + const periodLo = Math.max(Number.EPSILON, period - uncertainty); + const periodHi = period + uncertainty; + + const fpsLo = resolution / periodHi; + const fpsHi = resolution / periodLo; + const fittedFps = resolution / period; + + // Prefer established frame rates when they're supported by the measured interval. If several fit, use the one + // closest to the measured cadence. + let fps: number | null = null; + let bestKnownError = Infinity; + + for (const candidate of KNOWN_FRAME_RATES) { + if (candidate < fpsLo || candidate > fpsHi) { + continue; + } + + const error = Math.abs(candidate / fittedFps - 1); + + if (error < bestKnownError) { + fps = candidate; + bestKnownError = error; + } + } + + if (fps === null) { + // Otherwise, the natural fraction may be simpler either as FPS or as ticks/frame. + const periodFraction = simplestFractionBetween( + periodLo, + periodHi, + MAX_DENOMINATOR, + ); + + const fpsFraction = simplestFractionBetween( + fpsLo, + fpsHi, + MAX_DENOMINATOR, + ); + + if ( + fpsFraction + && ( + !periodFraction + || fpsFraction.den < periodFraction.den + || ( + fpsFraction.den === periodFraction.den + && fpsFraction.num <= periodFraction.num + ) + ) + ) { + fps = fpsFraction.num / fpsFraction.den; + } else if (periodFraction) { + fps = resolution * periodFraction.den / periodFraction.num; + } else { + return null; + } + } + + // Make sure the chosen rate still explains the deltas. + const finalPeriod = resolution / fps; + let finalInlierCount = 0; + + for (const gap of gaps) { + const multiple = Math.max(1, Math.round(gap / finalPeriod)); + + if ( + Math.abs(gap - multiple * finalPeriod) + < DELTA_TOLERANCE + ) { + finalInlierCount++; + } + } + + if (finalInlierCount / gaps.length < MIN_INLIER_RATIO) { + return null; + } + + return fps; +}; + +const simplestFractionBetween = ( + lo: number, + hi: number, + maxDenominator: number, +): Rational | null => { + for (let den = 1; den <= maxDenominator; den++) { + const num = Math.floor(lo * den) + 1; + + if (num / den < hi) { + return simplifyRational({ num, den }); + } + } + + return null; +}; + +const getBestGuessFrameRate = (frameRate: number) => { + const SPECIAL_FRAME_RATES = [ + 24 / 1.001, + 30 / 1.001, + 60 / 1.001, + 120 / 1.001, + ]; + + const COMMON_FRAME_RATES = [ + 12, + 15, + 20, + 24, + 25, + 30, + 48, + 50, + 60, + 100, + 120, + 144, + 240, + ]; + + const SPECIAL_TOLERANCE = 0.0005; + const COMMON_TOLERANCE = 0.025; + + for (const candidate of SPECIAL_FRAME_RATES) { + if (Math.abs(candidate / frameRate - 1) <= SPECIAL_TOLERANCE) { + return candidate; + } + } + + let best = frameRate; + let bestError = Infinity; + + for (const candidate of COMMON_FRAME_RATES) { + const error = Math.abs(candidate / frameRate - 1); + + if (error <= COMMON_TOLERANCE && error < bestError) { + best = candidate; + bestError = error; + } + } + + return best; +}; diff --git a/test/node/frame-rate.test.ts b/test/node/frame-rate.test.ts new file mode 100644 index 0000000..e34e4be --- /dev/null +++ b/test/node/frame-rate.test.ts @@ -0,0 +1,164 @@ +import { expect, test } from 'vitest'; +import path from 'node:path'; +import { ALL_FORMATS, Input, FilePathSource, EncodedPacketSink } from '../../src/index.js'; +import { findUnderlyingFrameRate } from '../../src/input-track.js'; +import { assert } from '../../src/misc.js'; + +test('findUnderlyingFrameRate with 30 FPS at 30 Hz time resolution', () => { + const ticks = makeTicks(30, 30); + expect(findUnderlyingFrameRate(ticks, 30)).toBe(30); +}); + +test('findUnderlyingFrameRate with 30 FPS at 1000 Hz time resolution', () => { + expect(findUnderlyingFrameRate(makeTicks(30, 1000, { quantize: Math.floor }), 1000)).toBe(30); + expect(findUnderlyingFrameRate(makeTicks(30, 1000, { quantize: Math.round }), 1000)).toBe(30); + expect(findUnderlyingFrameRate(makeTicks(30, 1000, { quantize: Math.ceil }), 1000)).toBe(30); +}); + +test('findUnderlyingFrameRate with 30 FPS at 30 Hz with dropped frames', () => { + const ticks = makeTicks(30, 30, { dropFrame: dropSomeFrames }); + expect(findUnderlyingFrameRate(ticks, 30)).toBe(30); +}); + +test('findUnderlyingFrameRate with 30 FPS at 1000 Hz with dropped frames', () => { + expect(findUnderlyingFrameRate( + makeTicks(30, 1000, { quantize: Math.floor, dropFrame: dropSomeFrames }), + 1000, + )).toBe(30); + expect(findUnderlyingFrameRate( + makeTicks(30, 1000, { quantize: Math.round, dropFrame: dropSomeFrames }), + 1000, + )).toBe(30); + expect(findUnderlyingFrameRate( + makeTicks(30, 1000, { quantize: Math.ceil, dropFrame: dropSomeFrames }), + 1000, + )).toBe(30); +}); + +test('findUnderlyingFrameRate with 24/1.001 FPS at 24000 Hz time resolution', () => { + const ticks = makeTicks(24000 / 1001, 24000); + expect(findUnderlyingFrameRate(ticks, 24000)).toBe(24000 / 1001); +}); + +test('findUnderlyingFrameRate with 24/1.001 FPS at 1000 Hz time resolution (1 minute)', () => { + const frameCount = Math.floor(60 * 24000 / 1001); + const ticks = makeTicks(24000 / 1001, 1000, { frameCount }); + expect(findUnderlyingFrameRate(ticks, 1000)).toBe(24000 / 1001); +}); + +test('findUnderlyingFrameRate with 24/1.001 FPS at 1000 Hz time resolution (60 minutes)', () => { + const frameCount = Math.floor(3600 * 24000 / 1001); + const ticks = makeTicks(24000 / 1001, 1000, { frameCount }); + expect(findUnderlyingFrameRate(ticks, 1000)).toBe(24000 / 1001); +}); + +test('findUnderlyingFrameRate with 30 FPS at 1000 Hz starting at 1e9 seconds', () => { + const ticks = makeTicks(30, 1000, { startTime: 1e9 }); + expect(findUnderlyingFrameRate(ticks, 1000)).toBe(30); +}); + +test('findUnderlyingFrameRate with irregular timestamps yields null', () => { + const ticks = new Float64Array([0, 0.2, 0.5].map(x => Math.round(x * 100))); + expect(findUnderlyingFrameRate(ticks, 100)).toBe(null); +}); + +test('findUnderlyingFrameRate with cursed VFR timestamps yields null', async () => { + const { ticks, timeResolution } = await getSortedTrackTicks(publicPath('cursed-vfr.mp4')); + expect(findUnderlyingFrameRate(ticks, timeResolution)).toBe(null); +}); + +test('computeFrameRateMetrics with constant frame rate video', async () => { + using input = new Input({ + source: new FilePathSource(publicPath('video.mp4')), + formats: ALL_FORMATS, + }); + + const videoTrack = await input.getPrimaryVideoTrack(); + assert(videoTrack); + + const metrics = await videoTrack.computeFrameRateMetrics(); + + expect(metrics.underlyingFrameRate).toBe(25); + expect(metrics.bestGuessFrameRate).toBe(25); + expect(metrics.minFrameRate).toBe(25); + expect(metrics.maxFrameRate).toBe(25); + expect(metrics.averageFrameRate).toBe(25); + expect(metrics.medianFrameRate).toBe(25); + expect(metrics.frameRateIsConstant).toBe(true); +}); + +test('computeFrameRateMetrics with cursed VFR video', async () => { + using input = new Input({ + source: new FilePathSource(publicPath('cursed-vfr.mp4')), + formats: ALL_FORMATS, + }); + + const videoTrack = await input.getPrimaryVideoTrack(); + assert(videoTrack); + + const metrics = await videoTrack.computeFrameRateMetrics(); + + expect(metrics.underlyingFrameRate).toBe(null); + expect(metrics.bestGuessFrameRate).toBe(30); + expect(metrics.minFrameRate).toBe(0.6369426751592356); + expect(metrics.maxFrameRate).toBe(188.85245901639345); + expect(metrics.averageFrameRate).toBe(8.664659340979288); + expect(metrics.medianFrameRate).toBe(30.165016356826754); + expect(metrics.frameRateIsConstant).toBe(false); +}); + +const makeTicks = ( + frameRate: number, + timeResolution: number, + options: { + frameCount?: number; + startTime?: number; + quantize?: (value: number) => number; + dropFrame?: (index: number) => boolean; + } = {}, +) => { + const frameCount = options.frameCount ?? 300; + const startTime = options.startTime ?? 0; + const quantize = options.quantize ?? Math.round; + + const ticks: number[] = []; + + for (let i = 0; i < frameCount; i++) { + if (options.dropFrame?.(i)) { + continue; + } + + ticks.push(quantize((startTime + i / frameRate) * timeResolution)); + } + + return new Float64Array(ticks); +}; + +const dropSomeFrames = (index: number) => index % 11 === 4 || index % 17 === 9; + +const getSortedTrackTicks = async (filePath: string) => { + using input = new Input({ + source: new FilePathSource(filePath), + formats: ALL_FORMATS, + }); + + const videoTrack = await input.getPrimaryVideoTrack(); + assert(videoTrack); + + const timeResolution = await videoTrack.getTimeResolution(); + const sink = new EncodedPacketSink(videoTrack); + const ticks: number[] = []; + + for await (const packet of sink.packets(undefined, undefined, { metadataOnly: true })) { + ticks.push(Math.round(packet.timestamp * timeResolution)); + } + + ticks.sort((a, b) => a - b); + + const dedupedTicks = ticks.filter((tick, index) => index === 0 || tick !== ticks[index - 1]); + + return { ticks: new Float64Array(dedupedTicks), timeResolution }; +}; + +const __dirname = new URL('.', import.meta.url).pathname; +const publicPath = (file: string) => path.join(__dirname, '../public', file); diff --git a/test/public/cursed-vfr.mp4 b/test/public/cursed-vfr.mp4 new file mode 100644 index 0000000..d9584ac Binary files /dev/null and b/test/public/cursed-vfr.mp4 differ