Allow fractional frame rates in track metadata, document frameRate better, other small doc adjustments

dev/demux.html

@@ -45,11 +45,11 @@
 			a.download = filename;
 			a.click();
 			URL.revokeObjectURL(url);
-		}	
-		
+		}
+
 		console.log(output.target.buffer)
-		download(new Blob([output.target.buffer]), 'converted.mp3');	
-	
+		download(new Blob([output.target.buffer]), 'converted.mp3');
+
 
 		/*
 		const videoTrack = await input.getPrimaryVideoTrack();
@@ -184,7 +184,7 @@
 		}
 		*/
 
-		
+
 
 		/*
 		const videoTrack = await input.getPrimaryVideoTrack();
@@ -245,7 +245,7 @@
 		*/
 
 		//chunks.return();
-		
+
 		//console.log("Done")
 
 		/*
@@ -441,31 +441,31 @@
 		const drain = new Metamuxer.VideoFrameDrain(videoTrack);
 
 		for await (const frame of drain.frames()) {
-            // ...
-        }
+			// ...
+		}
 
 		console.log(await videoTrack.getDuration());
 		*/
 
 		/*
-        const drain = new Metamuxer.VideoFrameDrain(videoTrack);
+		const drain = new Metamuxer.VideoFrameDrain(videoTrack);
 
 		
-        const width = await videoTrack.getWidth();
-        const height = await videoTrack.getHeight();
+		const width = await videoTrack.getWidth();
+		const height = await videoTrack.getHeight();
 
-        const canvas = document.createElement('canvas');
-        canvas.width = width;
-        canvas.height = height;
-        document.body.append(canvas);
+		const canvas = document.createElement('canvas');
+		canvas.width = width;
+		canvas.height = height;
+		document.body.append(canvas);
 
-        const context = canvas.getContext('2d');
+		const context = canvas.getContext('2d');
 
-        // Top-level for await loop inside the event listener
-        for await (const frame of drain.frames()) {
-            context.drawImage(frame, 0, 0, width, height);
-            frame.close();
-        }
+		// Top-level for await loop inside the event listener
+		for await (const frame of drain.frames()) {
+			context.drawImage(frame, 0, 0, width, height);
+			frame.close();
+		}
 		*/
 
 		/*

dev/mux.html

@@ -111,7 +111,7 @@
 Example entry 1: Hello <b>world</b>.
 `;
 
-if (false) simpleWebvttFile = 
+	if (false) simpleWebvttFile =
 `WEBVTT
 
 1
@@ -127,7 +127,7 @@ We are looking straight down 5th Avenue.
 Testing... <00:17.350>One... <00:18.125>Two...
 `
 
-	simpleWebvttFile = 
+	simpleWebvttFile =
 `WEBVTT
 
 00:00:01.000 --> 00:00:01.500 align:start line:1
@@ -178,5 +178,5 @@ Testing... <00:17.350>One... <00:18.125>Two...
 	await output.finalize();
 
 	console.log(target);
-	download(new Blob([target.buffer]), 'test' + format.fileExtension);
+	//download(new Blob([target.buffer]), 'test' + format.fileExtension);
 </script>

docs/guide/reading-overview.md

@@ -94,7 +94,7 @@ track.type; // => 'video' | 'audio' | 'subtitle';
 track.isVideoTrack(); // => boolean
 track.isAudioTrack(); // => boolean
 
-// Retrieve the track's language as an ISO 639-2 language code.
+// Retrieve the track's language as an ISO 639-2/T language code.
 // Resolves to 'und' (undetermined) if the language isn't known.
 track.languageCode; // => string
 ```

docs/guide/writing-overview.md

@@ -42,11 +42,13 @@ For each track you want to add, you'll need to create a unique [media source](./
 
 Optionally, you can specify additional track metadata when adding tracks:
 ```ts
-// This specifies that the video track should be rotated by 90 degrees clockwise
-// before being displayed by video players, and that a frame rate of 30 FPS is
-// expected.
+// This specifies that the video track should be rotated by 90 degrees
+// clockwise before being displayed by video players, and that a frame rate
+// of 30 FPS is expected.
 output.addVideoTrack(videoSource, {
-	rotation: 90, // Clockwise rotation in degrees
+	// Clockwise rotation in degrees
+	rotation: 90,
+	// Expected frame rate in hertz
 	frameRate: 30,
 });
 
@@ -66,6 +68,15 @@ output.addSubtitleTrack(subtitleSourceFre, { language: 'fre' });
 output.addSubtitleTrack(subtitleSourceIta, { language: 'ita' });
 ```
 
+::: info
+The optional `frameRate` video track metadata option specifies the expected frame rate of the video. All timestamps and durations of frames that will be added to this track will be snapped to the specified frame rate. You should avoid adding frames more often than the rate permits, as this will lead to multiple frames having the same timestamp.
+
+To precisely achieve common fractional frame rates, make sure to use their exact fractional forms:
+$23.976 \rightarrow 24000/1001$\
+$29.97 \rightarrow 30000/1001$\
+$59.94 \rightarrow 60000/1001$
+:::
+
 As an example, let's add two tracks to our output:
 - A video track driven by the contents of a `<canvas>` element, encoded using AVC
 - An audio track driven by the user's microphone input, encoded using AAC

docs/index.md

@@ -35,5 +35,4 @@ things the guide needs to cover:
 - utility functions
 - supported containers / codecs
 - samples & packets
-- conversion
-- do a part that "frameRate" must be an integer. and make sure to document numbers that can only be integer in the docblocks
+- conversion

src/input-track.ts

@@ -70,7 +70,7 @@ export abstract class InputTrack {
 		return this._backing.getId();
 	}
 
-	/** The ISO 639-2 language code for this track. If the language is unknown, this field is 'und' (undetermined). */
+	/** The ISO 639-2/T language code for this track. If the language is unknown, this field is 'und' (undetermined). */
 	get languageCode() {
 		return this._backing.getLanguageCode();
 	}

src/isobmff/isobmff-muxer.ts

@@ -2,7 +2,7 @@ import { Box, ftyp, IsobmffBoxWriter, mdat, mfra, moof, moov, vtta, vttc, vtte }
 import { Muxer } from '../muxer';
 import { Output, OutputAudioTrack, OutputSubtitleTrack, OutputTrack, OutputVideoTrack } from '../output';
 import { BufferTargetWriter, Writer } from '../writer';
-import { assert, last } from '../misc';
+import { assert, computeRationalApproximation, last } from '../misc';
 import { IsobmffOutputFormatOptions, IsobmffOutputFormat, MovOutputFormat } from '../output-format';
 import { inlineTimestampRegex, SubtitleConfig, SubtitleCue, SubtitleMetadata } from '../subtitles';
 import {
@@ -196,6 +196,11 @@ export class IsobmffMuxer extends Muxer {
 		assert(meta.decoderConfig.codedWidth !== undefined);
 		assert(meta.decoderConfig.codedHeight !== undefined);
 
+		// The frame rate set by the user may not be an integer. Since timescale is an integer, we'll approximate the
+		// frame time (inverse of frame rate) with a rational number, then use that approximation's denominator
+		// as the timescale.
+		const timescale = computeRationalApproximation(1 / (track.metadata.frameRate ?? 57600), 1e6).denominator;
+
 		const newTrackData: IsobmffVideoTrackData = {
 			track,
 			type: 'video',
@@ -204,7 +209,7 @@ export class IsobmffMuxer extends Muxer {
 				height: meta.decoderConfig.codedHeight,
 				decoderConfig: meta.decoderConfig,
 			},
-			timescale: track.metadata.frameRate ?? 57600,
+			timescale,
 			samples: [],
 			sampleQueue: [],
 			timestampProcessingQueue: [],

src/matroska/matroska-muxer.ts

@@ -7,6 +7,7 @@ import {
 	colorSpaceIsComplete,
 	normalizeRotation,
 	readBits,
+	roundToMultiple,
 	textEncoder,
 	toUint8Array,
 	writeBits,
@@ -510,8 +511,16 @@ export class MatroskaMuxer extends Muxer {
 			const trackData = this.getVideoTrackData(track, meta);
 
 			const isKeyFrame = packet.type === 'key';
-			const timestamp = this.validateAndNormalizeTimestamp(trackData.track, packet.timestamp, isKeyFrame);
-			const videoChunk = this.createInternalChunk(packet.data, timestamp, packet.duration, packet.type);
+			let timestamp = this.validateAndNormalizeTimestamp(trackData.track, packet.timestamp, isKeyFrame);
+			let duration = packet.duration;
+
+			if (track.metadata.frameRate !== undefined) {
+				// Constrain the time values to the frame rate
+				timestamp = roundToMultiple(timestamp, 1 / track.metadata.frameRate);
+				duration = roundToMultiple(duration, 1 / track.metadata.frameRate);
+			}
+
+			const videoChunk = this.createInternalChunk(packet.data, timestamp, duration, packet.type);
 			if (track.source._codec === 'vp9') this.fixVP9ColorSpace(trackData, videoChunk);
 
 			trackData.chunkQueue.push(videoChunk);

src/misc.ts

@@ -450,3 +450,47 @@ export const retriedFetch = async (
 		}
 	}
 };
+
+export const computeRationalApproximation = (x: number, maxDenominator: number) => {
+	// Handle negative numbers
+	const sign = x < 0 ? -1 : 1;
+	x = Math.abs(x);
+
+	let prevNumerator = 0, prevDenominator = 1;
+	let currNumerator = 1, currDenominator = 0;
+
+	// Continued fraction algorithm
+	let remainder = x;
+
+	while (true) {
+		const integer = Math.floor(remainder);
+
+		// Calculate next convergent
+		const nextNumerator = integer * currNumerator + prevNumerator;
+		const nextDenominator = integer * currDenominator + prevDenominator;
+
+		if (nextDenominator > maxDenominator) {
+			return {
+				numerator: sign * currNumerator,
+				denominator: currDenominator,
+			};
+		}
+
+		prevNumerator = currNumerator;
+		prevDenominator = currDenominator;
+		currNumerator = nextNumerator;
+		currDenominator = nextDenominator;
+
+		remainder = 1 / (remainder - integer);
+
+		// Guard against precision issues
+		if (!isFinite(remainder)) {
+			break;
+		}
+	}
+
+	return {
+		numerator: sign * currNumerator,
+		denominator: currDenominator,
+	};
+};

src/output.ts

@@ -57,7 +57,7 @@ export type OutputSubtitleTrack = OutputTrack & { type: 'subtitle' };
  * @public
  */
 export type BaseTrackMetadata = {
-	/** The three-letter, ISO 639-2 language code specifying the language of this track. */
+	/** The three-letter, ISO 639-2/T language code specifying the language of this track. */
 	languageCode?: string;
 };
 
@@ -68,7 +68,11 @@ export type BaseTrackMetadata = {
 export type VideoTrackMetadata = BaseTrackMetadata & {
 	/** The angle in degrees by which the track's frames should be rotated (clockwise). */
 	rotation?: Rotation;
-	/** The expected video frame rate. You should not exceed the value you set here. */
+	/**
+	 * The expected video frame rate in hertz. If set, all timestamps and durations of this track will be snapped to
+	 * this frame rate. You should avoid adding more frames than the rate allows, as this will lead to multiple frames
+	 * with the same timestamp.
+	 */
 	frameRate?: number;
 };
 /**
@@ -87,7 +91,7 @@ const validateBaseTrackMetadata = (metadata: BaseTrackMetadata) => {
 		throw new TypeError('metadata must be an object.');
 	}
 	if (metadata.languageCode !== undefined && !isIso639Dash2LanguageCode(metadata.languageCode)) {
-		throw new TypeError('metadata.languageCode must be a three-letter, ISO 639-2 language code.');
+		throw new TypeError('metadata.languageCode must be a three-letter, ISO 639-2/T language code.');
 	}
 };
 
@@ -158,10 +162,10 @@ export class Output<
 		}
 		if (
 			metadata.frameRate !== undefined
-			&& (!Number.isInteger(metadata.frameRate) || metadata.frameRate <= 0)
+			&& (!Number.isFinite(metadata.frameRate) || metadata.frameRate <= 0)
 		) {
 			throw new TypeError(
-				`Invalid video frame rate: ${metadata.frameRate}. Must be a positive integer.`,
+				`Invalid video frame rate: ${metadata.frameRate}. Must be a positive number.`,
 			);
 		}