diff --git a/dev/demux.html b/dev/demux.html index e55f89a..f4d059d 100644 --- a/dev/demux.html +++ b/dev/demux.html @@ -45,11 +45,11 @@ a.download = filename; a.click(); URL.revokeObjectURL(url); - } - + } + console.log(output.target.buffer) - download(new Blob([output.target.buffer]), 'converted.mp3'); - + download(new Blob([output.target.buffer]), 'converted.mp3'); + /* const videoTrack = await input.getPrimaryVideoTrack(); @@ -184,7 +184,7 @@ } */ - + /* const videoTrack = await input.getPrimaryVideoTrack(); @@ -245,7 +245,7 @@ */ //chunks.return(); - + //console.log("Done") /* @@ -441,31 +441,31 @@ const drain = new Metamuxer.VideoFrameDrain(videoTrack); for await (const frame of drain.frames()) { - // ... - } + // ... + } console.log(await videoTrack.getDuration()); */ /* - const drain = new Metamuxer.VideoFrameDrain(videoTrack); + const drain = new Metamuxer.VideoFrameDrain(videoTrack); - const width = await videoTrack.getWidth(); - const height = await videoTrack.getHeight(); + const width = await videoTrack.getWidth(); + const height = await videoTrack.getHeight(); - const canvas = document.createElement('canvas'); - canvas.width = width; - canvas.height = height; - document.body.append(canvas); + const canvas = document.createElement('canvas'); + canvas.width = width; + canvas.height = height; + document.body.append(canvas); - const context = canvas.getContext('2d'); + const context = canvas.getContext('2d'); - // Top-level for await loop inside the event listener - for await (const frame of drain.frames()) { - context.drawImage(frame, 0, 0, width, height); - frame.close(); - } + // Top-level for await loop inside the event listener + for await (const frame of drain.frames()) { + context.drawImage(frame, 0, 0, width, height); + frame.close(); + } */ /* diff --git a/dev/mux.html b/dev/mux.html index 55a3558..8acf0c2 100644 --- a/dev/mux.html +++ b/dev/mux.html @@ -111,7 +111,7 @@ Example entry 1: Hello world. `; -if (false) simpleWebvttFile = + if (false) simpleWebvttFile = `WEBVTT 1 @@ -127,7 +127,7 @@ We are looking straight down 5th Avenue. Testing... <00:17.350>One... <00:18.125>Two... ` - simpleWebvttFile = + simpleWebvttFile = `WEBVTT 00:00:01.000 --> 00:00:01.500 align:start line:1 @@ -178,5 +178,5 @@ Testing... <00:17.350>One... <00:18.125>Two... await output.finalize(); console.log(target); - download(new Blob([target.buffer]), 'test' + format.fileExtension); + //download(new Blob([target.buffer]), 'test' + format.fileExtension); \ No newline at end of file diff --git a/docs/guide/reading-overview.md b/docs/guide/reading-overview.md index 5cc76c6..be22a4b 100644 --- a/docs/guide/reading-overview.md +++ b/docs/guide/reading-overview.md @@ -94,7 +94,7 @@ track.type; // => 'video' | 'audio' | 'subtitle'; track.isVideoTrack(); // => boolean track.isAudioTrack(); // => boolean -// Retrieve the track's language as an ISO 639-2 language code. +// Retrieve the track's language as an ISO 639-2/T language code. // Resolves to 'und' (undetermined) if the language isn't known. track.languageCode; // => string ``` diff --git a/docs/guide/writing-overview.md b/docs/guide/writing-overview.md index f40fe22..fff546e 100644 --- a/docs/guide/writing-overview.md +++ b/docs/guide/writing-overview.md @@ -42,11 +42,13 @@ For each track you want to add, you'll need to create a unique [media source](./ Optionally, you can specify additional track metadata when adding tracks: ```ts -// This specifies that the video track should be rotated by 90 degrees clockwise -// before being displayed by video players, and that a frame rate of 30 FPS is -// expected. +// This specifies that the video track should be rotated by 90 degrees +// clockwise before being displayed by video players, and that a frame rate +// of 30 FPS is expected. output.addVideoTrack(videoSource, { - rotation: 90, // Clockwise rotation in degrees + // Clockwise rotation in degrees + rotation: 90, + // Expected frame rate in hertz frameRate: 30, }); @@ -66,6 +68,15 @@ output.addSubtitleTrack(subtitleSourceFre, { language: 'fre' }); output.addSubtitleTrack(subtitleSourceIta, { language: 'ita' }); ``` +::: info +The optional `frameRate` video track metadata option specifies the expected frame rate of the video. All timestamps and durations of frames that will be added to this track will be snapped to the specified frame rate. You should avoid adding frames more often than the rate permits, as this will lead to multiple frames having the same timestamp. + +To precisely achieve common fractional frame rates, make sure to use their exact fractional forms: +$23.976 \rightarrow 24000/1001$\ +$29.97 \rightarrow 30000/1001$\ +$59.94 \rightarrow 60000/1001$ +::: + As an example, let's add two tracks to our output: - A video track driven by the contents of a `` element, encoded using AVC - An audio track driven by the user's microphone input, encoded using AAC diff --git a/docs/index.md b/docs/index.md index d528b38..85e049c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -35,5 +35,4 @@ things the guide needs to cover: - utility functions - supported containers / codecs - samples & packets -- conversion -- do a part that "frameRate" must be an integer. and make sure to document numbers that can only be integer in the docblocks \ No newline at end of file +- conversion \ No newline at end of file diff --git a/src/input-track.ts b/src/input-track.ts index 009f45f..5ddf534 100644 --- a/src/input-track.ts +++ b/src/input-track.ts @@ -70,7 +70,7 @@ export abstract class InputTrack { return this._backing.getId(); } - /** The ISO 639-2 language code for this track. If the language is unknown, this field is 'und' (undetermined). */ + /** The ISO 639-2/T language code for this track. If the language is unknown, this field is 'und' (undetermined). */ get languageCode() { return this._backing.getLanguageCode(); } diff --git a/src/isobmff/isobmff-muxer.ts b/src/isobmff/isobmff-muxer.ts index b7068b9..b5c76fc 100644 --- a/src/isobmff/isobmff-muxer.ts +++ b/src/isobmff/isobmff-muxer.ts @@ -2,7 +2,7 @@ import { Box, ftyp, IsobmffBoxWriter, mdat, mfra, moof, moov, vtta, vttc, vtte } import { Muxer } from '../muxer'; import { Output, OutputAudioTrack, OutputSubtitleTrack, OutputTrack, OutputVideoTrack } from '../output'; import { BufferTargetWriter, Writer } from '../writer'; -import { assert, last } from '../misc'; +import { assert, computeRationalApproximation, last } from '../misc'; import { IsobmffOutputFormatOptions, IsobmffOutputFormat, MovOutputFormat } from '../output-format'; import { inlineTimestampRegex, SubtitleConfig, SubtitleCue, SubtitleMetadata } from '../subtitles'; import { @@ -196,6 +196,11 @@ export class IsobmffMuxer extends Muxer { assert(meta.decoderConfig.codedWidth !== undefined); assert(meta.decoderConfig.codedHeight !== undefined); + // The frame rate set by the user may not be an integer. Since timescale is an integer, we'll approximate the + // frame time (inverse of frame rate) with a rational number, then use that approximation's denominator + // as the timescale. + const timescale = computeRationalApproximation(1 / (track.metadata.frameRate ?? 57600), 1e6).denominator; + const newTrackData: IsobmffVideoTrackData = { track, type: 'video', @@ -204,7 +209,7 @@ export class IsobmffMuxer extends Muxer { height: meta.decoderConfig.codedHeight, decoderConfig: meta.decoderConfig, }, - timescale: track.metadata.frameRate ?? 57600, + timescale, samples: [], sampleQueue: [], timestampProcessingQueue: [], diff --git a/src/matroska/matroska-muxer.ts b/src/matroska/matroska-muxer.ts index 74cb2be..60d231e 100644 --- a/src/matroska/matroska-muxer.ts +++ b/src/matroska/matroska-muxer.ts @@ -7,6 +7,7 @@ import { colorSpaceIsComplete, normalizeRotation, readBits, + roundToMultiple, textEncoder, toUint8Array, writeBits, @@ -510,8 +511,16 @@ export class MatroskaMuxer extends Muxer { const trackData = this.getVideoTrackData(track, meta); const isKeyFrame = packet.type === 'key'; - const timestamp = this.validateAndNormalizeTimestamp(trackData.track, packet.timestamp, isKeyFrame); - const videoChunk = this.createInternalChunk(packet.data, timestamp, packet.duration, packet.type); + let timestamp = this.validateAndNormalizeTimestamp(trackData.track, packet.timestamp, isKeyFrame); + let duration = packet.duration; + + if (track.metadata.frameRate !== undefined) { + // Constrain the time values to the frame rate + timestamp = roundToMultiple(timestamp, 1 / track.metadata.frameRate); + duration = roundToMultiple(duration, 1 / track.metadata.frameRate); + } + + const videoChunk = this.createInternalChunk(packet.data, timestamp, duration, packet.type); if (track.source._codec === 'vp9') this.fixVP9ColorSpace(trackData, videoChunk); trackData.chunkQueue.push(videoChunk); diff --git a/src/misc.ts b/src/misc.ts index 117fbcc..cb9c75a 100644 --- a/src/misc.ts +++ b/src/misc.ts @@ -450,3 +450,47 @@ export const retriedFetch = async ( } } }; + +export const computeRationalApproximation = (x: number, maxDenominator: number) => { + // Handle negative numbers + const sign = x < 0 ? -1 : 1; + x = Math.abs(x); + + let prevNumerator = 0, prevDenominator = 1; + let currNumerator = 1, currDenominator = 0; + + // Continued fraction algorithm + let remainder = x; + + while (true) { + const integer = Math.floor(remainder); + + // Calculate next convergent + const nextNumerator = integer * currNumerator + prevNumerator; + const nextDenominator = integer * currDenominator + prevDenominator; + + if (nextDenominator > maxDenominator) { + return { + numerator: sign * currNumerator, + denominator: currDenominator, + }; + } + + prevNumerator = currNumerator; + prevDenominator = currDenominator; + currNumerator = nextNumerator; + currDenominator = nextDenominator; + + remainder = 1 / (remainder - integer); + + // Guard against precision issues + if (!isFinite(remainder)) { + break; + } + } + + return { + numerator: sign * currNumerator, + denominator: currDenominator, + }; +}; diff --git a/src/output.ts b/src/output.ts index 418ae16..7538b56 100644 --- a/src/output.ts +++ b/src/output.ts @@ -57,7 +57,7 @@ export type OutputSubtitleTrack = OutputTrack & { type: 'subtitle' }; * @public */ export type BaseTrackMetadata = { - /** The three-letter, ISO 639-2 language code specifying the language of this track. */ + /** The three-letter, ISO 639-2/T language code specifying the language of this track. */ languageCode?: string; }; @@ -68,7 +68,11 @@ export type BaseTrackMetadata = { export type VideoTrackMetadata = BaseTrackMetadata & { /** The angle in degrees by which the track's frames should be rotated (clockwise). */ rotation?: Rotation; - /** The expected video frame rate. You should not exceed the value you set here. */ + /** + * The expected video frame rate in hertz. If set, all timestamps and durations of this track will be snapped to + * this frame rate. You should avoid adding more frames than the rate allows, as this will lead to multiple frames + * with the same timestamp. + */ frameRate?: number; }; /** @@ -87,7 +91,7 @@ const validateBaseTrackMetadata = (metadata: BaseTrackMetadata) => { throw new TypeError('metadata must be an object.'); } if (metadata.languageCode !== undefined && !isIso639Dash2LanguageCode(metadata.languageCode)) { - throw new TypeError('metadata.languageCode must be a three-letter, ISO 639-2 language code.'); + throw new TypeError('metadata.languageCode must be a three-letter, ISO 639-2/T language code.'); } }; @@ -158,10 +162,10 @@ export class Output< } if ( metadata.frameRate !== undefined - && (!Number.isInteger(metadata.frameRate) || metadata.frameRate <= 0) + && (!Number.isFinite(metadata.frameRate) || metadata.frameRate <= 0) ) { throw new TypeError( - `Invalid video frame rate: ${metadata.frameRate}. Must be a positive integer.`, + `Invalid video frame rate: ${metadata.frameRate}. Must be a positive number.`, ); }