diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0312605..6556ff8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,6 +15,16 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Merge main into release branch + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git checkout release + git merge origin/main --no-ff -m "Merge main into release for tag ${{ github.event.release.tag_name }}" + git push origin release - name: Set up Node.js uses: actions/setup-node@v4 diff --git a/dev/convert.html b/dev/convert.html index 0bcba3a..38b9d23 100644 --- a/dev/convert.html +++ b/dev/convert.html @@ -70,8 +70,10 @@ }, */ video: { - forceTranscode: true, - codec: 'av1', + frameRate: 27.123, + //width: 320, + //forceTranscode: true, + //codec: 'av1', //discard: true, //width: 1280, //discard: true, diff --git a/dev/demux.html b/dev/demux.html index 406c017..df38943 100644 --- a/dev/demux.html +++ b/dev/demux.html @@ -16,6 +16,28 @@ source }); + const audioTrack = await input.getPrimaryAudioTrack(); + const sink = new Mediabunny.EncodedPacketSink(audioTrack); + + let thing = await sink.getFirstPacket(); + while (thing) { + console.log(thing); + if (thing.timestamp >= 2.4) break; + + thing = await sink.getNextPacket(thing); + + } + console.log("done") + + /* + for await (const packet of sink.packets()) { + console.log(packet); + + if (packet.timestamp >= 2.4) break; + } + */ + + /* const videoTrack = await input.getPrimaryVideoTrack(); const packetSink = new Mediabunny.EncodedPacketSink(videoTrack); const sampleSink = new Mediabunny.VideoSampleSink(videoTrack); @@ -29,6 +51,7 @@ } } console.log("don") + */ /* console.time() diff --git a/docs/guide/converting-media-files.md b/docs/guide/converting-media-files.md index b3547cd..69c9e51 100644 --- a/docs/guide/converting-media-files.md +++ b/docs/guide/converting-media-files.md @@ -11,6 +11,7 @@ It has the following features: - Trimming - Video resizing & fitting - Video rotation +- Video frame rate adjustment - Audio resampling - Audio up/downmixing @@ -101,6 +102,7 @@ type ConversionOptions = { height?: number; fit?: 'fill' | 'contain' | 'cover'; rotate?: 0 | 90 | 180 | 270; + frameRate?: number; codec?: VideoCodec; bitrate?: number | Quality; forceTranscode?: boolean; @@ -141,6 +143,10 @@ The `width`, `height` and `fit` properties control how the video is resized. If If `width` or `height` is used in conjunction with `rotation`, they control the post-rotation dimensions. +### Adjusting frame rate + +The `frameRate` property can be used to set the frame rate of the output video in Hz. If not specified, the original input frame rate will be used (which may be variable). + ### Transcoding video Use the `codec` property to control the codec of the output track. This should be set to a [codec](./supported-formats-and-codecs#video-codecs) supported by the output file, or else the track will be [discarded](#discarded-tracks). diff --git a/package.json b/package.json index 72a0ee1..34b623a 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "mediabunny", "author": "Vanilagy", - "version": "1.4.4", + "version": "1.5.0", "description": "Pure TypeScript media toolkit for reading, writing, and converting media files, directly in the browser.", "type": "module", "main": "./dist/bundles/mediabunny.cjs", diff --git a/src/conversion.ts b/src/conversion.ts index 4079759..7f4e156 100644 --- a/src/conversion.ts +++ b/src/conversion.ts @@ -77,6 +77,11 @@ export type ConversionOptions = { * rotation is _in addition to_ the natural rotation of the input video as specified in input file's metadata. */ rotate?: Rotation; + /** + * The desired frame rate of the output video, in hertz. If not specified, the original input frame rate will + * be used (which may be variable). + */ + frameRate?: number; /** The desired output video codec. */ codec?: VideoCodec; /** The desired bitrate of the output video. */ @@ -261,8 +266,14 @@ export class Conversion { if (options.video?.rotate !== undefined && ![0, 90, 180, 270].includes(options.video.rotate)) { throw new TypeError('options.video.rotate, when provided, must be 0, 90, 180 or 270.'); } + if ( + options.video?.frameRate !== undefined + && (!Number.isFinite(options.video.frameRate) || options.video.frameRate <= 0) + ) { + throw new TypeError('options.video.frameRate, when provided, must be a finite positive number.'); + } if (options.audio !== undefined && (!options.audio || typeof options.audio !== 'object')) { - throw new TypeError('options.video, when provided, must be an object.'); + throw new TypeError('options.audio, when provided, must be an object.'); } if (options.audio?.discard !== undefined && typeof options.audio.discard !== 'boolean') { throw new TypeError('options.audio.discard, when provided, must be a boolean.'); @@ -470,7 +481,10 @@ export class Conversion { } const firstTimestamp = await track.getFirstTimestamp(); - const needsTranscode = !!this._options.video?.forceTranscode || this._startTimestamp > 0 || firstTimestamp < 0; + const needsTranscode = !!this._options.video?.forceTranscode + || this._startTimestamp > 0 + || firstTimestamp < 0 + || !!this._options.video?.frameRate; const needsRerender = width !== originalWidth || height !== originalHeight || (totalRotation !== 0 && !outputSupportsRotation); @@ -547,10 +561,10 @@ export class Conversion { onEncodedPacket: sample => this._reportProgress(track.id, sample.timestamp + sample.duration), }; - if (needsRerender) { - const source = new VideoSampleSource(encodingConfig); - videoSource = source; + const source = new VideoSampleSource(encodingConfig); + videoSource = source; + if (needsRerender) { this._trackPromises.push((async () => { await this._started; @@ -562,6 +576,27 @@ export class Conversion { poolSize: 1, }); const iterator = sink.canvases(this._startTimestamp, this._endTimestamp); + const frameRate = this._options.video?.frameRate; + + let lastCanvas: HTMLCanvasElement | OffscreenCanvas | null = null; + let lastCanvasTimestamp: number | null = null; + let lastCanvasEndTimestamp: number | null = null; + + /** Repeats the last sample to pad out the time until the specified timestamp. */ + const padFrames = async (until: number) => { + assert(lastCanvas); + assert(frameRate !== undefined); + + const frameDifference = Math.round((until - lastCanvasTimestamp!) * frameRate); + + for (let i = 1; i < frameDifference; i++) { + const sample = new VideoSample(lastCanvas, { + timestamp: lastCanvasTimestamp! + i / frameRate, + duration: 1 / frameRate, + }); + await source.add(sample); + } + }; for await (const { canvas, timestamp, duration } of iterator) { if (this._synchronizer.shouldWait(track.id, timestamp)) { @@ -572,37 +607,134 @@ export class Conversion { return; } + let adjustedSampleTimestamp = Math.max(timestamp - this._startTimestamp, 0); + lastCanvasEndTimestamp = timestamp + duration; + + if (frameRate !== undefined) { + // Logic for skipping/repeating frames when a frame rate is set + const alignedTimestamp = Math.floor(adjustedSampleTimestamp * frameRate) / frameRate; + + if (lastCanvas !== null) { + if (alignedTimestamp <= lastCanvasTimestamp!) { + lastCanvas = canvas; + lastCanvasTimestamp = alignedTimestamp; + + // Skip this sample, since we already added one for this frame + continue; + } else { + // Check if we may need to repeat the previous frame + await padFrames(alignedTimestamp); + } + } + + adjustedSampleTimestamp = alignedTimestamp; + } + const sample = new VideoSample(canvas, { - timestamp: Math.max(timestamp - this._startTimestamp, 0), - duration, + timestamp: adjustedSampleTimestamp, + duration: frameRate !== undefined ? 1 / frameRate : duration, }); await source.add(sample); - sample.close(); + + if (frameRate !== undefined) { + lastCanvas = canvas; + lastCanvasTimestamp = adjustedSampleTimestamp; + } else { + sample.close(); + } } + + if (lastCanvas) { + assert(lastCanvasEndTimestamp !== null); + assert(frameRate !== undefined); + + // If necessary, pad until the end timestamp of the last sample + await padFrames(Math.floor(lastCanvasEndTimestamp * frameRate) / frameRate); + } + + source.close(); + this._synchronizer.closeTrack(track.id); })()); } else { - const source = new VideoSampleSource(encodingConfig); - videoSource = source; - this._trackPromises.push((async () => { await this._started; const sink = new VideoSampleSink(track); + const frameRate = this._options.video?.frameRate; + + let lastSample: VideoSample | null = null; + let lastSampleTimestamp: number | null = null; + let lastSampleEndTimestamp: number | null = null; + + /** Repeats the last sample to pad out the time until the specified timestamp. */ + const padFrames = async (until: number) => { + assert(lastSample); + assert(frameRate !== undefined); + + const frameDifference = Math.round((until - lastSampleTimestamp!) * frameRate); + + for (let i = 1; i < frameDifference; i++) { + lastSample.setTimestamp(lastSampleTimestamp! + i / frameRate); + lastSample.setDuration(1 / frameRate); + await source.add(lastSample); + } + + lastSample.close(); + }; for await (const sample of sink.samples(this._startTimestamp, this._endTimestamp)) { if (this._synchronizer.shouldWait(track.id, sample.timestamp)) { await this._synchronizer.wait(sample.timestamp); } - sample.setTimestamp(Math.max(sample.timestamp - this._startTimestamp, 0)); - if (this._canceled) { + lastSample?.close(); return; } + let adjustedSampleTimestamp = Math.max(sample.timestamp - this._startTimestamp, 0); + lastSampleEndTimestamp = sample.timestamp + sample.duration; + + if (frameRate !== undefined) { + // Logic for skipping/repeating frames when a frame rate is set + const alignedTimestamp = Math.floor(adjustedSampleTimestamp * frameRate) / frameRate; + + if (lastSample !== null) { + if (alignedTimestamp <= lastSampleTimestamp!) { + lastSample.close(); + lastSample = sample; + lastSampleTimestamp = alignedTimestamp; + + // Skip this sample, since we already added one for this frame + continue; + } else { + // Check if we may need to repeat the previous frame + await padFrames(alignedTimestamp); + } + } + + adjustedSampleTimestamp = alignedTimestamp; + sample.setDuration(1 / frameRate); + } + + sample.setTimestamp(adjustedSampleTimestamp); await source.add(sample); - sample.close(); + + if (frameRate !== undefined) { + lastSample = sample; + lastSampleTimestamp = adjustedSampleTimestamp; + } else { + sample.close(); + } + } + + if (lastSample) { + assert(lastSampleEndTimestamp !== null); + assert(frameRate !== undefined); + + // If necessary, pad until the end timestamp of the last sample + await padFrames(Math.floor(lastSampleEndTimestamp * frameRate) / frameRate); } source.close(); @@ -612,6 +744,7 @@ export class Conversion { } this.output.addVideoTrack(videoSource, { + frameRate: this._options.video?.frameRate, languageCode: track.languageCode, rotation: needsRerender ? 0 : totalRotation, // Rerendering will bake the rotation into the output }); diff --git a/src/isobmff/isobmff-demuxer.ts b/src/isobmff/isobmff-demuxer.ts index a3c3379..c0b3a2d 100644 --- a/src/isobmff/isobmff-demuxer.ts +++ b/src/isobmff/isobmff-demuxer.ts @@ -2024,7 +2024,6 @@ abstract class IsobmffTrackBacking implements InputTrackBacking { } const trackData = locationInFragment.fragment.trackData.get(this.internalTrack.id)!; - const fragmentSample = trackData.samples[locationInFragment.sampleIndex]!; const fragmentIndex = binarySearchExact( this.internalTrack.fragments, @@ -2072,7 +2071,7 @@ abstract class IsobmffTrackBacking implements InputTrackBacking { }; } }, - fragmentSample.presentationTimestamp, + -Infinity, // Use -Infinity as a search timestamp to avoid using the lookup entries Infinity, options, ); @@ -2116,7 +2115,6 @@ abstract class IsobmffTrackBacking implements InputTrackBacking { } const trackData = locationInFragment.fragment.trackData.get(this.internalTrack.id)!; - const fragmentSample = trackData.samples[locationInFragment.sampleIndex]!; const fragmentIndex = binarySearchExact( this.internalTrack.fragments, @@ -2171,7 +2169,7 @@ abstract class IsobmffTrackBacking implements InputTrackBacking { }; } }, - fragmentSample.presentationTimestamp, + -Infinity, // Use -Infinity as a search timestamp to avoid using the lookup entries Infinity, options, ); diff --git a/src/matroska/ebml.ts b/src/matroska/ebml.ts index ecc4e8b..ace8b18 100644 --- a/src/matroska/ebml.ts +++ b/src/matroska/ebml.ts @@ -594,6 +594,38 @@ export const CODEC_STRING_MAP: Partial> = { 'webvtt': 'S_TEXT/WEBVTT', }; +export const readVarInt = (data: Uint8Array, offset: number) => { + if (offset >= data.length) { + throw new Error('Offset out of bounds.'); + } + + // Read the first byte to determine the width of the variable-length integer + const firstByte = data[offset]!; + + // Find the position of VINT_MARKER, which determines the width + let width = 1; + let mask = 1 << 7; + while ((firstByte & mask) === 0 && width < 8) { + width++; + mask >>= 1; + } + + if (offset + width > data.length) { + throw new Error('VarInt extends beyond data bounds.'); + } + + // First byte's value needs the marker bit cleared + let value = firstByte & (mask - 1); + + // Read remaining bytes + for (let i = 1; i < width; i++) { + value *= 1 << 8; + value += data[offset + i]!; + } + + return { value, width }; +}; + export function assertDefinedSize(size: number | null): asserts size is number { if (size === null) { throw new Error('Undefined element size is used in a place where it is not supported.'); diff --git a/src/matroska/matroska-demuxer.ts b/src/matroska/matroska-demuxer.ts index 95392a6..8d61174 100644 --- a/src/matroska/matroska-demuxer.ts +++ b/src/matroska/matroska-demuxer.ts @@ -57,6 +57,7 @@ import { LEVEL_0_AND_1_EBML_IDS, MAX_HEADER_SIZE, MIN_HEADER_SIZE, + readVarInt, } from './ebml'; import { buildMatroskaMimeType } from './matroska-misc'; @@ -107,12 +108,20 @@ type ClusterTrackData = { }[]; }; +enum BlockLacing { + None, + Xiph, + FixedSize, + Ebml, +} + type ClusterBlock = { timestamp: number; duration: number; isKeyFrame: boolean; referencedTimestamps: number[]; data: Uint8Array; + lacing: BlockLacing; }; type CuePoint = { @@ -133,6 +142,7 @@ type InternalTrack = { inputTrack: InputTrack | null; codecId: string | null; codecPrivate: Uint8Array | null; + defaultDuration: number | null; languageCode: string; info: | null @@ -494,16 +504,20 @@ export class MatroskaDemuxer extends Demuxer { this.readContiguousElements(this.clusterReader, size); for (const [trackId, trackData] of cluster.trackData) { - let blockReferencesExist = false; + const track = segment.tracks.find(x => x.id === trackId) ?? null; // This must hold, as track datas only get created if a block for that track is encountered assert(trackData.blocks.length > 0); + let blockReferencesExist = false; + let hasLacedBlocks = false; + for (let i = 0; i < trackData.blocks.length; i++) { const block = trackData.blocks[i]!; block.timestamp += cluster.timestamp; blockReferencesExist ||= block.referencedTimestamps.length > 0; + hasLacedBlocks ||= block.lacing !== BlockLacing.None; } if (blockReferencesExist) { @@ -526,16 +540,35 @@ export class MatroskaDemuxer extends Demuxer { // Update block durations based on presentation order const nextEntry = trackData.presentationTimestamps[i + 1]!; currentBlock.duration = nextEntry.timestamp - currentBlock.timestamp; + } else if (currentBlock.duration === 0) { + if (track?.defaultDuration != null) { + if (currentBlock.lacing === BlockLacing.None) { + currentBlock.duration = track.defaultDuration; + } else { + // Handled by the lace resolution code + } + } } } + if (hasLacedBlocks) { + // Perform lace resolution. Here, we expand each laced block into multiple blocks where each contains + // one frame of the lace. We do this after determining block timestamps so we can properly distribute + // the block's duration across the laced frames. + this.expandLacedBlocks(trackData.blocks, track); + + // Recompute since blocks have changed + trackData.presentationTimestamps = trackData.blocks + .map((block, i) => ({ timestamp: block.timestamp, blockIndex: i })) + .sort((a, b) => a.timestamp - b.timestamp); + } + const firstBlock = trackData.blocks[trackData.presentationTimestamps[0]!.blockIndex]!; const lastBlock = trackData.blocks[last(trackData.presentationTimestamps)!.blockIndex]!; trackData.startTimestamp = firstBlock.timestamp; trackData.endTimestamp = lastBlock.timestamp + lastBlock.duration; - const track = segment.tracks.find(x => x.id === trackId); if (track) { const insertionIndex = binarySearchLessOrEqual( track.clusters, @@ -584,6 +617,120 @@ export class MatroskaDemuxer extends Demuxer { return trackData; } + expandLacedBlocks(blocks: ClusterBlock[], track: InternalTrack | null) { + // https://www.matroska.org/technical/notes.html#block-lacing + + for (let blockIndex = 0; blockIndex < blocks.length; blockIndex++) { + const originalBlock = blocks[blockIndex]!; + if (originalBlock.lacing === BlockLacing.None) { + continue; + } + + const data = originalBlock.data; + let pos = 0; + + const frameSizes: number[] = []; + const frameCount = data[pos]! + 1; + pos++; + + switch (originalBlock.lacing) { + case BlockLacing.Xiph: { + let totalUsedSize = 0; + + // Xiph lacing, just like in Ogg + for (let i = 0; i < frameCount - 1; i++) { + let frameSize = 0; + + while (pos < data.length) { + const value = data[pos]!; + frameSize += value; + pos++; + + if (value < 255) { + frameSizes.push(frameSize); + totalUsedSize += frameSize; + + break; + } + } + } + + // Compute the last frame's size from whatever's left + frameSizes.push(data.length - (pos + totalUsedSize)); + }; break; + + case BlockLacing.FixedSize: { + // Fixed size lacing: all frames have same size + const totalDataSize = data.length - 1; // Minus the frame count byte + const frameSize = Math.floor(totalDataSize / frameCount); + + for (let i = 0; i < frameCount; i++) { + frameSizes.push(frameSize); + } + }; break; + + case BlockLacing.Ebml: { + // EBML lacing: first size absolute, subsequent ones are coded as signed differences from the last + const firstResult = readVarInt(data, pos); + let currentSize = firstResult.value; + frameSizes.push(currentSize); + pos += firstResult.width; + + let totalUsedSize = currentSize; + + for (let i = 1; i < frameCount - 1; i++) { + const diffResult = readVarInt(data, pos); + const unsignedDiff = diffResult.value; + const bias = (1 << (diffResult.width * 7 - 1)) - 1; // Typo-corrected version of 2^((7*n)-1)^-1 + const diff = unsignedDiff - bias; + + currentSize += diff; + frameSizes.push(currentSize); + pos += diffResult.width; + + totalUsedSize += currentSize; + } + + // Compute the last frame's size from whatever's left + frameSizes.push(data.length - (pos + totalUsedSize)); + }; break; + + default: assert(false); + } + + assert(frameSizes.length === frameCount); + + blocks.splice(blockIndex, 1); // Remove the original block + let dataOffset = pos; + + // Now, let's insert each frame as its own block + for (let i = 0; i < frameCount; i++) { + const frameSize = frameSizes[i]!; + const frameData = data.subarray(dataOffset, dataOffset + frameSize); + + const blockDuration = originalBlock.duration || (frameCount * (track?.defaultDuration ?? 0)); + + // Distribute timestamps evenly across the block duration + const frameTimestamp = originalBlock.timestamp + (blockDuration * i / frameCount); + const frameDuration = blockDuration / frameCount; + + blocks.splice(blockIndex + i, 0, { + timestamp: frameTimestamp, + duration: frameDuration, + isKeyFrame: originalBlock.isKeyFrame, + referencedTimestamps: originalBlock.referencedTimestamps, + data: frameData, + lacing: BlockLacing.None, + }); + + dataOffset += frameSize; + } + + blockIndex += frameCount; // Skip the blocks we just added + blockIndex--; + } + } + readContiguousElements(reader: EBMLReader, totalSize: number) { const startIndex = reader.pos; @@ -655,6 +802,7 @@ export class MatroskaDemuxer extends Demuxer { inputTrack: null, codecId: null, codecPrivate: null, + defaultDuration: null, languageCode: UNDETERMINED_LANGUAGE, info: null, }; @@ -816,6 +964,13 @@ export class MatroskaDemuxer extends Demuxer { this.currentTrack.codecPrivate = reader.readBytes(size); }; break; + case EBMLId.DefaultDuration: { + if (!this.currentTrack) break; + + this.currentTrack.defaultDuration + = this.currentTrack.segment.timestampFactor * reader.readUnsignedInt(size) / 1e9; + }; break; + case EBMLId.Language: { if (!this.currentTrack) break; @@ -977,14 +1132,16 @@ export class MatroskaDemuxer extends Demuxer { const flags = reader.readU8(); const isKeyFrame = !!(flags & 0x80); + const lacing = (flags >> 1) & 0x3 as BlockLacing; // If the block is laced, we'll expand it later const trackData = this.getTrackDataInCluster(this.currentCluster, trackNumber); trackData.blocks.push({ timestamp: relativeTimestamp, // We'll add the cluster's timestamp to this later - duration: 0, + duration: 0, // Will set later isKeyFrame, referencedTimestamps: [], data: reader.readBytes(size - (reader.pos - dataStartPos)), + lacing, }); }; break; @@ -1008,16 +1165,17 @@ export class MatroskaDemuxer extends Demuxer { const trackNumber = reader.readVarInt(); const relativeTimestamp = reader.readS16(); - // eslint-disable-next-line @typescript-eslint/no-unused-vars const flags = reader.readU8(); + const lacing = (flags >> 1) & 0x3 as BlockLacing; // If the block is laced, we'll expand it later const trackData = this.getTrackDataInCluster(this.currentCluster, trackNumber); this.currentBlock = { timestamp: relativeTimestamp, // We'll add the cluster's timestamp to this later - duration: 0, + duration: 0, // Will set later isKeyFrame: true, referencedTimestamps: [], data: reader.readBytes(size - (reader.pos - dataStartPos)), + lacing, }; trackData.blocks.push(this.currentBlock); }; break; @@ -1140,7 +1298,6 @@ abstract class MatroskaTrackBacking implements InputTrackBacking { } const trackData = locationInCluster.cluster.trackData.get(this.internalTrack.id)!; - const block = trackData.blocks[locationInCluster.blockIndex]!; const clusterIndex = binarySearchExact( this.internalTrack.clusters, @@ -1188,7 +1345,7 @@ abstract class MatroskaTrackBacking implements InputTrackBacking { }; } }, - block.timestamp, + -Infinity, // Use -Infinity as a search timestamp to avoid using the cues Infinity, options, ); @@ -1212,7 +1369,6 @@ abstract class MatroskaTrackBacking implements InputTrackBacking { } const trackData = locationInCluster.cluster.trackData.get(this.internalTrack.id)!; - const block = trackData.blocks[locationInCluster.blockIndex]!; const clusterIndex = binarySearchExact( this.internalTrack.clusters, @@ -1267,7 +1423,7 @@ abstract class MatroskaTrackBacking implements InputTrackBacking { }; } }, - block.timestamp, + -Infinity, // Use -Infinity as a search timestamp to avoid using the cues Infinity, options, );