From 7988b6c2f838bae0c3c1474f89ea272b510cace6 Mon Sep 17 00:00:00 2001 From: Vanilagy <1696106+Vanilagy@users.noreply.github.com> Date: Tue, 18 Feb 2025 22:13:21 +0100 Subject: [PATCH] Multiple miscellaneous fixes --- src/conversion.ts | 6 +++--- src/isobmff/isobmff-demuxer.ts | 23 +++++++++++------------ src/isobmff/isobmff-muxer.ts | 8 +++++--- src/matroska/matroska-demuxer.ts | 5 +++++ src/matroska/matroska-muxer.ts | 4 ++-- src/misc.ts | 9 +++++++++ src/muxer.ts | 26 ++++++++++---------------- src/output.ts | 4 ++-- todo.txt | 4 +++- 9 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/conversion.ts b/src/conversion.ts index 5759310..0583141 100644 --- a/src/conversion.ts +++ b/src/conversion.ts @@ -641,7 +641,7 @@ class Conversion { } if (needsResample) { - audioSource = await this.resampleAudio(track, codecOfChoice, numberOfChannels, sampleRate); + audioSource = this.resampleAudio(track, codecOfChoice, numberOfChannels, sampleRate); } else { const source = new AudioDataSource({ codec: codecOfChoice, @@ -654,7 +654,7 @@ class Conversion { await this.started; const sink = new AudioDataSink(track); - for await (const { data, timestamp } of sink.data(undefined, this.startTimestamp)) { + for await (const { data, timestamp } of sink.data(undefined, this.endTimestamp)) { if (this.synchronizer.shouldWait(track.id, timestamp)) { await this.synchronizer.wait(timestamp); } @@ -686,7 +686,7 @@ class Conversion { * Resamples the audio by decoding it, playing it onto an OfflineAudioContext and encoding the * resulting AudioBuffer. */ - async resampleAudio( + resampleAudio( track: InputAudioTrack, codec: AudioCodec, targetNumberOfChannels: number, diff --git a/src/isobmff/isobmff-demuxer.ts b/src/isobmff/isobmff-demuxer.ts index a8f2698..2c863f3 100644 --- a/src/isobmff/isobmff-demuxer.ts +++ b/src/isobmff/isobmff-demuxer.ts @@ -27,7 +27,6 @@ import { COLOR_PRIMARIES_MAP_INVERSE, MATRIX_COEFFICIENTS_MAP_INVERSE, TRANSFER_CHARACTERISTICS_MAP_INVERSE, - rotationMatrix, binarySearchLessOrEqual, binarySearchExact, Rotation, @@ -38,6 +37,8 @@ import { TransformationMatrix, extractRotationFromMatrix, roundToPrecision, + isIso639Dash2LanguageCode, + roundToMultiple, } from '../misc'; import { Reader } from '../reader'; import { EncodedAudioSample, EncodedVideoSample, PLACEHOLDER_DATA, SampleType } from '../sample'; @@ -175,8 +176,6 @@ type Fragment = { isKnownToBeFirstFragment: boolean; }; -const knownMatrixes = [rotationMatrix(0), rotationMatrix(90), rotationMatrix(180), rotationMatrix(270)]; - export class IsobmffDemuxer extends Demuxer { metadataReader: IsobmffReader; currentTrack: InternalTrack | null = null; @@ -657,16 +656,11 @@ export class IsobmffDemuxer extends Demuxer { this.metadataReader.readFixed_2_30(), ]; - const rotation = extractRotationFromMatrix(matrix); - const comparisonMatrix = rotationMatrix(rotation); + const rotation = (roundToMultiple(extractRotationFromMatrix(matrix), 90) + 360) % 360 as Rotation; + assert(rotation === 0 || rotation === 90 || rotation === 180 || rotation === 270); - const matrixIndex = knownMatrixes.findIndex(mat => mat.every((y, i) => y === comparisonMatrix[i])); - if (matrixIndex === -1) { - console.warn(`Wacky rotation matrix ${comparisonMatrix.join(',')}; sticking with no rotation.`); - track.rotation = 0; - } else { - track.rotation = (90 * matrixIndex) as Rotation; - } + // Flip clockwise to counter-clockwise + track.rotation = (-rotation + 360) % 360 as Rotation; }; break; case 'elst': { @@ -739,6 +733,11 @@ export class IsobmffDemuxer extends Demuxer { track.languageCode = String.fromCharCode(0x60 + (language & 0b11111)) + track.languageCode; language >>= 5; } + + if (!isIso639Dash2LanguageCode(track.languageCode)) { + // Sometimes the bytes are garbage + track.languageCode = UNDETERMINED_LANGUAGE; + } } }; break; diff --git a/src/isobmff/isobmff-muxer.ts b/src/isobmff/isobmff-muxer.ts index 1ae60c4..3f56b2c 100644 --- a/src/isobmff/isobmff-muxer.ts +++ b/src/isobmff/isobmff-muxer.ts @@ -13,7 +13,7 @@ import { validateSubtitleMetadata, validateVideoChunkMetadata, } from '../codec'; -import { EncodedAudioSample, EncodedVideoSample } from '../sample'; +import { EncodedAudioSample, EncodedVideoSample, SampleType } from '../sample'; import { BufferTarget } from '../target'; export const GLOBAL_TIMESCALE = 1000; @@ -25,7 +25,7 @@ export type Sample = { duration: number; data: Uint8Array | null; size: number; - type: 'key' | 'delta'; + type: SampleType; timescaleUnitsToNextSample: number; }; @@ -492,7 +492,7 @@ export class IsobmffMuxer extends Muxer { data: Uint8Array, timestamp: number, duration: number, - type: 'key' | 'delta', + type: SampleType, ) { const sample: Sample = { timestamp, @@ -564,6 +564,8 @@ export class IsobmffMuxer extends Muxer { const timescaleUnits = intoTimescale(sample.decodeTimestamp, trackData.timescale, false); const delta = Math.round(timescaleUnits - trackData.lastTimescaleUnits); + assert(delta >= 0); + trackData.lastTimescaleUnits += delta; trackData.lastSample.timescaleUnitsToNextSample = delta; diff --git a/src/matroska/matroska-demuxer.ts b/src/matroska/matroska-demuxer.ts index 0ad6903..8cd9d4c 100644 --- a/src/matroska/matroska-demuxer.ts +++ b/src/matroska/matroska-demuxer.ts @@ -26,6 +26,7 @@ import { binarySearchLessOrEqual, COLOR_PRIMARIES_MAP_INVERSE, findLastIndex, + isIso639Dash2LanguageCode, last, MATRIX_COEFFICIENTS_MAP_INVERSE, Rotation, @@ -749,6 +750,10 @@ export class MatroskaDemuxer extends Demuxer { if (!this.currentTrack) break; this.currentTrack.languageCode = reader.readString(size); + + if (!isIso639Dash2LanguageCode(this.currentTrack.languageCode)) { + this.currentTrack.languageCode = UNDETERMINED_LANGUAGE; + } }; break; case EBMLId.Video: { diff --git a/src/matroska/matroska-muxer.ts b/src/matroska/matroska-muxer.ts index 1c1b62b..1efebe3 100644 --- a/src/matroska/matroska-muxer.ts +++ b/src/matroska/matroska-muxer.ts @@ -687,8 +687,8 @@ export class MatroskaMuxer extends Muxer { } const relativeTimestamp = msTimestamp - this.currentClusterStartMsTimestamp!; - if (relativeTimestamp < 0) { - // The chunk lies outside of the current cluster + if (relativeTimestamp < -(2 ** 15)) { + // The block lies too far in the past, it's not representable within this cluster return; } diff --git a/src/misc.ts b/src/misc.ts index 0f321b6..d88b5fb 100644 --- a/src/misc.ts +++ b/src/misc.ts @@ -393,6 +393,10 @@ export const roundToPrecision = (value: number, digits: number) => { return Math.round(value * factor) / factor; }; +export const roundToMultiple = (value: number, multiple: number) => { + return Math.round(value / multiple) * multiple; +}; + export const ilog = (x: number) => { let ret = 0; while (x) { @@ -401,3 +405,8 @@ export const ilog = (x: number) => { } return ret; }; + +const ISO_639_2_REGEX = /^[a-z]{3}$/; +export const isIso639Dash2LanguageCode = (x: string) => { + return ISO_639_2_REGEX.test(x); +}; diff --git a/src/muxer.ts b/src/muxer.ts index aafb3af..d7bf13d 100644 --- a/src/muxer.ts +++ b/src/muxer.ts @@ -31,7 +31,7 @@ export abstract class Muxer { private trackTimestampInfo = new WeakMap(); protected validateAndNormalizeTimestamp(track: OutputTrack, timestampInSeconds: number, isKeyFrame: boolean) { @@ -44,7 +44,7 @@ export abstract class Muxer { timestampInfo = { timestampOffset: timestampInSeconds, maxTimestamp: track.source._offsetTimestamps ? 0 : timestampInSeconds, - lastKeyFrameTimestamp: track.source._offsetTimestamps ? 0 : timestampInSeconds, + maxTimestampBeforeLastKeyFrame: track.source._offsetTimestamps ? 0 : timestampInSeconds, }; this.trackTimestampInfo.set(track, timestampInfo); } @@ -57,22 +57,16 @@ export abstract class Muxer { throw new Error(`Timestamps must be non-negative (got ${timestampInSeconds}s).`); } - if (timestampInSeconds < timestampInfo.lastKeyFrameTimestamp) { - throw new Error( - `Timestamp cannot be smaller than last key frame's timestamp (got ${timestampInSeconds}s,` - + ` last key frame at ${timestampInfo.lastKeyFrameTimestamp}s).`, - ); + if (isKeyFrame) { + timestampInfo.maxTimestampBeforeLastKeyFrame = timestampInfo.maxTimestamp; } - if (isKeyFrame) { - if (timestampInSeconds < timestampInfo.maxTimestamp) { - throw new Error( - `Key frame timestamps cannot be smaller than any timestamp that came before` - + ` (got ${timestampInSeconds}s, max timestamp was ${timestampInfo.maxTimestamp}s).`, - ); - } - - timestampInfo.lastKeyFrameTimestamp = timestampInSeconds; + if (timestampInSeconds < timestampInfo.maxTimestampBeforeLastKeyFrame) { + throw new Error( + `Timestamps cannot be smaller than the highest timestamp of the previous run (a run begins with a` + + ` key frame and ends right before the next key frame). Got ${timestampInSeconds}s, but highest` + + ` timestamp is ${timestampInfo.maxTimestampBeforeLastKeyFrame}s.`, + ); } timestampInfo.maxTimestamp = Math.max(timestampInfo.maxTimestamp, timestampInSeconds); diff --git a/src/output.ts b/src/output.ts index 302deca..4ead55e 100644 --- a/src/output.ts +++ b/src/output.ts @@ -1,4 +1,4 @@ -import { AsyncMutex, TransformationMatrix } from './misc'; +import { AsyncMutex, isIso639Dash2LanguageCode, TransformationMatrix } from './misc'; import { Muxer } from './muxer'; import { OutputFormat } from './output-format'; import { AudioSource, MediaSource, SubtitleSource, VideoSource } from './media-source'; @@ -60,7 +60,7 @@ const validateBaseTrackMetadata = (metadata: BaseTrackMetadata) => { if (!metadata || typeof metadata !== 'object') { throw new TypeError('metadata must be an object.'); } - if (metadata.languageCode !== undefined && !/^[a-z]{3}$/.test(metadata.languageCode)) { + if (metadata.languageCode !== undefined && !isIso639Dash2LanguageCode(metadata.languageCode)) { throw new TypeError('metadata.languageCode must be a three-letter, ISO 639-2 language code.'); } }; diff --git a/todo.txt b/todo.txt index cf3e353..6afd86e 100644 --- a/todo.txt +++ b/todo.txt @@ -1,3 +1,5 @@ - onHeader, etc callbacks for Matroska - https://github.com/Vanilagy/mp4-muxer/issues/83 tell him it's possible now -- is this fixed? https://github.com/Vanilagy/webm-muxer/issues/50 \ No newline at end of file +- is this fixed? https://github.com/Vanilagy/webm-muxer/issues/50 +- cross-track offset for streaming sources +- configurable fragmented mp4 fragment size, like the mp4-muxer PR \ No newline at end of file