diff --git a/dev/convert.html b/dev/convert.html new file mode 100644 index 0000000..fbff6d4 --- /dev/null +++ b/dev/convert.html @@ -0,0 +1,72 @@ + + + + + \ No newline at end of file diff --git a/dev/demux.html b/dev/demux.html index 8057547..3062e52 100644 --- a/dev/demux.html +++ b/dev/demux.html @@ -6,32 +6,23 @@ const fileInput = document.createElement('input'); fileInput.type = 'file'; document.body.append(fileInput); - - /* - const encoder = new AudioEncoder({ - output: console.log, - error: console.error - }); - encoder.configure({ - codec: 'vorbis', - numberOfChannels: 2, - sampleRate: 48000, - bitrate: 128000 - }); - - let audioData = new AudioData({ - format: 'f32', - numberOfChannels: 2, - sampleRate: 48000, - timestamp: 0, - data: new Float32Array(2 * 48000), - numberOfFrames: 48000 - }); - encoder.encode(audioData); - */ fileInput.addEventListener('change', async () => { const file = fileInput.files[0]; + const source = new Metamuxer.BlobSource(file); + + const input = new Metamuxer.Input({ + formats: Metamuxer.ALL_FORMATS, + source + }); + + const videoTrack = await input.getPrimaryVideoTrack(); + const sink = new Metamuxer.EncodedVideoSampleSink(videoTrack); + + console.log(await sink.getSample(16)); + console.log(await sink.getFirstSample()); + + /* const source = new Metamuxer.BufferSource(await file.arrayBuffer()) ?? new Metamuxer.BlobSource(file); const start = performance.now(); @@ -45,6 +36,7 @@ target: new Metamuxer.BufferTarget() }); output.start(); + */ /* const audioTrack = await input.getPrimaryAudioTrack(); diff --git a/src/codec.ts b/src/codec.ts index 522e22e..2d33f18 100644 --- a/src/codec.ts +++ b/src/codec.ts @@ -11,17 +11,18 @@ import { } from './misc'; import { SubtitleMetadata } from './subtitles'; +// Codecs are ordered by encoding preference: + /** @public */ export const VIDEO_CODECS = [ 'avc', 'hevc', - 'vp8', 'vp9', 'av1', + 'vp8', ] as const; -export const PCM_CODECS = [ - 'pcm-u8', - 'pcm-s8', +/** @public */ +export const PCM_AUDIO_CODECS = [ 'pcm-s16', // We don't prefix 'le' so we're compatible with the WebCodecs-registered PCM codec strings 'pcm-s16be', 'pcm-s24', @@ -30,26 +31,34 @@ export const PCM_CODECS = [ 'pcm-s32be', 'pcm-f32', 'pcm-f32be', + 'pcm-u8', + 'pcm-s8', 'ulaw', 'alaw', ] as const; /** @public */ -export const AUDIO_CODECS = [ +export const NON_PCM_AUDIO_CODECS = [ 'aac', - 'mp3', 'opus', + 'mp3', 'vorbis', 'flac', - ...PCM_CODECS, ] as const; /** @public */ -export const SUBTITLE_CODECS = ['webvtt'] as const; // TODO add the rest +export const AUDIO_CODECS = [ + ...NON_PCM_AUDIO_CODECS, + ...PCM_AUDIO_CODECS, +] as const; +/** @public */ +export const SUBTITLE_CODECS = [ + 'webvtt', +] as const; // TODO add the rest /** @public */ export type VideoCodec = typeof VIDEO_CODECS[number]; /** @public */ export type AudioCodec = typeof AUDIO_CODECS[number]; -export type PcmAudioCodec = typeof PCM_CODECS[number]; +export type PcmAudioCodec = typeof PCM_AUDIO_CODECS[number]; /** @public */ export type SubtitleCodec = typeof SUBTITLE_CODECS[number]; /** @public */ @@ -855,7 +864,7 @@ export const buildAudioCodecString = (codec: AudioCodec, numberOfChannels: numbe return 'vorbis'; } else if (codec === 'flac') { return 'flac'; - } else if ((PCM_CODECS as readonly string[]).includes(codec)) { + } else if ((PCM_AUDIO_CODECS as readonly string[]).includes(codec)) { return codec; } @@ -892,7 +901,7 @@ export const extractAudioCodecString = (trackInfo: { return 'vorbis'; } else if (codec === 'flac') { return 'flac'; - } else if (codec && (PCM_CODECS as readonly string[]).includes(codec)) { + } else if (codec && (PCM_AUDIO_CODECS as readonly string[]).includes(codec)) { return codec; } @@ -958,7 +967,7 @@ export const parseAacAudioSpecificConfig = (bytes: Uint8Array | null) => { const PCM_CODEC_REGEX = /^pcm-([usf])(\d+)+(be)?$/; export const parsePcmCodec = (codec: PcmAudioCodec) => { - assert(PCM_CODECS.includes(codec)); + assert(PCM_AUDIO_CODECS.includes(codec)); if (codec === 'ulaw') { return { dataType: 'ulaw' as const, sampleSize: 1 as const, littleEndian: true, silentValue: 255 }; @@ -1021,6 +1030,91 @@ export const getAudioEncoderConfigExtension = (codec: AudioCodec) => { return {}; }; +/** @public */ +export class Quality { + /** @internal */ + _factor: number; + + /** @internal */ + constructor(factor: number) { + this._factor = factor; + } + + /** @internal */ + _toVideoBitrate(codec: VideoCodec, width: number, height: number) { + const pixels = width * height; + + const codecEfficiencyFactors = { + avc: 1.0, // H.264/AVC (baseline) + hevc: 0.6, // H.265/HEVC (~40% more efficient than AVC) + vp9: 0.6, // Similar to HEVC + av1: 0.4, // ~60% more efficient than AVC + vp8: 1.2, // Slightly less efficient than AVC + }; + + const getBaseBitrateForPixels = (pixelCount: number): number => { + const referencePixels = 1920 * 1080; + const referenceBitrate = 2000000; + + // Non-linear scaling + const scaleFactor = Math.pow(pixelCount / referencePixels, 0.75); + return referenceBitrate * scaleFactor; + }; + + const baseBitrate = getBaseBitrateForPixels(pixels); + const codecAdjustedBitrate = baseBitrate * codecEfficiencyFactors[codec]; + const finalBitrate = codecAdjustedBitrate * this._factor; + + return Math.ceil(finalBitrate / 1000) * 1000; + } + + /** @internal */ + _toAudioBitrate(codec: AudioCodec) { + if ((PCM_AUDIO_CODECS as readonly string[]).includes(codec) || codec === 'flac') { + return -1; + } + + const baseRates = { + aac: 128000, // 128kbps base for AAC + opus: 64000, // 64kbps base for Opus + mp3: 160000, // 160kbps base for MP3 + vorbis: 64000, // 64kbps base for Vorbis + }; + + const baseBitrate = baseRates[codec as keyof typeof baseRates]; + if (!baseBitrate) { + throw new Error(`Unhandled codec: ${codec}`); + } + + let finalBitrate = baseBitrate * this._factor; + + if (codec === 'aac') { + // AAC only works with specific bitrates, let's find the closest + const validRates = [96000, 128000, 160000, 192000]; + finalBitrate = validRates.reduce((prev, curr) => + Math.abs(curr - finalBitrate) < Math.abs(prev - finalBitrate) ? curr : prev, + ); + } else if (codec === 'opus' || codec === 'vorbis') { + finalBitrate = Math.max(6000, finalBitrate); + } else if (codec === 'mp3') { + finalBitrate = Math.round(finalBitrate / 32000) * 32000; + } + + return Math.round(finalBitrate / 1000) * 1000; + } +} + +/** @public */ +export const QUALITY_VERY_LOW = new Quality(0.4); +/** @public */ +export const QUALITY_LOW = new Quality(0.6); +/** @public */ +export const QUALITY_MEDIUM = new Quality(1); +/** @public */ +export const QUALITY_HIGH = new Quality(2); +/** @public */ +export const QUALITY_VERY_HIGH = new Quality(4); + const VALID_VIDEO_CODEC_STRING_PREFIXES = ['avc1', 'avc3', 'hev1', 'hvc1', 'vp8', 'vp09', 'av01']; const AVC_CODEC_STRING_REGEX = /^(avc1|avc3)\.[0-9a-fA-F]{6}$/; const HEVC_CODEC_STRING_REGEX = /^(hev1|hvc1)\.(?:[ABC]?\d+)\.[0-9a-fA-F]{1,8}\.[LH]\d+(?:\.[0-9a-fA-F]{1,2}){0,6}$/; @@ -1287,10 +1381,10 @@ export const validateAudioChunkMetadata = (metadata: EncodedAudioChunkMetadata | || metadata.decoderConfig.codec.startsWith('ulaw') || metadata.decoderConfig.codec.startsWith('alaw') ) { - if (!(PCM_CODECS as readonly string[]).includes(metadata.decoderConfig.codec)) { + if (!(PCM_AUDIO_CODECS as readonly string[]).includes(metadata.decoderConfig.codec)) { throw new TypeError( 'Audio chunk metadata decoder configuration codec string for PCM must be one of the supported PCM' - + ` codecs (${PCM_CODECS.join(', ')}).`, + + ` codecs (${PCM_AUDIO_CODECS.join(', ')}).`, ); } } @@ -1331,7 +1425,7 @@ export const canEncode = (codec: MediaCodec) => { export const canEncodeVideo = async (codec: VideoCodec, { width = 1280, height = 720, bitrate = 1e6 }: { width?: number; height?: number; - bitrate?: 1e6; + bitrate?: number; } = {}) => { if (!VIDEO_CODECS.includes(codec)) { return false; @@ -1380,7 +1474,7 @@ export const canEncodeAudio = async (codec: AudioCodec, { numberOfChannels = 2, throw new TypeError('bitrate must be a positive integer.'); } - if ((PCM_CODECS as readonly string[]).includes(codec)) { + if ((PCM_AUDIO_CODECS as readonly string[]).includes(codec)) { return true; // Because we encode these ourselves } @@ -1420,19 +1514,35 @@ export const getEncodableCodecs = async (): Promise => { }; /** @public */ -export const getEncodableVideoCodecs = async (): Promise => { - const bools = await Promise.all(VIDEO_CODECS.map(canEncode)); - return VIDEO_CODECS.filter((_, i) => bools[i]); +export const getEncodableVideoCodecs = async ( + checkedCodecs = VIDEO_CODECS as unknown as VideoCodec[], + options?: { + width?: number; + height?: number; + bitrate?: number; + }, +): Promise => { + const bools = await Promise.all(checkedCodecs.map(codec => canEncodeVideo(codec, options))); + return checkedCodecs.filter((_, i) => bools[i]); }; /** @public */ -export const getEncodableAudioCodecs = async (): Promise => { - const bools = await Promise.all(AUDIO_CODECS.map(canEncode)); - return AUDIO_CODECS.filter((_, i) => bools[i]); +export const getEncodableAudioCodecs = async ( + checkedCodecs = AUDIO_CODECS as unknown as AudioCodec[], + options?: { + numberOfChannels?: number; + sampleRate?: number; + bitrate?: number; + }, +): Promise => { + const bools = await Promise.all(checkedCodecs.map(codec => canEncodeAudio(codec, options))); + return checkedCodecs.filter((_, i) => bools[i]); }; /** @public */ -export const getEncodableSubtitleCodecs = async (): Promise => { - const bools = await Promise.all(SUBTITLE_CODECS.map(canEncode)); - return SUBTITLE_CODECS.filter((_, i) => bools[i]); +export const getEncodableSubtitleCodecs = async ( + checkedCodecs = SUBTITLE_CODECS as unknown as SubtitleCodec[], +): Promise => { + const bools = await Promise.all(checkedCodecs.map(canEncodeSubtitles)); + return checkedCodecs.filter((_, i) => bools[i]); }; diff --git a/src/conversion.ts b/src/conversion.ts new file mode 100644 index 0000000..2cdd60a --- /dev/null +++ b/src/conversion.ts @@ -0,0 +1,833 @@ +import { + AUDIO_CODECS, + AudioCodec, + getEncodableAudioCodecs, + getEncodableVideoCodecs, + NON_PCM_AUDIO_CODECS, + Quality, + QUALITY_HIGH, + VIDEO_CODECS, + VideoCodec, +} from './codec'; +import { Input } from './input'; +import { InputAudioTrack, InputTrack, InputVideoTrack } from './input-track'; +import { + AudioBufferSink, + AudioDataSink, + CanvasSink, + EncodedAudioSampleSink, + EncodedVideoSampleSink, + VideoFrameSink, +} from './media-sink'; +import { + AudioBufferSource, + AudioDataSource, + AudioEncodingConfig, + AudioSource, + CanvasSource, + EncodedAudioSampleSource, + EncodedVideoSampleSource, + VideoEncodingConfig, + VideoFrameSource, + VideoSource, +} from './media-source'; +import { assert, clamp, promiseWithResolvers, Rotation, setVideoFrameTiming } from './misc'; +import { Output, TrackType } from './output'; + +/** @public */ +export type ConversionOptions = { + input: Input; + output: Output; + + video?: { + discard?: boolean; + codec?: VideoCodec; + bitrate?: VideoEncodingConfig['bitrate']; + width?: number; + height?: number; + fit?: 'fill' | 'contain' | 'cover'; + rotate?: Rotation; + forceReencode?: boolean; + }; + + audio?: { + discard?: boolean; + codec?: AudioCodec; + bitrate?: AudioEncodingConfig['bitrate']; + numberOfChannels?: number; + sampleRate?: number; + forceReencode?: boolean; + }; + + trim?: { + start: number; + end: number; + }; + + onProgress?: (event: { completion: number }) => unknown; +}; + +/** @public */ +export type ConversionInfo = { + utilizedTracks: InputTrack[]; + discardedTracks: { + track: InputTrack; + reason: + | 'discardedByUser' + | 'maxTrackCountReached' + | 'maxTrackCountOfTypeReached' + | 'unknownSourceCodec' + | 'undecodableSourceCodec' + | 'noEncodableTargetCodec'; + }[]; +}; + +const FALLBACK_NUMBER_OF_CHANNELS = 2; +const FALLBACK_SAMPLE_RATE = 48000; + +/** @public */ +export const convert = (options: ConversionOptions) => { + const conversion = new Conversion(options); + return conversion.execute(); +}; + +class Conversion { + input: Input; + output: Output; + startTimestamp: number; + endTimestamp: number; + + addedCounts: Record = { + video: 0, + audio: 0, + subtitle: 0, + }; + + totalTrackCount = 0; + + trackPromises: Promise[] = []; + + started: Promise; + start: () => void; + + synchronizer = new TrackSynchronizer(); + + totalDuration: number | null = null; + maxTimestamps = new Map(); // Track ID -> timestamp + + result: ConversionInfo; + + constructor(public options: ConversionOptions) { + if (!options || typeof options !== 'object') { + throw new TypeError('options must be an object.'); + } + if (!(options.input instanceof Input)) { + throw new TypeError('options.input must be an Input.'); + } + if (!(options.output instanceof Output)) { + throw new TypeError('options.output must be an Output.'); + } + if (options.video !== undefined && (!options.video || typeof options.video !== 'object')) { + throw new TypeError('options.video, when provided, must be an object.'); + } + if (options.video?.discard !== undefined && typeof options.video.discard !== 'boolean') { + throw new TypeError('options.video.discard, when provided, must be a boolean.'); + } + if (options.video?.forceReencode !== undefined && typeof options.video.forceReencode !== 'boolean') { + throw new TypeError('options.video.forceReencode, when provided, must be a boolean.'); + } + if (options.video?.codec !== undefined && !VIDEO_CODECS.includes(options.video.codec)) { + throw new TypeError( + `options.video.codec, when provided, must be one of: ${VIDEO_CODECS.join(', ')}.`, + ); + } + if ( + options.video?.bitrate !== undefined + && !(options.video.bitrate instanceof Quality) + && (!Number.isInteger(options.video.bitrate) || options.video.bitrate <= 0) + ) { + throw new TypeError('options.video.bitrate, when provided, must be a positive integer or a quality.'); + } + if ( + options.video?.width !== undefined + && (!Number.isInteger(options.video.width) || options.video.width <= 0) + ) { + throw new TypeError('options.video.width, when provided, must be a positive integer.'); + } + if ( + options.video?.height !== undefined + && (!Number.isInteger(options.video.height) || options.video.height <= 0) + ) { + throw new TypeError('options.video.height, when provided, must be a positive integer.'); + } + if (options.video?.fit !== undefined && !['fill', 'contain', 'cover'].includes(options.video.fit)) { + throw new TypeError('options.video.fit, when provided, must be one of "fill", "contain", or "cover".'); + } + if ( + options.video?.width !== undefined + && options.video.height !== undefined + && options.video.fit === undefined + ) { + throw new TypeError( + 'When both options.video.width and options.video.height are provided, options.video.fit must also be' + + ' provided.', + ); + } + if (options.video?.rotate !== undefined && ![0, 90, 180, 270].includes(options.video.rotate)) { + throw new TypeError('options.video.rotate, when provided, must be 0, 90, 180 or 270.'); + } + if (options.audio !== undefined && (!options.audio || typeof options.audio !== 'object')) { + throw new TypeError('options.video, when provided, must be an object.'); + } + if (options.audio?.discard !== undefined && typeof options.audio.discard !== 'boolean') { + throw new TypeError('options.audio.discard, when provided, must be a boolean.'); + } + if (options.audio?.forceReencode !== undefined && typeof options.audio.forceReencode !== 'boolean') { + throw new TypeError('options.audio.forceReencode, when provided, must be a boolean.'); + } + if (options.audio?.codec !== undefined && !AUDIO_CODECS.includes(options.audio.codec)) { + throw new TypeError( + `options.audio.codec, when provided, must be one of: ${AUDIO_CODECS.join(', ')}.`, + ); + } + if ( + options.audio?.bitrate !== undefined + && !(options.audio.bitrate instanceof Quality) + && (!Number.isInteger(options.audio.bitrate) || options.audio.bitrate <= 0) + ) { + throw new TypeError('options.audio.bitrate, when provided, must be a positive integer or a quality.'); + } + if ( + options.audio?.numberOfChannels !== undefined + && (!Number.isInteger(options.audio.numberOfChannels) || options.audio.numberOfChannels <= 0) + ) { + throw new TypeError('options.audio.numberOfChannels, when provided, must be a positive integer.'); + } + if ( + options.audio?.sampleRate !== undefined + && (!Number.isInteger(options.audio.sampleRate) || options.audio.sampleRate <= 0) + ) { + throw new TypeError('options.audio.sampleRate, when provided, must be a positive integer.'); + } + if (options.trim !== undefined && (!options.trim || typeof options.trim !== 'object')) { + throw new TypeError('options.trim, when provided, must be an object.'); + } + if (options.trim?.start !== undefined && (!Number.isFinite(options.trim.start) || options.trim.start < 0)) { + throw new TypeError('options.trim.start, when provided, must be a non-negative number.'); + } + if (options.trim?.end !== undefined && (!Number.isFinite(options.trim.end) || options.trim.end < 0)) { + throw new TypeError('options.trim.end, when provided, must be a non-negative number.'); + } + if ( + options.trim?.start !== undefined + && options.trim.end !== undefined + && options.trim.start >= options.trim.end) { + throw new TypeError('options.trim.start must be less than options.trim.end.'); + } + if (options.onProgress !== undefined && typeof options.onProgress !== 'function') { + throw new TypeError('options.onProgress, when provided, must be a function.'); + } + + this.input = options.input; + this.output = options.output; + + this.startTimestamp = options.trim?.start ?? 0; + this.endTimestamp = options.trim?.end ?? Infinity; + + const { promise: started, resolve: start } = promiseWithResolvers(); + this.started = started; + this.start = start; + + this.result = { + utilizedTracks: [], + discardedTracks: [], + }; + } + + async execute() { + const inputTracks = await this.input.getTracks(); + const outputTrackCounts = this.output.format.getSupportedTrackCounts(); + + for (const track of inputTracks) { + if (this.totalTrackCount === outputTrackCounts.total.max) { + this.result.discardedTracks.push({ + track, + reason: 'maxTrackCountReached', + }); + continue; + } + + const type = track.getType(); + if (this.addedCounts[type] === outputTrackCounts[type].max) { + this.result.discardedTracks.push({ + track, + reason: 'maxTrackCountOfTypeReached', + }); + continue; + } + + if (track.isVideoTrack()) { + if (this.options.video?.discard) { + this.result.discardedTracks.push({ + track, + reason: 'discardedByUser', + }); + continue; + } + + await this.processVideoTrack(track); + } else if (track.isAudioTrack()) { + if (this.options.audio?.discard) { + this.result.discardedTracks.push({ + track, + reason: 'discardedByUser', + }); + continue; + } + + await this.processAudioTrack(track); + } + } + + if (this.options.onProgress) { + this.totalDuration = Math.min( + await this.input.computeDuration() - this.startTimestamp, + this.endTimestamp - this.startTimestamp, + ); + this.options.onProgress({ completion: 0 }); + } + + await this.output.start(); + this.start(); + + await Promise.all(this.trackPromises); + + await this.output.finalize(); + + this.options.onProgress?.({ completion: 1 }); + + return this.result; + } + + async processVideoTrack(track: InputVideoTrack) { + const trackId = track.getId(); + + const sourceCodec = await track.getCodec(); + if (!sourceCodec) { + this.result.discardedTracks.push({ + track, + reason: 'unknownSourceCodec', + }); + return; + } + + let videoSource: VideoSource; + + const originalWidth = await track.getCodedWidth(); + const originalHeight = await track.getCodedHeight(); + const originalAspectRatio = originalWidth / originalHeight; + + let width = originalWidth; + let height = originalHeight; + + // A lot of video encoders require that the dimensions be multiples of 2 + const ceilToMultipleOfTwo = (value: number) => Math.ceil(value / 2) * 2; + + if (this.options.video?.width !== undefined && this.options.video.height === undefined) { + width = ceilToMultipleOfTwo(this.options.video.width); + height = ceilToMultipleOfTwo(Math.round(width / originalAspectRatio)); + } else if (this.options.video?.width === undefined && this.options.video?.height !== undefined) { + height = ceilToMultipleOfTwo(this.options.video.height); + width = ceilToMultipleOfTwo(Math.round(height * originalAspectRatio)); + } else if (this.options.video?.width !== undefined && this.options.video.height !== undefined) { + width = ceilToMultipleOfTwo(this.options.video.width); + height = ceilToMultipleOfTwo(this.options.video.height); + } + + const needsReencode = !!this.options.video?.forceReencode || this.startTimestamp > 0; + const needsResize = width !== originalWidth || height !== originalHeight; + + let videoCodecs = this.output.format.getSupportedVideoCodecs(); + if ( + !needsReencode + && !this.options.video?.bitrate + && !needsResize + && videoCodecs.includes(sourceCodec) + && (!this.options.video?.codec || this.options.video?.codec === sourceCodec) + ) { + // Fast path, we can simply copy over the encoded samples + + const source = new EncodedVideoSampleSource(sourceCodec); + videoSource = source; + + this.trackPromises.push((async () => { + await this.started; + + const sink = new EncodedVideoSampleSink(track); + const decoderConfig = await track.getDecoderConfig(); + const meta: EncodedVideoChunkMetadata = { decoderConfig: decoderConfig ?? undefined }; + + for await (const sample of sink.samples(undefined, this.endTimestamp)) { + if (this.synchronizer.shouldWait(trackId, sample.timestamp)) { + await this.synchronizer.wait(sample.timestamp); + } + + await source.digest(sample, meta); + this.reportProgress(trackId, sample.timestamp + sample.duration); + } + + await source.close(); + this.synchronizer.closeTrack(trackId); + })()); + } else { + // We need to decode & reencode the video + + const canDecode = await track.canDecode(); + if (!canDecode) { + this.result.discardedTracks.push({ + track, + reason: 'undecodableSourceCodec', + }); + return; + } + + if (this.options.video?.codec) { + videoCodecs = videoCodecs.filter(codec => codec === this.options.video?.codec); + } + + const encodableCodecs = await getEncodableVideoCodecs(videoCodecs, { + width: needsResize ? width : await track.getCodedWidth(), + height: needsResize ? height : await track.getCodedHeight(), + }); + if (encodableCodecs.length === 0) { + this.result.discardedTracks.push({ + track, + reason: 'noEncodableTargetCodec', + }); + return; + } + + const encodingConfig: VideoEncodingConfig = { + codec: encodableCodecs[0]!, + bitrate: this.options.video?.bitrate ?? QUALITY_HIGH, + onEncodedSample: sample => this.reportProgress(trackId, sample.timestamp + sample.duration), + }; + + if (needsResize) { + // For resizing, we draw the frame onto a canvas and then encode the canvas + const canvas = document.createElement('canvas'); + canvas.width = width; + canvas.height = height; + const context = canvas.getContext('2d', { + alpha: false, + })!; + + const source = new CanvasSource(canvas, encodingConfig); + videoSource = source; + + this.trackPromises.push((async () => { + await this.started; + + const sink = new CanvasSink(track); + const iterator = sink.canvases(this.startTimestamp, this.endTimestamp); + + for await (const { canvas, timestamp, duration } of iterator) { + if (this.synchronizer.shouldWait(trackId, timestamp)) { + await this.synchronizer.wait(timestamp); + } + + if (!this.options.video?.fit || this.options.video.fit === 'fill') { + context.drawImage(canvas, 0, 0, width, height); + } else if (this.options.video.fit === 'contain') { + const scale = Math.min(width / canvas.width, height / canvas.height); + const newWidth = canvas.width * scale; + const newHeight = canvas.height * scale; + const dx = (width - newWidth) / 2; + const dy = (height - newHeight) / 2; + context.drawImage(canvas, 0, 0, canvas.width, canvas.height, dx, dy, newWidth, newHeight); + } else if (this.options.video.fit === 'cover') { + const scale = Math.max(width / canvas.width, height / canvas.height); + const newWidth = canvas.width * scale; + const newHeight = canvas.height * scale; + const dx = (width - newWidth) / 2; + const dy = (height - newHeight) / 2; + context.drawImage(canvas, 0, 0, canvas.width, canvas.height, dx, dy, newWidth, newHeight); + } + + await source.digest(Math.max(timestamp - this.startTimestamp, 0), duration); + } + + await source.close(); + this.synchronizer.closeTrack(trackId); + })()); + } else { + const source = new VideoFrameSource(encodingConfig); + videoSource = source; + + this.trackPromises.push((async () => { + await this.started; + + const sink = new VideoFrameSink(track); + + for await (const { frame, timestamp } of sink.frames(this.startTimestamp, this.endTimestamp)) { + if (this.synchronizer.shouldWait(trackId, timestamp)) { + await this.synchronizer.wait(timestamp); + } + + const clone = setVideoFrameTiming(frame, { + timestamp: Math.max(timestamp - this.startTimestamp, 0), + }); + + await source.digest(clone); + clone.close(); + } + + await source.close(); + this.synchronizer.closeTrack(trackId); + })()); + } + } + + // Rotation metadata is reset if we do resizing + const baseRotation = needsResize ? 0 : await track.getRotation(); + + this.output.addVideoTrack(videoSource, { + languageCode: await track.getLanguageCode(), + rotation: (baseRotation + (this.options.video?.rotate ?? 0)) % 360 as Rotation, + }); + this.addedCounts.video++; + this.totalTrackCount++; + + this.result.utilizedTracks.push(track); + } + + async processAudioTrack(track: InputAudioTrack) { + const trackId = track.getId(); + + const sourceCodec = await track.getCodec(); + if (!sourceCodec) { + this.result.discardedTracks.push({ + track, + reason: 'unknownSourceCodec', + }); + return; + } + + let audioSource: AudioSource; + + const originalNumberOfChannels = await track.getNumberOfChannels(); + const originalSampleRate = await track.getSampleRate(); + + let numberOfChannels = this.options.audio?.numberOfChannels ?? originalNumberOfChannels; + let sampleRate = this.options.audio?.sampleRate ?? originalSampleRate; + let needsResample = numberOfChannels !== originalNumberOfChannels + || sampleRate !== originalSampleRate + || this.startTimestamp > 0; + + let audioCodecs = this.output.format.getSupportedAudioCodecs(); + if ( + !this.options.audio?.forceReencode + && !this.options.audio?.bitrate + && !needsResample + && audioCodecs.includes(sourceCodec) + && (!this.options.audio?.codec || this.options.audio.codec === sourceCodec) + ) { + // Fast path, we can simply copy over the encoded samples + + const source = new EncodedAudioSampleSource(sourceCodec); + audioSource = source; + + this.trackPromises.push((async () => { + await this.started; + + const sink = new EncodedAudioSampleSink(track); + const decoderConfig = await track.getDecoderConfig(); + const meta: EncodedAudioChunkMetadata = { decoderConfig: decoderConfig ?? undefined }; + + for await (const sample of sink.samples(undefined, this.endTimestamp)) { + if (this.synchronizer.shouldWait(trackId, sample.timestamp)) { + await this.synchronizer.wait(sample.timestamp); + } + + await source.digest(sample, meta); + this.reportProgress(trackId, sample.timestamp + sample.duration); + } + + await source.close(); + this.synchronizer.closeTrack(trackId); + })()); + } else { + // We need to decode & reencode the audio + + const canDecode = await track.canDecode(); + if (!canDecode) { + this.result.discardedTracks.push({ + track, + reason: 'undecodableSourceCodec', + }); + return; + } + + let codecOfChoice: AudioCodec | null = null; + + if (this.options.audio?.codec) { + audioCodecs = audioCodecs.filter(codec => codec === this.options.audio!.codec); + } + + const encodableCodecs = await getEncodableAudioCodecs(audioCodecs, { + numberOfChannels, + sampleRate, + }); + + if ( + !encodableCodecs.some(codec => (NON_PCM_AUDIO_CODECS as readonly string[]).includes(codec)) + && audioCodecs.some(codec => (NON_PCM_AUDIO_CODECS as readonly string[]).includes(codec)) + && (numberOfChannels !== FALLBACK_NUMBER_OF_CHANNELS || sampleRate !== FALLBACK_SAMPLE_RATE) + ) { + // We could not find a compatible non-PCM codec despite the container supporting them. This can be + // caused by strange channel count or sample rate configurations. Therefore, let's try again but with + // fallback parameters. + + const encodableCodecsWithDefaultParams = await getEncodableAudioCodecs(audioCodecs, { + numberOfChannels: FALLBACK_NUMBER_OF_CHANNELS, + sampleRate: FALLBACK_SAMPLE_RATE, + }); + + if ( + encodableCodecsWithDefaultParams + .some(codec => (NON_PCM_AUDIO_CODECS as readonly string[]).includes(codec)) + ) { + // We are able to encode using a non-PCM codec, but it'll require resampling + needsResample = true; + codecOfChoice = encodableCodecsWithDefaultParams[0]!; + numberOfChannels = FALLBACK_NUMBER_OF_CHANNELS; + sampleRate = FALLBACK_SAMPLE_RATE; + } + } else { + codecOfChoice = encodableCodecs[0] ?? null; + } + + if (codecOfChoice === null) { + this.result.discardedTracks.push({ + track, + reason: 'noEncodableTargetCodec', + }); + return; + } + + if (needsResample) { + audioSource = await this.resampleAudio(track, codecOfChoice, numberOfChannels, sampleRate); + } else { + const source = new AudioDataSource({ + codec: codecOfChoice, + bitrate: this.options.audio?.bitrate ?? QUALITY_HIGH, + onEncodedSample: sample => this.reportProgress(trackId, sample.timestamp + sample.duration), + }); + audioSource = source; + + this.trackPromises.push((async () => { + await this.started; + + const sink = new AudioDataSink(track); + for await (const { data, timestamp } of sink.data(undefined, this.startTimestamp)) { + if (this.synchronizer.shouldWait(trackId, timestamp)) { + await this.synchronizer.wait(timestamp); + } + + await source.digest(data); + data.close(); + } + + await source.close(); + this.synchronizer.closeTrack(trackId); + })()); + } + } + + this.output.addAudioTrack(audioSource, { + languageCode: await track.getLanguageCode(), + }); + this.addedCounts.audio++; + this.totalTrackCount++; + + this.result.utilizedTracks.push(track); + } + + /** + * Resamples the audio by decoding it, playing it onto an OfflineAudioContext and encoding the + * resulting AudioBuffer. + */ + async resampleAudio( + track: InputAudioTrack, + codec: AudioCodec, + targetNumberOfChannels: number, + targetSampleRate: number, + ) { + const trackId = track.getId(); + const source = new AudioBufferSource({ + codec, + bitrate: this.options.audio?.bitrate ?? QUALITY_HIGH, + onEncodedSample: sample => this.reportProgress(trackId, sample.timestamp + sample.duration), + }); + + this.trackPromises.push((async () => { + await this.started; + + const trackDuration = Math.min( + await track.computeDuration() - this.startTimestamp, + this.endTimestamp - this.startTimestamp, + ); + const totalFrameCount = Math.round(trackDuration * FALLBACK_SAMPLE_RATE); + + const MAX_CHUNK_LENGTH = 5 * FALLBACK_SAMPLE_RATE; + + let currentContextStartFrame = 0; + let currentContext: OfflineAudioContext | null = new OfflineAudioContext({ + length: Math.min(totalFrameCount - currentContextStartFrame, MAX_CHUNK_LENGTH), + numberOfChannels: targetNumberOfChannels, + sampleRate: targetSampleRate, + }); + + const sink = new AudioBufferSink(track); + for await (const { buffer, timestamp, duration } of sink.buffers(this.startTimestamp, this.endTimestamp)) { + if (this.synchronizer.shouldWait(trackId, timestamp)) { + await this.synchronizer.wait(timestamp); + } + + const offsetTimestamp = timestamp - this.startTimestamp; + const endTimestamp = offsetTimestamp + duration; + + // while loop, as a single source buffer may span multiple audio contexts + while (currentContext) { + const currentContextStartTime = currentContextStartFrame / targetSampleRate; + const currentContextEndTime + = (currentContextStartFrame + currentContext.length) / targetSampleRate; + + if (offsetTimestamp < currentContextEndTime) { + // The buffer lies within the context, let's play it + const node = currentContext.createBufferSource(); + node.buffer = buffer; + node.connect(currentContext.destination); + + if (offsetTimestamp < currentContextStartTime) { + node.start(0, currentContextStartTime - offsetTimestamp); + } else { + node.start(offsetTimestamp - currentContextStartTime); + } + } + + if (endTimestamp >= currentContextEndTime) { + // Render the audio + const renderedBuffer = await currentContext.startRendering(); + await source.digest(renderedBuffer); + + currentContextStartFrame += currentContext.length; + + const newLength = Math.min( + totalFrameCount - currentContextStartFrame, + MAX_CHUNK_LENGTH, + ); + currentContext = newLength > 0 + ? new OfflineAudioContext({ + length: newLength, + numberOfChannels: targetNumberOfChannels, + sampleRate: targetSampleRate, + }) + : null; + } else { + break; + } + } + } + + if (currentContext) { + const renderedBuffer = await currentContext.startRendering(); + await source.digest(renderedBuffer); + } + + await source.close(); + this.synchronizer.closeTrack(trackId); + })()); + + return source; + } + + reportProgress(trackId: number, endTimestamp: number) { + if (!this.options.onProgress) { + return; + } + assert(this.totalDuration !== null); + + this.maxTimestamps.set(trackId, Math.max(endTimestamp, this.maxTimestamps.get(trackId) ?? -Infinity)); + + let totalTimestamps = 0; + for (const [, timestamp] of this.maxTimestamps) { + totalTimestamps += timestamp; + } + + const averageTimestamp = totalTimestamps / this.totalTrackCount; + + this.options.onProgress({ + completion: 0.99 * clamp(averageTimestamp / this.totalDuration, 0, 1), + }); + } +} + +const MAX_TIMESTAMP_GAP = 5; + +/** + * Utility class for synchronizing multiple track sample consumers with one another. We don't want one consumer to get + * too out-of-sync with the others, as that may lead to a large number of samples that need to be internally buffered + * before they can be written. Therefore, we use this class to slow down a consumer if it is too far ahead of the + * slowest consumer. + */ +class TrackSynchronizer { + maxTimestamps = new Map(); // Track ID -> timestamp + resolvers: { + timestamp: number; + resolve: () => void; + }[] = []; + + computeMinAndMaybeResolve() { + let newMin = Infinity; + for (const [, timestamp] of this.maxTimestamps) { + newMin = Math.min(newMin, timestamp); + } + + for (let i = 0; i < this.resolvers.length; i++) { + const entry = this.resolvers[i]!; + + if (entry.timestamp - newMin < MAX_TIMESTAMP_GAP) { + // The gap has gotten small enough again, the consumer can continue again + entry.resolve(); + this.resolvers.splice(i, 1); + i--; + } + } + + return newMin; + } + + shouldWait(trackId: number, timestamp: number) { + this.maxTimestamps.set(trackId, Math.max(timestamp, this.maxTimestamps.get(trackId) ?? -Infinity)); + + const newMin = this.computeMinAndMaybeResolve(); + return timestamp - newMin >= MAX_TIMESTAMP_GAP; // Should wait if it is too far ahead of the slowest consumer + } + + wait(timestamp: number) { + const { promise, resolve } = promiseWithResolvers(); + + this.resolvers.push({ + timestamp, + resolve, + }); + + return promise; + } + + closeTrack(trackId: number) { + this.maxTimestamps.delete(trackId); + this.computeMinAndMaybeResolve(); + } +} diff --git a/src/index.ts b/src/index.ts index 40b82e5..fb4af5f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -42,11 +42,19 @@ export { export { VIDEO_CODECS, VideoCodec, + PCM_AUDIO_CODECS, + NON_PCM_AUDIO_CODECS, AUDIO_CODECS, AudioCodec, SUBTITLE_CODECS, SubtitleCodec, MediaCodec, + Quality, + QUALITY_VERY_LOW, + QUALITY_LOW, + QUALITY_MEDIUM, + QUALITY_HIGH, + QUALITY_VERY_HIGH, canEncode, canEncodeVideo, canEncodeAudio, @@ -57,7 +65,7 @@ export { getEncodableSubtitleCodecs, } from './codec'; export { Target, BufferTarget, StreamTarget, StreamTargetChunk, StreamTargetOptions } from './target'; -export { Rotation, TransformationMatrix, AnyIterable } from './misc'; +export { Rotation, TransformationMatrix, AnyIterable, setVideoFrameTiming } from './misc'; export { Source, BufferSource, BlobSource, UrlSource } from './source'; export { InputFormat, @@ -96,5 +104,6 @@ export { AudioBufferSink, WrappedAudioBuffer, } from './media-sink'; +export { convert, ConversionOptions, ConversionInfo } from './conversion'; // 🐡🦔 diff --git a/src/input-track.ts b/src/input-track.ts index 1282bf7..fb228ce 100644 --- a/src/input-track.ts +++ b/src/input-track.ts @@ -1,9 +1,11 @@ import { AudioCodec, MediaCodec, VideoCodec } from './codec'; import { EncodedAudioSampleSink, EncodedVideoSampleSink, SampleRetrievalOptions } from './media-sink'; import { Rotation } from './misc'; +import { TrackType } from './output'; import { EncodedAudioSample, EncodedVideoSample } from './sample'; export interface InputTrackBacking { + getId(): number; getCodec(): Promise; getFirstTimestamp(): Promise; computeDuration(): Promise; @@ -20,6 +22,7 @@ export abstract class InputTrack { this._backing = backing; } + abstract getType(): TrackType; abstract getCodec(): Promise; abstract getCodecMimeType(): Promise; abstract canDecode(): Promise; @@ -33,6 +36,10 @@ export abstract class InputTrack { return this instanceof InputAudioTrack; } + getId() { + return this._backing.getId(); + } + getFirstTimestamp() { return this._backing.getFirstTimestamp(); } @@ -51,6 +58,7 @@ export interface InputVideoTrackBacking extends InputTrackBacking { getCodedWidth(): Promise; getCodedHeight(): Promise; getRotation(): Promise; + getColorSpace(): Promise; getDecoderConfig(): Promise; getFirstSample(options: SampleRetrievalOptions): Promise; getSample(timestamp: number, options: SampleRetrievalOptions): Promise; @@ -71,6 +79,10 @@ export class InputVideoTrack extends InputTrack { this._backing = backing; } + getType(): TrackType { + return 'video'; + } + getCodec(): Promise { return this._backing.getCodec(); } @@ -97,6 +109,18 @@ export class InputVideoTrack extends InputTrack { return rotation % 180 === 0 ? this._backing.getCodedHeight() : this._backing.getCodedWidth(); } + getColorSpace() { + return this._backing.getColorSpace(); + } + + async hasHighDynamicRange() { + const colorSpace = await this._backing.getColorSpace(); + + return (colorSpace.primaries as string) === 'bt2020' || (colorSpace.primaries as string) === 'smpte432' + || (colorSpace.transfer as string) === 'pg' || (colorSpace.transfer as string) === 'hlg' + || (colorSpace.matrix as string) === 'bt2020-ncl'; + } + getDecoderConfig() { return this._backing.getDecoderConfig(); } @@ -150,6 +174,10 @@ export class InputAudioTrack extends InputTrack { this._backing = backing; } + getType(): TrackType { + return 'audio'; + } + getCodec(): Promise { return this._backing.getCodec(); } diff --git a/src/isobmff/isobmff-boxes.ts b/src/isobmff/isobmff-boxes.ts index 55259ab..1baa707 100644 --- a/src/isobmff/isobmff-boxes.ts +++ b/src/isobmff/isobmff-boxes.ts @@ -17,7 +17,7 @@ import { AudioCodec, generateAv1CodecConfigurationFromCodecString, parsePcmCodec, - PCM_CODECS, + PCM_AUDIO_CODECS, PcmAudioCodec, SubtitleCodec, VideoCodec, @@ -680,7 +680,7 @@ export const soundSampleDescription = ( let contents: NestedNumberArray; let sampleSizeInBits = 16; - if ((PCM_CODECS as readonly AudioCodec[]).includes(trackData.track.source._codec)) { + if ((PCM_AUDIO_CODECS as readonly AudioCodec[]).includes(trackData.track.source._codec)) { const codec = trackData.track.source._codec as PcmAudioCodec; const { sampleSize } = parsePcmCodec(codec); sampleSizeInBits = 8 * sampleSize; diff --git a/src/isobmff/isobmff-demuxer.ts b/src/isobmff/isobmff-demuxer.ts index 9f191b8..a79bffd 100644 --- a/src/isobmff/isobmff-demuxer.ts +++ b/src/isobmff/isobmff-demuxer.ts @@ -7,7 +7,7 @@ import { extractVp9CodecInfoFromFrame, MediaCodec, parseAacAudioSpecificConfig, - PCM_CODECS, + PCM_AUDIO_CODECS, VideoCodec, Vp9CodecInfo, } from '../codec'; @@ -35,6 +35,8 @@ import { AsyncMutex, findLastIndex, UNDETERMINED_LANGUAGE, + TransformationMatrix, + extractRotationFromMatrix, } from '../misc'; import { Reader } from '../reader'; import { EncodedAudioSample, EncodedVideoSample, PLACEHOLDER_DATA, SampleType } from '../sample'; @@ -165,6 +167,7 @@ type Fragment = { dataStart: number; dataEnd: number; nextFragment: Fragment | null; + isKnownToBeFirstFragment: boolean; }; const knownMatrixes = [rotationMatrix(0), rotationMatrix(90), rotationMatrix(180), rotationMatrix(270)]; @@ -295,7 +298,7 @@ export class IsobmffDemuxer extends Demuxer { const isPcmCodec = internalTrack.info?.type === 'audio' && internalTrack.info.codec - && (PCM_CODECS as readonly string[]).includes(internalTrack.info.codec); + && (PCM_AUDIO_CODECS as readonly string[]).includes(internalTrack.info.codec); if (isPcmCodec && sampleTable.sampleCompositionTimeOffsets.length === 0) { // If the audio has PCM samples, the way the samples are defined in the sample table is somewhat @@ -462,6 +465,8 @@ export class IsobmffDemuxer extends Demuxer { this.isobmffReader.pos = currentFragment.moofOffset + currentFragment.moofSize; } + let nextFragmentIsFirstFragment = this.isobmffReader.pos === 0; + while (this.isobmffReader.pos < startPos) { if (currentFragment?.nextFragment) { currentFragment = currentFragment.nextFragment; @@ -477,18 +482,23 @@ export class IsobmffDemuxer extends Demuxer { if (boxInfo.name === 'moof') { const index = binarySearchExact(this.fragments, startPos, x => x.moofOffset); + let fragment: Fragment; if (index === -1) { this.isobmffReader.pos = startPos; - const fragment = await this.readFragment(); // Recursive call - if (currentFragment) currentFragment.nextFragment = fragment; - currentFragment = fragment; + fragment = await this.readFragment(); // Recursive call } else { // We already know this fragment - const fragment = this.fragments[index]!; - // Even if we already know the fragment, we might not yet know its predecessor - if (currentFragment) currentFragment.nextFragment = fragment; - currentFragment = fragment; + fragment = this.fragments[index]!; + } + + // Even if we already know the fragment, we might not yet know its predecessor; always do this + if (currentFragment) currentFragment.nextFragment = fragment; + currentFragment = fragment; + + if (nextFragmentIsFirstFragment) { + fragment.isKnownToBeFirstFragment = true; + nextFragmentIsFirstFragment = false; } } @@ -613,16 +623,24 @@ export class IsobmffDemuxer extends Demuxer { } this.isobmffReader.pos += 2 * 4 + 2 + 2 + 2 + 2; - const values: number[] = []; - values.push(this.isobmffReader.readFixed_16_16(), this.isobmffReader.readFixed_16_16()); - this.isobmffReader.pos += 4; - values.push(this.isobmffReader.readFixed_16_16(), this.isobmffReader.readFixed_16_16()); + const matrix: TransformationMatrix = [ + this.isobmffReader.readFixed_16_16(), + this.isobmffReader.readFixed_16_16(), + this.isobmffReader.readFixed_2_30(), + this.isobmffReader.readFixed_16_16(), + this.isobmffReader.readFixed_16_16(), + this.isobmffReader.readFixed_2_30(), + this.isobmffReader.readFixed_16_16(), + this.isobmffReader.readFixed_16_16(), + this.isobmffReader.readFixed_2_30(), + ]; - const matrixIndex = knownMatrixes.findIndex((x) => { - return x[0] === values[0] && x[1] === values[1] && x[3] === values[2] && x[4] === values[3]; - }); + const rotation = extractRotationFromMatrix(matrix); + const comparisonMatrix = rotationMatrix(rotation); + + const matrixIndex = knownMatrixes.findIndex(mat => mat.every((y, i) => y === comparisonMatrix[i])); if (matrixIndex === -1) { - console.warn(`Wacky rotation matrix ${values.join(',')}; sticking with no rotation.`); + console.warn(`Wacky rotation matrix ${comparisonMatrix.join(',')}; sticking with no rotation.`); track.rotation = 0; } else { track.rotation = (90 * matrixIndex) as Rotation; @@ -960,7 +978,7 @@ export class IsobmffDemuxer extends Demuxer { transfer: TRANSFER_CHARACTERISTICS_MAP_INVERSE[transferCharacteristics], matrix: MATRIX_COEFFICIENTS_MAP_INVERSE[matrixCoefficients], fullRange: fullRangeFlag, - }; + } as VideoColorSpaceInit; }; break; case 'wave': { @@ -1418,6 +1436,7 @@ export class IsobmffDemuxer extends Demuxer { dataStart: Infinity, dataEnd: 0, nextFragment: null, + isKnownToBeFirstFragment: false, }; this.readContiguousBoxes(boxInfo.contentSize); @@ -1694,6 +1713,10 @@ abstract class IsobmffTrackBacking< constructor(public internalTrack: InternalTrack) {} + getId() { + return this.internalTrack.id; + } + getCodec(): Promise { throw new Error('Not implemented on base class.'); } @@ -1724,14 +1747,35 @@ abstract class IsobmffTrackBacking< if (this.internalTrack.demuxer.isFragmented) { return this.performFragmentedLookup( () => { - const fragment = this.internalTrack.fragments[0]; + const startFragment = this.internalTrack.demuxer.fragments[0] ?? null; + if (startFragment?.isKnownToBeFirstFragment) { + // Walk from the very first fragment in the file until we find one with our track in it + let currentFragment: Fragment | null = startFragment; + while (currentFragment) { + const trackData = currentFragment.trackData.get(this.internalTrack.id); + if (trackData) { + return { + fragmentIndex: binarySearchExact( + this.internalTrack.fragments, + currentFragment.moofOffset, + x => x.moofOffset, + ), + sampleIndex: 0, + correctSampleFound: true, + }; + } + + currentFragment = currentFragment.nextFragment; + } + } + return { - fragmentIndex: fragment ? 0 : -1, - sampleIndex: fragment ? 0 : -1, - correctSampleFound: !!fragment, + fragmentIndex: -1, + sampleIndex: -1, + correctSampleFound: false, }; }, - 0, + -Infinity, // Use -Infinity as a search timestamp to avoid using the lookup entries Infinity, options, ); @@ -2108,8 +2152,11 @@ abstract class IsobmffTrackBacking< ? this.internalTrack.fragmentLookupTable![lookupEntryIndex]! : null; + let nextFragmentIsFirstFragment = false; + if (fragmentIndex === -1) { isobmffReader.pos = lookupEntry?.moofOffset ?? 0; + nextFragmentIsFirstFragment = isobmffReader.pos === 0; } else { const fragment = this.internalTrack.fragments[fragmentIndex]!; @@ -2150,16 +2197,19 @@ abstract class IsobmffTrackBacking< if (index === -1) { // This is the first time we've seen this fragment isobmffReader.pos = startPos; - fragment = await demuxer.readFragment(); - if (prevFragment) prevFragment.nextFragment = fragment; - prevFragment = fragment; } else { // We already know this fragment fragment = demuxer.fragments[index]!; - // Even if we already know the fragment, we might not yet know its predecessor - if (prevFragment) prevFragment.nextFragment = fragment; - prevFragment = fragment; + } + + // Even if we already know the fragment, we might not yet know its predecessor, so always do this + if (prevFragment) prevFragment.nextFragment = fragment; + prevFragment = fragment; + + if (nextFragmentIsFirstFragment) { + fragment.isKnownToBeFirstFragment = true; + nextFragmentIsFirstFragment = false; } const { fragmentIndex, sampleIndex, correctSampleFound } = getBestMatch(); @@ -2224,6 +2274,15 @@ class IsobmffVideoTrackBacking extends IsobmffTrackBacking i return this.internalTrack.rotation; } + async getColorSpace(): Promise { + return { + primaries: this.internalTrack.info.colorSpace?.primaries, + transfer: this.internalTrack.info.colorSpace?.transfer, + matrix: this.internalTrack.info.colorSpace?.matrix, + fullRange: this.internalTrack.info.colorSpace?.fullRange, + }; + } + async getDecoderConfig(): Promise { if (!this.internalTrack.info.codec) { return null; diff --git a/src/isobmff/isobmff-muxer.ts b/src/isobmff/isobmff-muxer.ts index a163477..1ae60c4 100644 --- a/src/isobmff/isobmff-muxer.ts +++ b/src/isobmff/isobmff-muxer.ts @@ -7,7 +7,7 @@ import { IsobmffOutputFormatOptions, IsobmffOutputFormat, MovOutputFormat } from import { inlineTimestampRegex, SubtitleConfig, SubtitleCue, SubtitleMetadata } from '../subtitles'; import { parsePcmCodec, - PCM_CODECS, + PCM_AUDIO_CODECS, PcmAudioCodec, validateAudioChunkMetadata, validateSubtitleMetadata, @@ -239,7 +239,7 @@ export class IsobmffMuxer extends Muxer { compactlyCodedChunkTable: [], requiresPcmTransformation: !this.isFragmented - && (PCM_CODECS as readonly string[]).includes(track.source._codec), + && (PCM_AUDIO_CODECS as readonly string[]).includes(track.source._codec), }; this.trackDatas.push(newTrackData); @@ -668,16 +668,16 @@ export class IsobmffMuxer extends Muxer { // We can only finalize this fragment (and begin a new one) if we know that each track will be able to // start the new one with a key frame. const keyFrameQueuedEverywhere = this.trackDatas.every((otherTrackData) => { - if (otherTrackData.track.source._closed) { - return true; - } - if (trackData === otherTrackData) { return sample.type === 'key'; } const firstQueuedSample = otherTrackData.sampleQueue[0]; - return firstQueuedSample && firstQueuedSample.type === 'key'; + if (firstQueuedSample) { + return firstQueuedSample.type === 'key'; + } + + return otherTrackData.track.source._closed; }); if ( diff --git a/src/matroska/ebml.ts b/src/matroska/ebml.ts index 99c65c1..ff9d0c9 100644 --- a/src/matroska/ebml.ts +++ b/src/matroska/ebml.ts @@ -450,7 +450,7 @@ export class EBMLReader { } if (width !== 4 && width !== 8) { - throw new Error('Bad FLOAT size ' + width); + throw new Error('Bad float size ' + width); } const { view, offset } = this.reader.getViewAndOffset(this.pos, this.pos + width); diff --git a/src/matroska/matroska-demuxer.ts b/src/matroska/matroska-demuxer.ts index 8b82991..58863cd 100644 --- a/src/matroska/matroska-demuxer.ts +++ b/src/matroska/matroska-demuxer.ts @@ -69,6 +69,7 @@ type Cluster = { timestamp: number; trackData: Map; nextCluster: Cluster | null; + isKnownToBeFirstCluster: boolean; }; type ClusterTrackData = { @@ -415,6 +416,7 @@ export class MatroskaDemuxer extends Demuxer { timestamp: -1, trackData: new Map(), nextCluster: null, + isKnownToBeFirstCluster: false, }; this.currentCluster = cluster; this.readContiguousElements(this.clusterReader, size); @@ -772,7 +774,7 @@ export class MatroskaDemuxer extends Demuxer { const matrixCoefficients = reader.readUnsignedInt(size); const mapped = MATRIX_COEFFICIENTS_MAP_INVERSE[matrixCoefficients] ?? null; - this.currentTrack.info.colorSpace.matrix = mapped; + this.currentTrack.info.colorSpace.matrix = mapped as VideoColorSpaceInit['matrix']; }; break; case EBMLId.Range: { @@ -786,7 +788,7 @@ export class MatroskaDemuxer extends Demuxer { const transferCharacteristics = reader.readUnsignedInt(size); const mapped = TRANSFER_CHARACTERISTICS_MAP_INVERSE[transferCharacteristics] ?? null; - this.currentTrack.info.colorSpace.transfer = mapped; + this.currentTrack.info.colorSpace.transfer = mapped as VideoColorSpaceInit['transfer']; }; break; case EBMLId.Primaries: { @@ -794,7 +796,7 @@ export class MatroskaDemuxer extends Demuxer { const primaries = reader.readUnsignedInt(size); const mapped = COLOR_PRIMARIES_MAP_INVERSE[primaries] ?? null; - this.currentTrack.info.colorSpace.primaries = mapped; + this.currentTrack.info.colorSpace.primaries = mapped as VideoColorSpaceInit['primaries']; }; break; case EBMLId.Projection: { @@ -967,6 +969,10 @@ abstract class MatroskaTrackBacking< constructor(public internalTrack: InternalTrack) {} + getId() { + return this.internalTrack.id; + } + getCodec(): Promise { throw new Error('Not implemented on base class.'); } @@ -996,14 +1002,35 @@ abstract class MatroskaTrackBacking< async getFirstSample(options: SampleRetrievalOptions) { return this.performClusterLookup( () => { - const cluster = this.internalTrack.clusters[0]; + const startCluster = this.internalTrack.segment.clusters[0] ?? null; + if (startCluster?.isKnownToBeFirstCluster) { + // Walk from the very first cluster in the file until we find one with our track in it + let currentCluster: Cluster | null = startCluster; + while (currentCluster) { + const trackData = currentCluster.trackData.get(this.internalTrack.id); + if (trackData) { + return { + clusterIndex: binarySearchExact( + this.internalTrack.clusters, + currentCluster.elementStartPos, + x => x.elementStartPos, + ), + blockIndex: 0, + correctBlockFound: true, + }; + } + + currentCluster = currentCluster.nextCluster; + } + } + return { - clusterIndex: cluster ? 0 : -1, - blockIndex: cluster ? 0 : -1, - correctBlockFound: !!cluster, + clusterIndex: -1, + blockIndex: -1, + correctBlockFound: false, }; }, - 0, + -Infinity, // Use -Infinity as a search timestamp to avoid using the cues Infinity, options, ); @@ -1300,8 +1327,11 @@ abstract class MatroskaTrackBacking< ); const cuePoint = cuePointIndex !== -1 ? this.internalTrack.cuePoints[cuePointIndex]! : null; + let nextClusterIsFirstCluster = false; + if (clusterIndex === -1) { metadataReader.pos = cuePoint?.clusterPosition ?? segment.clusterSeekStartPos; + nextClusterIsFirstCluster = metadataReader.pos === segment.clusterSeekStartPos; } else { const cluster = this.internalTrack.clusters[clusterIndex]!; @@ -1344,15 +1374,18 @@ abstract class MatroskaTrackBacking< // This is the first time we've seen this cluster metadataReader.pos = elementStartPos; cluster = await demuxer.readCluster(segment); - - if (prevCluster) prevCluster.nextCluster = cluster; - prevCluster = cluster; } else { // We already know this cluster cluster = segment.clusters[index]!; - // Even if we already know the cluster, we might not yet know its predecessor - if (prevCluster) prevCluster.nextCluster = cluster; - prevCluster = cluster; + } + + // Even if we already know the cluster, we might not yet know its predecessor, so always do this + if (prevCluster) prevCluster.nextCluster = cluster; + prevCluster = cluster; + + if (nextClusterIsFirstCluster) { + cluster.isKnownToBeFirstCluster = true; + nextClusterIsFirstCluster = false; } const { clusterIndex, blockIndex, correctBlockFound } = getBestMatch(); @@ -1417,6 +1450,15 @@ class MatroskaVideoTrackBacking extends MatroskaTrackBacking return this.internalTrack.info.rotation; } + async getColorSpace(): Promise { + return { + primaries: this.internalTrack.info.colorSpace?.primaries, + transfer: this.internalTrack.info.colorSpace?.transfer, + matrix: this.internalTrack.info.colorSpace?.matrix, + fullRange: this.internalTrack.info.colorSpace?.fullRange, + }; + } + async getDecoderConfig(): Promise { if (!this.internalTrack.info.codec) { return null; diff --git a/src/matroska/matroska-muxer.ts b/src/matroska/matroska-muxer.ts index 36f8210..7bd0318 100644 --- a/src/matroska/matroska-muxer.ts +++ b/src/matroska/matroska-muxer.ts @@ -31,7 +31,7 @@ import { parseSubtitleTimestamp, } from '../subtitles'; import { - PCM_CODECS, + PCM_AUDIO_CODECS, PcmAudioCodec, generateAv1CodecConfigurationFromCodecString, parsePcmCodec, @@ -308,7 +308,7 @@ export class MatroskaMuxer extends Muxer { }, { id: EBMLId.ProjectionPoseRoll, - data: (rotation + 180) % 360 - 180, // [0, 270] -> [-180, 90] + data: new EBMLFloat32((rotation + 180) % 360 - 180), // [0, 270] -> [-180, 90] }, ], } @@ -321,7 +321,7 @@ export class MatroskaMuxer extends Muxer { } private audioSpecificTrackInfo(trackData: MatroskaAudioTrackData) { - const pcmInfo = (PCM_CODECS as readonly string[]).includes(trackData.track.source._codec) + const pcmInfo = (PCM_AUDIO_CODECS as readonly string[]).includes(trackData.track.source._codec) ? parsePcmCodec(trackData.track.source._codec as PcmAudioCodec) : null; @@ -659,16 +659,16 @@ export class MatroskaMuxer extends Muxer { // We can only finalize this cluster (and begin a new one) if we know that each track will be able to // start the new one with a key frame. const keyFrameQueuedEverywhere = this.trackDatas.every((otherTrackData) => { - if (otherTrackData.track.source._closed) { - return true; - } - if (trackData === otherTrackData) { return chunk.type === 'key'; } const firstQueuedSample = otherTrackData.chunkQueue[0]; - return firstQueuedSample && firstQueuedSample.type === 'key'; + if (firstQueuedSample) { + return firstQueuedSample.type === 'key'; + } + + return otherTrackData.track.source._closed; }); if ( diff --git a/src/media-sink.ts b/src/media-sink.ts index 229cc04..e472844 100644 --- a/src/media-sink.ts +++ b/src/media-sink.ts @@ -1,4 +1,4 @@ -import { parsePcmCodec, PCM_CODECS, PcmAudioCodec } from './codec'; +import { parsePcmCodec, PCM_AUDIO_CODECS, PcmAudioCodec } from './codec'; import { InputAudioTrack, InputVideoTrack } from './input-track'; import { AnyIterable, @@ -920,7 +920,7 @@ class PcmAudioDecoderWrapper extends DecoderWrapper | null = null; /** @internal */ _closed = false; /** @internal */ @@ -59,30 +60,42 @@ export abstract class MediaSource { /** @internal */ async _flush() {} - async close() { - if (this._closing) { + close() { + if (this._closingPromise) { throw new Error('Source already closed.'); } - if (!this._connectedTrack) { + const connectedTrack = this._connectedTrack; + + if (!connectedTrack) { throw new Error('Cannot call close without connecting the source to an output track.'); } - if (!this._connectedTrack.output._started) { + if (!connectedTrack.output._started) { throw new Error('Cannot call close before output has been started.'); } - this._closing = true; + return this._closingPromise = (async () => { + await this._flush(); - await this._flush(); + this._closed = true; - this._closed = true; + if (connectedTrack.output._finalizing) { + return; + } - if (this._connectedTrack.output._finalizing) { - return; + connectedTrack.output._muxer.onTrackClose(connectedTrack); + })(); + } + + /** @internal */ + async _flushOrWaitForClose() { + if (this._closingPromise) { + // Since closing also flushes, we don't want to do it twice + return this._closingPromise; + } else { + return this._flush(); } - - this._connectedTrack.output._muxer.onTrackClose(this._connectedTrack); } } @@ -126,10 +139,10 @@ export class EncodedVideoSampleSource extends VideoSource { /** @public */ export type VideoEncodingConfig = { codec: VideoCodec; - bitrate: number; + bitrate: number | Quality; latencyMode?: VideoEncoderConfig['latencyMode']; keyFrameInterval?: number; - onEncodedSample?: (chunk: EncodedVideoSample, meta: EncodedVideoChunkMetadata | undefined) => unknown; + onEncodedSample?: (sample: EncodedVideoSample, meta: EncodedVideoChunkMetadata | undefined) => unknown; onEncodingError?: (error: Error) => unknown; }; @@ -140,8 +153,8 @@ const validateVideoEncodingConfig = (config: VideoEncodingConfig) => { if (!VIDEO_CODECS.includes(config.codec)) { throw new TypeError(`Invalid video codec '${config.codec}'. Must be one of: ${VIDEO_CODECS.join(', ')}.`); } - if (!Number.isInteger(config.bitrate) || config.bitrate <= 0) { - throw new TypeError('config.bitrate must be a positive integer.'); + if (!(config.bitrate instanceof Quality) && (!Number.isInteger(config.bitrate) || config.bitrate <= 0)) { + throw new TypeError('config.bitrate must be a positive integer or a quality.'); } if (config.latencyMode !== undefined && !['quality', 'realtime'].includes(config.latencyMode)) { throw new TypeError('config.latencyMode, when provided, must be \'quality\' or \'realtime\'.'); @@ -205,7 +218,9 @@ class VideoEncoderWrapper { // in Matroska. this.encoder.encode(videoFrame, { ...encodeOptions, - keyFrame: keyFrameInterval === 0 || multipleOfKeyFrameInterval !== this.lastMultipleOfKeyFrameInterval, + keyFrame: encodeOptions?.keyFrame + || keyFrameInterval === 0 + || multipleOfKeyFrameInterval !== this.lastMultipleOfKeyFrameInterval, }); if (shouldClose) { @@ -234,16 +249,22 @@ class VideoEncoderWrapper { const { promise, resolve } = promiseWithResolvers(); this.ensureEncoderPromise = promise; + const width = videoFrame.codedWidth; + const height = videoFrame.codedHeight; + const bitrate = this.encodingConfig.bitrate instanceof Quality + ? this.encodingConfig.bitrate._toVideoBitrate(this.encodingConfig.codec, width, height) + : this.encodingConfig.bitrate; + const encoderConfig: VideoEncoderConfig = { codec: buildVideoCodecString( this.encodingConfig.codec, - videoFrame.codedWidth, - videoFrame.codedHeight, - this.encodingConfig.bitrate, + width, + height, + bitrate, ), - width: videoFrame.codedWidth, - height: videoFrame.codedHeight, - bitrate: this.encodingConfig.bitrate, + width, + height, + bitrate, framerate: this.source._connectedTrack?.metadata.frameRate, latencyMode: this.encodingConfig.latencyMode, ...getVideoEncoderConfigExtension(this.encodingConfig.codec), @@ -444,8 +465,8 @@ export class EncodedAudioSampleSource extends AudioSource { /** @public */ export type AudioEncodingConfig = { codec: AudioCodec; - bitrate?: number; - onEncodedSample?: (chunk: EncodedAudioSample, meta: EncodedAudioChunkMetadata | undefined) => unknown; + bitrate?: number | Quality; + onEncodedSample?: (sample: EncodedAudioSample, meta: EncodedAudioChunkMetadata | undefined) => unknown; onEncodingError?: (error: Error) => unknown; }; @@ -456,11 +477,18 @@ const validateAudioEncodingConfig = (config: AudioEncodingConfig) => { if (!AUDIO_CODECS.includes(config.codec)) { throw new TypeError(`Invalid audio codec '${config.codec}'. Must be one of: ${AUDIO_CODECS.join(', ')}.`); } - if (config.bitrate === undefined && !(PCM_CODECS as readonly string[]).includes(config.codec)) { + if ( + config.bitrate === undefined + && (!(PCM_AUDIO_CODECS as readonly string[]).includes(config.codec) || config.codec === 'flac') + ) { throw new TypeError('config.bitrate must be provided for compressed audio codecs.'); } - if (config.bitrate !== undefined && (!Number.isInteger(config.bitrate) || config.bitrate <= 0)) { - throw new TypeError('config.bitrate must be a positive integer.'); + if ( + config.bitrate !== undefined + && !(config.bitrate instanceof Quality) + && (!Number.isInteger(config.bitrate) || config.bitrate <= 0) + ) { + throw new TypeError('config.bitrate, when provided, must be a positive integer or a quality.'); } if (config.onEncodingError !== undefined && typeof config.onEncodingError !== 'function') { throw new TypeError('config.onEncodingError, when provided, must be a function.'); @@ -610,22 +638,27 @@ class AudioEncoderWrapper { const { promise, resolve } = promiseWithResolvers(); this.ensureEncoderPromise = promise; - if ((PCM_CODECS as readonly string[]).includes(this.encodingConfig.codec)) { + if ((PCM_AUDIO_CODECS as readonly string[]).includes(this.encodingConfig.codec)) { this.initPcmEncoder(); } else { if (typeof AudioEncoder === 'undefined') { throw new Error('AudioEncoder is not supported by this browser.'); } + const { numberOfChannels, sampleRate } = audioData; + const bitrate = this.encodingConfig.bitrate instanceof Quality + ? this.encodingConfig.bitrate._toAudioBitrate(this.encodingConfig.codec) + : this.encodingConfig.bitrate; + const encoderConfig: AudioEncoderConfig = { codec: buildAudioCodecString( this.encodingConfig.codec, - audioData.numberOfChannels, - audioData.sampleRate, + numberOfChannels, + sampleRate, ), - numberOfChannels: audioData.numberOfChannels, - sampleRate: audioData.sampleRate, - bitrate: this.encodingConfig.bitrate, + numberOfChannels, + sampleRate, + bitrate, ...getAudioEncoderConfigExtension(this.encodingConfig.codec), }; const support = await AudioEncoder.isConfigSupported(encoderConfig); diff --git a/src/misc.ts b/src/misc.ts index 500eea6..209e88a 100644 --- a/src/misc.ts +++ b/src/misc.ts @@ -60,27 +60,33 @@ const invertObject = (object: Reco return Object.fromEntries(Object.entries(object).map(([key, value]) => [value, key])) as Record; }; -// These maps are taken from https://www.matroska.org/technical/elements.html, -// which references the tables in ITU-T H.273 - they should be valid for Matroska and ISOBMFF. -export const COLOR_PRIMARIES_MAP: Record = { +// For the color space mappings, see Rec. ITU-T H.273. + +export const COLOR_PRIMARIES_MAP = { bt709: 1, // ITU-R BT.709 bt470bg: 5, // ITU-R BT.470BG smpte170m: 6, // ITU-R BT.601 525 - SMPTE 170M + bt2020: 9, // ITU-R BT.202 + smpte432: 12, // SMPTE EG 432-1 }; export const COLOR_PRIMARIES_MAP_INVERSE = invertObject(COLOR_PRIMARIES_MAP); -export const TRANSFER_CHARACTERISTICS_MAP: Record = { +export const TRANSFER_CHARACTERISTICS_MAP = { 'bt709': 1, // ITU-R BT.709 'smpte170m': 6, // SMPTE 170M + 'linear': 8, // Linear transfer characteristics 'iec61966-2-1': 13, // IEC 61966-2-1 + 'pg': 16, // Rec. ITU-R BT.2100-2 perceptual quantization (PQ) system + 'hlg': 18, // Rec. ITU-R BT.2100-2 hybrid loggamma (HLG) system }; export const TRANSFER_CHARACTERISTICS_MAP_INVERSE = invertObject(TRANSFER_CHARACTERISTICS_MAP); -export const MATRIX_COEFFICIENTS_MAP: Record = { - rgb: 0, // Identity - bt709: 1, // ITU-R BT.709 - bt470bg: 5, // ITU-R BT.470BG - smpte170m: 6, // SMPTE 170M +export const MATRIX_COEFFICIENTS_MAP = { + 'rgb': 0, // Identity + 'bt709': 1, // ITU-R BT.709 + 'bt470bg': 5, // ITU-R BT.470BG + 'smpte170m': 6, // SMPTE 170M + 'bt2020-ncl': 9, // ITU-R BT.2020-2 (non-constant luminance) }; export const MATRIX_COEFFICIENTS_MAP_INVERSE = invertObject(MATRIX_COEFFICIENTS_MAP); @@ -127,7 +133,7 @@ export class AsyncMutex { } } -export const rotationMatrix = (rotationInDegrees: Rotation): TransformationMatrix => { +export const rotationMatrix = (rotationInDegrees: number): TransformationMatrix => { const theta = rotationInDegrees * (Math.PI / 180); const cosTheta = Math.round(Math.cos(theta)); const sinTheta = Math.round(Math.sin(theta)); @@ -140,6 +146,18 @@ export const rotationMatrix = (rotationInDegrees: Rotation): TransformationMatri ]; }; +/** Extracts the rotation component from a transformation matrix, in degrees. */ +export const extractRotationFromMatrix = (matrix: TransformationMatrix) => { + const [m11, , , m21] = matrix; + + const scaleX = Math.hypot(m11, m21); + + const cosTheta = m11 / scaleX; + const sinTheta = m21 / scaleX; + + return Math.atan2(sinTheta, cosTheta) * (180 / Math.PI); +}; + export const IDENTITY_MATRIX = rotationMatrix(0); export const bytesToHexString = (bytes: Uint8Array) => { @@ -331,3 +349,17 @@ export const clamp = (value: number, min: number, max: number) => { }; export const UNDETERMINED_LANGUAGE = 'und'; + +/** @public */ +export const setVideoFrameTiming = (frame: VideoFrame, timing: { + timestamp?: number; + duration?: number; +}) => { + const clone = new VideoFrame(frame, { + timestamp: timing.timestamp && 1e6 * timing.timestamp, + duration: timing.duration && 1e6 * timing.duration, + }); + frame.close(); + + return clone; +}; diff --git a/src/output-format.ts b/src/output-format.ts index 0a426b9..d70890e 100644 --- a/src/output-format.ts +++ b/src/output-format.ts @@ -1,4 +1,14 @@ -import { AUDIO_CODECS, MediaCodec, PCM_CODECS, SUBTITLE_CODECS, VIDEO_CODECS } from './codec'; +import { + AUDIO_CODECS, + AudioCodec, + MediaCodec, + NON_PCM_AUDIO_CODECS, + PCM_AUDIO_CODECS, + SUBTITLE_CODECS, + SubtitleCodec, + VIDEO_CODECS, + VideoCodec, +} from './codec'; import { IsobmffMuxer } from './isobmff/isobmff-muxer'; import { MatroskaMuxer } from './matroska/matroska-muxer'; import { Muxer } from './muxer'; @@ -26,15 +36,18 @@ export abstract class OutputFormat { abstract getSupportedTrackCounts(): TrackCountLimits; getSupportedVideoCodecs() { - return this.getSupportedCodecs().filter(codec => (VIDEO_CODECS as readonly string[]).includes(codec)); + return this.getSupportedCodecs() + .filter(codec => (VIDEO_CODECS as readonly string[]).includes(codec)) as VideoCodec[]; } getSupportedAudioCodecs() { - return this.getSupportedCodecs().filter(codec => (AUDIO_CODECS as readonly string[]).includes(codec)); + return this.getSupportedCodecs() + .filter(codec => (AUDIO_CODECS as readonly string[]).includes(codec)) as AudioCodec[]; } getSupportedSubtitleCodecs() { - return this.getSupportedCodecs().filter(codec => (SUBTITLE_CODECS as readonly string[]).includes(codec)); + return this.getSupportedCodecs() + .filter(codec => (SUBTITLE_CODECS as readonly string[]).includes(codec)) as SubtitleCodec[]; } /** @internal */ @@ -99,9 +112,9 @@ export class Mp4OutputFormat extends IsobmffOutputFormat { static getSupportedCodecs(): MediaCodec[] { return [ - 'avc', 'hevc', 'vp8', 'vp9', 'av1', - 'aac', 'mp3', 'opus', 'vorbis', 'flac', // No PCM codecs - 'webvtt', + ...VIDEO_CODECS, + ...NON_PCM_AUDIO_CODECS, + ...SUBTITLE_CODECS, ]; } @@ -132,8 +145,8 @@ export class MovOutputFormat extends IsobmffOutputFormat { static getSupportedCodecs(): MediaCodec[] { return [ - 'avc', 'hevc', 'vp8', 'vp9', 'av1', - 'aac', 'mp3', 'opus', 'vorbis', 'flac', ...PCM_CODECS, + ...VIDEO_CODECS, + ...AUDIO_CODECS, ]; } @@ -199,11 +212,10 @@ export class MkvOutputFormat extends OutputFormat { static getSupportedCodecs(): MediaCodec[] { return [ - 'avc', 'hevc', 'vp8', 'vp9', 'av1', - 'aac', 'mp3', 'opus', 'vorbis', 'flac', - // pcm-s8, pcm-f32be, ulaw and alaw are not supported - 'pcm-u8', 'pcm-s16', 'pcm-s16be', 'pcm-s24', 'pcm-s24be', 'pcm-s32', 'pcm-s32be', 'pcm-f32', - 'webvtt', + ...VIDEO_CODECS, + ...NON_PCM_AUDIO_CODECS, + ...PCM_AUDIO_CODECS.filter(codec => !['pcm-s8', 'pcm-f32be', 'ulaw', 'alaw'].includes(codec)), + ...SUBTITLE_CODECS, ]; } } @@ -228,9 +240,9 @@ export class WebMOutputFormat extends MkvOutputFormat { static override getSupportedCodecs(): MediaCodec[] { return [ - 'vp8', 'vp9', 'av1', - 'opus', 'vorbis', - 'webvtt', + ...VIDEO_CODECS.filter(codec => ['vp8', 'vp9', 'av1'].includes(codec)), + ...AUDIO_CODECS.filter(codec => ['opus', 'vorbis'].includes(codec)), + ...SUBTITLE_CODECS, ]; } @@ -275,7 +287,9 @@ export class WaveOutputFormat extends OutputFormat { static getSupportedCodecs(): MediaCodec[] { return [ - 'pcm-u8', 'pcm-s16', 'pcm-s24', 'pcm-s32', 'pcm-f32', 'ulaw', 'alaw', + ...PCM_AUDIO_CODECS.filter(codec => + ['pcm-s16', 'pcm-s24', 'pcm-s32', 'pcm-f32', 'pcm-u8', 'ulaw', 'alaw'].includes(codec), + ), ]; } } diff --git a/src/output.ts b/src/output.ts index 7e31b57..11162a3 100644 --- a/src/output.ts +++ b/src/output.ts @@ -6,9 +6,12 @@ import { Target } from './target'; import { Writer } from './writer'; /** @public */ -export type OutputOptions = { - format: OutputFormat; - target: Target; +export type OutputOptions< + F extends OutputFormat = OutputFormat, + T extends Target = Target, +> = { + format: F; + target: T; }; /** @public */ @@ -63,11 +66,13 @@ const validateBaseTrackMetadata = (metadata: BaseTrackMetadata) => { }; /** @public */ -export class Output { - /** @internal */ - _format: OutputFormat; - /** @internal */ - _target: Target; +export class Output< + F extends OutputFormat = OutputFormat, + T extends Target = Target, +> { + format: F; + target: T; + /** @internal */ _muxer: Muxer; /** @internal */ @@ -83,7 +88,7 @@ export class Output { /** @internal */ _mutex = new AsyncMutex(); - constructor(options: OutputOptions) { + constructor(options: OutputOptions) { if (!options || typeof options !== 'object') { throw new TypeError('options must be an object.'); } @@ -99,8 +104,8 @@ export class Output { } options.target._output = this; - this._format = options.format; - this._target = options.target; + this.format = options.format; + this.target = options.target; this._writer = options.target._createWriter(); this._muxer = options.format._createMuxer(this); @@ -159,7 +164,7 @@ export class Output { } // Verify maximum track count constraints - const supportedTrackCounts = this._format.getSupportedTrackCounts(); + const supportedTrackCounts = this.format.getSupportedTrackCounts(); const presentTracksOfThisType = this._tracks.reduce( (count, track) => count + (track.type === type ? 1 : 0), 0, @@ -168,15 +173,15 @@ export class Output { if (presentTracksOfThisType === maxCount) { throw new Error( maxCount === 0 - ? `${this._format._getName()} does not support ${type} tracks.` - : (`${this._format._getName()} does not support more than ${maxCount} ${type} track` + ? `${this.format._getName()} does not support ${type} tracks.` + : (`${this.format._getName()} does not support more than ${maxCount} ${type} track` + `${maxCount === 1 ? '' : 's'}.`), ); } const maxTotalCount = supportedTrackCounts.total.max; if (this._tracks.length === maxTotalCount) { throw new Error( - `${this._format._getName()} does not support more than ${maxTotalCount} tracks` + `${this.format._getName()} does not support more than ${maxTotalCount} tracks` + `${maxTotalCount === 1 ? '' : 's'} in total.`, ); } @@ -190,48 +195,48 @@ export class Output { } as OutputTrack; if (track.type === 'video') { - const supportedVideoCodecs = this._format.getSupportedVideoCodecs(); + const supportedVideoCodecs = this.format.getSupportedVideoCodecs(); if (supportedVideoCodecs.length === 0) { throw new Error( - `${this._format._getName()} does not support video tracks.` - + this._format._codecUnsupportedHint(track.source._codec), + `${this.format._getName()} does not support video tracks.` + + this.format._codecUnsupportedHint(track.source._codec), ); } else if (!supportedVideoCodecs.includes(track.source._codec)) { throw new Error( - `Codec '${track.source._codec}' cannot be contained within ${this._format._getName()}. Supported` + `Codec '${track.source._codec}' cannot be contained within ${this.format._getName()}. Supported` + ` video codecs are: ${supportedVideoCodecs.map(codec => `'${codec}'`).join(', ')}.` - + this._format._codecUnsupportedHint(track.source._codec), + + this.format._codecUnsupportedHint(track.source._codec), ); } } else if (track.type === 'audio') { - const supportedAudioCodecs = this._format.getSupportedAudioCodecs(); + const supportedAudioCodecs = this.format.getSupportedAudioCodecs(); if (supportedAudioCodecs.length === 0) { throw new Error( - `${this._format._getName()} does not support audio tracks.` - + this._format._codecUnsupportedHint(track.source._codec), + `${this.format._getName()} does not support audio tracks.` + + this.format._codecUnsupportedHint(track.source._codec), ); } else if (!supportedAudioCodecs.includes(track.source._codec)) { throw new Error( - `Codec '${track.source._codec}' cannot be contained within ${this._format._getName()}. Supported` + `Codec '${track.source._codec}' cannot be contained within ${this.format._getName()}. Supported` + ` audio codecs are: ${supportedAudioCodecs.map(codec => `'${codec}'`).join(', ')}.` - + this._format._codecUnsupportedHint(track.source._codec), + + this.format._codecUnsupportedHint(track.source._codec), ); } } else if (track.type === 'subtitle') { - const supportedSubtitleCodecs = this._format.getSupportedSubtitleCodecs(); + const supportedSubtitleCodecs = this.format.getSupportedSubtitleCodecs(); if (supportedSubtitleCodecs.length === 0) { throw new Error( - `${this._format._getName()} does not support subtitle tracks.` - + this._format._codecUnsupportedHint(track.source._codec), + `${this.format._getName()} does not support subtitle tracks.` + + this.format._codecUnsupportedHint(track.source._codec), ); } else if (!supportedSubtitleCodecs.includes(track.source._codec)) { throw new Error( - `Codec '${track.source._codec}' cannot be contained within ${this._format._getName()}. Supported` + `Codec '${track.source._codec}' cannot be contained within ${this.format._getName()}. Supported` + ` subtitle codecs are: ${supportedSubtitleCodecs.map(codec => `'${codec}'`).join(', ')}.` - + this._format._codecUnsupportedHint(track.source._codec), + + this.format._codecUnsupportedHint(track.source._codec), ); } } @@ -249,7 +254,7 @@ export class Output { } // Verify minimum track count constraints - const supportedTrackCounts = this._format.getSupportedTrackCounts(); + const supportedTrackCounts = this.format.getSupportedTrackCounts(); for (const trackType of ALL_TRACK_TYPES) { const presentTracksOfThisType = this._tracks.reduce( (count, track) => count + (track.type === trackType ? 1 : 0), @@ -259,9 +264,9 @@ export class Output { if (presentTracksOfThisType < minCount) { throw new Error( minCount === supportedTrackCounts[trackType].max - ? (`${this._format._getName()} requires exactly ${minCount} ${trackType}` + ? (`${this.format._getName()} requires exactly ${minCount} ${trackType}` + ` track${minCount === 1 ? '' : 's'}.`) - : (`${this._format._getName()} requires at least ${minCount} ${trackType}` + : (`${this.format._getName()} requires at least ${minCount} ${trackType}` + ` track${minCount === 1 ? '' : 's'}.`), ); } @@ -270,9 +275,9 @@ export class Output { if (this._tracks.length < totalMinCount) { throw new Error( totalMinCount === supportedTrackCounts.total.max - ? (`${this._format._getName()} requires exactly ${totalMinCount} track` + ? (`${this.format._getName()} requires exactly ${totalMinCount} track` + `${totalMinCount === 1 ? '' : 's'}.`) - : (`${this._format._getName()} requires at least ${totalMinCount} track` + : (`${this.format._getName()} requires at least ${totalMinCount} track` + `${totalMinCount === 1 ? '' : 's'}.`), ); } @@ -302,7 +307,7 @@ export class Output { const release = await this._mutex.acquire(); - const promises = this._tracks.map(x => x.source._flush()); + const promises = this._tracks.map(x => x.source._flushOrWaitForClose()); await Promise.all(promises); await this._writer.close(); @@ -321,7 +326,7 @@ export class Output { const release = await this._mutex.acquire(); - const promises = this._tracks.map(x => x.source._flush()); + const promises = this._tracks.map(x => x.source._flushOrWaitForClose()); await Promise.all(promises); await this._muxer.finalize(); diff --git a/src/sample.ts b/src/sample.ts index 4a53810..4d31560 100644 --- a/src/sample.ts +++ b/src/sample.ts @@ -43,6 +43,10 @@ export class EncodedVideoSample { } is(otherSample: EncodedVideoSample) { + if (!(otherSample instanceof EncodedVideoSample)) { + throw new TypeError('otherSample must be an EncodedVideoSample.'); + } + return ( this.type === otherSample.type && this.timestamp === otherSample.timestamp @@ -51,9 +55,32 @@ export class EncodedVideoSample { ); } + clone(options?: { + timestamp?: number; + duration?: number; + }) { + if (options !== undefined && (!options || typeof options !== 'object')) { + throw new TypeError('options, when provided, must be an object.'); + } + if (options?.timestamp !== undefined && !Number.isFinite(options.timestamp)) { + throw new TypeError('options.timestamp, when provided, must be a number.'); + } + if (options?.duration !== undefined && !Number.isFinite(options.duration)) { + throw new TypeError('options.duration, when provided, must be a number.'); + } + + return new EncodedVideoSample( + this.data, + this.type, + options?.timestamp ?? this.timestamp, + options?.duration ?? this.duration, + this.byteLength, + ); + } + static fromEncodedVideoChunk(chunk: EncodedVideoChunk) { - if (typeof EncodedVideoChunk === 'undefined') { - throw new Error('Your browser does not support EncodedVideoChunk.'); + if (!(chunk instanceof EncodedVideoChunk)) { + throw new TypeError('chunk must be an EncodedVideoChunk.'); } const data = new Uint8Array(chunk.byteLength); @@ -108,6 +135,10 @@ export class EncodedAudioSample { } is(otherSample: EncodedAudioSample) { + if (!(otherSample instanceof EncodedAudioSample)) { + throw new TypeError('otherSample must be an EncodedAudioSample.'); + } + return ( this.type === otherSample.type && this.timestamp === otherSample.timestamp @@ -116,9 +147,32 @@ export class EncodedAudioSample { ); } + clone(options?: { + timestamp?: number; + duration?: number; + }) { + if (options !== undefined && (!options || typeof options !== 'object')) { + throw new TypeError('options, when provided, must be an object.'); + } + if (options?.timestamp !== undefined && !Number.isFinite(options.timestamp)) { + throw new TypeError('options.timestamp, when provided, must be a number.'); + } + if (options?.duration !== undefined && !Number.isFinite(options.duration)) { + throw new TypeError('options.duration, when provided, must be a number.'); + } + + return new EncodedAudioSample( + this.data, + this.type, + options?.timestamp ?? this.timestamp, + options?.duration ?? this.duration, + this.byteLength, + ); + } + static fromEncodedAudioChunk(chunk: EncodedAudioChunk) { - if (typeof EncodedAudioChunk === 'undefined') { - throw new Error('Your browser does not support EncodedAudioChunk.'); + if (!(chunk instanceof EncodedAudioChunk)) { + throw new TypeError('chunk must be an EncodedAudioChunk.'); } const data = new Uint8Array(chunk.byteLength); diff --git a/src/wave/wave-demuxer.ts b/src/wave/wave-demuxer.ts index 68e856e..31e04d8 100644 --- a/src/wave/wave-demuxer.ts +++ b/src/wave/wave-demuxer.ts @@ -183,6 +183,10 @@ const SAMPLE_SIZE_IN_FRAMES = 2048; class WaveAudioTrackBacking implements InputAudioTrackBacking { constructor(public demuxer: WaveDemuxer) {} + getId() { + return 1; + } + async getCodec() { return this.demuxer.getCodec(); }