arcodange/mediabunny
1
0
Fork 0
mirror of https://github.com/arcodange-org/mediabunny.git synced 2026-09-27 10:53:50 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add media conversion utility function, fix fragment/cluster reading issues, expand color space logic, add qualitative bitrates, & many other fixes and improvements

Browse Source
This commit is contained in:
Vanilagy
2025-01-19 18:55:12 +01:00
parent d323c2be8e
commit 70b88332b2
19 changed files with 1504 additions and 217 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dev/convert.html
+72
View File
@@ -0,0 +1,72 @@
<!DOCTYPE html>
<script src="../dist/metamuxer.js"></script>
<script type="module">
const fileInput = document.createElement('input');
fileInput.type = 'file';
document.body.append(fileInput);
const progress = document.createElement('progress');
progress.max = 1;
document.body.append(progress);
fileInput.addEventListener('change', async () => {
const file = fileInput.files[0];
const source = new Metamuxer.BlobSource(file);
const target = new Metamuxer.BufferTarget();
const outputFormat = new Metamuxer.Mp4OutputFormat()
console.time()
const res = await Metamuxer.convert({
input: new Metamuxer.Input({
formats: Metamuxer.ALL_FORMATS,
source
}),
output: new Metamuxer.Output({
format: outputFormat,
target
}),
/*
video: {
discard: true,
bitrate: Metamuxer.QUALITY_VERY_HIGH,
//bitrate: Metamuxer.QUALITY_VERY_HIGH
codec: 'av1',
//width: 800,
//fit: 'fill',
//rotate: 90,
},
audio: {
bitrate: Metamuxer.QUALITY_HIGH,
codec: 'aac',
},
*/
audio: {
discard: true
},
trim: {
start: 0,
end: 60
},
onProgress: (event) => {
progress.value = event.completion;
}
});
console.timeEnd()
console.log(res)
console.log("Done", target.buffer);
download(new Blob([target.buffer]), 'converted' + outputFormat.getFileExtension());
function download(blob, filename) {
const url = URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url;
a.download = filename;
a.click();
URL.revokeObjectURL(url);
}
});
</script>
dev/demux.html
+15 -23
View File
@@ -7,31 +7,22 @@
fileInput.type = 'file';
document.body.append(fileInput);
/*
const encoder = new AudioEncoder({
output: console.log,
error: console.error
});
encoder.configure({
codec: 'vorbis',
numberOfChannels: 2,
sampleRate: 48000,
bitrate: 128000
});
let audioData = new AudioData({
format: 'f32',
numberOfChannels: 2,
sampleRate: 48000,
timestamp: 0,
data: new Float32Array(2 * 48000),
numberOfFrames: 48000
});
encoder.encode(audioData);
*/
fileInput.addEventListener('change', async () => {
const file = fileInput.files[0];
const source = new Metamuxer.BlobSource(file);
const input = new Metamuxer.Input({
formats: Metamuxer.ALL_FORMATS,
source
});
const videoTrack = await input.getPrimaryVideoTrack();
const sink = new Metamuxer.EncodedVideoSampleSink(videoTrack);
console.log(await sink.getSample(16));
console.log(await sink.getFirstSample());
/*
const source = new Metamuxer.BufferSource(await file.arrayBuffer()) ?? new Metamuxer.BlobSource(file);
const start = performance.now();
@@ -45,6 +36,7 @@
target: new Metamuxer.BufferTarget()
});
output.start();
*/
/*
const audioTrack = await input.getPrimaryAudioTrack();
src/codec.ts
+135 -25
View File
@@ -11,17 +11,18 @@ import {
} from './misc';
import { SubtitleMetadata } from './subtitles';
// Codecs are ordered by encoding preference:
/** @public */
export const VIDEO_CODECS = [
'avc',
'hevc',
'vp8',
'vp9',
'av1',
'vp8',
] as const;
export const PCM_CODECS = [
'pcm-u8',
'pcm-s8',
/** @public */
export const PCM_AUDIO_CODECS = [
'pcm-s16', // We don't prefix 'le' so we're compatible with the WebCodecs-registered PCM codec strings
'pcm-s16be',
'pcm-s24',
@@ -30,26 +31,34 @@ export const PCM_CODECS = [
'pcm-s32be',
'pcm-f32',
'pcm-f32be',
'pcm-u8',
'pcm-s8',
'ulaw',
'alaw',
] as const;
/** @public */
export const AUDIO_CODECS = [
export const NON_PCM_AUDIO_CODECS = [
'aac',
'mp3',
'opus',
'mp3',
'vorbis',
'flac',
...PCM_CODECS,
] as const;
/** @public */
export const SUBTITLE_CODECS = ['webvtt'] as const; // TODO add the rest
export const AUDIO_CODECS = [
...NON_PCM_AUDIO_CODECS,
...PCM_AUDIO_CODECS,
] as const;
/** @public */
export const SUBTITLE_CODECS = [
'webvtt',
] as const; // TODO add the rest
/** @public */
export type VideoCodec = typeof VIDEO_CODECS[number];
/** @public */
export type AudioCodec = typeof AUDIO_CODECS[number];
export type PcmAudioCodec = typeof PCM_CODECS[number];
export type PcmAudioCodec = typeof PCM_AUDIO_CODECS[number];
/** @public */
export type SubtitleCodec = typeof SUBTITLE_CODECS[number];
/** @public */
@@ -855,7 +864,7 @@ export const buildAudioCodecString = (codec: AudioCodec, numberOfChannels: numbe
return 'vorbis';
} else if (codec === 'flac') {
return 'flac';
} else if ((PCM_CODECS as readonly string[]).includes(codec)) {
} else if ((PCM_AUDIO_CODECS as readonly string[]).includes(codec)) {
return codec;
}
@@ -892,7 +901,7 @@ export const extractAudioCodecString = (trackInfo: {
return 'vorbis';
} else if (codec === 'flac') {
return 'flac';
} else if (codec && (PCM_CODECS as readonly string[]).includes(codec)) {
} else if (codec && (PCM_AUDIO_CODECS as readonly string[]).includes(codec)) {
return codec;
}
@@ -958,7 +967,7 @@ export const parseAacAudioSpecificConfig = (bytes: Uint8Array | null) => {
const PCM_CODEC_REGEX = /^pcm-([usf])(\d+)+(be)?$/;
export const parsePcmCodec = (codec: PcmAudioCodec) => {
assert(PCM_CODECS.includes(codec));
assert(PCM_AUDIO_CODECS.includes(codec));
if (codec === 'ulaw') {
return { dataType: 'ulaw' as const, sampleSize: 1 as const, littleEndian: true, silentValue: 255 };
@@ -1021,6 +1030,91 @@ export const getAudioEncoderConfigExtension = (codec: AudioCodec) => {
return {};
};
/** @public */
export class Quality {
/** @internal */
_factor: number;
/** @internal */
constructor(factor: number) {
this._factor = factor;
}
/** @internal */
_toVideoBitrate(codec: VideoCodec, width: number, height: number) {
const pixels = width * height;
const codecEfficiencyFactors = {
avc: 1.0, // H.264/AVC (baseline)
hevc: 0.6, // H.265/HEVC (~40% more efficient than AVC)
vp9: 0.6, // Similar to HEVC
av1: 0.4, // ~60% more efficient than AVC
vp8: 1.2, // Slightly less efficient than AVC
};
const getBaseBitrateForPixels = (pixelCount: number): number => {
const referencePixels = 1920 * 1080;
const referenceBitrate = 2000000;
// Non-linear scaling
const scaleFactor = Math.pow(pixelCount / referencePixels, 0.75);
return referenceBitrate * scaleFactor;
};
const baseBitrate = getBaseBitrateForPixels(pixels);
const codecAdjustedBitrate = baseBitrate * codecEfficiencyFactors[codec];
const finalBitrate = codecAdjustedBitrate * this._factor;
return Math.ceil(finalBitrate / 1000) * 1000;
}
/** @internal */
_toAudioBitrate(codec: AudioCodec) {
if ((PCM_AUDIO_CODECS as readonly string[]).includes(codec) || codec === 'flac') {
return -1;
}
const baseRates = {
aac: 128000, // 128kbps base for AAC
opus: 64000, // 64kbps base for Opus
mp3: 160000, // 160kbps base for MP3
vorbis: 64000, // 64kbps base for Vorbis
};
const baseBitrate = baseRates[codec as keyof typeof baseRates];
if (!baseBitrate) {
throw new Error(`Unhandled codec: ${codec}`);
}
let finalBitrate = baseBitrate * this._factor;
if (codec === 'aac') {
// AAC only works with specific bitrates, let's find the closest
const validRates = [96000, 128000, 160000, 192000];
finalBitrate = validRates.reduce((prev, curr) =>
Math.abs(curr - finalBitrate) < Math.abs(prev - finalBitrate) ? curr : prev,
);
} else if (codec === 'opus' || codec === 'vorbis') {
finalBitrate = Math.max(6000, finalBitrate);
} else if (codec === 'mp3') {
finalBitrate = Math.round(finalBitrate / 32000) * 32000;
}
return Math.round(finalBitrate / 1000) * 1000;
}
}
/** @public */
export const QUALITY_VERY_LOW = new Quality(0.4);
/** @public */
export const QUALITY_LOW = new Quality(0.6);
/** @public */
export const QUALITY_MEDIUM = new Quality(1);
/** @public */
export const QUALITY_HIGH = new Quality(2);
/** @public */
export const QUALITY_VERY_HIGH = new Quality(4);
const VALID_VIDEO_CODEC_STRING_PREFIXES = ['avc1', 'avc3', 'hev1', 'hvc1', 'vp8', 'vp09', 'av01'];
const AVC_CODEC_STRING_REGEX = /^(avc1|avc3)\.[0-9a-fA-F]{6}$/;
const HEVC_CODEC_STRING_REGEX = /^(hev1|hvc1)\.(?:[ABC]?\d+)\.[0-9a-fA-F]{1,8}\.[LH]\d+(?:\.[0-9a-fA-F]{1,2}){0,6}$/;
@@ -1287,10 +1381,10 @@ export const validateAudioChunkMetadata = (metadata: EncodedAudioChunkMetadata |
|| metadata.decoderConfig.codec.startsWith('ulaw')
|| metadata.decoderConfig.codec.startsWith('alaw')
) {
if (!(PCM_CODECS as readonly string[]).includes(metadata.decoderConfig.codec)) {
if (!(PCM_AUDIO_CODECS as readonly string[]).includes(metadata.decoderConfig.codec)) {
throw new TypeError(
'Audio chunk metadata decoder configuration codec string for PCM must be one of the supported PCM'
+ ` codecs (${PCM_CODECS.join(', ')}).`,
+ ` codecs (${PCM_AUDIO_CODECS.join(', ')}).`,
);
}
}
@@ -1331,7 +1425,7 @@ export const canEncode = (codec: MediaCodec) => {
export const canEncodeVideo = async (codec: VideoCodec, { width = 1280, height = 720, bitrate = 1e6 }: {
width?: number;
height?: number;
bitrate?: 1e6;
bitrate?: number;
} = {}) => {
if (!VIDEO_CODECS.includes(codec)) {
return false;
@@ -1380,7 +1474,7 @@ export const canEncodeAudio = async (codec: AudioCodec, { numberOfChannels = 2,
throw new TypeError('bitrate must be a positive integer.');
}
if ((PCM_CODECS as readonly string[]).includes(codec)) {
if ((PCM_AUDIO_CODECS as readonly string[]).includes(codec)) {
return true; // Because we encode these ourselves
}
@@ -1420,19 +1514,35 @@ export const getEncodableCodecs = async (): Promise<MediaCodec[]> => {
};
/** @public */
export const getEncodableVideoCodecs = async (): Promise<VideoCodec[]> => {
const bools = await Promise.all(VIDEO_CODECS.map(canEncode));
return VIDEO_CODECS.filter((_, i) => bools[i]);
export const getEncodableVideoCodecs = async (
checkedCodecs = VIDEO_CODECS as unknown as VideoCodec[],
options?: {
width?: number;
height?: number;
bitrate?: number;
},
): Promise<VideoCodec[]> => {
const bools = await Promise.all(checkedCodecs.map(codec => canEncodeVideo(codec, options)));
return checkedCodecs.filter((_, i) => bools[i]);
};
/** @public */
export const getEncodableAudioCodecs = async (): Promise<AudioCodec[]> => {
const bools = await Promise.all(AUDIO_CODECS.map(canEncode));
return AUDIO_CODECS.filter((_, i) => bools[i]);
export const getEncodableAudioCodecs = async (
checkedCodecs = AUDIO_CODECS as unknown as AudioCodec[],
options?: {
numberOfChannels?: number;
sampleRate?: number;
bitrate?: number;
},
): Promise<AudioCodec[]> => {
const bools = await Promise.all(checkedCodecs.map(codec => canEncodeAudio(codec, options)));
return checkedCodecs.filter((_, i) => bools[i]);
};
/** @public */
export const getEncodableSubtitleCodecs = async (): Promise<SubtitleCodec[]> => {
const bools = await Promise.all(SUBTITLE_CODECS.map(canEncode));
return SUBTITLE_CODECS.filter((_, i) => bools[i]);
export const getEncodableSubtitleCodecs = async (
checkedCodecs = SUBTITLE_CODECS as unknown as SubtitleCodec[],
): Promise<SubtitleCodec[]> => {
const bools = await Promise.all(checkedCodecs.map(canEncodeSubtitles));
return checkedCodecs.filter((_, i) => bools[i]);
};
src/conversion.ts
+833
View File
@@ -0,0 +1,833 @@
import {
AUDIO_CODECS,
AudioCodec,
getEncodableAudioCodecs,
getEncodableVideoCodecs,
NON_PCM_AUDIO_CODECS,
Quality,
QUALITY_HIGH,
VIDEO_CODECS,
VideoCodec,
} from './codec';
import { Input } from './input';
import { InputAudioTrack, InputTrack, InputVideoTrack } from './input-track';
import {
AudioBufferSink,
AudioDataSink,
CanvasSink,
EncodedAudioSampleSink,
EncodedVideoSampleSink,
VideoFrameSink,
} from './media-sink';
import {
AudioBufferSource,
AudioDataSource,
AudioEncodingConfig,
AudioSource,
CanvasSource,
EncodedAudioSampleSource,
EncodedVideoSampleSource,
VideoEncodingConfig,
VideoFrameSource,
VideoSource,
} from './media-source';
import { assert, clamp, promiseWithResolvers, Rotation, setVideoFrameTiming } from './misc';
import { Output, TrackType } from './output';
/** @public */
export type ConversionOptions = {
input: Input;
output: Output;
video?: {
discard?: boolean;
codec?: VideoCodec;
bitrate?: VideoEncodingConfig['bitrate'];
width?: number;
height?: number;
fit?: 'fill' | 'contain' | 'cover';
rotate?: Rotation;
forceReencode?: boolean;
};
audio?: {
discard?: boolean;
codec?: AudioCodec;
bitrate?: AudioEncodingConfig['bitrate'];
numberOfChannels?: number;
sampleRate?: number;
forceReencode?: boolean;
};
trim?: {
start: number;
end: number;
};
onProgress?: (event: { completion: number }) => unknown;
};
/** @public */
export type ConversionInfo = {
utilizedTracks: InputTrack[];
discardedTracks: {
track: InputTrack;
reason:
| 'discardedByUser'
| 'maxTrackCountReached'
| 'maxTrackCountOfTypeReached'
| 'unknownSourceCodec'
| 'undecodableSourceCodec'
| 'noEncodableTargetCodec';
}[];
};
const FALLBACK_NUMBER_OF_CHANNELS = 2;
const FALLBACK_SAMPLE_RATE = 48000;
/** @public */
export const convert = (options: ConversionOptions) => {
const conversion = new Conversion(options);
return conversion.execute();
};
class Conversion {
input: Input;
output: Output;
startTimestamp: number;
endTimestamp: number;
addedCounts: Record<TrackType, number> = {
video: 0,
audio: 0,
subtitle: 0,
};
totalTrackCount = 0;
trackPromises: Promise<void>[] = [];
started: Promise<void>;
start: () => void;
synchronizer = new TrackSynchronizer();
totalDuration: number | null = null;
maxTimestamps = new Map<number, number>(); // Track ID -> timestamp
result: ConversionInfo;
constructor(public options: ConversionOptions) {
if (!options || typeof options !== 'object') {
throw new TypeError('options must be an object.');
}
if (!(options.input instanceof Input)) {
throw new TypeError('options.input must be an Input.');
}
if (!(options.output instanceof Output)) {
throw new TypeError('options.output must be an Output.');
}
if (options.video !== undefined && (!options.video || typeof options.video !== 'object')) {
throw new TypeError('options.video, when provided, must be an object.');
}
if (options.video?.discard !== undefined && typeof options.video.discard !== 'boolean') {
throw new TypeError('options.video.discard, when provided, must be a boolean.');
}
if (options.video?.forceReencode !== undefined && typeof options.video.forceReencode !== 'boolean') {
throw new TypeError('options.video.forceReencode, when provided, must be a boolean.');
}
if (options.video?.codec !== undefined && !VIDEO_CODECS.includes(options.video.codec)) {
throw new TypeError(
`options.video.codec, when provided, must be one of: ${VIDEO_CODECS.join(', ')}.`,
);
}
if (
options.video?.bitrate !== undefined
&& !(options.video.bitrate instanceof Quality)
&& (!Number.isInteger(options.video.bitrate) || options.video.bitrate <= 0)
) {
throw new TypeError('options.video.bitrate, when provided, must be a positive integer or a quality.');
}
if (
options.video?.width !== undefined
&& (!Number.isInteger(options.video.width) || options.video.width <= 0)
) {
throw new TypeError('options.video.width, when provided, must be a positive integer.');
}
if (
options.video?.height !== undefined
&& (!Number.isInteger(options.video.height) || options.video.height <= 0)
) {
throw new TypeError('options.video.height, when provided, must be a positive integer.');
}
if (options.video?.fit !== undefined && !['fill', 'contain', 'cover'].includes(options.video.fit)) {
throw new TypeError('options.video.fit, when provided, must be one of "fill", "contain", or "cover".');
}
if (
options.video?.width !== undefined
&& options.video.height !== undefined
&& options.video.fit === undefined
) {
throw new TypeError(
'When both options.video.width and options.video.height are provided, options.video.fit must also be'
+ ' provided.',
);
}
if (options.video?.rotate !== undefined && ![0, 90, 180, 270].includes(options.video.rotate)) {
throw new TypeError('options.video.rotate, when provided, must be 0, 90, 180 or 270.');
}
if (options.audio !== undefined && (!options.audio || typeof options.audio !== 'object')) {
throw new TypeError('options.video, when provided, must be an object.');
}
if (options.audio?.discard !== undefined && typeof options.audio.discard !== 'boolean') {
throw new TypeError('options.audio.discard, when provided, must be a boolean.');
}
if (options.audio?.forceReencode !== undefined && typeof options.audio.forceReencode !== 'boolean') {
throw new TypeError('options.audio.forceReencode, when provided, must be a boolean.');
}
if (options.audio?.codec !== undefined && !AUDIO_CODECS.includes(options.audio.codec)) {
throw new TypeError(
`options.audio.codec, when provided, must be one of: ${AUDIO_CODECS.join(', ')}.`,
);
}
if (
options.audio?.bitrate !== undefined
&& !(options.audio.bitrate instanceof Quality)
&& (!Number.isInteger(options.audio.bitrate) || options.audio.bitrate <= 0)
) {
throw new TypeError('options.audio.bitrate, when provided, must be a positive integer or a quality.');
}
if (
options.audio?.numberOfChannels !== undefined
&& (!Number.isInteger(options.audio.numberOfChannels) || options.audio.numberOfChannels <= 0)
) {
throw new TypeError('options.audio.numberOfChannels, when provided, must be a positive integer.');
}
if (
options.audio?.sampleRate !== undefined
&& (!Number.isInteger(options.audio.sampleRate) || options.audio.sampleRate <= 0)
) {
throw new TypeError('options.audio.sampleRate, when provided, must be a positive integer.');
}
if (options.trim !== undefined && (!options.trim || typeof options.trim !== 'object')) {
throw new TypeError('options.trim, when provided, must be an object.');
}
if (options.trim?.start !== undefined && (!Number.isFinite(options.trim.start) || options.trim.start < 0)) {
throw new TypeError('options.trim.start, when provided, must be a non-negative number.');
}
if (options.trim?.end !== undefined && (!Number.isFinite(options.trim.end) || options.trim.end < 0)) {
throw new TypeError('options.trim.end, when provided, must be a non-negative number.');
}
if (
options.trim?.start !== undefined
&& options.trim.end !== undefined
&& options.trim.start >= options.trim.end) {
throw new TypeError('options.trim.start must be less than options.trim.end.');
}
if (options.onProgress !== undefined && typeof options.onProgress !== 'function') {
throw new TypeError('options.onProgress, when provided, must be a function.');
}
this.input = options.input;
this.output = options.output;
this.startTimestamp = options.trim?.start ?? 0;
this.endTimestamp = options.trim?.end ?? Infinity;
const { promise: started, resolve: start } = promiseWithResolvers();
this.started = started;
this.start = start;
this.result = {
utilizedTracks: [],
discardedTracks: [],
};
}
async execute() {
const inputTracks = await this.input.getTracks();
const outputTrackCounts = this.output.format.getSupportedTrackCounts();
for (const track of inputTracks) {
if (this.totalTrackCount === outputTrackCounts.total.max) {
this.result.discardedTracks.push({
track,
reason: 'maxTrackCountReached',
});
continue;
}
const type = track.getType();
if (this.addedCounts[type] === outputTrackCounts[type].max) {
this.result.discardedTracks.push({
track,
reason: 'maxTrackCountOfTypeReached',
});
continue;
}
if (track.isVideoTrack()) {
if (this.options.video?.discard) {
this.result.discardedTracks.push({
track,
reason: 'discardedByUser',
});
continue;
}
await this.processVideoTrack(track);
} else if (track.isAudioTrack()) {
if (this.options.audio?.discard) {
this.result.discardedTracks.push({
track,
reason: 'discardedByUser',
});
continue;
}
await this.processAudioTrack(track);
}
}
if (this.options.onProgress) {
this.totalDuration = Math.min(
await this.input.computeDuration() - this.startTimestamp,
this.endTimestamp - this.startTimestamp,
);
this.options.onProgress({ completion: 0 });
}
await this.output.start();
this.start();
await Promise.all(this.trackPromises);
await this.output.finalize();
this.options.onProgress?.({ completion: 1 });
return this.result;
}
async processVideoTrack(track: InputVideoTrack) {
const trackId = track.getId();
const sourceCodec = await track.getCodec();
if (!sourceCodec) {
this.result.discardedTracks.push({
track,
reason: 'unknownSourceCodec',
});
return;
}
let videoSource: VideoSource;
const originalWidth = await track.getCodedWidth();
const originalHeight = await track.getCodedHeight();
const originalAspectRatio = originalWidth / originalHeight;
let width = originalWidth;
let height = originalHeight;
// A lot of video encoders require that the dimensions be multiples of 2
const ceilToMultipleOfTwo = (value: number) => Math.ceil(value / 2) * 2;
if (this.options.video?.width !== undefined && this.options.video.height === undefined) {
width = ceilToMultipleOfTwo(this.options.video.width);
height = ceilToMultipleOfTwo(Math.round(width / originalAspectRatio));
} else if (this.options.video?.width === undefined && this.options.video?.height !== undefined) {
height = ceilToMultipleOfTwo(this.options.video.height);
width = ceilToMultipleOfTwo(Math.round(height * originalAspectRatio));
} else if (this.options.video?.width !== undefined && this.options.video.height !== undefined) {
width = ceilToMultipleOfTwo(this.options.video.width);
height = ceilToMultipleOfTwo(this.options.video.height);
}
const needsReencode = !!this.options.video?.forceReencode || this.startTimestamp > 0;
const needsResize = width !== originalWidth || height !== originalHeight;
let videoCodecs = this.output.format.getSupportedVideoCodecs();
if (
!needsReencode
&& !this.options.video?.bitrate
&& !needsResize
&& videoCodecs.includes(sourceCodec)
&& (!this.options.video?.codec || this.options.video?.codec === sourceCodec)
) {
// Fast path, we can simply copy over the encoded samples
const source = new EncodedVideoSampleSource(sourceCodec);
videoSource = source;
this.trackPromises.push((async () => {
await this.started;
const sink = new EncodedVideoSampleSink(track);
const decoderConfig = await track.getDecoderConfig();
const meta: EncodedVideoChunkMetadata = { decoderConfig: decoderConfig ?? undefined };
for await (const sample of sink.samples(undefined, this.endTimestamp)) {
if (this.synchronizer.shouldWait(trackId, sample.timestamp)) {
await this.synchronizer.wait(sample.timestamp);
}
await source.digest(sample, meta);
this.reportProgress(trackId, sample.timestamp + sample.duration);
}
await source.close();
this.synchronizer.closeTrack(trackId);
})());
} else {
// We need to decode & reencode the video
const canDecode = await track.canDecode();
if (!canDecode) {
this.result.discardedTracks.push({
track,
reason: 'undecodableSourceCodec',
});
return;
}
if (this.options.video?.codec) {
videoCodecs = videoCodecs.filter(codec => codec === this.options.video?.codec);
}
const encodableCodecs = await getEncodableVideoCodecs(videoCodecs, {
width: needsResize ? width : await track.getCodedWidth(),
height: needsResize ? height : await track.getCodedHeight(),
});
if (encodableCodecs.length === 0) {
this.result.discardedTracks.push({
track,
reason: 'noEncodableTargetCodec',
});
return;
}
const encodingConfig: VideoEncodingConfig = {
codec: encodableCodecs[0]!,
bitrate: this.options.video?.bitrate ?? QUALITY_HIGH,
onEncodedSample: sample => this.reportProgress(trackId, sample.timestamp + sample.duration),
};
if (needsResize) {
// For resizing, we draw the frame onto a canvas and then encode the canvas
const canvas = document.createElement('canvas');
canvas.width = width;
canvas.height = height;
const context = canvas.getContext('2d', {
alpha: false,
})!;
const source = new CanvasSource(canvas, encodingConfig);
videoSource = source;
this.trackPromises.push((async () => {
await this.started;
const sink = new CanvasSink(track);
const iterator = sink.canvases(this.startTimestamp, this.endTimestamp);
for await (const { canvas, timestamp, duration } of iterator) {
if (this.synchronizer.shouldWait(trackId, timestamp)) {
await this.synchronizer.wait(timestamp);
}
if (!this.options.video?.fit || this.options.video.fit === 'fill') {
context.drawImage(canvas, 0, 0, width, height);
} else if (this.options.video.fit === 'contain') {
const scale = Math.min(width / canvas.width, height / canvas.height);
const newWidth = canvas.width * scale;
const newHeight = canvas.height * scale;
const dx = (width - newWidth) / 2;
const dy = (height - newHeight) / 2;
context.drawImage(canvas, 0, 0, canvas.width, canvas.height, dx, dy, newWidth, newHeight);
} else if (this.options.video.fit === 'cover') {
const scale = Math.max(width / canvas.width, height / canvas.height);
const newWidth = canvas.width * scale;
const newHeight = canvas.height * scale;
const dx = (width - newWidth) / 2;
const dy = (height - newHeight) / 2;
context.drawImage(canvas, 0, 0, canvas.width, canvas.height, dx, dy, newWidth, newHeight);
}
await source.digest(Math.max(timestamp - this.startTimestamp, 0), duration);
}
await source.close();
this.synchronizer.closeTrack(trackId);
})());
} else {
const source = new VideoFrameSource(encodingConfig);
videoSource = source;
this.trackPromises.push((async () => {
await this.started;
const sink = new VideoFrameSink(track);
for await (const { frame, timestamp } of sink.frames(this.startTimestamp, this.endTimestamp)) {
if (this.synchronizer.shouldWait(trackId, timestamp)) {
await this.synchronizer.wait(timestamp);
}
const clone = setVideoFrameTiming(frame, {
timestamp: Math.max(timestamp - this.startTimestamp, 0),
});
await source.digest(clone);
clone.close();
}
await source.close();
this.synchronizer.closeTrack(trackId);
})());
}
}
// Rotation metadata is reset if we do resizing
const baseRotation = needsResize ? 0 : await track.getRotation();
this.output.addVideoTrack(videoSource, {
languageCode: await track.getLanguageCode(),
rotation: (baseRotation + (this.options.video?.rotate ?? 0)) % 360 as Rotation,
});
this.addedCounts.video++;
this.totalTrackCount++;
this.result.utilizedTracks.push(track);
}
async processAudioTrack(track: InputAudioTrack) {
const trackId = track.getId();
const sourceCodec = await track.getCodec();
if (!sourceCodec) {
this.result.discardedTracks.push({
track,
reason: 'unknownSourceCodec',
});
return;
}
let audioSource: AudioSource;
const originalNumberOfChannels = await track.getNumberOfChannels();
const originalSampleRate = await track.getSampleRate();
let numberOfChannels = this.options.audio?.numberOfChannels ?? originalNumberOfChannels;
let sampleRate = this.options.audio?.sampleRate ?? originalSampleRate;
let needsResample = numberOfChannels !== originalNumberOfChannels
|| sampleRate !== originalSampleRate
|| this.startTimestamp > 0;
let audioCodecs = this.output.format.getSupportedAudioCodecs();
if (
!this.options.audio?.forceReencode
&& !this.options.audio?.bitrate
&& !needsResample
&& audioCodecs.includes(sourceCodec)
&& (!this.options.audio?.codec || this.options.audio.codec === sourceCodec)
) {
// Fast path, we can simply copy over the encoded samples
const source = new EncodedAudioSampleSource(sourceCodec);
audioSource = source;
this.trackPromises.push((async () => {
await this.started;
const sink = new EncodedAudioSampleSink(track);
const decoderConfig = await track.getDecoderConfig();
const meta: EncodedAudioChunkMetadata = { decoderConfig: decoderConfig ?? undefined };
for await (const sample of sink.samples(undefined, this.endTimestamp)) {
if (this.synchronizer.shouldWait(trackId, sample.timestamp)) {
await this.synchronizer.wait(sample.timestamp);
}
await source.digest(sample, meta);
this.reportProgress(trackId, sample.timestamp + sample.duration);
}
await source.close();
this.synchronizer.closeTrack(trackId);
})());
} else {
// We need to decode & reencode the audio
const canDecode = await track.canDecode();
if (!canDecode) {
this.result.discardedTracks.push({
track,
reason: 'undecodableSourceCodec',
});
return;
}
let codecOfChoice: AudioCodec | null = null;
if (this.options.audio?.codec) {
audioCodecs = audioCodecs.filter(codec => codec === this.options.audio!.codec);
}
const encodableCodecs = await getEncodableAudioCodecs(audioCodecs, {
numberOfChannels,
sampleRate,
});
if (
!encodableCodecs.some(codec => (NON_PCM_AUDIO_CODECS as readonly string[]).includes(codec))
&& audioCodecs.some(codec => (NON_PCM_AUDIO_CODECS as readonly string[]).includes(codec))
&& (numberOfChannels !== FALLBACK_NUMBER_OF_CHANNELS || sampleRate !== FALLBACK_SAMPLE_RATE)
) {
// We could not find a compatible non-PCM codec despite the container supporting them. This can be
// caused by strange channel count or sample rate configurations. Therefore, let's try again but with
// fallback parameters.
const encodableCodecsWithDefaultParams = await getEncodableAudioCodecs(audioCodecs, {
numberOfChannels: FALLBACK_NUMBER_OF_CHANNELS,
sampleRate: FALLBACK_SAMPLE_RATE,
});
if (
encodableCodecsWithDefaultParams
.some(codec => (NON_PCM_AUDIO_CODECS as readonly string[]).includes(codec))
) {
// We are able to encode using a non-PCM codec, but it'll require resampling
needsResample = true;
codecOfChoice = encodableCodecsWithDefaultParams[0]!;
numberOfChannels = FALLBACK_NUMBER_OF_CHANNELS;
sampleRate = FALLBACK_SAMPLE_RATE;
}
} else {
codecOfChoice = encodableCodecs[0] ?? null;
}
if (codecOfChoice === null) {
this.result.discardedTracks.push({
track,
reason: 'noEncodableTargetCodec',
});
return;
}
if (needsResample) {
audioSource = await this.resampleAudio(track, codecOfChoice, numberOfChannels, sampleRate);
} else {
const source = new AudioDataSource({
codec: codecOfChoice,
bitrate: this.options.audio?.bitrate ?? QUALITY_HIGH,
onEncodedSample: sample => this.reportProgress(trackId, sample.timestamp + sample.duration),
});
audioSource = source;
this.trackPromises.push((async () => {
await this.started;
const sink = new AudioDataSink(track);
for await (const { data, timestamp } of sink.data(undefined, this.startTimestamp)) {
if (this.synchronizer.shouldWait(trackId, timestamp)) {
await this.synchronizer.wait(timestamp);
}
await source.digest(data);
data.close();
}
await source.close();
this.synchronizer.closeTrack(trackId);
})());
}
}
this.output.addAudioTrack(audioSource, {
languageCode: await track.getLanguageCode(),
});
this.addedCounts.audio++;
this.totalTrackCount++;
this.result.utilizedTracks.push(track);
}
/**
* Resamples the audio by decoding it, playing it onto an OfflineAudioContext and encoding the
* resulting AudioBuffer.
*/
async resampleAudio(
track: InputAudioTrack,
codec: AudioCodec,
targetNumberOfChannels: number,
targetSampleRate: number,
) {
const trackId = track.getId();
const source = new AudioBufferSource({
codec,
bitrate: this.options.audio?.bitrate ?? QUALITY_HIGH,
onEncodedSample: sample => this.reportProgress(trackId, sample.timestamp + sample.duration),
});
this.trackPromises.push((async () => {
await this.started;
const trackDuration = Math.min(
await track.computeDuration() - this.startTimestamp,
this.endTimestamp - this.startTimestamp,
);
const totalFrameCount = Math.round(trackDuration * FALLBACK_SAMPLE_RATE);
const MAX_CHUNK_LENGTH = 5 * FALLBACK_SAMPLE_RATE;
let currentContextStartFrame = 0;
let currentContext: OfflineAudioContext | null = new OfflineAudioContext({
length: Math.min(totalFrameCount - currentContextStartFrame, MAX_CHUNK_LENGTH),
numberOfChannels: targetNumberOfChannels,
sampleRate: targetSampleRate,
});
const sink = new AudioBufferSink(track);
for await (const { buffer, timestamp, duration } of sink.buffers(this.startTimestamp, this.endTimestamp)) {
if (this.synchronizer.shouldWait(trackId, timestamp)) {
await this.synchronizer.wait(timestamp);
}
const offsetTimestamp = timestamp - this.startTimestamp;
const endTimestamp = offsetTimestamp + duration;
// while loop, as a single source buffer may span multiple audio contexts
while (currentContext) {
const currentContextStartTime = currentContextStartFrame / targetSampleRate;
const currentContextEndTime
= (currentContextStartFrame + currentContext.length) / targetSampleRate;
if (offsetTimestamp < currentContextEndTime) {
// The buffer lies within the context, let's play it
const node = currentContext.createBufferSource();
node.buffer = buffer;
node.connect(currentContext.destination);
if (offsetTimestamp < currentContextStartTime) {
node.start(0, currentContextStartTime - offsetTimestamp);
} else {
node.start(offsetTimestamp - currentContextStartTime);
}
}
if (endTimestamp >= currentContextEndTime) {
// Render the audio
const renderedBuffer = await currentContext.startRendering();
await source.digest(renderedBuffer);
currentContextStartFrame += currentContext.length;
const newLength = Math.min(
totalFrameCount - currentContextStartFrame,
MAX_CHUNK_LENGTH,
);
currentContext = newLength > 0
? new OfflineAudioContext({
length: newLength,
numberOfChannels: targetNumberOfChannels,
sampleRate: targetSampleRate,
})
: null;
} else {
break;
}
}
}
if (currentContext) {
const renderedBuffer = await currentContext.startRendering();
await source.digest(renderedBuffer);
}
await source.close();
this.synchronizer.closeTrack(trackId);
})());
return source;
}
reportProgress(trackId: number, endTimestamp: number) {
if (!this.options.onProgress) {
return;
}
assert(this.totalDuration !== null);
this.maxTimestamps.set(trackId, Math.max(endTimestamp, this.maxTimestamps.get(trackId) ?? -Infinity));
let totalTimestamps = 0;
for (const [, timestamp] of this.maxTimestamps) {
totalTimestamps += timestamp;
}
const averageTimestamp = totalTimestamps / this.totalTrackCount;
this.options.onProgress({
completion: 0.99 * clamp(averageTimestamp / this.totalDuration, 0, 1),
});
}
}
const MAX_TIMESTAMP_GAP = 5;
/**
* Utility class for synchronizing multiple track sample consumers with one another. We don't want one consumer to get
* too out-of-sync with the others, as that may lead to a large number of samples that need to be internally buffered
* before they can be written. Therefore, we use this class to slow down a consumer if it is too far ahead of the
* slowest consumer.
*/
class TrackSynchronizer {
maxTimestamps = new Map<number, number>(); // Track ID -> timestamp
resolvers: {
timestamp: number;
resolve: () => void;
}[] = [];
computeMinAndMaybeResolve() {
let newMin = Infinity;
for (const [, timestamp] of this.maxTimestamps) {
newMin = Math.min(newMin, timestamp);
}
for (let i = 0; i < this.resolvers.length; i++) {
const entry = this.resolvers[i]!;
if (entry.timestamp - newMin < MAX_TIMESTAMP_GAP) {
// The gap has gotten small enough again, the consumer can continue again
entry.resolve();
this.resolvers.splice(i, 1);
i--;
}
}
return newMin;
}
shouldWait(trackId: number, timestamp: number) {
this.maxTimestamps.set(trackId, Math.max(timestamp, this.maxTimestamps.get(trackId) ?? -Infinity));
const newMin = this.computeMinAndMaybeResolve();
return timestamp - newMin >= MAX_TIMESTAMP_GAP; // Should wait if it is too far ahead of the slowest consumer
}
wait(timestamp: number) {
const { promise, resolve } = promiseWithResolvers();
this.resolvers.push({
timestamp,
resolve,
});
return promise;
}
closeTrack(trackId: number) {
this.maxTimestamps.delete(trackId);
this.computeMinAndMaybeResolve();
}
}
src/index.ts
+10 -1
View File
@@ -42,11 +42,19 @@ export {
export {
VIDEO_CODECS,
VideoCodec,
PCM_AUDIO_CODECS,
NON_PCM_AUDIO_CODECS,
AUDIO_CODECS,
AudioCodec,
SUBTITLE_CODECS,
SubtitleCodec,
MediaCodec,
Quality,
QUALITY_VERY_LOW,
QUALITY_LOW,
QUALITY_MEDIUM,
QUALITY_HIGH,
QUALITY_VERY_HIGH,
canEncode,
canEncodeVideo,
canEncodeAudio,
@@ -57,7 +65,7 @@ export {
getEncodableSubtitleCodecs,
} from './codec';
export { Target, BufferTarget, StreamTarget, StreamTargetChunk, StreamTargetOptions } from './target';
export { Rotation, TransformationMatrix, AnyIterable } from './misc';
export { Rotation, TransformationMatrix, AnyIterable, setVideoFrameTiming } from './misc';
export { Source, BufferSource, BlobSource, UrlSource } from './source';
export {
InputFormat,
@@ -96,5 +104,6 @@ export {
AudioBufferSink,
WrappedAudioBuffer,
} from './media-sink';
export { convert, ConversionOptions, ConversionInfo } from './conversion';
// 🐡🦔
src/input-track.ts
+28
View File
@@ -1,9 +1,11 @@
import { AudioCodec, MediaCodec, VideoCodec } from './codec';
import { EncodedAudioSampleSink, EncodedVideoSampleSink, SampleRetrievalOptions } from './media-sink';
import { Rotation } from './misc';
import { TrackType } from './output';
import { EncodedAudioSample, EncodedVideoSample } from './sample';
export interface InputTrackBacking {
getId(): number;
getCodec(): Promise<MediaCodec | null>;
getFirstTimestamp(): Promise<number>;
computeDuration(): Promise<number>;
@@ -20,6 +22,7 @@ export abstract class InputTrack {
this._backing = backing;
}
abstract getType(): TrackType;
abstract getCodec(): Promise<MediaCodec | null>;
abstract getCodecMimeType(): Promise<string | null>;
abstract canDecode(): Promise<boolean>;
@@ -33,6 +36,10 @@ export abstract class InputTrack {
return this instanceof InputAudioTrack;
}
getId() {
return this._backing.getId();
}
getFirstTimestamp() {
return this._backing.getFirstTimestamp();
}
@@ -51,6 +58,7 @@ export interface InputVideoTrackBacking extends InputTrackBacking {
getCodedWidth(): Promise<number>;
getCodedHeight(): Promise<number>;
getRotation(): Promise<Rotation>;
getColorSpace(): Promise<VideoColorSpaceInit>;
getDecoderConfig(): Promise<VideoDecoderConfig | null>;
getFirstSample(options: SampleRetrievalOptions): Promise<EncodedVideoSample | null>;
getSample(timestamp: number, options: SampleRetrievalOptions): Promise<EncodedVideoSample | null>;
@@ -71,6 +79,10 @@ export class InputVideoTrack extends InputTrack {
this._backing = backing;
}
getType(): TrackType {
return 'video';
}
getCodec(): Promise<VideoCodec | null> {
return this._backing.getCodec();
}
@@ -97,6 +109,18 @@ export class InputVideoTrack extends InputTrack {
return rotation % 180 === 0 ? this._backing.getCodedHeight() : this._backing.getCodedWidth();
}
getColorSpace() {
return this._backing.getColorSpace();
}
async hasHighDynamicRange() {
const colorSpace = await this._backing.getColorSpace();
return (colorSpace.primaries as string) === 'bt2020' || (colorSpace.primaries as string) === 'smpte432'
|| (colorSpace.transfer as string) === 'pg' || (colorSpace.transfer as string) === 'hlg'
|| (colorSpace.matrix as string) === 'bt2020-ncl';
}
getDecoderConfig() {
return this._backing.getDecoderConfig();
}
@@ -150,6 +174,10 @@ export class InputAudioTrack extends InputTrack {
this._backing = backing;
}
getType(): TrackType {
return 'audio';
}
getCodec(): Promise<AudioCodec | null> {
return this._backing.getCodec();
}
src/isobmff/isobmff-boxes.ts
+2 -2
View File
@@ -17,7 +17,7 @@ import {
AudioCodec,
generateAv1CodecConfigurationFromCodecString,
parsePcmCodec,
PCM_CODECS,
PCM_AUDIO_CODECS,
PcmAudioCodec,
SubtitleCodec,
VideoCodec,
@@ -680,7 +680,7 @@ export const soundSampleDescription = (
let contents: NestedNumberArray;
let sampleSizeInBits = 16;
if ((PCM_CODECS as readonly AudioCodec[]).includes(trackData.track.source._codec)) {
if ((PCM_AUDIO_CODECS as readonly AudioCodec[]).includes(trackData.track.source._codec)) {
const codec = trackData.track.source._codec as PcmAudioCodec;
const { sampleSize } = parsePcmCodec(codec);
sampleSizeInBits = 8 * sampleSize;
src/isobmff/isobmff-demuxer.ts
+84 -25
View File
@@ -7,7 +7,7 @@ import {
extractVp9CodecInfoFromFrame,
MediaCodec,
parseAacAudioSpecificConfig,
PCM_CODECS,
PCM_AUDIO_CODECS,
VideoCodec,
Vp9CodecInfo,
} from '../codec';
@@ -35,6 +35,8 @@ import {
AsyncMutex,
findLastIndex,
UNDETERMINED_LANGUAGE,
TransformationMatrix,
extractRotationFromMatrix,
} from '../misc';
import { Reader } from '../reader';
import { EncodedAudioSample, EncodedVideoSample, PLACEHOLDER_DATA, SampleType } from '../sample';
@@ -165,6 +167,7 @@ type Fragment = {
dataStart: number;
dataEnd: number;
nextFragment: Fragment | null;
isKnownToBeFirstFragment: boolean;
};
const knownMatrixes = [rotationMatrix(0), rotationMatrix(90), rotationMatrix(180), rotationMatrix(270)];
@@ -295,7 +298,7 @@ export class IsobmffDemuxer extends Demuxer {
const isPcmCodec = internalTrack.info?.type === 'audio'
&& internalTrack.info.codec
&& (PCM_CODECS as readonly string[]).includes(internalTrack.info.codec);
&& (PCM_AUDIO_CODECS as readonly string[]).includes(internalTrack.info.codec);
if (isPcmCodec && sampleTable.sampleCompositionTimeOffsets.length === 0) {
// If the audio has PCM samples, the way the samples are defined in the sample table is somewhat
@@ -462,6 +465,8 @@ export class IsobmffDemuxer extends Demuxer {
this.isobmffReader.pos = currentFragment.moofOffset + currentFragment.moofSize;
}
let nextFragmentIsFirstFragment = this.isobmffReader.pos === 0;
while (this.isobmffReader.pos < startPos) {
if (currentFragment?.nextFragment) {
currentFragment = currentFragment.nextFragment;
@@ -477,18 +482,23 @@ export class IsobmffDemuxer extends Demuxer {
if (boxInfo.name === 'moof') {
const index = binarySearchExact(this.fragments, startPos, x => x.moofOffset);
let fragment: Fragment;
if (index === -1) {
this.isobmffReader.pos = startPos;
const fragment = await this.readFragment(); // Recursive call
if (currentFragment) currentFragment.nextFragment = fragment;
currentFragment = fragment;
fragment = await this.readFragment(); // Recursive call
} else {
// We already know this fragment
const fragment = this.fragments[index]!;
// Even if we already know the fragment, we might not yet know its predecessor
fragment = this.fragments[index]!;
}
// Even if we already know the fragment, we might not yet know its predecessor; always do this
if (currentFragment) currentFragment.nextFragment = fragment;
currentFragment = fragment;
if (nextFragmentIsFirstFragment) {
fragment.isKnownToBeFirstFragment = true;
nextFragmentIsFirstFragment = false;
}
}
@@ -613,16 +623,24 @@ export class IsobmffDemuxer extends Demuxer {
}
this.isobmffReader.pos += 2 * 4 + 2 + 2 + 2 + 2;
const values: number[] = [];
values.push(this.isobmffReader.readFixed_16_16(), this.isobmffReader.readFixed_16_16());
this.isobmffReader.pos += 4;
values.push(this.isobmffReader.readFixed_16_16(), this.isobmffReader.readFixed_16_16());
const matrix: TransformationMatrix = [
this.isobmffReader.readFixed_16_16(),
this.isobmffReader.readFixed_16_16(),
this.isobmffReader.readFixed_2_30(),
this.isobmffReader.readFixed_16_16(),
this.isobmffReader.readFixed_16_16(),
this.isobmffReader.readFixed_2_30(),
this.isobmffReader.readFixed_16_16(),
this.isobmffReader.readFixed_16_16(),
this.isobmffReader.readFixed_2_30(),
];
const matrixIndex = knownMatrixes.findIndex((x) => {
return x[0] === values[0] && x[1] === values[1] && x[3] === values[2] && x[4] === values[3];
});
const rotation = extractRotationFromMatrix(matrix);
const comparisonMatrix = rotationMatrix(rotation);
const matrixIndex = knownMatrixes.findIndex(mat => mat.every((y, i) => y === comparisonMatrix[i]));
if (matrixIndex === -1) {
console.warn(`Wacky rotation matrix ${values.join(',')}; sticking with no rotation.`);
console.warn(`Wacky rotation matrix ${comparisonMatrix.join(',')}; sticking with no rotation.`);
track.rotation = 0;
} else {
track.rotation = (90 * matrixIndex) as Rotation;
@@ -960,7 +978,7 @@ export class IsobmffDemuxer extends Demuxer {
transfer: TRANSFER_CHARACTERISTICS_MAP_INVERSE[transferCharacteristics],
matrix: MATRIX_COEFFICIENTS_MAP_INVERSE[matrixCoefficients],
fullRange: fullRangeFlag,
};
} as VideoColorSpaceInit;
}; break;
case 'wave': {
@@ -1418,6 +1436,7 @@ export class IsobmffDemuxer extends Demuxer {
dataStart: Infinity,
dataEnd: 0,
nextFragment: null,
isKnownToBeFirstFragment: false,
};
this.readContiguousBoxes(boxInfo.contentSize);
@@ -1694,6 +1713,10 @@ abstract class IsobmffTrackBacking<
constructor(public internalTrack: InternalTrack) {}
getId() {
return this.internalTrack.id;
}
getCodec(): Promise<MediaCodec | null> {
throw new Error('Not implemented on base class.');
}
@@ -1724,14 +1747,35 @@ abstract class IsobmffTrackBacking<
if (this.internalTrack.demuxer.isFragmented) {
return this.performFragmentedLookup(
() => {
const fragment = this.internalTrack.fragments[0];
const startFragment = this.internalTrack.demuxer.fragments[0] ?? null;
if (startFragment?.isKnownToBeFirstFragment) {
// Walk from the very first fragment in the file until we find one with our track in it
let currentFragment: Fragment | null = startFragment;
while (currentFragment) {
const trackData = currentFragment.trackData.get(this.internalTrack.id);
if (trackData) {
return {
fragmentIndex: fragment ? 0 : -1,
sampleIndex: fragment ? 0 : -1,
correctSampleFound: !!fragment,
fragmentIndex: binarySearchExact(
this.internalTrack.fragments,
currentFragment.moofOffset,
x => x.moofOffset,
),
sampleIndex: 0,
correctSampleFound: true,
};
}
currentFragment = currentFragment.nextFragment;
}
}
return {
fragmentIndex: -1,
sampleIndex: -1,
correctSampleFound: false,
};
},
0,
-Infinity, // Use -Infinity as a search timestamp to avoid using the lookup entries
Infinity,
options,
);
@@ -2108,8 +2152,11 @@ abstract class IsobmffTrackBacking<
? this.internalTrack.fragmentLookupTable![lookupEntryIndex]!
: null;
let nextFragmentIsFirstFragment = false;
if (fragmentIndex === -1) {
isobmffReader.pos = lookupEntry?.moofOffset ?? 0;
nextFragmentIsFirstFragment = isobmffReader.pos === 0;
} else {
const fragment = this.internalTrack.fragments[fragmentIndex]!;
@@ -2150,16 +2197,19 @@ abstract class IsobmffTrackBacking<
if (index === -1) {
// This is the first time we've seen this fragment
isobmffReader.pos = startPos;
fragment = await demuxer.readFragment();
if (prevFragment) prevFragment.nextFragment = fragment;
prevFragment = fragment;
} else {
// We already know this fragment
fragment = demuxer.fragments[index]!;
// Even if we already know the fragment, we might not yet know its predecessor
}
// Even if we already know the fragment, we might not yet know its predecessor, so always do this
if (prevFragment) prevFragment.nextFragment = fragment;
prevFragment = fragment;
if (nextFragmentIsFirstFragment) {
fragment.isKnownToBeFirstFragment = true;
nextFragmentIsFirstFragment = false;
}
const { fragmentIndex, sampleIndex, correctSampleFound } = getBestMatch();
@@ -2224,6 +2274,15 @@ class IsobmffVideoTrackBacking extends IsobmffTrackBacking<EncodedVideoSample> i
return this.internalTrack.rotation;
}
async getColorSpace(): Promise<VideoColorSpaceInit> {
return {
primaries: this.internalTrack.info.colorSpace?.primaries,
transfer: this.internalTrack.info.colorSpace?.transfer,
matrix: this.internalTrack.info.colorSpace?.matrix,
fullRange: this.internalTrack.info.colorSpace?.fullRange,
};
}
async getDecoderConfig(): Promise<VideoDecoderConfig | null> {
if (!this.internalTrack.info.codec) {
return null;
src/isobmff/isobmff-muxer.ts
+7 -7
View File
@@ -7,7 +7,7 @@ import { IsobmffOutputFormatOptions, IsobmffOutputFormat, MovOutputFormat } from
import { inlineTimestampRegex, SubtitleConfig, SubtitleCue, SubtitleMetadata } from '../subtitles';
import {
parsePcmCodec,
PCM_CODECS,
PCM_AUDIO_CODECS,
PcmAudioCodec,
validateAudioChunkMetadata,
validateSubtitleMetadata,
@@ -239,7 +239,7 @@ export class IsobmffMuxer extends Muxer {
compactlyCodedChunkTable: [],
requiresPcmTransformation:
!this.isFragmented
&& (PCM_CODECS as readonly string[]).includes(track.source._codec),
&& (PCM_AUDIO_CODECS as readonly string[]).includes(track.source._codec),
};
this.trackDatas.push(newTrackData);
@@ -668,16 +668,16 @@ export class IsobmffMuxer extends Muxer {
// We can only finalize this fragment (and begin a new one) if we know that each track will be able to
// start the new one with a key frame.
const keyFrameQueuedEverywhere = this.trackDatas.every((otherTrackData) => {
if (otherTrackData.track.source._closed) {
return true;
}
if (trackData === otherTrackData) {
return sample.type === 'key';
}
const firstQueuedSample = otherTrackData.sampleQueue[0];
return firstQueuedSample && firstQueuedSample.type === 'key';
if (firstQueuedSample) {
return firstQueuedSample.type === 'key';
}
return otherTrackData.track.source._closed;
});
if (
src/matroska/ebml.ts
+1 -1
View File
@@ -450,7 +450,7 @@ export class EBMLReader {
}
if (width !== 4 && width !== 8) {
throw new Error('Bad FLOAT size ' + width);
throw new Error('Bad float size ' + width);
}
const { view, offset } = this.reader.getViewAndOffset(this.pos, this.pos + width);
src/matroska/matroska-demuxer.ts
+54 -12
View File
@@ -69,6 +69,7 @@ type Cluster = {
timestamp: number;
trackData: Map<number, ClusterTrackData>;
nextCluster: Cluster | null;
isKnownToBeFirstCluster: boolean;
};
type ClusterTrackData = {
@@ -415,6 +416,7 @@ export class MatroskaDemuxer extends Demuxer {
timestamp: -1,
trackData: new Map(),
nextCluster: null,
isKnownToBeFirstCluster: false,
};
this.currentCluster = cluster;
this.readContiguousElements(this.clusterReader, size);
@@ -772,7 +774,7 @@ export class MatroskaDemuxer extends Demuxer {
const matrixCoefficients = reader.readUnsignedInt(size);
const mapped = MATRIX_COEFFICIENTS_MAP_INVERSE[matrixCoefficients] ?? null;
this.currentTrack.info.colorSpace.matrix = mapped;
this.currentTrack.info.colorSpace.matrix = mapped as VideoColorSpaceInit['matrix'];
}; break;
case EBMLId.Range: {
@@ -786,7 +788,7 @@ export class MatroskaDemuxer extends Demuxer {
const transferCharacteristics = reader.readUnsignedInt(size);
const mapped = TRANSFER_CHARACTERISTICS_MAP_INVERSE[transferCharacteristics] ?? null;
this.currentTrack.info.colorSpace.transfer = mapped;
this.currentTrack.info.colorSpace.transfer = mapped as VideoColorSpaceInit['transfer'];
}; break;
case EBMLId.Primaries: {
@@ -794,7 +796,7 @@ export class MatroskaDemuxer extends Demuxer {
const primaries = reader.readUnsignedInt(size);
const mapped = COLOR_PRIMARIES_MAP_INVERSE[primaries] ?? null;
this.currentTrack.info.colorSpace.primaries = mapped;
this.currentTrack.info.colorSpace.primaries = mapped as VideoColorSpaceInit['primaries'];
}; break;
case EBMLId.Projection: {
@@ -967,6 +969,10 @@ abstract class MatroskaTrackBacking<
constructor(public internalTrack: InternalTrack) {}
getId() {
return this.internalTrack.id;
}
getCodec(): Promise<MediaCodec | null> {
throw new Error('Not implemented on base class.');
}
@@ -996,14 +1002,35 @@ abstract class MatroskaTrackBacking<
async getFirstSample(options: SampleRetrievalOptions) {
return this.performClusterLookup(
() => {
const cluster = this.internalTrack.clusters[0];
const startCluster = this.internalTrack.segment.clusters[0] ?? null;
if (startCluster?.isKnownToBeFirstCluster) {
// Walk from the very first cluster in the file until we find one with our track in it
let currentCluster: Cluster | null = startCluster;
while (currentCluster) {
const trackData = currentCluster.trackData.get(this.internalTrack.id);
if (trackData) {
return {
clusterIndex: cluster ? 0 : -1,
blockIndex: cluster ? 0 : -1,
correctBlockFound: !!cluster,
clusterIndex: binarySearchExact(
this.internalTrack.clusters,
currentCluster.elementStartPos,
x => x.elementStartPos,
),
blockIndex: 0,
correctBlockFound: true,
};
}
currentCluster = currentCluster.nextCluster;
}
}
return {
clusterIndex: -1,
blockIndex: -1,
correctBlockFound: false,
};
},
0,
-Infinity, // Use -Infinity as a search timestamp to avoid using the cues
Infinity,
options,
);
@@ -1300,8 +1327,11 @@ abstract class MatroskaTrackBacking<
);
const cuePoint = cuePointIndex !== -1 ? this.internalTrack.cuePoints[cuePointIndex]! : null;
let nextClusterIsFirstCluster = false;
if (clusterIndex === -1) {
metadataReader.pos = cuePoint?.clusterPosition ?? segment.clusterSeekStartPos;
nextClusterIsFirstCluster = metadataReader.pos === segment.clusterSeekStartPos;
} else {
const cluster = this.internalTrack.clusters[clusterIndex]!;
@@ -1344,15 +1374,18 @@ abstract class MatroskaTrackBacking<
// This is the first time we've seen this cluster
metadataReader.pos = elementStartPos;
cluster = await demuxer.readCluster(segment);
if (prevCluster) prevCluster.nextCluster = cluster;
prevCluster = cluster;
} else {
// We already know this cluster
cluster = segment.clusters[index]!;
// Even if we already know the cluster, we might not yet know its predecessor
}
// Even if we already know the cluster, we might not yet know its predecessor, so always do this
if (prevCluster) prevCluster.nextCluster = cluster;
prevCluster = cluster;
if (nextClusterIsFirstCluster) {
cluster.isKnownToBeFirstCluster = true;
nextClusterIsFirstCluster = false;
}
const { clusterIndex, blockIndex, correctBlockFound } = getBestMatch();
@@ -1417,6 +1450,15 @@ class MatroskaVideoTrackBacking extends MatroskaTrackBacking<EncodedVideoSample>
return this.internalTrack.info.rotation;
}
async getColorSpace(): Promise<VideoColorSpaceInit> {
return {
primaries: this.internalTrack.info.colorSpace?.primaries,
transfer: this.internalTrack.info.colorSpace?.transfer,
matrix: this.internalTrack.info.colorSpace?.matrix,
fullRange: this.internalTrack.info.colorSpace?.fullRange,
};
}
async getDecoderConfig(): Promise<VideoDecoderConfig | null> {
if (!this.internalTrack.info.codec) {
return null;
src/matroska/matroska-muxer.ts
+8 -8
View File
@@ -31,7 +31,7 @@ import {
parseSubtitleTimestamp,
} from '../subtitles';
import {
PCM_CODECS,
PCM_AUDIO_CODECS,
PcmAudioCodec,
generateAv1CodecConfigurationFromCodecString,
parsePcmCodec,
@@ -308,7 +308,7 @@ export class MatroskaMuxer extends Muxer {
},
{
id: EBMLId.ProjectionPoseRoll,
data: (rotation + 180) % 360 - 180, // [0, 270] -> [-180, 90]
data: new EBMLFloat32((rotation + 180) % 360 - 180), // [0, 270] -> [-180, 90]
},
],
}
@@ -321,7 +321,7 @@ export class MatroskaMuxer extends Muxer {
}
private audioSpecificTrackInfo(trackData: MatroskaAudioTrackData) {
const pcmInfo = (PCM_CODECS as readonly string[]).includes(trackData.track.source._codec)
const pcmInfo = (PCM_AUDIO_CODECS as readonly string[]).includes(trackData.track.source._codec)
? parsePcmCodec(trackData.track.source._codec as PcmAudioCodec)
: null;
@@ -659,16 +659,16 @@ export class MatroskaMuxer extends Muxer {
// We can only finalize this cluster (and begin a new one) if we know that each track will be able to
// start the new one with a key frame.
const keyFrameQueuedEverywhere = this.trackDatas.every((otherTrackData) => {
if (otherTrackData.track.source._closed) {
return true;
}
if (trackData === otherTrackData) {
return chunk.type === 'key';
}
const firstQueuedSample = otherTrackData.chunkQueue[0];
return firstQueuedSample && firstQueuedSample.type === 'key';
if (firstQueuedSample) {
return firstQueuedSample.type === 'key';
}
return otherTrackData.track.source._closed;
});
if (
src/media-sink.ts
+3 -3
View File
@@ -1,4 +1,4 @@
import { parsePcmCodec, PCM_CODECS, PcmAudioCodec } from './codec';
import { parsePcmCodec, PCM_AUDIO_CODECS, PcmAudioCodec } from './codec';
import { InputAudioTrack, InputVideoTrack } from './input-track';
import {
AnyIterable,
@@ -920,7 +920,7 @@ class PcmAudioDecoderWrapper extends DecoderWrapper<EncodedAudioSample, AudioDat
) {
super(onData, onError);
assert((PCM_CODECS as readonly string[]).includes(decoderConfig.codec));
assert((PCM_AUDIO_CODECS as readonly string[]).includes(decoderConfig.codec));
this.codec = decoderConfig.codec as PcmAudioCodec;
const { dataType, sampleSize, littleEndian } = parsePcmCodec(this.codec);
@@ -1104,7 +1104,7 @@ export class AudioDataSink extends BaseMediaFrameSink<EncodedAudioSample, AudioD
const decoderConfig = await this._audioTrack.getDecoderConfig();
assert(decoderConfig);
if ((PCM_CODECS as readonly string[]).includes(decoderConfig.codec)) {
if ((PCM_AUDIO_CODECS as readonly string[]).includes(decoderConfig.codec)) {
return new PcmAudioDecoderWrapper(onData, onError, decoderConfig);
} else {
return new AudioDecoderWrapper(onData, onError, decoderConfig);
src/media-source.ts
+65 -32
View File
@@ -6,8 +6,9 @@ import {
getAudioEncoderConfigExtension,
getVideoEncoderConfigExtension,
parsePcmCodec,
PCM_CODECS,
PCM_AUDIO_CODECS,
PcmAudioCodec,
Quality,
SUBTITLE_CODECS,
SubtitleCodec,
VIDEO_CODECS,
@@ -25,7 +26,7 @@ export abstract class MediaSource {
/** @internal */
_connectedTrack: OutputTrack | null = null;
/** @internal */
_closing = false;
_closingPromise: Promise<void> | null = null;
/** @internal */
_closed = false;
/** @internal */
@@ -59,30 +60,42 @@ export abstract class MediaSource {
/** @internal */
async _flush() {}
async close() {
if (this._closing) {
close() {
if (this._closingPromise) {
throw new Error('Source already closed.');
}
if (!this._connectedTrack) {
const connectedTrack = this._connectedTrack;
if (!connectedTrack) {
throw new Error('Cannot call close without connecting the source to an output track.');
}
if (!this._connectedTrack.output._started) {
if (!connectedTrack.output._started) {
throw new Error('Cannot call close before output has been started.');
}
this._closing = true;
return this._closingPromise = (async () => {
await this._flush();
this._closed = true;
if (this._connectedTrack.output._finalizing) {
if (connectedTrack.output._finalizing) {
return;
}
this._connectedTrack.output._muxer.onTrackClose(this._connectedTrack);
connectedTrack.output._muxer.onTrackClose(connectedTrack);
})();
}
/** @internal */
async _flushOrWaitForClose() {
if (this._closingPromise) {
// Since closing also flushes, we don't want to do it twice
return this._closingPromise;
} else {
return this._flush();
}
}
}
@@ -126,10 +139,10 @@ export class EncodedVideoSampleSource extends VideoSource {
/** @public */
export type VideoEncodingConfig = {
codec: VideoCodec;
bitrate: number;
bitrate: number | Quality;
latencyMode?: VideoEncoderConfig['latencyMode'];
keyFrameInterval?: number;
onEncodedSample?: (chunk: EncodedVideoSample, meta: EncodedVideoChunkMetadata | undefined) => unknown;
onEncodedSample?: (sample: EncodedVideoSample, meta: EncodedVideoChunkMetadata | undefined) => unknown;
onEncodingError?: (error: Error) => unknown;
};
@@ -140,8 +153,8 @@ const validateVideoEncodingConfig = (config: VideoEncodingConfig) => {
if (!VIDEO_CODECS.includes(config.codec)) {
throw new TypeError(`Invalid video codec '${config.codec}'. Must be one of: ${VIDEO_CODECS.join(', ')}.`);
}
if (!Number.isInteger(config.bitrate) || config.bitrate <= 0) {
throw new TypeError('config.bitrate must be a positive integer.');
if (!(config.bitrate instanceof Quality) && (!Number.isInteger(config.bitrate) || config.bitrate <= 0)) {
throw new TypeError('config.bitrate must be a positive integer or a quality.');
}
if (config.latencyMode !== undefined && !['quality', 'realtime'].includes(config.latencyMode)) {
throw new TypeError('config.latencyMode, when provided, must be \'quality\' or \'realtime\'.');
@@ -205,7 +218,9 @@ class VideoEncoderWrapper {
// in Matroska.
this.encoder.encode(videoFrame, {
...encodeOptions,
keyFrame: keyFrameInterval === 0 || multipleOfKeyFrameInterval !== this.lastMultipleOfKeyFrameInterval,
keyFrame: encodeOptions?.keyFrame
|| keyFrameInterval === 0
|| multipleOfKeyFrameInterval !== this.lastMultipleOfKeyFrameInterval,
});
if (shouldClose) {
@@ -234,16 +249,22 @@ class VideoEncoderWrapper {
const { promise, resolve } = promiseWithResolvers();
this.ensureEncoderPromise = promise;
const width = videoFrame.codedWidth;
const height = videoFrame.codedHeight;
const bitrate = this.encodingConfig.bitrate instanceof Quality
? this.encodingConfig.bitrate._toVideoBitrate(this.encodingConfig.codec, width, height)
: this.encodingConfig.bitrate;
const encoderConfig: VideoEncoderConfig = {
codec: buildVideoCodecString(
this.encodingConfig.codec,
videoFrame.codedWidth,
videoFrame.codedHeight,
this.encodingConfig.bitrate,
width,
height,
bitrate,
),
width: videoFrame.codedWidth,
height: videoFrame.codedHeight,
bitrate: this.encodingConfig.bitrate,
width,
height,
bitrate,
framerate: this.source._connectedTrack?.metadata.frameRate,
latencyMode: this.encodingConfig.latencyMode,
...getVideoEncoderConfigExtension(this.encodingConfig.codec),
@@ -444,8 +465,8 @@ export class EncodedAudioSampleSource extends AudioSource {
/** @public */
export type AudioEncodingConfig = {
codec: AudioCodec;
bitrate?: number;
onEncodedSample?: (chunk: EncodedAudioSample, meta: EncodedAudioChunkMetadata | undefined) => unknown;
bitrate?: number | Quality;
onEncodedSample?: (sample: EncodedAudioSample, meta: EncodedAudioChunkMetadata | undefined) => unknown;
onEncodingError?: (error: Error) => unknown;
};
@@ -456,11 +477,18 @@ const validateAudioEncodingConfig = (config: AudioEncodingConfig) => {
if (!AUDIO_CODECS.includes(config.codec)) {
throw new TypeError(`Invalid audio codec '${config.codec}'. Must be one of: ${AUDIO_CODECS.join(', ')}.`);
}
if (config.bitrate === undefined && !(PCM_CODECS as readonly string[]).includes(config.codec)) {
if (
config.bitrate === undefined
&& (!(PCM_AUDIO_CODECS as readonly string[]).includes(config.codec) || config.codec === 'flac')
) {
throw new TypeError('config.bitrate must be provided for compressed audio codecs.');
}
if (config.bitrate !== undefined && (!Number.isInteger(config.bitrate) || config.bitrate <= 0)) {
throw new TypeError('config.bitrate must be a positive integer.');
if (
config.bitrate !== undefined
&& !(config.bitrate instanceof Quality)
&& (!Number.isInteger(config.bitrate) || config.bitrate <= 0)
) {
throw new TypeError('config.bitrate, when provided, must be a positive integer or a quality.');
}
if (config.onEncodingError !== undefined && typeof config.onEncodingError !== 'function') {
throw new TypeError('config.onEncodingError, when provided, must be a function.');
@@ -610,22 +638,27 @@ class AudioEncoderWrapper {
const { promise, resolve } = promiseWithResolvers();
this.ensureEncoderPromise = promise;
if ((PCM_CODECS as readonly string[]).includes(this.encodingConfig.codec)) {
if ((PCM_AUDIO_CODECS as readonly string[]).includes(this.encodingConfig.codec)) {
this.initPcmEncoder();
} else {
if (typeof AudioEncoder === 'undefined') {
throw new Error('AudioEncoder is not supported by this browser.');
}
const { numberOfChannels, sampleRate } = audioData;
const bitrate = this.encodingConfig.bitrate instanceof Quality
? this.encodingConfig.bitrate._toAudioBitrate(this.encodingConfig.codec)
: this.encodingConfig.bitrate;
const encoderConfig: AudioEncoderConfig = {
codec: buildAudioCodecString(
this.encodingConfig.codec,
audioData.numberOfChannels,
audioData.sampleRate,
numberOfChannels,
sampleRate,
),
numberOfChannels: audioData.numberOfChannels,
sampleRate: audioData.sampleRate,
bitrate: this.encodingConfig.bitrate,
numberOfChannels,
sampleRate,
bitrate,
...getAudioEncoderConfigExtension(this.encodingConfig.codec),
};
const support = await AudioEncoder.isConfigSupported(encoderConfig);
src/misc.ts
+42 -10
View File
@@ -60,27 +60,33 @@ const invertObject = <K extends PropertyKey, V extends PropertyKey>(object: Reco
return Object.fromEntries(Object.entries(object).map(([key, value]) => [value, key])) as Record<V, K>;
};
// These maps are taken from https://www.matroska.org/technical/elements.html,
// which references the tables in ITU-T H.273 - they should be valid for Matroska and ISOBMFF.
export const COLOR_PRIMARIES_MAP: Record<VideoColorPrimaries, number> = {
// For the color space mappings, see Rec. ITU-T H.273.
export const COLOR_PRIMARIES_MAP = {
bt709: 1, // ITU-R BT.709
bt470bg: 5, // ITU-R BT.470BG
smpte170m: 6, // ITU-R BT.601 525 - SMPTE 170M
bt2020: 9, // ITU-R BT.202
smpte432: 12, // SMPTE EG 432-1
};
export const COLOR_PRIMARIES_MAP_INVERSE = invertObject(COLOR_PRIMARIES_MAP);
export const TRANSFER_CHARACTERISTICS_MAP: Record<VideoTransferCharacteristics, number> = {
export const TRANSFER_CHARACTERISTICS_MAP = {
'bt709': 1, // ITU-R BT.709
'smpte170m': 6, // SMPTE 170M
'linear': 8, // Linear transfer characteristics
'iec61966-2-1': 13, // IEC 61966-2-1
'pg': 16, // Rec. ITU-R BT.2100-2 perceptual quantization (PQ) system
'hlg': 18, // Rec. ITU-R BT.2100-2 hybrid loggamma (HLG) system
};
export const TRANSFER_CHARACTERISTICS_MAP_INVERSE = invertObject(TRANSFER_CHARACTERISTICS_MAP);
export const MATRIX_COEFFICIENTS_MAP: Record<VideoMatrixCoefficients, number> = {
rgb: 0, // Identity
bt709: 1, // ITU-R BT.709
bt470bg: 5, // ITU-R BT.470BG
smpte170m: 6, // SMPTE 170M
export const MATRIX_COEFFICIENTS_MAP = {
'rgb': 0, // Identity
'bt709': 1, // ITU-R BT.709
'bt470bg': 5, // ITU-R BT.470BG
'smpte170m': 6, // SMPTE 170M
'bt2020-ncl': 9, // ITU-R BT.2020-2 (non-constant luminance)
};
export const MATRIX_COEFFICIENTS_MAP_INVERSE = invertObject(MATRIX_COEFFICIENTS_MAP);
@@ -127,7 +133,7 @@ export class AsyncMutex {
}
}
export const rotationMatrix = (rotationInDegrees: Rotation): TransformationMatrix => {
export const rotationMatrix = (rotationInDegrees: number): TransformationMatrix => {
const theta = rotationInDegrees * (Math.PI / 180);
const cosTheta = Math.round(Math.cos(theta));
const sinTheta = Math.round(Math.sin(theta));
@@ -140,6 +146,18 @@ export const rotationMatrix = (rotationInDegrees: Rotation): TransformationMatri
];
};
/** Extracts the rotation component from a transformation matrix, in degrees. */
export const extractRotationFromMatrix = (matrix: TransformationMatrix) => {
const [m11, , , m21] = matrix;
const scaleX = Math.hypot(m11, m21);
const cosTheta = m11 / scaleX;
const sinTheta = m21 / scaleX;
return Math.atan2(sinTheta, cosTheta) * (180 / Math.PI);
};
export const IDENTITY_MATRIX = rotationMatrix(0);
export const bytesToHexString = (bytes: Uint8Array) => {
@@ -331,3 +349,17 @@ export const clamp = (value: number, min: number, max: number) => {
};
export const UNDETERMINED_LANGUAGE = 'und';
/** @public */
export const setVideoFrameTiming = (frame: VideoFrame, timing: {
timestamp?: number;
duration?: number;
}) => {
const clone = new VideoFrame(frame, {
timestamp: timing.timestamp && 1e6 * timing.timestamp,
duration: timing.duration && 1e6 * timing.duration,
});
frame.close();
return clone;
};
src/output-format.ts
+32 -18
View File
@@ -1,4 +1,14 @@
import { AUDIO_CODECS, MediaCodec, PCM_CODECS, SUBTITLE_CODECS, VIDEO_CODECS } from './codec';
import {
AUDIO_CODECS,
AudioCodec,
MediaCodec,
NON_PCM_AUDIO_CODECS,
PCM_AUDIO_CODECS,
SUBTITLE_CODECS,
SubtitleCodec,
VIDEO_CODECS,
VideoCodec,
} from './codec';
import { IsobmffMuxer } from './isobmff/isobmff-muxer';
import { MatroskaMuxer } from './matroska/matroska-muxer';
import { Muxer } from './muxer';
@@ -26,15 +36,18 @@ export abstract class OutputFormat {
abstract getSupportedTrackCounts(): TrackCountLimits;
getSupportedVideoCodecs() {
return this.getSupportedCodecs().filter(codec => (VIDEO_CODECS as readonly string[]).includes(codec));
return this.getSupportedCodecs()
.filter(codec => (VIDEO_CODECS as readonly string[]).includes(codec)) as VideoCodec[];
}
getSupportedAudioCodecs() {
return this.getSupportedCodecs().filter(codec => (AUDIO_CODECS as readonly string[]).includes(codec));
return this.getSupportedCodecs()
.filter(codec => (AUDIO_CODECS as readonly string[]).includes(codec)) as AudioCodec[];
}
getSupportedSubtitleCodecs() {
return this.getSupportedCodecs().filter(codec => (SUBTITLE_CODECS as readonly string[]).includes(codec));
return this.getSupportedCodecs()
.filter(codec => (SUBTITLE_CODECS as readonly string[]).includes(codec)) as SubtitleCodec[];
}
/** @internal */
@@ -99,9 +112,9 @@ export class Mp4OutputFormat extends IsobmffOutputFormat {
static getSupportedCodecs(): MediaCodec[] {
return [
'avc', 'hevc', 'vp8', 'vp9', 'av1',
'aac', 'mp3', 'opus', 'vorbis', 'flac', // No PCM codecs
'webvtt',
...VIDEO_CODECS,
...NON_PCM_AUDIO_CODECS,
...SUBTITLE_CODECS,
];
}
@@ -132,8 +145,8 @@ export class MovOutputFormat extends IsobmffOutputFormat {
static getSupportedCodecs(): MediaCodec[] {
return [
'avc', 'hevc', 'vp8', 'vp9', 'av1',
'aac', 'mp3', 'opus', 'vorbis', 'flac', ...PCM_CODECS,
...VIDEO_CODECS,
...AUDIO_CODECS,
];
}
@@ -199,11 +212,10 @@ export class MkvOutputFormat extends OutputFormat {
static getSupportedCodecs(): MediaCodec[] {
return [
'avc', 'hevc', 'vp8', 'vp9', 'av1',
'aac', 'mp3', 'opus', 'vorbis', 'flac',
// pcm-s8, pcm-f32be, ulaw and alaw are not supported
'pcm-u8', 'pcm-s16', 'pcm-s16be', 'pcm-s24', 'pcm-s24be', 'pcm-s32', 'pcm-s32be', 'pcm-f32',
'webvtt',
...VIDEO_CODECS,
...NON_PCM_AUDIO_CODECS,
...PCM_AUDIO_CODECS.filter(codec => !['pcm-s8', 'pcm-f32be', 'ulaw', 'alaw'].includes(codec)),
...SUBTITLE_CODECS,
];
}
}
@@ -228,9 +240,9 @@ export class WebMOutputFormat extends MkvOutputFormat {
static override getSupportedCodecs(): MediaCodec[] {
return [
'vp8', 'vp9', 'av1',
'opus', 'vorbis',
'webvtt',
...VIDEO_CODECS.filter(codec => ['vp8', 'vp9', 'av1'].includes(codec)),
...AUDIO_CODECS.filter(codec => ['opus', 'vorbis'].includes(codec)),
...SUBTITLE_CODECS,
];
}
@@ -275,7 +287,9 @@ export class WaveOutputFormat extends OutputFormat {
static getSupportedCodecs(): MediaCodec[] {
return [
'pcm-u8', 'pcm-s16', 'pcm-s24', 'pcm-s32', 'pcm-f32', 'ulaw', 'alaw',
...PCM_AUDIO_CODECS.filter(codec =>
['pcm-s16', 'pcm-s24', 'pcm-s32', 'pcm-f32', 'pcm-u8', 'ulaw', 'alaw'].includes(codec),
),
];
}
}
src/output.ts
+42 -37
View File
@@ -6,9 +6,12 @@ import { Target } from './target';
import { Writer } from './writer';
/** @public */
export type OutputOptions = {
format: OutputFormat;
target: Target;
export type OutputOptions<
F extends OutputFormat = OutputFormat,
T extends Target = Target,
> = {
format: F;
target: T;
};
/** @public */
@@ -63,11 +66,13 @@ const validateBaseTrackMetadata = (metadata: BaseTrackMetadata) => {
};
/** @public */
export class Output {
/** @internal */
_format: OutputFormat;
/** @internal */
_target: Target;
export class Output<
F extends OutputFormat = OutputFormat,
T extends Target = Target,
> {
format: F;
target: T;
/** @internal */
_muxer: Muxer;
/** @internal */
@@ -83,7 +88,7 @@ export class Output {
/** @internal */
_mutex = new AsyncMutex();
constructor(options: OutputOptions) {
constructor(options: OutputOptions<F, T>) {
if (!options || typeof options !== 'object') {
throw new TypeError('options must be an object.');
}
@@ -99,8 +104,8 @@ export class Output {
}
options.target._output = this;
this._format = options.format;
this._target = options.target;
this.format = options.format;
this.target = options.target;
this._writer = options.target._createWriter();
this._muxer = options.format._createMuxer(this);
@@ -159,7 +164,7 @@ export class Output {
}
// Verify maximum track count constraints
const supportedTrackCounts = this._format.getSupportedTrackCounts();
const supportedTrackCounts = this.format.getSupportedTrackCounts();
const presentTracksOfThisType = this._tracks.reduce(
(count, track) => count + (track.type === type ? 1 : 0),
0,
@@ -168,15 +173,15 @@ export class Output {
if (presentTracksOfThisType === maxCount) {
throw new Error(
maxCount === 0
? `${this._format._getName()} does not support ${type} tracks.`
: (`${this._format._getName()} does not support more than ${maxCount} ${type} track`
? `${this.format._getName()} does not support ${type} tracks.`
: (`${this.format._getName()} does not support more than ${maxCount} ${type} track`
+ `${maxCount === 1 ? '' : 's'}.`),
);
}
const maxTotalCount = supportedTrackCounts.total.max;
if (this._tracks.length === maxTotalCount) {
throw new Error(
`${this._format._getName()} does not support more than ${maxTotalCount} tracks`
`${this.format._getName()} does not support more than ${maxTotalCount} tracks`
+ `${maxTotalCount === 1 ? '' : 's'} in total.`,
);
}
@@ -190,48 +195,48 @@ export class Output {
} as OutputTrack;
if (track.type === 'video') {
const supportedVideoCodecs = this._format.getSupportedVideoCodecs();
const supportedVideoCodecs = this.format.getSupportedVideoCodecs();
if (supportedVideoCodecs.length === 0) {
throw new Error(
`${this._format._getName()} does not support video tracks.`
+ this._format._codecUnsupportedHint(track.source._codec),
`${this.format._getName()} does not support video tracks.`
+ this.format._codecUnsupportedHint(track.source._codec),
);
} else if (!supportedVideoCodecs.includes(track.source._codec)) {
throw new Error(
`Codec '${track.source._codec}' cannot be contained within ${this._format._getName()}. Supported`
`Codec '${track.source._codec}' cannot be contained within ${this.format._getName()}. Supported`
+ ` video codecs are: ${supportedVideoCodecs.map(codec => `'${codec}'`).join(', ')}.`
+ this._format._codecUnsupportedHint(track.source._codec),
+ this.format._codecUnsupportedHint(track.source._codec),
);
}
} else if (track.type === 'audio') {
const supportedAudioCodecs = this._format.getSupportedAudioCodecs();
const supportedAudioCodecs = this.format.getSupportedAudioCodecs();
if (supportedAudioCodecs.length === 0) {
throw new Error(
`${this._format._getName()} does not support audio tracks.`
+ this._format._codecUnsupportedHint(track.source._codec),
`${this.format._getName()} does not support audio tracks.`
+ this.format._codecUnsupportedHint(track.source._codec),
);
} else if (!supportedAudioCodecs.includes(track.source._codec)) {
throw new Error(
`Codec '${track.source._codec}' cannot be contained within ${this._format._getName()}. Supported`
`Codec '${track.source._codec}' cannot be contained within ${this.format._getName()}. Supported`
+ ` audio codecs are: ${supportedAudioCodecs.map(codec => `'${codec}'`).join(', ')}.`
+ this._format._codecUnsupportedHint(track.source._codec),
+ this.format._codecUnsupportedHint(track.source._codec),
);
}
} else if (track.type === 'subtitle') {
const supportedSubtitleCodecs = this._format.getSupportedSubtitleCodecs();
const supportedSubtitleCodecs = this.format.getSupportedSubtitleCodecs();
if (supportedSubtitleCodecs.length === 0) {
throw new Error(
`${this._format._getName()} does not support subtitle tracks.`
+ this._format._codecUnsupportedHint(track.source._codec),
`${this.format._getName()} does not support subtitle tracks.`
+ this.format._codecUnsupportedHint(track.source._codec),
);
} else if (!supportedSubtitleCodecs.includes(track.source._codec)) {
throw new Error(
`Codec '${track.source._codec}' cannot be contained within ${this._format._getName()}. Supported`
`Codec '${track.source._codec}' cannot be contained within ${this.format._getName()}. Supported`
+ ` subtitle codecs are: ${supportedSubtitleCodecs.map(codec => `'${codec}'`).join(', ')}.`
+ this._format._codecUnsupportedHint(track.source._codec),
+ this.format._codecUnsupportedHint(track.source._codec),
);
}
}
@@ -249,7 +254,7 @@ export class Output {
}
// Verify minimum track count constraints
const supportedTrackCounts = this._format.getSupportedTrackCounts();
const supportedTrackCounts = this.format.getSupportedTrackCounts();
for (const trackType of ALL_TRACK_TYPES) {
const presentTracksOfThisType = this._tracks.reduce(
(count, track) => count + (track.type === trackType ? 1 : 0),
@@ -259,9 +264,9 @@ export class Output {
if (presentTracksOfThisType < minCount) {
throw new Error(
minCount === supportedTrackCounts[trackType].max
? (`${this._format._getName()} requires exactly ${minCount} ${trackType}`
? (`${this.format._getName()} requires exactly ${minCount} ${trackType}`
+ ` track${minCount === 1 ? '' : 's'}.`)
: (`${this._format._getName()} requires at least ${minCount} ${trackType}`
: (`${this.format._getName()} requires at least ${minCount} ${trackType}`
+ ` track${minCount === 1 ? '' : 's'}.`),
);
}
@@ -270,9 +275,9 @@ export class Output {
if (this._tracks.length < totalMinCount) {
throw new Error(
totalMinCount === supportedTrackCounts.total.max
? (`${this._format._getName()} requires exactly ${totalMinCount} track`
? (`${this.format._getName()} requires exactly ${totalMinCount} track`
+ `${totalMinCount === 1 ? '' : 's'}.`)
: (`${this._format._getName()} requires at least ${totalMinCount} track`
: (`${this.format._getName()} requires at least ${totalMinCount} track`
+ `${totalMinCount === 1 ? '' : 's'}.`),
);
}
@@ -302,7 +307,7 @@ export class Output {
const release = await this._mutex.acquire();
const promises = this._tracks.map(x => x.source._flush());
const promises = this._tracks.map(x => x.source._flushOrWaitForClose());
await Promise.all(promises);
await this._writer.close();
@@ -321,7 +326,7 @@ export class Output {
const release = await this._mutex.acquire();
const promises = this._tracks.map(x => x.source._flush());
const promises = this._tracks.map(x => x.source._flushOrWaitForClose());
await Promise.all(promises);
await this._muxer.finalize();
src/sample.ts
+58 -4
View File
@@ -43,6 +43,10 @@ export class EncodedVideoSample {
}
is(otherSample: EncodedVideoSample) {
if (!(otherSample instanceof EncodedVideoSample)) {
throw new TypeError('otherSample must be an EncodedVideoSample.');
}
return (
this.type === otherSample.type
&& this.timestamp === otherSample.timestamp
@@ -51,9 +55,32 @@ export class EncodedVideoSample {
);
}
clone(options?: {
timestamp?: number;
duration?: number;
}) {
if (options !== undefined && (!options || typeof options !== 'object')) {
throw new TypeError('options, when provided, must be an object.');
}
if (options?.timestamp !== undefined && !Number.isFinite(options.timestamp)) {
throw new TypeError('options.timestamp, when provided, must be a number.');
}
if (options?.duration !== undefined && !Number.isFinite(options.duration)) {
throw new TypeError('options.duration, when provided, must be a number.');
}
return new EncodedVideoSample(
this.data,
this.type,
options?.timestamp ?? this.timestamp,
options?.duration ?? this.duration,
this.byteLength,
);
}
static fromEncodedVideoChunk(chunk: EncodedVideoChunk) {
if (typeof EncodedVideoChunk === 'undefined') {
throw new Error('Your browser does not support EncodedVideoChunk.');
if (!(chunk instanceof EncodedVideoChunk)) {
throw new TypeError('chunk must be an EncodedVideoChunk.');
}
const data = new Uint8Array(chunk.byteLength);
@@ -108,6 +135,10 @@ export class EncodedAudioSample {
}
is(otherSample: EncodedAudioSample) {
if (!(otherSample instanceof EncodedAudioSample)) {
throw new TypeError('otherSample must be an EncodedAudioSample.');
}
return (
this.type === otherSample.type
&& this.timestamp === otherSample.timestamp
@@ -116,9 +147,32 @@ export class EncodedAudioSample {
);
}
clone(options?: {
timestamp?: number;
duration?: number;
}) {
if (options !== undefined && (!options || typeof options !== 'object')) {
throw new TypeError('options, when provided, must be an object.');
}
if (options?.timestamp !== undefined && !Number.isFinite(options.timestamp)) {
throw new TypeError('options.timestamp, when provided, must be a number.');
}
if (options?.duration !== undefined && !Number.isFinite(options.duration)) {
throw new TypeError('options.duration, when provided, must be a number.');
}
return new EncodedAudioSample(
this.data,
this.type,
options?.timestamp ?? this.timestamp,
options?.duration ?? this.duration,
this.byteLength,
);
}
static fromEncodedAudioChunk(chunk: EncodedAudioChunk) {
if (typeof EncodedAudioChunk === 'undefined') {
throw new Error('Your browser does not support EncodedAudioChunk.');
if (!(chunk instanceof EncodedAudioChunk)) {
throw new TypeError('chunk must be an EncodedAudioChunk.');
}
const data = new Uint8Array(chunk.byteLength);
src/wave/wave-demuxer.ts
+4
View File
@@ -183,6 +183,10 @@ const SAMPLE_SIZE_IN_FRAMES = 2048;
class WaveAudioTrackBacking implements InputAudioTrackBacking {
constructor(public demuxer: WaveDemuxer) {}
getId() {
return 1;
}
async getCodec() {
return this.demuxer.getCodec();
}