Compare commits

..
35 Commits
Author SHA1 Message Date
Vanilagy e5aeff89e3 Wtf 2025-12-04 10:03:09 +01:00
Vanilagy 3620b04caf Fix incomplete AV1 sequence header OBU parsing, fix missing saample close() calls in audio resampling path, add error logging when open samples are GC'd, bump patch 2025-12-04 09:58:12 +01:00
Vanilagy c0c353dbe8 Bump patch 2025-11-26 23:31:29 +01:00
Vanilagy 75096edb23 Update TypeScript, fix generic ArrayBufferView type errors, allow SharedArrayBuffer as input for BufferSource 2025-11-26 12:04:23 +01:00
Vanilagy f33dd4f2cd UrlSource: Deal gracefully with servers that cap unbounded range requests (fixes #231) 2025-11-26 11:26:09 +01:00
Vanilagy cc3d48451e Bump patch 2025-11-25 17:26:12 +01:00
Vanilagy 34b3ef0d44 Fix URLs that get redirected 2025-11-24 17:41:41 +01:00
Vanilagy 896242a5a5 Enable sitemap generation for docs 2025-11-24 17:34:07 +01:00
Vanilagy ef7a40944e Stronger encodability checks for Firefox (fixes #222) 2025-11-24 17:18:46 +01:00
Vanilagy f62ecb0eb6 Fix wrong type being used 2025-11-24 16:53:53 +01:00
Vanilagy da159bb063 Stop retrying requests when the source is disposed, log warning in possible CORS detection case (see #158) 2025-11-24 16:28:10 +01:00
Vanilagy c5d1efc18d Bump patch 2025-11-20 09:22:26 +01:00
Vanilagy 9464adf22f Remove pure annotations for non-function call expressions (fixes #225) 2025-11-18 11:36:20 +01:00
Vanilagy 17dfd2c68a Bump minor 2025-11-13 20:18:00 +01:00
Vanilagy abe6185ddd Fix test? 2025-11-13 11:02:36 +01:00
Vanilagy 217b383f12 Add read/write support for track disposition information 2025-11-13 11:00:42 +01:00
2c96ec0f1b feature(isobmff): read disabled tracks (#214)
* feature(isobmff): read disabled tracks

* Put enabled ISOBMFF tracks first

---------

Co-authored-by: Vanilagy <[email protected]>
2025-11-13 09:34:29 +01:00
Vanilagy 89d48d1bf9 Bump patch 2025-11-13 08:52:48 +01:00
Vanilagy 6684984e7e Fix faulty timestamp rounding to nearest integer (fixes #207) 2025-11-12 21:35:18 +01:00
Vanilagy 0f030dc2a6 Add Ponder 2025-11-12 19:28:43 +01:00
Vanilagy 02b08e036b Fix Matroska files where no audio packets are key packets (fixes #192) 2025-11-12 16:07:17 +01:00
Vanilagy 327696666b Expand doc blocks 2025-11-12 14:40:47 +01:00
Vanilagy 07b2e70863 Force software decoding in Chromium for interlaced AVC 2025-11-12 14:29:56 +01:00
Vanilagy 3cb6ed82ce Fix globals not being treeshakable by some bundlers (fixes #172) 2025-11-12 13:11:18 +01:00
Vanilagy 12216ae29e Remove reordering Matroska blocks by references (fixes #178) 2025-11-12 10:49:47 +01:00
Vanilagy 198b3d2eae Merge branch 'main' of https://github.com/Vanilagy/mediabunny 2025-11-12 10:05:41 +01:00
Vanilagy e33e9f13fe Fix missing VideoSample.close() calls in CanvasSink conversion path 2025-11-12 10:05:41 +01:00
Vanilagy fa1f44cf92 Bump patch 2025-11-11 22:26:46 +01:00
Vanilagy 966ac133e9 Merge branch 'main' of https://github.com/Vanilagy/mediabunny 2025-11-11 18:03:28 +01:00
Vanilagy 23f814679e Modify audio encoder to fill gaps between timestamps with silence to ensure AV sync (fixes #176) 2025-11-11 18:03:28 +01:00
Jonny BurgerandGitHub f3a2d57156 Don't overlook potential packets in FLAC bitstream (#219)
* Don't overlook potential packets in FLAC bitstream

* Update way of going back
2025-11-11 17:07:49 +01:00
Vanilagy 9d477dfb14 Allow muxing of avc3 into ISOBMFF (see #216) 2025-11-11 17:06:10 +01:00
Vanilagy 5b45c51d40 Support AVC3 in ISOBMFF (fixes #216) 2025-11-11 16:55:54 +01:00
Vanilagy 71dd2c565b Merge branch 'main' of https://github.com/Vanilagy/metamuxer 2025-11-11 11:05:17 +01:00
Vanilagy 3db4aa8b91 Fixed doubly stacked rotation correction when converting rotated video with a custom process function (fixes #218) 2025-11-11 11:05:15 +01:00
51 changed files with 1300 additions and 755 deletions
+4
View File
@@ -39,6 +39,10 @@ Mediabunny is a JavaScript library for reading, writing, and converting media fi
<a href="https://pqina.nl/pintura/" target="_blank" rel="sponsored">
<img src="./docs/public/sponsors/pintura-labs.png" width="50" height="50" alt="Pintura Labs">
</a>
&nbsp;&nbsp;&nbsp;&nbsp;
<a href="https://ponder.ai/" target="_blank" rel="sponsored">
<img src="./docs/public/sponsors/ponder.png" width="50" height="50" alt="Ponder">
</a>
</div>
### Bronze sponsors
+12 -3
View File
@@ -48,6 +48,7 @@
target
});
let ctx = null;
const conversion = await Mediabunny.Conversion.init({
input: new Mediabunny.Input({
formats: Mediabunny.ALL_FORMATS,
@@ -56,6 +57,8 @@
output,
audio: (_, n) => ({
discard: n > 1,
//codec: 'opus',
//codec: 'opus',
/*
process: (sample) => {
return sample;
@@ -97,6 +100,10 @@
},
*/
video: () => ({
width: 720,
frameRate: 30,
bitrate: Mediabunny.QUALITY_VERY_LOW,
//discard: true,
/*
process: (sample) => {
if (!ctx) {
@@ -108,9 +115,11 @@
ctx = canvas.getContext('2d');
}
console.log(ctx.canvas.width, ctx.canvas.height);
ctx.clearRect(0, 0, ctx.canvas.width, ctx.canvas.height);
sample.draw(ctx, 0, 0);
ctx.drawImage(watermark, 32, 32);
sample.drawWithFit(ctx, { fit: 'fill' });
//ctx.drawImage(watermark, 32, 32);
return ctx.canvas;
},
@@ -171,7 +180,7 @@
},
trim: {
start: 0,
end: 10
end: 20
},
});
console.log(conversion);
+42 -1
View File
@@ -14,7 +14,47 @@
source: new Mediabunny.BlobSource(file),
});
const videoTrack = await input.getPrimaryAudioTrack();
const audioTrack = await input.getPrimaryAudioTrack();
const decoderConfig = await audioTrack.getDecoderConfig();
console.log(decoderConfig);
/*
const videoTrack = await input.getPrimaryVideoTrack();
const sink = new Mediabunny.EncodedPacketSink(videoTrack);
for await (const packet of sink.packets()) {
console.log(packet);
}
*/
/*
return;
const audioTrack = await input.getPrimaryAudioTrack();
const sink = new Mediabunny.EncodedPacketSink(audioTrack);
for await (const packet of sink.packets()) {
console.log(packet.timestamp, packet.duration, packet.type)
}
*/
/*
const audioTrack = await input.getPrimaryAudioTrack();
const sink = new Mediabunny.AudioSampleSink(audioTrack);
let lastEnd = 0;
for await (const sample of sink.samples()) {
if (sample.timestamp - lastEnd > 0) {
console.warn(sample.timestamp - lastEnd)
console.log(sample.timestamp, sample.duration, "diffie", sample.timestamp - lastEnd);
}
lastEnd = sample.timestamp + sample.duration;
sample.close();
}
*/
/*
const sink = new Mediabunny.EncodedPacketSink(videoTrack);
for await (const packet of sink.packets()) {
@@ -22,6 +62,7 @@
if (packet.timestamp > 10) break;
}
*/
/*
const sink = new Mediabunny.VideoSampleSink(videoTrack);
+4 -1
View File
@@ -14,6 +14,9 @@ export default withMermaid({
title: 'Mediabunny',
description: DESCRIPTION,
cleanUrls: true,
sitemap: {
hostname: 'https://mediabunny.dev',
},
head: [
['link', { rel: 'icon', type: 'image/png', href: '/mediabunny-logo.png' }],
['link', { rel: 'icon', type: 'image/svg+xml', href: '/mediabunny-logo.svg' }],
@@ -34,7 +37,7 @@ export default withMermaid({
// https://vitepress.dev/reference/default-theme-config
nav: [
{ text: 'Guide', link: '/guide/introduction', activeMatch: '/guide' },
{ text: 'API', link: '/api', activeMatch: '/api' },
{ text: 'API', link: '/api/', activeMatch: '/api' },
{ text: 'LLMs', link: '/llms', activeMatch: '/llms' },
{ text: 'Examples', link: '/examples', activeMatch: '/examples' },
{ text: 'Sponsors', link: '/#sponsors', activeMatch: '/#sponsors' },
+6 -6
View File
@@ -9,37 +9,37 @@ hero:
features:
- title: Metadata extraction
details: Extract various metadata from an input media file.
link: /examples/metadata-extraction
link: /examples/metadata-extraction/
target: _self
icon:
src: /mingcute--file-info-line.svg
- title: Thumbnail generation
details: Generate multiple small thumbnails for a video track.
link: /examples/thumbnail-generation
link: /examples/thumbnail-generation/
target: _self
icon:
src: /mingcute--photo-album-line.svg
- title: Media player (advanced)
details: "A full video & audio media player, implemented from scratch with Mediabunny, with microsecond playback accuracy."
link: /examples/media-player
link: /examples/media-player/
target: _self
icon:
src: /mingcute--video-line.svg
- title: File compression
details: Convert an input file to a highly-compressed MP4 file.
link: /examples/file-compression
link: /examples/file-compression/
target: _self
icon:
src: /mingcute--file-zip-line.svg
- title: Procedural video generation
details: Generate a video file as fast as the hardware allows.
link: /examples/procedural-generation
link: /examples/procedural-generation/
target: _self
icon:
src: /mingcute--magic-3-line.svg
- title: Live recording & streaming
details: Record a video from live sources and stream it to a video element.
link: /examples/live-recording
link: /examples/live-recording/
target: _self
icon:
src: /mingcute--microphone-line.svg
+9 -1
View File
@@ -433,4 +433,12 @@ await textSource.add('00:00:00.000 --> 00:00:02.000\nHello there!\n\n');
await textSource.add('00:00:02.500 --> 00:00:04.000\nChunky chunks.\n\n');
```
The chunks have certain constraints: A cue must be fully contained within a chunk and cannot be split across multiple smaller chunks (although a chunk can contain multiple cues). Also, the WebVTT preamble must be added first and all at once.
The chunks have certain constraints: A cue must be fully contained within a chunk and cannot be split across multiple smaller chunks (although a chunk can contain multiple cues). Also, the WebVTT preamble must be added first and all at once.
::: info
For QuickTime to display WebVTT subtitles, it typically expects alignment information to be specified:
```
00:00:00.000 --> 00:00:02.000 align:center
This is your last chance.
```
:::
+4
View File
@@ -106,6 +106,10 @@ track.languageCode; // => string
// A user-defined name for this track.
track.name; // => string
// Information about the intended usage of the track
// (default, commentary, hearing-impaired, visually-impaired, etc.)
track.disposition; // TrackDisposition
```
#### Codec information
+1
View File
@@ -56,6 +56,7 @@ output.addVideoTrack(videoSource, {
output.addAudioTrack(audioSourceEng, {
language: 'eng', // ISO 639-2/T language code
name: 'Developer Commentary', // Sets a user-defined track name
disposition: { commentary: true }, // Sets additional flags in the file
});
output.addAudioTrack(audioSourceGer, {
language: 'ger',
+1
View File
@@ -98,6 +98,7 @@ const sponsors = {
],
silver: [
{ image: '/sponsors/pintura-labs.png', name: 'Pintura Labs', url: 'https://pqina.nl/pintura/' },
{ image: '/sponsors/ponder.png', name: 'Ponder', url: 'https://ponder.ai/' },
],
bronze: [
{ image: '/sponsors/rve.png', name: 'React Video Editor', url: 'https://www.reactvideoeditor.com/' },
Binary file not shown.

After

Width:  |  Height:  |  Size: 17 KiB

+1 -1
View File
@@ -21,7 +21,7 @@ const context = canvas.getContext('2d', { alpha: false, desynchronized: true })!
const frameRate = 30;
const chunks: Uint8Array[] = [];
const chunks: Uint8Array<ArrayBuffer>[] = [];
let recording = false;
let output: Output;
let videoSource: CanvasSource;
@@ -97,7 +97,7 @@ const extractMetadata = (resource: File | string) => {
'Lyrics': tags.lyrics,
'Comment': tags.comment,
'Images': tags.images?.map((image) => {
const blob = new Blob([image.data], { type: image.mimeType });
const blob = new Blob([image.data.slice()], { type: image.mimeType });
const element = new Image();
element.src = URL.createObjectURL(blob);
+285 -462
View File
File diff suppressed because it is too large Load Diff
+4 -4
View File
@@ -1,7 +1,7 @@
{
"name": "mediabunny",
"author": "Vanilagy",
"version": "1.24.5",
"version": "1.25.4",
"description": "Pure TypeScript media toolkit for reading, writing, and converting media files, directly in the browser.",
"type": "module",
"workspaces": [
@@ -70,7 +70,7 @@
"@eslint/js": "^9.22.0",
"@fontsource-variable/rubik": "^5.2.6",
"@fontsource/dm-mono": "^5.2.6",
"@microsoft/api-extractor": "^7.52.1",
"@microsoft/api-extractor": "^7.55.1",
"@stylistic/eslint-plugin": "^4.2.0",
"@tailwindcss/vite": "^4.1.7",
"@types/markdown-it-footnote": "^3.0.4",
@@ -84,8 +84,8 @@
"mermaid": "^11.6.0",
"tailwindcss": "^4.1.7",
"tsx": "^4.19.4",
"typescript": "^5.8.2",
"typescript-eslint": "^8.26.1",
"typescript": "^5.9.3",
"typescript-eslint": "^8.48.0",
"vite": "^6.3.5",
"vitepress": "^1.6.3",
"vitepress-plugin-llms": "^1.5.1",
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@mediabunny/mp3-encoder",
"author": "Vanilagy",
"version": "1.24.5",
"version": "1.25.4",
"description": "MP3 encoder extension for Mediabunny, based on LAME.",
"main": "./dist/bundles/mediabunny-mp3-encoder.mjs",
"module": "./dist/bundles/mediabunny-mp3-encoder.mjs",
+7
View File
@@ -21,6 +21,7 @@ import {
} from '../misc';
import { EncodedPacket, PLACEHOLDER_DATA } from '../packet';
import { readBytes, Reader } from '../reader';
import { DEFAULT_TRACK_DISPOSITION } from '../metadata';
import { FrameHeader, MAX_FRAME_HEADER_SIZE, MIN_FRAME_HEADER_SIZE, readFrameHeader } from './adts-reader';
const SAMPLES_PER_AAC_FRAME = 1024;
@@ -188,6 +189,12 @@ class AdtsAudioTrackBacking implements InputAudioTrackBacking {
return sampleRate;
}
getDisposition() {
return {
...DEFAULT_TRACK_DISPOSITION,
};
}
async getDecoderConfig(): Promise<AudioDecoderConfig> {
assert(this.demuxer.firstFrameHeader);
+280 -52
View File
@@ -24,7 +24,7 @@ import {
toUint8Array,
} from './misc';
import { PacketType } from './packet';
import { MetadataTags } from './tags';
import { MetadataTags } from './metadata';
// References for AVC/HEVC code:
// ISO 14496-15
@@ -229,7 +229,7 @@ const extractNalUnitTypeForAvc = (data: Uint8Array) => {
};
/** Builds an AvcDecoderConfigurationRecord from an AVC packet in Annex B format. */
export const extractAvcDecoderConfigurationRecord = (packetData: Uint8Array) => {
export const extractAvcDecoderConfigurationRecord = (packetData: Uint8Array): AvcDecoderConfigurationRecord | null => {
try {
const nalUnits = findNalUnitsInAnnexB(packetData);
@@ -247,60 +247,27 @@ export const extractAvcDecoderConfigurationRecord = (packetData: Uint8Array) =>
// Let's get the first SPS for profile and level information
const spsData = spsUnits[0]!;
const bitstream = new Bitstream(removeEmulationPreventionBytes(spsData));
const spsInfo = parseAvcSps(spsData);
assert(spsInfo !== null);
bitstream.skipBits(1); // forbidden_zero_bit
bitstream.skipBits(2); // nal_ref_idc
const nal_unit_type = bitstream.readBits(5);
const hasExtendedData = spsInfo.profileIdc === 100
|| spsInfo.profileIdc === 110
|| spsInfo.profileIdc === 122
|| spsInfo.profileIdc === 144;
if (nal_unit_type !== 7) { // SPS NAL unit type is 7
console.error('Invalid SPS NAL unit type');
return null;
}
const profile_idc = bitstream.readAlignedByte();
const constraint_flags = bitstream.readAlignedByte();
const level_idc = bitstream.readAlignedByte();
const record: AvcDecoderConfigurationRecord = {
return {
configurationVersion: 1,
avcProfileIndication: profile_idc,
profileCompatibility: constraint_flags,
avcLevelIndication: level_idc,
avcProfileIndication: spsInfo.profileIdc,
profileCompatibility: spsInfo.constraintFlags,
avcLevelIndication: spsInfo.levelIdc,
lengthSizeMinusOne: 3, // Typically 4 bytes for length field
sequenceParameterSets: spsUnits,
pictureParameterSets: ppsUnits,
chromaFormat: null,
bitDepthLumaMinus8: null,
bitDepthChromaMinus8: null,
sequenceParameterSetExt: null,
chromaFormat: hasExtendedData ? spsInfo.chromaFormatIdc : null,
bitDepthLumaMinus8: hasExtendedData ? spsInfo.bitDepthLumaMinus8 : null,
bitDepthChromaMinus8: hasExtendedData ? spsInfo.bitDepthChromaMinus8 : null,
sequenceParameterSetExt: hasExtendedData ? spsExtUnits : null,
};
if (
profile_idc === 100
|| profile_idc === 110
|| profile_idc === 122
|| profile_idc === 144
) {
readExpGolomb(bitstream); // seq_parameter_set_id
const chroma_format_idc = readExpGolomb(bitstream);
if (chroma_format_idc === 3) {
bitstream.skipBits(1); // separate_colour_plane_flag
}
const bit_depth_luma_minus8 = readExpGolomb(bitstream);
const bit_depth_chroma_minus8 = readExpGolomb(bitstream);
record.chromaFormat = chroma_format_idc;
record.bitDepthLumaMinus8 = bit_depth_luma_minus8;
record.bitDepthChromaMinus8 = bit_depth_chroma_minus8;
record.sequenceParameterSetExt = spsExtUnits;
}
return record;
} catch (error) {
console.error('Error building AVC Decoder Configuration Record:', error);
return null;
@@ -377,6 +344,206 @@ export const serializeAvcDecoderConfigurationRecord = (record: AvcDecoderConfigu
return new Uint8Array(bytes);
};
/** Deserializes an AvcDecoderConfigurationRecord from the format specified in Section 5.3.3.1 of ISO 14496-15. */
export const deserializeAvcDecoderConfigurationRecord = (data: Uint8Array): AvcDecoderConfigurationRecord | null => {
try {
const view = toDataView(data);
let offset = 0;
// Read header
const configurationVersion = view.getUint8(offset++);
const avcProfileIndication = view.getUint8(offset++);
const profileCompatibility = view.getUint8(offset++);
const avcLevelIndication = view.getUint8(offset++);
const lengthSizeMinusOne = view.getUint8(offset++) & 0x03;
const numOfSequenceParameterSets = view.getUint8(offset++) & 0x1F;
// Read SPS
const sequenceParameterSets: Uint8Array[] = [];
for (let i = 0; i < numOfSequenceParameterSets; i++) {
const length = view.getUint16(offset, false);
offset += 2;
sequenceParameterSets.push(data.subarray(offset, offset + length));
offset += length;
}
const numOfPictureParameterSets = view.getUint8(offset++);
// Read PPS
const pictureParameterSets: Uint8Array[] = [];
for (let i = 0; i < numOfPictureParameterSets; i++) {
const length = view.getUint16(offset, false);
offset += 2;
pictureParameterSets.push(data.subarray(offset, offset + length));
offset += length;
}
const record: AvcDecoderConfigurationRecord = {
configurationVersion,
avcProfileIndication,
profileCompatibility,
avcLevelIndication,
lengthSizeMinusOne,
sequenceParameterSets,
pictureParameterSets,
chromaFormat: null,
bitDepthLumaMinus8: null,
bitDepthChromaMinus8: null,
sequenceParameterSetExt: null,
};
// Check if there are extended profile fields
if (
(
avcProfileIndication === 100
|| avcProfileIndication === 110
|| avcProfileIndication === 122
|| avcProfileIndication === 144
)
&& offset + 4 <= data.length
) {
const chromaFormat = view.getUint8(offset++) & 0x03;
const bitDepthLumaMinus8 = view.getUint8(offset++) & 0x07;
const bitDepthChromaMinus8 = view.getUint8(offset++) & 0x07;
const numOfSequenceParameterSetExt = view.getUint8(offset++);
record.chromaFormat = chromaFormat;
record.bitDepthLumaMinus8 = bitDepthLumaMinus8;
record.bitDepthChromaMinus8 = bitDepthChromaMinus8;
// Read SPS Ext
const sequenceParameterSetExt: Uint8Array[] = [];
for (let i = 0; i < numOfSequenceParameterSetExt; i++) {
const length = view.getUint16(offset, false);
offset += 2;
sequenceParameterSetExt.push(data.subarray(offset, offset + length));
offset += length;
}
record.sequenceParameterSetExt = sequenceParameterSetExt;
}
return record;
} catch (error) {
console.error('Error deserializing AVC Decoder Configuration Record:', error);
return null;
}
};
export type AvcSpsInfo = {
profileIdc: number;
constraintFlags: number;
levelIdc: number;
frameMbsOnlyFlag: number;
chromaFormatIdc: number | null;
bitDepthLumaMinus8: number | null;
bitDepthChromaMinus8: number | null;
};
/** Parses an AVC SPS (Sequence Parameter Set) to extract basic information. */
export const parseAvcSps = (sps: Uint8Array): AvcSpsInfo | null => {
try {
const bitstream = new Bitstream(removeEmulationPreventionBytes(sps));
bitstream.skipBits(1); // forbidden_zero_bit
bitstream.skipBits(2); // nal_ref_idc
const nalUnitType = bitstream.readBits(5);
if (nalUnitType !== 7) { // SPS NAL unit type is 7
return null;
}
const profileIdc = bitstream.readAlignedByte();
const constraintFlags = bitstream.readAlignedByte();
const levelIdc = bitstream.readAlignedByte();
readExpGolomb(bitstream); // seq_parameter_set_id
let chromaFormatIdc: number | null = null;
let bitDepthLumaMinus8: number | null = null;
let bitDepthChromaMinus8: number | null = null;
// Handle high profile chroma_format_idc
if (
profileIdc === 100
|| profileIdc === 110
|| profileIdc === 122
|| profileIdc === 244
|| profileIdc === 44
|| profileIdc === 83
|| profileIdc === 86
|| profileIdc === 118
|| profileIdc === 128
) {
chromaFormatIdc = readExpGolomb(bitstream);
if (chromaFormatIdc === 3) {
bitstream.skipBits(1); // separate_colour_plane_flag
}
bitDepthLumaMinus8 = readExpGolomb(bitstream);
bitDepthChromaMinus8 = readExpGolomb(bitstream);
bitstream.skipBits(1); // qpprime_y_zero_transform_bypass_flag
const seqScalingMatrixPresentFlag = bitstream.readBits(1);
if (seqScalingMatrixPresentFlag) {
for (let i = 0; i < (chromaFormatIdc !== 3 ? 8 : 12); i++) {
const seqScalingListPresentFlag = bitstream.readBits(1);
if (seqScalingListPresentFlag) {
const sizeOfScalingList = i < 6 ? 16 : 64;
let lastScale = 8;
let nextScale = 8;
for (let j = 0; j < sizeOfScalingList; j++) {
if (nextScale !== 0) {
const deltaScale = readSignedExpGolomb(bitstream);
nextScale = (lastScale + deltaScale + 256) % 256;
}
lastScale = nextScale === 0 ? lastScale : nextScale;
}
}
}
}
}
readExpGolomb(bitstream); // log2_max_frame_num_minus4
const picOrderCntType = readExpGolomb(bitstream);
if (picOrderCntType === 0) {
readExpGolomb(bitstream); // log2_max_pic_order_cnt_lsb_minus4
} else if (picOrderCntType === 1) {
bitstream.skipBits(1); // delta_pic_order_always_zero_flag
readSignedExpGolomb(bitstream); // offset_for_non_ref_pic
readSignedExpGolomb(bitstream); // offset_for_top_to_bottom_field
const numRefFramesInPicOrderCntCycle = readExpGolomb(bitstream);
for (let i = 0; i < numRefFramesInPicOrderCntCycle; i++) {
readSignedExpGolomb(bitstream); // offset_for_ref_frame[i]
}
}
readExpGolomb(bitstream); // max_num_ref_frames
bitstream.skipBits(1); // gaps_in_frame_num_value_allowed_flag
readExpGolomb(bitstream); // pic_width_in_mbs_minus1
readExpGolomb(bitstream); // pic_height_in_map_units_minus1
const frameMbsOnlyFlag = bitstream.readBits(1);
return {
profileIdc,
constraintFlags,
levelIdc,
frameMbsOnlyFlag,
chromaFormatIdc,
bitDepthLumaMinus8,
bitDepthChromaMinus8,
};
} catch (error) {
console.error('Error parsing AVC SPS:', error);
return null;
}
};
// Data specified in ISO 14496-15
export type HevcDecoderConfigurationRecord = {
configurationVersion: number;
@@ -423,9 +590,7 @@ export const extractNalUnitTypeForHevc = (data: Uint8Array) => {
};
/** Builds a HevcDecoderConfigurationRecord from an HEVC packet in Annex B format. */
export const extractHevcDecoderConfigurationRecord = (
packetData: Uint8Array,
) => {
export const extractHevcDecoderConfigurationRecord = (packetData: Uint8Array) => {
try {
const nalUnits = findNalUnitsInAnnexB(packetData);
@@ -1289,6 +1454,69 @@ export const extractAv1CodecInfoFromPacket = (
}
}
// Frame size
const frameWidthBitsMinus1 = bitstream.readBits(4);
const frameHeightBitsMinus1 = bitstream.readBits(4);
const n1 = frameWidthBitsMinus1 + 1;
bitstream.skipBits(n1); // max_frame_width_minus_1
const n2 = frameHeightBitsMinus1 + 1;
bitstream.skipBits(n2); // max_frame_height_minus_1
// Frame IDs
let frameIdNumbersPresentFlag = 0;
if (reducedStillPictureHeader) {
frameIdNumbersPresentFlag = 0;
} else {
frameIdNumbersPresentFlag = bitstream.readBits(1);
}
if (frameIdNumbersPresentFlag) {
bitstream.skipBits(4); // delta_frame_id_length_minus_2
bitstream.skipBits(3); // additional_frame_id_length_minus_1
}
bitstream.skipBits(1); // use_128x128_superblock
bitstream.skipBits(1); // enable_filter_intra
bitstream.skipBits(1); // enable_intra_edge_filter
if (!reducedStillPictureHeader) {
bitstream.skipBits(1); // enable_interintra_compound
bitstream.skipBits(1); // enable_masked_compound
bitstream.skipBits(1); // enable_warped_motion
bitstream.skipBits(1); // enable_dual_filter
const enableOrderHint = bitstream.readBits(1);
if (enableOrderHint) {
bitstream.skipBits(1); // enable_jnt_comp
bitstream.skipBits(1); // enable_ref_frame_mvs
}
const seqChooseScreenContentTools = bitstream.readBits(1);
let seqForceScreenContentTools = 0;
if (seqChooseScreenContentTools) {
seqForceScreenContentTools = 2; // SELECT_SCREEN_CONTENT_TOOLS
} else {
seqForceScreenContentTools = bitstream.readBits(1);
}
if (seqForceScreenContentTools > 0) {
const seqChooseIntegerMv = bitstream.readBits(1);
if (!seqChooseIntegerMv) {
bitstream.skipBits(1); // seq_force_integer_mv
}
}
if (enableOrderHint) {
bitstream.skipBits(3); // order_hint_bits_minus_1
}
}
bitstream.skipBits(1); // enable_superres
bitstream.skipBits(1); // enable_cdef
bitstream.skipBits(1); // enable_restoration
// color_config()
const highBitdepth = bitstream.readBits(1);
let bitDepth = 8;
+5 -2
View File
@@ -337,6 +337,7 @@ export const extractVideoCodecString = (trackInfo: {
codec: VideoCodec | null;
codecDescription: Uint8Array | null;
colorSpace: VideoColorSpaceInit | null;
avcType: 1 | 3 | null;
avcCodecInfo: AvcDecoderConfigurationRecord | null;
hevcCodecInfo: HevcDecoderConfigurationRecord | null;
vp9CodecInfo: Vp9CodecInfo | null;
@@ -345,6 +346,8 @@ export const extractVideoCodecString = (trackInfo: {
const { codec, codecDescription, colorSpace, avcCodecInfo, hevcCodecInfo, vp9CodecInfo, av1CodecInfo } = trackInfo;
if (codec === 'avc') {
assert(trackInfo.avcType !== null);
if (avcCodecInfo) {
const bytes = new Uint8Array([
avcCodecInfo.avcProfileIndication,
@@ -352,14 +355,14 @@ export const extractVideoCodecString = (trackInfo: {
avcCodecInfo.avcLevelIndication,
]);
return `avc1.${bytesToHexString(bytes)}`;
return `avc${trackInfo.avcType}.${bytesToHexString(bytes)}`;
}
if (!codecDescription || codecDescription.byteLength < 4) {
throw new TypeError('AVC decoder description is not provided or is not at least 4 bytes long.');
}
return `avc1.${bytesToHexString(codecDescription.subarray(1, 4))}`;
return `avc${trackInfo.avcType}.${bytesToHexString(codecDescription.subarray(1, 4))}`;
} else if (codec === 'hevc') {
let generalProfileSpace: number;
let generalProfileIdc: number;
+18 -8
View File
@@ -48,7 +48,7 @@ import {
import { Output, TrackType } from './output';
import { Mp4OutputFormat } from './output-format';
import { AudioSample, clampCropRectangle, validateCropRectangle, VideoSample } from './sample';
import { MetadataTags, validateMetadataTags } from './tags';
import { MetadataTags, validateMetadataTags } from './metadata';
import { NullTarget } from './target';
/**
@@ -182,7 +182,8 @@ export type ConversionVideoOptions = {
* corrections.
*
* Must return a {@link VideoSample} or a `CanvasImageSource`, an array of them, or `null` for dropping the frame.
* When non-timestamped data is returned, the timestamp and duration from the source sample will be used.
* When non-timestamped data is returned, the timestamp and duration from the source sample will be used. Rotation
* metadata of the returned sample will be ignored.
*
* This function can also be used to manually resize frames. When doing so, you should signal the post-process
* dimensions using the `processedWidth` and `processedHeight` fields, which enables the encoder to better know what
@@ -873,7 +874,10 @@ export class Conversion {
|| trackOptions.process !== undefined;
let needsRerender = width !== originalWidth
|| height !== originalHeight
|| (totalRotation !== 0 && !outputSupportsRotation)
// TODO This is suboptimal: Forcing a rerender when both rotation and process are set is not
// performance-optimal, but right now there's no other way because we can't change the track rotation
// metadata after the output has already started. Should be possible with API changes in v2, though!
|| (totalRotation !== 0 && (!outputSupportsRotation || trackOptions.process !== undefined))
|| !!crop;
const alpha = trackOptions.alpha ?? 'discard';
@@ -1039,6 +1043,7 @@ export class Conversion {
duration: 1 / frameRate,
});
await this._registerVideoSample(track, trackOptions, source, sample);
sample.close();
}
};
@@ -1075,12 +1080,11 @@ export class Conversion {
duration: frameRate !== undefined ? 1 / frameRate : duration,
});
await this._registerVideoSample(track, trackOptions, source, sample);
sample.close();
if (frameRate !== undefined) {
lastCanvas = canvas;
lastCanvasTimestamp = adjustedSampleTimestamp;
} else {
sample.close();
}
}
@@ -1180,9 +1184,10 @@ export class Conversion {
this.output.addVideoTrack(videoSource, {
frameRate: trackOptions.frameRate,
// TEMP: This condition can be removed when all demuxers properly homogenize to BCP47 in v2
// TODO: This condition can be removed when all demuxers properly homogenize to BCP47 in v2
languageCode: isIso639Dash2LanguageCode(track.languageCode) ? track.languageCode : undefined,
name: track.name ?? undefined,
disposition: track.disposition,
rotation: needsRerender ? 0 : totalRotation, // Rerendering will bake the rotation into the output
});
this._addedCounts.video++;
@@ -1419,9 +1424,10 @@ export class Conversion {
}
this.output.addAudioTrack(audioSource, {
// TEMP: This condition can be removed when all demuxers properly homogenize to BCP47 in v2
// TODO: This condition can be removed when all demuxers properly homogenize to BCP47 in v2
languageCode: isIso639Dash2LanguageCode(track.languageCode) ? track.languageCode : undefined,
name: track.name ?? undefined,
disposition: track.disposition,
});
this._addedCounts.audio++;
this._totalTrackCount++;
@@ -1503,7 +1509,10 @@ export class Conversion {
targetSampleRate,
startTime: this._startTimestamp,
endTime: this._endTimestamp,
onSample: sample => this._registerAudioSample(track, trackOptions, source, sample),
onSample: async (sample) => {
await this._registerAudioSample(track, trackOptions, source, sample);
sample.close();
},
});
const sink = new AudioSampleSink(track);
@@ -1515,6 +1524,7 @@ export class Conversion {
}
await resampler.add(sample);
sample.close();
}
await resampler.finalize();
+1 -1
View File
@@ -8,7 +8,7 @@
import { Input } from './input';
import { InputTrack } from './input-track';
import { MetadataTags } from './tags';
import { MetadataTags } from './metadata';
export abstract class Demuxer {
input: Input;
+55 -9
View File
@@ -22,6 +22,7 @@ import {
VideoCodec,
} from './codec';
import { customAudioEncoders, customVideoEncoders } from './custom-coder';
import { isFirefox } from './misc';
import { EncodedPacket } from './packet';
/**
@@ -115,9 +116,15 @@ export type VideoEncodingAdditionalOptions = {
* format that supports transparency (such as WebM or Matroska).
*/
alpha?: 'discard' | 'keep';
/** Configures the bitrate mode. */
/** Configures the bitrate mode; defaults to `'variable'`. */
bitrateMode?: 'constant' | 'variable';
/** The latency mode used by the encoder; controls the performance-quality tradeoff. */
/**
* The latency mode used by the encoder; controls the performance-quality tradeoff.
*
* - `'quality'` (default): The encoder prioritizes quality over latency, and no frames can be dropped.
* - `'realtime'`: The encoder prioritizes low latency over quality, and may drop frames if the encoder becomes
* overloaded to keep up with real-time requirements.
*/
latencyMode?: 'quality' | 'realtime';
/**
* The full codec string as specified in the WebCodecs Codec Registry. This string must match the codec
@@ -125,7 +132,8 @@ export type VideoEncodingAdditionalOptions = {
*/
fullCodecString?: string;
/**
* A hint that configures the hardware acceleration method of this codec. This is best left on `'no-preference'`.
* A hint that configures the hardware acceleration method of this codec. This is best left on `'no-preference'`,
* the default.
*/
hardwareAcceleration?: 'no-preference' | 'prefer-hardware' | 'prefer-software';
/**
@@ -403,31 +411,31 @@ export class Quality {
* @group Encoding
* @public
*/
export const QUALITY_VERY_LOW = new Quality(0.3);
export const QUALITY_VERY_LOW = /* #__PURE__ */ new Quality(0.3);
/**
* Represents a low media quality.
* @group Encoding
* @public
*/
export const QUALITY_LOW = new Quality(0.6);
export const QUALITY_LOW = /* #__PURE__ */ new Quality(0.6);
/**
* Represents a medium media quality.
* @group Encoding
* @public
*/
export const QUALITY_MEDIUM = new Quality(1);
export const QUALITY_MEDIUM = /* #__PURE__ */ new Quality(1);
/**
* Represents a high media quality.
* @group Encoding
* @public
*/
export const QUALITY_HIGH = new Quality(2);
export const QUALITY_HIGH = /* #__PURE__ */ new Quality(2);
/**
* Represents a very high media quality.
* @group Encoding
* @public
*/
export const QUALITY_VERY_HIGH = new Quality(4);
export const QUALITY_VERY_HIGH = /* #__PURE__ */ new Quality(4);
/**
* Checks if the browser is able to encode the given codec.
@@ -522,7 +530,45 @@ export const canEncodeVideo = async (
});
const support = await VideoEncoder.isConfigSupported(encoderConfig);
return support.supported === true;
if (!support.supported) {
return false;
}
if (isFirefox()) {
// isConfigSupported on Firefox appears to unreliably indicate if encoding will actually succeed. Therefore, we
// just try encoding a frame to see if it actually works.
// https://github.com/Vanilagy/mediabunny/issues/222
// eslint-disable-next-line @typescript-eslint/no-misused-promises, no-async-promise-executor
return new Promise<boolean>(async (resolve) => {
try {
const encoder = new VideoEncoder({
output: () => {},
error: () => resolve(false),
});
encoder.configure(encoderConfig);
const frameData = new Uint8Array(width * height * 4);
const frame = new VideoFrame(frameData, {
format: 'RGBA',
codedWidth: width,
codedHeight: height,
timestamp: 0,
});
encoder.encode(frame);
frame.close();
await encoder.flush();
resolve(true);
} catch {
resolve(false);
}
});
} else {
return true;
}
};
/**
+13 -5
View File
@@ -28,7 +28,7 @@ import {
readU32Be,
readU8,
} from '../reader';
import { MetadataTags } from '../tags';
import { DEFAULT_TRACK_DISPOSITION, MetadataTags } from '../metadata';
import {
calculateCrc8,
readBlockSize,
@@ -328,11 +328,13 @@ export class FlacDemuxer extends Demuxer {
const nextByte = readU8(slice);
if (nextByte === 0xff) {
const positionBeforeReading = slice.filePos;
const byteAfterNextByte = readU8(slice);
const expected = this.blockingBit === 1 ? 0b1111_1001 : 0b1111_1000;
if (byteAfterNextByte !== expected) {
slice.skip(-1);
slice.filePos = positionBeforeReading;
continue;
}
@@ -345,7 +347,7 @@ export class FlacDemuxer extends Demuxer {
});
if (!nextFrameHeader) {
slice.skip(-1);
slice.filePos = positionBeforeReading;
continue;
}
@@ -355,14 +357,14 @@ export class FlacDemuxer extends Demuxer {
if (this.blockingBit === 0) {
// Case A: If the stream is fixed block size, this is the frame number, which increments by 1
if (nextFrameHeader.num - frameHeader.num !== 1) {
slice.skip(-1);
slice.filePos = positionBeforeReading;
continue;
}
} else {
// Case B: If the stream is variable block size, this is the sample number, which increments by
// amount of samples in a frame.
if (nextFrameHeader.num - frameHeader.num !== frameHeader.blockSize) {
slice.skip(-1);
slice.filePos = positionBeforeReading;
continue;
}
}
@@ -563,6 +565,12 @@ class FlacAudioTrackBacking implements InputAudioTrackBacking {
return this.demuxer.audioInfo.sampleRate;
}
getDisposition() {
return {
...DEFAULT_TRACK_DISPOSITION,
};
}
async getFirstTimestamp() {
return 0;
}
+2 -2
View File
@@ -20,7 +20,7 @@ import { Output, OutputAudioTrack } from '../output';
import { FlacOutputFormat } from '../output-format';
import { EncodedPacket } from '../packet';
import { FileSlice, readBytes } from '../reader';
import { AttachedImage, metadataTagsAreEmpty } from '../tags';
import { AttachedImage, metadataTagsAreEmpty } from '../metadata';
import { Writer } from '../writer';
import {
readBlockSize,
@@ -28,7 +28,7 @@ import {
readCodedNumber,
} from './flac-misc';
const FLAC_HEADER = new Uint8Array([0x66, 0x4c, 0x61, 0x43]); // 'fLaC'
const FLAC_HEADER = /* #__PURE__ */ new Uint8Array([0x66, 0x4c, 0x61, 0x43]); // 'fLaC'
const STREAMINFO_SIZE = 38;
const STREAMINFO_BLOCK_SIZE = 34;
+1 -1
View File
@@ -7,7 +7,7 @@
*/
import { decodeSynchsafe, encodeSynchsafe } from '../shared/mp3-misc';
import { MetadataTags } from './tags';
import { MetadataTags } from './metadata';
import {
coalesceIndex,
textDecoder,
+2 -1
View File
@@ -202,6 +202,7 @@ export {
AttachedImage,
RichImageData,
AttachedFile,
} from './tags';
TrackDisposition,
} from './metadata';
// 🐡🦔
+9 -9
View File
@@ -481,56 +481,56 @@ export class AdtsInputFormat extends InputFormat {
* @group Input formats
* @public
*/
export const MP4 = new Mp4InputFormat();
export const MP4 = /* #__PURE__ */ new Mp4InputFormat();
/**
* QuickTime File Format input format singleton.
* @group Input formats
* @public
*/
export const QTFF = new QuickTimeInputFormat();
export const QTFF = /* #__PURE__ */ new QuickTimeInputFormat();
/**
* Matroska input format singleton.
* @group Input formats
* @public
*/
export const MATROSKA = new MatroskaInputFormat();
export const MATROSKA = /* #__PURE__ */ new MatroskaInputFormat();
/**
* WebM input format singleton.
* @group Input formats
* @public
*/
export const WEBM = new WebMInputFormat();
export const WEBM = /* #__PURE__ */ new WebMInputFormat();
/**
* MP3 input format singleton.
* @group Input formats
* @public
*/
export const MP3 = new Mp3InputFormat();
export const MP3 = /* #__PURE__ */ new Mp3InputFormat();
/**
* WAVE input format singleton.
* @group Input formats
* @public
*/
export const WAVE = new WaveInputFormat();
export const WAVE = /* #__PURE__ */ new WaveInputFormat();
/**
* Ogg input format singleton.
* @group Input formats
* @public
*/
export const OGG = new OggInputFormat();
export const OGG = /* #__PURE__ */ new OggInputFormat();
/**
* ADTS input format singleton.
* @group Input formats
* @public
*/
export const ADTS = new AdtsInputFormat();
export const ADTS = /* #__PURE__ */ new AdtsInputFormat();
/**
* FLAC input format singleton.
* @group Input formats
* @public
*/
export const FLAC = new FlacInputFormat();
export const FLAC = /* #__PURE__ */ new FlacInputFormat();
/**
* List of all input format singletons. If you don't need to support all input formats, you should specify the
+7
View File
@@ -14,6 +14,7 @@ import { EncodedPacketSink, PacketRetrievalOptions } from './media-sink';
import { assert, Rotation } from './misc';
import { TrackType } from './output';
import { EncodedPacket, PacketType } from './packet';
import { TrackDisposition } from './metadata';
/**
* Contains aggregate statistics about the encoded packets of a track.
@@ -36,6 +37,7 @@ export interface InputTrackBacking {
getName(): string | null;
getLanguageCode(): string;
getTimeResolution(): number;
getDisposition(): TrackDisposition;
getFirstTimestamp(): Promise<number>;
computeDuration(): Promise<number>;
@@ -128,6 +130,11 @@ export abstract class InputTrack {
return this._backing.getTimeResolution();
}
/** The track's disposition, i.e. information about its intended usage. */
get disposition() {
return this._backing.getDisposition();
}
/**
* Returns the start timestamp of the first packet of this track, in seconds. While often near zero, this value
* may be positive or even negative. A negative starting timestamp means the track's timing has been offset. Samples
+20 -13
View File
@@ -44,7 +44,7 @@ import {
Sample,
} from './isobmff-muxer';
import { parseOpusIdentificationHeader } from '../codec-data';
import { MetadataTags, RichImageData } from '../tags';
import { MetadataTags, RichImageData } from '../metadata';
export class IsobmffBoxWriter {
private helper = new Uint8Array(8);
@@ -135,8 +135,8 @@ export class IsobmffBoxWriter {
}
}
const bytes = new Uint8Array(8);
const view = new DataView(bytes.buffer);
const bytes = /* #__PURE__ */ new Uint8Array(8);
const view = /* #__PURE__ */ new DataView(bytes.buffer);
const u8 = (value: number) => {
return [(value % 0x100 + 0x100) % 0x100];
@@ -243,7 +243,7 @@ const rotationMatrix = (rotationInDegrees: number): TransformationMatrix => {
0, 0, 1,
];
};
const IDENTITY_MATRIX = rotationMatrix(0);
const IDENTITY_MATRIX = /* #__PURE__ */ rotationMatrix(0);
const matrixToBytes = (matrix: TransformationMatrix) => {
return [
@@ -421,7 +421,12 @@ export const tkhd = (
matrix = IDENTITY_MATRIX;
}
return fullBox('tkhd', +needsU64, 3, [
let flags = 0x2; // Track in movie
if (trackData.track.metadata.disposition?.default !== false) {
flags |= 0x1; // Track enabled
}
return fullBox('tkhd', +needsU64, flags, [
u32OrU64(creationTime), // Creation time
u32OrU64(creationTime), // Modification time
u32(trackData.track.id), // Track ID
@@ -577,7 +582,7 @@ export const stsd = (trackData: IsobmffTrackData) => {
if (trackData.type === 'video') {
sampleDescription = videoSampleDescription(
VIDEO_CODEC_TO_BOX_NAME[trackData.track.source._codec],
videoCodecToBoxName(trackData.track.source._codec, trackData.info.decoderConfig.codec),
trackData,
);
} else if (trackData.type === 'audio') {
@@ -845,7 +850,7 @@ export const dOps = (trackData: IsobmffAudioTrackData) => {
let inputSampleRate = trackData.info.sampleRate;
let outputGain = 0;
let channelMappingFamily = 0;
let channelMappingTable = new Uint8Array(0);
let channelMappingTable: Uint8Array<ArrayBufferLike> = new Uint8Array(0);
// Read preskip and from codec private data from the encoder
// https://www.rfc-editor.org/rfc/rfc7845#section-5
@@ -1572,12 +1577,14 @@ const dataStringBoxLong = (value: string) => {
]);
};
const VIDEO_CODEC_TO_BOX_NAME: Record<VideoCodec, string> = {
avc: 'avc1',
hevc: 'hvc1',
vp8: 'vp08',
vp9: 'vp09',
av1: 'av01',
const videoCodecToBoxName = (codec: VideoCodec, fullCodecString: string) => {
switch (codec) {
case 'avc': return fullCodecString.startsWith('avc3') ? 'avc3' : 'avc1';
case 'hevc': return 'hvc1';
case 'vp8': return 'vp08';
case 'vp9': return 'vp09';
case 'av1': return 'av01';
}
};
const VIDEO_CODEC_TO_CONFIGURATION_BOX: Record<VideoCodec, (trackData: IsobmffVideoTrackData) => Box | null> = {
+22 -8
View File
@@ -51,13 +51,13 @@ import {
MATRIX_COEFFICIENTS_MAP_INVERSE,
normalizeRotation,
roundToMultiple,
roundToPrecision,
Rotation,
textDecoder,
TransformationMatrix,
TRANSFER_CHARACTERISTICS_MAP_INVERSE,
UNDETERMINED_LANGUAGE,
toDataView,
roundIfAlmostInteger,
} from '../misc';
import { EncodedPacket, PLACEHOLDER_DATA } from '../packet';
import { buildIsobmffMimeType } from './isobmff-misc';
@@ -86,12 +86,13 @@ import {
readU8,
readAscii,
} from '../reader';
import { MetadataTags, RichImageData } from '../tags';
import { DEFAULT_TRACK_DISPOSITION, MetadataTags, RichImageData, TrackDisposition } from '../metadata';
type InternalTrack = {
id: number;
demuxer: IsobmffDemuxer;
inputTrack: InputTrack | null;
disposition: TrackDisposition;
timescale: number;
durationInMovieTimescale: number;
durationInMediaTimescale: number;
@@ -126,6 +127,7 @@ type InternalTrack = {
codec: VideoCodec | null;
codecDescription: Uint8Array | null;
colorSpace: VideoColorSpaceInit | null;
avcType: 1 | 3 | null;
avcCodecInfo: AvcDecoderConfigurationRecord | null;
hevcCodecInfo: HevcDecoderConfigurationRecord | null;
vp9CodecInfo: Vp9CodecInfo | null;
@@ -314,6 +316,9 @@ export class IsobmffDemuxer extends Demuxer {
this.moovSlice = moovSlice;
this.readContiguousBoxes(this.moovSlice);
// Put default tracks first
this.tracks.sort((a, b) => Number(b.disposition.default) - Number(a.disposition.default));
for (const track of this.tracks) {
// Modify the edit list offset based on the previous segment durations. They are in different
// timescales, so we first convert to seconds and then into the track timescale.
@@ -650,6 +655,9 @@ export class IsobmffDemuxer extends Demuxer {
id: -1,
demuxer: this,
inputTrack: null,
disposition: {
...DEFAULT_TRACK_DISPOSITION,
},
info: null,
timescale: -1,
durationInMovieTimescale: -1,
@@ -694,10 +702,10 @@ export class IsobmffDemuxer extends Demuxer {
const version = readU8(slice);
const flags = readU24Be(slice);
const trackEnabled = (flags & 0x1) !== 0;
if (!trackEnabled) {
break;
}
// Spec says disabled tracks are to be treated like they don't exist, but in practice, they are treated
// more like non-default tracks.
const trackEnabled = !!(flags & 0x1);
track.disposition.default = trackEnabled;
// Skip over creation & modification time to reach the track ID
if (version === 0) {
@@ -836,6 +844,7 @@ export class IsobmffDemuxer extends Demuxer {
codec: null,
codecDescription: null,
colorSpace: null,
avcType: null,
avcCodecInfo: null,
hevcCodecInfo: null,
vp9CodecInfo: null,
@@ -890,8 +899,9 @@ export class IsobmffDemuxer extends Demuxer {
const lowercaseBoxName = sampleBoxInfo.name.toLowerCase();
if (track.info.type === 'video') {
if (lowercaseBoxName === 'avc1') {
if (lowercaseBoxName === 'avc1' || lowercaseBoxName === 'avc3') {
track.info.codec = 'avc';
track.info.avcType = lowercaseBoxName === 'avc1' ? 1 : 3;
} else if (lowercaseBoxName === 'hvc1' || lowercaseBoxName === 'hev1') {
track.info.codec = 'hevc';
} else if (lowercaseBoxName === 'vp08') {
@@ -2342,6 +2352,10 @@ abstract class IsobmffTrackBacking implements InputTrackBacking {
return this.internalTrack.timescale;
}
getDisposition() {
return this.internalTrack.disposition;
}
async computeDuration() {
const lastPacket = await this.getPacket(Infinity, { metadataOnly: true });
return (lastPacket?.timestamp ?? 0) + (lastPacket?.duration ?? 0);
@@ -2385,7 +2399,7 @@ abstract class IsobmffTrackBacking implements InputTrackBacking {
// Do a little rounding to catch cases where the result is very close to an integer. If it is, it's likely
// that the number was originally an integer divided by the timescale. For stability, it's best
// to return the integer in this case.
return roundToPrecision(timestamp * this.internalTrack.timescale, 14) + this.internalTrack.editListOffset;
return roundIfAlmostInteger(timestamp * this.internalTrack.timescale) + this.internalTrack.editListOffset;
}
async getPacket(timestamp: number, options: PacketRetrievalOptions) {
+1 -1
View File
@@ -6,7 +6,7 @@
* file, You can obtain one at https://mozilla.org/MPL/2.0/.
*/
import { RichImageData } from '../tags';
import { RichImageData } from '../metadata';
import { textDecoder } from '../misc';
import { FileSlice, readAscii, readBytes, readI32Be, readU16Be, readU32Be, readU64Be, readU8 } from '../reader';
+4
View File
@@ -88,6 +88,10 @@ export enum EBMLId {
FlagEnabled = 0xb9,
FlagDefault = 0x88,
FlagForced = 0x55aa,
FlagOriginal = 0x55ae,
FlagHearingImpaired = 0x55ab,
FlagVisualImpaired = 0x55ac,
FlagCommentary = 0x55af,
FlagLacing = 0x9c,
Name = 0x536e,
Language = 0x22b59c,
+56 -71
View File
@@ -31,7 +31,7 @@ import {
InputVideoTrack,
InputVideoTrackBacking,
} from '../input-track';
import { AttachedFile, MetadataTags } from '../tags';
import { AttachedFile, DEFAULT_TRACK_DISPOSITION, MetadataTags, TrackDisposition } from '../metadata';
import { PacketRetrievalOptions } from '../media-sink';
import {
assert,
@@ -43,7 +43,7 @@ import {
MATRIX_COEFFICIENTS_MAP_INVERSE,
normalizeRotation,
Rotation,
roundToPrecision,
roundIfAlmostInteger,
TRANSFER_CHARACTERISTICS_MAP_INVERSE,
UNDETERMINED_LANGUAGE,
} from '../misc';
@@ -61,7 +61,6 @@ import {
readElementHeader,
readElementId,
readFloat,
readSignedInt,
readUnsignedInt,
readVarInt,
resync,
@@ -137,7 +136,6 @@ type ClusterBlock = {
timestamp: number;
duration: number;
isKeyFrame: boolean;
referencedTimestamps: number[];
data: Uint8Array;
lacing: BlockLacing;
decoded: boolean;
@@ -190,7 +188,7 @@ type InternalTrack = {
}[];
cuePoints: CuePoint[];
isDefault: boolean;
disposition: TrackDisposition;
inputTrack: InputTrack | null;
codecId: string | null;
codecPrivate: Uint8Array | null;
@@ -539,7 +537,7 @@ export class MatroskaDemuxer extends Demuxer {
}
// Put default tracks first
this.currentSegment.tracks.sort((a, b) => Number(b.isDefault) - Number(a.isDefault));
this.currentSegment.tracks.sort((a, b) => Number(b.disposition.default) - Number(a.disposition.default));
// Now, let's distribute the cue points to the tracks
const idToTrack = new Map(this.currentSegment.tracks.map(x => [x.id, x]));
@@ -649,21 +647,15 @@ export class MatroskaDemuxer extends Demuxer {
// This must hold, as track datas only get created if a block for that track is encountered
assert(trackData.blocks.length > 0);
let blockReferencesExist = false;
let hasLacedBlocks = false;
for (let i = 0; i < trackData.blocks.length; i++) {
const block = trackData.blocks[i]!;
block.timestamp += cluster.timestamp;
blockReferencesExist ||= block.referencedTimestamps.length > 0;
hasLacedBlocks ||= block.lacing !== BlockLacing.None;
}
if (blockReferencesExist) {
trackData.blocks = sortBlocksByReferences(trackData.blocks);
}
trackData.presentationTimestamps = trackData.blocks
.map((block, i) => ({ timestamp: block.timestamp, blockIndex: i }))
.sort((a, b) => a.timestamp - b.timestamp);
@@ -859,7 +851,6 @@ export class MatroskaDemuxer extends Demuxer {
timestamp: frameTimestamp,
duration: frameDuration,
isKeyFrame: originalBlock.isKeyFrame,
referencedTimestamps: originalBlock.referencedTimestamps,
data: frameData,
lacing: BlockLacing.None,
decoded: true,
@@ -998,7 +989,9 @@ export class MatroskaDemuxer extends Demuxer {
clusterPositionCache: [],
cuePoints: [],
isDefault: false,
disposition: {
...DEFAULT_TRACK_DISPOSITION,
},
inputTrack: null,
codecId: null,
codecPrivate: null,
@@ -1163,7 +1156,37 @@ export class MatroskaDemuxer extends Demuxer {
case EBMLId.FlagDefault: {
if (!this.currentTrack) break;
this.currentTrack.isDefault = !!readUnsignedInt(slice, size);
this.currentTrack.disposition.default = !!readUnsignedInt(slice, size);
}; break;
case EBMLId.FlagForced: {
if (!this.currentTrack) break;
this.currentTrack.disposition.forced = !!readUnsignedInt(slice, size);
}; break;
case EBMLId.FlagOriginal: {
if (!this.currentTrack) break;
this.currentTrack.disposition.original = !!readUnsignedInt(slice, size);
}; break;
case EBMLId.FlagHearingImpaired: {
if (!this.currentTrack) break;
this.currentTrack.disposition.hearingImpaired = !!readUnsignedInt(slice, size);
}; break;
case EBMLId.FlagVisualImpaired: {
if (!this.currentTrack) break;
this.currentTrack.disposition.visuallyImpaired = !!readUnsignedInt(slice, size);
}; break;
case EBMLId.FlagCommentary: {
if (!this.currentTrack) break;
this.currentTrack.disposition.commentary = !!readUnsignedInt(slice, size);
}; break;
case EBMLId.CodecID: {
@@ -1383,9 +1406,17 @@ export class MatroskaDemuxer extends Demuxer {
const relativeTimestamp = readI16Be(slice);
const flags = readU8(slice);
const isKeyFrame = !!(flags & 0x80);
const lacing = (flags >> 1) & 0x3 as BlockLacing; // If the block is laced, we'll expand it later
let isKeyFrame = !!(flags & 0x80);
if (trackData.track.info?.type === 'audio' && trackData.track.info.codec) {
// Some files don't mark their audio packets as key packets (I'm looking at you, Firefox). But, we
// can fix this in most cases: if we recognize the codec of the track, then we know every packet is
// necessarily a key packet, no matter what the container says.
// https://github.com/Vanilagy/mediabunny/issues/192
isKeyFrame = true;
}
const blockData = readBytes(slice, size - (slice.filePos - dataStartPos));
const hasDecodingInstructions = trackData.track.decodingInstructions.length > 0;
@@ -1393,7 +1424,6 @@ export class MatroskaDemuxer extends Demuxer {
timestamp: relativeTimestamp, // We'll add the cluster's timestamp to this later
duration: 0, // Will set later
isKeyFrame,
referencedTimestamps: [],
data: blockData,
lacing,
decoded: !hasDecodingInstructions,
@@ -1406,13 +1436,7 @@ export class MatroskaDemuxer extends Demuxer {
this.readContiguousElements(slice.slice(dataStartPos, size));
if (this.currentBlock) {
for (let i = 0; i < this.currentBlock.referencedTimestamps.length; i++) {
this.currentBlock.referencedTimestamps[i]! += this.currentBlock.timestamp;
}
this.currentBlock = null;
}
this.currentBlock = null;
}; break;
case EBMLId.Block: {
@@ -1436,7 +1460,6 @@ export class MatroskaDemuxer extends Demuxer {
timestamp: relativeTimestamp, // We'll add the cluster's timestamp to this later
duration: 0, // Will set later
isKeyFrame: true,
referencedTimestamps: [],
data: blockData,
lacing,
decoded: !hasDecodingInstructions,
@@ -1487,11 +1510,8 @@ export class MatroskaDemuxer extends Demuxer {
if (!this.currentBlock) break;
this.currentBlock.isKeyFrame = false;
const relativeTimestamp = readSignedInt(slice, size);
// We'll offset this by the block's timestamp later
this.currentBlock.referencedTimestamps.push(relativeTimestamp);
// We ignore the actual value here, we just use the reference as an indicator for "not a key frame".
// This is in line with FFmpeg's behavior.
}; break;
case EBMLId.Tag: {
@@ -1859,6 +1879,10 @@ abstract class MatroskaTrackBacking implements InputTrackBacking {
return this.internalTrack.segment.timestampFactor;
}
getDisposition() {
return this.internalTrack.disposition;
}
async getFirstPacket(options: PacketRetrievalOptions) {
return this.performClusterLookup(
null,
@@ -1886,7 +1910,7 @@ abstract class MatroskaTrackBacking implements InputTrackBacking {
// Do a little rounding to catch cases where the result is very close to an integer. If it is, it's likely
// that the number was originally an integer divided by the timescale. For stability, it's best
// to return the integer in this case.
return roundToPrecision(timestamp * this.internalTrack.segment.timestampFactor, 14);
return roundIfAlmostInteger(timestamp * this.internalTrack.segment.timestampFactor);
}
async getPacket(timestamp: number, options: PacketRetrievalOptions) {
@@ -2319,6 +2343,7 @@ class MatroskaVideoTrackBacking extends MatroskaTrackBacking implements InputVid
codec: this.internalTrack.info.codec,
codecDescription: this.internalTrack.info.codecDescription,
colorSpace: this.internalTrack.info.colorSpace,
avcType: 1, // We don't know better (or do we?) so just assume 'avc1'
avcCodecInfo: this.internalTrack.info.codec === 'avc' && firstPacket
? extractAvcDecoderConfigurationRecord(firstPacket.data)
: null,
@@ -2379,43 +2404,3 @@ class MatroskaAudioTrackBacking extends MatroskaTrackBacking implements InputAud
};
}
}
/** Sorts blocks such that referenced blocks come before the blocks that reference them. */
const sortBlocksByReferences = (blocks: ClusterBlock[]) => {
const timestampToBlock = new Map<number, ClusterBlock>();
for (let i = 0; i < blocks.length; i++) {
const block = blocks[i]!;
timestampToBlock.set(block.timestamp, block);
}
const processedBlocks = new Set<ClusterBlock>();
const result: ClusterBlock[] = [];
const processBlock = (block: ClusterBlock) => {
if (processedBlocks.has(block)) {
return;
}
// Marking the block as processed here already; prevents this algorithm from dying on cycles
processedBlocks.add(block);
for (let j = 0; j < block.referencedTimestamps.length; j++) {
const timestamp = block.referencedTimestamps[j]!;
const otherBlock = timestampToBlock.get(timestamp);
if (!otherBlock) {
continue;
}
processBlock(otherBlock);
}
result.push(block);
};
for (let i = 0; i < blocks.length; i++) {
processBlock(blocks[i]!);
}
return result;
};
+19 -1
View File
@@ -63,7 +63,7 @@ import { Muxer } from '../muxer';
import { Writer } from '../writer';
import { EncodedPacket } from '../packet';
import { parseOpusIdentificationHeader } from '../codec-data';
import { AttachedFile } from '../tags';
import { AttachedFile } from '../metadata';
const MIN_CLUSTER_TIMESTAMP_MS = -(2 ** 15);
const MAX_CLUSTER_TIMESTAMP_MS = 2 ** 15 - 1;
@@ -304,6 +304,24 @@ export class MatroskaMuxer extends Muxer {
{ id: EBMLId.TrackNumber, data: trackData.track.id },
{ id: EBMLId.TrackUID, data: trackData.track.id },
{ id: EBMLId.TrackType, data: TRACK_TYPE_MAP[trackData.type] },
trackData.track.metadata.disposition?.default === false
? { id: EBMLId.FlagDefault, data: 0 }
: null,
trackData.track.metadata.disposition?.forced
? { id: EBMLId.FlagForced, data: 1 }
: null,
trackData.track.metadata.disposition?.hearingImpaired
? { id: EBMLId.FlagHearingImpaired, data: 1 }
: null,
trackData.track.metadata.disposition?.visuallyImpaired
? { id: EBMLId.FlagVisualImpaired, data: 1 }
: null,
trackData.track.metadata.disposition?.original
? { id: EBMLId.FlagOriginal, data: 1 }
: null,
trackData.track.metadata.disposition?.commentary
? { id: EBMLId.FlagCommentary, data: 1 }
: null,
{ id: EBMLId.FlagLacing, data: 0 },
{ id: EBMLId.Language, data: trackData.track.metadata.languageCode ?? UNDETERMINED_LANGUAGE },
{ id: EBMLId.CodecID, data: codecId },
+21 -1
View File
@@ -8,10 +8,12 @@
import { parsePcmCodec, PCM_AUDIO_CODECS, PcmAudioCodec, VideoCodec, AudioCodec } from './codec';
import {
deserializeAvcDecoderConfigurationRecord,
determineVideoPacketType,
extractHevcNalUnits,
extractNalUnitTypeForHevc,
HevcNalUnitType,
parseAvcSps,
} from './codec-data';
import { CustomVideoDecoder, customVideoDecoders, CustomAudioDecoder, customAudioDecoders } from './custom-coder';
import { InputDisposedError } from './input';
@@ -24,6 +26,7 @@ import {
getInt24,
getUint24,
insertSorted,
isChromium,
isFirefox,
isNumber,
isWebKit,
@@ -33,6 +36,7 @@ import {
Rotation,
toAsyncIterator,
toDataView,
toUint8Array,
validateAnyIterable,
} from './misc';
import { EncodedPacket } from './packet';
@@ -872,6 +876,22 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
}
};
if (codec === 'avc' && this.decoderConfig.description && isChromium()) {
// Chromium has/had a bug with playing interlaced AVC (https://issues.chromium.org/issues/456919096)
// which can be worked around by requesting that software decoding be used. So, here we peek into the
// AVC description, if present, and switch to software decoding if we find interlaced content.
const record = deserializeAvcDecoderConfigurationRecord(toUint8Array(this.decoderConfig.description));
if (record && record.sequenceParameterSets.length > 0) {
const sps = parseAvcSps(record.sequenceParameterSets[0]!);
if (sps && sps.frameMbsOnlyFlag === 0) {
this.decoderConfig = {
...this.decoderConfig,
hardwareAcceleration: 'prefer-software',
};
}
}
}
this.decoder = new VideoDecoder({
output: (frame) => {
try {
@@ -882,7 +902,7 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
},
error: onError,
});
this.decoder.configure(decoderConfig);
this.decoder.configure(this.decoderConfig);
}
}
+31
View File
@@ -1294,6 +1294,8 @@ class AudioEncoderWrapper {
private customEncoderCallSerializer = new CallSerializer();
private customEncoderQueueSize = 0;
private lastEndSampleIndex: number | null = null;
/**
* Encoders typically throw their errors "out of band", meaning asynchronously in some other execution context.
* However, we want to surface these errors to the user within the normal control flow, so they don't go uncaught.
@@ -1342,6 +1344,35 @@ class AudioEncoderWrapper {
}
assert(this.encoderInitialized);
// Handle padding of gaps with silence to avoid audio drift over time, like in
// https://github.com/Vanilagy/mediabunny/issues/176
// TODO An open question is how encoders deal with the first AudioData having a non-zero timestamp, and with
// AudioDatas that have an overlapping timestamp range.
{
const startSampleIndex = Math.round(
audioSample.timestamp * audioSample.sampleRate,
);
const endSampleIndex = Math.round(
(audioSample.timestamp + audioSample.duration) * audioSample.sampleRate,
);
if (this.lastEndSampleIndex !== null && startSampleIndex > this.lastEndSampleIndex) {
const sampleCount = startSampleIndex - this.lastEndSampleIndex;
const fillSample = new AudioSample({
data: new Float32Array(sampleCount * audioSample.numberOfChannels),
format: 'f32-planar',
sampleRate: audioSample.sampleRate,
numberOfChannels: audioSample.numberOfChannels,
numberOfFrames: sampleCount,
timestamp: this.lastEndSampleIndex / audioSample.sampleRate,
});
await this.add(fillSample, true); // Recursive call
}
this.lastEndSampleIndex = endSampleIndex;
}
if (this.customEncoder) {
this.customEncoderQueueSize++;
+59
View File
@@ -259,3 +259,62 @@ export const metadataTagsAreEmpty = (tags: MetadataTags) => {
&& tags.comment === undefined
&& (tags.raw === undefined || Object.keys(tags.raw).length === 0);
};
/**
* Specifies a track's disposition, i.e. information about its intended usage.
* @public
* @group Miscellaneous
*/
export type TrackDisposition = {
/**
* Indicates that this track is eligible for automatic selection by a player; that it is the main track among other,
* non-default tracks of the same type.
*/
default: boolean;
/**
* Indicates that players should always display this track by default, even if it goes against the user's default
* preferences. For example, a subtitle track only containing translations of foreign-language audio.
*/
forced: boolean;
/** Indicates that this track is in the content's original language. */
original: boolean;
/** Indicates that this track contains commentary. */
commentary: boolean;
/** Indicates that this track is intended for hearing-impaired users. */
hearingImpaired: boolean;
/** Indicates that this track is intended for visually-impaired users. */
visuallyImpaired: boolean;
};
export const DEFAULT_TRACK_DISPOSITION: TrackDisposition = {
default: true,
forced: false,
original: false,
commentary: false,
hearingImpaired: false,
visuallyImpaired: false,
};
export const validateTrackDisposition = (disposition: Partial<TrackDisposition>) => {
if (!disposition || typeof disposition !== 'object') {
throw new TypeError('disposition must be an object.');
}
if (disposition.default !== undefined && typeof disposition.default !== 'boolean') {
throw new TypeError('disposition.default must be a boolean.');
}
if (disposition.forced !== undefined && typeof disposition.forced !== 'boolean') {
throw new TypeError('disposition.forced must be a boolean.');
}
if (disposition.original !== undefined && typeof disposition.original !== 'boolean') {
throw new TypeError('disposition.original must be a boolean.');
}
if (disposition.commentary !== undefined && typeof disposition.commentary !== 'boolean') {
throw new TypeError('disposition.commentary must be a boolean.');
}
if (disposition.hearingImpaired !== undefined && typeof disposition.hearingImpaired !== 'boolean') {
throw new TypeError('disposition.hearingImpaired must be a boolean.');
}
if (disposition.visuallyImpaired !== undefined && typeof disposition.visuallyImpaired !== 'boolean') {
throw new TypeError('disposition.visuallyImpaired must be a boolean.');
}
};
+39 -19
View File
@@ -157,25 +157,25 @@ export const writeBits = (bytes: Uint8Array, start: number, end: number, value:
export const toUint8Array = (source: AllowSharedBufferSource): Uint8Array => {
if (source.constructor === Uint8Array) { // We want a true Uint8Array, not something that extends it like Buffer
return source;
} else if (source instanceof ArrayBuffer) {
return new Uint8Array(source);
} else {
} else if (ArrayBuffer.isView(source)) {
return new Uint8Array(source.buffer, source.byteOffset, source.byteLength);
} else {
return new Uint8Array(source);
}
};
export const toDataView = (source: AllowSharedBufferSource) => {
export const toDataView = (source: AllowSharedBufferSource): DataView => {
if (source.constructor === DataView) {
return source;
} else if (source instanceof ArrayBuffer) {
return new DataView(source);
} else {
} else if (ArrayBuffer.isView(source)) {
return new DataView(source.buffer, source.byteOffset, source.byteLength);
} else {
return new DataView(source);
}
};
export const textDecoder = new TextDecoder();
export const textEncoder = new TextEncoder();
export const textDecoder = /* #__PURE__ */ new TextDecoder();
export const textEncoder = /* #__PURE__ */ new TextEncoder();
export const isIso88591Compatible = (text: string) => {
for (let i = 0; i < text.length; i++) {
@@ -201,7 +201,7 @@ export const COLOR_PRIMARIES_MAP = {
bt2020: 9, // ITU-R BT.202
smpte432: 12, // SMPTE EG 432-1
};
export const COLOR_PRIMARIES_MAP_INVERSE = invertObject(COLOR_PRIMARIES_MAP);
export const COLOR_PRIMARIES_MAP_INVERSE = /* #__PURE__ */ invertObject(COLOR_PRIMARIES_MAP);
export const TRANSFER_CHARACTERISTICS_MAP = {
'bt709': 1, // ITU-R BT.709
@@ -211,7 +211,7 @@ export const TRANSFER_CHARACTERISTICS_MAP = {
'pq': 16, // Rec. ITU-R BT.2100-2 perceptual quantization (PQ) system
'hlg': 18, // Rec. ITU-R BT.2100-2 hybrid loggamma (HLG) system
};
export const TRANSFER_CHARACTERISTICS_MAP_INVERSE = invertObject(TRANSFER_CHARACTERISTICS_MAP);
export const TRANSFER_CHARACTERISTICS_MAP_INVERSE = /* #__PURE__ */ invertObject(TRANSFER_CHARACTERISTICS_MAP);
export const MATRIX_COEFFICIENTS_MAP = {
'rgb': 0, // Identity
@@ -220,7 +220,7 @@ export const MATRIX_COEFFICIENTS_MAP = {
'smpte170m': 6, // SMPTE 170M
'bt2020-ncl': 9, // ITU-R BT.2020-2 (non-constant luminance)
};
export const MATRIX_COEFFICIENTS_MAP_INVERSE = invertObject(MATRIX_COEFFICIENTS_MAP);
export const MATRIX_COEFFICIENTS_MAP_INVERSE = /* #__PURE__ */ invertObject(MATRIX_COEFFICIENTS_MAP);
export type RequiredNonNull<T> = {
[K in keyof T]-?: NonNullable<T[K]>;
@@ -486,9 +486,14 @@ export const clamp = (value: number, min: number, max: number) => {
export const UNDETERMINED_LANGUAGE = 'und';
export const roundToPrecision = (value: number, digits: number) => {
const factor = 10 ** digits;
return Math.round(value * factor) / factor;
export const roundIfAlmostInteger = (value: number) => {
const rounded = Math.round(value);
if (Math.abs(value / rounded - 1) < 10 * Number.EPSILON) {
return rounded;
} else {
return value;
}
};
export const roundToMultiple = (value: number, multiple: number) => {
@@ -579,6 +584,7 @@ export const retriedFetch = async (
url: string | URL | Request,
requestInit: RequestInit,
getRetryDelay: (previousAttempts: number, error: unknown, url: string | URL | Request) => number | null,
shouldStop: () => boolean,
) => {
let attempts = 0;
@@ -586,6 +592,10 @@ export const retriedFetch = async (
try {
return await fetchFn(url, requestInit);
} catch (error) {
if (shouldStop()) {
throw error;
}
attempts++;
const retryDelayInSeconds = getRetryDelay(attempts, error, url);
@@ -602,6 +612,10 @@ export const retriedFetch = async (
if (retryDelayInSeconds > 0) {
await new Promise(resolve => setTimeout(resolve, 1000 * retryDelayInSeconds));
}
if (shouldStop()) {
throw error;
}
}
}
};
@@ -665,10 +679,7 @@ export const isWebKit = () => {
}
// This even returns true for WebKit-wrapping browsers such as Chrome on iOS
const result = !!(typeof navigator !== 'undefined' && navigator.vendor?.match(/apple/i));
isWebKitCache = result;
return result;
return isWebKitCache = !!(typeof navigator !== 'undefined' && navigator.vendor?.match(/apple/i));
};
let isFirefoxCache: boolean | null = null;
@@ -680,6 +691,15 @@ export const isFirefox = () => {
return isFirefoxCache = typeof navigator !== 'undefined' && navigator.userAgent?.includes('Firefox');
};
let isChromiumCache: boolean | null = null;
export const isChromium = () => {
if (isChromiumCache !== null) {
return isChromiumCache;
}
return isChromiumCache = !!(typeof navigator !== 'undefined' && navigator.vendor?.includes('Google Inc'));
};
/**
* T or a promise that resolves to T.
* @group Miscellaneous
+7 -1
View File
@@ -10,7 +10,7 @@ import { AudioCodec } from '../codec';
import { Demuxer } from '../demuxer';
import { Input } from '../input';
import { InputAudioTrack, InputAudioTrackBacking } from '../input-track';
import { MetadataTags } from '../tags';
import { DEFAULT_TRACK_DISPOSITION, MetadataTags } from '../metadata';
import { PacketRetrievalOptions } from '../media-sink';
import { assert, AsyncMutex, binarySearchExact, binarySearchLessOrEqual, UNDETERMINED_LANGUAGE } from '../misc';
import { EncodedPacket, PLACEHOLDER_DATA } from '../packet';
@@ -257,6 +257,12 @@ class Mp3AudioTrackBacking implements InputAudioTrackBacking {
return this.demuxer.firstFrameHeader.sampleRate;
}
getDisposition() {
return {
...DEFAULT_TRACK_DISPOSITION,
};
}
async getDecoderConfig(): Promise<AudioDecoderConfig> {
assert(this.demuxer.firstFrameHeader);
+1 -1
View File
@@ -7,7 +7,7 @@
*/
import { assert, toDataView } from '../misc';
import { metadataTagsAreEmpty } from '../tags';
import { metadataTagsAreEmpty } from '../metadata';
import { Muxer } from '../muxer';
import { Output, OutputAudioTrack } from '../output';
import { Mp3OutputFormat } from '../output-format';
+11 -11
View File
@@ -46,21 +46,21 @@ export abstract class Muxer {
private trackTimestampInfo = new WeakMap<OutputTrack, {
maxTimestamp: number;
maxTimestampBeforeLastKeyFrame: number;
maxTimestampBeforeLastKeyPacket: number;
}>();
protected validateAndNormalizeTimestamp(track: OutputTrack, timestampInSeconds: number, isKeyFrame: boolean) {
protected validateAndNormalizeTimestamp(track: OutputTrack, timestampInSeconds: number, isKeyPacket: boolean) {
timestampInSeconds += track.source._timestampOffset;
let timestampInfo = this.trackTimestampInfo.get(track);
if (!timestampInfo) {
if (!isKeyFrame) {
throw new Error('First frame must be a key frame.');
if (!isKeyPacket) {
throw new Error('First packet must be a key packet.');
}
timestampInfo = {
maxTimestamp: timestampInSeconds,
maxTimestampBeforeLastKeyFrame: timestampInSeconds,
maxTimestampBeforeLastKeyPacket: timestampInSeconds,
};
this.trackTimestampInfo.set(track, timestampInfo);
}
@@ -69,15 +69,15 @@ export abstract class Muxer {
throw new Error(`Timestamps must be non-negative (got ${timestampInSeconds}s).`);
}
if (isKeyFrame) {
timestampInfo.maxTimestampBeforeLastKeyFrame = timestampInfo.maxTimestamp;
if (isKeyPacket) {
timestampInfo.maxTimestampBeforeLastKeyPacket = timestampInfo.maxTimestamp;
}
if (timestampInSeconds < timestampInfo.maxTimestampBeforeLastKeyFrame) {
if (timestampInSeconds < timestampInfo.maxTimestampBeforeLastKeyPacket) {
throw new Error(
`Timestamps cannot be smaller than the highest timestamp of the previous GOP (a GOP begins with a key`
+ ` frame and ends right before the next key frame). Got ${timestampInSeconds}s, but highest timestamp`
+ ` is ${timestampInfo.maxTimestampBeforeLastKeyFrame}s.`,
`Timestamps cannot be smaller than the largest timestamp of the previous GOP (a GOP begins with a key`
+ ` packet and ends right before the next key packet). Got ${timestampInSeconds}s, but largest`
+ ` timestamp is ${timestampInfo.maxTimestampBeforeLastKeyPacket}s.`,
);
}
+10 -4
View File
@@ -12,14 +12,14 @@ import { Demuxer } from '../demuxer';
import { Input } from '../input';
import { InputAudioTrack, InputAudioTrackBacking } from '../input-track';
import { PacketRetrievalOptions } from '../media-sink';
import { MetadataTags } from '../tags';
import { DEFAULT_TRACK_DISPOSITION, MetadataTags } from '../metadata';
import {
assert,
AsyncMutex,
binarySearchLessOrEqual,
findLast,
last,
roundToPrecision,
roundIfAlmostInteger,
toDataView,
UNDETERMINED_LANGUAGE,
} from '../misc';
@@ -463,6 +463,12 @@ class OggAudioTrackBacking implements InputAudioTrackBacking {
return UNDETERMINED_LANGUAGE;
}
getDisposition() {
return {
...DEFAULT_TRACK_DISPOSITION,
};
}
async getFirstTimestamp() {
return 0;
}
@@ -577,7 +583,7 @@ class OggAudioTrackBacking implements InputAudioTrackBacking {
return this.getPacketSequential(timestamp, options);
}
const timestampInSamples = roundToPrecision(timestamp * this.internalSampleRate, 14);
const timestampInSamples = roundIfAlmostInteger(timestamp * this.internalSampleRate);
if (timestampInSamples === 0) {
// Fast path for timestamp 0 - avoids binary search when playing back from the start
return this.getFirstPacket(options);
@@ -910,7 +916,7 @@ class OggAudioTrackBacking implements InputAudioTrackBacking {
const release = await this.sequentialScanMutex.acquire(); // Requires exclusivity because we write to a cache
try {
const timestampInSamples = roundToPrecision(timestamp * this.internalSampleRate, 14);
const timestampInSamples = roundIfAlmostInteger(timestamp * this.internalSampleRate);
timestamp = timestampInSamples / this.internalSampleRate;
const index = binarySearchLessOrEqual(
+6 -1
View File
@@ -7,7 +7,7 @@
*/
import { AsyncMutex, isIso639Dash2LanguageCode, Rotation } from './misc';
import { MetadataTags, validateMetadataTags } from './tags';
import { MetadataTags, TrackDisposition, validateMetadataTags, validateTrackDisposition } from './metadata';
import { Muxer } from './muxer';
import { OutputFormat } from './output-format';
import { AudioSource, MediaSource, SubtitleSource, VideoSource } from './media-source';
@@ -74,6 +74,8 @@ export type BaseTrackMetadata = {
languageCode?: string;
/** A user-defined name for this track, like "English" or "Director Commentary". */
name?: string;
/** The track's disposition, i.e. information about its intended usage. */
disposition?: Partial<TrackDisposition>;
/**
* The maximum amount of encoded packets that will be added to this track. Setting this field provides the muxer
* with an additional signal that it can use to preallocate space in the file.
@@ -129,6 +131,9 @@ const validateBaseTrackMetadata = (metadata: BaseTrackMetadata) => {
if (metadata.name !== undefined && typeof metadata.name !== 'string') {
throw new TypeError('metadata.name, when provided, must be a string.');
}
if (metadata.disposition !== undefined) {
validateTrackDisposition(metadata.disposition);
}
if (
metadata.maximumPacketCount !== undefined
&& (!Number.isInteger(metadata.maximumPacketCount) || metadata.maximumPacketCount < 0)
+1 -1
View File
@@ -8,7 +8,7 @@
import { SECOND_TO_MICROSECOND_FACTOR } from './misc';
export const PLACEHOLDER_DATA = new Uint8Array(0);
export const PLACEHOLDER_DATA = /* #__PURE__ */ new Uint8Array(0);
/**
* The type of a packet. Key packets can be decoded without previous packets, while delta packets depend on previous
+69 -5
View File
@@ -21,6 +21,54 @@ import {
polyfillSymbolDispose();
type FinalizationRegistryValue = {
type: 'video';
data: VideoFrame | OffscreenCanvas | Uint8Array;
} | {
type: 'audio';
data: AudioData | Uint8Array;
};
// Let's manually handle logging the garbage collection errors that are typically logged by the browser. This way, they
// also kick for audio samples (which is normally not the case), making sure any incorrect code is quickly caught.
let lastVideoGcErrorLog = -Infinity;
let lastAudioGcErrorLog = -Infinity;
let finalizationRegistry: FinalizationRegistry<FinalizationRegistryValue> | null = null;
if (typeof FinalizationRegistry !== 'undefined') {
finalizationRegistry = new FinalizationRegistry<FinalizationRegistryValue>((value) => {
const now = Date.now();
if (value.type === 'video') {
if (now - lastVideoGcErrorLog >= 1000) {
// This error is annoying but oh so important
console.error(
`A VideoSample was garbage collected without first being closed. For proper resource management,`
+ ` make sure to call close() on all your VideoSamples as soon as you're done using them.`,
);
lastVideoGcErrorLog = now;
}
if (typeof VideoFrame !== 'undefined' && value.data instanceof VideoFrame) {
value.data.close(); // Prevent the browser error since we're logging our own
}
} else {
if (now - lastAudioGcErrorLog >= 1000) {
console.error(
`An AudioSample was garbage collected without first being closed. For proper resource management,`
+ ` make sure to call close() on all your AudioSamples as soon as you're done using them.`,
);
lastAudioGcErrorLog = now;
}
if (typeof AudioData !== 'undefined' && value.data instanceof AudioData) {
value.data.close();
}
}
});
}
/**
* Metadata used for VideoSample initialization.
* @group Samples
@@ -133,7 +181,11 @@ export class VideoSample implements Disposable {
data: VideoFrame | CanvasImageSource | AllowSharedBufferSource,
init?: VideoSampleInit,
) {
if (data instanceof ArrayBuffer || ArrayBuffer.isView(data)) {
if (
data instanceof ArrayBuffer
|| (typeof SharedArrayBuffer !== 'undefined' && data instanceof SharedArrayBuffer)
|| ArrayBuffer.isView(data)
) {
if (!init || typeof init !== 'object') {
throw new TypeError('init must be an object.');
}
@@ -265,6 +317,8 @@ export class VideoSample implements Disposable {
} else {
throw new TypeError('Invalid data type: Must be a BufferSource or CanvasImageSource.');
}
finalizationRegistry?.register(this, { type: 'video', data: this._data }, this);
}
/** Clones this video sample. */
@@ -313,6 +367,8 @@ export class VideoSample implements Disposable {
return;
}
finalizationRegistry?.unregister(this);
if (isVideoFrame(this._data)) {
this._data.close();
} else {
@@ -552,7 +608,6 @@ export class VideoSample implements Disposable {
dHeight,
);
// Restore the previous transformation state
context.restore();
}
@@ -644,6 +699,8 @@ export class VideoSample implements Disposable {
dy = (canvasHeight - newHeight) / 2;
}
context.save();
const aspectRatioChange = rotation % 180 === 0 ? 1 : newWidth / newHeight;
context.translate(canvasWidth / 2, canvasHeight / 2);
context.rotate(rotation * Math.PI / 180);
@@ -655,6 +712,8 @@ export class VideoSample implements Disposable {
// Important that we don't use .draw() here since that would take rotation into account, but we wanna handle it
// ourselves here
context.drawImage(this.toCanvasImageSource(), sx, sy, sWidth, sHeight, dx, dy, newWidth, newHeight);
context.restore();
}
/** @internal */
@@ -952,6 +1011,8 @@ export class AudioSample implements Disposable {
this._data = dataBuffer;
}
finalizationRegistry?.register(this, { type: 'audio', data: this._data }, this);
}
/** Returns the number of bytes required to hold the audio sample's data as specified by the given options. */
@@ -1113,9 +1174,8 @@ export class AudioSample implements Disposable {
}
}
} else {
// Branch for Uint8Array data (non-AudioData)
const uint8Data = this._data;
const srcView = new DataView(uint8Data.buffer, uint8Data.byteOffset, uint8Data.byteLength);
const srcView = toDataView(uint8Data);
const srcFormat = this.format;
const readFn = getReadFunction(srcFormat);
@@ -1185,6 +1245,8 @@ export class AudioSample implements Disposable {
return;
}
finalizationRegistry?.unregister(this);
if (isAudioData(this._data)) {
this._data.close();
} else {
@@ -1247,7 +1309,9 @@ export class AudioSample implements Disposable {
numberOfFrames: this.numberOfFrames,
numberOfChannels: this.numberOfChannels,
timestamp: this.microsecondTimestamp,
data: this._data,
data: this._data.buffer instanceof ArrayBuffer
? this._data.buffer
: this._data.slice(), // In the case of SharedArrayBuffer, convert to ArrayBuffer
});
}
}
+44 -40
View File
@@ -102,10 +102,17 @@ export class BufferSource extends Source {
/** @internal */
_onreadCalled = false;
/** Creates a new {@link BufferSource} backed the specified `ArrayBuffer` or `ArrayBufferView`. */
constructor(buffer: ArrayBuffer | ArrayBufferView) {
if (!(buffer instanceof ArrayBuffer) && !ArrayBuffer.isView(buffer)) {
throw new TypeError('buffer must be an ArrayBuffer or ArrayBufferView.');
/**
* Creates a new {@link BufferSource} backed by the specified `ArrayBuffer`, `SharedArrayBuffer`,
* or `ArrayBufferView`.
*/
constructor(buffer: AllowSharedBufferSource) {
if (
!(buffer instanceof ArrayBuffer)
&& !(typeof SharedArrayBuffer !== 'undefined' && buffer instanceof SharedArrayBuffer)
&& !ArrayBuffer.isView(buffer)
) {
throw new TypeError('buffer must be an ArrayBuffer, SharedArrayBuffer, or ArrayBufferView.');
}
super();
@@ -234,12 +241,7 @@ export class BlobSource extends Source {
const { done, value } = await reader.read();
if (done) {
this._orchestrator.forgetWorker(worker);
if (worker.currentPos < worker.targetPos) { // I think this `if` should always hit?
throw new Error('Blob reader stopped unexpectedly before all requested data was read.');
}
break;
throw new Error('Blob reader stopped unexpectedly before all requested data was read.');
}
if (worker.aborted) {
@@ -297,6 +299,10 @@ const DEFAULT_RETRY_DELAY
= typeof navigator !== 'undefined' && typeof navigator.onLine === 'boolean' ? navigator.onLine : true;
if (isOnline && originOfSrc !== null && originOfSrc !== window.location.origin) {
console.warn(
`Request will not be retried because a CORS error was suspected due to different origins. You can`
+ ` modify this behavior by providing your own function for the 'getRetryDelay' option.`,
);
return null;
}
}
@@ -425,6 +431,7 @@ export class UrlSource extends Source {
signal: abortController.signal,
}),
this._getRetryDelay,
() => this._disposed,
);
if (!response.ok) {
@@ -436,7 +443,7 @@ export class UrlSource extends Source {
let fileSize: number;
if (response.status === 206) {
fileSize = this._getPartialLengthFromRangeResponse(response);
fileSize = this._getTotalLengthFromRangeResponse(response);
worker = this._orchestrator.createWorker(0, Math.min(fileSize, URL_SOURCE_MIN_LOAD_AMOUNT));
} else {
// Server probably returned a 200.
@@ -492,6 +499,7 @@ export class UrlSource extends Source {
signal: abortController.signal,
}),
this._getRetryDelay,
() => this._disposed,
);
}
@@ -509,14 +517,6 @@ export class UrlSource extends Source {
);
}
const length = this._getPartialLengthFromRangeResponse(response);
const required = worker.targetPos - worker.currentPos;
if (length < required) {
throw new Error(
`HTTP response unexpectedly too short: Needed at least ${required} bytes, got only ${length}.`,
);
}
if (!response.body) {
throw new Error(
'Missing HTTP response body stream. The used fetch function must provide the response body as a'
@@ -539,6 +539,11 @@ export class UrlSource extends Source {
try {
readResult = await reader.read();
} catch (error) {
if (this._disposed) {
// No need to try to retry
throw error;
}
const retryDelayInSeconds = this._getRetryDelay(1, error, this._url);
if (retryDelayInSeconds !== null) {
console.error('Error while reading response stream. Attempting to resume.', error);
@@ -557,16 +562,17 @@ export class UrlSource extends Source {
const { done, value } = readResult;
if (done) {
this._orchestrator.forgetWorker(worker);
if (worker.currentPos < worker.targetPos) {
throw new Error(
'Response stream reader stopped unexpectedly before all requested data was read.',
);
if (worker.currentPos >= worker.targetPos) {
// All data was delivered, we're good
this._orchestrator.forgetWorker(worker);
worker.running = false;
return;
}
worker.running = false;
return;
// The response stopped early, before the target. This can happen if server decides to cap range
// requests arbitrarily, even if the request had an uncapped end. In this case, let's fetch the rest
// of the data using a new request.
break;
}
this.onread?.(worker.currentPos, worker.currentPos + value.length);
@@ -586,25 +592,23 @@ export class UrlSource extends Source {
}
/** @internal */
private _getPartialLengthFromRangeResponse(response: Response) {
private _getTotalLengthFromRangeResponse(response: Response) {
const contentRange = response.headers.get('Content-Range');
if (contentRange) {
const match = /\/(\d+)/.exec(contentRange);
if (match) {
return Number(match[1]);
} else {
throw new Error(`Invalid Content-Range header: ${contentRange}`);
}
}
const contentLength = response.headers.get('Content-Length');
if (contentLength) {
return Number(contentLength);
} else {
const contentLength = response.headers.get('Content-Length');
if (contentLength) {
return Number(contentLength);
} else {
throw new Error(
'Partial HTTP response (status 206) must surface either Content-Range or'
+ ' Content-Length header.',
);
}
throw new Error(
'Partial HTTP response (status 206) must surface either Content-Range or'
+ ' Content-Length header.',
);
}
}
@@ -639,7 +643,7 @@ export class FilePathSource extends Source {
_fileHandle: FileHandle | null = null;
/** Creates a new {@link FilePathSource} backed by the file at the specified file path. */
constructor(filePath: string, options: BlobSourceOptions = {}) {
constructor(filePath: string, options: FilePathSourceOptions = {}) {
if (typeof filePath !== 'string') {
throw new TypeError('filePath must be a string.');
}
+1 -1
View File
@@ -62,7 +62,7 @@ export type StreamTargetChunk = {
/** The operation type. */
type: 'write'; // This ensures automatic compatibility with FileSystemWritableFileStream
/** The data to write. */
data: Uint8Array;
data: Uint8Array<ArrayBuffer>;
/** The byte offset in the output file at which to write the data. */
position: number;
};
+7 -1
View File
@@ -11,7 +11,7 @@ import { Demuxer } from '../demuxer';
import { Input } from '../input';
import { InputAudioTrack, InputAudioTrackBacking } from '../input-track';
import { PacketRetrievalOptions } from '../media-sink';
import { MetadataTags } from '../tags';
import { DEFAULT_TRACK_DISPOSITION, MetadataTags } from '../metadata';
import { assert, UNDETERMINED_LANGUAGE } from '../misc';
import { EncodedPacket, PLACEHOLDER_DATA } from '../packet';
import { readAscii, readBytes, Reader, readU16, readU32, readU64 } from '../reader';
@@ -407,6 +407,12 @@ class WaveAudioTrackBacking implements InputAudioTrackBacking {
return UNDETERMINED_LANGUAGE;
}
getDisposition() {
return {
...DEFAULT_TRACK_DISPOSITION,
};
}
async getFirstTimestamp() {
return 0;
}
+1 -1
View File
@@ -15,7 +15,7 @@ import { Writer } from '../writer';
import { EncodedPacket } from '../packet';
import { WavOutputFormat } from '../output-format';
import { assert, assertNever, isIso88591Compatible, keyValueIterator } from '../misc';
import { MetadataTags, metadataTagsAreEmpty } from '../tags';
import { MetadataTags, metadataTagsAreEmpty } from '../metadata';
import { Id3V2Writer } from '../id3';
export class WaveMuxer extends Muxer {
+2 -2
View File
@@ -190,7 +190,7 @@ const MAX_CHUNKS_AT_ONCE = 2;
interface Chunk {
start: number;
written: ChunkSection[];
data: Uint8Array;
data: Uint8Array<ArrayBuffer>;
shouldFlush: boolean;
}
@@ -281,7 +281,7 @@ export class StreamTargetWriter extends Writer {
const chunks: {
start: number;
size: number;
data?: Uint8Array;
data?: Uint8Array<ArrayBuffer>;
}[] = [];
const sorted = [...this.sections].sort((a, b) => a.start - b.start);
+92
View File
@@ -0,0 +1,92 @@
import { expect, test } from 'vitest';
import { Output } from '../../src/output.js';
import { MkvOutputFormat } from '../../src/output-format.js';
import { BufferTarget } from '../../src/target.js';
import { EncodedVideoPacketSource } from '../../src/media-source.js';
import { EncodedPacket } from '../../src/packet.js';
import { Input } from '../../src/input.js';
import { BufferSource } from '../../src/source.js';
import { ALL_FORMATS } from '../../src/input-format.js';
test('Default track disposition', async () => {
const output = new Output({
format: new MkvOutputFormat(),
target: new BufferTarget(),
});
const source = new EncodedVideoPacketSource('avc');
output.addVideoTrack(source);
await output.start();
await source.add(new EncodedPacket(new Uint8Array(1024), 'key', 0, 1), {
decoderConfig: {
codec: 'avc1.123456',
codedWidth: 1920,
codedHeight: 1080,
},
});
await output.finalize();
using input = new Input({
source: new BufferSource(output.target.buffer!),
formats: ALL_FORMATS,
});
const track = (await input.getPrimaryVideoTrack())!;
expect(track.disposition).toEqual({
default: true,
forced: false,
original: false,
hearingImpaired: false,
visuallyImpaired: false,
commentary: false,
});
});
test('Customized track disposition', async () => {
const output = new Output({
format: new MkvOutputFormat(),
target: new BufferTarget(),
});
const source = new EncodedVideoPacketSource('avc');
output.addVideoTrack(source, {
disposition: {
default: false,
forced: true,
original: true,
hearingImpaired: true,
visuallyImpaired: true,
commentary: true,
},
});
await output.start();
await source.add(new EncodedPacket(new Uint8Array(1024), 'key', 0, 1), {
decoderConfig: {
codec: 'avc1.123456',
codedWidth: 1920,
codedHeight: 1080,
},
});
await output.finalize();
using input = new Input({
source: new BufferSource(output.target.buffer!),
formats: ALL_FORMATS,
});
const track = (await input.getPrimaryVideoTrack())!;
expect(track.disposition).toEqual({
default: false,
forced: true,
original: true,
hearingImpaired: true,
visuallyImpaired: true,
commentary: true,
});
});
+1 -1
View File
@@ -15,7 +15,7 @@ import { EncodedPacket } from '../../src/packet.js';
import { Input } from '../../src/input.js';
import { BufferSource, FilePathSource } from '../../src/source.js';
import { ALL_FORMATS } from '../../src/input-format.js';
import { AttachedFile, MetadataTags } from '../../src/tags.js';
import { AttachedFile, MetadataTags } from '../../src/metadata.js';
import path from 'node:path';
import { AudioCodec, buildAudioCodecString } from '../../src/codec.js';
import { Conversion } from '../../src/conversion.js';