Compare commits

...
11 Commits
18 changed files with 561 additions and 538 deletions
+12 -1
View File
@@ -14,7 +14,18 @@
source: new Mediabunny.BlobSource(file),
});
console.log(await input.getTracks());
const audioTrack = await input.getPrimaryAudioTrack();
const decoderConfig = await audioTrack.getDecoderConfig();
console.log(decoderConfig);
/*
const videoTrack = await input.getPrimaryVideoTrack();
const sink = new Mediabunny.EncodedPacketSink(videoTrack);
for await (const packet of sink.packets()) {
console.log(packet);
}
*/
/*
return;
+4 -1
View File
@@ -14,6 +14,9 @@ export default withMermaid({
title: 'Mediabunny',
description: DESCRIPTION,
cleanUrls: true,
sitemap: {
hostname: 'https://mediabunny.dev',
},
head: [
['link', { rel: 'icon', type: 'image/png', href: '/mediabunny-logo.png' }],
['link', { rel: 'icon', type: 'image/svg+xml', href: '/mediabunny-logo.svg' }],
@@ -34,7 +37,7 @@ export default withMermaid({
// https://vitepress.dev/reference/default-theme-config
nav: [
{ text: 'Guide', link: '/guide/introduction', activeMatch: '/guide' },
{ text: 'API', link: '/api', activeMatch: '/api' },
{ text: 'API', link: '/api/', activeMatch: '/api' },
{ text: 'LLMs', link: '/llms', activeMatch: '/llms' },
{ text: 'Examples', link: '/examples', activeMatch: '/examples' },
{ text: 'Sponsors', link: '/#sponsors', activeMatch: '/#sponsors' },
+6 -6
View File
@@ -9,37 +9,37 @@ hero:
features:
- title: Metadata extraction
details: Extract various metadata from an input media file.
link: /examples/metadata-extraction
link: /examples/metadata-extraction/
target: _self
icon:
src: /mingcute--file-info-line.svg
- title: Thumbnail generation
details: Generate multiple small thumbnails for a video track.
link: /examples/thumbnail-generation
link: /examples/thumbnail-generation/
target: _self
icon:
src: /mingcute--photo-album-line.svg
- title: Media player (advanced)
details: "A full video & audio media player, implemented from scratch with Mediabunny, with microsecond playback accuracy."
link: /examples/media-player
link: /examples/media-player/
target: _self
icon:
src: /mingcute--video-line.svg
- title: File compression
details: Convert an input file to a highly-compressed MP4 file.
link: /examples/file-compression
link: /examples/file-compression/
target: _self
icon:
src: /mingcute--file-zip-line.svg
- title: Procedural video generation
details: Generate a video file as fast as the hardware allows.
link: /examples/procedural-generation
link: /examples/procedural-generation/
target: _self
icon:
src: /mingcute--magic-3-line.svg
- title: Live recording & streaming
details: Record a video from live sources and stream it to a video element.
link: /examples/live-recording
link: /examples/live-recording/
target: _self
icon:
src: /mingcute--microphone-line.svg
+9 -1
View File
@@ -433,4 +433,12 @@ await textSource.add('00:00:00.000 --> 00:00:02.000\nHello there!\n\n');
await textSource.add('00:00:02.500 --> 00:00:04.000\nChunky chunks.\n\n');
```
The chunks have certain constraints: A cue must be fully contained within a chunk and cannot be split across multiple smaller chunks (although a chunk can contain multiple cues). Also, the WebVTT preamble must be added first and all at once.
The chunks have certain constraints: A cue must be fully contained within a chunk and cannot be split across multiple smaller chunks (although a chunk can contain multiple cues). Also, the WebVTT preamble must be added first and all at once.
::: info
For QuickTime to display WebVTT subtitles, it typically expects alignment information to be specified:
```
00:00:00.000 --> 00:00:02.000 align:center
This is your last chance.
```
:::
+1 -1
View File
@@ -21,7 +21,7 @@ const context = canvas.getContext('2d', { alpha: false, desynchronized: true })!
const frameRate = 30;
const chunks: Uint8Array[] = [];
const chunks: Uint8Array<ArrayBuffer>[] = [];
let recording = false;
let output: Output;
let videoSource: CanvasSource;
@@ -97,7 +97,7 @@ const extractMetadata = (resource: File | string) => {
'Lyrics': tags.lyrics,
'Comment': tags.comment,
'Images': tags.images?.map((image) => {
const blob = new Blob([image.data], { type: image.mimeType });
const blob = new Blob([image.data.slice()], { type: image.mimeType });
const element = new Image();
element.src = URL.createObjectURL(blob);
+285 -462
View File
File diff suppressed because it is too large Load Diff
+4 -4
View File
@@ -1,7 +1,7 @@
{
"name": "mediabunny",
"author": "Vanilagy",
"version": "1.25.1",
"version": "1.25.4",
"description": "Pure TypeScript media toolkit for reading, writing, and converting media files, directly in the browser.",
"type": "module",
"workspaces": [
@@ -70,7 +70,7 @@
"@eslint/js": "^9.22.0",
"@fontsource-variable/rubik": "^5.2.6",
"@fontsource/dm-mono": "^5.2.6",
"@microsoft/api-extractor": "^7.52.1",
"@microsoft/api-extractor": "^7.55.1",
"@stylistic/eslint-plugin": "^4.2.0",
"@tailwindcss/vite": "^4.1.7",
"@types/markdown-it-footnote": "^3.0.4",
@@ -84,8 +84,8 @@
"mermaid": "^11.6.0",
"tailwindcss": "^4.1.7",
"tsx": "^4.19.4",
"typescript": "^5.8.2",
"typescript-eslint": "^8.26.1",
"typescript": "^5.9.3",
"typescript-eslint": "^8.48.0",
"vite": "^6.3.5",
"vitepress": "^1.6.3",
"vitepress-plugin-llms": "^1.5.1",
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@mediabunny/mp3-encoder",
"author": "Vanilagy",
"version": "1.25.1",
"version": "1.25.4",
"description": "MP3 encoder extension for Mediabunny, based on LAME.",
"main": "./dist/bundles/mediabunny-mp3-encoder.mjs",
"module": "./dist/bundles/mediabunny-mp3-encoder.mjs",
+64 -3
View File
@@ -590,9 +590,7 @@ export const extractNalUnitTypeForHevc = (data: Uint8Array) => {
};
/** Builds a HevcDecoderConfigurationRecord from an HEVC packet in Annex B format. */
export const extractHevcDecoderConfigurationRecord = (
packetData: Uint8Array,
) => {
export const extractHevcDecoderConfigurationRecord = (packetData: Uint8Array) => {
try {
const nalUnits = findNalUnitsInAnnexB(packetData);
@@ -1456,6 +1454,69 @@ export const extractAv1CodecInfoFromPacket = (
}
}
// Frame size
const frameWidthBitsMinus1 = bitstream.readBits(4);
const frameHeightBitsMinus1 = bitstream.readBits(4);
const n1 = frameWidthBitsMinus1 + 1;
bitstream.skipBits(n1); // max_frame_width_minus_1
const n2 = frameHeightBitsMinus1 + 1;
bitstream.skipBits(n2); // max_frame_height_minus_1
// Frame IDs
let frameIdNumbersPresentFlag = 0;
if (reducedStillPictureHeader) {
frameIdNumbersPresentFlag = 0;
} else {
frameIdNumbersPresentFlag = bitstream.readBits(1);
}
if (frameIdNumbersPresentFlag) {
bitstream.skipBits(4); // delta_frame_id_length_minus_2
bitstream.skipBits(3); // additional_frame_id_length_minus_1
}
bitstream.skipBits(1); // use_128x128_superblock
bitstream.skipBits(1); // enable_filter_intra
bitstream.skipBits(1); // enable_intra_edge_filter
if (!reducedStillPictureHeader) {
bitstream.skipBits(1); // enable_interintra_compound
bitstream.skipBits(1); // enable_masked_compound
bitstream.skipBits(1); // enable_warped_motion
bitstream.skipBits(1); // enable_dual_filter
const enableOrderHint = bitstream.readBits(1);
if (enableOrderHint) {
bitstream.skipBits(1); // enable_jnt_comp
bitstream.skipBits(1); // enable_ref_frame_mvs
}
const seqChooseScreenContentTools = bitstream.readBits(1);
let seqForceScreenContentTools = 0;
if (seqChooseScreenContentTools) {
seqForceScreenContentTools = 2; // SELECT_SCREEN_CONTENT_TOOLS
} else {
seqForceScreenContentTools = bitstream.readBits(1);
}
if (seqForceScreenContentTools > 0) {
const seqChooseIntegerMv = bitstream.readBits(1);
if (!seqChooseIntegerMv) {
bitstream.skipBits(1); // seq_force_integer_mv
}
}
if (enableOrderHint) {
bitstream.skipBits(3); // order_hint_bits_minus_1
}
}
bitstream.skipBits(1); // enable_superres
bitstream.skipBits(1); // enable_cdef
bitstream.skipBits(1); // enable_restoration
// color_config()
const highBitdepth = bitstream.readBits(1);
let bitDepth = 8;
+5 -1
View File
@@ -1509,7 +1509,10 @@ export class Conversion {
targetSampleRate,
startTime: this._startTimestamp,
endTime: this._endTimestamp,
onSample: sample => this._registerAudioSample(track, trackOptions, source, sample),
onSample: async (sample) => {
await this._registerAudioSample(track, trackOptions, source, sample);
sample.close();
},
});
const sink = new AudioSampleSink(track);
@@ -1521,6 +1524,7 @@ export class Conversion {
}
await resampler.add(sample);
sample.close();
}
await resampler.finalize();
+40 -1
View File
@@ -22,6 +22,7 @@ import {
VideoCodec,
} from './codec';
import { customAudioEncoders, customVideoEncoders } from './custom-coder';
import { isFirefox } from './misc';
import { EncodedPacket } from './packet';
/**
@@ -529,7 +530,45 @@ export const canEncodeVideo = async (
});
const support = await VideoEncoder.isConfigSupported(encoderConfig);
return support.supported === true;
if (!support.supported) {
return false;
}
if (isFirefox()) {
// isConfigSupported on Firefox appears to unreliably indicate if encoding will actually succeed. Therefore, we
// just try encoding a frame to see if it actually works.
// https://github.com/Vanilagy/mediabunny/issues/222
// eslint-disable-next-line @typescript-eslint/no-misused-promises, no-async-promise-executor
return new Promise<boolean>(async (resolve) => {
try {
const encoder = new VideoEncoder({
output: () => {},
error: () => resolve(false),
});
encoder.configure(encoderConfig);
const frameData = new Uint8Array(width * height * 4);
const frame = new VideoFrame(frameData, {
format: 'RGBA',
codedWidth: width,
codedHeight: height,
timestamp: 0,
});
encoder.encode(frame);
frame.close();
await encoder.flush();
resolve(true);
} catch {
resolve(false);
}
});
} else {
return true;
}
};
/**
+1 -1
View File
@@ -850,7 +850,7 @@ export const dOps = (trackData: IsobmffAudioTrackData) => {
let inputSampleRate = trackData.info.sampleRate;
let outputGain = 0;
let channelMappingFamily = 0;
let channelMappingTable = new Uint8Array(0);
let channelMappingTable: Uint8Array<ArrayBufferLike> = new Uint8Array(0);
// Read preskip and from codec private data from the encoder
// https://www.rfc-editor.org/rfc/rfc7845#section-5
+16 -7
View File
@@ -157,20 +157,20 @@ export const writeBits = (bytes: Uint8Array, start: number, end: number, value:
export const toUint8Array = (source: AllowSharedBufferSource): Uint8Array => {
if (source.constructor === Uint8Array) { // We want a true Uint8Array, not something that extends it like Buffer
return source;
} else if (source instanceof ArrayBuffer) {
return new Uint8Array(source);
} else {
} else if (ArrayBuffer.isView(source)) {
return new Uint8Array(source.buffer, source.byteOffset, source.byteLength);
} else {
return new Uint8Array(source);
}
};
export const toDataView = (source: AllowSharedBufferSource) => {
export const toDataView = (source: AllowSharedBufferSource): DataView => {
if (source.constructor === DataView) {
return source;
} else if (source instanceof ArrayBuffer) {
return new DataView(source);
} else {
} else if (ArrayBuffer.isView(source)) {
return new DataView(source.buffer, source.byteOffset, source.byteLength);
} else {
return new DataView(source);
}
};
@@ -584,6 +584,7 @@ export const retriedFetch = async (
url: string | URL | Request,
requestInit: RequestInit,
getRetryDelay: (previousAttempts: number, error: unknown, url: string | URL | Request) => number | null,
shouldStop: () => boolean,
) => {
let attempts = 0;
@@ -591,6 +592,10 @@ export const retriedFetch = async (
try {
return await fetchFn(url, requestInit);
} catch (error) {
if (shouldStop()) {
throw error;
}
attempts++;
const retryDelayInSeconds = getRetryDelay(attempts, error, url);
@@ -607,6 +612,10 @@ export const retriedFetch = async (
if (retryDelayInSeconds > 0) {
await new Promise(resolve => setTimeout(resolve, 1000 * retryDelayInSeconds));
}
if (shouldStop()) {
throw error;
}
}
}
};
+65 -4
View File
@@ -21,6 +21,54 @@ import {
polyfillSymbolDispose();
type FinalizationRegistryValue = {
type: 'video';
data: VideoFrame | OffscreenCanvas | Uint8Array;
} | {
type: 'audio';
data: AudioData | Uint8Array;
};
// Let's manually handle logging the garbage collection errors that are typically logged by the browser. This way, they
// also kick for audio samples (which is normally not the case), making sure any incorrect code is quickly caught.
let lastVideoGcErrorLog = -Infinity;
let lastAudioGcErrorLog = -Infinity;
let finalizationRegistry: FinalizationRegistry<FinalizationRegistryValue> | null = null;
if (typeof FinalizationRegistry !== 'undefined') {
finalizationRegistry = new FinalizationRegistry<FinalizationRegistryValue>((value) => {
const now = Date.now();
if (value.type === 'video') {
if (now - lastVideoGcErrorLog >= 1000) {
// This error is annoying but oh so important
console.error(
`A VideoSample was garbage collected without first being closed. For proper resource management,`
+ ` make sure to call close() on all your VideoSamples as soon as you're done using them.`,
);
lastVideoGcErrorLog = now;
}
if (typeof VideoFrame !== 'undefined' && value.data instanceof VideoFrame) {
value.data.close(); // Prevent the browser error since we're logging our own
}
} else {
if (now - lastAudioGcErrorLog >= 1000) {
console.error(
`An AudioSample was garbage collected without first being closed. For proper resource management,`
+ ` make sure to call close() on all your AudioSamples as soon as you're done using them.`,
);
lastAudioGcErrorLog = now;
}
if (typeof AudioData !== 'undefined' && value.data instanceof AudioData) {
value.data.close();
}
}
});
}
/**
* Metadata used for VideoSample initialization.
* @group Samples
@@ -133,7 +181,11 @@ export class VideoSample implements Disposable {
data: VideoFrame | CanvasImageSource | AllowSharedBufferSource,
init?: VideoSampleInit,
) {
if (data instanceof ArrayBuffer || ArrayBuffer.isView(data)) {
if (
data instanceof ArrayBuffer
|| (typeof SharedArrayBuffer !== 'undefined' && data instanceof SharedArrayBuffer)
|| ArrayBuffer.isView(data)
) {
if (!init || typeof init !== 'object') {
throw new TypeError('init must be an object.');
}
@@ -265,6 +317,8 @@ export class VideoSample implements Disposable {
} else {
throw new TypeError('Invalid data type: Must be a BufferSource or CanvasImageSource.');
}
finalizationRegistry?.register(this, { type: 'video', data: this._data }, this);
}
/** Clones this video sample. */
@@ -313,6 +367,8 @@ export class VideoSample implements Disposable {
return;
}
finalizationRegistry?.unregister(this);
if (isVideoFrame(this._data)) {
this._data.close();
} else {
@@ -955,6 +1011,8 @@ export class AudioSample implements Disposable {
this._data = dataBuffer;
}
finalizationRegistry?.register(this, { type: 'audio', data: this._data }, this);
}
/** Returns the number of bytes required to hold the audio sample's data as specified by the given options. */
@@ -1116,9 +1174,8 @@ export class AudioSample implements Disposable {
}
}
} else {
// Branch for Uint8Array data (non-AudioData)
const uint8Data = this._data;
const srcView = new DataView(uint8Data.buffer, uint8Data.byteOffset, uint8Data.byteLength);
const srcView = toDataView(uint8Data);
const srcFormat = this.format;
const readFn = getReadFunction(srcFormat);
@@ -1188,6 +1245,8 @@ export class AudioSample implements Disposable {
return;
}
finalizationRegistry?.unregister(this);
if (isAudioData(this._data)) {
this._data.close();
} else {
@@ -1250,7 +1309,9 @@ export class AudioSample implements Disposable {
numberOfFrames: this.numberOfFrames,
numberOfChannels: this.numberOfChannels,
timestamp: this.microsecondTimestamp,
data: this._data,
data: this._data.buffer instanceof ArrayBuffer
? this._data.buffer
: this._data.slice(), // In the case of SharedArrayBuffer, convert to ArrayBuffer
});
}
}
+44 -40
View File
@@ -102,10 +102,17 @@ export class BufferSource extends Source {
/** @internal */
_onreadCalled = false;
/** Creates a new {@link BufferSource} backed the specified `ArrayBuffer` or `ArrayBufferView`. */
constructor(buffer: ArrayBuffer | ArrayBufferView) {
if (!(buffer instanceof ArrayBuffer) && !ArrayBuffer.isView(buffer)) {
throw new TypeError('buffer must be an ArrayBuffer or ArrayBufferView.');
/**
* Creates a new {@link BufferSource} backed by the specified `ArrayBuffer`, `SharedArrayBuffer`,
* or `ArrayBufferView`.
*/
constructor(buffer: AllowSharedBufferSource) {
if (
!(buffer instanceof ArrayBuffer)
&& !(typeof SharedArrayBuffer !== 'undefined' && buffer instanceof SharedArrayBuffer)
&& !ArrayBuffer.isView(buffer)
) {
throw new TypeError('buffer must be an ArrayBuffer, SharedArrayBuffer, or ArrayBufferView.');
}
super();
@@ -234,12 +241,7 @@ export class BlobSource extends Source {
const { done, value } = await reader.read();
if (done) {
this._orchestrator.forgetWorker(worker);
if (worker.currentPos < worker.targetPos) { // I think this `if` should always hit?
throw new Error('Blob reader stopped unexpectedly before all requested data was read.');
}
break;
throw new Error('Blob reader stopped unexpectedly before all requested data was read.');
}
if (worker.aborted) {
@@ -297,6 +299,10 @@ const DEFAULT_RETRY_DELAY
= typeof navigator !== 'undefined' && typeof navigator.onLine === 'boolean' ? navigator.onLine : true;
if (isOnline && originOfSrc !== null && originOfSrc !== window.location.origin) {
console.warn(
`Request will not be retried because a CORS error was suspected due to different origins. You can`
+ ` modify this behavior by providing your own function for the 'getRetryDelay' option.`,
);
return null;
}
}
@@ -425,6 +431,7 @@ export class UrlSource extends Source {
signal: abortController.signal,
}),
this._getRetryDelay,
() => this._disposed,
);
if (!response.ok) {
@@ -436,7 +443,7 @@ export class UrlSource extends Source {
let fileSize: number;
if (response.status === 206) {
fileSize = this._getPartialLengthFromRangeResponse(response);
fileSize = this._getTotalLengthFromRangeResponse(response);
worker = this._orchestrator.createWorker(0, Math.min(fileSize, URL_SOURCE_MIN_LOAD_AMOUNT));
} else {
// Server probably returned a 200.
@@ -492,6 +499,7 @@ export class UrlSource extends Source {
signal: abortController.signal,
}),
this._getRetryDelay,
() => this._disposed,
);
}
@@ -509,14 +517,6 @@ export class UrlSource extends Source {
);
}
const length = this._getPartialLengthFromRangeResponse(response);
const required = worker.targetPos - worker.currentPos;
if (length < required) {
throw new Error(
`HTTP response unexpectedly too short: Needed at least ${required} bytes, got only ${length}.`,
);
}
if (!response.body) {
throw new Error(
'Missing HTTP response body stream. The used fetch function must provide the response body as a'
@@ -539,6 +539,11 @@ export class UrlSource extends Source {
try {
readResult = await reader.read();
} catch (error) {
if (this._disposed) {
// No need to try to retry
throw error;
}
const retryDelayInSeconds = this._getRetryDelay(1, error, this._url);
if (retryDelayInSeconds !== null) {
console.error('Error while reading response stream. Attempting to resume.', error);
@@ -557,16 +562,17 @@ export class UrlSource extends Source {
const { done, value } = readResult;
if (done) {
this._orchestrator.forgetWorker(worker);
if (worker.currentPos < worker.targetPos) {
throw new Error(
'Response stream reader stopped unexpectedly before all requested data was read.',
);
if (worker.currentPos >= worker.targetPos) {
// All data was delivered, we're good
this._orchestrator.forgetWorker(worker);
worker.running = false;
return;
}
worker.running = false;
return;
// The response stopped early, before the target. This can happen if server decides to cap range
// requests arbitrarily, even if the request had an uncapped end. In this case, let's fetch the rest
// of the data using a new request.
break;
}
this.onread?.(worker.currentPos, worker.currentPos + value.length);
@@ -586,25 +592,23 @@ export class UrlSource extends Source {
}
/** @internal */
private _getPartialLengthFromRangeResponse(response: Response) {
private _getTotalLengthFromRangeResponse(response: Response) {
const contentRange = response.headers.get('Content-Range');
if (contentRange) {
const match = /\/(\d+)/.exec(contentRange);
if (match) {
return Number(match[1]);
} else {
throw new Error(`Invalid Content-Range header: ${contentRange}`);
}
}
const contentLength = response.headers.get('Content-Length');
if (contentLength) {
return Number(contentLength);
} else {
const contentLength = response.headers.get('Content-Length');
if (contentLength) {
return Number(contentLength);
} else {
throw new Error(
'Partial HTTP response (status 206) must surface either Content-Range or'
+ ' Content-Length header.',
);
}
throw new Error(
'Partial HTTP response (status 206) must surface either Content-Range or'
+ ' Content-Length header.',
);
}
}
@@ -639,7 +643,7 @@ export class FilePathSource extends Source {
_fileHandle: FileHandle | null = null;
/** Creates a new {@link FilePathSource} backed by the file at the specified file path. */
constructor(filePath: string, options: BlobSourceOptions = {}) {
constructor(filePath: string, options: FilePathSourceOptions = {}) {
if (typeof filePath !== 'string') {
throw new TypeError('filePath must be a string.');
}
+1 -1
View File
@@ -62,7 +62,7 @@ export type StreamTargetChunk = {
/** The operation type. */
type: 'write'; // This ensures automatic compatibility with FileSystemWritableFileStream
/** The data to write. */
data: Uint8Array;
data: Uint8Array<ArrayBuffer>;
/** The byte offset in the output file at which to write the data. */
position: number;
};
+2 -2
View File
@@ -190,7 +190,7 @@ const MAX_CHUNKS_AT_ONCE = 2;
interface Chunk {
start: number;
written: ChunkSection[];
data: Uint8Array;
data: Uint8Array<ArrayBuffer>;
shouldFlush: boolean;
}
@@ -281,7 +281,7 @@ export class StreamTargetWriter extends Writer {
const chunks: {
start: number;
size: number;
data?: Uint8Array;
data?: Uint8Array<ArrayBuffer>;
}[] = [];
const sorted = [...this.sections].sort((a, b) => a.start - b.start);