Compare commits

..
16 Commits
Author SHA1 Message Date
Vanilagy fa1f44cf92 Bump patch 2025-11-11 22:26:46 +01:00
Vanilagy 966ac133e9 Merge branch 'main' of https://github.com/Vanilagy/mediabunny 2025-11-11 18:03:28 +01:00
Vanilagy 23f814679e Modify audio encoder to fill gaps between timestamps with silence to ensure AV sync (fixes #176) 2025-11-11 18:03:28 +01:00
Jonny BurgerandGitHub f3a2d57156 Don't overlook potential packets in FLAC bitstream (#219)
* Don't overlook potential packets in FLAC bitstream

* Update way of going back
2025-11-11 17:07:49 +01:00
Vanilagy 9d477dfb14 Allow muxing of avc3 into ISOBMFF (see #216) 2025-11-11 17:06:10 +01:00
Vanilagy 5b45c51d40 Support AVC3 in ISOBMFF (fixes #216) 2025-11-11 16:55:54 +01:00
Vanilagy 71dd2c565b Merge branch 'main' of https://github.com/Vanilagy/metamuxer 2025-11-11 11:05:17 +01:00
Vanilagy 3db4aa8b91 Fixed doubly stacked rotation correction when converting rotated video with a custom process function (fixes #218) 2025-11-11 11:05:15 +01:00
Vanilagy 8b39eb0d43 Revert Matroska default duration flooring 2025-11-10 21:14:43 +01:00
Vanilagy 6022781e8c Bump patch 2025-11-10 20:53:28 +01:00
Vanilagy c0da28fb6c Floor Matroska default packet duration (fixes #170) 2025-11-10 16:14:25 +01:00
Vanilagy fd8c74d4db Fix invalid encoder AAC description on WebKit (fixes #206) 2025-11-10 14:44:35 +01:00
David P.andGitHub 6fb3103c01 Merge pull request #215 from hexcowboy/issue-#184
Use `ArrayBuffer` instead of `ReadableStreamReader`on WebKit browsers
2025-11-10 11:36:13 +01:00
Vanilagy d969266662 Simplify isWebKit, get rid of isSafari 2025-11-10 11:30:26 +01:00
hex e0dda582b1 fix webkit video conversion bug 2025-11-07 16:58:20 -07:00
Vanilagy 5aabf6350a Clarify docs 2025-11-06 16:31:02 +01:00
17 changed files with 233 additions and 51 deletions
+11 -5
View File
@@ -48,6 +48,7 @@
target target
}); });
let ctx = null;
const conversion = await Mediabunny.Conversion.init({ const conversion = await Mediabunny.Conversion.init({
input: new Mediabunny.Input({ input: new Mediabunny.Input({
formats: Mediabunny.ALL_FORMATS, formats: Mediabunny.ALL_FORMATS,
@@ -56,6 +57,8 @@
output, output,
audio: (_, n) => ({ audio: (_, n) => ({
discard: n > 1, discard: n > 1,
codec: 'aac',
//codec: 'opus',
/* /*
process: (sample) => { process: (sample) => {
return sample; return sample;
@@ -70,7 +73,7 @@
//numberOfChannels: 1, //numberOfChannels: 1,
//sampleRate: 4000 //sampleRate: 4000
//discard: true //discard: true
//forceTranscode: true, forceTranscode: true,
}), }),
/* /*
video: { video: {
@@ -97,6 +100,7 @@
}, },
*/ */
video: () => ({ video: () => ({
//discard: true,
/* /*
process: (sample) => { process: (sample) => {
if (!ctx) { if (!ctx) {
@@ -108,9 +112,11 @@
ctx = canvas.getContext('2d'); ctx = canvas.getContext('2d');
} }
console.log(ctx.canvas.width, ctx.canvas.height);
ctx.clearRect(0, 0, ctx.canvas.width, ctx.canvas.height); ctx.clearRect(0, 0, ctx.canvas.width, ctx.canvas.height);
sample.draw(ctx, 0, 0); sample.drawWithFit(ctx, { fit: 'fill' });
ctx.drawImage(watermark, 32, 32); //ctx.drawImage(watermark, 32, 32);
return ctx.canvas; return ctx.canvas;
}, },
@@ -170,8 +176,8 @@
} }
}, },
trim: { trim: {
start: 0, //start: 0,
end: 10 //end: 10
}, },
}); });
console.log(conversion); console.log(conversion);
+44 -1
View File
@@ -14,7 +14,49 @@
source: new Mediabunny.BlobSource(file), source: new Mediabunny.BlobSource(file),
}); });
const videoTrack = await input.getPrimaryVideoTrack(); const audioTrack = await input.getPrimaryAudioTrack();
const sink = new Mediabunny.AudioSampleSink(audioTrack);
let lastEnd = 0;
for await (const sample of sink.samples()) {
if (sample.timestamp - lastEnd > 0) {
console.warn(sample.timestamp - lastEnd)
console.log(sample.timestamp, sample.duration, "diffie", sample.timestamp - lastEnd);
}
lastEnd = sample.timestamp + sample.duration;
sample.close();
}
/*
const sink = new Mediabunny.EncodedPacketSink(videoTrack);
for await (const packet of sink.packets()) {
console.log(packet.timestamp, packet.duration, packet.timestamp + packet.duration);
if (packet.timestamp > 10) break;
}
*/
/*
const sink = new Mediabunny.VideoSampleSink(videoTrack);
for await (const sample of sink.samples()) {
console.log(sample.timestamp, sample.duration);
sample.close();
}
*/
/*
const sink = new Mediabunny.EncodedPacketSink(videoTrack);
for await (const packet of sink.packets()) {
console.log(packet.timestamp, packet.timestamp + packet.duration);
}
*/
/*
const sink = new Mediabunny.VideoSampleSink(videoTrack); const sink = new Mediabunny.VideoSampleSink(videoTrack);
console.log(await videoTrack.getDecoderConfig()) console.log(await videoTrack.getDecoderConfig())
@@ -30,6 +72,7 @@
for await (const sample of sink.samples()) { for await (const sample of sink.samples()) {
console.log(sample); console.log(sample);
} }
*/
/* /*
let total = 0; let total = 0;
+6 -6
View File
@@ -1,12 +1,12 @@
{ {
"name": "mediabunny", "name": "mediabunny",
"version": "1.24.4", "version": "1.24.6",
"lockfileVersion": 3, "lockfileVersion": 3,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "mediabunny", "name": "mediabunny",
"version": "1.24.4", "version": "1.24.6",
"license": "MPL-2.0", "license": "MPL-2.0",
"workspaces": [ "workspaces": [
"packages/*" "packages/*"
@@ -7749,9 +7749,9 @@
} }
}, },
"node_modules/mediabunny": { "node_modules/mediabunny": {
"version": "1.24.3", "version": "1.24.5",
"resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.24.3.tgz", "resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.24.5.tgz",
"integrity": "sha512-uQu5lXg6c3z2YUKLimAsem1qYPdGfGziQ5CKaAMJAs0C0NNw5T/Zkn0foPML0gWqQJ5ZeC0wTgCxpiBQY8m1Tw==", "integrity": "sha512-4Sd5x8WXQiuDKZcJDAiQjyD0Lg/cg9p+dBShCM02F1pMvKdB9R6+WHZ9NFCFSqViVPX2fYSu95ZIpBUg9+TwTQ==",
"license": "MPL-2.0", "license": "MPL-2.0",
"peer": true, "peer": true,
"workspaces": [ "workspaces": [
@@ -12242,7 +12242,7 @@
}, },
"packages/mp3-encoder": { "packages/mp3-encoder": {
"name": "@mediabunny/mp3-encoder", "name": "@mediabunny/mp3-encoder",
"version": "1.24.4", "version": "1.24.6",
"license": "MPL-2.0", "license": "MPL-2.0",
"devDependencies": { "devDependencies": {
"@types/emscripten": "^1.40.1" "@types/emscripten": "^1.40.1"
+1 -1
View File
@@ -1,7 +1,7 @@
{ {
"name": "mediabunny", "name": "mediabunny",
"author": "Vanilagy", "author": "Vanilagy",
"version": "1.24.4", "version": "1.24.6",
"description": "Pure TypeScript media toolkit for reading, writing, and converting media files, directly in the browser.", "description": "Pure TypeScript media toolkit for reading, writing, and converting media files, directly in the browser.",
"type": "module", "type": "module",
"workspaces": [ "workspaces": [
+1 -1
View File
@@ -1,7 +1,7 @@
{ {
"name": "@mediabunny/mp3-encoder", "name": "@mediabunny/mp3-encoder",
"author": "Vanilagy", "author": "Vanilagy",
"version": "1.24.4", "version": "1.24.6",
"description": "MP3 encoder extension for Mediabunny, based on LAME.", "description": "MP3 encoder extension for Mediabunny, based on LAME.",
"main": "./dist/bundles/mediabunny-mp3-encoder.mjs", "main": "./dist/bundles/mediabunny-mp3-encoder.mjs",
"module": "./dist/bundles/mediabunny-mp3-encoder.mjs", "module": "./dist/bundles/mediabunny-mp3-encoder.mjs",
+53 -2
View File
@@ -337,6 +337,7 @@ export const extractVideoCodecString = (trackInfo: {
codec: VideoCodec | null; codec: VideoCodec | null;
codecDescription: Uint8Array | null; codecDescription: Uint8Array | null;
colorSpace: VideoColorSpaceInit | null; colorSpace: VideoColorSpaceInit | null;
avcType: 1 | 3 | null;
avcCodecInfo: AvcDecoderConfigurationRecord | null; avcCodecInfo: AvcDecoderConfigurationRecord | null;
hevcCodecInfo: HevcDecoderConfigurationRecord | null; hevcCodecInfo: HevcDecoderConfigurationRecord | null;
vp9CodecInfo: Vp9CodecInfo | null; vp9CodecInfo: Vp9CodecInfo | null;
@@ -345,6 +346,8 @@ export const extractVideoCodecString = (trackInfo: {
const { codec, codecDescription, colorSpace, avcCodecInfo, hevcCodecInfo, vp9CodecInfo, av1CodecInfo } = trackInfo; const { codec, codecDescription, colorSpace, avcCodecInfo, hevcCodecInfo, vp9CodecInfo, av1CodecInfo } = trackInfo;
if (codec === 'avc') { if (codec === 'avc') {
assert(trackInfo.avcType !== null);
if (avcCodecInfo) { if (avcCodecInfo) {
const bytes = new Uint8Array([ const bytes = new Uint8Array([
avcCodecInfo.avcProfileIndication, avcCodecInfo.avcProfileIndication,
@@ -352,14 +355,14 @@ export const extractVideoCodecString = (trackInfo: {
avcCodecInfo.avcLevelIndication, avcCodecInfo.avcLevelIndication,
]); ]);
return `avc1.${bytesToHexString(bytes)}`; return `avc${trackInfo.avcType}.${bytesToHexString(bytes)}`;
} }
if (!codecDescription || codecDescription.byteLength < 4) { if (!codecDescription || codecDescription.byteLength < 4) {
throw new TypeError('AVC decoder description is not provided or is not at least 4 bytes long.'); throw new TypeError('AVC decoder description is not provided or is not at least 4 bytes long.');
} }
return `avc1.${bytesToHexString(codecDescription.subarray(1, 4))}`; return `avc${trackInfo.avcType}.${bytesToHexString(codecDescription.subarray(1, 4))}`;
} else if (codec === 'hevc') { } else if (codec === 'hevc') {
let generalProfileSpace: number; let generalProfileSpace: number;
let generalProfileIdc: number; let generalProfileIdc: number;
@@ -621,6 +624,54 @@ export const parseAacAudioSpecificConfig = (bytes: Uint8Array | null): AacAudioS
}; };
}; };
export const buildAacAudioSpecificConfig = (config: {
objectType: number;
sampleRate: number;
numberOfChannels: number;
}) => {
let frequencyIndex = aacFrequencyTable.indexOf(config.sampleRate);
let customSampleRate: number | null = null;
if (frequencyIndex === -1) {
frequencyIndex = 15;
customSampleRate = config.sampleRate;
}
const channelConfiguration = aacChannelMap.indexOf(config.numberOfChannels);
if (channelConfiguration === -1) {
throw new TypeError(`Unsupported number of channels: ${config.numberOfChannels}`);
}
let bitCount = 5 + 4 + 4;
if (config.objectType >= 32) {
bitCount += 6;
}
if (frequencyIndex === 15) {
bitCount += 24;
}
const byteCount = Math.ceil(bitCount / 8);
const bytes = new Uint8Array(byteCount);
const bitstream = new Bitstream(bytes);
if (config.objectType < 32) {
bitstream.writeBits(5, config.objectType);
} else {
bitstream.writeBits(5, 31);
bitstream.writeBits(6, config.objectType - 32);
}
bitstream.writeBits(4, frequencyIndex);
if (frequencyIndex === 15) {
bitstream.writeBits(24, customSampleRate!);
}
bitstream.writeBits(4, channelConfiguration);
return bytes;
};
export const OPUS_SAMPLE_RATE = 48_000; export const OPUS_SAMPLE_RATE = 48_000;
const PCM_CODEC_REGEX = /^pcm-([usf])(\d+)+(be)?$/; const PCM_CODEC_REGEX = /^pcm-([usf])(\d+)+(be)?$/;
+6 -2
View File
@@ -182,7 +182,8 @@ export type ConversionVideoOptions = {
* corrections. * corrections.
* *
* Must return a {@link VideoSample} or a `CanvasImageSource`, an array of them, or `null` for dropping the frame. * Must return a {@link VideoSample} or a `CanvasImageSource`, an array of them, or `null` for dropping the frame.
* When non-timestamped data is returned, the timestamp and duration from the source sample will be used. * When non-timestamped data is returned, the timestamp and duration from the source sample will be used. Rotation
* metadata of the returned sample will be ignored.
* *
* This function can also be used to manually resize frames. When doing so, you should signal the post-process * This function can also be used to manually resize frames. When doing so, you should signal the post-process
* dimensions using the `processedWidth` and `processedHeight` fields, which enables the encoder to better know what * dimensions using the `processedWidth` and `processedHeight` fields, which enables the encoder to better know what
@@ -873,7 +874,10 @@ export class Conversion {
|| trackOptions.process !== undefined; || trackOptions.process !== undefined;
let needsRerender = width !== originalWidth let needsRerender = width !== originalWidth
|| height !== originalHeight || height !== originalHeight
|| (totalRotation !== 0 && !outputSupportsRotation) // TODO This is suboptimal: Forcing a rerender when both rotation and process are set is not
// performance-optimal, but right now there's no other way because we can't change the track rotation
// metadata after the output has already started. Should be possible with API changes in v2, though!
|| (totalRotation !== 0 && (!outputSupportsRotation || trackOptions.process !== undefined))
|| !!crop; || !!crop;
const alpha = trackOptions.alpha ?? 'discard'; const alpha = trackOptions.alpha ?? 'discard';
+6 -4
View File
@@ -328,11 +328,13 @@ export class FlacDemuxer extends Demuxer {
const nextByte = readU8(slice); const nextByte = readU8(slice);
if (nextByte === 0xff) { if (nextByte === 0xff) {
const positionBeforeReading = slice.filePos;
const byteAfterNextByte = readU8(slice); const byteAfterNextByte = readU8(slice);
const expected = this.blockingBit === 1 ? 0b1111_1001 : 0b1111_1000; const expected = this.blockingBit === 1 ? 0b1111_1001 : 0b1111_1000;
if (byteAfterNextByte !== expected) { if (byteAfterNextByte !== expected) {
slice.skip(-1); slice.filePos = positionBeforeReading;
continue; continue;
} }
@@ -345,7 +347,7 @@ export class FlacDemuxer extends Demuxer {
}); });
if (!nextFrameHeader) { if (!nextFrameHeader) {
slice.skip(-1); slice.filePos = positionBeforeReading;
continue; continue;
} }
@@ -355,14 +357,14 @@ export class FlacDemuxer extends Demuxer {
if (this.blockingBit === 0) { if (this.blockingBit === 0) {
// Case A: If the stream is fixed block size, this is the frame number, which increments by 1 // Case A: If the stream is fixed block size, this is the frame number, which increments by 1
if (nextFrameHeader.num - frameHeader.num !== 1) { if (nextFrameHeader.num - frameHeader.num !== 1) {
slice.skip(-1); slice.filePos = positionBeforeReading;
continue; continue;
} }
} else { } else {
// Case B: If the stream is variable block size, this is the sample number, which increments by // Case B: If the stream is variable block size, this is the sample number, which increments by
// amount of samples in a frame. // amount of samples in a frame.
if (nextFrameHeader.num - frameHeader.num !== frameHeader.blockSize) { if (nextFrameHeader.num - frameHeader.num !== frameHeader.blockSize) {
slice.skip(-1); slice.filePos = positionBeforeReading;
continue; continue;
} }
} }
+9 -7
View File
@@ -577,7 +577,7 @@ export const stsd = (trackData: IsobmffTrackData) => {
if (trackData.type === 'video') { if (trackData.type === 'video') {
sampleDescription = videoSampleDescription( sampleDescription = videoSampleDescription(
VIDEO_CODEC_TO_BOX_NAME[trackData.track.source._codec], videoCodecToBoxName(trackData.track.source._codec, trackData.info.decoderConfig.codec),
trackData, trackData,
); );
} else if (trackData.type === 'audio') { } else if (trackData.type === 'audio') {
@@ -1572,12 +1572,14 @@ const dataStringBoxLong = (value: string) => {
]); ]);
}; };
const VIDEO_CODEC_TO_BOX_NAME: Record<VideoCodec, string> = { const videoCodecToBoxName = (codec: VideoCodec, fullCodecString: string) => {
avc: 'avc1', switch (codec) {
hevc: 'hvc1', case 'avc': return fullCodecString.startsWith('avc3') ? 'avc3' : 'avc1';
vp8: 'vp08', case 'hevc': return 'hvc1';
vp9: 'vp09', case 'vp8': return 'vp08';
av1: 'av01', case 'vp9': return 'vp09';
case 'av1': return 'av01';
}
}; };
const VIDEO_CODEC_TO_CONFIGURATION_BOX: Record<VideoCodec, (trackData: IsobmffVideoTrackData) => Box | null> = { const VIDEO_CODEC_TO_CONFIGURATION_BOX: Record<VideoCodec, (trackData: IsobmffVideoTrackData) => Box | null> = {
+4 -1
View File
@@ -126,6 +126,7 @@ type InternalTrack = {
codec: VideoCodec | null; codec: VideoCodec | null;
codecDescription: Uint8Array | null; codecDescription: Uint8Array | null;
colorSpace: VideoColorSpaceInit | null; colorSpace: VideoColorSpaceInit | null;
avcType: 1 | 3 | null;
avcCodecInfo: AvcDecoderConfigurationRecord | null; avcCodecInfo: AvcDecoderConfigurationRecord | null;
hevcCodecInfo: HevcDecoderConfigurationRecord | null; hevcCodecInfo: HevcDecoderConfigurationRecord | null;
vp9CodecInfo: Vp9CodecInfo | null; vp9CodecInfo: Vp9CodecInfo | null;
@@ -836,6 +837,7 @@ export class IsobmffDemuxer extends Demuxer {
codec: null, codec: null,
codecDescription: null, codecDescription: null,
colorSpace: null, colorSpace: null,
avcType: null,
avcCodecInfo: null, avcCodecInfo: null,
hevcCodecInfo: null, hevcCodecInfo: null,
vp9CodecInfo: null, vp9CodecInfo: null,
@@ -890,8 +892,9 @@ export class IsobmffDemuxer extends Demuxer {
const lowercaseBoxName = sampleBoxInfo.name.toLowerCase(); const lowercaseBoxName = sampleBoxInfo.name.toLowerCase();
if (track.info.type === 'video') { if (track.info.type === 'video') {
if (lowercaseBoxName === 'avc1') { if (lowercaseBoxName === 'avc1' || lowercaseBoxName === 'avc3') {
track.info.codec = 'avc'; track.info.codec = 'avc';
track.info.avcType = lowercaseBoxName === 'avc1' ? 1 : 3;
} else if (lowercaseBoxName === 'hvc1' || lowercaseBoxName === 'hev1') { } else if (lowercaseBoxName === 'hvc1' || lowercaseBoxName === 'hev1') {
track.info.codec = 'hevc'; track.info.codec = 'hevc';
} else if (lowercaseBoxName === 'vp08') { } else if (lowercaseBoxName === 'vp08') {
+3 -2
View File
@@ -844,13 +844,13 @@ export class MatroskaDemuxer extends Demuxer {
blocks.splice(blockIndex, 1); // Remove the original block blocks.splice(blockIndex, 1); // Remove the original block
const blockDuration = originalBlock.duration || frameCount * (track.defaultDuration ?? 0);
// Now, let's insert each frame as its own block // Now, let's insert each frame as its own block
for (let i = 0; i < frameCount; i++) { for (let i = 0; i < frameCount; i++) {
const frameSize = frameSizes[i]!; const frameSize = frameSizes[i]!;
const frameData = readBytes(slice, frameSize); const frameData = readBytes(slice, frameSize);
const blockDuration = originalBlock.duration || (frameCount * (track.defaultDuration ?? 0));
// Distribute timestamps evenly across the block duration // Distribute timestamps evenly across the block duration
const frameTimestamp = originalBlock.timestamp + (blockDuration * i / frameCount); const frameTimestamp = originalBlock.timestamp + (blockDuration * i / frameCount);
const frameDuration = blockDuration / frameCount; const frameDuration = blockDuration / frameCount;
@@ -2319,6 +2319,7 @@ class MatroskaVideoTrackBacking extends MatroskaTrackBacking implements InputVid
codec: this.internalTrack.info.codec, codec: this.internalTrack.info.codec,
codecDescription: this.internalTrack.info.codecDescription, codecDescription: this.internalTrack.info.codecDescription,
colorSpace: this.internalTrack.info.colorSpace, colorSpace: this.internalTrack.info.colorSpace,
avcType: 1, // We don't know better (or do we?) so just assume 'avc1'
avcCodecInfo: this.internalTrack.info.codec === 'avc' && firstPacket avcCodecInfo: this.internalTrack.info.codec === 'avc' && firstPacket
? extractAvcDecoderConfigurationRecord(firstPacket.data) ? extractAvcDecoderConfigurationRecord(firstPacket.data)
: null, : null,
+4 -4
View File
@@ -26,7 +26,7 @@ import {
insertSorted, insertSorted,
isFirefox, isFirefox,
isNumber, isNumber,
isSafari, isWebKit,
last, last,
mapAsyncGenerator, mapAsyncGenerator,
promiseWithResolvers, promiseWithResolvers,
@@ -918,7 +918,7 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
} else { } else {
assert(this.decoder); assert(this.decoder);
if (!isSafari()) { if (!isWebKit()) {
insertSorted(this.inputTimestamps, packet.timestamp, x => x); insertSorted(this.inputTimestamps, packet.timestamp, x => x);
} }
@@ -1049,7 +1049,7 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
/** Handler for the WebCodecs VideoDecoder for ironing out browser differences. */ /** Handler for the WebCodecs VideoDecoder for ironing out browser differences. */
sampleHandler(sample: VideoSample) { sampleHandler(sample: VideoSample) {
if (isSafari()) { if (isWebKit()) {
// For correct B-frame handling, we don't just hand over the frames directly but instead add them to // For correct B-frame handling, we don't just hand over the frames directly but instead add them to
// a queue, because we want to ensure frames are emitted in presentation order. We flush the queue // a queue, because we want to ensure frames are emitted in presentation order. We flush the queue
// each time we receive a frame with a timestamp larger than the highest we've seen so far, as we // each time we receive a frame with a timestamp larger than the highest we've seen so far, as we
@@ -1137,7 +1137,7 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
this.alphaRaslSkipped = false; this.alphaRaslSkipped = false;
} }
if (isSafari()) { if (isWebKit()) {
for (const sample of this.sampleQueue) { for (const sample of this.sampleQueue) {
this.finalizeAndEmitSample(sample); this.finalizeAndEmitSample(sample);
} }
+61
View File
@@ -9,6 +9,8 @@
import { import {
AUDIO_CODECS, AUDIO_CODECS,
AudioCodec, AudioCodec,
buildAacAudioSpecificConfig,
parseAacAudioSpecificConfig,
parsePcmCodec, parsePcmCodec,
PCM_AUDIO_CODECS, PCM_AUDIO_CODECS,
PcmAudioCodec, PcmAudioCodec,
@@ -24,9 +26,11 @@ import {
CallSerializer, CallSerializer,
clamp, clamp,
isFirefox, isFirefox,
last,
promiseWithResolvers, promiseWithResolvers,
setInt24, setInt24,
setUint24, setUint24,
toUint8Array,
} from './misc'; } from './misc';
import { Muxer } from './muxer'; import { Muxer } from './muxer';
import { SubtitleParser } from './subtitles'; import { SubtitleParser } from './subtitles';
@@ -1290,6 +1294,8 @@ class AudioEncoderWrapper {
private customEncoderCallSerializer = new CallSerializer(); private customEncoderCallSerializer = new CallSerializer();
private customEncoderQueueSize = 0; private customEncoderQueueSize = 0;
private lastEndSampleIndex: number | null = null;
/** /**
* Encoders typically throw their errors "out of band", meaning asynchronously in some other execution context. * Encoders typically throw their errors "out of band", meaning asynchronously in some other execution context.
* However, we want to surface these errors to the user within the normal control flow, so they don't go uncaught. * However, we want to surface these errors to the user within the normal control flow, so they don't go uncaught.
@@ -1338,6 +1344,35 @@ class AudioEncoderWrapper {
} }
assert(this.encoderInitialized); assert(this.encoderInitialized);
// Handle padding of gaps with silence to avoid audio drift over time, like in
// https://github.com/Vanilagy/mediabunny/issues/176
// TODO An open question is how encoders deal with the first AudioData having a non-zero timestamp, and with
// AudioDatas that have an overlapping timestamp range.
{
const startSampleIndex = Math.round(
audioSample.timestamp * audioSample.sampleRate,
);
const endSampleIndex = Math.round(
(audioSample.timestamp + audioSample.duration) * audioSample.sampleRate,
);
if (this.lastEndSampleIndex !== null && startSampleIndex > this.lastEndSampleIndex) {
const sampleCount = startSampleIndex - this.lastEndSampleIndex;
const fillSample = new AudioSample({
data: new Float32Array(sampleCount * audioSample.numberOfChannels),
format: 'f32-planar',
sampleRate: audioSample.sampleRate,
numberOfChannels: audioSample.numberOfChannels,
numberOfFrames: sampleCount,
timestamp: this.lastEndSampleIndex / audioSample.sampleRate,
});
await this.add(fillSample, true); // Recursive call
}
this.lastEndSampleIndex = endSampleIndex;
}
if (this.customEncoder) { if (this.customEncoder) {
this.customEncoderQueueSize++; this.customEncoderQueueSize++;
@@ -1515,6 +1550,32 @@ class AudioEncoderWrapper {
this.encoder = new AudioEncoder({ this.encoder = new AudioEncoder({
output: (chunk, meta) => { output: (chunk, meta) => {
// WebKit emits an invalid description for AAC (https://bugs.webkit.org/show_bug.cgi?id=302253),
// which we try to detect here. If detected, we'll provide our own description instead, derived
// from the codec string and audio parameters.
if (this.encodingConfig.codec === 'aac' && meta?.decoderConfig) {
let needsDescriptionOverwrite = false;
if (!meta.decoderConfig.description || meta.decoderConfig.description.byteLength < 2) {
needsDescriptionOverwrite = true;
} else {
const audioSpecificConfig = parseAacAudioSpecificConfig(
toUint8Array(meta.decoderConfig.description),
);
needsDescriptionOverwrite = audioSpecificConfig.objectType === 0;
}
if (needsDescriptionOverwrite) {
const objectType = Number(last(encoderConfig.codec.split('.')));
meta.decoderConfig.description = buildAacAudioSpecificConfig({
objectType,
numberOfChannels: meta.decoderConfig.numberOfChannels,
sampleRate: meta.decoderConfig.sampleRate,
});
}
}
const packet = EncodedPacket.fromEncodedChunk(chunk); const packet = EncodedPacket.fromEncodedChunk(chunk);
this.encodingConfig.onEncodedPacket?.(packet, meta); this.encodingConfig.onEncodedPacket?.(packet, meta);
+7 -12
View File
@@ -658,21 +658,16 @@ export class CallSerializer {
} }
} }
let isSafariCache: boolean | null = null; let isWebKitCache: boolean | null = null;
export const isSafari = () => { export const isWebKit = () => {
if (isSafariCache !== null) { if (isWebKitCache !== null) {
return isSafariCache; return isWebKitCache;
} }
const result = !!( // This even returns true for WebKit-wrapping browsers such as Chrome on iOS
typeof navigator !== 'undefined' const result = !!(typeof navigator !== 'undefined' && navigator.vendor?.match(/apple/i));
&& navigator.vendor?.match(/apple/i)
&& !navigator.userAgent?.match(/crios/i)
&& !navigator.userAgent?.match(/fxios/i)
&& !navigator.userAgent?.match(/Opera|OPT\//)
);
isSafariCache = result; isWebKitCache = result;
return result; return result;
}; };
+4 -1
View File
@@ -126,7 +126,10 @@ export class EncodedPacket {
} }
} }
/** If this packet is a metadata-only packet. Metadata-only packets don't contain their packet data. */ /**
* If this packet is a metadata-only packet. Metadata-only packets don't contain their packet data. They are the
* result of retrieving packets with {@link PacketRetrievalOptions.metadataOnly} set to `true`.
*/
get isMetadataOnly() { get isMetadataOnly() {
return this.data === PLACEHOLDER_DATA; return this.data === PLACEHOLDER_DATA;
} }
+4 -1
View File
@@ -552,7 +552,6 @@ export class VideoSample implements Disposable {
dHeight, dHeight,
); );
// Restore the previous transformation state
context.restore(); context.restore();
} }
@@ -644,6 +643,8 @@ export class VideoSample implements Disposable {
dy = (canvasHeight - newHeight) / 2; dy = (canvasHeight - newHeight) / 2;
} }
context.save();
const aspectRatioChange = rotation % 180 === 0 ? 1 : newWidth / newHeight; const aspectRatioChange = rotation % 180 === 0 ? 1 : newWidth / newHeight;
context.translate(canvasWidth / 2, canvasHeight / 2); context.translate(canvasWidth / 2, canvasHeight / 2);
context.rotate(rotation * Math.PI / 180); context.rotate(rotation * Math.PI / 180);
@@ -655,6 +656,8 @@ export class VideoSample implements Disposable {
// Important that we don't use .draw() here since that would take rotation into account, but we wanna handle it // Important that we don't use .draw() here since that would take rotation into account, but we wanna handle it
// ourselves here // ourselves here
context.drawImage(this.toCanvasImageSource(), sx, sy, sWidth, sHeight, dx, dy, newWidth, newHeight); context.drawImage(this.toCanvasImageSource(), sx, sy, sWidth, sHeight, dx, dy, newWidth, newHeight);
context.restore();
} }
/** @internal */ /** @internal */
+9 -1
View File
@@ -12,6 +12,7 @@ import {
binarySearchLessOrEqual, binarySearchLessOrEqual,
closedIntervalsOverlap, closedIntervalsOverlap,
isNumber, isNumber,
isWebKit,
MaybePromise, MaybePromise,
mergeRequestInit, mergeRequestInit,
promiseWithResolvers, promiseWithResolvers,
@@ -209,7 +210,14 @@ export class BlobSource extends Source {
private async _runWorker(worker: ReadWorker) { private async _runWorker(worker: ReadWorker) {
let reader = this._readers.get(worker); let reader = this._readers.get(worker);
if (reader === undefined) { if (reader === undefined) {
if ('stream' in this._blob) { // https://github.com/Vanilagy/mediabunny/issues/184
// WebKit has critical bugs with blob.stream():
// - WebKitBlobResource error 1 when streaming large files
// - Memory buildup and reload loops on iOS (network process crashes)
// - ReadableStream stalls under backpressure (especially video)
// Affects Safari and all iOS browsers (Chrome, Firefox, etc.).
// Use arrayBuffer() fallback for WebKit browsers.
if ('stream' in this._blob && !isWebKit()) {
// Get a reader of the blob starting at the required offset, and then keep it around // Get a reader of the blob starting at the required offset, and then keep it around
const slice = this._blob.slice(worker.currentPos); const slice = this._blob.slice(worker.currentPos);
reader = slice.stream().getReader(); reader = slice.stream().getReader();