diff --git a/dev/convert.html b/dev/convert.html
index bc1f9d1..79b7ae3 100644
--- a/dev/convert.html
+++ b/dev/convert.html
@@ -48,6 +48,7 @@
target
});
+ let ctx = null;
const conversion = await Mediabunny.Conversion.init({
input: new Mediabunny.Input({
formats: Mediabunny.ALL_FORMATS,
@@ -56,6 +57,7 @@
output,
audio: (_, n) => ({
discard: n > 1,
+ //codec: 'opus',
/*
process: (sample) => {
return sample;
@@ -97,7 +99,7 @@
},
*/
video: () => ({
- /*
+ //discard: true,
process: (sample) => {
if (!ctx) {
// Create a canvas for image compositing
@@ -108,13 +110,14 @@
ctx = canvas.getContext('2d');
}
+ console.log(ctx.canvas.width, ctx.canvas.height);
+
ctx.clearRect(0, 0, ctx.canvas.width, ctx.canvas.height);
- sample.draw(ctx, 0, 0);
- ctx.drawImage(watermark, 32, 32);
+ sample.drawWithFit(ctx, { fit: 'fill' });
+ //ctx.drawImage(watermark, 32, 32);
return ctx.canvas;
},
- */
//width: 300,
//alpha: 'keep',
//width: 320,
@@ -170,8 +173,8 @@
}
},
trim: {
- start: 0,
- end: 10
+ //start: 0,
+ //end: 10
},
});
console.log(conversion);
diff --git a/dev/demux.html b/dev/demux.html
index 65b062a..fc1e694 100644
--- a/dev/demux.html
+++ b/dev/demux.html
@@ -45,7 +45,34 @@
source: new Mediabunny.BlobSource(file),
});
- const videoTrack = await input.getPrimaryVideoTrack();
+ const videoTrack = await input.getPrimaryAudioTrack();
+ const sink = new Mediabunny.EncodedPacketSink(videoTrack);
+
+ for await (const packet of sink.packets()) {
+ console.log(packet.timestamp, packet.duration, packet.timestamp + packet.duration);
+
+ if (packet.timestamp > 10) break;
+ }
+
+ /*
+ const sink = new Mediabunny.VideoSampleSink(videoTrack);
+
+ for await (const sample of sink.samples()) {
+ console.log(sample.timestamp, sample.duration);
+ sample.close();
+ }
+ */
+
+ /*
+ const sink = new Mediabunny.EncodedPacketSink(videoTrack);
+
+ for await (const packet of sink.packets()) {
+ console.log(packet.timestamp, packet.timestamp + packet.duration);
+ }
+ */
+
+
+ /*
const sink = new Mediabunny.VideoSampleSink(videoTrack);
console.log(await videoTrack.getDecoderConfig())
@@ -61,6 +88,7 @@
for await (const sample of sink.samples()) {
console.log(sample);
}
+ */
/*
let total = 0;
diff --git a/package-lock.json b/package-lock.json
index 76daac5..271da7e 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "mediabunny",
- "version": "1.24.3",
+ "version": "1.24.5",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "mediabunny",
- "version": "1.24.3",
+ "version": "1.24.5",
"license": "MPL-2.0",
"workspaces": [
"packages/*"
@@ -7749,9 +7749,9 @@
}
},
"node_modules/mediabunny": {
- "version": "1.24.2",
- "resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.24.2.tgz",
- "integrity": "sha512-+H2Jha7D8C92F1fysdN+nmBBAOgVV+pNv/Pxc+c//qo2E/idnv8MsZrft7VJsFE5WjK30ESkwW5ezM0oceFIkg==",
+ "version": "1.24.4",
+ "resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.24.4.tgz",
+ "integrity": "sha512-dpWYBPTtMg152yNLXZQ7xb6hsXdYbKp9EuK8qq4npS+SZ08FVc1XHlXYhrOm31T+tUVJKgm95Yaqy69wTpZP9Q==",
"license": "MPL-2.0",
"peer": true,
"workspaces": [
@@ -12242,7 +12242,7 @@
},
"packages/mp3-encoder": {
"name": "@mediabunny/mp3-encoder",
- "version": "1.24.3",
+ "version": "1.24.5",
"license": "MPL-2.0",
"devDependencies": {
"@types/emscripten": "^1.40.1"
diff --git a/package.json b/package.json
index 98efb29..3b11949 100644
--- a/package.json
+++ b/package.json
@@ -1,7 +1,7 @@
{
"name": "mediabunny",
"author": "Vanilagy",
- "version": "1.24.3",
+ "version": "1.24.5",
"description": "Pure TypeScript media toolkit for reading, writing, and converting media files, directly in the browser.",
"type": "module",
"workspaces": [
diff --git a/packages/mp3-encoder/package.json b/packages/mp3-encoder/package.json
index c5acbf8..b7d133b 100644
--- a/packages/mp3-encoder/package.json
+++ b/packages/mp3-encoder/package.json
@@ -1,7 +1,7 @@
{
"name": "@mediabunny/mp3-encoder",
"author": "Vanilagy",
- "version": "1.24.3",
+ "version": "1.24.5",
"description": "MP3 encoder extension for Mediabunny, based on LAME.",
"main": "./dist/bundles/mediabunny-mp3-encoder.mjs",
"module": "./dist/bundles/mediabunny-mp3-encoder.mjs",
diff --git a/src/codec.ts b/src/codec.ts
index 2e90cfd..a396e70 100644
--- a/src/codec.ts
+++ b/src/codec.ts
@@ -621,6 +621,54 @@ export const parseAacAudioSpecificConfig = (bytes: Uint8Array | null): AacAudioS
};
};
+export const buildAacAudioSpecificConfig = (config: {
+ objectType: number;
+ sampleRate: number;
+ numberOfChannels: number;
+}) => {
+ let frequencyIndex = aacFrequencyTable.indexOf(config.sampleRate);
+ let customSampleRate: number | null = null;
+
+ if (frequencyIndex === -1) {
+ frequencyIndex = 15;
+ customSampleRate = config.sampleRate;
+ }
+
+ const channelConfiguration = aacChannelMap.indexOf(config.numberOfChannels);
+ if (channelConfiguration === -1) {
+ throw new TypeError(`Unsupported number of channels: ${config.numberOfChannels}`);
+ }
+
+ let bitCount = 5 + 4 + 4;
+ if (config.objectType >= 32) {
+ bitCount += 6;
+ }
+ if (frequencyIndex === 15) {
+ bitCount += 24;
+ }
+
+ const byteCount = Math.ceil(bitCount / 8);
+ const bytes = new Uint8Array(byteCount);
+ const bitstream = new Bitstream(bytes);
+
+ if (config.objectType < 32) {
+ bitstream.writeBits(5, config.objectType);
+ } else {
+ bitstream.writeBits(5, 31);
+ bitstream.writeBits(6, config.objectType - 32);
+ }
+
+ bitstream.writeBits(4, frequencyIndex);
+
+ if (frequencyIndex === 15) {
+ bitstream.writeBits(24, customSampleRate!);
+ }
+
+ bitstream.writeBits(4, channelConfiguration);
+
+ return bytes;
+};
+
export const OPUS_SAMPLE_RATE = 48_000;
const PCM_CODEC_REGEX = /^pcm-([usf])(\d+)+(be)?$/;
diff --git a/src/conversion.ts b/src/conversion.ts
index abf820a..0cabe1a 100644
--- a/src/conversion.ts
+++ b/src/conversion.ts
@@ -182,7 +182,8 @@ export type ConversionVideoOptions = {
* corrections.
*
* Must return a {@link VideoSample} or a `CanvasImageSource`, an array of them, or `null` for dropping the frame.
- * When non-timestamped data is returned, the timestamp and duration from the source sample will be used.
+ * When non-timestamped data is returned, the timestamp and duration from the source sample will be used. Rotation
+ * metadata of the returned sample will be ignored.
*
* This function can also be used to manually resize frames. When doing so, you should signal the post-process
* dimensions using the `processedWidth` and `processedHeight` fields, which enables the encoder to better know what
@@ -873,7 +874,10 @@ export class Conversion {
|| trackOptions.process !== undefined;
let needsRerender = width !== originalWidth
|| height !== originalHeight
- || (totalRotation !== 0 && !outputSupportsRotation)
+ // TODO This is suboptimal: Forcing a rerender when both rotation and process are set is not
+ // performance-optimal, but right now there's no other way because we can't change the track rotation
+ // metadata after the output has already started. Should be possible with API changes in v2, though!
+ || (totalRotation !== 0 && (!outputSupportsRotation || trackOptions.process !== undefined))
|| !!crop;
const alpha = trackOptions.alpha ?? 'discard';
diff --git a/src/flac/flac-demuxer.ts b/src/flac/flac-demuxer.ts
index bf71ee4..5248394 100644
--- a/src/flac/flac-demuxer.ts
+++ b/src/flac/flac-demuxer.ts
@@ -339,16 +339,34 @@ export class FlacDemuxer extends Demuxer {
slice.skip(-2);
const lengthIfNextFlacFrameHeaderIsLegit = slice.filePos - startPos;
- const nextIsLegit = this.readFlacFrameHeader({
+ const nextFrameHeader = this.readFlacFrameHeader({
slice,
isFirstPacket: false,
});
- if (!nextIsLegit) {
+ if (!nextFrameHeader) {
slice.skip(-1);
continue;
}
+ // Ensure the frameOrSampleNum is consecutive.
+ // https://github.com/Vanilagy/mediabunny/issues/194
+
+ if (this.blockingBit === 0) {
+ // Case A: If the stream is fixed block size, this is the frame number, which increments by 1
+ if (nextFrameHeader.num - frameHeader.num !== 1) {
+ slice.skip(-1);
+ continue;
+ }
+ } else {
+ // Case B: If the stream is variable block size, this is the sample number, which increments by
+ // amount of samples in a frame.
+ if (nextFrameHeader.num - frameHeader.num !== frameHeader.blockSize) {
+ slice.skip(-1);
+ continue;
+ }
+ }
+
return {
num: frameHeader.num,
blockSize: frameHeader.blockSize,
@@ -442,6 +460,11 @@ export class FlacDemuxer extends Demuxer {
return null;
}
+ if (sampleRate !== this.audioInfo.sampleRate) {
+ // This cannot be a valid FLAC frame, the sample rate is not the same as in the stream info
+ return null;
+ }
+
const size = slice.filePos - startOffset;
const crc = readU8(slice);
diff --git a/src/matroska/matroska-demuxer.ts b/src/matroska/matroska-demuxer.ts
index c9655d5..e964bef 100644
--- a/src/matroska/matroska-demuxer.ts
+++ b/src/matroska/matroska-demuxer.ts
@@ -844,13 +844,13 @@ export class MatroskaDemuxer extends Demuxer {
blocks.splice(blockIndex, 1); // Remove the original block
+ const blockDuration = originalBlock.duration || frameCount * (track.defaultDuration ?? 0);
+
// Now, let's insert each frame as its own block
for (let i = 0; i < frameCount; i++) {
const frameSize = frameSizes[i]!;
const frameData = readBytes(slice, frameSize);
- const blockDuration = originalBlock.duration || (frameCount * (track.defaultDuration ?? 0));
-
// Distribute timestamps evenly across the block duration
const frameTimestamp = originalBlock.timestamp + (blockDuration * i / frameCount);
const frameDuration = blockDuration / frameCount;
diff --git a/src/media-sink.ts b/src/media-sink.ts
index 1d0665a..195ff2a 100644
--- a/src/media-sink.ts
+++ b/src/media-sink.ts
@@ -26,7 +26,7 @@ import {
insertSorted,
isFirefox,
isNumber,
- isSafari,
+ isWebKit,
last,
mapAsyncGenerator,
promiseWithResolvers,
@@ -918,7 +918,7 @@ class VideoDecoderWrapper extends DecoderWrapper {
} else {
assert(this.decoder);
- if (!isSafari()) {
+ if (!isWebKit()) {
insertSorted(this.inputTimestamps, packet.timestamp, x => x);
}
@@ -1049,7 +1049,7 @@ class VideoDecoderWrapper extends DecoderWrapper {
/** Handler for the WebCodecs VideoDecoder for ironing out browser differences. */
sampleHandler(sample: VideoSample) {
- if (isSafari()) {
+ if (isWebKit()) {
// For correct B-frame handling, we don't just hand over the frames directly but instead add them to
// a queue, because we want to ensure frames are emitted in presentation order. We flush the queue
// each time we receive a frame with a timestamp larger than the highest we've seen so far, as we
@@ -1137,7 +1137,7 @@ class VideoDecoderWrapper extends DecoderWrapper {
this.alphaRaslSkipped = false;
}
- if (isSafari()) {
+ if (isWebKit()) {
for (const sample of this.sampleQueue) {
this.finalizeAndEmitSample(sample);
}
diff --git a/src/media-source.ts b/src/media-source.ts
index c89b24a..86989a2 100644
--- a/src/media-source.ts
+++ b/src/media-source.ts
@@ -9,6 +9,8 @@
import {
AUDIO_CODECS,
AudioCodec,
+ buildAacAudioSpecificConfig,
+ parseAacAudioSpecificConfig,
parsePcmCodec,
PCM_AUDIO_CODECS,
PcmAudioCodec,
@@ -24,9 +26,11 @@ import {
CallSerializer,
clamp,
isFirefox,
+ last,
promiseWithResolvers,
setInt24,
setUint24,
+ toUint8Array,
} from './misc';
import { Muxer } from './muxer';
import { SubtitleParser } from './subtitles';
@@ -1515,6 +1519,32 @@ class AudioEncoderWrapper {
this.encoder = new AudioEncoder({
output: (chunk, meta) => {
+ // WebKit emits an invalid description for AAC (https://bugs.webkit.org/show_bug.cgi?id=302253),
+ // which we try to detect here. If detected, we'll provide our own description instead, derived
+ // from the codec string and audio parameters.
+ if (this.encodingConfig.codec === 'aac' && meta?.decoderConfig) {
+ let needsDescriptionOverwrite = false;
+ if (!meta.decoderConfig.description || meta.decoderConfig.description.byteLength < 2) {
+ needsDescriptionOverwrite = true;
+ } else {
+ const audioSpecificConfig = parseAacAudioSpecificConfig(
+ toUint8Array(meta.decoderConfig.description),
+ );
+
+ needsDescriptionOverwrite = audioSpecificConfig.objectType === 0;
+ }
+
+ if (needsDescriptionOverwrite) {
+ const objectType = Number(last(encoderConfig.codec.split('.')));
+
+ meta.decoderConfig.description = buildAacAudioSpecificConfig({
+ objectType,
+ numberOfChannels: meta.decoderConfig.numberOfChannels,
+ sampleRate: meta.decoderConfig.sampleRate,
+ });
+ }
+ }
+
const packet = EncodedPacket.fromEncodedChunk(chunk);
this.encodingConfig.onEncodedPacket?.(packet, meta);
diff --git a/src/misc.ts b/src/misc.ts
index 6bdb52b..c6f5119 100644
--- a/src/misc.ts
+++ b/src/misc.ts
@@ -658,21 +658,16 @@ export class CallSerializer {
}
}
-let isSafariCache: boolean | null = null;
-export const isSafari = () => {
- if (isSafariCache !== null) {
- return isSafariCache;
+let isWebKitCache: boolean | null = null;
+export const isWebKit = () => {
+ if (isWebKitCache !== null) {
+ return isWebKitCache;
}
- const result = !!(
- typeof navigator !== 'undefined'
- && navigator.vendor?.match(/apple/i)
- && !navigator.userAgent?.match(/crios/i)
- && !navigator.userAgent?.match(/fxios/i)
- && !navigator.userAgent?.match(/Opera|OPT\//)
- );
+ // This even returns true for WebKit-wrapping browsers such as Chrome on iOS
+ const result = !!(typeof navigator !== 'undefined' && navigator.vendor?.match(/apple/i));
- isSafariCache = result;
+ isWebKitCache = result;
return result;
};
diff --git a/src/packet.ts b/src/packet.ts
index c71890c..f5de60e 100644
--- a/src/packet.ts
+++ b/src/packet.ts
@@ -126,7 +126,10 @@ export class EncodedPacket {
}
}
- /** If this packet is a metadata-only packet. Metadata-only packets don't contain their packet data. */
+ /**
+ * If this packet is a metadata-only packet. Metadata-only packets don't contain their packet data. They are the
+ * result of retrieving packets with {@link PacketRetrievalOptions.metadataOnly} set to `true`.
+ */
get isMetadataOnly() {
return this.data === PLACEHOLDER_DATA;
}
diff --git a/src/sample.ts b/src/sample.ts
index 2a9bb43..770f450 100644
--- a/src/sample.ts
+++ b/src/sample.ts
@@ -591,7 +591,6 @@ export class VideoSample implements Disposable {
dHeight,
);
- // Restore the previous transformation state
context.restore();
}
@@ -690,6 +689,8 @@ export class VideoSample implements Disposable {
dy = (canvasHeight - newHeight) / 2;
}
+ context.save();
+
const aspectRatioChange = relativeRotation % 180 === 0 ? 1 : newWidth / newHeight;
context.translate(canvasWidth / 2, canvasHeight / 2);
context.rotate(relativeRotation * Math.PI / 180);
@@ -701,6 +702,8 @@ export class VideoSample implements Disposable {
// Important that we don't use .draw() here since that would take rotation into account, but we wanna handle it
// ourselves here
context.drawImage(source, sx, sy, sWidth, sHeight, dx, dy, newWidth, newHeight);
+
+ context.restore();
}
/** @internal */
diff --git a/src/source.ts b/src/source.ts
index adafa2e..68ebac1 100644
--- a/src/source.ts
+++ b/src/source.ts
@@ -12,6 +12,7 @@ import {
binarySearchLessOrEqual,
closedIntervalsOverlap,
isNumber,
+ isWebKit,
MaybePromise,
mergeRequestInit,
promiseWithResolvers,
@@ -209,7 +210,14 @@ export class BlobSource extends Source {
private async _runWorker(worker: ReadWorker) {
let reader = this._readers.get(worker);
if (reader === undefined) {
- if ('stream' in this._blob) {
+ // https://github.com/Vanilagy/mediabunny/issues/184
+ // WebKit has critical bugs with blob.stream():
+ // - WebKitBlobResource error 1 when streaming large files
+ // - Memory buildup and reload loops on iOS (network process crashes)
+ // - ReadableStream stalls under backpressure (especially video)
+ // Affects Safari and all iOS browsers (Chrome, Firefox, etc.).
+ // Use arrayBuffer() fallback for WebKit browsers.
+ if ('stream' in this._blob && !isWebKit()) {
// Get a reader of the blob starting at the required offset, and then keep it around
const slice = this._blob.slice(worker.currentPos);
reader = slice.stream().getReader();
diff --git a/test/browser/transparency.test.ts b/test/browser/transparency.test.ts
index e460e55..35527ae 100644
--- a/test/browser/transparency.test.ts
+++ b/test/browser/transparency.test.ts
@@ -148,8 +148,8 @@ test('Can encode transparent video', async () => {
// Red (within error)
expect(imageData.data[index2 + 0]).greaterThanOrEqual(253);
- expect(imageData.data[index2 + 1]).lessThanOrEqual(2);
- expect(imageData.data[index2 + 2]).lessThanOrEqual(2);
+ expect(imageData.data[index2 + 1]).lessThanOrEqual(3);
+ expect(imageData.data[index2 + 2]).lessThanOrEqual(3);
expect(imageData.data[index2 + 3]).toBe(255); // Opaque