Merge branch 'main' into fork-branch

dev/convert.html

@@ -48,6 +48,7 @@
 			target
 		});
 
+		let ctx = null;
 		const conversion = await Mediabunny.Conversion.init({
 			input: new Mediabunny.Input({
 				formats: Mediabunny.ALL_FORMATS,	
@@ -56,6 +57,7 @@
 			output,
 			audio: (_, n) => ({
 				discard: n > 1,
+				//codec: 'opus',
 				/*
 				process: (sample) => {
 					return sample;
@@ -97,7 +99,7 @@
 			},
 			*/
 			video: () => ({
-				/*
+				//discard: true,
 				process: (sample) => {
 					if (!ctx) {
 						// Create a canvas for image compositing
@@ -108,13 +110,14 @@
 						ctx = canvas.getContext('2d');
 					}
 
+					console.log(ctx.canvas.width, ctx.canvas.height);
+
 					ctx.clearRect(0, 0, ctx.canvas.width, ctx.canvas.height);
-					sample.draw(ctx, 0, 0);
+					sample.drawWithFit(ctx, { fit: 'fill' });
-					ctx.drawImage(watermark, 32, 32);
+					//ctx.drawImage(watermark, 32, 32);
 
 					return ctx.canvas;
 				},
-				*/
 				//width: 300,
 				//alpha: 'keep',
 				//width: 320,
@@ -170,8 +173,8 @@
 				}
 			},
 			trim: {
-				start: 0,
+				//start: 0,
-				end: 10
+				//end: 10
 			},
 		});
 		console.log(conversion);

dev/demux.html

@@ -45,7 +45,34 @@
 			source: new Mediabunny.BlobSource(file),
 		});
 
-		const videoTrack = await input.getPrimaryVideoTrack();
+		const videoTrack = await input.getPrimaryAudioTrack();
+		const sink = new Mediabunny.EncodedPacketSink(videoTrack);
+
+		for await (const packet of sink.packets()) {
+			console.log(packet.timestamp, packet.duration, packet.timestamp + packet.duration);
+
+			if (packet.timestamp > 10) break;
+		}
+
+		/*
+		const sink = new Mediabunny.VideoSampleSink(videoTrack);
+
+		for await (const sample of sink.samples()) {
+			console.log(sample.timestamp, sample.duration);
+			sample.close();
+		}
+		*/
+
+		/*
+		const sink = new Mediabunny.EncodedPacketSink(videoTrack);
+
+		for await (const packet of sink.packets()) {
+			console.log(packet.timestamp, packet.timestamp + packet.duration);
+		}
+		*/
+
+
+		/*
 		const sink = new Mediabunny.VideoSampleSink(videoTrack);
 
 		console.log(await videoTrack.getDecoderConfig())
@@ -61,6 +88,7 @@
 		for await (const sample of sink.samples()) {
 			console.log(sample);
 		}
+		*/
 
 		/*
 		let total = 0;

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "mediabunny",
-  "version": "1.24.3",
+  "version": "1.24.5",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "mediabunny",
-      "version": "1.24.3",
+      "version": "1.24.5",
       "license": "MPL-2.0",
       "workspaces": [
         "packages/*"
@@ -7749,9 +7749,9 @@
       }
     },
     "node_modules/mediabunny": {
-      "version": "1.24.2",
+      "version": "1.24.4",
-      "resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.24.2.tgz",
+      "resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.24.4.tgz",
-      "integrity": "sha512-+H2Jha7D8C92F1fysdN+nmBBAOgVV+pNv/Pxc+c//qo2E/idnv8MsZrft7VJsFE5WjK30ESkwW5ezM0oceFIkg==",
+      "integrity": "sha512-dpWYBPTtMg152yNLXZQ7xb6hsXdYbKp9EuK8qq4npS+SZ08FVc1XHlXYhrOm31T+tUVJKgm95Yaqy69wTpZP9Q==",
       "license": "MPL-2.0",
       "peer": true,
       "workspaces": [
@@ -12242,7 +12242,7 @@
     },
     "packages/mp3-encoder": {
       "name": "@mediabunny/mp3-encoder",
-      "version": "1.24.3",
+      "version": "1.24.5",
       "license": "MPL-2.0",
       "devDependencies": {
         "@types/emscripten": "^1.40.1"

package.json

@@ -1,7 +1,7 @@
 {
   "name": "mediabunny",
   "author": "Vanilagy",
-  "version": "1.24.3",
+  "version": "1.24.5",
   "description": "Pure TypeScript media toolkit for reading, writing, and converting media files, directly in the browser.",
   "type": "module",
   "workspaces": [

packages/mp3-encoder/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@mediabunny/mp3-encoder",
   "author": "Vanilagy",
-  "version": "1.24.3",
+  "version": "1.24.5",
   "description": "MP3 encoder extension for Mediabunny, based on LAME.",
   "main": "./dist/bundles/mediabunny-mp3-encoder.mjs",
   "module": "./dist/bundles/mediabunny-mp3-encoder.mjs",

src/codec.ts

@@ -621,6 +621,54 @@ export const parseAacAudioSpecificConfig = (bytes: Uint8Array | null): AacAudioS
 	};
 };
 
+export const buildAacAudioSpecificConfig = (config: {
+	objectType: number;
+	sampleRate: number;
+	numberOfChannels: number;
+}) => {
+	let frequencyIndex = aacFrequencyTable.indexOf(config.sampleRate);
+	let customSampleRate: number | null = null;
+
+	if (frequencyIndex === -1) {
+		frequencyIndex = 15;
+		customSampleRate = config.sampleRate;
+	}
+
+	const channelConfiguration = aacChannelMap.indexOf(config.numberOfChannels);
+	if (channelConfiguration === -1) {
+		throw new TypeError(`Unsupported number of channels: ${config.numberOfChannels}`);
+	}
+
+	let bitCount = 5 + 4 + 4;
+	if (config.objectType >= 32) {
+		bitCount += 6;
+	}
+	if (frequencyIndex === 15) {
+		bitCount += 24;
+	}
+
+	const byteCount = Math.ceil(bitCount / 8);
+	const bytes = new Uint8Array(byteCount);
+	const bitstream = new Bitstream(bytes);
+
+	if (config.objectType < 32) {
+		bitstream.writeBits(5, config.objectType);
+	} else {
+		bitstream.writeBits(5, 31);
+		bitstream.writeBits(6, config.objectType - 32);
+	}
+
+	bitstream.writeBits(4, frequencyIndex);
+
+	if (frequencyIndex === 15) {
+		bitstream.writeBits(24, customSampleRate!);
+	}
+
+	bitstream.writeBits(4, channelConfiguration);
+
+	return bytes;
+};
+
 export const OPUS_SAMPLE_RATE = 48_000;
 
 const PCM_CODEC_REGEX = /^pcm-([usf])(\d+)+(be)?$/;

src/conversion.ts

@@ -182,7 +182,8 @@ export type ConversionVideoOptions = {
 	 * corrections.
 	 *
 	 * Must return a {@link VideoSample} or a `CanvasImageSource`, an array of them, or `null` for dropping the frame.
-	 * When non-timestamped data is returned, the timestamp and duration from the source sample will be used.
+	 * When non-timestamped data is returned, the timestamp and duration from the source sample will be used. Rotation
+	 * metadata of the returned sample will be ignored.
 	 *
 	 * This function can also be used to manually resize frames. When doing so, you should signal the post-process
 	 * dimensions using the `processedWidth` and `processedHeight` fields, which enables the encoder to better know what
@@ -873,7 +874,10 @@ export class Conversion {
 			|| trackOptions.process !== undefined;
 		let needsRerender = width !== originalWidth
 			|| height !== originalHeight
-			|| (totalRotation !== 0 && !outputSupportsRotation)
+			// TODO This is suboptimal: Forcing a rerender when both rotation and process are set is not
+			// performance-optimal, but right now there's no other way because we can't change the track rotation
+			// metadata after the output has already started. Should be possible with API changes in v2, though!
+			|| (totalRotation !== 0 && (!outputSupportsRotation || trackOptions.process !== undefined))
 			|| !!crop;
 
 		const alpha = trackOptions.alpha ?? 'discard';

src/flac/flac-demuxer.ts

@@ -339,16 +339,34 @@ export class FlacDemuxer extends Demuxer {
 				slice.skip(-2);
 				const lengthIfNextFlacFrameHeaderIsLegit = slice.filePos - startPos;
 
-				const nextIsLegit = this.readFlacFrameHeader({
+				const nextFrameHeader = this.readFlacFrameHeader({
 					slice,
 					isFirstPacket: false,
 				});
 
-				if (!nextIsLegit) {
+				if (!nextFrameHeader) {
 					slice.skip(-1);
 					continue;
 				}
 
+				// Ensure the frameOrSampleNum is consecutive.
+				// https://github.com/Vanilagy/mediabunny/issues/194
+
+				if (this.blockingBit === 0) {
+					// Case A: If the stream is fixed block size, this is the frame number, which increments by 1
+					if (nextFrameHeader.num - frameHeader.num !== 1) {
+						slice.skip(-1);
+						continue;
+					}
+				} else {
+					// Case B: If the stream is variable block size, this is the sample number, which increments by
+					// amount of samples in a frame.
+					if (nextFrameHeader.num - frameHeader.num !== frameHeader.blockSize) {
+						slice.skip(-1);
+						continue;
+					}
+				}
+
 				return {
 					num: frameHeader.num,
 					blockSize: frameHeader.blockSize,
@@ -442,6 +460,11 @@ export class FlacDemuxer extends Demuxer {
 			return null;
 		}
 
+		if (sampleRate !== this.audioInfo.sampleRate) {
+			// This cannot be a valid FLAC frame, the sample rate is not the same as in the stream info
+			return null;
+		}
+
 		const size = slice.filePos - startOffset;
 		const crc = readU8(slice);
 

src/matroska/matroska-demuxer.ts

@@ -844,13 +844,13 @@ export class MatroskaDemuxer extends Demuxer {
 
 			blocks.splice(blockIndex, 1); // Remove the original block
 
+			const blockDuration = originalBlock.duration || frameCount * (track.defaultDuration ?? 0);
+
 			// Now, let's insert each frame as its own block
 			for (let i = 0; i < frameCount; i++) {
 				const frameSize = frameSizes[i]!;
 				const frameData = readBytes(slice, frameSize);
 
-				const blockDuration = originalBlock.duration || (frameCount * (track.defaultDuration ?? 0));
-
 				// Distribute timestamps evenly across the block duration
 				const frameTimestamp = originalBlock.timestamp + (blockDuration * i / frameCount);
 				const frameDuration = blockDuration / frameCount;

src/media-sink.ts

@@ -26,7 +26,7 @@ import {
 	insertSorted,
 	isFirefox,
 	isNumber,
-	isSafari,
+	isWebKit,
 	last,
 	mapAsyncGenerator,
 	promiseWithResolvers,
@@ -918,7 +918,7 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
 		} else {
 			assert(this.decoder);
 
-			if (!isSafari()) {
+			if (!isWebKit()) {
 				insertSorted(this.inputTimestamps, packet.timestamp, x => x);
 			}
 
@@ -1049,7 +1049,7 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
 
 	/** Handler for the WebCodecs VideoDecoder for ironing out browser differences. */
 	sampleHandler(sample: VideoSample) {
-		if (isSafari()) {
+		if (isWebKit()) {
 			// For correct B-frame handling, we don't just hand over the frames directly but instead add them to
 			// a queue, because we want to ensure frames are emitted in presentation order. We flush the queue
 			// each time we receive a frame with a timestamp larger than the highest we've seen so far, as we
@@ -1137,7 +1137,7 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
 			this.alphaRaslSkipped = false;
 		}
 
-		if (isSafari()) {
+		if (isWebKit()) {
 			for (const sample of this.sampleQueue) {
 				this.finalizeAndEmitSample(sample);
 			}

src/media-source.ts

@@ -9,6 +9,8 @@
 import {
 	AUDIO_CODECS,
 	AudioCodec,
+	buildAacAudioSpecificConfig,
+	parseAacAudioSpecificConfig,
 	parsePcmCodec,
 	PCM_AUDIO_CODECS,
 	PcmAudioCodec,
@@ -24,9 +26,11 @@ import {
 	CallSerializer,
 	clamp,
 	isFirefox,
+	last,
 	promiseWithResolvers,
 	setInt24,
 	setUint24,
+	toUint8Array,
 } from './misc';
 import { Muxer } from './muxer';
 import { SubtitleParser } from './subtitles';
@@ -1515,6 +1519,32 @@ class AudioEncoderWrapper {
 
 				this.encoder = new AudioEncoder({
 					output: (chunk, meta) => {
+						// WebKit emits an invalid description for AAC (https://bugs.webkit.org/show_bug.cgi?id=302253),
+						// which we try to detect here. If detected, we'll provide our own description instead, derived
+						// from the codec string and audio parameters.
+						if (this.encodingConfig.codec === 'aac' && meta?.decoderConfig) {
+							let needsDescriptionOverwrite = false;
+							if (!meta.decoderConfig.description || meta.decoderConfig.description.byteLength < 2) {
+								needsDescriptionOverwrite = true;
+							} else {
+								const audioSpecificConfig = parseAacAudioSpecificConfig(
+									toUint8Array(meta.decoderConfig.description),
+								);
+
+								needsDescriptionOverwrite = audioSpecificConfig.objectType === 0;
+							}
+
+							if (needsDescriptionOverwrite) {
+								const objectType = Number(last(encoderConfig.codec.split('.')));
+
+								meta.decoderConfig.description = buildAacAudioSpecificConfig({
+									objectType,
+									numberOfChannels: meta.decoderConfig.numberOfChannels,
+									sampleRate: meta.decoderConfig.sampleRate,
+								});
+							}
+						}
+
 						const packet = EncodedPacket.fromEncodedChunk(chunk);
 
 						this.encodingConfig.onEncodedPacket?.(packet, meta);

src/misc.ts

@@ -658,21 +658,16 @@ export class CallSerializer {
 	}
 }
 
-let isSafariCache: boolean | null = null;
+let isWebKitCache: boolean | null = null;
-export const isSafari = () => {
+export const isWebKit = () => {
-	if (isSafariCache !== null) {
+	if (isWebKitCache !== null) {
-		return isSafariCache;
+		return isWebKitCache;
 	}
 
-	const result = !!(
+	// This even returns true for WebKit-wrapping browsers such as Chrome on iOS
-		typeof navigator !== 'undefined'
+	const result = !!(typeof navigator !== 'undefined' && navigator.vendor?.match(/apple/i));
-		&& navigator.vendor?.match(/apple/i)
-		&& !navigator.userAgent?.match(/crios/i)
-		&& !navigator.userAgent?.match(/fxios/i)
-		&& !navigator.userAgent?.match(/Opera|OPT\//)
-	);
 
-	isSafariCache = result;
+	isWebKitCache = result;
 	return result;
 };
 

src/packet.ts

@@ -126,7 +126,10 @@ export class EncodedPacket {
 		}
 	}
 
-	/** If this packet is a metadata-only packet. Metadata-only packets don't contain their packet data. */
+	/**
+	 * If this packet is a metadata-only packet. Metadata-only packets don't contain their packet data. They are the
+	 * result of retrieving packets with {@link PacketRetrievalOptions.metadataOnly} set to `true`.
+	 */
 	get isMetadataOnly() {
 		return this.data === PLACEHOLDER_DATA;
 	}

src/sample.ts

@@ -591,7 +591,6 @@ export class VideoSample implements Disposable {
 			dHeight,
 		);
 
-		// Restore the previous transformation state
 		context.restore();
 	}
 
@@ -690,6 +689,8 @@ export class VideoSample implements Disposable {
 			dy = (canvasHeight - newHeight) / 2;
 		}
 
+		context.save();
+
 		const aspectRatioChange = relativeRotation % 180 === 0 ? 1 : newWidth / newHeight;
 		context.translate(canvasWidth / 2, canvasHeight / 2);
 		context.rotate(relativeRotation * Math.PI / 180);
@@ -701,6 +702,8 @@ export class VideoSample implements Disposable {
 		// Important that we don't use .draw() here since that would take rotation into account, but we wanna handle it
 		// ourselves here
 		context.drawImage(source, sx, sy, sWidth, sHeight, dx, dy, newWidth, newHeight);
+
+		context.restore();
 	}
 
 	/** @internal */

src/source.ts

@@ -12,6 +12,7 @@ import {
 	binarySearchLessOrEqual,
 	closedIntervalsOverlap,
 	isNumber,
+	isWebKit,
 	MaybePromise,
 	mergeRequestInit,
 	promiseWithResolvers,
@@ -209,7 +210,14 @@ export class BlobSource extends Source {
 	private async _runWorker(worker: ReadWorker) {
 		let reader = this._readers.get(worker);
 		if (reader === undefined) {
-			if ('stream' in this._blob) {
+			// https://github.com/Vanilagy/mediabunny/issues/184
+			// WebKit has critical bugs with blob.stream():
+			// - WebKitBlobResource error 1 when streaming large files
+			// - Memory buildup and reload loops on iOS (network process crashes)
+			// - ReadableStream stalls under backpressure (especially video)
+			// Affects Safari and all iOS browsers (Chrome, Firefox, etc.).
+			// Use arrayBuffer() fallback for WebKit browsers.
+			if ('stream' in this._blob && !isWebKit()) {
 				// Get a reader of the blob starting at the required offset, and then keep it around
 				const slice = this._blob.slice(worker.currentPos);
 				reader = slice.stream().getReader();

test/browser/transparency.test.ts

@@ -148,8 +148,8 @@ test('Can encode transparent video', async () => {
 
 	// Red (within error)
 	expect(imageData.data[index2 + 0]).greaterThanOrEqual(253);
-	expect(imageData.data[index2 + 1]).lessThanOrEqual(2);
+	expect(imageData.data[index2 + 1]).lessThanOrEqual(3);
-	expect(imageData.data[index2 + 2]).lessThanOrEqual(2);
+	expect(imageData.data[index2 + 2]).lessThanOrEqual(3);
 
 	expect(imageData.data[index2 + 3]).toBe(255); // Opaque