Bump patch

* Don't overlook potential packets in FLAC bitstream

* Update way of going back

README.md

@@ -45,12 +45,16 @@ Mediabunny is a JavaScript library for reading, writing, and converting media fi
 
 <div align="center">
   <a href="https://www.reactvideoeditor.com/" target="_blank" rel="sponsored">
-    <img src="./docs/public/sponsors/rve.svg" width="40" height="40" alt="React Video Editor">
+    <img src="./docs/public/sponsors/rve.png" width="40" height="40" alt="React Video Editor">
   </a>
   &nbsp;&nbsp;&nbsp;&nbsp;
   <a href="https://www.mux.com/" target="_blank" rel="sponsored">
     <img src="./docs/public/sponsors/mux.jpg" width="40" height="40" alt="Mux">
   </a>
+  &nbsp;&nbsp;&nbsp;&nbsp;
+  <a href="https://jellypod.ai/" target="_blank" rel="sponsored">
+    <img src="./docs/public/sponsors/jellypod.png" width="40" height="40" alt="Jellypod">
+  </a>
 </div>
 
 [Sponsor Mediabunny's development](https://github.com/sponsors/Vanilagy)

dev/convert.html

@@ -48,6 +48,7 @@
 			target
 		});
 
+		let ctx = null;
 		const conversion = await Mediabunny.Conversion.init({
 			input: new Mediabunny.Input({
 				formats: Mediabunny.ALL_FORMATS,	
@@ -56,6 +57,8 @@
 			output,
 			audio: (_, n) => ({
 				discard: n > 1,
+				codec: 'aac',
+				//codec: 'opus',
 				/*
 				process: (sample) => {
 					return sample;
@@ -70,7 +73,7 @@
 				//numberOfChannels: 1,
 				//sampleRate: 4000
 				//discard: true
-				//forceTranscode: true,
+				forceTranscode: true,
 			}),
 			/*
 			video: {
@@ -97,6 +100,7 @@
 			},
 			*/
 			video: () => ({
+				//discard: true,
 				/*
 				process: (sample) => {
 					if (!ctx) {
@@ -108,14 +112,16 @@
 						ctx = canvas.getContext('2d');
 					}
 
+					console.log(ctx.canvas.width, ctx.canvas.height);
+
 					ctx.clearRect(0, 0, ctx.canvas.width, ctx.canvas.height);
-					sample.draw(ctx, 0, 0);
-					ctx.drawImage(watermark, 32, 32);
+					sample.drawWithFit(ctx, { fit: 'fill' });
+					//ctx.drawImage(watermark, 32, 32);
 
 					return ctx.canvas;
 				},
 				*/
-				width: 300,
+				//width: 300,
 				//alpha: 'keep',
 				//width: 320,
 				//discard: true,
@@ -170,8 +176,8 @@
 				}
 			},
 			trim: {
-				start: 0,
-				end: 10
+				//start: 0,
+				//end: 10
 			},
 		});
 		console.log(conversion);

dev/demux.html

@@ -14,6 +14,67 @@
 			source: new Mediabunny.BlobSource(file),
 		});
 
+		const audioTrack = await input.getPrimaryAudioTrack();
+		const sink = new Mediabunny.AudioSampleSink(audioTrack);
+
+		let lastEnd = 0;
+		for await (const sample of sink.samples()) {
+			if (sample.timestamp - lastEnd > 0) {
+				console.warn(sample.timestamp - lastEnd)
+				console.log(sample.timestamp, sample.duration, "diffie", sample.timestamp - lastEnd);
+			}
+
+			lastEnd = sample.timestamp + sample.duration;
+			sample.close();
+		}
+		
+		/*
+		const sink = new Mediabunny.EncodedPacketSink(videoTrack);
+
+		for await (const packet of sink.packets()) {
+			console.log(packet.timestamp, packet.duration, packet.timestamp + packet.duration);
+
+			if (packet.timestamp > 10) break;
+		}
+		*/
+
+		/*
+		const sink = new Mediabunny.VideoSampleSink(videoTrack);
+
+		for await (const sample of sink.samples()) {
+			console.log(sample.timestamp, sample.duration);
+			sample.close();
+		}
+		*/
+
+		/*
+		const sink = new Mediabunny.EncodedPacketSink(videoTrack);
+
+		for await (const packet of sink.packets()) {
+			console.log(packet.timestamp, packet.timestamp + packet.duration);
+		}
+		*/
+
+
+		/*
+		const sink = new Mediabunny.VideoSampleSink(videoTrack);
+
+		console.log(await videoTrack.getDecoderConfig())
+
+		const sink2 = new Mediabunny.EncodedPacketSink(videoTrack);
+		for await (const packet of sink2.packets()) {
+			console.log(packet.data.join(', '))//, window.findNalUnitsInLengthPrefixed(packet.data, 4).map(x => window.extractNalUnitTypeForAvc(x)));
+			if (packet.timestamp > 1) break;
+		}
+
+		//return;
+
+		for await (const sample of sink.samples()) {
+			console.log(sample);
+		}
+		*/
+
+		/*
 		let total = 0;
 		input.source.onread = (start, end) => {
 			total += end - start;
@@ -22,6 +83,7 @@
 
 		const videoTrack = await input.getPrimaryVideoTrack();
 		const sink = new Mediabunny.EncodedPacketSink(videoTrack);
+		*/
 
 		//console.log(await sink.getPacket(0));
 		/*
@@ -38,7 +100,7 @@
 
 
 
-		const stats = await videoTrack.computePacketStats();
+		//const stats = await videoTrack.computePacketStats();
 		
 		/*
 		const sink = new Mediabunny.EncodedPacketSink(videoTrack);
@@ -47,9 +109,9 @@
 		}
 		*/
 		//console.log(await videoTrack.computeDuration());
-		console.log("Done", stats, total, file.size)
+		//console.log("Done", stats, total, file.size)
 
-		console.log(input);
+		//console.log(input);
 
 		/*
 		const videoTrack = await input.getPrimaryVideoTrack();

docs/index.md

@@ -100,10 +100,12 @@ const sponsors = {
 		{ image: '/sponsors/pintura-labs.png', name: 'Pintura Labs', url: 'https://pqina.nl/pintura/' },
 	],
 	bronze: [
-		{ image: '/sponsors/rve.svg', name: 'React Video Editor', url: 'https://www.reactvideoeditor.com/' },
+		{ image: '/sponsors/rve.png', name: 'React Video Editor', url: 'https://www.reactvideoeditor.com/' },
 		{ image: '/sponsors/mux.jpg', name: 'Mux', url: 'https://www.mux.com/' },
+		{ image: '/sponsors/jellypod.png', name: 'Jellypod', url: 'https://jellypod.ai/' },
 	],
 	individual: [
+		{ image: 'https://avatars.githubusercontent.com/u/197597', name: 'Christopher Chedeau', url: 'https://github.com/vjeux' },
 		{ image: 'https://avatars.githubusercontent.com/u/84167135', name: 'Memenome', url: 'https://github.com/memenome' },
 		{ image: 'https://avatars.githubusercontent.com/u/5913254', name: 'Brandon McConnell', url: 'https://github.com/brandonmcconnell' },
 		{ image: 'https://avatars.githubusercontent.com/u/9549394', name: 'studnitz', url: 'https://github.com/studnitz' },

docs/public/sponsors/rve.svg

@@ -1,4 +0,0 @@
-<svg width="52" height="51" viewBox="0 0 52 51" fill="none" xmlns="http://www.w3.org/2000/svg">
-<path d="M25.1186 1.50109C23.3805 0.147773 21.1954 -0.253724 19.1203 0.397437L5.08373 4.80289C2.19204 5.71105 0.248047 8.44236 0.248047 11.6022V42.9894C0.248047 45.2558 1.24465 47.3313 2.98266 48.6825C4.72099 50.036 6.9084 50.4373 8.98111 49.7863L19.4595 46.4967C19.1318 45.366 18.9658 44.1862 18.9658 42.9894V28.735C18.9658 25.7417 20.0093 22.8829 21.9088 20.6899C23.303 19.0744 25.1116 17.8826 27.1329 17.2463L27.8534 17.0309V7.19654C27.8534 4.93006 26.8568 2.85458 25.1185 1.50109H25.1186Z" fill="#1C2AC5"/>
-<path d="M51.2479 24.3293V38.5836C51.2479 41.7411 49.3061 44.4749 46.4122 45.3806L32.3756 49.7862C31.7253 49.9917 31.0632 50.0922 30.4081 50.0922C28.9739 50.0922 27.5703 49.6125 26.3771 48.6824C25.832 48.2565 25.3595 47.7645 24.9711 47.2139C24.4986 46.553 24.1453 45.8164 23.923 45.0257C23.9206 45.0257 23.9184 45.0257 23.916 45.0281L23.9206 45.0232C23.7406 44.3746 23.6423 43.6917 23.6423 42.9893V28.7349C23.6423 27.2712 24.0588 25.8983 24.7935 24.7576C25.528 23.6169 26.5807 22.7066 27.8534 22.1656C28.057 22.08 28.2652 22.0015 28.4782 21.9356L30.1953 21.3971L42.515 17.53C44.5899 16.8788 46.7751 17.2803 48.5132 18.6338C50.2516 19.9873 51.248 22.0628 51.248 24.3293H51.2479Z" fill="#3E8AF5"/>
-</svg>

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "mediabunny",
-  "version": "1.24.2",
+  "version": "1.24.6",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "mediabunny",
-      "version": "1.24.2",
+      "version": "1.24.6",
       "license": "MPL-2.0",
       "workspaces": [
         "packages/*"
@@ -7749,9 +7749,9 @@
       }
     },
     "node_modules/mediabunny": {
-      "version": "1.24.1",
-      "resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.24.1.tgz",
-      "integrity": "sha512-k7ki4KGgFE7F4XaUn0sDpPzdB8Hgms5Hq3QonzXyVSMh75oRuyjNhuAWJtAAlzs45gUerlIHZCvHd6/5H7duow==",
+      "version": "1.24.5",
+      "resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.24.5.tgz",
+      "integrity": "sha512-4Sd5x8WXQiuDKZcJDAiQjyD0Lg/cg9p+dBShCM02F1pMvKdB9R6+WHZ9NFCFSqViVPX2fYSu95ZIpBUg9+TwTQ==",
       "license": "MPL-2.0",
       "peer": true,
       "workspaces": [
@@ -12242,7 +12242,7 @@
     },
     "packages/mp3-encoder": {
       "name": "@mediabunny/mp3-encoder",
-      "version": "1.24.2",
+      "version": "1.24.6",
       "license": "MPL-2.0",
       "devDependencies": {
         "@types/emscripten": "^1.40.1"

package.json

@@ -1,7 +1,7 @@
 {
   "name": "mediabunny",
   "author": "Vanilagy",
-  "version": "1.24.2",
+  "version": "1.24.6",
   "description": "Pure TypeScript media toolkit for reading, writing, and converting media files, directly in the browser.",
   "type": "module",
   "workspaces": [

packages/mp3-encoder/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@mediabunny/mp3-encoder",
   "author": "Vanilagy",
-  "version": "1.24.2",
+  "version": "1.24.6",
   "description": "MP3 encoder extension for Mediabunny, based on LAME.",
   "main": "./dist/bundles/mediabunny-mp3-encoder.mjs",
   "module": "./dist/bundles/mediabunny-mp3-encoder.mjs",

src/codec.ts

@@ -337,6 +337,7 @@ export const extractVideoCodecString = (trackInfo: {
 	codec: VideoCodec | null;
 	codecDescription: Uint8Array | null;
 	colorSpace: VideoColorSpaceInit | null;
+	avcType: 1 | 3 | null;
 	avcCodecInfo: AvcDecoderConfigurationRecord | null;
 	hevcCodecInfo: HevcDecoderConfigurationRecord | null;
 	vp9CodecInfo: Vp9CodecInfo | null;
@@ -345,6 +346,8 @@ export const extractVideoCodecString = (trackInfo: {
 	const { codec, codecDescription, colorSpace, avcCodecInfo, hevcCodecInfo, vp9CodecInfo, av1CodecInfo } = trackInfo;
 
 	if (codec === 'avc') {
+		assert(trackInfo.avcType !== null);
+
 		if (avcCodecInfo) {
 			const bytes = new Uint8Array([
 				avcCodecInfo.avcProfileIndication,
@@ -352,14 +355,14 @@ export const extractVideoCodecString = (trackInfo: {
 				avcCodecInfo.avcLevelIndication,
 			]);
 
-			return `avc1.${bytesToHexString(bytes)}`;
+			return `avc${trackInfo.avcType}.${bytesToHexString(bytes)}`;
 		}
 
 		if (!codecDescription || codecDescription.byteLength < 4) {
 			throw new TypeError('AVC decoder description is not provided or is not at least 4 bytes long.');
 		}
 
-		return `avc1.${bytesToHexString(codecDescription.subarray(1, 4))}`;
+		return `avc${trackInfo.avcType}.${bytesToHexString(codecDescription.subarray(1, 4))}`;
 	} else if (codec === 'hevc') {
 		let generalProfileSpace: number;
 		let generalProfileIdc: number;
@@ -621,6 +624,54 @@ export const parseAacAudioSpecificConfig = (bytes: Uint8Array | null): AacAudioS
 	};
 };
 
+export const buildAacAudioSpecificConfig = (config: {
+	objectType: number;
+	sampleRate: number;
+	numberOfChannels: number;
+}) => {
+	let frequencyIndex = aacFrequencyTable.indexOf(config.sampleRate);
+	let customSampleRate: number | null = null;
+
+	if (frequencyIndex === -1) {
+		frequencyIndex = 15;
+		customSampleRate = config.sampleRate;
+	}
+
+	const channelConfiguration = aacChannelMap.indexOf(config.numberOfChannels);
+	if (channelConfiguration === -1) {
+		throw new TypeError(`Unsupported number of channels: ${config.numberOfChannels}`);
+	}
+
+	let bitCount = 5 + 4 + 4;
+	if (config.objectType >= 32) {
+		bitCount += 6;
+	}
+	if (frequencyIndex === 15) {
+		bitCount += 24;
+	}
+
+	const byteCount = Math.ceil(bitCount / 8);
+	const bytes = new Uint8Array(byteCount);
+	const bitstream = new Bitstream(bytes);
+
+	if (config.objectType < 32) {
+		bitstream.writeBits(5, config.objectType);
+	} else {
+		bitstream.writeBits(5, 31);
+		bitstream.writeBits(6, config.objectType - 32);
+	}
+
+	bitstream.writeBits(4, frequencyIndex);
+
+	if (frequencyIndex === 15) {
+		bitstream.writeBits(24, customSampleRate!);
+	}
+
+	bitstream.writeBits(4, channelConfiguration);
+
+	return bytes;
+};
+
 export const OPUS_SAMPLE_RATE = 48_000;
 
 const PCM_CODEC_REGEX = /^pcm-([usf])(\d+)+(be)?$/;

src/conversion.ts

@@ -182,7 +182,8 @@ export type ConversionVideoOptions = {
 	 * corrections.
 	 *
 	 * Must return a {@link VideoSample} or a `CanvasImageSource`, an array of them, or `null` for dropping the frame.
-	 * When non-timestamped data is returned, the timestamp and duration from the source sample will be used.
+	 * When non-timestamped data is returned, the timestamp and duration from the source sample will be used. Rotation
+	 * metadata of the returned sample will be ignored.
 	 *
 	 * This function can also be used to manually resize frames. When doing so, you should signal the post-process
 	 * dimensions using the `processedWidth` and `processedHeight` fields, which enables the encoder to better know what
@@ -873,7 +874,10 @@ export class Conversion {
 			|| trackOptions.process !== undefined;
 		let needsRerender = width !== originalWidth
 			|| height !== originalHeight
-			|| (totalRotation !== 0 && !outputSupportsRotation)
+			// TODO This is suboptimal: Forcing a rerender when both rotation and process are set is not
+			// performance-optimal, but right now there's no other way because we can't change the track rotation
+			// metadata after the output has already started. Should be possible with API changes in v2, though!
+			|| (totalRotation !== 0 && (!outputSupportsRotation || trackOptions.process !== undefined))
 			|| !!crop;
 
 		const alpha = trackOptions.alpha ?? 'discard';

src/flac/flac-demuxer.ts

@@ -328,27 +328,47 @@ export class FlacDemuxer extends Demuxer {
 
 			const nextByte = readU8(slice);
 			if (nextByte === 0xff) {
+				const positionBeforeReading = slice.filePos;
+
 				const byteAfterNextByte = readU8(slice);
 
 				const expected = this.blockingBit === 1 ? 0b1111_1001 : 0b1111_1000;
 				if (byteAfterNextByte !== expected) {
-					slice.skip(-1);
+					slice.filePos = positionBeforeReading;
 					continue;
 				}
 
 				slice.skip(-2);
 				const lengthIfNextFlacFrameHeaderIsLegit = slice.filePos - startPos;
 
-				const nextIsLegit = this.readFlacFrameHeader({
+				const nextFrameHeader = this.readFlacFrameHeader({
 					slice,
 					isFirstPacket: false,
 				});
 
-				if (!nextIsLegit) {
-					slice.skip(-1);
+				if (!nextFrameHeader) {
+					slice.filePos = positionBeforeReading;
 					continue;
 				}
 
+				// Ensure the frameOrSampleNum is consecutive.
+				// https://github.com/Vanilagy/mediabunny/issues/194
+
+				if (this.blockingBit === 0) {
+					// Case A: If the stream is fixed block size, this is the frame number, which increments by 1
+					if (nextFrameHeader.num - frameHeader.num !== 1) {
+						slice.filePos = positionBeforeReading;
+						continue;
+					}
+				} else {
+					// Case B: If the stream is variable block size, this is the sample number, which increments by
+					// amount of samples in a frame.
+					if (nextFrameHeader.num - frameHeader.num !== frameHeader.blockSize) {
+						slice.filePos = positionBeforeReading;
+						continue;
+					}
+				}
+
 				return {
 					num: frameHeader.num,
 					blockSize: frameHeader.blockSize,
@@ -442,6 +462,11 @@ export class FlacDemuxer extends Demuxer {
 			return null;
 		}
 
+		if (sampleRate !== this.audioInfo.sampleRate) {
+			// This cannot be a valid FLAC frame, the sample rate is not the same as in the stream info
+			return null;
+		}
+
 		const size = slice.filePos - startOffset;
 		const crc = readU8(slice);
 

src/isobmff/isobmff-boxes.ts

@@ -577,7 +577,7 @@ export const stsd = (trackData: IsobmffTrackData) => {
 
 	if (trackData.type === 'video') {
 		sampleDescription = videoSampleDescription(
-			VIDEO_CODEC_TO_BOX_NAME[trackData.track.source._codec],
+			videoCodecToBoxName(trackData.track.source._codec, trackData.info.decoderConfig.codec),
 			trackData,
 		);
 	} else if (trackData.type === 'audio') {
@@ -1572,12 +1572,14 @@ const dataStringBoxLong = (value: string) => {
 	]);
 };
 
-const VIDEO_CODEC_TO_BOX_NAME: Record<VideoCodec, string> = {
-	avc: 'avc1',
-	hevc: 'hvc1',
-	vp8: 'vp08',
-	vp9: 'vp09',
-	av1: 'av01',
+const videoCodecToBoxName = (codec: VideoCodec, fullCodecString: string) => {
+	switch (codec) {
+		case 'avc': return fullCodecString.startsWith('avc3') ? 'avc3' : 'avc1';
+		case 'hevc': return 'hvc1';
+		case 'vp8': return 'vp08';
+		case 'vp9': return 'vp09';
+		case 'av1': return 'av01';
+	}
 };
 
 const VIDEO_CODEC_TO_CONFIGURATION_BOX: Record<VideoCodec, (trackData: IsobmffVideoTrackData) => Box | null> = {

src/isobmff/isobmff-demuxer.ts

@@ -126,6 +126,7 @@ type InternalTrack = {
 		codec: VideoCodec | null;
 		codecDescription: Uint8Array | null;
 		colorSpace: VideoColorSpaceInit | null;
+		avcType: 1 | 3 | null;
 		avcCodecInfo: AvcDecoderConfigurationRecord | null;
 		hevcCodecInfo: HevcDecoderConfigurationRecord | null;
 		vp9CodecInfo: Vp9CodecInfo | null;
@@ -836,6 +837,7 @@ export class IsobmffDemuxer extends Demuxer {
 						codec: null,
 						codecDescription: null,
 						colorSpace: null,
+						avcType: null,
 						avcCodecInfo: null,
 						hevcCodecInfo: null,
 						vp9CodecInfo: null,
@@ -890,8 +892,9 @@ export class IsobmffDemuxer extends Demuxer {
 					const lowercaseBoxName = sampleBoxInfo.name.toLowerCase();
 
 					if (track.info.type === 'video') {
-						if (lowercaseBoxName === 'avc1') {
+						if (lowercaseBoxName === 'avc1' || lowercaseBoxName === 'avc3') {
 							track.info.codec = 'avc';
+							track.info.avcType = lowercaseBoxName === 'avc1' ? 1 : 3;
 						} else if (lowercaseBoxName === 'hvc1' || lowercaseBoxName === 'hev1') {
 							track.info.codec = 'hevc';
 						} else if (lowercaseBoxName === 'vp08') {

src/matroska/matroska-demuxer.ts

@@ -844,13 +844,13 @@ export class MatroskaDemuxer extends Demuxer {
 
 			blocks.splice(blockIndex, 1); // Remove the original block
 
+			const blockDuration = originalBlock.duration || frameCount * (track.defaultDuration ?? 0);
+
 			// Now, let's insert each frame as its own block
 			for (let i = 0; i < frameCount; i++) {
 				const frameSize = frameSizes[i]!;
 				const frameData = readBytes(slice, frameSize);
 
-				const blockDuration = originalBlock.duration || (frameCount * (track.defaultDuration ?? 0));
-
 				// Distribute timestamps evenly across the block duration
 				const frameTimestamp = originalBlock.timestamp + (blockDuration * i / frameCount);
 				const frameDuration = blockDuration / frameCount;
@@ -2319,6 +2319,7 @@ class MatroskaVideoTrackBacking extends MatroskaTrackBacking implements InputVid
 					codec: this.internalTrack.info.codec,
 					codecDescription: this.internalTrack.info.codecDescription,
 					colorSpace: this.internalTrack.info.colorSpace,
+					avcType: 1, // We don't know better (or do we?) so just assume 'avc1'
 					avcCodecInfo: this.internalTrack.info.codec === 'avc' && firstPacket
 						? extractAvcDecoderConfigurationRecord(firstPacket.data)
 						: null,

src/media-sink.ts

@@ -26,7 +26,7 @@ import {
 	insertSorted,
 	isFirefox,
 	isNumber,
-	isSafari,
+	isWebKit,
 	last,
 	mapAsyncGenerator,
 	promiseWithResolvers,
@@ -918,7 +918,7 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
 		} else {
 			assert(this.decoder);
 
-			if (!isSafari()) {
+			if (!isWebKit()) {
 				insertSorted(this.inputTimestamps, packet.timestamp, x => x);
 			}
 
@@ -1049,7 +1049,7 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
 
 	/** Handler for the WebCodecs VideoDecoder for ironing out browser differences. */
 	sampleHandler(sample: VideoSample) {
-		if (isSafari()) {
+		if (isWebKit()) {
 			// For correct B-frame handling, we don't just hand over the frames directly but instead add them to
 			// a queue, because we want to ensure frames are emitted in presentation order. We flush the queue
 			// each time we receive a frame with a timestamp larger than the highest we've seen so far, as we
@@ -1137,7 +1137,7 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
 			this.alphaRaslSkipped = false;
 		}
 
-		if (isSafari()) {
+		if (isWebKit()) {
 			for (const sample of this.sampleQueue) {
 				this.finalizeAndEmitSample(sample);
 			}

src/media-source.ts

@@ -9,6 +9,8 @@
 import {
 	AUDIO_CODECS,
 	AudioCodec,
+	buildAacAudioSpecificConfig,
+	parseAacAudioSpecificConfig,
 	parsePcmCodec,
 	PCM_AUDIO_CODECS,
 	PcmAudioCodec,
@@ -24,9 +26,11 @@ import {
 	CallSerializer,
 	clamp,
 	isFirefox,
+	last,
 	promiseWithResolvers,
 	setInt24,
 	setUint24,
+	toUint8Array,
 } from './misc';
 import { Muxer } from './muxer';
 import { SubtitleParser } from './subtitles';
@@ -1159,7 +1163,7 @@ export class MediaStreamVideoTrackSource extends VideoSource {
 					type: 'videoTrack',
 					trackId: this._workerTrackId,
 					track: this._track,
-				}, [this._track]);
+				});
 
 				this._workerListener = (event: MessageEvent) => {
 					const message = event.data as MediaStreamTrackProcessorWorkerMessage;
@@ -1290,6 +1294,8 @@ class AudioEncoderWrapper {
 	private customEncoderCallSerializer = new CallSerializer();
 	private customEncoderQueueSize = 0;
 
+	private lastEndSampleIndex: number | null = null;
+
 	/**
 	 * Encoders typically throw their errors "out of band", meaning asynchronously in some other execution context.
 	 * However, we want to surface these errors to the user within the normal control flow, so they don't go uncaught.
@@ -1338,6 +1344,35 @@ class AudioEncoderWrapper {
 			}
 			assert(this.encoderInitialized);
 
+			// Handle padding of gaps with silence to avoid audio drift over time, like in
+			// https://github.com/Vanilagy/mediabunny/issues/176
+			// TODO An open question is how encoders deal with the first AudioData having a non-zero timestamp, and with
+			// AudioDatas that have an overlapping timestamp range.
+			{
+				const startSampleIndex = Math.round(
+					audioSample.timestamp * audioSample.sampleRate,
+				);
+				const endSampleIndex = Math.round(
+					(audioSample.timestamp + audioSample.duration) * audioSample.sampleRate,
+				);
+
+				if (this.lastEndSampleIndex !== null && startSampleIndex > this.lastEndSampleIndex) {
+					const sampleCount = startSampleIndex - this.lastEndSampleIndex;
+					const fillSample = new AudioSample({
+						data: new Float32Array(sampleCount * audioSample.numberOfChannels),
+						format: 'f32-planar',
+						sampleRate: audioSample.sampleRate,
+						numberOfChannels: audioSample.numberOfChannels,
+						numberOfFrames: sampleCount,
+						timestamp: this.lastEndSampleIndex / audioSample.sampleRate,
+					});
+
+					await this.add(fillSample, true); // Recursive call
+				}
+
+				this.lastEndSampleIndex = endSampleIndex;
+			}
+
 			if (this.customEncoder) {
 				this.customEncoderQueueSize++;
 
@@ -1515,6 +1550,32 @@ class AudioEncoderWrapper {
 
 				this.encoder = new AudioEncoder({
 					output: (chunk, meta) => {
+						// WebKit emits an invalid description for AAC (https://bugs.webkit.org/show_bug.cgi?id=302253),
+						// which we try to detect here. If detected, we'll provide our own description instead, derived
+						// from the codec string and audio parameters.
+						if (this.encodingConfig.codec === 'aac' && meta?.decoderConfig) {
+							let needsDescriptionOverwrite = false;
+							if (!meta.decoderConfig.description || meta.decoderConfig.description.byteLength < 2) {
+								needsDescriptionOverwrite = true;
+							} else {
+								const audioSpecificConfig = parseAacAudioSpecificConfig(
+									toUint8Array(meta.decoderConfig.description),
+								);
+
+								needsDescriptionOverwrite = audioSpecificConfig.objectType === 0;
+							}
+
+							if (needsDescriptionOverwrite) {
+								const objectType = Number(last(encoderConfig.codec.split('.')));
+
+								meta.decoderConfig.description = buildAacAudioSpecificConfig({
+									objectType,
+									numberOfChannels: meta.decoderConfig.numberOfChannels,
+									sampleRate: meta.decoderConfig.sampleRate,
+								});
+							}
+						}
+
 						const packet = EncodedPacket.fromEncodedChunk(chunk);
 
 						this.encodingConfig.onEncodedPacket?.(packet, meta);
@@ -1978,17 +2039,19 @@ const mediaStreamTrackProcessorWorkerCode = () => {
 	});
 
 	const abortControllers = new Map<number, AbortController>();
-	const stoppedTracks = new Set<number>();
+	const activeTracks = new Map<number, MediaStreamVideoTrack>();
 
 	self.addEventListener('message', (event) => {
 		const message = event.data as MediaStreamTrackProcessorControllerMessage;
 
 		switch (message.type) {
 			case 'videoTrack': {
+				activeTracks.set(message.trackId, message.track);
+
 				const processor = new MediaStreamTrackProcessor({ track: message.track });
 				const consumer = new WritableStream<VideoFrame>({
 					write: (videoFrame) => {
-						if (stoppedTracks.has(message.trackId)) {
+						if (!activeTracks.has(message.trackId)) {
 							videoFrame.close();
 							return;
 						}
@@ -2026,7 +2089,9 @@ const mediaStreamTrackProcessorWorkerCode = () => {
 					abortControllers.delete(message.trackId);
 				}
 
-				stoppedTracks.add(message.trackId);
+				const track = activeTracks.get(message.trackId);
+				track?.stop();
+				activeTracks.delete(message.trackId);
 
 				sendMessage({
 					type: 'trackStopped',

src/misc.ts

@@ -658,21 +658,16 @@ export class CallSerializer {
 	}
 }
 
-let isSafariCache: boolean | null = null;
-export const isSafari = () => {
-	if (isSafariCache !== null) {
-		return isSafariCache;
+let isWebKitCache: boolean | null = null;
+export const isWebKit = () => {
+	if (isWebKitCache !== null) {
+		return isWebKitCache;
 	}
 
-	const result = !!(
-		typeof navigator !== 'undefined'
-		&& navigator.vendor?.match(/apple/i)
-		&& !navigator.userAgent?.match(/crios/i)
-		&& !navigator.userAgent?.match(/fxios/i)
-		&& !navigator.userAgent?.match(/Opera|OPT\//)
-	);
+	// This even returns true for WebKit-wrapping browsers such as Chrome on iOS
+	const result = !!(typeof navigator !== 'undefined' && navigator.vendor?.match(/apple/i));
 
-	isSafariCache = result;
+	isWebKitCache = result;
 	return result;
 };
 

src/packet.ts

@@ -126,7 +126,10 @@ export class EncodedPacket {
 		}
 	}
 
-	/** If this packet is a metadata-only packet. Metadata-only packets don't contain their packet data. */
+	/**
+	 * If this packet is a metadata-only packet. Metadata-only packets don't contain their packet data. They are the
+	 * result of retrieving packets with {@link PacketRetrievalOptions.metadataOnly} set to `true`.
+	 */
 	get isMetadataOnly() {
 		return this.data === PLACEHOLDER_DATA;
 	}

src/sample.ts

@@ -552,7 +552,6 @@ export class VideoSample implements Disposable {
 			dHeight,
 		);
 
-		// Restore the previous transformation state
 		context.restore();
 	}
 
@@ -644,6 +643,8 @@ export class VideoSample implements Disposable {
 			dy = (canvasHeight - newHeight) / 2;
 		}
 
+		context.save();
+
 		const aspectRatioChange = rotation % 180 === 0 ? 1 : newWidth / newHeight;
 		context.translate(canvasWidth / 2, canvasHeight / 2);
 		context.rotate(rotation * Math.PI / 180);
@@ -655,6 +656,8 @@ export class VideoSample implements Disposable {
 		// Important that we don't use .draw() here since that would take rotation into account, but we wanna handle it
 		// ourselves here
 		context.drawImage(this.toCanvasImageSource(), sx, sy, sWidth, sHeight, dx, dy, newWidth, newHeight);
+
+		context.restore();
 	}
 
 	/** @internal */

src/source.ts

@@ -12,6 +12,7 @@ import {
 	binarySearchLessOrEqual,
 	closedIntervalsOverlap,
 	isNumber,
+	isWebKit,
 	MaybePromise,
 	mergeRequestInit,
 	promiseWithResolvers,
@@ -209,7 +210,14 @@ export class BlobSource extends Source {
 	private async _runWorker(worker: ReadWorker) {
 		let reader = this._readers.get(worker);
 		if (reader === undefined) {
-			if ('stream' in this._blob) {
+			// https://github.com/Vanilagy/mediabunny/issues/184
+			// WebKit has critical bugs with blob.stream():
+			// - WebKitBlobResource error 1 when streaming large files
+			// - Memory buildup and reload loops on iOS (network process crashes)
+			// - ReadableStream stalls under backpressure (especially video)
+			// Affects Safari and all iOS browsers (Chrome, Firefox, etc.).
+			// Use arrayBuffer() fallback for WebKit browsers.
+			if ('stream' in this._blob && !isWebKit()) {
 				// Get a reader of the blob starting at the required offset, and then keep it around
 				const slice = this._blob.slice(worker.currentPos);
 				reader = slice.stream().getReader();
@@ -234,11 +242,19 @@ export class BlobSource extends Source {
 					break;
 				}
 
+				if (worker.aborted) {
+					break;
+				}
+
 				this.onread?.(worker.currentPos, worker.currentPos + value.length);
 				this._orchestrator.supplyWorkerData(worker, value);
 			} else {
 				const data = await this._blob.slice(worker.currentPos, worker.targetPos).arrayBuffer();
 
+				if (worker.aborted) {
+					break;
+				}
+
 				this.onread?.(worker.currentPos, worker.currentPos + data.byteLength);
 				this._orchestrator.supplyWorkerData(worker, new Uint8Array(data));
 			}
@@ -457,7 +473,7 @@ export class UrlSource extends Source {
 	/** @internal */
 	private async _runWorker(worker: ReadWorker) {
 		// The outer loop is for resuming a request if it dies mid-response
-		while (!worker.aborted) {
+		while (true) {
 			const existing = this._existingResponses.get(worker);
 			this._existingResponses.delete(worker);
 
@@ -534,6 +550,10 @@ export class UrlSource extends Source {
 					}
 				}
 
+				if (worker.aborted) {
+					break;
+				}
+
 				const { done, value } = readResult;
 
 				if (done) {
@@ -552,6 +572,10 @@ export class UrlSource extends Source {
 				this.onread?.(worker.currentPos, worker.currentPos + value.length);
 				this._orchestrator.supplyWorkerData(worker, value);
 			}
+
+			if (worker.aborted) {
+				break;
+			}
 		}
 
 		worker.running = false;
@@ -796,6 +820,10 @@ export class StreamSource extends Source {
 			let data = this._options.read(worker.currentPos, originalTargetPos);
 			if (data instanceof Promise) data = await data;
 
+			if (worker.aborted) {
+				break;
+			}
+
 			if (data instanceof Uint8Array) {
 				data = toUint8Array(data); // Normalize things like Node.js Buffer to Uint8Array
 
@@ -833,6 +861,10 @@ export class StreamSource extends Source {
 						throw new TypeError('ReadableStream returned by options.read must yield Uint8Array chunks.');
 					}
 
+					if (worker.aborted) {
+						break;
+					}
+
 					const data = toUint8Array(value); // Normalize things like Node.js Buffer to Uint8Array
 
 					this.onread?.(worker.currentPos, worker.currentPos + data.length);
@@ -1420,7 +1452,10 @@ class ReadOrchestrator {
 			currentPos: startPos,
 			targetPos,
 			running: false,
-			aborted: false,
+			// Due to async shenanigans, it can happen that workers are started after disposal. In this case, instead of
+			// simply not creating the worker, we allow it to run but immediately label it as aborted, so it can then
+			// shut itself down.
+			aborted: this.disposed,
 			pendingSlices: [],
 			age: this.nextAge++,
 		};
@@ -1473,10 +1508,7 @@ class ReadOrchestrator {
 
 	/** Called by a worker when it has read some data. */
 	supplyWorkerData(worker: ReadWorker, bytes: Uint8Array) {
-		if (this.disposed) {
-			// Writes may still come in after disposal, but we just ignore those
-			return;
-		}
+		assert(!worker.aborted);
 
 		const start = worker.currentPos;
 		const end = start + bytes.length;

test/browser/transparency.test.ts

@@ -139,17 +139,19 @@ test('Can encode transparent video', async () => {
 	probeContext.drawImage(video, 0, 0);
 
 	let imageData = probeContext.getImageData(0, 0, probeCanvas.width, probeCanvas.height);
-	expect(imageData.data[3]).lessThanOrEqual(2); // Transparent (within error)
+	const pos1 = { x: 100, y: 100 };
+	const index1 = (pos1.x + pos1.y * probeCanvas.width) * 4;
+	expect(imageData.data[index1 + 3]).toBe(0); // Transparent
 
-	const pos = { x: 300, y: 300 }; // Dead center in the red square
-	const index = (pos.x + pos.y * probeCanvas.width) * 4;
+	const pos2 = { x: 300, y: 300 }; // Dead center in the red square
+	const index2 = (pos2.x + pos2.y * probeCanvas.width) * 4;
 
 	// Red (within error)
-	expect(imageData.data[index + 0]).greaterThanOrEqual(253);
-	expect(imageData.data[index + 1]).lessThanOrEqual(2);
-	expect(imageData.data[index + 2]).lessThanOrEqual(2);
+	expect(imageData.data[index2 + 0]).greaterThanOrEqual(253);
+	expect(imageData.data[index2 + 1]).lessThanOrEqual(3);
+	expect(imageData.data[index2 + 2]).lessThanOrEqual(3);
 
-	expect(imageData.data[index + 3]).greaterThanOrEqual(253); // Opaque (within error)
+	expect(imageData.data[index2 + 3]).toBe(255); // Opaque
 
 	// Let's also check it's read correctly by Mediabunny
 	using input = new Input({
@@ -169,7 +171,8 @@ test('Can encode transparent video', async () => {
 	firstSample.draw(probeContext, 0, 0);
 
 	imageData = probeContext.getImageData(0, 0, probeCanvas.width, probeCanvas.height);
-	expect(imageData.data[3]).lessThanOrEqual(2); // Transparent (within error)
+
+	expect(imageData.data[3]).toBe(0); // Transparent
 });
 
 test('Can encode video with alternating transparency', async () => {