Merge main into release for tag v1.38.1

dev/convert.html

@@ -26,7 +26,7 @@
 			chunked: true,
 			chunkSize: 2**20
 		});
-		const outputFormat = new Mediabunny.Mp4OutputFormat({});
+		const outputFormat = new Mediabunny.Mp4OutputFormat({fastStart: 'fragmented'});
 
 		const button = document.createElement('button');
 		button.textContent = 'Cancel';

dev/demux.html

@@ -18,10 +18,11 @@
 		});
 
 		const videoTrack = await input.getPrimaryVideoTrack();
-		const sink = new Mediabunny.VideoSampleSink(videoTrack);
+		const sink = new Mediabunny.EncodedPacketSink(videoTrack);
 
-		const sample = await sink.getSample(await videoTrack.getFirstTimestamp());
+		for await (const packet of sink.packets()) {
-		console.log(sample);
+			console.log(packet.timestamp);
+		}
 
 		/*
 		

dev/live.html

@@ -19,7 +19,7 @@
     button.addEventListener('click', async () => {
         const stream = await navigator.mediaDevices.getDisplayMedia({ video: true, audio: false });
         const videoTrack = stream.getVideoTracks()[0];
-        const audioTrack = null;//stream.getAudioTracks()[0];
+        const audioTrack = stream.getAudioTracks()[0];
 
         const output = new Mediabunny.Output({
             target: new Mediabunny.BufferTarget(),
@@ -29,7 +29,7 @@
 		let audioSource = null;
         if (videoTrack) {
             videoSource = new Mediabunny.MediaStreamVideoTrackSource(videoTrack, {
-                codec: 'avc',
+                codec: 'vp9',
                 bitrate: Mediabunny.QUALITY_MEDIUM,
 				sizeChangeBehavior: 'passThrough',
             });
@@ -40,13 +40,13 @@
         }
         if (audioTrack) {
             audioSource = new Mediabunny.MediaStreamAudioTrackSource(audioTrack, {
-                codec: 'mp3',
+                codec: 'opus',
                 bitrate: Mediabunny.QUALITY_MEDIUM
             });
 
             audioSource.errorPromise.catch((d) => console.log("Hello!!???", d));
 
-            //output.addAudioTrack(audioSource);
+            output.addAudioTrack(audioSource);
         }
 
         await output.start();

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "mediabunny",
-  "version": "1.38.0",
+  "version": "1.38.1",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "mediabunny",
-      "version": "1.38.0",
+      "version": "1.38.1",
       "license": "MPL-2.0",
       "workspaces": [
         "packages/*"
@@ -7751,9 +7751,9 @@
       }
     },
     "node_modules/mediabunny": {
-      "version": "1.37.0",
+      "version": "1.38.0",
-      "resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.37.0.tgz",
+      "resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.38.0.tgz",
-      "integrity": "sha512-eV7M9IJ29pr/8RNL1sYtIxNbdMfDMN1hMwMaOFfNLhwuKKGSC+eKwiJFpdVjEJ3zrMA4LGerF4Hps0SENFSAlg==",
+      "integrity": "sha512-FiZtTWLY4rmEw4jBx8r5Tk+sE+PcBb03UYUZNv9DujEy9wdmX7JaQOvHNUZLVTxPWT0h+LYmWFnWKRkl4KmzyA==",
       "license": "MPL-2.0",
       "peer": true,
       "workspaces": [
@@ -12077,7 +12077,7 @@
     },
     "packages/aac-encoder": {
       "name": "@mediabunny/aac-encoder",
-      "version": "1.38.0",
+      "version": "1.38.1",
       "license": "MPL-2.0",
       "devDependencies": {
         "@types/emscripten": "^1.40.1"
@@ -12092,7 +12092,7 @@
     },
     "packages/ac3": {
       "name": "@mediabunny/ac3",
-      "version": "1.38.0",
+      "version": "1.38.1",
       "license": "MPL-2.0",
       "devDependencies": {
         "@types/emscripten": "^1.40.1"
@@ -12107,7 +12107,7 @@
     },
     "packages/flac-encoder": {
       "name": "@mediabunny/flac-encoder",
-      "version": "1.38.0",
+      "version": "1.38.1",
       "license": "MPL-2.0",
       "devDependencies": {
         "@types/emscripten": "^1.40.1"
@@ -12122,7 +12122,7 @@
     },
     "packages/mp3-encoder": {
       "name": "@mediabunny/mp3-encoder",
-      "version": "1.38.0",
+      "version": "1.38.1",
       "license": "MPL-2.0",
       "devDependencies": {
         "@types/emscripten": "^1.40.1"

package.json

@@ -1,7 +1,7 @@
 {
   "name": "mediabunny",
   "author": "Vanilagy",
-  "version": "1.38.0",
+  "version": "1.38.1",
   "description": "Pure TypeScript media toolkit for reading, writing, and converting media files, directly in the browser.",
   "type": "module",
   "workspaces": [

packages/aac-encoder/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@mediabunny/aac-encoder",
   "author": "Vanilagy",
-  "version": "1.38.0",
+  "version": "1.38.1",
   "description": "AAC encoder extension for Mediabunny, based on FFmpeg.",
   "main": "./dist/bundles/mediabunny-aac-encoder.mjs",
   "module": "./dist/bundles/mediabunny-aac-encoder.mjs",

packages/ac3/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@mediabunny/ac3",
   "author": "Vanilagy",
-  "version": "1.38.0",
+  "version": "1.38.1",
   "description": "AC-3 and E-AC-3 (Dolby Digital) decoder and encoder extension for Mediabunny, based on FFmpeg.",
   "main": "./dist/bundles/mediabunny-ac3.mjs",
   "module": "./dist/bundles/mediabunny-ac3.mjs",

packages/flac-encoder/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@mediabunny/flac-encoder",
   "author": "Vanilagy",
-  "version": "1.38.0",
+  "version": "1.38.1",
   "description": "FLAC encoder extension for Mediabunny, based on libFLAC.",
   "main": "./dist/bundles/mediabunny-flac-encoder.mjs",
   "module": "./dist/bundles/mediabunny-flac-encoder.mjs",

packages/mp3-encoder/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@mediabunny/mp3-encoder",
   "author": "Vanilagy",
-  "version": "1.38.0",
+  "version": "1.38.1",
   "description": "MP3 encoder extension for Mediabunny, based on LAME.",
   "main": "./dist/bundles/mediabunny-mp3-encoder.mjs",
   "module": "./dist/bundles/mediabunny-mp3-encoder.mjs",

src/isobmff/isobmff-muxer.ts

@@ -1245,11 +1245,13 @@ export class IsobmffMuxer extends Muxer {
 	override async onTrackClose(track: OutputTrack) {
 		const release = await this.mutex.acquire();
 
-		if (track.type === 'subtitle' && track.source._codec === 'webvtt') {
+		const trackData = this.trackDatas.find(x => x.track === track);
-			const trackData = this.trackDatas.find(x => x.track === track) as IsobmffSubtitleTrackData;
 		if (trackData) {
+			if (trackData.type === 'subtitle' && track.source._codec === 'webvtt') {
 				await this.processWebVTTCues(trackData, Infinity);
 			}
+
+			this.processTimestamps(trackData);
 		}
 
 		if (this.allTracksAreKnown()) {
@@ -1274,19 +1276,15 @@ export class IsobmffMuxer extends Muxer {
 			if (trackData.type === 'subtitle' && trackData.track.source._codec === 'webvtt') {
 				await this.processWebVTTCues(trackData, Infinity);
 			}
+
+			this.processTimestamps(trackData);
 		}
 
 		if (this.isFragmented) {
 			await this.interleaveSamples(true);
-
-			for (const trackData of this.trackDatas) {
-				this.processTimestamps(trackData);
-			}
-
 			await this.finalizeFragment(false); // Don't flush the last fragment as we will flush it with the mfra box
 		} else {
 			for (const trackData of this.trackDatas) {
-				this.processTimestamps(trackData);
 				await this.finalizeCurrentChunk(trackData);
 			}
 		}

src/media-source.ts

@@ -1076,6 +1076,8 @@ export class MediaStreamVideoTrackSource extends VideoSource {
 	private _lastVideoFrame: VideoFrame | null = null;
 	/** @internal */
 	private _timerHandle: UnthrottledTimerHandle | null = null;
+	/** @internal */
+	private _videoElement: HTMLVideoElement | null = null;
 
 	/** A promise that rejects upon any error within this source. This promise never resolves. */
 	get errorPromise() {
@@ -1160,12 +1162,13 @@ export class MediaStreamVideoTrackSource extends VideoSource {
 			while (now - lastFrameTime > 1000 / frameRate) {
 				lastFrameTime += 1000 / frameRate;
 				const timestamp = firstVideoFrameTimestamp + frameCount / frameRate;
-				const clone = new VideoFrame(this._lastVideoFrame, {
+
+				const frame = new VideoFrame(this._videoElement ?? this._lastVideoFrame, {
 					timestamp: 1e6 * timestamp,
 					duration: 1e6 / frameRate,
 				});
 
-				addVideoFrame(clone, now);
+				addVideoFrame(frame, now);
 			}
 		};
 
@@ -1298,8 +1301,44 @@ export class MediaStreamVideoTrackSource extends VideoSource {
 				};
 
 				mediaStreamTrackProcessorWorker!.addEventListener('message', this._workerListener);
+			} else if (frameRate !== null) {
+				// No MediaStreamTrackProcessor support at all (e.g. Firefox), but we have a frame rate, so we can
+				// manually sample from a hidden <video> element instead.
+				const video = document.createElement('video');
+				video.style.position = 'fixed';
+				video.style.left = '-10000px';
+				video.style.top = '-10000px';
+				video.style.width = '1px';
+				video.style.height = '1px';
+				video.style.opacity = '0';
+				video.style.pointerEvents = 'none';
+				video.muted = true;
+				video.srcObject = new MediaStream([this._track]);
+				document.body.appendChild(video);
+				this._videoElement = video;
+
+				video.addEventListener('loadeddata', () => {
+					if (errored || !this._videoElement) {
+						return;
+					}
+
+					// This will be the first frame
+					const frame = new VideoFrame(video, {
+						timestamp: 1000 * performance.now(),
+					});
+					onVideoFrame(frame);
+					frame.close();
+				}, { once: true });
+
+				void video.play().catch((error) => {
+					errored = true;
+					this._promiseWithResolvers.reject(error);
+				});
 			} else {
-				throw new Error('MediaStreamTrackProcessor is required but not supported by this browser.');
+				throw new Error(
+					'When no explicit frame rate is set, MediaStreamTrackProcessor is required; but it\'s not supported'
+					+ ' by this browser.',
+				);
 			}
 		}
 	}
@@ -1329,6 +1368,12 @@ export class MediaStreamVideoTrackSource extends VideoSource {
 		}
 		this._lastVideoFrame?.close();
 
+		if (this._videoElement) {
+			this._videoElement.srcObject = null;
+			this._videoElement.remove();
+			this._videoElement = null;
+		}
+
 		if (this._workerTrackId !== null) {
 			assert(this._workerListener);
 

test/node/isobmff-muxer.test.ts

@@ -61,3 +61,46 @@ test('ISOBMFF muxer internally converts ADTS to AAC', async () => {
 
 	expect(count).toBe(4557);
 });
+
+test('Fragmented fMP4 with video+audio preserves B-frame CTS', async () => {
+	using input = new Input({
+		source: new FilePathSource(path.join(__dirname, '../public/video.mp4')),
+		formats: ALL_FORMATS,
+	});
+
+	const videoTrack = await input.getPrimaryVideoTrack();
+	const audioTrack = await input.getPrimaryAudioTrack();
+	assert(videoTrack);
+	assert(audioTrack);
+
+	const originalVideoSink = new EncodedPacketSink(videoTrack);
+	const originalTimestamps: number[] = [];
+	for await (const packet of originalVideoSink.packets()) {
+		originalTimestamps.push(packet.timestamp);
+	}
+
+	const output = new Output({
+		format: new Mp4OutputFormat({ fastStart: 'fragmented' }),
+		target: new BufferTarget(),
+	});
+
+	const conversion = await Conversion.init({ input, output, showWarnings: false });
+	await conversion.execute();
+
+	using outputAsInput = new Input({
+		source: new BufferSource(output.target.buffer!),
+		formats: ALL_FORMATS,
+	});
+
+	const outputVideoTrack = await outputAsInput.getPrimaryVideoTrack();
+	assert(outputVideoTrack);
+
+	const videoSink = new EncodedPacketSink(outputVideoTrack);
+
+	const timestamps: number[] = [];
+	for await (const packet of videoSink.packets()) {
+		timestamps.push(packet.timestamp);
+	}
+
+	expect(timestamps).toEqual(originalTimestamps);
+});