Fix MP3 encoder emitting incorrect timestamps, fixed overaggressive audio sample gap filling (fixes #233)

dev/live.html

@@ -1,6 +1,7 @@
 <button>Go</button>
 
 <script src="../dist/bundles/mediabunny.cjs"></script>
+<script src="../packages/mp3-encoder/dist/bundles/mediabunny-mp3-encoder.js"></script>
 
 <script type="module">
     function download(blob, filename) {
@@ -12,6 +13,8 @@
 		URL.revokeObjectURL(url);
 	}
 
+	MediabunnyMp3Encoder.registerMp3Encoder();
+
     const button = document.querySelector('button');
     button.addEventListener('click', async () => {
         const stream = await navigator.mediaDevices.getUserMedia({ video: true, audio: true });
@@ -34,7 +37,7 @@
         }
         if (audioTrack) {
             const source = new Mediabunny.MediaStreamAudioTrackSource(audioTrack, {
-                codec: 'aac',
+                codec: 'mp3',
                 bitrate: Mediabunny.QUALITY_MEDIUM
             });
 

packages/mp3-encoder/src/index.ts

@@ -22,7 +22,7 @@ class Mp3Encoder extends CustomAudioEncoder {
 
 	private buffer = new Uint8Array(2 ** 16);
 	private currentBufferOffset = 0;
-	private currentTimestamp = 0;
+	private currentTimestamp: number | null = null;
 	private chunkMetadata: EncodedAudioChunkMetadata = {};
 
 	static override supports(codec: AudioCodec, config: AudioDecoderConfig): boolean {
@@ -79,6 +79,11 @@ class Mp3Encoder extends CustomAudioEncoder {
 	}
 
 	async encode(audioSample: AudioSample) {
+		if (this.currentTimestamp === null) {
+			// The first sample's timestamp determines where we start
+			this.currentTimestamp = audioSample.timestamp;
+		}
+
 		const sizePerChannel = audioSample.allocationSize({
 			format: 's16-planar',
 			planeIndex: 0,
@@ -123,6 +128,8 @@ class Mp3Encoder extends CustomAudioEncoder {
 	 * these chunks and extract the MP3 frames only when they're complete.
 	 */
 	private digestOutput(bytes: Uint8Array) {
+		assert(this.currentTimestamp !== null);
+
 		const requiredBufferSize = this.currentBufferOffset + bytes.length;
 		if (requiredBufferSize > this.buffer.length) {
 			// Grow the buffer to the required size

src/media-source.ts

@@ -1356,21 +1356,27 @@ class AudioEncoderWrapper {
 					(audioSample.timestamp + audioSample.duration) * audioSample.sampleRate,
 				);
 
-				if (this.lastEndSampleIndex !== null && startSampleIndex > this.lastEndSampleIndex) {
+				if (this.lastEndSampleIndex === null) {
-					const sampleCount = startSampleIndex - this.lastEndSampleIndex;
+					this.lastEndSampleIndex = endSampleIndex;
-					const fillSample = new AudioSample({
+				} else {
-						data: new Float32Array(sampleCount * audioSample.numberOfChannels),
+					const sampleDiff = startSampleIndex - this.lastEndSampleIndex;
-						format: 'f32-planar',
-						sampleRate: audioSample.sampleRate,
-						numberOfChannels: audioSample.numberOfChannels,
-						numberOfFrames: sampleCount,
-						timestamp: this.lastEndSampleIndex / audioSample.sampleRate,
-					});
 
-					await this.add(fillSample, true); // Recursive call
+					if (sampleDiff >= 64) {
+						// The gap is big enough, let's add a correction sample
+						const fillSample = new AudioSample({
+							data: new Float32Array(sampleDiff * audioSample.numberOfChannels),
+							format: 'f32-planar',
+							sampleRate: audioSample.sampleRate,
+							numberOfChannels: audioSample.numberOfChannels,
+							numberOfFrames: sampleDiff,
+							timestamp: this.lastEndSampleIndex / audioSample.sampleRate,
+						});
+
+						await this.add(fillSample, true); // Recursive call
+					}
+
+					this.lastEndSampleIndex += audioSample.numberOfFrames;
 				}
-
-				this.lastEndSampleIndex = endSampleIndex;
 			}
 
 			if (this.customEncoder) {