Modify audio encoder to fill gaps between timestamps with silence to ensure AV sync (fixes #176)

dev/convert.html

@@ -57,6 +57,7 @@
 			output,
 			audio: (_, n) => ({
 				discard: n > 1,
+				codec: 'aac',
 				//codec: 'opus',
 				/*
 				process: (sample) => {
@@ -72,7 +73,7 @@
 				//numberOfChannels: 1,
 				//sampleRate: 4000
 				//discard: true
-				//forceTranscode: true,
+				forceTranscode: true,
 			}),
 			/*
 			video: {
@@ -100,6 +101,7 @@
 			*/
 			video: () => ({
 				//discard: true,
+				/*
 				process: (sample) => {
 					if (!ctx) {
 						// Create a canvas for image compositing
@@ -118,6 +120,7 @@
 
 					return ctx.canvas;
 				},
+				*/
 				//width: 300,
 				//alpha: 'keep',
 				//width: 320,

dev/demux.html

@@ -14,7 +14,21 @@
 			source: new Mediabunny.BlobSource(file),
 		});
 
-		const videoTrack = await input.getPrimaryAudioTrack();
+		const audioTrack = await input.getPrimaryAudioTrack();
+		const sink = new Mediabunny.AudioSampleSink(audioTrack);
+
+		let lastEnd = 0;
+		for await (const sample of sink.samples()) {
+			if (sample.timestamp - lastEnd > 0) {
+				console.warn(sample.timestamp - lastEnd)
+				console.log(sample.timestamp, sample.duration, "diffie", sample.timestamp - lastEnd);
+			}
+
+			lastEnd = sample.timestamp + sample.duration;
+			sample.close();
+		}
+		
+		/*
 		const sink = new Mediabunny.EncodedPacketSink(videoTrack);
 
 		for await (const packet of sink.packets()) {
@@ -22,6 +36,7 @@
 
 			if (packet.timestamp > 10) break;
 		}
+		*/
 
 		/*
 		const sink = new Mediabunny.VideoSampleSink(videoTrack);

src/media-source.ts

@@ -1294,6 +1294,8 @@ class AudioEncoderWrapper {
 	private customEncoderCallSerializer = new CallSerializer();
 	private customEncoderQueueSize = 0;
 
+	private lastEndSampleIndex: number | null = null;
+
 	/**
 	 * Encoders typically throw their errors "out of band", meaning asynchronously in some other execution context.
 	 * However, we want to surface these errors to the user within the normal control flow, so they don't go uncaught.
@@ -1342,6 +1344,35 @@ class AudioEncoderWrapper {
 			}
 			assert(this.encoderInitialized);
 
+			// Handle padding of gaps with silence to avoid audio drift over time, like in
+			// https://github.com/Vanilagy/mediabunny/issues/176
+			// TODO An open question is how encoders deal with the first AudioData having a non-zero timestamp, and with
+			// AudioDatas that have an overlapping timestamp range.
+			{
+				const startSampleIndex = Math.round(
+					audioSample.timestamp * audioSample.sampleRate,
+				);
+				const endSampleIndex = Math.round(
+					(audioSample.timestamp + audioSample.duration) * audioSample.sampleRate,
+				);
+
+				if (this.lastEndSampleIndex !== null && startSampleIndex > this.lastEndSampleIndex) {
+					const sampleCount = startSampleIndex - this.lastEndSampleIndex;
+					const fillSample = new AudioSample({
+						data: new Float32Array(sampleCount * audioSample.numberOfChannels),
+						format: 'f32-planar',
+						sampleRate: audioSample.sampleRate,
+						numberOfChannels: audioSample.numberOfChannels,
+						numberOfFrames: sampleCount,
+						timestamp: this.lastEndSampleIndex / audioSample.sampleRate,
+					});
+
+					await this.add(fillSample, true); // Recursive call
+				}
+
+				this.lastEndSampleIndex = endSampleIndex;
+			}
+
 			if (this.customEncoder) {
 				this.customEncoderQueueSize++;