diff --git a/dev/convert.html b/dev/convert.html index 79b7ae3..bb9b7a4 100644 --- a/dev/convert.html +++ b/dev/convert.html @@ -57,6 +57,7 @@ output, audio: (_, n) => ({ discard: n > 1, + codec: 'aac', //codec: 'opus', /* process: (sample) => { @@ -72,7 +73,7 @@ //numberOfChannels: 1, //sampleRate: 4000 //discard: true - //forceTranscode: true, + forceTranscode: true, }), /* video: { @@ -100,6 +101,7 @@ */ video: () => ({ //discard: true, + /* process: (sample) => { if (!ctx) { // Create a canvas for image compositing @@ -118,6 +120,7 @@ return ctx.canvas; }, + */ //width: 300, //alpha: 'keep', //width: 320, diff --git a/dev/demux.html b/dev/demux.html index 1a864a3..c423bfe 100644 --- a/dev/demux.html +++ b/dev/demux.html @@ -14,7 +14,21 @@ source: new Mediabunny.BlobSource(file), }); - const videoTrack = await input.getPrimaryAudioTrack(); + const audioTrack = await input.getPrimaryAudioTrack(); + const sink = new Mediabunny.AudioSampleSink(audioTrack); + + let lastEnd = 0; + for await (const sample of sink.samples()) { + if (sample.timestamp - lastEnd > 0) { + console.warn(sample.timestamp - lastEnd) + console.log(sample.timestamp, sample.duration, "diffie", sample.timestamp - lastEnd); + } + + lastEnd = sample.timestamp + sample.duration; + sample.close(); + } + + /* const sink = new Mediabunny.EncodedPacketSink(videoTrack); for await (const packet of sink.packets()) { @@ -22,6 +36,7 @@ if (packet.timestamp > 10) break; } + */ /* const sink = new Mediabunny.VideoSampleSink(videoTrack); diff --git a/src/media-source.ts b/src/media-source.ts index 86989a2..c2b4a6b 100644 --- a/src/media-source.ts +++ b/src/media-source.ts @@ -1294,6 +1294,8 @@ class AudioEncoderWrapper { private customEncoderCallSerializer = new CallSerializer(); private customEncoderQueueSize = 0; + private lastEndSampleIndex: number | null = null; + /** * Encoders typically throw their errors "out of band", meaning asynchronously in some other execution context. * However, we want to surface these errors to the user within the normal control flow, so they don't go uncaught. @@ -1342,6 +1344,35 @@ class AudioEncoderWrapper { } assert(this.encoderInitialized); + // Handle padding of gaps with silence to avoid audio drift over time, like in + // https://github.com/Vanilagy/mediabunny/issues/176 + // TODO An open question is how encoders deal with the first AudioData having a non-zero timestamp, and with + // AudioDatas that have an overlapping timestamp range. + { + const startSampleIndex = Math.round( + audioSample.timestamp * audioSample.sampleRate, + ); + const endSampleIndex = Math.round( + (audioSample.timestamp + audioSample.duration) * audioSample.sampleRate, + ); + + if (this.lastEndSampleIndex !== null && startSampleIndex > this.lastEndSampleIndex) { + const sampleCount = startSampleIndex - this.lastEndSampleIndex; + const fillSample = new AudioSample({ + data: new Float32Array(sampleCount * audioSample.numberOfChannels), + format: 'f32-planar', + sampleRate: audioSample.sampleRate, + numberOfChannels: audioSample.numberOfChannels, + numberOfFrames: sampleCount, + timestamp: this.lastEndSampleIndex / audioSample.sampleRate, + }); + + await this.add(fillSample, true); // Recursive call + } + + this.lastEndSampleIndex = endSampleIndex; + } + if (this.customEncoder) { this.customEncoderQueueSize++;