diff --git a/dev/demux.html b/dev/demux.html index 65aea09..ac9fba2 100644 --- a/dev/demux.html +++ b/dev/demux.html @@ -17,6 +17,30 @@ source: new Mediabunny.BlobSource(file), }); + const track = await input.getPrimaryAudioTrack(); + const packetSink = new Mediabunny.EncodedPacketSink(track); + + for await (const packet of packetSink.packets()) { + //console.log(packet); + break; + } + + const sink = new Mediabunny.AudioSampleSink(track); + + let count = 0; + for await (const sample of sink.samples()) { + const buf = new Float32Array(new ArrayBuffer(sample.allocationSize({ format: 'f32-planar', planeIndex: 0 }))); + sample.copyTo(buf, { format: 'f32-planar', planeIndex: 0 }); + let max = Math.max(...buf.map(x => Math.abs(x))); + + console.log(sample, buf, max) + + if (++count === 3) { + break; + } + } + + /* const track = await input.getPrimaryAudioTrack(); const sink = new Mediabunny.EncodedPacketSink(track); @@ -31,9 +55,9 @@ const conversion = await Mediabunny.Conversion.init({ input, output }); await conversion.execute(); - + return; - + // Download it now const blob = new Blob([output.target.buffer]); const url = URL.createObjectURL(blob); @@ -42,6 +66,7 @@ a.download = file.name.replace(/\.\w+$/, '.mp4'); a.click(); URL.revokeObjectURL(url); + */ /* const track = await input.getPrimaryAudioTrack(); diff --git a/src/media-sink.ts b/src/media-sink.ts index 61bc0b7..fda943f 100644 --- a/src/media-sink.ts +++ b/src/media-sink.ts @@ -451,7 +451,7 @@ export abstract class BaseMediaSampleSink< /** @internal */ protected mediaSamplesInRange( - startTimestamp = 0, + startTimestamp = -Infinity, endTimestamp = Infinity, options: PacketRetrievalOptions, ): AsyncGenerator { @@ -1811,7 +1811,7 @@ export class VideoSampleSink extends BaseMediaSampleSink { * @param endTimestamp - The timestamp in seconds at which to stop yielding samples (exclusive). * @param options - Options used for the underlying packet retrieval. */ - samples(startTimestamp = 0, endTimestamp = Infinity, options: PacketRetrievalOptions = {}) { + samples(startTimestamp?: number, endTimestamp?: number, options: PacketRetrievalOptions = {}) { return this.mediaSamplesInRange(startTimestamp, endTimestamp, options); } @@ -2109,7 +2109,7 @@ export class CanvasSink { * @param endTimestamp - The timestamp in seconds at which to stop yielding canvases (exclusive). * @param options - Options used for the underlying packet retrieval. */ - async* canvases(startTimestamp = 0, endTimestamp = Infinity, options?: PacketRetrievalOptions) { + async* canvases(startTimestamp?: number, endTimestamp?: number, options?: PacketRetrievalOptions) { await this._ensureInit(); yield* mapAsyncGenerator( this._videoSampleSink.samples(startTimestamp, endTimestamp, options), @@ -2148,6 +2148,9 @@ class AudioDecoderWrapper extends DecoderWrapper { // Internal state to accumulate a precise current timestamp based on audio durations, not the (potentially // inaccurate) packet timestamps. currentTimestamp: number | null = null; + // Chromium does not respect negative packet timestamps, so we must do the fixin' ourselves + expectedFirstTimestamp: number | null = null; + timestampOffset = 0; constructor( onSample: (sample: AudioSample) => unknown, @@ -2158,12 +2161,20 @@ class AudioDecoderWrapper extends DecoderWrapper { super(onSample, onError); const sampleHandler = (sample: AudioSample) => { + let sampleTimestamp = sample.timestamp; + + if (this.expectedFirstTimestamp && this.currentTimestamp === null) { + this.timestampOffset = this.expectedFirstTimestamp - sampleTimestamp; ; + } + + sampleTimestamp += this.timestampOffset; + if ( this.currentTimestamp === null - || Math.abs(sample.timestamp - this.currentTimestamp) >= sample.duration + || Math.abs(sampleTimestamp - this.currentTimestamp) >= sample.duration ) { // We need to sync with the sample timestamp again - this.currentTimestamp = sample.timestamp; + this.currentTimestamp = sampleTimestamp; } const preciseTimestamp = this.currentTimestamp; @@ -2238,17 +2249,23 @@ class AudioDecoderWrapper extends DecoderWrapper { .then(() => this.customDecoderQueueSize--); } else { assert(this.decoder); + + this.expectedFirstTimestamp ??= packet.timestamp; this.decoder.decode(packet.toEncodedAudioChunk()); } } - flush() { + async flush() { if (this.customDecoder) { - return this.customDecoderCallSerializer.call(() => this.customDecoder!.flush()); + await this.customDecoderCallSerializer.call(() => this.customDecoder!.flush()); } else { assert(this.decoder); - return this.decoder.flush(); + await this.decoder.flush(); } + + this.currentTimestamp = null; + this.expectedFirstTimestamp = null; + this.timestampOffset = 0; } close() { @@ -2519,7 +2536,7 @@ export class AudioSampleSink extends BaseMediaSampleSink { * @param endTimestamp - The timestamp in seconds at which to stop yielding samples (exclusive). * @param options - Options used for the underlying packet retrieval. */ - samples(startTimestamp = 0, endTimestamp = Infinity, options: PacketRetrievalOptions = {}) { + samples(startTimestamp?: number, endTimestamp?: number, options: PacketRetrievalOptions = {}) { return this.mediaSamplesInRange(startTimestamp, endTimestamp, options); } @@ -2609,7 +2626,7 @@ export class AudioBufferSink { * @param endTimestamp - The timestamp in seconds at which to stop yielding buffers (exclusive). * @param options - Options used for the underlying packet retrieval. */ - buffers(startTimestamp = 0, endTimestamp = Infinity, options?: PacketRetrievalOptions) { + buffers(startTimestamp?: number, endTimestamp?: number, options?: PacketRetrievalOptions) { return mapAsyncGenerator( this._audioSampleSink.samples(startTimestamp, endTimestamp, options), data => this._audioSampleToWrappedArrayBuffer(data), diff --git a/test/browser/media-sinks.test.ts b/test/browser/media-sinks.test.ts new file mode 100644 index 0000000..3a8840b --- /dev/null +++ b/test/browser/media-sinks.test.ts @@ -0,0 +1,26 @@ +import { expect, test } from 'vitest'; +import { Input } from '../../src/input.js'; +import { UrlSource } from '../../src/source.js'; +import { ALL_FORMATS } from '../../src/input-format.js'; +import { assert } from '../../src/misc.js'; +import { AudioSampleSink } from '../../src/media-sink.js'; + +// https://github.com/Vanilagy/mediabunny/issues/370 +test('Negative audio timestamps are preserved', async () => { + using input = new Input({ + source: new UrlSource('/edts.mp4'), + formats: ALL_FORMATS, + }); + + const track = await input.getPrimaryAudioTrack(); + assert(track); + + expect(await track.getFirstTimestamp()).toBeLessThan(0); + + const sink = new AudioSampleSink(track); + + for await (using sample of sink.samples()) { + expect(sample.timestamp).toBe(await track.getFirstTimestamp()); + break; + } +}); diff --git a/test/public/edts.mp4 b/test/public/edts.mp4 new file mode 100644 index 0000000..49fb44c Binary files /dev/null and b/test/public/edts.mp4 differ