Fix decoded audio always starting at 0, fix startTimestamp=0 for ranged media iterators (fixes #370)

dev/demux.html

@@ -17,6 +17,30 @@
 			source: new Mediabunny.BlobSource(file),
 		});
 
+		const track = await input.getPrimaryAudioTrack();
+		const packetSink = new Mediabunny.EncodedPacketSink(track);
+
+		for await (const packet of packetSink.packets()) {
+			//console.log(packet);
+			break;
+		}
+
+		const sink = new Mediabunny.AudioSampleSink(track);
+
+		let count = 0;
+		for await (const sample of sink.samples()) {
+			const buf = new Float32Array(new ArrayBuffer(sample.allocationSize({ format: 'f32-planar', planeIndex: 0 })));
+			sample.copyTo(buf, { format: 'f32-planar', planeIndex: 0 });
+			let max = Math.max(...buf.map(x => Math.abs(x)));
+
+			console.log(sample, buf, max)
+			
+			if (++count === 3) {
+				break;
+			}
+		}
+
+		/*
 		const track = await input.getPrimaryAudioTrack();
 		const sink = new Mediabunny.EncodedPacketSink(track);
 		
@@ -31,9 +55,9 @@
 
 		const conversion = await Mediabunny.Conversion.init({ input, output });
 		await conversion.execute();
-
+		
 		return;
-
+		
 		// Download it now
 		const blob = new Blob([output.target.buffer]);
 		const url = URL.createObjectURL(blob);
@@ -42,6 +66,7 @@
 		a.download = file.name.replace(/\.\w+$/, '.mp4');
 		a.click();
 		URL.revokeObjectURL(url);
+		*/
 
 		/*
 		const track = await input.getPrimaryAudioTrack();

src/media-sink.ts

@@ -451,7 +451,7 @@ export abstract class BaseMediaSampleSink<
 
 	/** @internal */
 	protected mediaSamplesInRange(
-		startTimestamp = 0,
+		startTimestamp = -Infinity,
 		endTimestamp = Infinity,
 		options: PacketRetrievalOptions,
 	): AsyncGenerator<MediaSample, void, unknown> {
@@ -1811,7 +1811,7 @@ export class VideoSampleSink extends BaseMediaSampleSink<VideoSample> {
 	 * @param endTimestamp - The timestamp in seconds at which to stop yielding samples (exclusive).
 	 * @param options - Options used for the underlying packet retrieval.
 	 */
-	samples(startTimestamp = 0, endTimestamp = Infinity, options: PacketRetrievalOptions = {}) {
+	samples(startTimestamp?: number, endTimestamp?: number, options: PacketRetrievalOptions = {}) {
 		return this.mediaSamplesInRange(startTimestamp, endTimestamp, options);
 	}
 
@@ -2109,7 +2109,7 @@ export class CanvasSink {
 	 * @param endTimestamp - The timestamp in seconds at which to stop yielding canvases (exclusive).
 	 * @param options - Options used for the underlying packet retrieval.
 	 */
-	async* canvases(startTimestamp = 0, endTimestamp = Infinity, options?: PacketRetrievalOptions) {
+	async* canvases(startTimestamp?: number, endTimestamp?: number, options?: PacketRetrievalOptions) {
 		await this._ensureInit();
 		yield* mapAsyncGenerator(
 			this._videoSampleSink.samples(startTimestamp, endTimestamp, options),
@@ -2148,6 +2148,9 @@ class AudioDecoderWrapper extends DecoderWrapper<AudioSample> {
 	// Internal state to accumulate a precise current timestamp based on audio durations, not the (potentially
 	// inaccurate) packet timestamps.
 	currentTimestamp: number | null = null;
+	// Chromium does not respect negative packet timestamps, so we must do the fixin' ourselves
+	expectedFirstTimestamp: number | null = null;
+	timestampOffset = 0;
 
 	constructor(
 		onSample: (sample: AudioSample) => unknown,
@@ -2158,12 +2161,20 @@ class AudioDecoderWrapper extends DecoderWrapper<AudioSample> {
 		super(onSample, onError);
 
 		const sampleHandler = (sample: AudioSample) => {
+			let sampleTimestamp = sample.timestamp;
+
+			if (this.expectedFirstTimestamp && this.currentTimestamp === null) {
+				this.timestampOffset = this.expectedFirstTimestamp - sampleTimestamp; ;
+			}
+
+			sampleTimestamp += this.timestampOffset;
+
 			if (
 				this.currentTimestamp === null
-				|| Math.abs(sample.timestamp - this.currentTimestamp) >= sample.duration
+				|| Math.abs(sampleTimestamp - this.currentTimestamp) >= sample.duration
 			) {
 				// We need to sync with the sample timestamp again
-				this.currentTimestamp = sample.timestamp;
+				this.currentTimestamp = sampleTimestamp;
 			}
 
 			const preciseTimestamp = this.currentTimestamp;
@@ -2238,17 +2249,23 @@ class AudioDecoderWrapper extends DecoderWrapper<AudioSample> {
 				.then(() => this.customDecoderQueueSize--);
 		} else {
 			assert(this.decoder);
+
+			this.expectedFirstTimestamp ??= packet.timestamp;
 			this.decoder.decode(packet.toEncodedAudioChunk());
 		}
 	}
 
-	flush() {
+	async flush() {
 		if (this.customDecoder) {
-			return this.customDecoderCallSerializer.call(() => this.customDecoder!.flush());
+			await this.customDecoderCallSerializer.call(() => this.customDecoder!.flush());
 		} else {
 			assert(this.decoder);
-			return this.decoder.flush();
+			await this.decoder.flush();
 		}
+
+		this.currentTimestamp = null;
+		this.expectedFirstTimestamp = null;
+		this.timestampOffset = 0;
 	}
 
 	close() {
@@ -2519,7 +2536,7 @@ export class AudioSampleSink extends BaseMediaSampleSink<AudioSample> {
 	 * @param endTimestamp - The timestamp in seconds at which to stop yielding samples (exclusive).
 	 * @param options - Options used for the underlying packet retrieval.
 	 */
-	samples(startTimestamp = 0, endTimestamp = Infinity, options: PacketRetrievalOptions = {}) {
+	samples(startTimestamp?: number, endTimestamp?: number, options: PacketRetrievalOptions = {}) {
 		return this.mediaSamplesInRange(startTimestamp, endTimestamp, options);
 	}
 
@@ -2609,7 +2626,7 @@ export class AudioBufferSink {
 	 * @param endTimestamp - The timestamp in seconds at which to stop yielding buffers (exclusive).
 	 * @param options - Options used for the underlying packet retrieval.
 	 */
-	buffers(startTimestamp = 0, endTimestamp = Infinity, options?: PacketRetrievalOptions) {
+	buffers(startTimestamp?: number, endTimestamp?: number, options?: PacketRetrievalOptions) {
 		return mapAsyncGenerator(
 			this._audioSampleSink.samples(startTimestamp, endTimestamp, options),
 			data => this._audioSampleToWrappedArrayBuffer(data),

test/browser/media-sinks.test.ts

@@ -0,0 +1,26 @@
+import { expect, test } from 'vitest';
+import { Input } from '../../src/input.js';
+import { UrlSource } from '../../src/source.js';
+import { ALL_FORMATS } from '../../src/input-format.js';
+import { assert } from '../../src/misc.js';
+import { AudioSampleSink } from '../../src/media-sink.js';
+
+// https://github.com/Vanilagy/mediabunny/issues/370
+test('Negative audio timestamps are preserved', async () => {
+	using input = new Input({
+		source: new UrlSource('/edts.mp4'),
+		formats: ALL_FORMATS,
+	});
+
+	const track = await input.getPrimaryAudioTrack();
+	assert(track);
+
+	expect(await track.getFirstTimestamp()).toBeLessThan(0);
+
+	const sink = new AudioSampleSink(track);
+
+	for await (using sample of sink.samples()) {
+		expect(sample.timestamp).toBe(await track.getFirstTimestamp());
+		break;
+	}
+});