Allow negative timestamps in media sources, add negative timestamp support to ISOBMFF muxer

dev/mux.html

@@ -47,7 +47,7 @@
 	format = new Mediabunny.MkvOutputFormat();
 	format = new Mediabunny.MovOutputFormat();
 	format = new Mediabunny.Mp4OutputFormat({ fastStart: 'reserve' });
-	format = new Mediabunny.Mp4OutputFormat();
+	format = new Mediabunny.Mp4OutputFormat({  });
 	let target = new Mediabunny.BufferTarget();
 
 	/*
@@ -191,17 +191,17 @@ Testing... <00:17.350>One... <00:18.125>Two...
 9. <b>justify (bottom, right)</b>.
 `;
 
-	subtitleSource.add(simpleWebvttFile);
-	subtitleSource.close();
+	//subtitleSource.add(simpleWebvttFile);
+	//subtitleSource.close();
 	
 	const p = document.createElement('p');
 	document.body.append(p);
 
-	for (let i = 0; i < 100; i++) {
+	for (let i = 0; i < 1; i++) {
 		context.fillStyle = ['red', 'green', 'blue', 'yellow'][i % 4];
 		context.fillRect(canvas.width * Math.random(), canvas.height * Math.random(), canvas.width * Math.random(), canvas.height * Math.random());
 
-		await videoSource.add(i / 10, 1 / 10);
+		await videoSource.add(i / 10 - 1, 1 / 10);
 
 		p.textContent = i;
 	}
@@ -211,7 +211,7 @@ Testing... <00:17.350>One... <00:18.125>Two...
 	let length = 10;
 	let slicedAudioBuffer = sliceAudioBuffer(audioBuffer, length, audioContext);
 
-	await audioSource.add(slicedAudioBuffer);
+	//await audioSource.add(slicedAudioBuffer);
 
 	await output.finalize();
 

examples/media-player/media-player.ts

@@ -128,6 +128,9 @@ const initMediaPlayer = async (resource: File | string) => {
 		isRelativeToUnixEpoch = (await Promise.all(tracks.map(t => t.isRelativeToUnixEpoch()))).some(Boolean);
 		playbackTimeAtStart = firstTimestamp;
 
+		// For degenerate cases where the end timestamp is less than 0
+		endTimestamp = Math.max(firstTimestamp, endTimestamp);
+
 		// Configure the time display elements accordingly
 		const timestampFontSize = isRelativeToUnixEpoch ? '12px' : '';
 		const timestampWhiteSpace = isRelativeToUnixEpoch ? 'pre' : '';

src/index.ts

@@ -73,6 +73,7 @@ export {
 	AudioSource,
 	SubtitleSource,
 	AudioBufferSource,
+	type AudioBufferSourceOptions,
 	AudioSampleSource,
 	CanvasSource,
 	EncodedAudioPacketSource,

src/isobmff/isobmff-boxes.ts

@@ -9,6 +9,7 @@
 import {
 	toUint8Array,
 	assert,
+	isI32,
 	isU32,
 	last,
 	TransformationMatrix,
@@ -356,7 +357,8 @@ export const styp = () => box('styp', [
 
 /** Segment Index Box */
 export const sidx = (muxer: IsobmffMuxer, referencedSize: number) => {
-	let duration = muxer.maxWrittenEndTimestamp - muxer.minWrittenTimestamp;
+	const earliestPresentationTime = Math.max(0, muxer.minWrittenTimestamp);
+	let duration = Math.max(0, muxer.maxWrittenEndTimestamp - earliestPresentationTime);
 	if (!Number.isFinite(duration)) {
 		duration = 0;
 	}
@@ -364,7 +366,7 @@ export const sidx = (muxer: IsobmffMuxer, referencedSize: number) => {
 	return fullBox('sidx', 1, 0, [
 		u32(1), // Reference ID
 		u32(GLOBAL_TIMESCALE), // Timescale
-		u64(intoTimescale(muxer.minWrittenTimestamp, GLOBAL_TIMESCALE)), // Earliest presentation time
+		u64(intoTimescale(earliestPresentationTime, GLOBAL_TIMESCALE)), // Earliest presentation time
 		u64(0), // First offset
 		u16(0), // Reserved
 		u16(1), // Reference count
@@ -402,8 +404,12 @@ export const mvhd = (
 		0,
 		...trackDatas
 			.map(trackData => (
-				intoTimescale(presentationSpan(trackData), GLOBAL_TIMESCALE)
-				+ intoTimescale(trackData.startTimestampOffset ?? 0, GLOBAL_TIMESCALE)
+				// Round separately to match the edit list
+				Math.max(
+					0,
+					intoTimescale(presentationSpan(trackData), GLOBAL_TIMESCALE)
+					+ intoTimescale(trackData.startTimestampOffset ?? 0, GLOBAL_TIMESCALE),
+				)
 			)),
 	);
 	const nextTrackId = Math.max(0, ...trackDatas.map(x => x.track.id)) + 1;
@@ -459,11 +465,11 @@ const presentationSpan = (trackData: IsobmffTrackData) => {
  */
 export const trak = (trackData: IsobmffTrackData, creationTime: number) => {
 	const trackMetadata = getTrackMetadata(trackData);
-	const needsEditList = trackData.startTimestampOffset !== null && trackData.startTimestampOffset > 0;
+	const needsEditList = trackData.startTimestampOffset !== null && trackData.startTimestampOffset !== 0;
 
 	return box('trak', undefined, [
 		tkhd(trackData, creationTime),
-		needsEditList ? edts(trackData, trackData.startTimestampOffset!) : null,
+		needsEditList ? edts(trackData) : null,
 		mdia(trackData, creationTime),
 		trackMetadata.name !== undefined
 			? box('udta', undefined, [
@@ -480,8 +486,12 @@ export const tkhd = (
 	trackData: IsobmffTrackData,
 	creationTime: number,
 ) => {
-	const durationInGlobalTimescale = intoTimescale(presentationSpan(trackData), GLOBAL_TIMESCALE)
-		+ intoTimescale(trackData.startTimestampOffset ?? 0, GLOBAL_TIMESCALE);
+	// Round separately to match the edit list
+	const durationInGlobalTimescale = Math.max(
+		0,
+		intoTimescale(presentationSpan(trackData), GLOBAL_TIMESCALE)
+		+ intoTimescale(trackData.startTimestampOffset ?? 0, GLOBAL_TIMESCALE),
+	);
 
 	const needsU64 = !isU32(creationTime) || !isU32(durationInGlobalTimescale);
 	const u32OrU64 = needsU64 ? u64 : u32;
@@ -528,29 +538,63 @@ export const tkhd = (
 };
 
 /** Edit Box: Specifies edits to the track's media. */
-export const edts = (trackData: IsobmffTrackData, offset: number) => {
-	const startOffset = intoTimescale(offset, GLOBAL_TIMESCALE);
-	const mediaDuration = intoTimescale(presentationSpan(trackData), GLOBAL_TIMESCALE);
+export const edts = (trackData: IsobmffTrackData) => {
+	const offset = trackData.startTimestampOffset;
+	assert(offset !== null);
 
-	const needs64Bits = !isU32(startOffset) || !isU32(mediaDuration);
-	const u32OrU64 = needs64Bits ? u64 : u32;
-	const i32OrI64 = needs64Bits ? i64 : i32;
+	if (offset > 0) {
+		// Positive offset: empty segment at the start, then the full media afterwards
 
-	return box('edts', undefined, [
-		fullBox('elst', needs64Bits ? 1 : 0, 0, [
-			u32(2), // Entry count
+		const startOffset = intoTimescale(offset, GLOBAL_TIMESCALE);
+		const mediaDuration = intoTimescale(presentationSpan(trackData), GLOBAL_TIMESCALE);
 
-			// #1
-			u32OrU64(startOffset), // Segment duration
-			i32OrI64(-1), // Media time
-			fixed_16_16(1), // Media rate
+		const needs64Bits = !isU32(startOffset) || !isU32(mediaDuration);
+		const u32OrU64 = needs64Bits ? u64 : u32;
+		const i32OrI64 = needs64Bits ? i64 : i32;
 
-			// #2
-			u32OrU64(mediaDuration), // Segment duration
-			i32OrI64(0), // Media time
-			fixed_16_16(1), // Media rate
-		]),
-	]);
+		return box('edts', undefined, [
+			fullBox('elst', needs64Bits ? 1 : 0, 0, [
+				u32(2), // Entry count
+
+				// #1
+				u32OrU64(startOffset), // Segment duration
+				i32OrI64(-1), // Media time
+				fixed_16_16(1), // Media rate
+
+				// #2
+				u32OrU64(mediaDuration), // Segment duration
+				i32OrI64(0), // Media time
+				fixed_16_16(1), // Media rate
+			]),
+		]);
+	} else {
+		// Negative offset: the negative section of the media is trimmed off
+
+		const mediaTime = intoTimescale(-offset, trackData.timescale);
+		// Not the entire media is visible.
+		// For fragmented files, this value is zero, which simply means "unknown duration" in this case. Spec:
+		// "the segment_duration of this edit may be zero"
+		const mediaDuration = Math.max(
+			0,
+			intoTimescale(presentationSpan(trackData), GLOBAL_TIMESCALE)
+			+ intoTimescale(offset, GLOBAL_TIMESCALE),
+		);
+
+		const needs64Bits = !isI32(mediaTime) || !isU32(mediaDuration);
+		const u32OrU64 = needs64Bits ? u64 : u32;
+		const i32OrI64 = needs64Bits ? i64 : i32;
+
+		return box('edts', undefined, [
+			fullBox('elst', needs64Bits ? 1 : 0, 0, [
+				u32(1), // Entry count
+
+				// #1
+				u32OrU64(mediaDuration), // Segment duration
+				i32OrI64(mediaTime), // Media time
+				fixed_16_16(1), // Media rate
+			]),
+		]);
+	}
 };
 
 /** Media Box: Describes and define a track's media type and sample data. */
@@ -565,7 +609,7 @@ export const mdhd = (
 	trackData: IsobmffTrackData,
 	creationTime: number,
 ) => {
-	// Since the duration represents the raw media duration, edit list offsets are not taken into account here
+	// Since _this_ duration represents the raw media duration, edit list offsets are not taken into account here
 	const localDuration = intoTimescale(
 		presentationSpan(trackData),
 		trackData.timescale,

src/isobmff/isobmff-demuxer.ts

@@ -953,11 +953,6 @@ export class IsobmffDemuxer extends Demuxer {
 						: readI32Be(slice);
 					const mediaRate = readFixed_16_16(slice);
 
-					if (segmentDuration === 0) {
-						// Don't care
-						continue;
-					}
-
 					if (relevantEntryFound) {
 						Logging._warn(
 							'Unsupported edit list: multiple edits are not currently supported. Only using first edit.',

src/isobmff/isobmff-muxer.ts

@@ -139,7 +139,7 @@ export type IsobmffTrackData = {
 	info: {
 		config: SubtitleConfig;
 	};
-	lastCueEndTimestamp: number;
+	lastCueEndTimestamp: number | null;
 	cueQueue: SubtitleCue[];
 	nextSourceId: number;
 	cueToSourceId: WeakMap<SubtitleCue, number>;
@@ -613,7 +613,7 @@ export class IsobmffMuxer extends Muxer {
 			compactlyCodedChunkTable: [],
 			closed: false,
 
-			lastCueEndTimestamp: 0,
+			lastCueEndTimestamp: null,
 			cueQueue: [],
 			nextSourceId: 0,
 			cueToSourceId: new WeakMap(),
@@ -790,6 +790,8 @@ export class IsobmffMuxer extends Muxer {
 		// overlapping samples require special logic. The algorithm produces the format specified in ISO 14496-30.
 
 		while (trackData.cueQueue.length > 0) {
+			trackData.lastCueEndTimestamp ??= Math.min(0, trackData.cueQueue[0]!.timestamp);
+
 			const timestamps = new Set<number>([]);
 			for (const cue of trackData.cueQueue) {
 				assert(cue.timestamp <= until);
@@ -904,10 +906,10 @@ export class IsobmffMuxer extends Muxer {
 		}
 
 		if (trackData.type === 'audio' && trackData.info.requiresPcmTransformation) {
-			if (!this.isFragmented) {
-				// The first timestamp is the lowest
-				trackData.startTimestampOffset ??= trackData.timestampProcessingQueue[0]!.timestamp;
-			}
+			assert(!this.isFragmented);
+
+			// The first timestamp is the lowest
+			trackData.startTimestampOffset ??= trackData.timestampProcessingQueue[0]!.timestamp;
 
 			let totalDuration = 0;
 
@@ -936,7 +938,9 @@ export class IsobmffMuxer extends Muxer {
 
 		const sortedTimestamps = trackData.timestampProcessingQueue.map(x => x.timestamp).sort((a, b) => a - b);
 
-		if (!this.isFragmented) {
+		if (this.isFragmented) {
+			trackData.startTimestampOffset ??= Math.min(sortedTimestamps[0]!, 0);
+		} else {
 			trackData.startTimestampOffset ??= sortedTimestamps[0]!;
 		}
 
@@ -1292,16 +1296,22 @@ export class IsobmffMuxer extends Muxer {
 		let fragmentStartTimestamp = Infinity;
 		for (let i = 0; i < tracksInFragment.length; i++) {
 			const trackData = tracksInFragment[i]!;
+			assert(trackData.currentChunk);
+			assert(trackData.startTimestampOffset !== null);
 
-			trackData.currentChunk!.offset = currentPos;
-			trackData.currentChunk!.moofOffset = moofOffset;
-			trackData.currentChunk!.trafIndex = i;
+			trackData.currentChunk.offset = currentPos;
+			trackData.currentChunk.moofOffset = moofOffset;
+			trackData.currentChunk.trafIndex = i;
+			trackData.currentChunk.startTimestamp -= trackData.startTimestampOffset;
 
-			for (const sample of trackData.currentChunk!.samples) {
+			for (const sample of trackData.currentChunk.samples) {
 				currentPos += sample.size;
+
+				sample.timestamp -= trackData.startTimestampOffset;
+				sample.decodeTimestamp -= trackData.startTimestampOffset;
 			}
 
-			fragmentStartTimestamp = Math.min(fragmentStartTimestamp, trackData.currentChunk!.startTimestamp);
+			fragmentStartTimestamp = Math.min(fragmentStartTimestamp, trackData.currentChunk.startTimestamp);
 		}
 
 		const mdatSize = currentPos - mdatStartPos;

src/media-source.ts

@@ -1340,8 +1340,8 @@ export class CanvasSource extends VideoSource {
 	 * to respect writer and encoder backpressure.
 	 */
 	add(timestamp: number, duration = 0, encodeOptions?: VideoEncoderEncodeOptions) {
-		if (!Number.isFinite(timestamp) || timestamp < 0) {
-			throw new TypeError('timestamp must be a non-negative number.');
+		if (!Number.isFinite(timestamp)) {
+			throw new TypeError('timestamp must be a finite number.');
 		}
 		if (!Number.isFinite(duration) || duration < 0) {
 			throw new TypeError('duration must be a non-negative number.');
@@ -2486,6 +2486,19 @@ export class AudioSampleSource extends AudioSource {
 	}
 }
 
+/**
+ * Options for {@link AudioBufferSource}.
+ * @group Media sources
+ * @public
+ */
+export type AudioBufferSourceOptions = {
+	/**
+	 * The timestamp of the first `AudioBuffer`, in seconds. Subsequent buffers are placed directly after the previous
+	 * one. Defaults to 0.
+	 */
+	startTimestamp?: number;
+};
+
 /**
  * This source can be used to add audio data from an AudioBuffer to the output track. This is useful when working with
  * the Web Audio API.
@@ -2496,23 +2509,30 @@ export class AudioBufferSource extends AudioSource {
 	/** @internal */
 	private _encoder: AudioEncoderWrapper;
 	/** @internal */
-	private _accumulatedTime = 0;
+	private _accumulatedTime: number;
 
 	/**
 	 * Creates a new {@link AudioBufferSource} whose `AudioBuffer` instances are encoded according to the specified
-	 * {@link AudioEncodingConfig}.
+	 * {@link AudioEncodingConfig} and {@link AudioBufferSourceOptions}.
 	 */
-	constructor(encodingConfig: AudioEncodingConfig) {
+	constructor(encodingConfig: AudioEncodingConfig, options: AudioBufferSourceOptions = {}) {
 		validateAudioEncodingConfig(encodingConfig);
+		if (typeof options !== 'object' || !options) {
+			throw new TypeError('options must be an object.');
+		}
+		if (options.startTimestamp !== undefined && !Number.isFinite(options.startTimestamp)) {
+			throw new TypeError('options.startTimestamp, when provided, must be a finite number.');
+		}
 
 		super(encodingConfig.codec);
 		this._encoder = new AudioEncoderWrapper(this, encodingConfig);
+		this._accumulatedTime = options.startTimestamp ?? 0;
 	}
 
 	/**
-	 * Converts an AudioBuffer to audio samples, encodes them and adds them to the output. The first AudioBuffer will
-	 * be played at timestamp 0, and any subsequent AudioBuffer will have a timestamp equal to the total duration of
-	 * all previous AudioBuffers.
+	 * Converts an AudioBuffer to audio samples, encodes them and adds them to the output. The first `AudioBuffer` will
+	 * be played at the configured start timestamp (the default is 0), and each subsequent `AudioBuffer` will be placed
+	 * directly after the previous one.
 	 *
 	 * @returns A Promise that resolves once the output is ready to receive more samples. You should await this Promise
 	 * to respect writer and encoder backpressure.

src/misc.ts

@@ -42,6 +42,10 @@ export const isU32 = (value: number) => {
 	return value >= 0 && value < 2 ** 32;
 };
 
+export const isI32 = (value: number) => {
+	return value >= -(2 ** 31) && value < 2 ** 31;
+};
+
 /** Reads an exponential-Golomb universal code from a Bitstream.  */
 export const readExpGolomb = (bitstream: Bitstream) => {
 	let leadingZeroBits = 0;

src/muxer.ts

@@ -43,10 +43,6 @@ export abstract class Muxer {
 	}>();
 
 	protected validateTimestamp(track: OutputTrack, timestampInSeconds: number, isKeyPacket: boolean) {
-		if (timestampInSeconds < 0) {
-			throw new Error(`Timestamps must be non-negative (got ${timestampInSeconds}s).`);
-		}
-
 		let timestampInfo = this.trackTimestampInfo.get(track);
 		if (!timestampInfo) {
 			if (!isKeyPacket) {

test/node/isobmff-muxer.test.ts

@@ -220,6 +220,77 @@ test('Non-zero start timestamp, fragmented MP4', async () => {
 	expect(durations).toEqual([0.1, 0.1, 0.1, 0.1]);
 });
 
+test('Negative start timestamps, regular MP4', async () => {
+	await testNegativeTimestampRoundTrip([-1, 0, 1, 2, 3], 1, false);
+});
+
+test('Negative start timestamps, fragmented MP4', async () => {
+	await testNegativeTimestampRoundTrip([-1, 0, 1, 2, 3], 1, true);
+});
+
+test('Wholly negative timestamps, regular MP4', async () => {
+	await testNegativeTimestampRoundTrip([-1, -0.9, -0.8, -0.7, -0.6], 0.1, false);
+});
+
+test('Wholly negative timestamps, fragmented MP4', async () => {
+	await testNegativeTimestampRoundTrip([-1, -0.9, -0.8, -0.7, -0.6], 0.1, true);
+});
+
+const testNegativeTimestampRoundTrip = async (
+	timestamps: number[],
+	duration: number,
+	fragmented: boolean,
+) => {
+	const output = new Output({
+		format: new Mp4OutputFormat({ fastStart: fragmented ? 'fragmented' : false }),
+		target: new BufferTarget(),
+	});
+
+	const source = new EncodedVideoPacketSource('vp8');
+	output.addVideoTrack(source);
+
+	await output.start();
+
+	const meta = { decoderConfig: { codec: 'vp8', codedWidth: 1280, codedHeight: 720 } };
+	const inputPackets = timestamps.map((timestamp, index) => new EncodedPacket(
+		new Uint8Array(1024).fill(index),
+		'key',
+		timestamp,
+		duration,
+	));
+
+	for (let i = 0; i < inputPackets.length; i++) {
+		await source.add(inputPackets[i]!, i === 0 ? meta : undefined);
+	}
+
+	await output.finalize();
+
+	using input = new Input({
+		source: new BufferSource(output.target.buffer!),
+		formats: ALL_FORMATS,
+	});
+
+	const track = await input.getPrimaryVideoTrack();
+	assert(track);
+
+	const outputPackets: EncodedPacket[] = [];
+	for await (const packet of new EncodedPacketSink(track).packets()) {
+		outputPackets.push(packet);
+	}
+
+	expect(outputPackets.map(packet => ({
+		data: packet.data,
+		type: packet.type,
+		timestamp: packet.timestamp,
+		duration: packet.duration,
+	}))).toEqual(inputPackets.map(packet => ({
+		data: packet.data,
+		type: packet.type,
+		timestamp: packet.timestamp,
+		duration: packet.duration,
+	})));
+};
+
 test('PCM audio', async () => {
 	const output = new Output({
 		format: new Mp4OutputFormat(),