Add negative timestamp support to MPEG-TS

src/misc.ts

@@ -458,6 +458,10 @@ export const lerp = (from: number, to: number, t: number) => {
 	return from + (to - from) * t;
 };
 
+export const modEuclid = (value: number, modulus: number) => {
+	return value - Math.floor(value / modulus) * modulus;
+};
+
 export const UNDETERMINED_LANGUAGE = 'und';
 
 export const roundIfAlmostInteger = (value: number) => {

src/mpeg-ts/mpeg-ts-demuxer.ts

@@ -80,7 +80,13 @@ import {
 } from '../../shared/mp3-misc';
 import { EncodedPacket, PacketType, PLACEHOLDER_DATA } from '../packet';
 import { FileSlice, readBytes, Reader, readU16Be, readU32Be, readU8 } from '../reader';
-import { buildMpegTsMimeType, MpegTsStreamType, TIMESCALE, TS_PACKET_SIZE } from './mpeg-ts-misc';
+import {
+	buildMpegTsMimeType,
+	MpegTsStreamType,
+	TIMESCALE,
+	TIMESTAMP_MODULUS,
+	TS_PACKET_SIZE,
+} from './mpeg-ts-misc';
 import { AC3_SAMPLE_RATES } from '../../shared/ac3-misc';
 import { Bitstream } from '../../shared/bitstream';
 
@@ -180,6 +186,10 @@ export class MpegTsDemuxer extends Demuxer {
 	sectionEndPositions: number[] = [];
 	seekChunkSize = 5 * 1024 * 1024; // 5 MiB, picked because most HLS segments are below this size
 	minReferencePointByteDistance = -1;
+	timestampWrapInfo: {
+		reference: number;
+		offset: number;
+	} | null = null;
 
 	constructor(input: Input) {
 		super(input);
@@ -508,7 +518,7 @@ export class MpegTsDemuxer extends Demuxer {
 					const elementaryStream = this.elementaryStreams.find(x => x.pid === section.pid);
 					outer:
 					if (elementaryStream && !elementaryStream.initialized) {
-						const pesPacket = readPesPacket(section, true);
+						const pesPacket = readPesPacket(this, section, true);
 						if (!pesPacket) {
 							throw new Error(
 								`Couldn't read first PES packet for Elementary Stream with PID ${elementaryStream.pid}`,
@@ -1023,6 +1033,30 @@ export class MpegTsDemuxer extends Demuxer {
 			body: bytes.subarray(4),
 		};
 	}
+
+	normalizeTimestamp(timestamp: number) {
+		if (!this.timestampWrapInfo) {
+			// No info yet, let's initialize it anchored on this timestamp. Since we process packets sequentially when
+			// extracting metadata, this means we'll anchor to the first PTS in the file.
+			const tolerance = 60 * TIMESCALE;
+			this.timestampWrapInfo = {
+				reference: timestamp - tolerance,
+				offset: timestamp >= TIMESTAMP_MODULUS - tolerance
+					? -TIMESTAMP_MODULUS // Timestamps close to the modulus are treated as negative
+					: TIMESTAMP_MODULUS,
+			};
+		}
+
+		const { reference, offset } = this.timestampWrapInfo;
+		if (
+			(offset < 0 && timestamp >= reference)
+			|| (offset > 0 && timestamp < reference)
+		) {
+			return timestamp + offset;
+		}
+
+		return timestamp;
+	}
 }
 
 type PesPacketHeader = {
@@ -1045,6 +1079,7 @@ type TimestampedPesPacket = PesPacket & {
 };
 
 const readPesPacketHeader = <T extends boolean>(
+	demuxer: MpegTsDemuxer,
 	section: Section,
 	expectPts: T,
 ): (T extends true ? TimestampedPesPacketHeader : PesPacketHeader) | null => {
@@ -1091,6 +1126,8 @@ const readPesPacketHeader = <T extends boolean>(
 		pts += bitstream.readBits(15) * (1 << 15);
 		bitstream.skipBits(1);
 		pts += bitstream.readBits(15);
+
+		pts = demuxer.normalizeTimestamp(pts);
 	} else {
 		if (expectPts) {
 			throw new Error(MISSING_PTS_ERROR_MESSAGE);
@@ -1106,12 +1143,13 @@ const readPesPacketHeader = <T extends boolean>(
 };
 
 const readPesPacket = <T extends boolean>(
+	demuxer: MpegTsDemuxer,
 	section: Section,
 	expectPts: T,
 ): (T extends true ? TimestampedPesPacket : PesPacket) | null => {
 	assert(section.endPos !== null); // Can only read full PES packets from fully read sections
 
-	const header = readPesPacketHeader(section, expectPts);
+	const header = readPesPacketHeader(demuxer, section, expectPts);
 	if (!header) {
 		return null;
 	}
@@ -1266,7 +1304,7 @@ abstract class MpegTsTrackBacking implements InputTrackBacking {
 		const section = this.elementaryStream.firstSection;
 		assert(section);
 
-		const pesPacket = readPesPacket(section, true);
+		const pesPacket = readPesPacket(this.elementaryStream.demuxer, section, true);
 		assert(pesPacket);
 
 		const context = new PacketReadingContext(this.elementaryStream, pesPacket);
@@ -1314,7 +1352,7 @@ abstract class MpegTsTrackBacking implements InputTrackBacking {
 		const section = await demuxer.readSection(sectionStartPos, true);
 		assert(section);
 
-		const pesPacket = readPesPacket(section, true);
+		const pesPacket = readPesPacket(demuxer, section, true);
 		assert(pesPacket);
 
 		const context = new PacketReadingContext(this.elementaryStream, pesPacket);
@@ -1398,7 +1436,7 @@ abstract class MpegTsTrackBacking implements InputTrackBacking {
 						return null;
 					}
 
-					const pesPacketHeader = readPesPacketHeader(section, false);
+					const pesPacketHeader = readPesPacketHeader(demuxer, section, false);
 					if (pesPacketHeader && pesPacketHeader.pts !== null) {
 						return {
 							pesPacketHeader: pesPacketHeader as TimestampedPesPacketHeader,
@@ -1416,7 +1454,7 @@ abstract class MpegTsTrackBacking implements InputTrackBacking {
 		// Get the first PES packet of the track
 		const firstSection = this.elementaryStream.firstSection;
 		assert(firstSection);
-		const firstPesPacketHeader = readPesPacketHeader(firstSection, true);
+		const firstPesPacketHeader = readPesPacketHeader(demuxer, firstSection, true);
 		assert(firstPesPacketHeader);
 
 		if (searchPts < firstPesPacketHeader.pts) {
@@ -1499,7 +1537,7 @@ abstract class MpegTsTrackBacking implements InputTrackBacking {
 			const section = await demuxer.readSection(sectionStartPos, true);
 			assert(section);
 
-			const pesPacket = readPesPacket(section, true);
+			const pesPacket = readPesPacket(demuxer, section, true);
 			assert(pesPacket);
 
 			const context = new PacketReadingContext(this.elementaryStream, pesPacket);
@@ -1563,7 +1601,7 @@ abstract class MpegTsTrackBacking implements InputTrackBacking {
 					if (packetHeader.pid === pid && packetHeader.payloadUnitStartIndicator === 1) {
 						const section = await demuxer.readSection(currentPos, false);
 						if (section) {
-							const nextPesHeader = readPesPacketHeader(section, false);
+							const nextPesHeader = readPesPacketHeader(demuxer, section, false);
 							if (nextPesHeader && nextPesHeader.pts !== null) {
 								if (nextPesHeader.pts > searchPts) {
 									break outer;
@@ -1595,7 +1633,7 @@ abstract class MpegTsTrackBacking implements InputTrackBacking {
 					if (packetHeader.pid === pid && packetHeader.payloadUnitStartIndicator === 1) {
 						const section = await demuxer.readSection(pos, false);
 						if (section) {
-							const header = readPesPacketHeader(section, false);
+							const header = readPesPacketHeader(demuxer, section, false);
 							if (header && header.pts !== null) {
 								currentPesHeader = header as TimestampedPesPacketHeader;
 								break;
@@ -1655,7 +1693,7 @@ abstract class MpegTsTrackBacking implements InputTrackBacking {
 							isKeyPacket = pesHeader.randomAccessIndicator === 1;
 						} else {
 							assert(pesHeaderSection);
-							const pesPacket = readPesPacket(pesHeaderSection, true);
+							const pesPacket = readPesPacket(demuxer, pesHeaderSection, true);
 							assert(pesPacket);
 
 							const context = new PacketReadingContext(this.elementaryStream, pesPacket);
@@ -1694,7 +1732,7 @@ abstract class MpegTsTrackBacking implements InputTrackBacking {
 						if (packetHeader.pid === pid && packetHeader.payloadUnitStartIndicator === 1) {
 							const section = await demuxer.readSection(currentPos, readSectionsInFull);
 							if (section) {
-								const nextPesHeader = readPesPacketHeader(section, false);
+								const nextPesHeader = readPesPacketHeader(demuxer, section, false);
 
 								if (nextPesHeader && nextPesHeader.pts !== null) {
 									pesHeader = nextPesHeader as TimestampedPesPacketHeader;
@@ -1728,7 +1766,7 @@ abstract class MpegTsTrackBacking implements InputTrackBacking {
 								if (packetHeader.pid === pid && packetHeader.payloadUnitStartIndicator === 1) {
 									const section = await demuxer.readSection(pos, readSectionsInFull);
 									if (section) {
-										const header = readPesPacketHeader(section, false);
+										const header = readPesPacketHeader(demuxer, section, false);
 										if (header && header.pts !== null) {
 											startPesHeader = header as TimestampedPesPacketHeader;
 											break;
@@ -1971,7 +2009,7 @@ class PacketReadingContext {
 							return;
 						}
 
-						const nextPesPacket = readPesPacket(nextSection, false);
+						const nextPesPacket = readPesPacket(this.demuxer, nextSection, false);
 						if (nextPesPacket) {
 							pesPacket = nextPesPacket;
 							break;

src/mpeg-ts/mpeg-ts-misc.ts

@@ -7,6 +7,7 @@
  */
 
 export const TIMESCALE = 90_000; // MPEG-TS timestamps run on a 90 kHz clock
+export const TIMESTAMP_MODULUS = 2 ** 33;
 export const TS_PACKET_SIZE = 188;
 
 export const enum MpegTsStreamType {

src/mpeg-ts/mpeg-ts-muxer.ts

@@ -24,13 +24,19 @@ import {
 	iterateNalUnitsInLengthPrefixed,
 } from '../codec-data';
 import { Bitstream } from '../../shared/bitstream';
-import { assert, promiseWithResolvers, setUint24, toDataView, toUint8Array } from '../misc';
+import { assert, modEuclid, promiseWithResolvers, setUint24, toDataView, toUint8Array } from '../misc';
 import { Muxer } from '../muxer';
 import { Output, OutputAudioTrack, OutputTrack, OutputVideoTrack } from '../output';
 import { MpegTsOutputFormat } from '../output-format';
 import { EncodedPacket } from '../packet';
 import { Writer } from '../writer';
-import { buildMpegTsMimeType, MpegTsStreamType, TIMESCALE, TS_PACKET_SIZE } from './mpeg-ts-misc';
+import {
+	buildMpegTsMimeType,
+	MpegTsStreamType,
+	TIMESCALE,
+	TIMESTAMP_MODULUS,
+	TS_PACKET_SIZE,
+} from './mpeg-ts-misc';
 
 // Resources:
 // ISO/IEC 13818-1
@@ -589,26 +595,30 @@ export class MpegTsMuxer extends Muxer {
 		pesView.setUint8(7, includeDts ? 0xC0 : 0x80); // PTS_DTS_flags, other flags=0
 		pesView.setUint8(8, headerDataLength); // PES_header_data_length
 
-		const pts = Math.round(queuedPacket.presentationTimestamp * TIMESCALE);
+		// This logic supports writing negative timestamps by simplying Euclid-modding them. It is then up to the
+		// demuxer to interpret them as negative, if it wasnts to.
+		const unwrappedPts = Math.round(queuedPacket.presentationTimestamp * TIMESCALE);
+		const pts = modEuclid(unwrappedPts, TIMESTAMP_MODULUS);
 		ptsDtsBitstream.pos = 0;
 		ptsDtsBitstream.writeBits(4, includeDts ? 0b0011 : 0b0010); // marker
-		ptsDtsBitstream.writeBits(3, (pts >>> 30) & 0x7); // PTS[32:30]
+		ptsDtsBitstream.writeBits(3, Math.floor(pts / 2 ** 30)); // PTS[32:30]
 		ptsDtsBitstream.writeBits(1, 1); // marker_bit
-		ptsDtsBitstream.writeBits(15, (pts >>> 15) & 0x7FFF); // PTS[29:15]
+		ptsDtsBitstream.writeBits(15, Math.floor(pts / 2 ** 15) % 2 ** 15); // PTS[29:15]
 		ptsDtsBitstream.writeBits(1, 1); // marker_bit
-		ptsDtsBitstream.writeBits(15, pts & 0x7FFF); // PTS[14:0]
+		ptsDtsBitstream.writeBits(15, pts % 2 ** 15); // PTS[14:0]
 		ptsDtsBitstream.writeBits(1, 1); // marker_bit
 
 		if (includeDts) {
 			assert(queuedPacket.decodeTimestamp !== null);
 
-			const dts = Math.round(queuedPacket.decodeTimestamp * TIMESCALE);
+			const unwrappedDts = Math.round(queuedPacket.decodeTimestamp * TIMESCALE);
+			const dts = modEuclid(unwrappedDts, TIMESTAMP_MODULUS);
 			ptsDtsBitstream.writeBits(4, 0b0001);
-			ptsDtsBitstream.writeBits(3, (dts >>> 30) & 0x7); // DTS[32:30]
+			ptsDtsBitstream.writeBits(3, Math.floor(dts / 2 ** 30)); // DTS[32:30]
 			ptsDtsBitstream.writeBits(1, 1); // marker_bit
-			ptsDtsBitstream.writeBits(15, (dts >>> 15) & 0x7FFF); // DTS[29:15]
+			ptsDtsBitstream.writeBits(15, Math.floor(dts / 2 ** 15) % 2 ** 15); // DTS[29:15]
 			ptsDtsBitstream.writeBits(1, 1); // marker_bit
-			ptsDtsBitstream.writeBits(15, dts & 0x7FFF); // DTS[14:0]
+			ptsDtsBitstream.writeBits(15, dts % 2 ** 15); // DTS[14:0]
 			ptsDtsBitstream.writeBits(1, 1); // marker_bit
 		}
 

test/browser/mpeg-ts-muxing.test.ts

@@ -738,3 +738,67 @@ test('MPEG-TS muxing with StreamTarget', async () => {
 	}
 	expect(videoPacketCount).toBe(frameCount);
 });
+
+test('MPEG-TS muxing with negative start timestamps', async () => {
+	await testNegativeTimestampRoundTrip(Array.from({ length: 50 }, (_, index) => (index - 10) / 10), 0.1);
+});
+
+test('MPEG-TS muxing with wholly negative timestamps', async () => {
+	await testNegativeTimestampRoundTrip([-1, -0.9, -0.8, -0.7, -0.6], 0.1);
+});
+
+const testNegativeTimestampRoundTrip = async (timestamps: number[], duration: number) => {
+	const output = new Output({
+		format: new MpegTsOutputFormat(),
+		target: new BufferTarget(),
+	});
+
+	const canvas = new OffscreenCanvas(640, 480);
+	const context = canvas.getContext('2d')!;
+	context.fillStyle = '#0000ff';
+	context.fillRect(0, 0, canvas.width, canvas.height);
+
+	const source = new CanvasSource(canvas, {
+		codec: 'avc',
+		quality: new Quality('high'),
+	});
+	output.addVideoTrack(source, { frameRate: 10 });
+
+	await output.start();
+
+	for (const timestamp of timestamps) {
+		await source.add(timestamp, duration);
+	}
+
+	await output.finalize();
+
+	using input = new Input({
+		source: new BufferSource(output.target.buffer!),
+		formats: ALL_FORMATS,
+	});
+
+	const track = await input.getPrimaryVideoTrack();
+	assert(track);
+	const sink = new EncodedPacketSink(track);
+
+	const outputPackets = [];
+	for await (const packet of sink.packets()) {
+		outputPackets.push(packet);
+	}
+
+	const expectedPackets = timestamps.map(timestamp => ({ timestamp, duration }));
+	expect(outputPackets.map(packet => ({
+		timestamp: packet.timestamp,
+		duration: packet.duration,
+	}))).toEqual(expectedPackets);
+
+	for (const expectedPacket of expectedPackets) {
+		const outputPacket = await sink.getPacket(expectedPacket.timestamp);
+		assert(outputPacket);
+
+		expect({
+			timestamp: outputPacket.timestamp,
+			duration: outputPacket.duration,
+		}).toEqual(expectedPacket);
+	}
+};