MPEG-TS muxer: Write out DTS for video tracks (fixes buggy QuickTime behavior)

dev/convert.html

@@ -26,7 +26,7 @@
 			chunked: true,
 			chunkSize: 2**20
 		});
-		const outputFormat = new Mediabunny.Mp4OutputFormat({});
+		const outputFormat = new Mediabunny.MpegTsOutputFormat({});
 
 		const button = document.createElement('button');
 		button.textContent = 'Cancel';
@@ -58,7 +58,8 @@
 			}),
 			output,
 			audio: (_, n) => ({
-				codec: 'eac3',
+				//codec: 'eac3',
+				//discard: true,
 				//discard: n > 1,
 				//codec: 'opus',
 				//codec: 'opus',
@@ -103,7 +104,8 @@
 			},
 			*/
 			video: () => ({
-				width: 320,
+				forceTranscode: true,
+				width: 1280,
 				//forceTranscode: true,
 				//allowRotationMetadata: false,
 				//width: 720,
@@ -186,7 +188,7 @@
 			},
 			trim: {
 				//start: 0,
-				end: 10
+				//end: 10
 			},
 		});
 		console.log(conversion);

src/mpeg-ts/mpeg-ts-demuxer.ts

@@ -72,6 +72,9 @@ import { FileSlice, readBytes, Reader, readU16Be, readU32Be, readU8 } from '../r
 import { buildMpegTsMimeType, MpegTsStreamType, TIMESCALE, TS_PACKET_SIZE } from './mpeg-ts-misc';
 import { AC3_SAMPLE_RATES } from '../../shared/ac3-misc';
 
+// Resources:
+// ISO/IEC 13818-1
+
 type ElementaryStream = {
 	demuxer: MpegTsDemuxer;
 	pid: number;

src/mpeg-ts/mpeg-ts-muxer.ts

@@ -25,12 +25,15 @@ import {
 } from '../codec-data';
 import { assert, Bitstream, promiseWithResolvers, setUint24, toDataView, toUint8Array } from '../misc';
 import { Muxer } from '../muxer';
-import { Output, OutputAudioTrack, OutputVideoTrack } from '../output';
+import { Output, OutputAudioTrack, OutputTrack, OutputVideoTrack } from '../output';
 import { MpegTsOutputFormat } from '../output-format';
 import { EncodedPacket } from '../packet';
 import { Writer } from '../writer';
 import { buildMpegTsMimeType, MpegTsStreamType, TIMESCALE, TS_PACKET_SIZE } from './mpeg-ts-misc';
 
+// Resources:
+// ISO/IEC 13818-1
+
 const PAT_PID = 0x0000;
 const PMT_PID = 0x1000;
 const FIRST_TRACK_PID = 0x0100;
@@ -47,6 +50,7 @@ type MpegTsTrackData = {
 	streamType: MpegTsStreamType;
 	streamId: number;
 	codecString: string;
+	timestampProcessingQueue: QueuedPacket[];
 	packetQueue: QueuedPacket[];
 	inputIsAnnexB: boolean | null;
 	inputIsAdts: boolean | null;
@@ -59,7 +63,8 @@ type MpegTsTrackData = {
 
 type QueuedPacket = {
 	data: Uint8Array;
-	timestamp: number;
+	presentationTimestamp: number;
+	decodeTimestamp: number | null;
 	isKeyframe: boolean;
 };
 
@@ -77,9 +82,6 @@ export class MpegTsMuxer extends Muxer {
 	private videoTrackIndex = 0;
 	private audioTrackIndex = 0;
 
-	private pesHeaderBuffer = new Uint8Array(14);
-	private pesHeaderView = toDataView(this.pesHeaderBuffer);
-	private ptsBitstream = new Bitstream(this.pesHeaderBuffer.subarray(9, 14));
 	private adaptationFieldBuffer = new Uint8Array(184);
 	private payloadBuffer = new Uint8Array(184);
 
@@ -124,6 +126,7 @@ export class MpegTsMuxer extends Muxer {
 			streamType,
 			streamId,
 			codecString: meta.decoderConfig.codec,
+			timestampProcessingQueue: [],
 			packetQueue: [],
 			inputIsAnnexB: null,
 			inputIsAdts: null,
@@ -188,6 +191,7 @@ export class MpegTsMuxer extends Muxer {
 			streamType,
 			streamId,
 			codecString: meta.decoderConfig.codec,
+			timestampProcessingQueue: [],
 			packetQueue: [],
 			inputIsAnnexB: null,
 			inputIsAdts: null,
@@ -225,13 +229,16 @@ export class MpegTsMuxer extends Muxer {
 
 			const preparedData = this.prepareVideoPacket(trackData, packet, meta);
 
-			trackData.packetQueue.push({
+			if (packet.type === 'key') {
+				await this.flushTimestampQueue(trackData);
+			}
+
+			trackData.timestampProcessingQueue.push({
 				data: preparedData,
-				timestamp,
+				presentationTimestamp: timestamp,
+				decodeTimestamp: null,
 				isKeyframe: packet.type === 'key',
 			});
-
-			await this.interleavePackets();
 		} finally {
 			release();
 		}
@@ -255,13 +262,16 @@ export class MpegTsMuxer extends Muxer {
 
 			const preparedData = this.prepareAudioPacket(trackData, packet, meta);
 
-			trackData.packetQueue.push({
+			if (packet.type === 'key') {
+				await this.flushTimestampQueue(trackData);
+			}
+
+			trackData.timestampProcessingQueue.push({
 				data: preparedData,
-				timestamp,
+				presentationTimestamp: timestamp,
+				decodeTimestamp: null,
 				isKeyframe: packet.type === 'key',
 			});
-
-			await this.interleavePackets();
 		} finally {
 			release();
 		}
@@ -444,6 +454,28 @@ export class MpegTsMuxer extends Muxer {
 		return true;
 	}
 
+	private async flushTimestampQueue(trackData: MpegTsTrackData, alsoInterleave = true) {
+		if (trackData.timestampProcessingQueue.length === 0) {
+			return;
+		}
+
+		const sortedTimestamps = trackData.timestampProcessingQueue
+			.map(packet => packet.presentationTimestamp)
+			.sort((a, b) => a - b);
+
+		for (let i = 0; i < trackData.timestampProcessingQueue.length; i++) {
+			const queuedPacket = trackData.timestampProcessingQueue[i]!;
+			queuedPacket.decodeTimestamp = sortedTimestamps[i]!;
+			trackData.packetQueue.push(queuedPacket);
+		}
+
+		trackData.timestampProcessingQueue.length = 0;
+
+		if (alsoInterleave) {
+			await this.interleavePackets();
+		}
+	}
+
 	private async interleavePackets(isFinalCall = false) {
 		if (!this.tablesWritten) {
 			if (!this.allTracksAreKnown() && !isFinalCall) {
@@ -469,10 +501,10 @@ export class MpegTsMuxer extends Muxer {
 
 				if (
 					trackData.packetQueue.length > 0
-					&& trackData.packetQueue[0]!.timestamp < minTimestamp
+					&& trackData.packetQueue[0]!.presentationTimestamp < minTimestamp
 				) {
 					trackWithMinTimestamp = trackData;
-					minTimestamp = trackData.packetQueue[0]!.timestamp;
+					minTimestamp = trackData.packetQueue[0]!.presentationTimestamp;
 				}
 			}
 
@@ -526,10 +558,14 @@ export class MpegTsMuxer extends Muxer {
 	}
 
 	private writePesPacket(trackData: MpegTsTrackData, queuedPacket: QueuedPacket) {
-		const pesView = this.pesHeaderView;
+		const includeDts = trackData.track.type === 'video';
+		const headerDataLength = includeDts ? 10 : 5;
+		const pesHeaderBuffer = new Uint8Array(9 + headerDataLength);
+		const pesView = toDataView(pesHeaderBuffer);
+		const ptsDtsBitstream = new Bitstream(pesHeaderBuffer.subarray(9));
 
 		setUint24(pesView, 0, 0x000001, false); // packet_start_code_prefix
-		this.pesHeaderBuffer[3] = trackData.streamId; // stream_id
+		pesHeaderBuffer[3] = trackData.streamId; // stream_id
 
 		const pesPacketLength = trackData.track.type === 'video'
 			? 0 // Unbounded
@@ -539,20 +575,33 @@ export class MpegTsMuxer extends Muxer {
 		// '10' marker, PES_scrambling_control=0, PES_priority=0,
 		// data_alignment_indicator=1, copyright=0, original_or_copy=0
 		pesView.setUint8(6, 0x84);
-		pesView.setUint8(7, 0x80); // PTS_DTS_flags=10 (PTS only), other flags=0
+		pesView.setUint8(7, includeDts ? 0xC0 : 0x80); // PTS_DTS_flags, other flags=0
-		pesView.setUint8(8, 5); // PES_header_data_length (5 bytes for PTS)
+		pesView.setUint8(8, headerDataLength); // PES_header_data_length
 
-		const pts = Math.round(queuedPacket.timestamp * TIMESCALE);
+		const pts = Math.round(queuedPacket.presentationTimestamp * TIMESCALE);
-		this.ptsBitstream.pos = 0;
+		ptsDtsBitstream.pos = 0;
-		this.ptsBitstream.writeBits(4, 0b0010); // marker
+		ptsDtsBitstream.writeBits(4, includeDts ? 0b0011 : 0b0010); // marker
-		this.ptsBitstream.writeBits(3, (pts >>> 30) & 0x7); // PTS[32:30]
+		ptsDtsBitstream.writeBits(3, (pts >>> 30) & 0x7); // PTS[32:30]
-		this.ptsBitstream.writeBits(1, 1); // marker_bit
+		ptsDtsBitstream.writeBits(1, 1); // marker_bit
-		this.ptsBitstream.writeBits(15, (pts >>> 15) & 0x7FFF); // PTS[29:15]
+		ptsDtsBitstream.writeBits(15, (pts >>> 15) & 0x7FFF); // PTS[29:15]
-		this.ptsBitstream.writeBits(1, 1); // marker_bit
+		ptsDtsBitstream.writeBits(1, 1); // marker_bit
-		this.ptsBitstream.writeBits(15, pts & 0x7FFF); // PTS[14:0]
+		ptsDtsBitstream.writeBits(15, pts & 0x7FFF); // PTS[14:0]
-		this.ptsBitstream.writeBits(1, 1); // marker_bit
+		ptsDtsBitstream.writeBits(1, 1); // marker_bit
 
-		const totalLength = this.pesHeaderBuffer.length + queuedPacket.data.length;
+		if (includeDts) {
+			assert(queuedPacket.decodeTimestamp !== null);
+
+			const dts = Math.round(queuedPacket.decodeTimestamp * TIMESCALE);
+			ptsDtsBitstream.writeBits(4, 0b0001);
+			ptsDtsBitstream.writeBits(3, (dts >>> 30) & 0x7); // DTS[32:30]
+			ptsDtsBitstream.writeBits(1, 1); // marker_bit
+			ptsDtsBitstream.writeBits(15, (dts >>> 15) & 0x7FFF); // DTS[29:15]
+			ptsDtsBitstream.writeBits(1, 1); // marker_bit
+			ptsDtsBitstream.writeBits(15, dts & 0x7FFF); // DTS[14:0]
+			ptsDtsBitstream.writeBits(1, 1); // marker_bit
+		}
+
+		const totalLength = pesHeaderBuffer.length + queuedPacket.data.length;
 		let offset = 0;
 		let isFirstTsPacket = true;
 
@@ -593,13 +642,13 @@ export class MpegTsMuxer extends Muxer {
 			const payload = this.payloadBuffer.subarray(0, payloadSize);
 
 			let payloadOffset = 0;
-			if (offset < this.pesHeaderBuffer.length) {
+			if (offset < pesHeaderBuffer.length) {
-				const headerBytes = Math.min(this.pesHeaderBuffer.length - offset, payloadSize);
+				const headerBytes = Math.min(pesHeaderBuffer.length - offset, payloadSize);
-				payload.set(this.pesHeaderBuffer.subarray(offset, offset + headerBytes), 0);
+				payload.set(pesHeaderBuffer.subarray(offset, offset + headerBytes), 0);
 				payloadOffset = headerBytes;
 			}
 
-			const dataStart = Math.max(0, offset - this.pesHeaderBuffer.length);
+			const dataStart = Math.max(0, offset - pesHeaderBuffer.length);
 			const dataEnd = dataStart + (payloadSize - payloadOffset);
 			if (payloadOffset < payloadSize) {
 				payload.set(queuedPacket.data.subarray(dataStart, dataEnd), payloadOffset);
@@ -658,13 +707,18 @@ export class MpegTsMuxer extends Muxer {
 	}
 
 	// eslint-disable-next-line @typescript-eslint/no-misused-promises
-	override async onTrackClose() {
+	override async onTrackClose(track: OutputTrack) {
 		const release = await this.mutex.acquire();
 
 		if (this.allTracksAreKnown()) {
 			this.allTracksKnown.resolve();
 		}
 
+		const trackData = this.trackDatas.find(x => x.track === track);
+		if (trackData) {
+			await this.flushTimestampQueue(trackData, false);
+		}
+
 		await this.interleavePackets();
 
 		release();
@@ -675,6 +729,10 @@ export class MpegTsMuxer extends Muxer {
 
 		this.allTracksKnown.resolve();
 
+		for (const trackData of this.trackDatas) {
+			await this.flushTimestampQueue(trackData, false);
+		}
+
 		await this.interleavePackets(true);
 
 		release();

todo.txt

@@ -1 +0,0 @@
-- Investigate quicktime stuttering with mediabunny-generated .ts file