Allow Matroska clusters that don't begin with a key frame

dev/mux.html

@@ -42,6 +42,7 @@
 	format = new Metamuxer.Mp4OutputFormat({ fastStart: 'fragmented' }); // new Metamuxer.MkvOutputFormat();// new Metamuxer.Mp4OutputFormat({ fastStart: false });
 	format = new Metamuxer.OggOutputFormat();
 	format = new Metamuxer.Mp4OutputFormat();
+	format = new Metamuxer.WebMOutputFormat();
 	let target = new Metamuxer.BufferTarget();
 
 	/*
@@ -85,10 +86,12 @@
 	download(new Blob([target.buffer]), 'test.mkv');
 	*/
 
+	/*
 	let videoSource = new Metamuxer.CanvasSource(canvas, {
 		codec: 'av1',
 		bitrate: 1e6
-	});
+	});*/
+	let videoSource = new Metamuxer.EncodedVideoPacketSource('av1');
 	let audioSource = new Metamuxer.AudioBufferSource({
 		codec: 'opus',
 		bitrate: 128e3,
@@ -159,12 +162,23 @@ Testing... <00:17.350>One... <00:18.125>Two...
 	//subtitleSource.add(simpleWebvttFile);
 	//subtitleSource.close();
 
+	/*
 	for (let i = 0; i < 100; i++) {
 		context.fillStyle = ['red', 'green', 'blue', 'yellow'][i % 4];
 		context.fillRect(canvas.width * Math.random(), canvas.height * Math.random(), canvas.width * Math.random(), canvas.height * Math.random());
 
 		await videoSource.add(i / 10, 1 / 10);
 	}
+	*/
+	for (let i = 0; i < 40; i += 0.1) {
+		await videoSource.add(new Metamuxer.EncodedPacket(new Uint8Array(1234), i === 0 ? 'key' : 'delta', i, 0.1), {
+			decoderConfig: {
+				codec: 'vp09.00.10.08',	
+				codedWidth: 64,
+				codedHeight: 64
+			}
+		});
+	}
 
 	let audioContext = new AudioContext();
 	let audioBuffer = await audioContext.decodeAudioData(await (await fetch('./CantinaBand60.wav')).arrayBuffer());
@@ -176,5 +190,5 @@ Testing... <00:17.350>One... <00:18.125>Two...
 	await output.finalize();
 
 	console.log(target);
-	download(new Blob([target.buffer]), 'test' + format.getFileExtension());
+	download(new Blob([target.buffer]), 'test' + format.fileExtension);
 </script>

src/isobmff/isobmff-demuxer.ts

@@ -3,6 +3,7 @@ import {
 	AudioCodec,
 	Av1CodecInfo,
 	extractAudioCodecString,
+	extractAv1CodecInfoFromFrame,
 	extractVideoCodecString,
 	extractVp9CodecInfoFromFrame,
 	MediaCodec,
@@ -2349,6 +2350,9 @@ class IsobmffVideoTrackBacking extends IsobmffTrackBacking implements InputVideo
 			if (this.internalTrack.info.codec === 'vp9' && !this.internalTrack.info.vp9CodecInfo) {
 				const firstPacket = await this.getFirstPacket({});
 				this.internalTrack.info.vp9CodecInfo = firstPacket && extractVp9CodecInfoFromFrame(firstPacket.data);
+			} else if (this.internalTrack.info.codec === 'av1' && !this.internalTrack.info.av1CodecInfo) {
+				const firstPacket = await this.getFirstPacket({});
+				this.internalTrack.info.av1CodecInfo = firstPacket && extractAv1CodecInfoFromFrame(firstPacket.data);
 			}
 
 			return {

src/matroska/matroska-muxer.ts

@@ -46,7 +46,8 @@ import { Muxer } from '../muxer';
 import { Writer } from '../writer';
 import { EncodedPacket } from '../packet';
 
-const MAX_CHUNK_LENGTH_MS = 2 ** 15;
+const MIN_CLUSTER_TIMESTAMP_MS = -(2 ** 15);
+const MAX_CLUSTER_TIMESTAMP_MS = 2 ** 15 - 1;
 const APP_NAME = 'https://github.com/Vanilagy/webm-muxer'; // TODO
 const SEGMENT_SIZE_BYTES = 6;
 const CLUSTER_SIZE_BYTES = 5;
@@ -661,7 +662,8 @@ export class MatroskaMuxer extends Muxer {
 		}
 
 		const msTimestamp = Math.round(1000 * chunk.timestamp);
-		// We can only finalize this cluster (and begin a new one) if we know that each track will be able to
+
+		// We wanna only finalize this cluster (and begin a new one) if we know that each track will be able to
 		// start the new one with a key frame.
 		const keyFrameQueuedEverywhere = this.trackDatas.every((otherTrackData) => {
 			if (trackData === otherTrackData) {
@@ -676,35 +678,39 @@ export class MatroskaMuxer extends Muxer {
 			return otherTrackData.track.source._closed;
 		});
 
-		if (
-			!this.currentCluster
-			|| (
+		let shouldCreateNewCluster = false;
+		if (!this.currentCluster) {
+			shouldCreateNewCluster = true;
+		} else {
+			assert(this.currentClusterStartMsTimestamp !== null);
+			assert(this.currentClusterMaxMsTimestamp !== null);
+
+			const relativeTimestamp = msTimestamp - this.currentClusterStartMsTimestamp;
+
+			shouldCreateNewCluster = (
 				keyFrameQueuedEverywhere
 				// This check is required because that means there is already a block with this timestamp in the
-				// CURRENT chunk, meaning that starting the next cluster at the same timestamp is forbidden (since the
-				// already-written block would belong into it instead).
-				&& msTimestamp > this.currentClusterMaxMsTimestamp!
-				&& msTimestamp - this.currentClusterStartMsTimestamp! >= 1000
+				// CURRENT chunk, meaning that starting the next cluster at the same timestamp is forbidden (since
+				// the already-written block would belong into it instead).
+				&& msTimestamp > this.currentClusterMaxMsTimestamp
+				&& relativeTimestamp >= 1000
 			)
-		) {
+			// The cluster would exceed its maximum allowed length. This puts us in an unfortunate position and forces
+			// us to begin the next cluster with a delta frame. Although this is undesirable, it is not forbidden by the
+			// spec and is supported by players.
+			|| relativeTimestamp > MAX_CLUSTER_TIMESTAMP_MS;
+		}
+
+		if (shouldCreateNewCluster) {
 			this.createNewCluster(msTimestamp);
 		}
 
 		const relativeTimestamp = msTimestamp - this.currentClusterStartMsTimestamp!;
-		if (relativeTimestamp < -(2 ** 15)) {
+		if (relativeTimestamp < MIN_CLUSTER_TIMESTAMP_MS) {
 			// The block lies too far in the past, it's not representable within this cluster
 			return;
 		}
 
-		const clusterIsTooLong = relativeTimestamp >= MAX_CHUNK_LENGTH_MS;
-		if (clusterIsTooLong) {
-			throw new Error(
-				`Current Matroska cluster exceeded its maximum allowed length of ${MAX_CHUNK_LENGTH_MS} `
-				+ `milliseconds. In order to produce a correct WebM file, you must pass in a key frame at least every `
-				+ `${MAX_CHUNK_LENGTH_MS} milliseconds.`,
-			);
-		}
-
 		const prelude = new Uint8Array(4);
 		const view = new DataView(prelude.buffer);
 		// 0x80 to indicate it's the last byte of a multi-byte number

src/media-source.ts

@@ -264,7 +264,7 @@ class VideoEncoderWrapper {
 
 		// Ensure a key frame every keyFrameInterval seconds. It is important that all video tracks follow the same
 		// "key frame" rhythm, because aligned key frames are required to start new fragments in ISOBMFF or clusters
-		// in Matroska.
+		// in Matroska (or at least desirable).
 		const finalEncodeOptions = {
 			...encodeOptions,
 			keyFrame: encodeOptions?.keyFrame

todo.txt

@@ -3,4 +3,5 @@
 - is this fixed? https://github.com/Vanilagy/webm-muxer/issues/50
 - cross-track offset for streaming sources
 - configurable fragmented mp4 fragment size, like the mp4-muxer PR
-- textsubtitlesource, chunked piping
+- textsubtitlesource, chunked piping
+- for matroska demuxing, don't blindly pipe codecPrivate into description. This should be if-else'd on a per-codec basis, for the codecs that actually need it