diff --git a/dev/demux.html b/dev/demux.html
index c423bfe..8c5310b 100644
--- a/dev/demux.html
+++ b/dev/demux.html
@@ -14,6 +14,14 @@
 			source: new Mediabunny.BlobSource(file),
 		});
 
+		const videoTrack = await input.getPrimaryVideoTrack();
+		const sink = new Mediabunny.EncodedPacketSink(videoTrack);
+
+		for await (const packet of sink.packets()) {
+			console.log(packet.timestamp, packet.duration, packet.type)
+		}
+
+		/*
 		const audioTrack = await input.getPrimaryAudioTrack();
 		const sink = new Mediabunny.AudioSampleSink(audioTrack);
 
@@ -27,6 +35,7 @@
 			lastEnd = sample.timestamp + sample.duration;
 			sample.close();
 		}
+		*/
 		
 		/*
 		const sink = new Mediabunny.EncodedPacketSink(videoTrack);
diff --git a/src/matroska/matroska-demuxer.ts b/src/matroska/matroska-demuxer.ts
index 1992205..e4e6c11 100644
--- a/src/matroska/matroska-demuxer.ts
+++ b/src/matroska/matroska-demuxer.ts
@@ -61,7 +61,6 @@ import {
 	readElementHeader,
 	readElementId,
 	readFloat,
-	readSignedInt,
 	readUnsignedInt,
 	readVarInt,
 	resync,
@@ -137,7 +136,6 @@ type ClusterBlock = {
 	timestamp: number;
 	duration: number;
 	isKeyFrame: boolean;
-	referencedTimestamps: number[];
 	data: Uint8Array;
 	lacing: BlockLacing;
 	decoded: boolean;
@@ -649,21 +647,15 @@ export class MatroskaDemuxer extends Demuxer {
 			// This must hold, as track datas only get created if a block for that track is encountered
 			assert(trackData.blocks.length > 0);
 
-			let blockReferencesExist = false;
 			let hasLacedBlocks = false;
 
 			for (let i = 0; i < trackData.blocks.length; i++) {
 				const block = trackData.blocks[i]!;
 				block.timestamp += cluster.timestamp;
 
-				blockReferencesExist ||= block.referencedTimestamps.length > 0;
 				hasLacedBlocks ||= block.lacing !== BlockLacing.None;
 			}
 
-			if (blockReferencesExist) {
-				trackData.blocks = sortBlocksByReferences(trackData.blocks);
-			}
-
 			trackData.presentationTimestamps = trackData.blocks
 				.map((block, i) => ({ timestamp: block.timestamp, blockIndex: i }))
 				.sort((a, b) => a.timestamp - b.timestamp);
@@ -859,7 +851,6 @@ export class MatroskaDemuxer extends Demuxer {
 					timestamp: frameTimestamp,
 					duration: frameDuration,
 					isKeyFrame: originalBlock.isKeyFrame,
-					referencedTimestamps: originalBlock.referencedTimestamps,
 					data: frameData,
 					lacing: BlockLacing.None,
 					decoded: true,
@@ -1393,7 +1384,6 @@ export class MatroskaDemuxer extends Demuxer {
 					timestamp: relativeTimestamp, // We'll add the cluster's timestamp to this later
 					duration: 0, // Will set later
 					isKeyFrame,
-					referencedTimestamps: [],
 					data: blockData,
 					lacing,
 					decoded: !hasDecodingInstructions,
@@ -1406,13 +1396,7 @@ export class MatroskaDemuxer extends Demuxer {
 
 				this.readContiguousElements(slice.slice(dataStartPos, size));
 
-				if (this.currentBlock) {
-					for (let i = 0; i < this.currentBlock.referencedTimestamps.length; i++) {
-						this.currentBlock.referencedTimestamps[i]! += this.currentBlock.timestamp;
-					}
-
-					this.currentBlock = null;
-				}
+				this.currentBlock = null;
 			}; break;
 
 			case EBMLId.Block: {
@@ -1436,7 +1420,6 @@ export class MatroskaDemuxer extends Demuxer {
 					timestamp: relativeTimestamp, // We'll add the cluster's timestamp to this later
 					duration: 0, // Will set later
 					isKeyFrame: true,
-					referencedTimestamps: [],
 					data: blockData,
 					lacing,
 					decoded: !hasDecodingInstructions,
@@ -1487,11 +1470,8 @@ export class MatroskaDemuxer extends Demuxer {
 				if (!this.currentBlock) break;
 
 				this.currentBlock.isKeyFrame = false;
-
-				const relativeTimestamp = readSignedInt(slice, size);
-
-				// We'll offset this by the block's timestamp later
-				this.currentBlock.referencedTimestamps.push(relativeTimestamp);
+				// We ignore the actual value here, we just use the reference as an indicator for "not a key frame".
+				// This is in line with FFmpeg's behavior.
 			}; break;
 
 			case EBMLId.Tag: {
@@ -2380,43 +2360,3 @@ class MatroskaAudioTrackBacking extends MatroskaTrackBacking implements InputAud
 		};
 	}
 }
-
-/** Sorts blocks such that referenced blocks come before the blocks that reference them. */
-const sortBlocksByReferences = (blocks: ClusterBlock[]) => {
-	const timestampToBlock = new Map<number, ClusterBlock>();
-
-	for (let i = 0; i < blocks.length; i++) {
-		const block = blocks[i]!;
-		timestampToBlock.set(block.timestamp, block);
-	}
-
-	const processedBlocks = new Set<ClusterBlock>();
-	const result: ClusterBlock[] = [];
-
-	const processBlock = (block: ClusterBlock) => {
-		if (processedBlocks.has(block)) {
-			return;
-		}
-
-		// Marking the block as processed here already; prevents this algorithm from dying on cycles
-		processedBlocks.add(block);
-
-		for (let j = 0; j < block.referencedTimestamps.length; j++) {
-			const timestamp = block.referencedTimestamps[j]!;
-			const otherBlock = timestampToBlock.get(timestamp);
-			if (!otherBlock) {
-				continue;
-			}
-
-			processBlock(otherBlock);
-		}
-
-		result.push(block);
-	};
-
-	for (let i = 0; i < blocks.length; i++) {
-		processBlock(blocks[i]!);
-	}
-
-	return result;
-};