diff --git a/src/adts/adts-muxer.ts b/src/adts/adts-muxer.ts
index def99c2..d0e49a0 100644
--- a/src/adts/adts-muxer.ts
+++ b/src/adts/adts-muxer.ts
@@ -60,7 +60,7 @@ export class AdtsMuxer extends Muxer {
 		const release = await this.mutex.acquire();
 
 		try {
-			this.validateAndNormalizeTimestamp(track, packet.timestamp, packet.type === 'key');
+			this.validateTimestamp(track, packet.timestamp, packet.type === 'key');
 
 			// First packet - determine input format from metadata
 			if (this.inputIsAdts === null) {
diff --git a/src/flac/flac-muxer.ts b/src/flac/flac-muxer.ts
index b022dbe..02e2067 100644
--- a/src/flac/flac-muxer.ts
+++ b/src/flac/flac-muxer.ts
@@ -214,7 +214,7 @@ export class FlacMuxer extends Muxer {
 		const release = await this.mutex.acquire();
 
 		try {
-			this.validateAndNormalizeTimestamp(
+			this.validateTimestamp(
 				track,
 				packet.timestamp,
 				packet.type === 'key',
diff --git a/src/hls/hls-muxer.ts b/src/hls/hls-muxer.ts
index f5d88e9..fb2e39e 100644
--- a/src/hls/hls-muxer.ts
+++ b/src/hls/hls-muxer.ts
@@ -614,17 +614,15 @@ export class HlsMuxer extends Muxer {
 		const release = await playlist.mutex.acquire();
 
 		try {
-			const timestamp = this.validateAndNormalizeTimestamp(track, packet.timestamp, packet.type === 'key');
-			const adjustedPacket = packet.clone({ timestamp });
-
-			trackData.packets.push(adjustedPacket);
+			this.validateTimestamp(track, packet.timestamp, packet.type === 'key');
+			trackData.packets.push(packet);
 
 			if (playlist.currentSegmentStartTimestamp === null) {
-				playlist.currentSegmentStartTimestamp = adjustedPacket.timestamp;
+				playlist.currentSegmentStartTimestamp = packet.timestamp;
 			} else if (!playlist.currentSegmentStartTimestampIsFixed) {
 				playlist.currentSegmentStartTimestamp = Math.min(
 					playlist.currentSegmentStartTimestamp,
-					adjustedPacket.timestamp,
+					packet.timestamp,
 				);
 			}
 
@@ -645,17 +643,15 @@ export class HlsMuxer extends Muxer {
 		const release = await playlist.mutex.acquire();
 
 		try {
-			const timestamp = this.validateAndNormalizeTimestamp(track, packet.timestamp, packet.type === 'key');
-			const adjustedPacket = packet.clone({ timestamp });
-
-			trackData.packets.push(adjustedPacket);
+			this.validateTimestamp(track, packet.timestamp, packet.type === 'key');
+			trackData.packets.push(packet);
 
 			if (playlist.currentSegmentStartTimestamp === null) {
-				playlist.currentSegmentStartTimestamp = adjustedPacket.timestamp;
+				playlist.currentSegmentStartTimestamp = packet.timestamp;
 			} else if (!playlist.currentSegmentStartTimestampIsFixed) {
 				playlist.currentSegmentStartTimestamp = Math.min(
 					playlist.currentSegmentStartTimestamp,
-					adjustedPacket.timestamp,
+					packet.timestamp,
 				);
 			}
 
diff --git a/src/isobmff/isobmff-muxer.ts b/src/isobmff/isobmff-muxer.ts
index 5e50127..a62e755 100644
--- a/src/isobmff/isobmff-muxer.ts
+++ b/src/isobmff/isobmff-muxer.ts
@@ -600,7 +600,7 @@ export class IsobmffMuxer extends Muxer {
 				packetData = concatNalUnitsInLengthPrefixed(nalUnits, 4);
 			}
 
-			const timestamp = this.validateAndNormalizeTimestamp(
+			this.validateTimestamp(
 				trackData.track,
 				packet.timestamp,
 				packet.type === 'key',
@@ -608,7 +608,7 @@ export class IsobmffMuxer extends Muxer {
 			const internalSample = this.createSampleForTrack(
 				trackData,
 				packetData,
-				timestamp,
+				packet.timestamp,
 				packet.duration,
 				packet.type,
 			);
@@ -638,11 +638,13 @@ export class IsobmffMuxer extends Muxer {
 				packetData = packetData.subarray(headerLength);
 			}
 
-			let timestamp = this.validateAndNormalizeTimestamp(
+			this.validateTimestamp(
 				trackData.track,
 				packet.timestamp,
 				packet.type === 'key',
 			);
+
+			let timestamp = packet.timestamp;
 			let duration = packet.duration;
 
 			if (trackData.info.requiresPcmTransformation) {
@@ -718,7 +720,7 @@ export class IsobmffMuxer extends Muxer {
 		try {
 			const trackData = this.getSubtitleTrackData(track, meta);
 
-			this.validateAndNormalizeTimestamp(trackData.track, cue.timestamp, true);
+			this.validateTimestamp(trackData.track, cue.timestamp, true);
 
 			if (track.source._codec === 'webvtt') {
 				trackData.cueQueue.push(cue);
diff --git a/src/matroska/matroska-muxer.ts b/src/matroska/matroska-muxer.ts
index da60565..dabeaa4 100644
--- a/src/matroska/matroska-muxer.ts
+++ b/src/matroska/matroska-muxer.ts
@@ -907,7 +907,9 @@ export class MatroskaMuxer extends Muxer {
 			const trackData = this.getVideoTrackData(track, packet, meta);
 
 			const isKeyFrame = packet.type === 'key';
-			let timestamp = this.validateAndNormalizeTimestamp(trackData.track, packet.timestamp, isKeyFrame);
+			this.validateTimestamp(trackData.track, packet.timestamp, isKeyFrame);
+
+			let timestamp = packet.timestamp;
 			let duration = packet.duration;
 
 			if (track.metadata.frameRate !== undefined) {
@@ -950,9 +952,9 @@ export class MatroskaMuxer extends Muxer {
 			}
 
 			const isKeyFrame = packet.type === 'key';
-			const timestamp = this.validateAndNormalizeTimestamp(trackData.track, packet.timestamp, isKeyFrame);
-			const audioChunk = this.createInternalChunk(packetData, timestamp, packet.duration, packet.type);
+			this.validateTimestamp(trackData.track, packet.timestamp, isKeyFrame);
 
+			const audioChunk = this.createInternalChunk(packetData, packet.timestamp, packet.duration, packet.type);
 			trackData.chunkQueue.push(audioChunk);
 			await this.interleaveChunks();
 		} finally {
@@ -966,10 +968,10 @@ export class MatroskaMuxer extends Muxer {
 		try {
 			const trackData = this.getSubtitleTrackData(track, meta);
 
-			const timestamp = this.validateAndNormalizeTimestamp(trackData.track, cue.timestamp, true);
+			this.validateTimestamp(trackData.track, cue.timestamp, true);
 
 			let bodyText = cue.text;
-			const timestampMs = Math.round(timestamp * 1000);
+			const timestampMs = Math.round(cue.timestamp * 1000);
 
 			// Replace in-body timestamps so that they're relative to the cue start time
 			inlineTimestampRegex.lastIndex = 0;
@@ -985,7 +987,7 @@ export class MatroskaMuxer extends Muxer {
 
 			const subtitleChunk = this.createInternalChunk(
 				body,
-				timestamp,
+				cue.timestamp,
 				cue.duration,
 				'key',
 				additions.trim() ? textEncoder.encode(additions) : null,
diff --git a/src/media-source.ts b/src/media-source.ts
index 34a054a..9eda359 100644
--- a/src/media-source.ts
+++ b/src/media-source.ts
@@ -81,11 +81,6 @@ export abstract class MediaSource {
 	_closingPromise: Promise<void> | null = null;
 	/** @internal */
 	_closed = false;
-	/**
-	 * @internal
-	 * A time offset in seconds that is added to all timestamps generated by this source.
-	 */
-	_timestampOffset = 0;
 
 	/** @internal */
 	_ensureValidAdd() {
@@ -1476,7 +1471,7 @@ export class MediaStreamVideoTrackSource extends VideoSource {
 		let frameCount = 0;
 		let errored = false;
 		let lastSampleTimestamp: number | null = null;
-		let pauseOffset = 0;
+		let timestampOffset = 0;
 
 		const tick = () => {
 			assert(frameRate !== null);
@@ -1544,9 +1539,7 @@ export class MediaStreamVideoTrackSource extends VideoSource {
 						// pause. Doing it like this instead of simply keeping track of the paused time is better since
 						// it retains the frame rate of the underlying source.
 						const timeDelta = currentTimestamp - lastSampleTimestamp;
-						// We modify this field instead of _timestampOffset since we still might have data in flight
-						// in the encoder, with which we don't want to mess.
-						pauseOffset -= timeDelta;
+						timestampOffset -= timeDelta;
 					}
 					lastSampleTimestamp = currentTimestamp;
 				}
@@ -1574,7 +1567,7 @@ export class MediaStreamVideoTrackSource extends VideoSource {
 					}
 				}
 
-				this._timestampOffset = target - firstVideoFrameTimestamp;
+				timestampOffset = target - firstVideoFrameTimestamp;
 			}
 
 			lastSampleTimestamp = currentTimestamp;
@@ -1586,7 +1579,7 @@ export class MediaStreamVideoTrackSource extends VideoSource {
 			}
 
 			const sample = new VideoSample(videoFrame, {
-				timestamp: currentTimestamp + pauseOffset,
+				timestamp: currentTimestamp + timestampOffset,
 			});
 
 			void this._encoder.add(sample, true)
@@ -2575,7 +2568,7 @@ export class MediaStreamAudioTrackSource extends AudioSource {
 		let firstAudioDataTimestamp: number | null = null;
 		let errored = false;
 		let lastSampleTimestamp: number | null = null;
-		let pauseOffset = 0;
+		let timestampOffset = 0;
 
 		const onAudioSample = (audioSample: AudioSample) => {
 			if (errored) {
@@ -2593,9 +2586,7 @@ export class MediaStreamAudioTrackSource extends AudioSource {
 						// the pause. Doing it like this instead of simply keeping track of the paused time is better
 						// since it retains the sample rate of the underlying source.
 						const timeDelta = currentTimestamp - lastSampleTimestamp;
-						// We modify this field instead of _timestampOffset since we still might have data in flight
-						// in the encoder, with which we don't want to mess.
-						pauseOffset -= timeDelta;
+						timestampOffset -= timeDelta;
 					}
 					lastSampleTimestamp = currentTimestamp;
 				}
@@ -2623,7 +2614,7 @@ export class MediaStreamAudioTrackSource extends AudioSource {
 					}
 				}
 
-				this._timestampOffset = target - firstAudioDataTimestamp;
+				timestampOffset = target - firstAudioDataTimestamp;
 			}
 
 			lastSampleTimestamp = currentTimestamp;
@@ -2634,7 +2625,7 @@ export class MediaStreamAudioTrackSource extends AudioSource {
 				return;
 			}
 
-			audioSample.setTimestamp(currentTimestamp + pauseOffset);
+			audioSample.setTimestamp(currentTimestamp + timestampOffset);
 
 			void this._encoder.add(audioSample, true)
 				.catch((error) => {
diff --git a/src/mp3/mp3-muxer.ts b/src/mp3/mp3-muxer.ts
index 97546b3..f1a1413 100644
--- a/src/mp3/mp3-muxer.ts
+++ b/src/mp3/mp3-muxer.ts
@@ -110,7 +110,7 @@ export class Mp3Muxer extends Muxer {
 				this.frameCount++;
 			}
 
-			this.validateAndNormalizeTimestamp(track, packet.timestamp, packet.type === 'key');
+			this.validateTimestamp(track, packet.timestamp, packet.type === 'key');
 
 			if (writeXingHeader) {
 				this.framePositions.push(this.writer.getPos());
diff --git a/src/mpeg-ts/mpeg-ts-muxer.ts b/src/mpeg-ts/mpeg-ts-muxer.ts
index dfa0f7e..014918a 100644
--- a/src/mpeg-ts/mpeg-ts-muxer.ts
+++ b/src/mpeg-ts/mpeg-ts-muxer.ts
@@ -227,7 +227,7 @@ export class MpegTsMuxer extends Muxer {
 		try {
 			const trackData = this.getVideoTrackData(track, meta);
 
-			const timestamp = this.validateAndNormalizeTimestamp(
+			this.validateTimestamp(
 				trackData.track,
 				packet.timestamp,
 				packet.type === 'key',
@@ -241,7 +241,7 @@ export class MpegTsMuxer extends Muxer {
 
 			trackData.timestampProcessingQueue.push({
 				data: preparedData,
-				presentationTimestamp: timestamp,
+				presentationTimestamp: packet.timestamp,
 				decodeTimestamp: null,
 				isKeyframe: packet.type === 'key',
 			});
@@ -260,7 +260,7 @@ export class MpegTsMuxer extends Muxer {
 		try {
 			const trackData = this.getAudioTrackData(track, meta);
 
-			const timestamp = this.validateAndNormalizeTimestamp(
+			this.validateTimestamp(
 				trackData.track,
 				packet.timestamp,
 				packet.type === 'key',
@@ -274,7 +274,7 @@ export class MpegTsMuxer extends Muxer {
 
 			trackData.timestampProcessingQueue.push({
 				data: preparedData,
-				presentationTimestamp: timestamp,
+				presentationTimestamp: packet.timestamp,
 				decodeTimestamp: null,
 				isKeyframe: packet.type === 'key',
 			});
diff --git a/src/muxer.ts b/src/muxer.ts
index fc9df5c..b945770 100644
--- a/src/muxer.ts
+++ b/src/muxer.ts
@@ -42,9 +42,7 @@ export abstract class Muxer {
 		maxTimestampBeforeLastKeyPacket: number | null;
 	}>();
 
-	protected validateAndNormalizeTimestamp(track: OutputTrack, timestampInSeconds: number, isKeyPacket: boolean) {
-		timestampInSeconds += track.source._timestampOffset;
-
+	protected validateTimestamp(track: OutputTrack, timestampInSeconds: number, isKeyPacket: boolean) {
 		if (timestampInSeconds < 0) {
 			throw new Error(`Timestamps must be non-negative (got ${timestampInSeconds}s).`);
 		}
@@ -78,7 +76,5 @@ export abstract class Muxer {
 
 			timestampInfo.maxTimestamp = Math.max(timestampInfo.maxTimestamp, timestampInSeconds);
 		}
-
-		return timestampInSeconds;
 	}
 }
diff --git a/src/ogg/ogg-muxer.ts b/src/ogg/ogg-muxer.ts
index 987eef8..7798f84 100644
--- a/src/ogg/ogg-muxer.ts
+++ b/src/ogg/ogg-muxer.ts
@@ -267,7 +267,7 @@ export class OggMuxer extends Muxer {
 		try {
 			const trackData = this.getTrackData(track, meta);
 
-			this.validateAndNormalizeTimestamp(trackData.track, packet.timestamp, packet.type === 'key');
+			this.validateTimestamp(trackData.track, packet.timestamp, packet.type === 'key');
 
 			const currentTimestampInSamples = trackData.currentTimestampInSamples;
 
diff --git a/src/wave/wave-muxer.ts b/src/wave/wave-muxer.ts
index 7e43a0f..68b1072 100644
--- a/src/wave/wave-muxer.ts
+++ b/src/wave/wave-muxer.ts
@@ -79,7 +79,7 @@ export class WaveMuxer extends Muxer {
 				this.headerWritten = true;
 			}
 
-			this.validateAndNormalizeTimestamp(track, packet.timestamp, packet.type === 'key');
+			this.validateTimestamp(track, packet.timestamp, packet.type === 'key');
 
 			if (!this.isRf64 && this.writer.getPos() + packet.data.byteLength >= 2 ** 32) {
 				throw new Error(