Add VideoSample and AudioSample abstractions

dev/player.html

@@ -53,44 +53,6 @@ if (!(await audioTrack?.canDecode())) {
 	audioTrack = null;	
 }
 
-
-//const startTime2 = audioContext.currentTime + 0.2;
-//const sink = new Metamuxer.AudioDataSink(audioTrack);
-//for await (const { data } of sink.data(0)) {
-//	const sample = new Metamuxer.AudioSample(data)
-//
-//	console.log(sample.toAudioData())
-//
-//	const audioBuffer = sample.toAudioBuffer()
-//
-//	//console.log(sample, sample.allocationSize({ planeIndex: 7, format: 'u8-planar' }))
-//
-//	//break;
-//
-//	/*
-//	const audioBuffer = new AudioBuffer({
-//		numberOfChannels: sample.numberOfChannels,
-//		length: sample.numberOfFrames,
-//		sampleRate: sample.sampleRate,
-//	});
-//
-//	// All user agents are required to support conversion to f32-planar
-//	const dataBytes = new Uint8Array(sample.allocationSize({ planeIndex: 0, format: 'u8-planar' }));
-//
-//	for (let i = 0; i < sample.numberOfChannels; i++) {
-//		sample.copyTo(dataBytes, { planeIndex: i, format: 'u8-planar' });
-//		audioBuffer.copyToChannel(new Float32Array(dataBytes).map(x => (x - 128)/128), i);
-//	}
-//		*/
-//
-//	const node = audioContext.createBufferSource();	
-//	node.buffer = audioBuffer;	
-//	node.connect(audioContext.destination);	
-//	node.start(startTime2 + sample.timestamp);
-//}
-//
-//await new Promise(() => {})
-
 const canvas = document.querySelector('canvas');
 const context = canvas.getContext('2d');
 let videoRotation = 0;

package.json

@@ -30,12 +30,12 @@
     "@types/dom-webcodecs": "^0.1.13"
   },
   "devDependencies": {
-    "@eslint/js": "^9.16.0",
-    "@microsoft/api-extractor": "^7.48.0",
-    "@stylistic/eslint-plugin": "^2.11.0",
-    "esbuild": "^0.23.1",
-    "eslint": "^9.16.0",
-    "typescript": "^5.5.4",
-    "typescript-eslint": "^8.16.0"
+    "@eslint/js": "^9.22.0",
+    "@microsoft/api-extractor": "^7.52.1",
+    "@stylistic/eslint-plugin": "^4.2.0",
+    "esbuild": "^0.25.1",
+    "eslint": "^9.22.0",
+    "typescript": "^5.8.2",
+    "typescript-eslint": "^8.26.1"
   }
 }

src/conversion.ts

@@ -13,24 +13,25 @@ import { Input } from './input';
 import { InputAudioTrack, InputTrack, InputVideoTrack } from './input-track';
 import {
 	AudioBufferSink,
-	AudioDataSink,
+	AudioSampleSink,
 	CanvasSink,
 	EncodedPacketSink,
-	VideoFrameSink,
+	VideoSampleSink,
 } from './media-sink';
 import {
 	AudioBufferSource,
-	AudioDataSource,
 	AudioEncodingConfig,
 	AudioSource,
 	EncodedVideoPacketSource,
 	EncodedAudioPacketSource,
 	VideoEncodingConfig,
-	VideoFrameSource,
 	VideoSource,
+	VideoSampleSource,
+	AudioSampleSource,
 } from './media-source';
-import { assert, clamp, normalizeRotation, promiseWithResolvers, Rotation, setVideoFrameTiming } from './misc';
+import { assert, clamp, normalizeRotation, promiseWithResolvers, Rotation } from './misc';
 import { Output, TrackType } from './output';
+import { VideoSample } from './sample';
 
 /** @public */
 export type ConversionOptions = {
@@ -467,7 +468,7 @@ export class Conversion {
 			};
 
 			if (needsRerender) {
-				const source = new VideoFrameSource(encodingConfig);
+				const source = new VideoSampleSource(encodingConfig);
 				videoSource = source;
 
 				this._trackPromises.push((async () => {
@@ -491,36 +492,37 @@ export class Conversion {
 							return;
 						}
 
-						await source.add(new VideoFrame(canvas, {
-							timestamp: 1e6 * Math.max(timestamp - this._startTimestamp, 0),
-							duration: 1e6 * duration,
-						}));
+						const sample = new VideoSample(canvas, {
+							timestamp: Math.max(timestamp - this._startTimestamp, 0),
+							duration,
+						});
+
+						await source.add(sample);
+						sample.close();
 					}
 				})());
 			} else {
-				const source = new VideoFrameSource(encodingConfig);
+				const source = new VideoSampleSource(encodingConfig);
 				videoSource = source;
 
 				this._trackPromises.push((async () => {
 					await this._started;
 
-					const sink = new VideoFrameSink(track);
+					const sink = new VideoSampleSink(track);
 
-					for await (const { frame, timestamp } of sink.frames(this._startTimestamp, this._endTimestamp)) {
-						if (this._synchronizer.shouldWait(track.id, timestamp)) {
-							await this._synchronizer.wait(timestamp);
+					for await (const sample of sink.samples(this._startTimestamp, this._endTimestamp)) {
+						if (this._synchronizer.shouldWait(track.id, sample.timestamp)) {
+							await this._synchronizer.wait(sample.timestamp);
 						}
 
-						const clone = setVideoFrameTiming(frame, {
-							timestamp: Math.max(timestamp - this._startTimestamp, 0),
-						});
+						sample.setTimestamp(Math.max(sample.timestamp - this._startTimestamp, 0));
 
 						if (this._canceled) {
 							return;
 						}
 
-						await source.add(clone);
-						clone.close();
+						await source.add(sample);
+						sample.close();
 					}
 
 					await source.close();
@@ -662,7 +664,7 @@ export class Conversion {
 			if (needsResample) {
 				audioSource = this._resampleAudio(track, codecOfChoice, numberOfChannels, sampleRate);
 			} else {
-				const source = new AudioDataSource({
+				const source = new AudioSampleSource({
 					codec: codecOfChoice,
 					bitrate: this._options.audio?.bitrate ?? QUALITY_HIGH,
 					onEncodedPacket: packet => this._reportProgress(track.id, packet.timestamp + packet.duration),
@@ -672,18 +674,18 @@ export class Conversion {
 				this._trackPromises.push((async () => {
 					await this._started;
 
-					const sink = new AudioDataSink(track);
-					for await (const { data, timestamp } of sink.data(undefined, this._endTimestamp)) {
-						if (this._synchronizer.shouldWait(track.id, timestamp)) {
-							await this._synchronizer.wait(timestamp);
+					const sink = new AudioSampleSink(track);
+					for await (const sample of sink.samples(undefined, this._endTimestamp)) {
+						if (this._synchronizer.shouldWait(track.id, sample.timestamp)) {
+							await this._synchronizer.wait(sample.timestamp);
 						}
 
 						if (this._canceled) {
 							return;
 						}
 
-						await source.add(data);
-						data.close();
+						await source.add(sample);
+						sample.close();
 					}
 
 					await source.close();

src/custom-coder.ts

@@ -1,11 +1,12 @@
 import { AudioCodec, VideoCodec } from './codec';
 import { EncodedPacket } from './packet';
+import { AudioSample, VideoSample } from './sample';
 
 /** @public */
 export abstract class CustomVideoDecoder {
 	codec!: VideoCodec;
 	config!: VideoDecoderConfig;
-	onFrame!: (frame: VideoFrame) => unknown;
+	onSample!: (sample: VideoSample) => unknown;
 
 	// eslint-disable-next-line @typescript-eslint/no-unused-vars
 	static supports(codec: VideoCodec, config: VideoDecoderConfig): boolean {
@@ -22,7 +23,7 @@ export abstract class CustomVideoDecoder {
 export abstract class CustomAudioDecoder {
 	codec!: AudioCodec;
 	config!: AudioDecoderConfig;
-	onData!: (data: AudioData) => unknown;
+	onSample!: (sample: AudioSample) => unknown;
 
 	// eslint-disable-next-line @typescript-eslint/no-unused-vars
 	static supports(codec: AudioCodec, config: AudioDecoderConfig): boolean {
@@ -47,7 +48,7 @@ export abstract class CustomVideoEncoder {
 	}
 
 	abstract init(): void;
-	abstract encode(videoFrame: VideoFrame, options: VideoEncoderEncodeOptions): Promise<void> | void;
+	abstract encode(videoSample: VideoSample, options: VideoEncoderEncodeOptions): Promise<void> | void;
 	abstract flush(): Promise<void> | void;
 	abstract close(): Promise<void> | void;
 }
@@ -64,7 +65,7 @@ export abstract class CustomAudioEncoder {
 	}
 
 	abstract init(): void;
-	abstract encode(audioData: AudioData): Promise<void> | void;
+	abstract encode(audioSample: AudioSample): Promise<void> | void;
 	abstract flush(): Promise<void> | void;
 	abstract close(): Promise<void> | void;
 }

src/index.ts

@@ -30,12 +30,12 @@ export {
 	MediaSource,
 	VideoSource,
 	EncodedVideoPacketSource,
-	VideoFrameSource,
+	VideoSampleSource,
 	CanvasSource,
 	MediaStreamVideoTrackSource,
 	AudioSource,
 	EncodedAudioPacketSource,
-	AudioDataSource,
+	AudioSampleSource,
 	AudioBufferSource,
 	MediaStreamAudioTrackSource,
 	SubtitleSource,
@@ -67,7 +67,7 @@ export {
 	getEncodableSubtitleCodecs,
 } from './codec';
 export { Target, BufferTarget, StreamTarget, StreamTargetChunk, StreamTargetOptions } from './target';
-export { Rotation, AnyIterable, setVideoFrameTiming } from './misc';
+export { Rotation, AnyIterable, SetRequired } from './misc';
 export { Source, BufferSource, StreamSource, StreamSourceOptions, BlobSource, UrlSource } from './source';
 export {
 	InputFormat,
@@ -92,20 +92,20 @@ export { Input, InputOptions } from './input';
 export { InputTrack, InputVideoTrack, InputAudioTrack, PacketStats } from './input-track';
 export { EncodedPacket, PacketType } from './packet';
 export {
+	VideoSample,
+	VideoSampleInit,
 	AudioSample,
 	AudioSampleInit,
 } from './sample';
 export {
 	PacketRetrievalOptions,
 	EncodedPacketSink,
-	BaseMediaFrameSink,
-	VideoFrameSink,
-	WrappedVideoFrame,
+	BaseMediaSampleSink,
+	VideoSampleSink,
 	CanvasSinkOptions,
 	CanvasSink,
 	WrappedCanvas,
-	AudioDataSink,
-	WrappedAudioData,
+	AudioSampleSink,
 	AudioBufferSink,
 	WrappedAudioBuffer,
 } from './media-sink';

src/media-sink.ts

@@ -17,6 +17,7 @@ import {
 } from './misc';
 import { EncodedPacket } from './packet';
 import { fromAlaw, fromUlaw } from './pcm';
+import { AudioSample, VideoSample } from './sample';
 
 /** @public */
 export type PacketRetrievalOptions = {
@@ -180,18 +181,11 @@ export class EncodedPacketSink {
 	}
 }
 
-export type WrappedMediaFrame<T extends VideoFrame | AudioData> = {
-	frame: T;
-	timestamp: number;
-	duration: number;
-};
-
 abstract class DecoderWrapper<
-	MediaFrame extends VideoFrame | AudioData,
-	WrappedFrame extends WrappedMediaFrame<MediaFrame> = WrappedMediaFrame<MediaFrame>,
+	MediaSample extends VideoSample | AudioSample,
 > {
 	constructor(
-		public onFrame: (frame: WrappedFrame) => unknown,
+		public onSample: (sample: MediaSample) => unknown,
 		public onError: (error: DOMException) => unknown,
 	) {}
 
@@ -202,36 +196,29 @@ abstract class DecoderWrapper<
 }
 
 /** @public */
-export abstract class BaseMediaFrameSink<
-	MediaFrame extends VideoFrame | AudioData,
-	/** @internal */
-	WrappedFrame extends WrappedMediaFrame<MediaFrame> = WrappedMediaFrame<MediaFrame>,
+export abstract class BaseMediaSampleSink<
+	MediaSample extends VideoSample | AudioSample,
 > {
 	/** @internal */
 	abstract _createDecoder(
-		onFrame: (frame: WrappedFrame) => unknown,
+		onSample: (sample: MediaSample) => unknown,
 		onError: (error: DOMException) => unknown
-	): Promise<DecoderWrapper<MediaFrame>>;
+	): Promise<DecoderWrapper<MediaSample>>;
 	/** @internal */
 	abstract _createPacketSink(): EncodedPacketSink;
 
 	/** @internal */
-	private _duplicateFrame(frame: WrappedFrame) {
-		return structuredClone(frame);
-	}
-
-	/** @internal */
-	protected mediaFramesInRange(
+	protected mediaSamplesInRange(
 		startTimestamp = 0,
 		endTimestamp = Infinity,
-	): AsyncGenerator<WrappedFrame, void, unknown> {
+	): AsyncGenerator<MediaSample, void, unknown> {
 		validateTimestamp(startTimestamp);
 		validateTimestamp(endTimestamp);
 
 		const MAX_QUEUE_SIZE = 8;
-		const frameQueue: WrappedFrame[] = [];
-		let firstFrameQueued = false;
-		let lastFrame: WrappedFrame | null = null;
+		const sampleQueue: MediaSample[] = [];
+		let firstSampleQueued = false;
+		let lastSample: MediaSample | null = null;
 		let { promise: queueNotEmpty, resolve: onQueueNotEmpty } = promiseWithResolvers();
 		let { promise: queueDequeue, resolve: onQueueDequeue } = promiseWithResolvers();
 		let decoderIsFlushed = false;
@@ -246,38 +233,38 @@ export abstract class BaseMediaFrameSink<
 		// The following is the "pump" process that keeps pumping packets into the decoder
 		(async () => {
 			const decoderError = new Error();
-			const decoder = await this._createDecoder((wrappedFrame) => {
+			const decoder = await this._createDecoder((sample) => {
 				onQueueDequeue();
-				if (wrappedFrame.timestamp >= endTimestamp) {
+				if (sample.timestamp >= endTimestamp) {
 					ended = true;
 				}
 
 				if (ended) {
-					wrappedFrame.frame.close();
+					sample.close();
 					return;
 				}
 
-				if (lastFrame) {
-					if (wrappedFrame.timestamp > startTimestamp) {
+				if (lastSample) {
+					if (sample.timestamp > startTimestamp) {
 						// We don't know ahead of time what the first first is. This is because the first first is the
 						// last first whose timestamp is less than or equal to the start timestamp. Therefore we need to
 						// wait for the first first after the start timestamp, and then we'll know that the previous
 						// first was the first first.
-						frameQueue.push(lastFrame);
-						firstFrameQueued = true;
+						sampleQueue.push(lastSample);
+						firstSampleQueued = true;
 					} else {
-						lastFrame.frame.close();
+						lastSample.close();
 					}
 				}
 
-				if (wrappedFrame.timestamp >= startTimestamp) {
-					frameQueue.push(wrappedFrame);
-					firstFrameQueued = true;
+				if (sample.timestamp >= startTimestamp) {
+					sampleQueue.push(sample);
+					firstSampleQueued = true;
 				}
 
-				lastFrame = firstFrameQueued ? null : wrappedFrame;
+				lastSample = firstSampleQueued ? null : sample;
 
-				if (frameQueue.length > 0) {
+				if (sampleQueue.length > 0) {
 					onQueueNotEmpty();
 					({ promise: queueNotEmpty, resolve: onQueueNotEmpty } = promiseWithResolvers());
 				}
@@ -319,7 +306,7 @@ export abstract class BaseMediaFrameSink<
 			await packets.next(); // Skip the start packet as we already have it
 
 			while (currentPacket && !ended) {
-				if (frameQueue.length + decoder.getDecodeQueueSize() > MAX_QUEUE_SIZE) {
+				if (sampleQueue.length + decoder.getDecodeQueueSize() > MAX_QUEUE_SIZE) {
 					({ promise: queueDequeue, resolve: onQueueDequeue } = promiseWithResolvers());
 					await queueDequeue;
 					continue;
@@ -340,8 +327,8 @@ export abstract class BaseMediaFrameSink<
 			if (!terminated) await decoder.flush();
 			decoder.close();
 
-			if (!firstFrameQueued && lastFrame) {
-				frameQueue.push(lastFrame);
+			if (!firstSampleQueued && lastSample) {
+				sampleQueue.push(lastSample);
 			}
 
 			decoderIsFlushed = true;
@@ -360,8 +347,8 @@ export abstract class BaseMediaFrameSink<
 						return { value: undefined, done: true };
 					} else if (outOfBandError) {
 						throw outOfBandError;
-					} else if (frameQueue.length > 0) {
-						const value = frameQueue.shift()!;
+					} else if (sampleQueue.length > 0) {
+						const value = sampleQueue.shift()!;
 						onQueueDequeue();
 						return { value, done: false };
 					} else if (!decoderIsFlushed) {
@@ -377,10 +364,10 @@ export abstract class BaseMediaFrameSink<
 				onQueueDequeue();
 				onQueueNotEmpty();
 
-				lastFrame?.frame.close();
+				lastSample?.close();
 
-				for (const frame of frameQueue) {
-					frame.frame.close();
+				for (const sample of sampleQueue) {
+					sample.close();
 				}
 
 				return { value: undefined, done: true };
@@ -395,15 +382,15 @@ export abstract class BaseMediaFrameSink<
 	}
 
 	/** @internal */
-	protected mediaFramesAtTimestamps(
+	protected mediaSamplesAtTimestamps(
 		timestamps: AnyIterable<number>,
-	): AsyncGenerator<WrappedFrame | null, void, unknown> {
+	): AsyncGenerator<MediaSample | null, void, unknown> {
 		validateAnyIterable(timestamps);
 		const timestampIterator = toAsyncIterator(timestamps);
 		const timestampsOfInterest: number[] = [];
 
 		const MAX_QUEUE_SIZE = 8;
-		const frameQueue: (WrappedFrame | null)[] = [];
+		const sampleQueue: (MediaSample | null)[] = [];
 		let { promise: queueNotEmpty, resolve: onQueueNotEmpty } = promiseWithResolvers();
 		let { promise: queueDequeue, resolve: onQueueDequeue } = promiseWithResolvers();
 		let decoderIsFlushed = false;
@@ -414,9 +401,9 @@ export abstract class BaseMediaFrameSink<
 		// the consumer.
 		let outOfBandError = null as Error | null;
 
-		let lastUsedFrame = null as WrappedFrame | null;
-		const pushToQueue = (frame: WrappedFrame | null) => {
-			frameQueue.push(frame);
+		let lastUsedSample = null as MediaSample | null;
+		const pushToQueue = (sample: MediaSample | null) => {
+			sampleQueue.push(sample);
 			onQueueNotEmpty();
 			({ promise: queueNotEmpty, resolve: onQueueNotEmpty } = promiseWithResolvers());
 		};
@@ -424,29 +411,29 @@ export abstract class BaseMediaFrameSink<
 		// The following is the "pump" process that keeps pumping packets into the decoder
 		(async () => {
 			const decoderError = new Error();
-			const decoder = await this._createDecoder((wrappedFrame) => {
+			const decoder = await this._createDecoder((sample) => {
 				onQueueDequeue();
 
 				if (terminated) {
-					wrappedFrame.frame.close();
+					sample.close();
 					return;
 				}
 
-				let frameUsed = false;
+				let sampleUsed = false;
 				while (
 					timestampsOfInterest.length > 0
-					&& wrappedFrame.timestamp - timestampsOfInterest[0]! > -1e-10 // Give it a little epsilon
+					&& sample.timestamp - timestampsOfInterest[0]! > -1e-10 // Give it a little epsilon
 				) {
-					pushToQueue(this._duplicateFrame(wrappedFrame));
+					pushToQueue(sample.clone() as MediaSample);
 					timestampsOfInterest.shift();
-					frameUsed = true;
+					sampleUsed = true;
 				}
 
-				if (frameUsed) {
-					lastUsedFrame?.frame.close();
-					lastUsedFrame = wrappedFrame;
+				if (sampleUsed) {
+					lastUsedSample?.close();
+					lastUsedSample = sample;
 				} else {
-					wrappedFrame.frame.close();
+					sample.close();
 				}
 			}, (error) => {
 				if (!outOfBandError) {
@@ -463,7 +450,7 @@ export abstract class BaseMediaFrameSink<
 			for await (const timestamp of timestampIterator) {
 				validateTimestamp(timestamp);
 
-				while (frameQueue.length + decoder.getDecodeQueueSize() > MAX_QUEUE_SIZE && !terminated) {
+				while (sampleQueue.length + decoder.getDecodeQueueSize() > MAX_QUEUE_SIZE && !terminated) {
 					({ promise: queueDequeue, resolve: onQueueDequeue } = promiseWithResolvers());
 					await queueDequeue;
 				}
@@ -501,10 +488,10 @@ export abstract class BaseMediaFrameSink<
 						targetPacket.sequenceNumber === lastPacket.sequenceNumber
 						&& timestampsOfInterest.length === 0
 					) {
-						// Special case: We have a repeat packet, but the frame for that packet has already been
-						// decoded. Therefore, we need to push the frame here instead of in the decoder callback.
-						if (lastUsedFrame) {
-							pushToQueue(this._duplicateFrame(lastUsedFrame));
+						// Special case: We have a repeat packet, but the sample for that packet has already been
+						// decoded. Therefore, we need to push the sample here instead of in the decoder callback.
+						if (lastUsedSample) {
+							pushToQueue(lastUsedSample.clone() as MediaSample);
 						}
 					} else {
 						timestampsOfInterest.push(targetPacket.timestamp);
@@ -528,7 +515,7 @@ export abstract class BaseMediaFrameSink<
 
 			if (!terminated) {
 				await decoder.flush();
-				lastUsedFrame?.frame.close();
+				lastUsedSample?.close();
 			}
 			decoder.close();
 
@@ -548,8 +535,8 @@ export abstract class BaseMediaFrameSink<
 						return { value: undefined, done: true };
 					} else if (outOfBandError) {
 						throw outOfBandError;
-					} else if (frameQueue.length > 0) {
-						const value = frameQueue.shift();
+					} else if (sampleQueue.length > 0) {
+						const value = sampleQueue.shift();
 						assert(value !== undefined);
 						onQueueDequeue();
 						return { value, done: false };
@@ -565,10 +552,10 @@ export abstract class BaseMediaFrameSink<
 				onQueueDequeue();
 				onQueueNotEmpty();
 
-				for (const frame of frameQueue) {
-					frame?.frame.close();
+				for (const sample of sampleQueue) {
+					sample?.close();
 				}
-				lastUsedFrame?.frame.close();
+				lastUsedSample?.close();
 
 				return { value: undefined, done: true };
 			},
@@ -582,44 +569,45 @@ export abstract class BaseMediaFrameSink<
 	}
 }
 
-class VideoDecoderWrapper extends DecoderWrapper<VideoFrame> {
+class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
 	decoder: VideoDecoder | null = null;
 
 	customDecoder: CustomVideoDecoder | null = null;
 	lastCustomDecoderPromise = Promise.resolve();
 	customDecoderQueueSize = 0;
 
-	frameQueue: VideoFrame[] = [];
+	sampleQueue: VideoSample[] = [];
 
 	constructor(
-		onFrame: (frame: WrappedMediaFrame<VideoFrame>) => unknown,
+		onSample: (sample: VideoSample) => unknown,
 		onError: (error: DOMException) => unknown,
 		codec: VideoCodec,
 		decoderConfig: VideoDecoderConfig,
+		public rotation: Rotation,
 		public timeResolution: number,
 	) {
-		super(onFrame, onError);
+		super(onSample, onError);
 
-		const frameHandler = (frame: VideoFrame) => {
+		const sampleHandler = (sample: VideoSample) => {
 			// For correct B-frame handling, we don't just hand over the frames directly but instead add them to a
 			// queue, because we want to ensure frames are emitted in presentation order. We flush the queue each time
 			// we receive a frame with a timestamp larger than the highest we've seen so far, as we can sure that is
 			// not a B-frame. Typically, WebCodecs automatically guarantees that frames are emitted in presentation
 			// order, but some browsers (Safari) don't always follow this rule.
-			if (this.frameQueue.length > 0 && (frame.timestamp >= last(this.frameQueue)!.timestamp)) {
-				for (const frame of this.frameQueue) {
-					this.wrapAndEmitFrame(frame);
+			if (this.sampleQueue.length > 0 && (sample.timestamp >= last(this.sampleQueue)!.timestamp)) {
+				for (const sample of this.sampleQueue) {
+					this.finalizeAndEmitSample(sample);
 				}
 
-				this.frameQueue.length = 0;
+				this.sampleQueue.length = 0;
 			}
 
 			const insertionIndex = binarySearchLessOrEqual(
-				this.frameQueue,
-				frame.timestamp,
+				this.sampleQueue,
+				sample.timestamp,
 				x => x.timestamp,
 			);
-			this.frameQueue.splice(insertionIndex + 1, 0, frame);
+			this.sampleQueue.splice(insertionIndex + 1, 0, sample);
 		};
 
 		const MatchingCustomDecoder = customVideoDecoders.find(x => x.supports(codec, decoderConfig));
@@ -628,28 +616,24 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoFrame> {
 			this.customDecoder = new MatchingCustomDecoder() as CustomVideoDecoder;
 			this.customDecoder.codec = codec;
 			this.customDecoder.config = decoderConfig;
-			this.customDecoder.onFrame = frameHandler;
+			this.customDecoder.onSample = sampleHandler;
 
 			this.customDecoder.init();
 		} else {
 			this.decoder = new VideoDecoder({
-				output: frameHandler,
+				output: frame => sampleHandler(new VideoSample(frame)),
 				error: onError,
 			});
 			this.decoder.configure(decoderConfig);
 		}
 	}
 
-	wrapAndEmitFrame(frame: VideoFrame) {
-		// Round the microsecond timestamps to the time resolution
-		const timestamp = Math.round(frame.timestamp / 1e6 * this.timeResolution) / this.timeResolution;
-		const duration = Math.round((frame.duration ?? 0) / 1e6 * this.timeResolution) / this.timeResolution;
+	finalizeAndEmitSample(sample: VideoSample) {
+		// Round the timestamps to the time resolution
+		sample.setTimestamp(Math.round(sample.timestamp * this.timeResolution) / this.timeResolution);
+		sample.setDuration(Math.round(sample.duration * this.timeResolution) / this.timeResolution);
 
-		this.onFrame({
-			frame,
-			timestamp,
-			duration,
-		});
+		this.onSample(sample);
 	}
 
 	getDecodeQueueSize() {
@@ -683,10 +667,10 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoFrame> {
 			await this.decoder.flush();
 		}
 
-		for (const frame of this.frameQueue) {
-			this.wrapAndEmitFrame(frame);
+		for (const sample of this.sampleQueue) {
+			this.finalizeAndEmitSample(sample);
 		}
-		this.frameQueue.length = 0;
+		this.sampleQueue.length = 0;
 	}
 
 	close() {
@@ -697,22 +681,15 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoFrame> {
 			this.decoder.close();
 		}
 
-		for (const frame of this.frameQueue) {
-			frame.close();
+		for (const sample of this.sampleQueue) {
+			sample.close();
 		}
-		this.frameQueue.length = 0;
+		this.sampleQueue.length = 0;
 	}
 }
 
 /** @public */
-export type WrappedVideoFrame = {
-	frame: VideoFrame;
-	timestamp: number;
-	duration: number;
-};
-
-/** @public */
-export class VideoFrameSink extends BaseMediaFrameSink<VideoFrame> {
+export class VideoSampleSink extends BaseMediaSampleSink<VideoSample> {
 	/** @internal */
 	_videoTrack: InputVideoTrack;
 
@@ -728,7 +705,7 @@ export class VideoFrameSink extends BaseMediaFrameSink<VideoFrame> {
 
 	/** @internal */
 	async _createDecoder(
-		onFrame: (frame: WrappedMediaFrame<VideoFrame>) => unknown,
+		onSample: (sample: VideoSample) => unknown,
 		onError: (error: DOMException) => unknown,
 	) {
 		if (!(await this._videoTrack.canDecode())) {
@@ -739,11 +716,12 @@ export class VideoFrameSink extends BaseMediaFrameSink<VideoFrame> {
 		}
 
 		const codec = this._videoTrack.codec;
+		const rotation = this._videoTrack.rotation;
 		const decoderConfig = await this._videoTrack.getDecoderConfig();
 		const timeResolution = this._videoTrack.timeResolution;
 		assert(codec && decoderConfig);
 
-		return new VideoDecoderWrapper(onFrame, onError, codec, decoderConfig, timeResolution);
+		return new VideoDecoderWrapper(onSample, onError, codec, decoderConfig, rotation, timeResolution);
 	}
 
 	/** @internal */
@@ -751,36 +729,21 @@ export class VideoFrameSink extends BaseMediaFrameSink<VideoFrame> {
 		return new EncodedPacketSink(this._videoTrack);
 	}
 
-	/** @internal */
-	_wrappedFrameToWrappedVideoFrame(frame: WrappedMediaFrame<VideoFrame>): WrappedVideoFrame {
-		return {
-			frame: frame.frame,
-			timestamp: frame.timestamp,
-			duration: frame.duration,
-		};
-	}
-
-	async getFrame(timestamp: number) {
+	async getSample(timestamp: number) {
 		validateTimestamp(timestamp);
 
-		for await (const frame of this.mediaFramesAtTimestamps([timestamp])) {
-			return frame && this._wrappedFrameToWrappedVideoFrame(frame);
+		for await (const sample of this.mediaSamplesAtTimestamps([timestamp])) {
+			return sample;
 		}
 		throw new Error('Internal error: Iterator returned nothing.');
 	}
 
-	frames(startTimestamp = 0, endTimestamp = Infinity) {
-		return mapAsyncGenerator(
-			this.mediaFramesInRange(startTimestamp, endTimestamp),
-			frame => this._wrappedFrameToWrappedVideoFrame(frame),
-		);
+	samples(startTimestamp = 0, endTimestamp = Infinity) {
+		return this.mediaSamplesInRange(startTimestamp, endTimestamp);
 	}
 
-	framesAtTimestamps(timestamps: AnyIterable<number>) {
-		return mapAsyncGenerator(
-			this.mediaFramesAtTimestamps(timestamps),
-			frame => frame && this._wrappedFrameToWrappedVideoFrame(frame),
-		);
+	samplesAtTimestamps(timestamps: AnyIterable<number>) {
+		return this.mediaSamplesAtTimestamps(timestamps);
 	}
 }
 
@@ -813,7 +776,7 @@ export class CanvasSink {
 	/** @internal */
 	_rotation: Rotation;
 	/** @internal */
-	_videoFrameSink: VideoFrameSink;
+	_videoSampleSink: VideoSampleSink;
 	/** @internal */
 	_canvasPool: (HTMLCanvasElement | OffscreenCanvas | null)[];
 	/** @internal */
@@ -877,12 +840,12 @@ export class CanvasSink {
 		this._height = height;
 		this._rotation = rotation;
 		this._fit = options.fit ?? 'fill';
-		this._videoFrameSink = new VideoFrameSink(videoTrack);
+		this._videoSampleSink = new VideoSampleSink(videoTrack);
 		this._canvasPool = Array.from({ length: options.poolSize ?? 0 }, () => null);
 	}
 
 	/** @internal */
-	_videoFrameToWrappedCanvas(frame: WrappedVideoFrame): WrappedCanvas {
+	_videoSampleToWrappedCanvas(sample: VideoSample): WrappedCanvas {
 		let canvas = this._canvasPool[this._nextCanvasIndex];
 		if (!canvas) {
 			if (typeof OffscreenCanvas !== 'undefined') {
@@ -903,12 +866,13 @@ export class CanvasSink {
 			this._nextCanvasIndex = (this._nextCanvasIndex + 1) % this._canvasPool.length;
 		}
 
-		const context = canvas.getContext('2d', { alpha: false });
+		const context
+			= canvas.getContext('2d', { alpha: false }) as CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;
 		assert(context);
 
 		context.resetTransform();
 
-		// These variables specify where the final frame will be drawn on the canvas
+		// These variables specify where the final sample will be drawn on the canvas
 		let dx: number;
 		let dy: number;
 		let newWidth: number;
@@ -920,15 +884,15 @@ export class CanvasSink {
 			newWidth = this._width;
 			newHeight = this._height;
 		} else {
-			const [frameWidth, frameHeight] = this._rotation % 180 === 0
-				? [frame.frame.codedWidth, frame.frame.codedHeight]
-				: [frame.frame.codedHeight, frame.frame.codedWidth];
+			const [sampleWidth, sampleHeight] = this._rotation % 180 === 0
+				? [sample.codedWidth, sample.codedHeight]
+				: [sample.codedHeight, sample.codedWidth];
 
 			const scale = this._fit === 'contain'
-				? Math.min(this._width / frameWidth, this._height / frameHeight)
-				: Math.max(this._width / frameWidth, this._height / frameHeight);
-			newWidth = frameWidth * scale;
-			newHeight = frameHeight * scale;
+				? Math.min(this._width / sampleWidth, this._height / sampleHeight)
+				: Math.max(this._width / sampleWidth, this._height / sampleHeight);
+			newWidth = sampleWidth * scale;
+			newHeight = sampleHeight * scale;
 			dx = (this._width - newWidth) / 2;
 			dy = (this._height - newHeight) / 2;
 		}
@@ -936,46 +900,46 @@ export class CanvasSink {
 		const aspectRatioChange = this._rotation % 180 === 0 ? 1 : newWidth / newHeight;
 		context.translate(this._width / 2, this._height / 2);
 		context.rotate(this._rotation * Math.PI / 180);
-		// This aspect ratio compensation is done so that we can draw the frame with the intended dimensions and
+		// This aspect ratio compensation is done so that we can draw the sample with the intended dimensions and
 		// don't need to think about how those dimensions change after the rotation
 		context.scale(1 / aspectRatioChange, aspectRatioChange);
 		context.translate(-this._width / 2, -this._height / 2);
 
-		context.drawImage(frame.frame, dx, dy, newWidth, newHeight);
+		context.drawImage(sample.toCanvasImageSource(), dx, dy, newWidth, newHeight);
 
 		const result = {
 			canvas,
-			timestamp: frame.timestamp,
-			duration: frame.duration,
+			timestamp: sample.timestamp,
+			duration: sample.duration,
 		};
 
-		frame.frame.close();
+		sample.close();
 		return result;
 	}
 
 	async getCanvas(timestamp: number) {
 		validateTimestamp(timestamp);
 
-		const frame = await this._videoFrameSink.getFrame(timestamp);
-		return frame && this._videoFrameToWrappedCanvas(frame);
+		const sample = await this._videoSampleSink.getSample(timestamp);
+		return sample && this._videoSampleToWrappedCanvas(sample);
 	}
 
 	canvases(startTimestamp = 0, endTimestamp = Infinity) {
 		return mapAsyncGenerator(
-			this._videoFrameSink.frames(startTimestamp, endTimestamp),
-			frame => this._videoFrameToWrappedCanvas(frame),
+			this._videoSampleSink.samples(startTimestamp, endTimestamp),
+			sample => this._videoSampleToWrappedCanvas(sample),
 		);
 	}
 
 	canvasesAtTimestamps(timestamps: AnyIterable<number>) {
 		return mapAsyncGenerator(
-			this._videoFrameSink.framesAtTimestamps(timestamps),
-			frame => frame && this._videoFrameToWrappedCanvas(frame),
+			this._videoSampleSink.samplesAtTimestamps(timestamps),
+			sample => sample && this._videoSampleToWrappedCanvas(sample),
 		);
 	}
 }
 
-class AudioDecoderWrapper extends DecoderWrapper<AudioData> {
+class AudioDecoderWrapper extends DecoderWrapper<AudioSample> {
 	decoder: AudioDecoder | null = null;
 
 	customDecoder: CustomAudioDecoder | null = null;
@@ -983,25 +947,20 @@ class AudioDecoderWrapper extends DecoderWrapper<AudioData> {
 	customDecoderQueueSize = 0;
 
 	constructor(
-		onData: (data: WrappedMediaFrame<AudioData>) => unknown,
+		onSample: (sample: AudioSample) => unknown,
 		onError: (error: DOMException) => unknown,
 		codec: AudioCodec,
 		decoderConfig: AudioDecoderConfig,
 	) {
-		super(onData, onError);
+		super(onSample, onError);
 
-		const dataHandler = (data: AudioData) => {
+		const sampleHandler = (sample: AudioSample) => {
 			const sampleRate = decoderConfig.sampleRate;
 
-			// Round the microsecond timestamps to the sample rate
-			const timestamp = Math.round(data.timestamp / 1e6 * sampleRate) / sampleRate;
-			const duration = Math.round(data.duration / 1e6 * sampleRate) / sampleRate;
+			// Round the timestamp to the sample rate
+			sample.setTimestamp(Math.round(sample.timestamp * sampleRate) / sampleRate);
 
-			onData({
-				frame: data,
-				timestamp,
-				duration,
-			});
+			onSample(sample);
 		};
 
 		const MatchingCustomDecoder = customAudioDecoders.find(x => x.supports(codec, decoderConfig));
@@ -1010,12 +969,12 @@ class AudioDecoderWrapper extends DecoderWrapper<AudioData> {
 			this.customDecoder = new MatchingCustomDecoder() as CustomAudioDecoder;
 			this.customDecoder.codec = codec;
 			this.customDecoder.config = decoderConfig;
-			this.customDecoder.onData = dataHandler;
+			this.customDecoder.onSample = sampleHandler;
 
 			this.customDecoder.init();
 		} else {
 			this.decoder = new AudioDecoder({
-				output: dataHandler,
+				output: data => sampleHandler(new AudioSample(data)),
 				error: onError,
 			});
 			this.decoder.configure(decoderConfig);
@@ -1066,7 +1025,7 @@ class AudioDecoderWrapper extends DecoderWrapper<AudioData> {
 
 // There are a lot of PCM variants not natively supported by the browser and by AudioData. Therefore we need a simple
 // decoder that maps any input PCM format into a PCM format supported by the browser.
-class PcmAudioDecoderWrapper extends DecoderWrapper<AudioData> {
+class PcmAudioDecoderWrapper extends DecoderWrapper<AudioSample> {
 	codec: PcmAudioCodec;
 
 	inputSampleSize: 1 | 2 | 3 | 4;
@@ -1081,11 +1040,11 @@ class PcmAudioDecoderWrapper extends DecoderWrapper<AudioData> {
 	currentTimestamp: number | null = null;
 
 	constructor(
-		onData: (data: WrappedMediaFrame<AudioData>) => unknown,
+		onSample: (sample: AudioSample) => unknown,
 		onError: (error: DOMException) => unknown,
 		public decoderConfig: AudioDecoderConfig,
 	) {
-		super(onData, onError);
+		super(onSample, onError);
 
 		assert((PCM_AUDIO_CODECS as readonly string[]).includes(decoderConfig.codec));
 		this.codec = decoderConfig.codec as PcmAudioCodec;
@@ -1207,21 +1166,16 @@ class PcmAudioDecoderWrapper extends DecoderWrapper<AudioData> {
 		const preciseTimestamp = this.currentTimestamp;
 		this.currentTimestamp += preciseDuration;
 
-		const audioData = new AudioData({
+		const audioSample = new AudioSample({
 			format: this.outputFormat,
 			data: outputBuffer,
 			numberOfChannels: this.decoderConfig.numberOfChannels,
 			sampleRate: this.decoderConfig.sampleRate,
 			numberOfFrames,
-			timestamp: 1e6 * preciseTimestamp,
+			timestamp: preciseTimestamp,
 		});
 
-		// Since all other decoders are async, we'll make this one behave async as well
-		queueMicrotask(() => this.onFrame({
-			frame: audioData,
-			timestamp: preciseTimestamp,
-			duration: preciseDuration,
-		}));
+		this.onSample(audioSample);
 	}
 
 	async flush() {
@@ -1234,14 +1188,7 @@ class PcmAudioDecoderWrapper extends DecoderWrapper<AudioData> {
 }
 
 /** @public */
-export type WrappedAudioData = {
-	data: AudioData;
-	timestamp: number;
-	duration: number;
-};
-
-/** @public */
-export class AudioDataSink extends BaseMediaFrameSink<AudioData> {
+export class AudioSampleSink extends BaseMediaSampleSink<AudioSample> {
 	/** @internal */
 	_audioTrack: InputAudioTrack;
 
@@ -1257,7 +1204,7 @@ export class AudioDataSink extends BaseMediaFrameSink<AudioData> {
 
 	/** @internal */
 	async _createDecoder(
-		onData: (data: WrappedMediaFrame<AudioData>) => unknown,
+		onSample: (sample: AudioSample) => unknown,
 		onError: (error: DOMException) => unknown,
 	) {
 		if (!(await this._audioTrack.canDecode())) {
@@ -1272,47 +1219,32 @@ export class AudioDataSink extends BaseMediaFrameSink<AudioData> {
 		assert(codec && decoderConfig);
 
 		if ((PCM_AUDIO_CODECS as readonly string[]).includes(decoderConfig.codec)) {
-			return new PcmAudioDecoderWrapper(onData, onError, decoderConfig);
+			return new PcmAudioDecoderWrapper(onSample, onError, decoderConfig);
 		} else {
-			return new AudioDecoderWrapper(onData, onError, codec, decoderConfig);
+			return new AudioDecoderWrapper(onSample, onError, codec, decoderConfig);
 		}
 	}
 
-	/** @internal */
-	_wrappedFrameToWrappedAudioData(frame: WrappedMediaFrame<AudioData>): WrappedAudioData {
-		return {
-			data: frame.frame,
-			timestamp: frame.timestamp,
-			duration: frame.duration,
-		};
-	}
-
 	/** @internal */
 	_createPacketSink() {
 		return new EncodedPacketSink(this._audioTrack);
 	}
 
-	async getData(timestamp: number) {
+	async getSample(timestamp: number) {
 		validateTimestamp(timestamp);
 
-		for await (const data of this.mediaFramesAtTimestamps([timestamp])) {
-			return data && this._wrappedFrameToWrappedAudioData(data);
+		for await (const sample of this.mediaSamplesAtTimestamps([timestamp])) {
+			return sample;
 		}
 		throw new Error('Internal error: Iterator returned nothing.');
 	}
 
-	data(startTimestamp = 0, endTimestamp = Infinity) {
-		return mapAsyncGenerator(
-			this.mediaFramesInRange(startTimestamp, endTimestamp),
-			data => this._wrappedFrameToWrappedAudioData(data),
-		);
+	samples(startTimestamp = 0, endTimestamp = Infinity) {
+		return this.mediaSamplesInRange(startTimestamp, endTimestamp);
 	}
 
-	dataAtTimestamps(timestamps: AnyIterable<number>) {
-		return mapAsyncGenerator(
-			this.mediaFramesAtTimestamps(timestamps),
-			data => data && this._wrappedFrameToWrappedAudioData(data),
-		);
+	samplesAtTimestamps(timestamps: AnyIterable<number>) {
+		return this.mediaSamplesAtTimestamps(timestamps);
 	}
 }
 
@@ -1326,60 +1258,43 @@ export type WrappedAudioBuffer = {
 /** @public */
 export class AudioBufferSink {
 	/** @internal */
-	_audioDataSink: AudioDataSink;
+	_audioSampleSink: AudioSampleSink;
 
 	constructor(audioTrack: InputAudioTrack) {
 		if (!(audioTrack instanceof InputAudioTrack)) {
 			throw new TypeError('audioTrack must be an InputAudioTrack.');
 		}
 
-		this._audioDataSink = new AudioDataSink(audioTrack);
+		this._audioSampleSink = new AudioSampleSink(audioTrack);
 	}
 
 	/** @internal */
-	_audioDataToWrappedArrayBuffer(data: WrappedAudioData): WrappedAudioBuffer {
-		const audioBuffer = new AudioBuffer({
-			numberOfChannels: data.data.numberOfChannels,
-			length: data.data.numberOfFrames,
-			sampleRate: data.data.sampleRate,
-		});
-
-		// All user agents are required to support conversion to f32-planar
-		const dataBytes = new Float32Array(data.data.allocationSize({ planeIndex: 0, format: 'f32-planar' }) / 4);
-
-		for (let i = 0; i < data.data.numberOfChannels; i++) {
-			data.data.copyTo(dataBytes, { planeIndex: i, format: 'f32-planar' });
-			audioBuffer.copyToChannel(dataBytes, i);
-		}
-
-		const result = {
-			buffer: audioBuffer,
-			timestamp: data.timestamp,
-			duration: data.duration,
+	_audioSampleToWrappedArrayBuffer(sample: AudioSample): WrappedAudioBuffer {
+		return {
+			buffer: sample.toAudioBuffer(),
+			timestamp: sample.timestamp,
+			duration: sample.duration,
 		};
-
-		data.data.close();
-		return result;
 	}
 
 	async getBuffer(timestamp: number) {
 		validateTimestamp(timestamp);
 
-		const data = await this._audioDataSink.getData(timestamp);
-		return data && this._audioDataToWrappedArrayBuffer(data);
+		const data = await this._audioSampleSink.getSample(timestamp);
+		return data && this._audioSampleToWrappedArrayBuffer(data);
 	}
 
 	buffers(startTimestamp = 0, endTimestamp = Infinity) {
 		return mapAsyncGenerator(
-			this._audioDataSink.data(startTimestamp, endTimestamp),
-			data => this._audioDataToWrappedArrayBuffer(data),
+			this._audioSampleSink.samples(startTimestamp, endTimestamp),
+			data => this._audioSampleToWrappedArrayBuffer(data),
 		);
 	}
 
 	buffersAtTimestamps(timestamps: AnyIterable<number>) {
 		return mapAsyncGenerator(
-			this._audioDataSink.dataAtTimestamps(timestamps),
-			data => data && this._audioDataToWrappedArrayBuffer(data),
+			this._audioSampleSink.samplesAtTimestamps(timestamps),
+			data => data && this._audioSampleToWrappedArrayBuffer(data),
 		);
 	}
 }

src/media-source.ts

@@ -26,6 +26,7 @@ import {
 	customAudioEncoders,
 } from './custom-coder';
 import { EncodedPacket } from './packet';
+import { AudioSample, VideoSample } from './sample';
 
 /** @public */
 export abstract class MediaSource {
@@ -194,33 +195,33 @@ class VideoEncoderWrapper {
 
 	constructor(private source: VideoSource, private encodingConfig: VideoEncodingConfig) {}
 
-	async add(videoFrame: VideoFrame, shouldClose: boolean, encodeOptions?: VideoEncoderEncodeOptions) {
+	async add(videoSample: VideoSample, shouldClose: boolean, encodeOptions?: VideoEncoderEncodeOptions) {
 		this.source._ensureValidAdd();
 
-		// Ensure video frame size remains constant
+		// Ensure video sample size remains constant
 		if (this.lastWidth !== null && this.lastHeight !== null) {
-			if (videoFrame.codedWidth !== this.lastWidth || videoFrame.codedHeight !== this.lastHeight) {
+			if (videoSample.codedWidth !== this.lastWidth || videoSample.codedHeight !== this.lastHeight) {
 				throw new Error(
-					`Video frame size must remain constant. Expected ${this.lastWidth}x${this.lastHeight},`
-					+ ` got ${videoFrame.codedWidth}x${videoFrame.codedHeight}.`,
+					`Video sample size must remain constant. Expected ${this.lastWidth}x${this.lastHeight},`
+					+ ` got ${videoSample.codedWidth}x${videoSample.codedHeight}.`,
 				);
 			}
 		} else {
-			this.lastWidth = videoFrame.codedWidth;
-			this.lastHeight = videoFrame.codedHeight;
+			this.lastWidth = videoSample.codedWidth;
+			this.lastHeight = videoSample.codedHeight;
 		}
 
 		if (!this.encoderInitialized) {
 			if (this.ensureEncoderPromise) {
 				await this.ensureEncoderPromise;
 			} else {
-				await this.ensureEncoder(videoFrame);
+				await this.ensureEncoder(videoSample);
 			}
 		}
 		assert(this.encoderInitialized);
 
 		const keyFrameInterval = this.encodingConfig.keyFrameInterval ?? 5;
-		const multipleOfKeyFrameInterval = Math.floor((videoFrame.timestamp / 1e6) / keyFrameInterval);
+		const multipleOfKeyFrameInterval = Math.floor(videoSample.timestamp / keyFrameInterval);
 
 		// Ensure a key frame every KEY_FRAME_INTERVAL seconds. It is important that all video tracks follow the same
 		// "key frame" rhythm, because aligned key frames are required to start new fragments in ISOBMFF or clusters
@@ -236,14 +237,14 @@ class VideoEncoderWrapper {
 		if (this.customEncoder) {
 			this.customEncoderQueueSize++;
 			this.lastCustomEncoderPromise = this.lastCustomEncoderPromise.then(() => {
-				return this.customEncoder!.encode(videoFrame, finalEncodeOptions);
+				return this.customEncoder!.encode(videoSample, finalEncodeOptions);
 			});
 
 			void this.lastCustomEncoderPromise.then(() => {
 				this.customEncoderQueueSize--;
 
 				if (shouldClose) {
-					videoFrame.close();
+					videoSample.close();
 				}
 			});
 
@@ -252,10 +253,12 @@ class VideoEncoderWrapper {
 			}
 		} else {
 			assert(this.encoder);
+			const videoFrame = videoSample.toVideoFrame();
 			this.encoder.encode(videoFrame, finalEncodeOptions);
+			videoFrame.close();
 
 			if (shouldClose) {
-				videoFrame.close();
+				videoSample.close();
 			}
 
 			// We need to do this after sending the frame to the encoder as the frame otherwise might be closed
@@ -267,7 +270,7 @@ class VideoEncoderWrapper {
 		await this.muxer!.mutex.currentPromise; // Allow the writer to apply backpressure
 	}
 
-	private async ensureEncoder(videoFrame: VideoFrame) {
+	private async ensureEncoder(videoSample: VideoSample) {
 		if (this.encoder) {
 			return;
 		}
@@ -275,8 +278,8 @@ class VideoEncoderWrapper {
 		const { promise, resolve } = promiseWithResolvers();
 		this.ensureEncoderPromise = promise;
 
-		const width = videoFrame.codedWidth;
-		const height = videoFrame.codedHeight;
+		const width = videoSample.codedWidth;
+		const height = videoSample.codedHeight;
 		const bitrate = this.encodingConfig.bitrate instanceof Quality
 			? this.encodingConfig.bitrate._toVideoBitrate(this.encodingConfig.codec, width, height)
 			: this.encodingConfig.bitrate;
@@ -365,7 +368,7 @@ class VideoEncoderWrapper {
 }
 
 /** @public */
-export class VideoFrameSource extends VideoSource {
+export class VideoSampleSource extends VideoSource {
 	/** @internal */
 	private _encoder: VideoEncoderWrapper;
 
@@ -376,12 +379,12 @@ export class VideoFrameSource extends VideoSource {
 		this._encoder = new VideoEncoderWrapper(this, encodingConfig);
 	}
 
-	add(videoFrame: VideoFrame, encodeOptions?: VideoEncoderEncodeOptions) {
-		if (!(videoFrame instanceof VideoFrame)) {
-			throw new TypeError('videoFrame must be a VideoFrame.');
+	add(videoSample: VideoSample, encodeOptions?: VideoEncoderEncodeOptions) {
+		if (!(videoSample instanceof VideoSample)) {
+			throw new TypeError('videoSample must be a VideoSample.');
 		}
 
-		return this._encoder.add(videoFrame, false, encodeOptions);
+		return this._encoder.add(videoSample, false, encodeOptions);
 	}
 
 	/** @internal */
@@ -419,13 +422,8 @@ export class CanvasSource extends VideoSource {
 			throw new TypeError('duration must be a non-negative number.');
 		}
 
-		const frame = new VideoFrame(this._canvas, {
-			timestamp: Math.round(1e6 * timestamp),
-			duration: Math.round(1e6 * duration) || undefined, // Drag 0 duration to undefined, glitches some codecs
-			alpha: 'discard',
-		});
-
-		return this._encoder.add(frame, true, encodeOptions);
+		const sample = new VideoSample(this._canvas, { timestamp, duration });
+		return this._encoder.add(sample, true, encodeOptions);
 	}
 
 	/** @internal */
@@ -475,7 +473,7 @@ export class MediaStreamVideoTrackSource extends VideoSource {
 					return;
 				}
 
-				void this._encoder.add(videoFrame, true);
+				void this._encoder.add(new VideoSample(videoFrame), true);
 			},
 		});
 
@@ -587,31 +585,31 @@ class AudioEncoderWrapper {
 
 	constructor(private source: AudioSource, private encodingConfig: AudioEncodingConfig) {}
 
-	async add(audioData: AudioData, shouldClose: boolean) {
+	async add(audioSample: AudioSample, shouldClose: boolean) {
 		this.source._ensureValidAdd();
 
 		// Ensure audio parameters remain constant
 		if (this.lastNumberOfChannels !== null && this.lastSampleRate !== null) {
 			if (
-				audioData.numberOfChannels !== this.lastNumberOfChannels
-				|| audioData.sampleRate !== this.lastSampleRate
+				audioSample.numberOfChannels !== this.lastNumberOfChannels
+				|| audioSample.sampleRate !== this.lastSampleRate
 			) {
 				throw new Error(
 					`Audio parameters must remain constant. Expected ${this.lastNumberOfChannels} channels at`
-					+ ` ${this.lastSampleRate} Hz, got ${audioData.numberOfChannels} channels at`
-					+ ` ${audioData.sampleRate} Hz.`,
+					+ ` ${this.lastSampleRate} Hz, got ${audioSample.numberOfChannels} channels at`
+					+ ` ${audioSample.sampleRate} Hz.`,
 				);
 			}
 		} else {
-			this.lastNumberOfChannels = audioData.numberOfChannels;
-			this.lastSampleRate = audioData.sampleRate;
+			this.lastNumberOfChannels = audioSample.numberOfChannels;
+			this.lastSampleRate = audioSample.sampleRate;
 		}
 
 		if (!this.encoderInitialized) {
 			if (this.ensureEncoderPromise) {
 				await this.ensureEncoderPromise;
 			} else {
-				await this.ensureEncoder(audioData);
+				await this.ensureEncoder(audioSample);
 			}
 		}
 		assert(this.encoderInitialized);
@@ -619,14 +617,14 @@ class AudioEncoderWrapper {
 		if (this.customEncoder) {
 			this.customEncoderQueueSize++;
 			this.lastCustomEncoderPromise = this.lastCustomEncoderPromise.then(() => {
-				return this.customEncoder!.encode(audioData);
+				return this.customEncoder!.encode(audioSample);
 			});
 
 			void this.lastCustomEncoderPromise.then(() => {
 				this.customEncoderQueueSize--;
 
 				if (shouldClose) {
-					audioData.close();
+					audioSample.close();
 				}
 			});
 
@@ -636,13 +634,15 @@ class AudioEncoderWrapper {
 
 			await this.muxer!.mutex.currentPromise; // Allow the writer to apply backpressure
 		} else if (this.isPcmEncoder) {
-			await this.doPcmEncoding(audioData, shouldClose);
+			await this.doPcmEncoding(audioSample, shouldClose);
 		} else {
 			assert(this.encoder);
+			const audioData = audioSample.toAudioData();
 			this.encoder.encode(audioData);
+			audioData.close();
 
 			if (shouldClose) {
-				audioData.close();
+				audioSample.close();
 			}
 
 			if (this.encoder.encodeQueueSize >= 4) {
@@ -653,12 +653,12 @@ class AudioEncoderWrapper {
 		}
 	}
 
-	private async doPcmEncoding(audioData: AudioData, shouldClose: boolean) {
+	private async doPcmEncoding(audioSample: AudioSample, shouldClose: boolean) {
 		assert(this.outputSampleSize);
 		assert(this.writeOutputValue);
 
 		// Need to extract data from the audio data before we close it
-		const { numberOfChannels, numberOfFrames, sampleRate, timestamp } = audioData;
+		const { numberOfChannels, numberOfFrames, sampleRate, timestamp } = audioSample;
 
 		const CHUNK_SIZE = 2048;
 		const outputs: {
@@ -668,7 +668,7 @@ class AudioEncoderWrapper {
 
 		// Prepare all of the output buffers, each being bounded by CHUNK_SIZE so we don't generate huge packets
 		for (let frame = 0; frame < numberOfFrames; frame += CHUNK_SIZE) {
-			const frameCount = Math.min(CHUNK_SIZE, audioData.numberOfFrames - frame);
+			const frameCount = Math.min(CHUNK_SIZE, audioSample.numberOfFrames - frame);
 			const outputSize = frameCount * numberOfChannels * this.outputSampleSize;
 			const outputBuffer = new ArrayBuffer(outputSize);
 			const outputView = new DataView(outputBuffer);
@@ -676,12 +676,11 @@ class AudioEncoderWrapper {
 			outputs.push({ frameCount, view: outputView });
 		}
 
-		// All user agents are required to support conversion to f32-planar
-		const allocationSize = audioData.allocationSize(({ planeIndex: 0, format: 'f32-planar' }));
+		const allocationSize = audioSample.allocationSize(({ planeIndex: 0, format: 'f32-planar' }));
 		const floats = new Float32Array(allocationSize / Float32Array.BYTES_PER_ELEMENT);
 
 		for (let i = 0; i < numberOfChannels; i++) {
-			audioData.copyTo(floats, { planeIndex: i, format: 'f32-planar' });
+			audioSample.copyTo(floats, { planeIndex: i, format: 'f32-planar' });
 
 			for (let j = 0; j < outputs.length; j++) {
 				const { frameCount, view } = outputs[j]!;
@@ -697,7 +696,7 @@ class AudioEncoderWrapper {
 		}
 
 		if (shouldClose) {
-			audioData.close();
+			audioSample.close();
 		}
 
 		const meta: EncodedAudioChunkMetadata = {
@@ -716,7 +715,7 @@ class AudioEncoderWrapper {
 			const packet = new EncodedPacket(
 				new Uint8Array(outputBuffer),
 				'key',
-				timestamp / 1e6 + startFrame / sampleRate,
+				timestamp + startFrame / sampleRate,
 				frameCount / sampleRate,
 			);
 
@@ -725,7 +724,7 @@ class AudioEncoderWrapper {
 		}
 	}
 
-	private async ensureEncoder(audioData: AudioData) {
+	private async ensureEncoder(audioSample: AudioSample) {
 		if (this.encoderInitialized) {
 			return;
 		}
@@ -733,7 +732,7 @@ class AudioEncoderWrapper {
 		const { promise, resolve } = promiseWithResolvers();
 		this.ensureEncoderPromise = promise;
 
-		const { numberOfChannels, sampleRate } = audioData;
+		const { numberOfChannels, sampleRate } = audioSample;
 		const bitrate = this.encodingConfig.bitrate instanceof Quality
 			? this.encodingConfig.bitrate._toAudioBitrate(this.encodingConfig.codec)
 			: this.encodingConfig.bitrate;
@@ -903,7 +902,7 @@ class AudioEncoderWrapper {
 }
 
 /** @public */
-export class AudioDataSource extends AudioSource {
+export class AudioSampleSource extends AudioSource {
 	/** @internal */
 	private _encoder: AudioEncoderWrapper;
 
@@ -914,12 +913,12 @@ export class AudioDataSource extends AudioSource {
 		this._encoder = new AudioEncoderWrapper(this, encodingConfig);
 	}
 
-	add(audioData: AudioData) {
-		if (!(audioData instanceof AudioData)) {
-			throw new TypeError('audioData must be an AudioData.');
+	add(audioSample: AudioSample) {
+		if (!(audioSample instanceof AudioSample)) {
+			throw new TypeError('audioSample must be an AudioSample.');
 		}
 
-		return this._encoder.add(audioData, false);
+		return this._encoder.add(audioSample, false);
 	}
 
 	/** @internal */
@@ -972,16 +971,16 @@ export class AudioBufferSource extends AudioSource {
 				);
 			}
 
-			const audioData = new AudioData({
+			const audioSample = new AudioSample({
 				format: 'f32-planar',
 				sampleRate,
 				numberOfFrames: framesToCopy,
 				numberOfChannels,
-				timestamp: (1e6 * (this._accumulatedFrameCount + currentRelativeFrame)) / sampleRate,
+				timestamp: (this._accumulatedFrameCount + currentRelativeFrame) / sampleRate,
 				data: chunkData,
 			});
 
-			promises.push(this._encoder.add(audioData, true));
+			promises.push(this._encoder.add(audioSample, true));
 
 			currentRelativeFrame += framesToCopy;
 			remainingFrames -= framesToCopy;
@@ -1033,7 +1032,7 @@ export class MediaStreamAudioTrackSource extends AudioSource {
 					return;
 				}
 
-				void this._encoder.add(audioData, true);
+				void this._encoder.add(new AudioSample(audioData), true);
 			},
 		});
 

src/misc.ts

@@ -63,8 +63,12 @@ export const toUint8Array = (source: AllowSharedBufferSource): Uint8Array => {
 	}
 };
 
-export const toDataView = (bytes: Uint8Array) => {
-	return new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+export const toDataView = (source: AllowSharedBufferSource) => {
+	if (source instanceof ArrayBuffer) {
+		return new DataView(source);
+	} else {
+		return new DataView(source.buffer, source.byteOffset, source.byteLength);
+	}
 };
 
 export const textEncoder = new TextEncoder();
@@ -356,20 +360,6 @@ export const clamp = (value: number, min: number, max: number) => {
 
 export const UNDETERMINED_LANGUAGE = 'und';
 
-/** @public */
-export const setVideoFrameTiming = (frame: VideoFrame, timing: {
-	timestamp?: number;
-	duration?: number;
-}) => {
-	const clone = new VideoFrame(frame, {
-		timestamp: timing.timestamp !== undefined ? 1e6 * timing.timestamp : undefined,
-		duration: timing.duration !== undefined ? 1e6 * timing.duration : undefined,
-	});
-	frame.close();
-
-	return clone;
-};
-
 export const roundToPrecision = (value: number, digits: number) => {
 	const factor = 10 ** digits;
 	return Math.round(value * factor) / factor;
@@ -393,5 +383,8 @@ export const isIso639Dash2LanguageCode = (x: string) => {
 	return ISO_639_2_REGEX.test(x);
 };
 
-// Since the result will be floored, add a bit of eps to compensate for floating point errors
+// Since the result will be truncated, add a bit of eps to compensate for floating point errors
 export const SECOND_TO_MICROSECOND_FACTOR = 1e6 * (1 + Number.EPSILON);
+
+/** @public */
+export type SetRequired<T, K extends keyof T> = T & Required<Pick<T, K>>;

src/packet.ts

@@ -40,11 +40,11 @@ export class EncodedPacket {
 	}
 
 	get microsecondTimestamp() {
-		return Math.floor(SECOND_TO_MICROSECOND_FACTOR * this.timestamp);
+		return Math.trunc(SECOND_TO_MICROSECOND_FACTOR * this.timestamp);
 	}
 
 	get microsecondDuration() {
-		return Math.floor(SECOND_TO_MICROSECOND_FACTOR * this.duration);
+		return Math.trunc(SECOND_TO_MICROSECOND_FACTOR * this.duration);
 	}
 
 	toEncodedVideoChunk() {