Add timestampBase option for live MediaStreamTrack sources

dev/live.html

@@ -19,7 +19,7 @@
     button.addEventListener('click', async () => {
         const stream = await navigator.mediaDevices.getDisplayMedia({ video: true, audio: false });
         const videoTrack = stream.getVideoTracks()[0];
-        const audioTrack = stream.getAudioTracks()[0];
+        const audioTrack = null && stream.getAudioTracks()[0];
 
         const output = new Mediabunny.Output({
             target: new Mediabunny.BufferTarget(),
@@ -32,7 +32,7 @@
                 codec: 'vp9',
                 bitrate: Mediabunny.QUALITY_MEDIUM,
 				sizeChangeBehavior: 'passThrough',
-            });
+            }, { timestampBase: 'unix' });
 
             videoSource.errorPromise.catch((d) => console.log("Hello?????", d));
 
@@ -67,9 +67,9 @@
         await output.finalize();
 
         console.log(output.target.buffer);
-        download(new Blob([output.target.buffer]), 'livetest' + output.format.fileExtension);
+        //download(new Blob([output.target.buffer]), 'livetest' + output.format.fileExtension);
 
         videoTrack?.stop();
         audioTrack?.stop();
     });
-</script>
+</script>

docs/blog/mediabunny-now-supports-hls.md

@@ -194,22 +194,22 @@ const output = new Output({
 const videoSourceFull = new MediaStreamVideoTrackSource(displayTrack, {
 	codec: 'avc',
 	bitrate: QUALITY_HIGH,
-});
+}, { timestampBase: 'unix' });
 // 480p video
 const videoSource480p = new MediaStreamVideoTrackSource(displayTrack, {
 	codec: 'avc',
 	bitrate: QUALITY_MEDIUM,
 	transform: { height: 480 },
-});
+}, { timestampBase: 'unix' });
 // Audio
 const audioSource = new MediaStreamAudioTrackSource(micTrack, {
 	codec: 'aac',
 	bitrate: QUALITY_HIGH,
-});
+}, { timestampBase: 'unix' });
 
-output.addVideoTrack(videoSourceFull);
+output.addVideoTrack(videoSourceFull, { isRelativeToUnixEpoch: true });
-output.addVideoTrack(videoSource480p);
+output.addVideoTrack(videoSource480p, { isRelativeToUnixEpoch: true });
-output.addAudioTrack(audioSource);
+output.addAudioTrack(audioSource, { isRelativeToUnixEpoch: true });
 
 await output.start();
 

scripts/generate-api-docs.ts

@@ -338,11 +338,11 @@ const generateDocs = (entryFiles: string[], apiConfigFile: string, dry = false)
 			if (linkText) {
 				// If custom link text is provided, always use it.
 				displayText = linkText.trim();
-			} else if (memberName) {
+			} else if (memberName && typeName === currentTypeName) {
-				// If it's a member link, default the text to just the member name.
+				// Member link on the current type: just the member name.
 				displayText = `\`${memberName}\``;
 			} else {
-				// Otherwise, it's a type link, so use the full type name.
+				// Type link, or member link on another type: use the full target.
 				displayText = `\`${cleanTarget}\``;
 			}
 

src/index.ts

@@ -76,8 +76,9 @@ export {
 	EncodedAudioPacketSource,
 	EncodedVideoPacketSource,
 	MediaStreamAudioTrackSource,
-	MediaStreamVideoTrackSourceOptions,
+	MediaStreamAudioTrackSourceOptions,
 	MediaStreamVideoTrackSource,
+	MediaStreamVideoTrackSourceOptions,
 	TextSubtitleSource,
 	VideoSampleSource,
 } from './media-source';

src/media-source.ts

@@ -1332,7 +1332,7 @@ export class CanvasSource extends VideoSource {
 }
 
 /**
- * Options for MediaStreamVideoTrackSource.
+ * Options for {@link MediaStreamVideoTrackSource}.
  * @group Media sources
  * @public
  */
@@ -1344,6 +1344,21 @@ export type MediaStreamVideoTrackSourceOptions = {
 	 * lead to wildly irregular FPS.
 	 */
 	frameRate?: number | null;
+	/**
+	 * Controls the basis (zero point) for video frame timestamps.
+	 *
+	 * When set to `'synced-zero'`, timestamps will be relative to the first chunk of media from a `MediaStreamTrack`
+	 * added to the {@link Output}.
+	 *
+	 * When set to `'zero'`, timestamps will be relative to the first video frame emitted by this source.
+	 *
+	 * When set to `'unix'`, timestamps will be relative to the Unix epoch, so clearly associated with a distinct point
+	 * in time. Here, pausing via {@link MediaStreamVideoTrackSource.pause} will also create gaps in timestamps. Be sure
+	 * to pair this mode with {@link BaseTrackMetadata.isRelativeToUnixEpoch}.
+	 *
+	 * Defaults to `'synced-zero'`.
+	 */
+	timestampBase?: 'synced-zero' | 'zero' | 'unix';
 };
 
 /**
@@ -1412,6 +1427,16 @@ export class MediaStreamVideoTrackSource extends VideoSource {
 		if (options.frameRate != null && (typeof options.frameRate !== 'number' || options.frameRate <= 0)) {
 			throw new TypeError('options.frameRate, when provided, must be either a positive number or null.');
 		}
+		if (
+			options.timestampBase !== undefined
+			&& options.timestampBase !== 'synced-zero'
+			&& options.timestampBase !== 'zero'
+			&& options.timestampBase !== 'unix'
+		) {
+			throw new TypeError(
+				'options.timestampBase, when provided, must be one of \'synced-zero\', \'zero\', or \'unix\'.',
+			);
+		}
 
 		encodingConfig = {
 			...encodingConfig,
@@ -1508,7 +1533,7 @@ export class MediaStreamVideoTrackSource extends VideoSource {
 			if (this._paused) {
 				const frameSeen = firstVideoFrameTimestamp !== null;
 				if (frameSeen) {
-					if (lastSampleTimestamp !== null) {
+					if (lastSampleTimestamp !== null && this._options.timestampBase !== 'unix') {
 						// In addition to dropping this frame, let's also keep track of the time we have lost due to the
 						// pause. Doing it like this instead of simply keeping track of the paused time is better since
 						// it retains the frame rate of the underlying source.
@@ -1527,14 +1552,23 @@ export class MediaStreamVideoTrackSource extends VideoSource {
 			if (firstVideoFrameTimestamp === null) {
 				firstVideoFrameTimestamp = currentTimestamp;
 
-				const muxer = this._connectedTrack!.output._muxer;
+				let target: number;
-				if (muxer.firstMediaStreamTimestamp === null) {
+				const timestampBase = this._options.timestampBase ?? 'synced-zero';
-					muxer.firstMediaStreamTimestamp = now / 1000;
+				if (timestampBase === 'unix') {
-					this._timestampOffset = -firstVideoFrameTimestamp;
+					target = Date.now() / 1000;
+				} else if (timestampBase === 'zero') {
+					target = 0;
 				} else {
-					this._timestampOffset = (now / 1000 - muxer.firstMediaStreamTimestamp)
+					const output = this._connectedTrack!.output;
-						- firstVideoFrameTimestamp;
+					if (output._firstMediaStreamTimestamp === null) {
+						output._firstMediaStreamTimestamp = now / 1000;
+						target = 0;
+					} else {
+						target = now / 1000 - output._firstMediaStreamTimestamp;
+					}
 				}
+
+				this._timestampOffset = target - firstVideoFrameTimestamp;
 			}
 
 			lastSampleTimestamp = currentTimestamp;
@@ -2405,6 +2439,29 @@ export class AudioBufferSource extends AudioSource {
 	}
 }
 
+/**
+ * Options for {@link MediaStreamAudioTrackSource}.
+ * @group Media sources
+ * @public
+ */
+export type MediaStreamAudioTrackSourceOptions = {
+	/**
+	 * Controls the basis (zero point) for audio sample timestamps.
+	 *
+	 * When set to `'synced-zero'`, timestamps will be relative to the first chunk of media from a `MediaStreamTrack`
+	 * added to the {@link Output}.
+	 *
+	 * When set to `'zero'`, timestamps will be relative to the first audio sample emitted by this source.
+	 *
+	 * When set to `'unix'`, timestamps will be relative to the Unix epoch, so clearly associated with a distinct point
+	 * in time. Here, pausing via {@link MediaStreamAudioTrackSource.pause} will also create gaps in timestamps. Be sure
+	 * to pair this mode with {@link BaseTrackMetadata.isRelativeToUnixEpoch}.
+	 *
+	 * Defaults to `'synced-zero'`.
+	 */
+	timestampBase?: 'synced-zero' | 'zero' | 'unix';
+};
+
 /**
  * Audio source that encodes the data of a
  * [`MediaStreamAudioTrack`](https://developer.mozilla.org/en-US/docs/Web/API/MediaStreamTrack) and pipes it into the
@@ -2415,6 +2472,8 @@ export class AudioBufferSource extends AudioSource {
  * @public
  */
 export class MediaStreamAudioTrackSource extends AudioSource {
+	/** @internal */
+	private _options: MediaStreamAudioTrackSourceOptions;
 	/** @internal */
 	private _encoder: AudioEncoderWrapper;
 	/** @internal */
@@ -2448,13 +2507,31 @@ export class MediaStreamAudioTrackSource extends AudioSource {
 	 * Creates a new {@link MediaStreamAudioTrackSource} from a `MediaStreamAudioTrack`, which will pull audio samples
 	 * from the stream in real time and encode them according to {@link AudioEncodingConfig}.
 	 */
-	constructor(track: MediaStreamAudioTrack, encodingConfig: AudioEncodingConfig) {
+	constructor(
+		track: MediaStreamAudioTrack,
+		encodingConfig: AudioEncodingConfig,
+		options: MediaStreamAudioTrackSourceOptions = {},
+	) {
 		if (!(track instanceof MediaStreamTrack) || track.kind !== 'audio') {
 			throw new TypeError('track must be an audio MediaStreamTrack.');
 		}
 		validateAudioEncodingConfig(encodingConfig);
+		if (typeof options !== 'object' || !options) {
+			throw new TypeError('options must be an object.');
+		}
+		if (
+			options.timestampBase !== undefined
+			&& options.timestampBase !== 'synced-zero'
+			&& options.timestampBase !== 'zero'
+			&& options.timestampBase !== 'unix'
+		) {
+			throw new TypeError(
+				'options.timestampBase, when provided, must be one of \'synced-zero\', \'zero\', or \'unix\'.',
+			);
+		}
 
 		super(encodingConfig.codec);
+		this._options = options;
 		this._encoder = new AudioEncoderWrapper(this, encodingConfig);
 		this._track = track;
 	}
@@ -2486,7 +2563,7 @@ export class MediaStreamAudioTrackSource extends AudioSource {
 			if (this._paused) {
 				const dataSeen = firstAudioDataTimestamp !== null;
 				if (dataSeen) {
-					if (lastSampleTimestamp !== null) {
+					if (lastSampleTimestamp !== null && this._options.timestampBase !== 'unix') {
 						// In addition to dropping this sample, let's also keep track of the time we have lost due to
 						// the pause. Doing it like this instead of simply keeping track of the paused time is better
 						// since it retains the sample rate of the underlying source.
@@ -2505,14 +2582,23 @@ export class MediaStreamAudioTrackSource extends AudioSource {
 			if (firstAudioDataTimestamp === null) {
 				firstAudioDataTimestamp = audioSample.timestamp;
 
-				const muxer = this._connectedTrack!.output._muxer;
+				let target: number;
-				if (muxer.firstMediaStreamTimestamp === null) {
+				const timestampBase = this._options.timestampBase ?? 'synced-zero';
-					muxer.firstMediaStreamTimestamp = performance.now() / 1000;
+				if (timestampBase === 'unix') {
-					this._timestampOffset = -firstAudioDataTimestamp;
+					target = Date.now() / 1000;
+				} else if (timestampBase === 'zero') {
+					target = 0;
 				} else {
-					this._timestampOffset = (performance.now() / 1000 - muxer.firstMediaStreamTimestamp)
+					const output = this._connectedTrack!.output;
-						- firstAudioDataTimestamp;
+					if (output._firstMediaStreamTimestamp === null) {
+						output._firstMediaStreamTimestamp = performance.now() / 1000;
+						target = 0;
+					} else {
+						target = performance.now() / 1000 - output._firstMediaStreamTimestamp;
+					}
 				}
+
+				this._timestampOffset = target - firstAudioDataTimestamp;
 			}
 
 			lastSampleTimestamp = currentTimestamp;

src/muxer.ts

@@ -15,13 +15,6 @@ export abstract class Muxer {
 	output: Output;
 	mutex = new AsyncMutex();
 
-	/**
-	 * This field is used to synchronize multiple MediaStreamTracks. They use the same time coordinate system across
-	 * tracks, and to ensure correct audio-video sync, we must use the same offset for all of them. The reason an offset
-	 * is needed at all is because the timestamps typically don't start at zero.
-	 */
-	firstMediaStreamTimestamp: number | null = null;
-
 	constructor(output: Output) {
 		this.output = output;
 	}

src/output.ts

@@ -397,6 +397,13 @@ export class Output<
 	_rootTarget: T | null = null;
 	/** @internal */
 	_rootTargetPromise: Promise<T> | null = null;
+	/**
+	 * This field is used to synchronize multiple MediaStreamTracks. They use the same time coordinate system across
+	 * tracks, and to ensure correct audio-video sync, we must use the same offset for all of them. The reason an offset
+	 * is needed at all is because the timestamps typically don't start at zero.
+	 * @internal
+	 */
+	_firstMediaStreamTimestamp: number | null = null;
 
 	/**
 	 * The target to which the root file will be written. Throws when using {@link PathedTarget} with an async callback;

todo.txt

@@ -1,3 +1,2 @@
 Thoughts:
 So, a certain "lookahead" logic is definitely needed. The question is if this is a per-demuxer thing or a general thing instead. The demuxer could get in a "packet query" that specifies things like "I am interested in the next 20 seconds guaranteed", allowing the demuxer to pre-fetch more intelligently. The alternative would be some sort of demuxer-agnostic approach where there is a magical "packet requester" that has to be segment-aware. I'm actually not sure if that's good.
-