arcodange/mediabunny
1
0
Fork 0
mirror of https://github.com/arcodange-org/mediabunny.git synced 2026-10-01 04:43:51 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add timestampBase option for live MediaStreamTrack sources

Browse Source
This commit is contained in:
Vanilagy
2026-04-27 19:38:45 +02:00
parent c89e7429f7
commit 9a1b8b2602
8 changed files with 124 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dev/live.html
+4 -4
View File
@@ -19,7 +19,7 @@
button.addEventListener('click', async () => {
const stream = await navigator.mediaDevices.getDisplayMedia({ video: true, audio: false });
const videoTrack = stream.getVideoTracks()[0];
const audioTrack = stream.getAudioTracks()[0];
const audioTrack = null && stream.getAudioTracks()[0];
const output = new Mediabunny.Output({
target: new Mediabunny.BufferTarget(),
@@ -32,7 +32,7 @@
codec: 'vp9',
bitrate: Mediabunny.QUALITY_MEDIUM,
sizeChangeBehavior: 'passThrough',
});
}, { timestampBase: 'unix' });
videoSource.errorPromise.catch((d) => console.log("Hello?????", d));
@@ -67,9 +67,9 @@
await output.finalize();
console.log(output.target.buffer);
download(new Blob([output.target.buffer]), 'livetest' + output.format.fileExtension);
//download(new Blob([output.target.buffer]), 'livetest' + output.format.fileExtension);
videoTrack?.stop();
audioTrack?.stop();
});
</script>
</script>
docs/blog/mediabunny-now-supports-hls.md
+6 -6
View File
@@ -194,22 +194,22 @@ const output = new Output({
const videoSourceFull = new MediaStreamVideoTrackSource(displayTrack, {
codec: 'avc',
bitrate: QUALITY_HIGH,
});
}, { timestampBase: 'unix' });
// 480p video
const videoSource480p = new MediaStreamVideoTrackSource(displayTrack, {
codec: 'avc',
bitrate: QUALITY_MEDIUM,
transform: { height: 480 },
});
}, { timestampBase: 'unix' });
// Audio
const audioSource = new MediaStreamAudioTrackSource(micTrack, {
codec: 'aac',
bitrate: QUALITY_HIGH,
});
}, { timestampBase: 'unix' });
output.addVideoTrack(videoSourceFull);
output.addVideoTrack(videoSource480p);
output.addAudioTrack(audioSource);
output.addVideoTrack(videoSourceFull, { isRelativeToUnixEpoch: true });
output.addVideoTrack(videoSource480p, { isRelativeToUnixEpoch: true });
output.addAudioTrack(audioSource, { isRelativeToUnixEpoch: true });
await output.start();
scripts/generate-api-docs.ts
+3 -3
View File
@@ -338,11 +338,11 @@ const generateDocs = (entryFiles: string[], apiConfigFile: string, dry = false)
if (linkText) {
// If custom link text is provided, always use it.
displayText = linkText.trim();
} else if (memberName) {
// If it's a member link, default the text to just the member name.
} else if (memberName && typeName === currentTypeName) {
// Member link on the current type: just the member name.
displayText = `\`${memberName}\``;
} else {
// Otherwise, it's a type link, so use the full type name.
// Type link, or member link on another type: use the full target.
displayText = `\`${cleanTarget}\``;
}
src/index.ts
+2 -1
View File
@@ -76,8 +76,9 @@ export {
EncodedAudioPacketSource,
EncodedVideoPacketSource,
MediaStreamAudioTrackSource,
MediaStreamVideoTrackSourceOptions,
MediaStreamAudioTrackSourceOptions,
MediaStreamVideoTrackSource,
MediaStreamVideoTrackSourceOptions,
TextSubtitleSource,
VideoSampleSource,
} from './media-source';
src/media-source.ts
+102 -16
View File
@@ -1332,7 +1332,7 @@ export class CanvasSource extends VideoSource {
}
/**
* Options for MediaStreamVideoTrackSource.
* Options for {@link MediaStreamVideoTrackSource}.
* @group Media sources
* @public
*/
@@ -1344,6 +1344,21 @@ export type MediaStreamVideoTrackSourceOptions = {
* lead to wildly irregular FPS.
*/
frameRate?: number | null;
/**
* Controls the basis (zero point) for video frame timestamps.
*
* When set to `'synced-zero'`, timestamps will be relative to the first chunk of media from a `MediaStreamTrack`
* added to the {@link Output}.
*
* When set to `'zero'`, timestamps will be relative to the first video frame emitted by this source.
*
* When set to `'unix'`, timestamps will be relative to the Unix epoch, so clearly associated with a distinct point
* in time. Here, pausing via {@link MediaStreamVideoTrackSource.pause} will also create gaps in timestamps. Be sure
* to pair this mode with {@link BaseTrackMetadata.isRelativeToUnixEpoch}.
*
* Defaults to `'synced-zero'`.
*/
timestampBase?: 'synced-zero' | 'zero' | 'unix';
};
/**
@@ -1412,6 +1427,16 @@ export class MediaStreamVideoTrackSource extends VideoSource {
if (options.frameRate != null && (typeof options.frameRate !== 'number' || options.frameRate <= 0)) {
throw new TypeError('options.frameRate, when provided, must be either a positive number or null.');
}
if (
options.timestampBase !== undefined
&& options.timestampBase !== 'synced-zero'
&& options.timestampBase !== 'zero'
&& options.timestampBase !== 'unix'
) {
throw new TypeError(
'options.timestampBase, when provided, must be one of \'synced-zero\', \'zero\', or \'unix\'.',
);
}
encodingConfig = {
...encodingConfig,
@@ -1508,7 +1533,7 @@ export class MediaStreamVideoTrackSource extends VideoSource {
if (this._paused) {
const frameSeen = firstVideoFrameTimestamp !== null;
if (frameSeen) {
if (lastSampleTimestamp !== null) {
if (lastSampleTimestamp !== null && this._options.timestampBase !== 'unix') {
// In addition to dropping this frame, let's also keep track of the time we have lost due to the
// pause. Doing it like this instead of simply keeping track of the paused time is better since
// it retains the frame rate of the underlying source.
@@ -1527,14 +1552,23 @@ export class MediaStreamVideoTrackSource extends VideoSource {
if (firstVideoFrameTimestamp === null) {
firstVideoFrameTimestamp = currentTimestamp;
const muxer = this._connectedTrack!.output._muxer;
if (muxer.firstMediaStreamTimestamp === null) {
muxer.firstMediaStreamTimestamp = now / 1000;
this._timestampOffset = -firstVideoFrameTimestamp;
let target: number;
const timestampBase = this._options.timestampBase ?? 'synced-zero';
if (timestampBase === 'unix') {
target = Date.now() / 1000;
} else if (timestampBase === 'zero') {
target = 0;
} else {
this._timestampOffset = (now / 1000 - muxer.firstMediaStreamTimestamp)
- firstVideoFrameTimestamp;
const output = this._connectedTrack!.output;
if (output._firstMediaStreamTimestamp === null) {
output._firstMediaStreamTimestamp = now / 1000;
target = 0;
} else {
target = now / 1000 - output._firstMediaStreamTimestamp;
}
}
this._timestampOffset = target - firstVideoFrameTimestamp;
}
lastSampleTimestamp = currentTimestamp;
@@ -2405,6 +2439,29 @@ export class AudioBufferSource extends AudioSource {
}
}
/**
* Options for {@link MediaStreamAudioTrackSource}.
* @group Media sources
* @public
*/
export type MediaStreamAudioTrackSourceOptions = {
/**
* Controls the basis (zero point) for audio sample timestamps.
*
* When set to `'synced-zero'`, timestamps will be relative to the first chunk of media from a `MediaStreamTrack`
* added to the {@link Output}.
*
* When set to `'zero'`, timestamps will be relative to the first audio sample emitted by this source.
*
* When set to `'unix'`, timestamps will be relative to the Unix epoch, so clearly associated with a distinct point
* in time. Here, pausing via {@link MediaStreamAudioTrackSource.pause} will also create gaps in timestamps. Be sure
* to pair this mode with {@link BaseTrackMetadata.isRelativeToUnixEpoch}.
*
* Defaults to `'synced-zero'`.
*/
timestampBase?: 'synced-zero' | 'zero' | 'unix';
};
/**
* Audio source that encodes the data of a
* [`MediaStreamAudioTrack`](https://developer.mozilla.org/en-US/docs/Web/API/MediaStreamTrack) and pipes it into the
@@ -2415,6 +2472,8 @@ export class AudioBufferSource extends AudioSource {
* @public
*/
export class MediaStreamAudioTrackSource extends AudioSource {
/** @internal */
private _options: MediaStreamAudioTrackSourceOptions;
/** @internal */
private _encoder: AudioEncoderWrapper;
/** @internal */
@@ -2448,13 +2507,31 @@ export class MediaStreamAudioTrackSource extends AudioSource {
* Creates a new {@link MediaStreamAudioTrackSource} from a `MediaStreamAudioTrack`, which will pull audio samples
* from the stream in real time and encode them according to {@link AudioEncodingConfig}.
*/
constructor(track: MediaStreamAudioTrack, encodingConfig: AudioEncodingConfig) {
constructor(
track: MediaStreamAudioTrack,
encodingConfig: AudioEncodingConfig,
options: MediaStreamAudioTrackSourceOptions = {},
) {
if (!(track instanceof MediaStreamTrack) || track.kind !== 'audio') {
throw new TypeError('track must be an audio MediaStreamTrack.');
}
validateAudioEncodingConfig(encodingConfig);
if (typeof options !== 'object' || !options) {
throw new TypeError('options must be an object.');
}
if (
options.timestampBase !== undefined
&& options.timestampBase !== 'synced-zero'
&& options.timestampBase !== 'zero'
&& options.timestampBase !== 'unix'
) {
throw new TypeError(
'options.timestampBase, when provided, must be one of \'synced-zero\', \'zero\', or \'unix\'.',
);
}
super(encodingConfig.codec);
this._options = options;
this._encoder = new AudioEncoderWrapper(this, encodingConfig);
this._track = track;
}
@@ -2486,7 +2563,7 @@ export class MediaStreamAudioTrackSource extends AudioSource {
if (this._paused) {
const dataSeen = firstAudioDataTimestamp !== null;
if (dataSeen) {
if (lastSampleTimestamp !== null) {
if (lastSampleTimestamp !== null && this._options.timestampBase !== 'unix') {
// In addition to dropping this sample, let's also keep track of the time we have lost due to
// the pause. Doing it like this instead of simply keeping track of the paused time is better
// since it retains the sample rate of the underlying source.
@@ -2505,14 +2582,23 @@ export class MediaStreamAudioTrackSource extends AudioSource {
if (firstAudioDataTimestamp === null) {
firstAudioDataTimestamp = audioSample.timestamp;
const muxer = this._connectedTrack!.output._muxer;
if (muxer.firstMediaStreamTimestamp === null) {
muxer.firstMediaStreamTimestamp = performance.now() / 1000;
this._timestampOffset = -firstAudioDataTimestamp;
let target: number;
const timestampBase = this._options.timestampBase ?? 'synced-zero';
if (timestampBase === 'unix') {
target = Date.now() / 1000;
} else if (timestampBase === 'zero') {
target = 0;
} else {
this._timestampOffset = (performance.now() / 1000 - muxer.firstMediaStreamTimestamp)
- firstAudioDataTimestamp;
const output = this._connectedTrack!.output;
if (output._firstMediaStreamTimestamp === null) {
output._firstMediaStreamTimestamp = performance.now() / 1000;
target = 0;
} else {
target = performance.now() / 1000 - output._firstMediaStreamTimestamp;
}
}
this._timestampOffset = target - firstAudioDataTimestamp;
}
lastSampleTimestamp = currentTimestamp;
src/muxer.ts
-7
View File
@@ -15,13 +15,6 @@ export abstract class Muxer {
output: Output;
mutex = new AsyncMutex();
/**
* This field is used to synchronize multiple MediaStreamTracks. They use the same time coordinate system across
* tracks, and to ensure correct audio-video sync, we must use the same offset for all of them. The reason an offset
* is needed at all is because the timestamps typically don't start at zero.
*/
firstMediaStreamTimestamp: number | null = null;
constructor(output: Output) {
this.output = output;
}
src/output.ts
+7
View File
@@ -397,6 +397,13 @@ export class Output<
_rootTarget: T | null = null;
/** @internal */
_rootTargetPromise: Promise<T> | null = null;
/**
* This field is used to synchronize multiple MediaStreamTracks. They use the same time coordinate system across
* tracks, and to ensure correct audio-video sync, we must use the same offset for all of them. The reason an offset
* is needed at all is because the timestamps typically don't start at zero.
* @internal
*/
_firstMediaStreamTimestamp: number | null = null;
/**
* The target to which the root file will be written. Throws when using {@link PathedTarget} with an async callback;
todo.txt
-1
View File
@@ -1,3 +1,2 @@
Thoughts:
So, a certain "lookahead" logic is definitely needed. The question is if this is a per-demuxer thing or a general thing instead. The demuxer could get in a "packet query" that specifies things like "I am interested in the next 20 seconds guaranteed", allowing the demuxer to pre-fetch more intelligently. The alternative would be some sort of demuxer-agnostic approach where there is a magical "packet requester" that has to be segment-aware. I'm actually not sure if that's good.