mirror of
https://github.com/arcodange-org/mediabunny.git
synced 2026-10-09 00:33:46 +02:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Vanilagy
|
0bfba89161 | ||
|
Vanilagy
|
93e9c256ab | ||
|
Vanilagy
|
c4bce00c4f | ||
|
Vanilagy
|
05ae21e5ea | ||
|
Vanilagy
|
31594f1977 | ||
|
Vanilagy
|
397c464846 | ||
|
Vanilagy
|
2087658eea | ||
|
|
6a11a1302c | ||
|
Vanilagy
|
472dc113e3 | ||
|
Vanilagy
|
193dcf0f14 | ||
|
Vanilagy
|
f79834c3c1 | ||
|
Vanilagy
|
66185fed15 | ||
|
Vanilagy
|
4172f81aa0 |
@@ -81,6 +81,11 @@ jobs:
|
||||
packages/aac-encoder/dist/bundles/mediabunny-aac-encoder.mjs
|
||||
packages/aac-encoder/dist/bundles/mediabunny-aac-encoder.min.mjs
|
||||
packages/aac-encoder/dist/mediabunny-aac-encoder.d.ts
|
||||
packages/flac-encoder/dist/bundles/mediabunny-flac-encoder.js
|
||||
packages/flac-encoder/dist/bundles/mediabunny-flac-encoder.min.js
|
||||
packages/flac-encoder/dist/bundles/mediabunny-flac-encoder.mjs
|
||||
packages/flac-encoder/dist/bundles/mediabunny-flac-encoder.min.mjs
|
||||
packages/flac-encoder/dist/mediabunny-flac-encoder.d.ts
|
||||
|
||||
- name: Publish Mediabunny to npm
|
||||
run: npm publish --access public
|
||||
|
||||
+1
-1
@@ -26,7 +26,7 @@
|
||||
chunked: true,
|
||||
chunkSize: 2**20
|
||||
});
|
||||
const outputFormat = new Mediabunny.Mp4OutputFormat({});
|
||||
const outputFormat = new Mediabunny.Mp4OutputFormat({fastStart: 'fragmented'});
|
||||
|
||||
const button = document.createElement('button');
|
||||
button.textContent = 'Cancel';
|
||||
|
||||
+4
-3
@@ -18,10 +18,11 @@
|
||||
});
|
||||
|
||||
const videoTrack = await input.getPrimaryVideoTrack();
|
||||
const sink = new Mediabunny.VideoSampleSink(videoTrack);
|
||||
const sink = new Mediabunny.EncodedPacketSink(videoTrack);
|
||||
|
||||
const sample = await sink.getSample(await videoTrack.getFirstTimestamp());
|
||||
console.log(sample);
|
||||
for await (const packet of sink.packets()) {
|
||||
console.log(packet.timestamp);
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
|
||||
+6
-4
@@ -17,7 +17,7 @@
|
||||
|
||||
const button = document.querySelector('button');
|
||||
button.addEventListener('click', async () => {
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ video: true, audio: true });
|
||||
const stream = await navigator.mediaDevices.getDisplayMedia({ video: true, audio: false });
|
||||
const videoTrack = stream.getVideoTracks()[0];
|
||||
const audioTrack = stream.getAudioTracks()[0];
|
||||
|
||||
@@ -29,8 +29,9 @@
|
||||
let audioSource = null;
|
||||
if (videoTrack) {
|
||||
videoSource = new Mediabunny.MediaStreamVideoTrackSource(videoTrack, {
|
||||
codec: 'avc',
|
||||
bitrate: Mediabunny.QUALITY_MEDIUM
|
||||
codec: 'vp9',
|
||||
bitrate: Mediabunny.QUALITY_MEDIUM,
|
||||
sizeChangeBehavior: 'passThrough',
|
||||
});
|
||||
|
||||
videoSource.errorPromise.catch((d) => console.log("Hello?????", d));
|
||||
@@ -39,7 +40,7 @@
|
||||
}
|
||||
if (audioTrack) {
|
||||
audioSource = new Mediabunny.MediaStreamAudioTrackSource(audioTrack, {
|
||||
codec: 'mp3',
|
||||
codec: 'opus',
|
||||
bitrate: Mediabunny.QUALITY_MEDIUM
|
||||
});
|
||||
|
||||
@@ -51,6 +52,7 @@
|
||||
await output.start();
|
||||
|
||||
setTimeout(() => {
|
||||
return;
|
||||
videoSource?.pause();
|
||||
audioSource?.pause();
|
||||
|
||||
|
||||
@@ -17,6 +17,7 @@ export default withMermaid({
|
||||
sitemap: {
|
||||
hostname: 'https://mediabunny.dev',
|
||||
},
|
||||
lastUpdated: true,
|
||||
head: [
|
||||
['link', { rel: 'icon', type: 'image/png', href: '/mediabunny-logo.png' }],
|
||||
['link', { rel: 'icon', type: 'image/svg+xml', href: '/mediabunny-logo.svg' }],
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# @mediabunny/flac-encoder
|
||||
|
||||
No browser currently supports FLAC encoding in their WebCodecs implementations. This extension package provides a reliable FLAC encoder for use with Mediabunny. It is implemented using Mediabunny's [custom coder API](../supported-formats-and-codecs#custom-coders) and uses a fast, size-optimized WASM build of [libFLAC](https://github.com/xiph/flac) (the reference FLAC encoder) under the hood.
|
||||
No browser currently supports FLAC encoding in their WebCodecs implementations. This extension package provides a reliable FLAC encoder for use with Mediabunny. It is implemented using Mediabunny's [custom coder API](../supported-formats-and-codecs#custom-coders) and uses a fast, size-optimized WASM build of [libFLAC](https://github.com/xiph/flac) under the hood.
|
||||
|
||||
<a class="!no-underline inline-flex items-center gap-1.5" :no-icon="true" href="https://github.com/Vanilagy/mediabunny/blob/main/packages/flac-encoder/README.md">
|
||||
GitHub page
|
||||
|
||||
@@ -97,9 +97,13 @@ await sink.getKeyPacket(5); // => EncodedPacket | null
|
||||
|
||||
When retrieving a packet using a timestamp, the last packet (in [presentation order](#decode-vs-presentation-order)) with a timestamp less than or equal to the search timestamp will be returned. The methods return `null` if there exists no such packet.
|
||||
|
||||
There is a special method for retrieving the first packet (in [decode order](#decode-vs-presentation-order)):
|
||||
There are special methods for retrieving the first packet (in [decode order](#decode-vs-presentation-order)):
|
||||
```ts
|
||||
await sink.getFirstPacket(); // => EncodedPacket | null
|
||||
|
||||
// The first packet is typically a key frame, but this is not required.
|
||||
// This method returns the first key frame:
|
||||
await sink.getFirstKeyPacket(); // => EncodedPacket | null
|
||||
```
|
||||
The last packet (in [presentation order](#decode-vs-presentation-order)) can be retrieved like so:
|
||||
```ts
|
||||
|
||||
@@ -37,13 +37,13 @@ Mediabunny ships with built-in decoders and encoders for all audio PCM codecs, m
|
||||
|
||||
### Audio codecs
|
||||
|
||||
- `'aac'` - Advanced Audio Coding (AAC) [^1]
|
||||
- `'aac'` - Advanced Audio Coding (AAC) [^aac]
|
||||
- `'opus'` - Opus
|
||||
- `'mp3'` - MP3 [^2]
|
||||
- `'mp3'` - MP3 [^mp3]
|
||||
- `'vorbis'` - Vorbis
|
||||
- `'flac'` - Free Lossless Audio Codec (FLAC)
|
||||
- `'ac3'` - Dolby Digital (AC-3) [^3]
|
||||
- `'eac3'` - Dolby Digital Plus (E-AC-3) [^3]
|
||||
- `'flac'` - Free Lossless Audio Codec (FLAC) [^flac]
|
||||
- `'ac3'` - Dolby Digital (AC-3) [^ac3]
|
||||
- `'eac3'` - Dolby Digital Plus (E-AC-3) [^ac3]
|
||||
- `'pcm-u8'` - 8-bit unsigned PCM
|
||||
- `'pcm-s8'` - 8-bit signed PCM
|
||||
- `'pcm-s16'` - 16-bit little-endian signed PCM
|
||||
@@ -67,7 +67,7 @@ Mediabunny ships with built-in decoders and encoders for all audio PCM codecs, m
|
||||
|
||||
Not all codecs can be used with all containers. The following table specifies the supported codec-container combinations:
|
||||
|
||||
| | .mp4 | .mov | .mkv | .webm[^4] | .ogg | .mp3 | .wav | .aac | .flac | .ts |
|
||||
| | .mp4 | .mov | .mkv | .webm[^webm] | .ogg | .mp3 | .wav | .aac | .flac | .ts |
|
||||
|:--------------:|:--------:|:-----:|:-----:|:---------:|:-----:|:-----:|:-----:|:-----:|:-----:|:-----:|
|
||||
| `'avc'` | ✓ | ✓ | ✓ | | | | | | | ✓ |
|
||||
| `'hevc'` | ✓ | ✓ | ✓ | | | | | | | ✓ |
|
||||
@@ -95,13 +95,14 @@ Not all codecs can be used with all containers. The following table specifies th
|
||||
| `'pcm-f64be'` | ✓ | ✓ | | | | | | | | |
|
||||
| `'ulaw'` | | ✓ | | | | | ✓ | | | |
|
||||
| `'alaw'` | | ✓ | | | | | ✓ | | | |
|
||||
| `'webvtt'`[^5] | (✓) | | (✓) | (✓) | | | | | | |
|
||||
| `'webvtt'`[^webvtt] | (✓) | | (✓) | (✓) | | | | | | |
|
||||
|
||||
[^1]: In some browsers, AAC encoding is not supported by WebCodecs. You can polyfill it with the [`@mediabunny/aac-encoder`](./extensions/aac-encoder) extension package, or provide your own [custom coder](#custom-coders).
|
||||
[^2]: MP3 encoding is not supported by WebCodecs. You can polyfill it with the [`@mediabunny/mp3-encoder`](./extensions/mp3-encoder) extension package, or provide your own [custom coder](#custom-coders).
|
||||
[^3]: AC-3 and E-AC-3 are not natively supported by WebCodecs. To encode or decode these codecs, you can use the [`@mediabunny/ac3`](./extensions/ac3) extension package, or provide your own [custom coder](#custom-coders).
|
||||
[^4]: WebM only supports a small subset of the codecs supported by Matroska. However, this library can technically read all codecs from a WebM that are supported by Matroska.
|
||||
[^5]: WebVTT can only be written, not read.
|
||||
[^aac]: In some browsers, AAC encoding is not supported by WebCodecs. You can polyfill it with the [`@mediabunny/aac-encoder`](./extensions/aac-encoder) extension package.
|
||||
[^mp3]: MP3 encoding is not supported by WebCodecs. You can polyfill it with the [`@mediabunny/mp3-encoder`](./extensions/mp3-encoder) extension package.
|
||||
[^flac]: FLAC encoding is not supported by WebCodecs. You can polyfill it with the [`@mediabunny/flac-encoder`](./extensions/flac-encoder) extension package.
|
||||
[^ac3]: AC-3 and E-AC-3 are not natively supported by WebCodecs. To encode or decode these codecs, you can use the [`@mediabunny/ac3`](./extensions/ac3) extension package.
|
||||
[^webm]: WebM only supports a small subset of the codecs supported by Matroska. However, this library can technically read all codecs from a WebM that are supported by Matroska.
|
||||
[^webvtt]: WebVTT can only be written, not read.
|
||||
|
||||
## Querying codec encodability
|
||||
|
||||
|
||||
@@ -112,7 +112,6 @@ const sponsors = {
|
||||
{ image: 'https://avatars.githubusercontent.com/u/82552321', name: 'Polotno', url: 'https://github.com/polotno-project' },
|
||||
{ image: 'https://avatars.githubusercontent.com/u/489051', name: 'Roman Rädle', url: 'https://github.com/raedle' },
|
||||
{ image: 'https://avatars.githubusercontent.com/u/197597', name: 'Christopher Chedeau', url: 'https://github.com/vjeux' },
|
||||
{ image: 'https://avatars.githubusercontent.com/u/84167135', name: 'Memenome', url: 'https://github.com/memenome' },
|
||||
{ image: 'https://avatars.githubusercontent.com/u/5913254', name: 'Brandon McConnell', url: 'https://github.com/brandonmcconnell' },
|
||||
{ image: 'https://avatars.githubusercontent.com/u/9549394', name: 'studnitz', url: 'https://github.com/studnitz' },
|
||||
{ image: 'https://avatars.githubusercontent.com/u/4714175', name: 'Phoomparin Mano', url: 'https://github.com/heypoom' },
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
User-agent: *
|
||||
Allow: /
|
||||
|
||||
Sitemap: https://mediabunny.dev/sitemap.xml
|
||||
Generated
+9
-9
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "mediabunny",
|
||||
"version": "1.37.0",
|
||||
"version": "1.39.1",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "mediabunny",
|
||||
"version": "1.37.0",
|
||||
"version": "1.39.1",
|
||||
"license": "MPL-2.0",
|
||||
"workspaces": [
|
||||
"packages/*"
|
||||
@@ -7751,9 +7751,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/mediabunny": {
|
||||
"version": "1.36.0",
|
||||
"resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.36.0.tgz",
|
||||
"integrity": "sha512-+RsDJ9O0JcrLenlgT0jzUA4+78UQhPDHnUax7QnfwOL4JMAitzp+6aV3WKUYaplb7wapvxpn/0k/S2UkAbAHWQ==",
|
||||
"version": "1.39.0",
|
||||
"resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.39.0.tgz",
|
||||
"integrity": "sha512-NDeht2Sf7kYfrUu7b/9AhVK4KdQhy6UeTF/EPNXW+PdpIYvSv36ZQ/SRBoFAJN2LRl6XrGb8JYY4MCnhsGQ2yg==",
|
||||
"license": "MPL-2.0",
|
||||
"peer": true,
|
||||
"workspaces": [
|
||||
@@ -12077,7 +12077,7 @@
|
||||
},
|
||||
"packages/aac-encoder": {
|
||||
"name": "@mediabunny/aac-encoder",
|
||||
"version": "1.37.0",
|
||||
"version": "1.39.1",
|
||||
"license": "MPL-2.0",
|
||||
"devDependencies": {
|
||||
"@types/emscripten": "^1.40.1"
|
||||
@@ -12092,7 +12092,7 @@
|
||||
},
|
||||
"packages/ac3": {
|
||||
"name": "@mediabunny/ac3",
|
||||
"version": "1.37.0",
|
||||
"version": "1.39.1",
|
||||
"license": "MPL-2.0",
|
||||
"devDependencies": {
|
||||
"@types/emscripten": "^1.40.1"
|
||||
@@ -12107,7 +12107,7 @@
|
||||
},
|
||||
"packages/flac-encoder": {
|
||||
"name": "@mediabunny/flac-encoder",
|
||||
"version": "1.37.0",
|
||||
"version": "1.39.1",
|
||||
"license": "MPL-2.0",
|
||||
"devDependencies": {
|
||||
"@types/emscripten": "^1.40.1"
|
||||
@@ -12122,7 +12122,7 @@
|
||||
},
|
||||
"packages/mp3-encoder": {
|
||||
"name": "@mediabunny/mp3-encoder",
|
||||
"version": "1.37.0",
|
||||
"version": "1.39.1",
|
||||
"license": "MPL-2.0",
|
||||
"devDependencies": {
|
||||
"@types/emscripten": "^1.40.1"
|
||||
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "mediabunny",
|
||||
"author": "Vanilagy",
|
||||
"version": "1.37.0",
|
||||
"version": "1.39.1",
|
||||
"description": "Pure TypeScript media toolkit for reading, writing, and converting media files, directly in the browser.",
|
||||
"type": "module",
|
||||
"workspaces": [
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "@mediabunny/aac-encoder",
|
||||
"author": "Vanilagy",
|
||||
"version": "1.37.0",
|
||||
"version": "1.39.1",
|
||||
"description": "AAC encoder extension for Mediabunny, based on FFmpeg.",
|
||||
"main": "./dist/bundles/mediabunny-aac-encoder.mjs",
|
||||
"module": "./dist/bundles/mediabunny-aac-encoder.mjs",
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "@mediabunny/ac3",
|
||||
"author": "Vanilagy",
|
||||
"version": "1.37.0",
|
||||
"version": "1.39.1",
|
||||
"description": "AC-3 and E-AC-3 (Dolby Digital) decoder and encoder extension for Mediabunny, based on FFmpeg.",
|
||||
"main": "./dist/bundles/mediabunny-ac3.mjs",
|
||||
"module": "./dist/bundles/mediabunny-ac3.mjs",
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "@mediabunny/flac-encoder",
|
||||
"author": "Vanilagy",
|
||||
"version": "1.37.0",
|
||||
"version": "1.39.1",
|
||||
"description": "FLAC encoder extension for Mediabunny, based on libFLAC.",
|
||||
"main": "./dist/bundles/mediabunny-flac-encoder.mjs",
|
||||
"module": "./dist/bundles/mediabunny-flac-encoder.mjs",
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "@mediabunny/mp3-encoder",
|
||||
"author": "Vanilagy",
|
||||
"version": "1.37.0",
|
||||
"version": "1.39.1",
|
||||
"description": "MP3 encoder extension for Mediabunny, based on LAME.",
|
||||
"main": "./dist/bundles/mediabunny-mp3-encoder.mjs",
|
||||
"module": "./dist/bundles/mediabunny-mp3-encoder.mjs",
|
||||
|
||||
@@ -515,6 +515,9 @@ export class Conversion {
|
||||
/**
|
||||
* Whether this conversion, as it has been configured, is valid and can be executed. If this field is `false`, check
|
||||
* the `discardedTracks` field for reasons.
|
||||
*
|
||||
* Note: a conversion having discarded tracks does not automatically mean it is invalid; if the remaining, utilized
|
||||
* tracks make for a valid output file, the conversion is still allowed.
|
||||
*/
|
||||
isValid = false;
|
||||
/** The list of tracks that are included in the output file. */
|
||||
|
||||
@@ -64,6 +64,7 @@ export {
|
||||
EncodedAudioPacketSource,
|
||||
EncodedVideoPacketSource,
|
||||
MediaStreamAudioTrackSource,
|
||||
MediaStreamVideoTrackSourceOptions,
|
||||
MediaStreamVideoTrackSource,
|
||||
TextSubtitleSource,
|
||||
VideoSampleSource,
|
||||
|
||||
@@ -1245,11 +1245,13 @@ export class IsobmffMuxer extends Muxer {
|
||||
override async onTrackClose(track: OutputTrack) {
|
||||
const release = await this.mutex.acquire();
|
||||
|
||||
if (track.type === 'subtitle' && track.source._codec === 'webvtt') {
|
||||
const trackData = this.trackDatas.find(x => x.track === track) as IsobmffSubtitleTrackData;
|
||||
if (trackData) {
|
||||
const trackData = this.trackDatas.find(x => x.track === track);
|
||||
if (trackData) {
|
||||
if (trackData.type === 'subtitle' && track.source._codec === 'webvtt') {
|
||||
await this.processWebVTTCues(trackData, Infinity);
|
||||
}
|
||||
|
||||
this.processTimestamps(trackData);
|
||||
}
|
||||
|
||||
if (this.allTracksAreKnown()) {
|
||||
@@ -1274,19 +1276,15 @@ export class IsobmffMuxer extends Muxer {
|
||||
if (trackData.type === 'subtitle' && trackData.track.source._codec === 'webvtt') {
|
||||
await this.processWebVTTCues(trackData, Infinity);
|
||||
}
|
||||
|
||||
this.processTimestamps(trackData);
|
||||
}
|
||||
|
||||
if (this.isFragmented) {
|
||||
await this.interleaveSamples(true);
|
||||
|
||||
for (const trackData of this.trackDatas) {
|
||||
this.processTimestamps(trackData);
|
||||
}
|
||||
|
||||
await this.finalizeFragment(false); // Don't flush the last fragment as we will flush it with the mfra box
|
||||
} else {
|
||||
for (const trackData of this.trackDatas) {
|
||||
this.processTimestamps(trackData);
|
||||
await this.finalizeCurrentChunk(trackData);
|
||||
}
|
||||
}
|
||||
|
||||
+18
-1
@@ -144,6 +144,23 @@ export class EncodedPacketSink {
|
||||
return maybeFixPacketType(this._track, this._track._backing.getFirstPacket(options), options);
|
||||
}
|
||||
|
||||
/** Retrieves the track's first key packet (in decode order), or null if it has no key packets. */
|
||||
async getFirstKeyPacket(options: PacketRetrievalOptions = {}) {
|
||||
validatePacketRetrievalOptions(options);
|
||||
|
||||
const firstPacket = await this.getFirstPacket(options);
|
||||
if (!firstPacket) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (firstPacket.type === 'key') {
|
||||
// Great
|
||||
return firstPacket;
|
||||
}
|
||||
|
||||
return this.getNextKeyPacket(firstPacket, options);
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieves the packet corresponding to the given timestamp, in seconds. More specifically, returns the last packet
|
||||
* (in presentation order) with a start timestamp less than or equal to the given timestamp. This method can be
|
||||
@@ -476,7 +493,7 @@ export abstract class BaseMediaSampleSink<
|
||||
|
||||
const packetSink = this._createPacketSink();
|
||||
const keyPacket = await packetSink.getKeyPacket(startTimestamp, { verifyKeyPackets: true })
|
||||
?? await packetSink.getFirstPacket();
|
||||
?? await packetSink.getFirstKeyPacket({ verifyKeyPackets: true });
|
||||
|
||||
let currentPacket: EncodedPacket | null = keyPacket;
|
||||
|
||||
|
||||
+162
-25
@@ -24,12 +24,15 @@ import {
|
||||
assertNever,
|
||||
CallSerializer,
|
||||
clamp,
|
||||
clearIntervalUnthrottled,
|
||||
isFirefox,
|
||||
last,
|
||||
promiseWithResolvers,
|
||||
setInt24,
|
||||
setIntervalUnthrottled,
|
||||
setUint24,
|
||||
toUint8Array,
|
||||
UnthrottledTimerHandle,
|
||||
} from './misc';
|
||||
import { Muxer } from './muxer';
|
||||
import { SubtitleParser } from './subtitles';
|
||||
@@ -1026,6 +1029,21 @@ export class CanvasSource extends VideoSource {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Options for MediaStreamVideoTrackSource.
|
||||
* @group Media sources
|
||||
* @public
|
||||
*/
|
||||
export type MediaStreamVideoTrackSourceOptions = {
|
||||
/**
|
||||
* The frame rate at which the underlying video track is sampled. Defaults to the frame rate specified in the
|
||||
* track's [`MediaTrackSettings`](https://developer.mozilla.org/en-US/docs/Web/API/MediaTrackSettings). Set to
|
||||
* `null` to only add a frame whenever the underlying track pushes one - this minimizes frame count but can
|
||||
* lead to wildly irregular FPS.
|
||||
*/
|
||||
frameRate?: number | null;
|
||||
};
|
||||
|
||||
/**
|
||||
* Video source that encodes the frames of a
|
||||
* [`MediaStreamVideoTrack`](https://developer.mozilla.org/en-US/docs/Web/API/MediaStreamTrack) and pipes them into the
|
||||
@@ -1036,6 +1054,8 @@ export class CanvasSource extends VideoSource {
|
||||
* @public
|
||||
*/
|
||||
export class MediaStreamVideoTrackSource extends VideoSource {
|
||||
/** @internal */
|
||||
private _options: MediaStreamVideoTrackSourceOptions;
|
||||
/** @internal */
|
||||
private _encoder: VideoEncoderWrapper;
|
||||
/** @internal */
|
||||
@@ -1053,9 +1073,11 @@ export class MediaStreamVideoTrackSource extends VideoSource {
|
||||
/** @internal */
|
||||
private _paused = false;
|
||||
/** @internal */
|
||||
private _lastSampleTimestamp: number | null = null;
|
||||
private _lastVideoFrame: VideoFrame | null = null;
|
||||
/** @internal */
|
||||
private _pauseOffset = 0;
|
||||
private _timerHandle: UnthrottledTimerHandle | null = null;
|
||||
/** @internal */
|
||||
private _videoElement: HTMLVideoElement | null = null;
|
||||
|
||||
/** A promise that rejects upon any error within this source. This promise never resolves. */
|
||||
get errorPromise() {
|
||||
@@ -1073,11 +1095,21 @@ export class MediaStreamVideoTrackSource extends VideoSource {
|
||||
* [`MediaStreamVideoTrack`](https://developer.mozilla.org/en-US/docs/Web/API/MediaStreamTrack), which will pull
|
||||
* video samples from the stream in real time and encode them according to {@link VideoEncodingConfig}.
|
||||
*/
|
||||
constructor(track: MediaStreamVideoTrack, encodingConfig: VideoEncodingConfig) {
|
||||
constructor(
|
||||
track: MediaStreamVideoTrack,
|
||||
encodingConfig: VideoEncodingConfig,
|
||||
options: MediaStreamVideoTrackSourceOptions = {},
|
||||
) {
|
||||
if (!(track instanceof MediaStreamTrack) || track.kind !== 'video') {
|
||||
throw new TypeError('track must be a video MediaStreamTrack.');
|
||||
}
|
||||
validateVideoEncodingConfig(encodingConfig);
|
||||
if (typeof options !== 'object' || !options) {
|
||||
throw new TypeError('options must be an object.');
|
||||
}
|
||||
if (options.frameRate != null && (typeof options.frameRate !== 'number' || options.frameRate <= 0)) {
|
||||
throw new TypeError('options.frameRate, when provided, must be either a positive number or null.');
|
||||
}
|
||||
|
||||
encodingConfig = {
|
||||
...encodingConfig,
|
||||
@@ -1085,6 +1117,8 @@ export class MediaStreamVideoTrackSource extends VideoSource {
|
||||
};
|
||||
|
||||
super(encodingConfig.codec);
|
||||
|
||||
this._options = options;
|
||||
this._encoder = new VideoEncoderWrapper(this, encodingConfig);
|
||||
this._track = track;
|
||||
}
|
||||
@@ -1098,32 +1132,90 @@ export class MediaStreamVideoTrackSource extends VideoSource {
|
||||
);
|
||||
}
|
||||
|
||||
const frameRate = this._options.frameRate !== undefined
|
||||
? this._options.frameRate
|
||||
: (this._track.getSettings().frameRate ?? null);
|
||||
|
||||
this._abortController = new AbortController();
|
||||
|
||||
let firstVideoFrameTimestamp: number | null = null;
|
||||
let lastFrameTime: number | null = null;
|
||||
let frameCount = 0;
|
||||
let errored = false;
|
||||
let lastSampleTimestamp: number | null = null;
|
||||
let pauseOffset = 0;
|
||||
|
||||
const tick = () => {
|
||||
assert(frameRate !== null);
|
||||
|
||||
if (!this._lastVideoFrame) {
|
||||
return;
|
||||
}
|
||||
|
||||
assert(lastFrameTime !== null);
|
||||
assert(firstVideoFrameTimestamp !== null);
|
||||
|
||||
const now = performance.now();
|
||||
|
||||
// Add as many frames as warranted by the elapsed time.
|
||||
// > instead of >= intentionally because tick() is called before the _lastVideoFrame is changed
|
||||
while (now - lastFrameTime > 1000 / frameRate) {
|
||||
lastFrameTime += 1000 / frameRate;
|
||||
const timestamp = firstVideoFrameTimestamp + frameCount / frameRate;
|
||||
|
||||
const frame = new VideoFrame(this._videoElement ?? this._lastVideoFrame, {
|
||||
timestamp: 1e6 * timestamp,
|
||||
duration: 1e6 / frameRate,
|
||||
});
|
||||
|
||||
addVideoFrame(frame, now);
|
||||
}
|
||||
};
|
||||
|
||||
if (frameRate !== null) {
|
||||
this._timerHandle = setIntervalUnthrottled(tick, 4); // Run it at 250 Hz
|
||||
}
|
||||
|
||||
const onVideoFrame = (videoFrame: VideoFrame) => {
|
||||
if (frameRate === null) {
|
||||
addVideoFrame(videoFrame);
|
||||
} else {
|
||||
const now = performance.now();
|
||||
|
||||
if (!this._lastVideoFrame) {
|
||||
addVideoFrame(videoFrame.clone(), now);
|
||||
lastFrameTime = now;
|
||||
this._lastVideoFrame = videoFrame;
|
||||
} else {
|
||||
tick();
|
||||
this._lastVideoFrame?.close();
|
||||
this._lastVideoFrame = videoFrame;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const addVideoFrame = (videoFrame: VideoFrame, now = performance.now()) => {
|
||||
if (errored) {
|
||||
videoFrame.close();
|
||||
return;
|
||||
}
|
||||
|
||||
frameCount++;
|
||||
const currentTimestamp = videoFrame.timestamp / 1e6;
|
||||
|
||||
if (this._paused) {
|
||||
const frameSeen = firstVideoFrameTimestamp !== null;
|
||||
if (frameSeen) {
|
||||
if (this._lastSampleTimestamp !== null) {
|
||||
if (lastSampleTimestamp !== null) {
|
||||
// In addition to dropping this frame, let's also keep track of the time we have lost due to the
|
||||
// pause. Doing it like this instead of simply keeping track of the paused time is better since
|
||||
// it retains the frame rate of the underlying source.
|
||||
const timeDelta = currentTimestamp - this._lastSampleTimestamp;
|
||||
const timeDelta = currentTimestamp - lastSampleTimestamp;
|
||||
// We modify this field instead of _timestampOffset since we still might have data in flight
|
||||
// in the encoder, with which we don't want to mess.
|
||||
this._pauseOffset -= timeDelta;
|
||||
pauseOffset -= timeDelta;
|
||||
}
|
||||
this._lastSampleTimestamp = currentTimestamp;
|
||||
lastSampleTimestamp = currentTimestamp;
|
||||
}
|
||||
|
||||
videoFrame.close();
|
||||
@@ -1135,24 +1227,24 @@ export class MediaStreamVideoTrackSource extends VideoSource {
|
||||
|
||||
const muxer = this._connectedTrack!.output._muxer;
|
||||
if (muxer.firstMediaStreamTimestamp === null) {
|
||||
muxer.firstMediaStreamTimestamp = performance.now() / 1000;
|
||||
muxer.firstMediaStreamTimestamp = now / 1000;
|
||||
this._timestampOffset = -firstVideoFrameTimestamp;
|
||||
} else {
|
||||
this._timestampOffset = (performance.now() / 1000 - muxer.firstMediaStreamTimestamp)
|
||||
this._timestampOffset = (now / 1000 - muxer.firstMediaStreamTimestamp)
|
||||
- firstVideoFrameTimestamp;
|
||||
}
|
||||
}
|
||||
|
||||
this._lastSampleTimestamp = currentTimestamp;
|
||||
lastSampleTimestamp = currentTimestamp;
|
||||
|
||||
if (this._encoder.getQueueSize() >= 4) {
|
||||
if (this._encoder.getQueueSize() >= 8) {
|
||||
// Drop frames if the encoder is overloaded
|
||||
videoFrame.close();
|
||||
return;
|
||||
}
|
||||
|
||||
const sample = new VideoSample(videoFrame, {
|
||||
timestamp: currentTimestamp + this._pauseOffset,
|
||||
timestamp: currentTimestamp + pauseOffset,
|
||||
});
|
||||
|
||||
void this._encoder.add(sample, true)
|
||||
@@ -1209,8 +1301,44 @@ export class MediaStreamVideoTrackSource extends VideoSource {
|
||||
};
|
||||
|
||||
mediaStreamTrackProcessorWorker!.addEventListener('message', this._workerListener);
|
||||
} else if (frameRate !== null) {
|
||||
// No MediaStreamTrackProcessor support at all (e.g. Firefox), but we have a frame rate, so we can
|
||||
// manually sample from a hidden <video> element instead.
|
||||
const video = document.createElement('video');
|
||||
video.style.position = 'fixed';
|
||||
video.style.left = '-10000px';
|
||||
video.style.top = '-10000px';
|
||||
video.style.width = '1px';
|
||||
video.style.height = '1px';
|
||||
video.style.opacity = '0';
|
||||
video.style.pointerEvents = 'none';
|
||||
video.muted = true;
|
||||
video.srcObject = new MediaStream([this._track]);
|
||||
document.body.appendChild(video);
|
||||
this._videoElement = video;
|
||||
|
||||
video.addEventListener('loadeddata', () => {
|
||||
if (errored || !this._videoElement) {
|
||||
return;
|
||||
}
|
||||
|
||||
// This will be the first frame
|
||||
const frame = new VideoFrame(video, {
|
||||
timestamp: 1000 * performance.now(),
|
||||
});
|
||||
onVideoFrame(frame);
|
||||
frame.close();
|
||||
}, { once: true });
|
||||
|
||||
void video.play().catch((error) => {
|
||||
errored = true;
|
||||
this._promiseWithResolvers.reject(error);
|
||||
});
|
||||
} else {
|
||||
throw new Error('MediaStreamTrackProcessor is required but not supported by this browser.');
|
||||
throw new Error(
|
||||
'When no explicit frame rate is set, MediaStreamTrackProcessor is required; but it\'s not supported'
|
||||
+ ' by this browser.',
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1235,6 +1363,17 @@ export class MediaStreamVideoTrackSource extends VideoSource {
|
||||
this._abortController = null;
|
||||
}
|
||||
|
||||
if (this._timerHandle) {
|
||||
clearIntervalUnthrottled(this._timerHandle);
|
||||
}
|
||||
this._lastVideoFrame?.close();
|
||||
|
||||
if (this._videoElement) {
|
||||
this._videoElement.srcObject = null;
|
||||
this._videoElement.remove();
|
||||
this._videoElement = null;
|
||||
}
|
||||
|
||||
if (this._workerTrackId !== null) {
|
||||
assert(this._workerListener);
|
||||
|
||||
@@ -1899,10 +2038,6 @@ export class MediaStreamAudioTrackSource extends AudioSource {
|
||||
private _errorPromiseAccessed = false;
|
||||
/** @internal */
|
||||
private _paused = false;
|
||||
/** @internal */
|
||||
private _lastSampleTimestamp: number | null = null;
|
||||
/** @internal */
|
||||
private _pauseOffset = 0;
|
||||
|
||||
/** A promise that rejects upon any error within this source. This promise never resolves. */
|
||||
get errorPromise() {
|
||||
@@ -1934,7 +2069,7 @@ export class MediaStreamAudioTrackSource extends AudioSource {
|
||||
override async _start() {
|
||||
if (!this._errorPromiseAccessed) {
|
||||
console.warn(
|
||||
'Make sure not to ignore the `errorPromise` field on MediaStreamVideoTrackSource, so that any internal'
|
||||
'Make sure not to ignore the `errorPromise` field on MediaStreamAudioTrackSource, so that any internal'
|
||||
+ ' errors get bubbled up properly.',
|
||||
);
|
||||
}
|
||||
@@ -1943,6 +2078,8 @@ export class MediaStreamAudioTrackSource extends AudioSource {
|
||||
|
||||
let firstAudioDataTimestamp: number | null = null;
|
||||
let errored = false;
|
||||
let lastSampleTimestamp: number | null = null;
|
||||
let pauseOffset = 0;
|
||||
|
||||
const onAudioSample = (audioSample: AudioSample) => {
|
||||
if (errored) {
|
||||
@@ -1955,16 +2092,16 @@ export class MediaStreamAudioTrackSource extends AudioSource {
|
||||
if (this._paused) {
|
||||
const dataSeen = firstAudioDataTimestamp !== null;
|
||||
if (dataSeen) {
|
||||
if (this._lastSampleTimestamp !== null) {
|
||||
if (lastSampleTimestamp !== null) {
|
||||
// In addition to dropping this sample, let's also keep track of the time we have lost due to
|
||||
// the pause. Doing it like this instead of simply keeping track of the paused time is better
|
||||
// since it retains the sample rate of the underlying source.
|
||||
const timeDelta = currentTimestamp - this._lastSampleTimestamp;
|
||||
const timeDelta = currentTimestamp - lastSampleTimestamp;
|
||||
// We modify this field instead of _timestampOffset since we still might have data in flight
|
||||
// in the encoder, with which we don't want to mess.
|
||||
this._pauseOffset -= timeDelta;
|
||||
pauseOffset -= timeDelta;
|
||||
}
|
||||
this._lastSampleTimestamp = currentTimestamp;
|
||||
lastSampleTimestamp = currentTimestamp;
|
||||
}
|
||||
|
||||
audioSample.close();
|
||||
@@ -1984,15 +2121,15 @@ export class MediaStreamAudioTrackSource extends AudioSource {
|
||||
}
|
||||
}
|
||||
|
||||
this._lastSampleTimestamp = currentTimestamp;
|
||||
lastSampleTimestamp = currentTimestamp;
|
||||
|
||||
if (this._encoder.getQueueSize() >= 4) {
|
||||
if (this._encoder.getQueueSize() >= 8) {
|
||||
// Drop data if the encoder is overloaded
|
||||
audioSample.close();
|
||||
return;
|
||||
}
|
||||
|
||||
audioSample.setTimestamp(currentTimestamp + this._pauseOffset);
|
||||
audioSample.setTimestamp(currentTimestamp + pauseOffset);
|
||||
|
||||
void this._encoder.add(audioSample, true)
|
||||
.catch((error) => {
|
||||
|
||||
+170
@@ -839,3 +839,173 @@ export const validateRectangle = (rect: Rectangle, propertyPath: string) => {
|
||||
throw new TypeError(`${propertyPath}.height must be a non-negative integer.`);
|
||||
}
|
||||
};
|
||||
|
||||
export type UnthrottledTimerHandle = {
|
||||
id: ReturnType<typeof setTimeout> | number;
|
||||
};
|
||||
|
||||
type UnthrottledTimerMessage =
|
||||
| { type: 'set-timeout'; timerId: number; delay: number }
|
||||
| { type: 'set-interval'; timerId: number; delay: number }
|
||||
| { type: 'clear-timeout'; timerId: number }
|
||||
| { type: 'clear-interval'; timerId: number };
|
||||
|
||||
type UnthrottledTimerEvent = { type: 'fire'; timerId: number };
|
||||
|
||||
let unthrottledTimerWorker: Worker | undefined;
|
||||
let nextUnthrottledTimerId = 1;
|
||||
const unthrottledTimeoutCallbacks = new Map<number, () => void>();
|
||||
const unthrottledIntervalCallbacks = new Map<number, () => void>();
|
||||
|
||||
const shouldUseNativeTimers = () => {
|
||||
return typeof window === 'undefined';
|
||||
};
|
||||
|
||||
const unthrottledTimerWorkerMain = () => {
|
||||
const timeoutHandles = new Map<number, ReturnType<typeof setTimeout>>();
|
||||
const intervalHandles = new Map<number, ReturnType<typeof setInterval>>();
|
||||
|
||||
self.onmessage = (event: MessageEvent<UnthrottledTimerMessage>) => {
|
||||
const message = event.data;
|
||||
|
||||
switch (message.type) {
|
||||
case 'set-timeout': {
|
||||
const handle = setTimeout(() => {
|
||||
timeoutHandles.delete(message.timerId);
|
||||
self.postMessage({ type: 'fire', timerId: message.timerId });
|
||||
}, message.delay);
|
||||
|
||||
timeoutHandles.set(message.timerId, handle);
|
||||
}; break;
|
||||
|
||||
case 'set-interval': {
|
||||
const handle = setInterval(() => {
|
||||
self.postMessage({ type: 'fire', timerId: message.timerId });
|
||||
}, message.delay);
|
||||
|
||||
intervalHandles.set(message.timerId, handle);
|
||||
}; break;
|
||||
|
||||
case 'clear-timeout': {
|
||||
const handle = timeoutHandles.get(message.timerId);
|
||||
if (handle !== undefined) {
|
||||
clearTimeout(handle);
|
||||
timeoutHandles.delete(message.timerId);
|
||||
}
|
||||
}; break;
|
||||
|
||||
case 'clear-interval': {
|
||||
const handle = intervalHandles.get(message.timerId);
|
||||
if (handle !== undefined) {
|
||||
clearInterval(handle);
|
||||
intervalHandles.delete(message.timerId);
|
||||
}
|
||||
}; break;
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
const getUnthrottledTimerWorker = () => {
|
||||
if (unthrottledTimerWorker) {
|
||||
return unthrottledTimerWorker;
|
||||
}
|
||||
|
||||
const workerSource = `(${unthrottledTimerWorkerMain.toString()})();`;
|
||||
const workerURL = URL.createObjectURL(new Blob([workerSource], { type: 'text/javascript' }));
|
||||
unthrottledTimerWorker = new Worker(workerURL);
|
||||
URL.revokeObjectURL(workerURL);
|
||||
|
||||
unthrottledTimerWorker.onmessage = (event: MessageEvent<UnthrottledTimerEvent>) => {
|
||||
const message = event.data;
|
||||
|
||||
const timeoutCallback = unthrottledTimeoutCallbacks.get(message.timerId);
|
||||
if (timeoutCallback) {
|
||||
unthrottledTimeoutCallbacks.delete(message.timerId);
|
||||
timeoutCallback();
|
||||
return;
|
||||
}
|
||||
|
||||
const intervalCallback = unthrottledIntervalCallbacks.get(message.timerId);
|
||||
if (intervalCallback) {
|
||||
intervalCallback();
|
||||
}
|
||||
};
|
||||
|
||||
return unthrottledTimerWorker;
|
||||
};
|
||||
|
||||
export const setTimeoutUnthrottled = (
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-function-type
|
||||
callback: Function,
|
||||
delay: number,
|
||||
): UnthrottledTimerHandle => {
|
||||
if (shouldUseNativeTimers()) {
|
||||
return { id: setTimeout(callback, delay) };
|
||||
}
|
||||
|
||||
const timerId = nextUnthrottledTimerId++;
|
||||
unthrottledTimeoutCallbacks.set(timerId, () => {
|
||||
(callback as () => void)();
|
||||
});
|
||||
|
||||
getUnthrottledTimerWorker().postMessage({
|
||||
type: 'set-timeout',
|
||||
timerId,
|
||||
delay,
|
||||
} satisfies UnthrottledTimerMessage);
|
||||
|
||||
return { id: timerId };
|
||||
};
|
||||
|
||||
export const clearTimeoutUnthrottled = (timer: UnthrottledTimerHandle) => {
|
||||
if (shouldUseNativeTimers()) {
|
||||
clearTimeout(timer.id);
|
||||
return;
|
||||
}
|
||||
|
||||
assert(typeof timer.id === 'number');
|
||||
unthrottledTimeoutCallbacks.delete(timer.id);
|
||||
|
||||
getUnthrottledTimerWorker().postMessage({
|
||||
type: 'clear-timeout',
|
||||
timerId: timer.id,
|
||||
} satisfies UnthrottledTimerMessage);
|
||||
};
|
||||
|
||||
export const setIntervalUnthrottled = (
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-function-type
|
||||
callback: Function,
|
||||
delay: number,
|
||||
): UnthrottledTimerHandle => {
|
||||
if (shouldUseNativeTimers()) {
|
||||
return { id: setInterval(callback, delay) };
|
||||
}
|
||||
|
||||
const timerId = nextUnthrottledTimerId++;
|
||||
unthrottledIntervalCallbacks.set(timerId, () => {
|
||||
(callback as () => void)();
|
||||
});
|
||||
|
||||
getUnthrottledTimerWorker().postMessage({
|
||||
type: 'set-interval',
|
||||
timerId,
|
||||
delay,
|
||||
} satisfies UnthrottledTimerMessage);
|
||||
|
||||
return { id: timerId };
|
||||
};
|
||||
|
||||
export const clearIntervalUnthrottled = (timer: UnthrottledTimerHandle) => {
|
||||
if (shouldUseNativeTimers()) {
|
||||
clearInterval(timer.id);
|
||||
return;
|
||||
}
|
||||
|
||||
assert(typeof timer.id === 'number');
|
||||
unthrottledIntervalCallbacks.delete(timer.id);
|
||||
|
||||
getUnthrottledTimerWorker().postMessage({
|
||||
type: 'clear-interval',
|
||||
timerId: timer.id,
|
||||
} satisfies UnthrottledTimerMessage);
|
||||
};
|
||||
|
||||
@@ -61,3 +61,46 @@ test('ISOBMFF muxer internally converts ADTS to AAC', async () => {
|
||||
|
||||
expect(count).toBe(4557);
|
||||
});
|
||||
|
||||
test('Fragmented fMP4 with video+audio preserves B-frame CTS', async () => {
|
||||
using input = new Input({
|
||||
source: new FilePathSource(path.join(__dirname, '../public/video.mp4')),
|
||||
formats: ALL_FORMATS,
|
||||
});
|
||||
|
||||
const videoTrack = await input.getPrimaryVideoTrack();
|
||||
const audioTrack = await input.getPrimaryAudioTrack();
|
||||
assert(videoTrack);
|
||||
assert(audioTrack);
|
||||
|
||||
const originalVideoSink = new EncodedPacketSink(videoTrack);
|
||||
const originalTimestamps: number[] = [];
|
||||
for await (const packet of originalVideoSink.packets()) {
|
||||
originalTimestamps.push(packet.timestamp);
|
||||
}
|
||||
|
||||
const output = new Output({
|
||||
format: new Mp4OutputFormat({ fastStart: 'fragmented' }),
|
||||
target: new BufferTarget(),
|
||||
});
|
||||
|
||||
const conversion = await Conversion.init({ input, output, showWarnings: false });
|
||||
await conversion.execute();
|
||||
|
||||
using outputAsInput = new Input({
|
||||
source: new BufferSource(output.target.buffer!),
|
||||
formats: ALL_FORMATS,
|
||||
});
|
||||
|
||||
const outputVideoTrack = await outputAsInput.getPrimaryVideoTrack();
|
||||
assert(outputVideoTrack);
|
||||
|
||||
const videoSink = new EncodedPacketSink(outputVideoTrack);
|
||||
|
||||
const timestamps: number[] = [];
|
||||
for await (const packet of videoSink.packets()) {
|
||||
timestamps.push(packet.timestamp);
|
||||
}
|
||||
|
||||
expect(timestamps).toEqual(originalTimestamps);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user