mirror of
https://github.com/arcodange-org/mediabunny.git
synced 2026-09-28 03:13:51 +02:00
Merge branch 'main' into cursors
This commit is contained in:
Vanilagy
@@ -5,6 +5,7 @@ on:
|
||||
types: [published]
|
||||
|
||||
permissions:
|
||||
id-token: write # Required for OIDC
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
@@ -14,7 +15,7 @@ jobs:
|
||||
permissions: write-all
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
@@ -27,12 +28,15 @@ jobs:
|
||||
git push origin release
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 22
|
||||
cache: 'npm'
|
||||
registry-url: https://registry.npmjs.org
|
||||
|
||||
- name: Update npm
|
||||
run: npm install -g npm@latest
|
||||
|
||||
- name: Get package.json version
|
||||
id: package-json-version
|
||||
run: echo "version=v$(cat package.json | jq '.version' --raw-output)" >> $GITHUB_OUTPUT
|
||||
@@ -58,11 +62,7 @@ jobs:
|
||||
run: gh release upload ${{ github.event.release.tag_name }} dist/bundles/mediabunny.cjs dist/bundles/mediabunny.min.cjs dist/bundles/mediabunny.mjs dist/bundles/mediabunny.min.mjs dist/mediabunny.d.ts packages/mp3-encoder/dist/bundles/mediabunny-mp3-encoder.js packages/mp3-encoder/dist/bundles/mediabunny-mp3-encoder.min.js packages/mp3-encoder/dist/bundles/mediabunny-mp3-encoder.mjs packages/mp3-encoder/dist/bundles/mediabunny-mp3-encoder.min.mjs packages/mp3-encoder/dist/mediabunny-mp3-encoder.d.ts
|
||||
|
||||
- name: Publish Mediabunny to npm
|
||||
run: npm publish --provenance --access public
|
||||
env:
|
||||
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
|
||||
run: npm publish --access public
|
||||
|
||||
- name: Publish workspace packages to npm
|
||||
run: npm publish --provenance --access public --workspaces
|
||||
env:
|
||||
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
|
||||
run: npm publish --access public --workspaces
|
||||
|
||||
@@ -17,7 +17,10 @@ Mediabunny is a JavaScript library for reading, writing, and converting media fi
|
||||
|
||||
<div align="center">
|
||||
<a href="https://remotion.dev/" target="_blank" rel="sponsored">
|
||||
<img src="./docs/public/sponsors/remotion.png" width="60" height="60" alt="Remotion">
|
||||
<picture>
|
||||
<source srcset="./docs/public/sponsors/remotion-dark.png" media="(prefers-color-scheme: dark)">
|
||||
<img src="./docs/public/sponsors/remotion-light.png" width="60" height="60" alt="Remotion">
|
||||
</picture>
|
||||
</a>
|
||||
|
||||
<a href="https://www.gling.ai/" target="_blank" rel="sponsored">
|
||||
|
||||
+5
-4
@@ -100,9 +100,10 @@
|
||||
},
|
||||
*/
|
||||
video: () => ({
|
||||
width: 720,
|
||||
frameRate: 30,
|
||||
bitrate: Mediabunny.QUALITY_VERY_LOW,
|
||||
allowRotationMetadata: false,
|
||||
//width: 720,
|
||||
//frameRate: 30,
|
||||
//bitrate: Mediabunny.QUALITY_VERY_LOW,
|
||||
//discard: true,
|
||||
/*
|
||||
process: (sample) => {
|
||||
@@ -180,7 +181,7 @@
|
||||
},
|
||||
trim: {
|
||||
start: 0,
|
||||
end: 20
|
||||
end: 4
|
||||
},
|
||||
});
|
||||
console.log(conversion);
|
||||
|
||||
+22
-7
@@ -1,6 +1,7 @@
|
||||
<button>Go</button>
|
||||
|
||||
<script src="../dist/bundles/mediabunny.cjs"></script>
|
||||
<script src="../packages/mp3-encoder/dist/bundles/mediabunny-mp3-encoder.js"></script>
|
||||
|
||||
<script type="module">
|
||||
function download(blob, filename) {
|
||||
@@ -12,6 +13,8 @@
|
||||
URL.revokeObjectURL(url);
|
||||
}
|
||||
|
||||
MediabunnyMp3Encoder.registerMp3Encoder();
|
||||
|
||||
const button = document.querySelector('button');
|
||||
button.addEventListener('click', async () => {
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ video: true, audio: true });
|
||||
@@ -22,29 +25,41 @@
|
||||
target: new Mediabunny.BufferTarget(),
|
||||
format: new Mediabunny.Mp4OutputFormat(),
|
||||
});
|
||||
let videoSource = null;
|
||||
let audioSource = null;
|
||||
if (videoTrack) {
|
||||
const source = new Mediabunny.MediaStreamVideoTrackSource(videoTrack, {
|
||||
videoSource = new Mediabunny.MediaStreamVideoTrackSource(videoTrack, {
|
||||
codec: 'avc',
|
||||
bitrate: Mediabunny.QUALITY_MEDIUM
|
||||
});
|
||||
|
||||
source.errorPromise.catch((d) => console.log("Hello?????", d));
|
||||
videoSource.errorPromise.catch((d) => console.log("Hello?????", d));
|
||||
|
||||
output.addVideoTrack(source);
|
||||
output.addVideoTrack(videoSource);
|
||||
}
|
||||
if (audioTrack) {
|
||||
const source = new Mediabunny.MediaStreamAudioTrackSource(audioTrack, {
|
||||
codec: 'aac',
|
||||
audioSource = new Mediabunny.MediaStreamAudioTrackSource(audioTrack, {
|
||||
codec: 'mp3',
|
||||
bitrate: Mediabunny.QUALITY_MEDIUM
|
||||
});
|
||||
|
||||
source.errorPromise.catch((d) => console.log("Hello!!???", d));
|
||||
audioSource.errorPromise.catch((d) => console.log("Hello!!???", d));
|
||||
|
||||
output.addAudioTrack(source);
|
||||
output.addAudioTrack(audioSource);
|
||||
}
|
||||
|
||||
await output.start();
|
||||
|
||||
setTimeout(() => {
|
||||
videoSource?.pause();
|
||||
audioSource?.pause();
|
||||
|
||||
setTimeout(() => {
|
||||
videoSource?.resume();
|
||||
audioSource?.resume();
|
||||
}, 1000);
|
||||
}, 1000);
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, 5000));
|
||||
|
||||
await output.finalize();
|
||||
|
||||
@@ -120,6 +120,7 @@ type ConversionVideoOptions = {
|
||||
height?: number;
|
||||
fit?: 'fill' | 'contain' | 'cover';
|
||||
rotate?: 0 | 90 | 180 | 270;
|
||||
allowRotationMetadata?: boolean;
|
||||
crop?: { left: number; top: number; width: number; height: number };
|
||||
frameRate?: number;
|
||||
codec?: VideoCodec;
|
||||
@@ -175,6 +176,8 @@ In the rare case that the input video changes size over time, the `fit` field ca
|
||||
|
||||
`rotation` rotates the video by the specified number of degrees clockwise. This rotation is applied on top of any rotation metadata in the original input file and happens before cropping and resizing.
|
||||
|
||||
By default, Mediabunny will try to make use of rotation metadata in the output file to perform the rotation whenever possible. However, if you don't want this to happen, or you want to use Mediabunny to strip all rotation metadata from a file, you can set `allowRotationMetadata` to `false`.
|
||||
|
||||
### Cropping video
|
||||
|
||||
`crop` can be used to extract a rectangular region from the original video. The rectangle is specified using `left`, `top`, `width` and `height` and is clamped to the dimensions of the video. Cropping is applied after rotation but before resizing.
|
||||
|
||||
@@ -187,6 +187,16 @@ videoTrackSource.errorPromise.catch((error) => ...);
|
||||
|
||||
This source requires no additional method calls; data will automatically be captured and piped to the output file as soon as `start()` is called on the `Output`. Make sure to `stop()` on `videoTrack` after finalizing the `Output` if you don't need the user's media anymore.
|
||||
|
||||
If you want to temporarily stop capturing video frames from this source, you can use the `pause()` and `resume()` methods:
|
||||
```ts
|
||||
videoTrackSource.pause();
|
||||
|
||||
// Later:
|
||||
videoTrackSource.resume();
|
||||
```
|
||||
|
||||
While paused, video frames emitted by the stream will be ignored. When resumed, video frames are let through again, offset in timestamp such that the result plays back continuously with no gap in playback. Note that pausing does *not* stop the underlying media stream.
|
||||
|
||||
::: info
|
||||
If this source is the only MediaStreamTrack source in the `Output`, then the first video sample added by it starts at timestamp 0. If there are multiple, then the earliest media sample across all tracks starts at timestamp 0, and all tracks will be perfectly synchronized with each other.
|
||||
:::
|
||||
@@ -344,6 +354,16 @@ audioTrackSource.errorPromise.catch((error) => ...);
|
||||
|
||||
This source requires no additional method calls; data will automatically be captured and piped to the output file as soon as `start()` is called on the `Output`. Make sure to `stop()` on `audioTrack` after finalizing the `Output` if you don't need the user's media anymore.
|
||||
|
||||
If you want to temporarily stop capturing audio data from this source, you can use the `pause()` and `resume()` methods:
|
||||
```ts
|
||||
audioTrackSource.pause();
|
||||
|
||||
// Later:
|
||||
audioTrackSource.resume();
|
||||
```
|
||||
|
||||
While paused, audio data emitted by the stream will be ignored. When resumed, audio data are let through again, offset in timestamp such that the result plays back continuously with no gap in playback. Note that pausing does *not* stop the underlying media stream.
|
||||
|
||||
::: info
|
||||
If this source is the only MediaStreamTrack source in the `Output`, then the first audio sample added by it starts at timestamp 0. If there are multiple, then the earliest media sample across all tracks starts at timestamp 0, and all tracks will be perfectly synchronized with each other.
|
||||
:::
|
||||
|
||||
+3
-1
@@ -91,7 +91,7 @@ const bundleSizes = [
|
||||
|
||||
const sponsors = {
|
||||
gold: [
|
||||
{ image: '/sponsors/remotion.png', name: 'Remotion', url: 'https://remotion.dev/' },
|
||||
{ image: '/sponsors/remotion-light.png', name: 'Remotion', url: 'https://remotion.dev/' },
|
||||
{ image: '/sponsors/gling.svg', name: 'Gling AI', url: 'https://www.gling.ai/' },
|
||||
{ image: '/sponsors/diffusionstudio.png', name: 'Diffusion Studio', url: 'https://diffusion.studio/' },
|
||||
{ image: '/sponsors/kino.jpg', name: 'Kino', url: 'https://kino.ai/' },
|
||||
@@ -114,6 +114,7 @@ const sponsors = {
|
||||
{ image: 'https://avatars.githubusercontent.com/u/2698271', name: 'Matthew Gardner', url: 'https://github.com/spheric' },
|
||||
{ image: 'https://avatars.githubusercontent.com/u/5475819', name: 'AJ Funk', url: 'https://github.com/AJFunk' },
|
||||
{ image: 'https://avatars.githubusercontent.com/u/30229596', name: 'Pablo Bonilla', url: 'https://github.com/devPablo' },
|
||||
{ image: 'https://avatars.githubusercontent.com/u/56988069', name: 'SyhabouthAlex', url: 'https://github.com/SyhabouthAlex' },
|
||||
{ image: 'https://avatars.githubusercontent.com/u/38181164', name: 'wcw', url: 'https://github.com/asd55667' },
|
||||
{ image: 'https://avatars.githubusercontent.com/u/1836701', name: 'Bean Deng', url: 'https://github.com/HADB' },
|
||||
{ image: 'https://avatars.githubusercontent.com/u/63088713', name: 'taf2000', url: 'https://github.com/taf2000' },
|
||||
@@ -123,6 +124,7 @@ const sponsors = {
|
||||
{ image: 'https://avatars.githubusercontent.com/u/97225946', name: '808vita', url: 'https://github.com/808vita' },
|
||||
{ image: 'https://avatars.githubusercontent.com/u/3709646', name: 'Rodrigo Belfiore', url: 'https://github.com/roprgm' },
|
||||
{ image: 'https://avatars.githubusercontent.com/u/31102694', name: 'Aiden Liu', url: 'https://github.com/aidenlx' },
|
||||
{ image: 'https://avatars.githubusercontent.com/u/41021374', name: 'arthco', url: 'https://github.com/arthtyagi' },
|
||||
],
|
||||
};
|
||||
</script>
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 13 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 19 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 9.4 KiB |
Generated
+6
-6
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "mediabunny",
|
||||
"version": "1.25.2",
|
||||
"version": "1.26.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "mediabunny",
|
||||
"version": "1.25.2",
|
||||
"version": "1.26.0",
|
||||
"license": "MPL-2.0",
|
||||
"workspaces": [
|
||||
"packages/*"
|
||||
@@ -7739,9 +7739,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/mediabunny": {
|
||||
"version": "1.25.1",
|
||||
"resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.25.1.tgz",
|
||||
"integrity": "sha512-0Rrd47PMCVJbTPA7IJaXPCupV5/RZ/icgr+a0qExRJAr0n5vB4fsGSo+fdHIehG0CrddXtVRvNZwFtJz709yfA==",
|
||||
"version": "1.25.8",
|
||||
"resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.25.8.tgz",
|
||||
"integrity": "sha512-2WCa9WtEbHOvg5rAWXQjVE+O/r01fpj65ymZdA2XhIeaWmdDQ40p32F0WY6tdBi+aeY+4ldwhAOpo0eyVrYTGg==",
|
||||
"license": "MPL-2.0",
|
||||
"peer": true,
|
||||
"workspaces": [
|
||||
@@ -12065,7 +12065,7 @@
|
||||
},
|
||||
"packages/mp3-encoder": {
|
||||
"name": "@mediabunny/mp3-encoder",
|
||||
"version": "1.25.2",
|
||||
"version": "1.26.0",
|
||||
"license": "MPL-2.0",
|
||||
"devDependencies": {
|
||||
"@types/emscripten": "^1.40.1"
|
||||
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "mediabunny",
|
||||
"author": "Vanilagy",
|
||||
"version": "1.25.2",
|
||||
"version": "1.26.0",
|
||||
"description": "Pure TypeScript media toolkit for reading, writing, and converting media files, directly in the browser.",
|
||||
"type": "module",
|
||||
"workspaces": [
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "@mediabunny/mp3-encoder",
|
||||
"author": "Vanilagy",
|
||||
"version": "1.25.2",
|
||||
"version": "1.26.0",
|
||||
"description": "MP3 encoder extension for Mediabunny, based on LAME.",
|
||||
"main": "./dist/bundles/mediabunny-mp3-encoder.mjs",
|
||||
"module": "./dist/bundles/mediabunny-mp3-encoder.mjs",
|
||||
|
||||
@@ -22,7 +22,7 @@ class Mp3Encoder extends CustomAudioEncoder {
|
||||
|
||||
private buffer = new Uint8Array(2 ** 16);
|
||||
private currentBufferOffset = 0;
|
||||
private currentTimestamp = 0;
|
||||
private currentTimestamp: number | null = null;
|
||||
private chunkMetadata: EncodedAudioChunkMetadata = {};
|
||||
|
||||
static override supports(codec: AudioCodec, config: AudioDecoderConfig): boolean {
|
||||
@@ -79,6 +79,11 @@ class Mp3Encoder extends CustomAudioEncoder {
|
||||
}
|
||||
|
||||
async encode(audioSample: AudioSample) {
|
||||
if (this.currentTimestamp === null) {
|
||||
// The first sample's timestamp determines where we start
|
||||
this.currentTimestamp = audioSample.timestamp;
|
||||
}
|
||||
|
||||
const sizePerChannel = audioSample.allocationSize({
|
||||
format: 's16-planar',
|
||||
planeIndex: 0,
|
||||
@@ -123,6 +128,8 @@ class Mp3Encoder extends CustomAudioEncoder {
|
||||
* these chunks and extract the MP3 frames only when they're complete.
|
||||
*/
|
||||
private digestOutput(bytes: Uint8Array) {
|
||||
assert(this.currentTimestamp !== null);
|
||||
|
||||
const requiredBufferSize = this.currentBufferOffset + bytes.length;
|
||||
if (requiredBufferSize > this.buffer.length) {
|
||||
// Grow the buffer to the required size
|
||||
|
||||
+129
-4
@@ -22,6 +22,8 @@ import {
|
||||
textEncoder,
|
||||
toDataView,
|
||||
toUint8Array,
|
||||
getChromiumVersion,
|
||||
isChromium,
|
||||
} from './misc';
|
||||
import { PacketType } from './packet';
|
||||
import { MetadataTags } from './metadata';
|
||||
@@ -34,6 +36,7 @@ import { MetadataTags } from './metadata';
|
||||
|
||||
export enum AvcNalUnitType {
|
||||
IDR = 5,
|
||||
SEI = 6,
|
||||
SPS = 7,
|
||||
PPS = 8,
|
||||
SPS_EXT = 13,
|
||||
@@ -590,9 +593,7 @@ export const extractNalUnitTypeForHevc = (data: Uint8Array) => {
|
||||
};
|
||||
|
||||
/** Builds a HevcDecoderConfigurationRecord from an HEVC packet in Annex B format. */
|
||||
export const extractHevcDecoderConfigurationRecord = (
|
||||
packetData: Uint8Array,
|
||||
) => {
|
||||
export const extractHevcDecoderConfigurationRecord = (packetData: Uint8Array) => {
|
||||
try {
|
||||
const nalUnits = findNalUnitsInAnnexB(packetData);
|
||||
|
||||
@@ -1456,6 +1457,69 @@ export const extractAv1CodecInfoFromPacket = (
|
||||
}
|
||||
}
|
||||
|
||||
// Frame size
|
||||
const frameWidthBitsMinus1 = bitstream.readBits(4);
|
||||
const frameHeightBitsMinus1 = bitstream.readBits(4);
|
||||
const n1 = frameWidthBitsMinus1 + 1;
|
||||
bitstream.skipBits(n1); // max_frame_width_minus_1
|
||||
const n2 = frameHeightBitsMinus1 + 1;
|
||||
bitstream.skipBits(n2); // max_frame_height_minus_1
|
||||
|
||||
// Frame IDs
|
||||
let frameIdNumbersPresentFlag = 0;
|
||||
if (reducedStillPictureHeader) {
|
||||
frameIdNumbersPresentFlag = 0;
|
||||
} else {
|
||||
frameIdNumbersPresentFlag = bitstream.readBits(1);
|
||||
}
|
||||
|
||||
if (frameIdNumbersPresentFlag) {
|
||||
bitstream.skipBits(4); // delta_frame_id_length_minus_2
|
||||
bitstream.skipBits(3); // additional_frame_id_length_minus_1
|
||||
}
|
||||
|
||||
bitstream.skipBits(1); // use_128x128_superblock
|
||||
bitstream.skipBits(1); // enable_filter_intra
|
||||
bitstream.skipBits(1); // enable_intra_edge_filter
|
||||
|
||||
if (!reducedStillPictureHeader) {
|
||||
bitstream.skipBits(1); // enable_interintra_compound
|
||||
bitstream.skipBits(1); // enable_masked_compound
|
||||
bitstream.skipBits(1); // enable_warped_motion
|
||||
bitstream.skipBits(1); // enable_dual_filter
|
||||
const enableOrderHint = bitstream.readBits(1);
|
||||
|
||||
if (enableOrderHint) {
|
||||
bitstream.skipBits(1); // enable_jnt_comp
|
||||
bitstream.skipBits(1); // enable_ref_frame_mvs
|
||||
}
|
||||
|
||||
const seqChooseScreenContentTools = bitstream.readBits(1);
|
||||
let seqForceScreenContentTools = 0;
|
||||
|
||||
if (seqChooseScreenContentTools) {
|
||||
seqForceScreenContentTools = 2; // SELECT_SCREEN_CONTENT_TOOLS
|
||||
} else {
|
||||
seqForceScreenContentTools = bitstream.readBits(1);
|
||||
}
|
||||
|
||||
if (seqForceScreenContentTools > 0) {
|
||||
const seqChooseIntegerMv = bitstream.readBits(1);
|
||||
if (!seqChooseIntegerMv) {
|
||||
bitstream.skipBits(1); // seq_force_integer_mv
|
||||
}
|
||||
}
|
||||
|
||||
if (enableOrderHint) {
|
||||
bitstream.skipBits(3); // order_hint_bits_minus_1
|
||||
}
|
||||
}
|
||||
|
||||
bitstream.skipBits(1); // enable_superres
|
||||
bitstream.skipBits(1); // enable_cdef
|
||||
bitstream.skipBits(1); // enable_restoration
|
||||
|
||||
// color_config()
|
||||
const highBitdepth = bitstream.readBits(1);
|
||||
|
||||
let bitDepth = 8;
|
||||
@@ -1653,7 +1717,68 @@ export const determineVideoPacketType = (
|
||||
switch (codec) {
|
||||
case 'avc': {
|
||||
const nalUnits = extractAvcNalUnits(packetData, decoderConfig);
|
||||
const isKeyframe = nalUnits.some(x => extractNalUnitTypeForAvc(x) === AvcNalUnitType.IDR);
|
||||
let isKeyframe = nalUnits.some(x => extractNalUnitTypeForAvc(x) === AvcNalUnitType.IDR);
|
||||
|
||||
if (!isKeyframe && (!isChromium() || getChromiumVersion()! >= 144)) {
|
||||
// In addition to IDR, Recovery Point SEI also counts as a valid H.264 keyframe by current consensus.
|
||||
// See https://github.com/w3c/webcodecs/issues/650 for the relevant discussion. WebKit and Firefox have
|
||||
// always supported them, but Chromium hasn't, therefore the (admittedly dirty) version check.
|
||||
|
||||
for (const nalUnit of nalUnits) {
|
||||
const type = extractNalUnitTypeForAvc(nalUnit);
|
||||
if (type !== AvcNalUnitType.SEI) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const bytes = removeEmulationPreventionBytes(nalUnit);
|
||||
let pos = 1; // Skip NALU header
|
||||
|
||||
// sei_rbsp()
|
||||
do {
|
||||
// sei_message()
|
||||
let payloadType = 0;
|
||||
while (true) {
|
||||
const nextByte = bytes[pos++];
|
||||
if (nextByte === undefined) break;
|
||||
payloadType += nextByte;
|
||||
|
||||
if (nextByte < 255) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let payloadSize = 0;
|
||||
while (true) {
|
||||
const nextByte = bytes[pos++];
|
||||
if (nextByte === undefined) break;
|
||||
payloadSize += nextByte;
|
||||
|
||||
if (nextByte < 255) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// sei_payload()
|
||||
const PAYLOAD_TYPE_RECOVERY_POINT = 6;
|
||||
if (payloadType === PAYLOAD_TYPE_RECOVERY_POINT) {
|
||||
const bitstream = new Bitstream(bytes);
|
||||
bitstream.pos = 8 * pos;
|
||||
|
||||
const recoveryFrameCount = readExpGolomb(bitstream);
|
||||
const exactMatchFlag = bitstream.readBits(1);
|
||||
|
||||
if (recoveryFrameCount === 0 && exactMatchFlag === 1) {
|
||||
// https://github.com/w3c/webcodecs/pull/910
|
||||
// "recovery_frame_cnt == 0 and exact_match_flag=1 in the SEI recovery payload"
|
||||
isKeyframe = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
pos += payloadSize;
|
||||
} while (pos < bytes.length - 1);
|
||||
}
|
||||
}
|
||||
|
||||
return isKeyframe ? 'key' : 'delta';
|
||||
};
|
||||
|
||||
+27
-7
@@ -82,10 +82,16 @@ export type ConversionOptions = {
|
||||
|
||||
/** Options to trim the input file. */
|
||||
trim?: {
|
||||
/** The time in the input file in seconds at which the output file should start. Must be less than `end`. */
|
||||
start: number;
|
||||
/** The time in the input file in seconds at which the output file should end. Must be greater than `start`. */
|
||||
end: number;
|
||||
/**
|
||||
* The time in the input file in seconds at which the output file should start. Must be less than `end`.
|
||||
* Defaults to 0 when omitted.
|
||||
*/
|
||||
start?: number;
|
||||
/**
|
||||
* The time in the input file in seconds at which the output file should end. Must be greater than `start`.
|
||||
* Defaults to the duration of the input when omitted.
|
||||
*/
|
||||
end?: number;
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -138,6 +144,12 @@ export type ConversionVideoOptions = {
|
||||
* metadata.
|
||||
*/
|
||||
rotate?: Rotation;
|
||||
/**
|
||||
* Defaults to `true`. When enabaled, Mediabunny will use the rotation metadata in the output file to perform video
|
||||
* rotation whenever possible. Set this field to `false` if you want to ensure the output file does not make use of
|
||||
* rotation metadata and that any rotation is baked into the video frames directly.
|
||||
*/
|
||||
allowRotationMetadata?: boolean;
|
||||
/**
|
||||
* Specifies the rectangular region of the input video to crop to. The crop region will automatically be clamped to
|
||||
* the dimensions of the input video track. Cropping is performed after rotation but before resizing.
|
||||
@@ -299,6 +311,9 @@ const validateVideoOptions = (videoOptions: ConversionVideoOptions | undefined)
|
||||
if (videoOptions?.rotate !== undefined && ![0, 90, 180, 270].includes(videoOptions.rotate)) {
|
||||
throw new TypeError('options.video.rotate, when provided, must be 0, 90, 180 or 270.');
|
||||
}
|
||||
if (videoOptions?.allowRotationMetadata !== undefined && typeof videoOptions.allowRotationMetadata !== 'boolean') {
|
||||
throw new TypeError('options.video.allowRotationMetadata, when provided, must be a boolean.');
|
||||
}
|
||||
if (videoOptions?.crop !== undefined) {
|
||||
validateCropRectangle(videoOptions.crop, 'options.video.');
|
||||
}
|
||||
@@ -833,7 +848,8 @@ export class Conversion {
|
||||
let videoSource: VideoSource;
|
||||
|
||||
const totalRotation = normalizeRotation(track.rotation + (trackOptions.rotate ?? 0));
|
||||
const outputSupportsRotation = this.output.format.supportsVideoRotationMetadata;
|
||||
const canUseRotationMetadata = this.output.format.supportsVideoRotationMetadata
|
||||
&& (trackOptions.allowRotationMetadata ?? true);
|
||||
|
||||
const [rotatedWidth, rotatedHeight] = totalRotation % 180 === 0
|
||||
? [track.codedWidth, track.codedHeight]
|
||||
@@ -878,7 +894,7 @@ export class Conversion {
|
||||
// TODO This is suboptimal: Forcing a rerender when both rotation and process are set is not
|
||||
// performance-optimal, but right now there's no other way because we can't change the track rotation
|
||||
// metadata after the output has already started. Should be possible with API changes in v2, though!
|
||||
|| (totalRotation !== 0 && (!outputSupportsRotation || trackOptions.process !== undefined))
|
||||
|| (totalRotation !== 0 && (!canUseRotationMetadata || trackOptions.process !== undefined))
|
||||
|| !!crop;
|
||||
|
||||
const alpha = trackOptions.alpha ?? 'discard';
|
||||
@@ -1510,7 +1526,10 @@ export class Conversion {
|
||||
targetSampleRate,
|
||||
startTime: this._startTimestamp,
|
||||
endTime: this._endTimestamp,
|
||||
onSample: sample => this._registerAudioSample(track, trackOptions, source, sample),
|
||||
onSample: async (sample) => {
|
||||
await this._registerAudioSample(track, trackOptions, source, sample);
|
||||
sample.close();
|
||||
},
|
||||
});
|
||||
|
||||
const sink = new AudioSampleSink(track);
|
||||
@@ -1522,6 +1541,7 @@ export class Conversion {
|
||||
}
|
||||
|
||||
await resampler.add(sample);
|
||||
sample.close();
|
||||
}
|
||||
|
||||
await resampler.finalize();
|
||||
|
||||
@@ -2524,12 +2524,8 @@ abstract class IsobmffTrackBacking implements InputTrackBacking {
|
||||
const timestampInTimescale = this.mapTimestampIntoTimescale(timestamp);
|
||||
|
||||
const sampleTable = this.internalTrack.demuxer.getSampleTableForTrack(this.internalTrack);
|
||||
const sampleIndex = getSampleIndexForTimestamp(sampleTable, timestampInTimescale);
|
||||
const keyFrameSampleIndex = sampleIndex === -1
|
||||
? -1
|
||||
: getRelevantKeyframeIndexForSample(sampleTable, sampleIndex);
|
||||
|
||||
const promise = this.fetchPacketForSampleIndex(res, keyFrameSampleIndex, options);
|
||||
const sampleIndex = getKeyframeSampleIndexForTimestamp(sampleTable, timestampInTimescale);
|
||||
const promise = this.fetchPacketForSampleIndex(res, sampleIndex, options);
|
||||
if (res.pending) await promise;
|
||||
|
||||
if (!sampleTableIsEmpty(sampleTable) || !this.internalTrack.demuxer.isFragmented) {
|
||||
@@ -2968,7 +2964,45 @@ const getSampleIndexForTimestamp = (sampleTable: SampleTable, timescaleUnits: nu
|
||||
|
||||
const entry = sampleTable.sampleTimingEntries[index]!;
|
||||
return entry.startIndex
|
||||
+ Math.min(Math.floor((timescaleUnits - entry.startDecodeTimestamp) / entry.delta), entry.count - 1);
|
||||
+ Math.min(
|
||||
Math.floor((timescaleUnits - entry.startDecodeTimestamp) / entry.delta),
|
||||
entry.count - 1,
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
const getKeyframeSampleIndexForTimestamp = (sampleTable: SampleTable, timescaleUnits: number) => {
|
||||
if (!sampleTable.keySampleIndices) {
|
||||
// Every sample is a keyframe
|
||||
return getSampleIndexForTimestamp(sampleTable, timescaleUnits);
|
||||
}
|
||||
|
||||
if (sampleTable.presentationTimestamps) {
|
||||
const index = binarySearchLessOrEqual(
|
||||
sampleTable.presentationTimestamps,
|
||||
timescaleUnits,
|
||||
x => x.presentationTimestamp,
|
||||
);
|
||||
if (index === -1) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
// Walk the samples in presentation order until we find one that's a keyframe
|
||||
for (let i = index; i >= 0; i--) {
|
||||
const sampleIndex = sampleTable.presentationTimestamps[i]!.sampleIndex;
|
||||
const isKeyFrame = binarySearchExact(sampleTable.keySampleIndices, sampleIndex, x => x) !== -1;
|
||||
|
||||
if (isKeyFrame) {
|
||||
return sampleIndex;
|
||||
}
|
||||
}
|
||||
|
||||
return -1;
|
||||
} else {
|
||||
const sampleIndex = getSampleIndexForTimestamp(sampleTable, timescaleUnits);
|
||||
|
||||
const index = binarySearchLessOrEqual(sampleTable.keySampleIndices, sampleIndex, x => x);
|
||||
return sampleTable.keySampleIndices[index] ?? -1;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -3046,15 +3080,6 @@ const getSampleInfo = (sampleTable: SampleTable, sampleIndex: number): SampleInf
|
||||
};
|
||||
};
|
||||
|
||||
const getRelevantKeyframeIndexForSample = (sampleTable: SampleTable, sampleIndex: number) => {
|
||||
if (!sampleTable.keySampleIndices) {
|
||||
return sampleIndex;
|
||||
}
|
||||
|
||||
const index = binarySearchLessOrEqual(sampleTable.keySampleIndices, sampleIndex, x => x);
|
||||
return sampleTable.keySampleIndices[index] ?? -1;
|
||||
};
|
||||
|
||||
const getNextKeyframeIndexForSample = (sampleTable: SampleTable, sampleIndex: number) => {
|
||||
if (!sampleTable.keySampleIndices) {
|
||||
return sampleIndex + 1;
|
||||
|
||||
+9
-7
@@ -216,9 +216,10 @@ export class EncodedPacketSink {
|
||||
}
|
||||
|
||||
const packet = await this._track._backing.getKeyPacket(timestamp, options);
|
||||
if (!packet || packet.type === 'delta') {
|
||||
if (!packet) {
|
||||
return packet;
|
||||
}
|
||||
assert(packet.type === 'key');
|
||||
|
||||
const determinedType = await this._track.determinePacketType(packet);
|
||||
if (determinedType === 'delta') {
|
||||
@@ -250,9 +251,10 @@ export class EncodedPacketSink {
|
||||
}
|
||||
|
||||
const nextPacket = await this._track._backing.getNextKeyPacket(packet, options);
|
||||
if (!nextPacket || nextPacket.type === 'delta') {
|
||||
if (!nextPacket) {
|
||||
return nextPacket;
|
||||
}
|
||||
assert(nextPacket.type === 'key');
|
||||
|
||||
const determinedType = await this._track.determinePacketType(nextPacket);
|
||||
if (determinedType === 'delta') {
|
||||
@@ -489,9 +491,6 @@ export abstract class BaseMediaSampleSink<
|
||||
const packetSink = this._createPacketSink();
|
||||
const keyPacket = await packetSink.getKeyPacket(startTimestamp, { verifyKeyPackets: true })
|
||||
?? await packetSink.getFirstPacket();
|
||||
if (!keyPacket) {
|
||||
return;
|
||||
}
|
||||
|
||||
let currentPacket: EncodedPacket | null = keyPacket;
|
||||
|
||||
@@ -513,7 +512,7 @@ export abstract class BaseMediaSampleSink<
|
||||
}
|
||||
}
|
||||
|
||||
const packets = packetSink.packets(keyPacket, endPacket);
|
||||
const packets = packetSink.packets(keyPacket ?? undefined, endPacket);
|
||||
await packets.next(); // Skip the start packet as we already have it
|
||||
|
||||
while (currentPacket && !ended && !this._track.input._disposed) {
|
||||
@@ -2177,11 +2176,14 @@ export class AudioBufferSink {
|
||||
|
||||
/** @internal */
|
||||
_audioSampleToWrappedArrayBuffer(sample: AudioSample): WrappedAudioBuffer {
|
||||
return {
|
||||
const result: WrappedAudioBuffer = {
|
||||
buffer: sample.toAudioBuffer(),
|
||||
timestamp: sample.timestamp,
|
||||
duration: sample.duration,
|
||||
};
|
||||
|
||||
sample.close();
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
+161
-66
@@ -1055,6 +1055,12 @@ export class MediaStreamVideoTrackSource extends VideoSource {
|
||||
private _promiseWithResolvers = promiseWithResolvers();
|
||||
/** @internal */
|
||||
private _errorPromiseAccessed = false;
|
||||
/** @internal */
|
||||
private _paused = false;
|
||||
/** @internal */
|
||||
private _lastSampleTimestamp: number | null = null;
|
||||
/** @internal */
|
||||
private _pauseOffset = 0;
|
||||
|
||||
/** A promise that rejects upon any error within this source. This promise never resolves. */
|
||||
get errorPromise() {
|
||||
@@ -1062,6 +1068,11 @@ export class MediaStreamVideoTrackSource extends VideoSource {
|
||||
return this._promiseWithResolvers.promise;
|
||||
}
|
||||
|
||||
/** Whether this source is currently paused as a result of calling `.pause()`. */
|
||||
get paused() {
|
||||
return this._paused;
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new {@link MediaStreamVideoTrackSource} from a
|
||||
* [`MediaStreamVideoTrack`](https://developer.mozilla.org/en-US/docs/Web/API/MediaStreamTrack), which will pull
|
||||
@@ -1103,8 +1114,29 @@ export class MediaStreamVideoTrackSource extends VideoSource {
|
||||
return;
|
||||
}
|
||||
|
||||
const currentTimestamp = videoFrame.timestamp / 1e6;
|
||||
|
||||
if (this._paused) {
|
||||
const frameSeen = firstVideoFrameTimestamp !== null;
|
||||
if (frameSeen) {
|
||||
if (this._lastSampleTimestamp !== null) {
|
||||
// In addition to dropping this frame, let's also keep track of the time we have lost due to the
|
||||
// pause. Doing it like this instead of simply keeping track of the paused time is better since
|
||||
// it retains the frame rate of the underlying source.
|
||||
const timeDelta = currentTimestamp - this._lastSampleTimestamp;
|
||||
// We modify this field instead of _timestampOffset since we still might have data in flight
|
||||
// in the encoder, with which we don't want to mess.
|
||||
this._pauseOffset -= timeDelta;
|
||||
}
|
||||
this._lastSampleTimestamp = currentTimestamp;
|
||||
}
|
||||
|
||||
videoFrame.close();
|
||||
return;
|
||||
}
|
||||
|
||||
if (firstVideoFrameTimestamp === null) {
|
||||
firstVideoFrameTimestamp = videoFrame.timestamp / 1e6;
|
||||
firstVideoFrameTimestamp = currentTimestamp;
|
||||
|
||||
const muxer = this._connectedTrack!.output._muxer;
|
||||
if (muxer.firstMediaStreamTimestamp === null) {
|
||||
@@ -1116,13 +1148,19 @@ export class MediaStreamVideoTrackSource extends VideoSource {
|
||||
}
|
||||
}
|
||||
|
||||
this._lastSampleTimestamp = currentTimestamp;
|
||||
|
||||
if (this._encoder.getQueueSize() >= 4) {
|
||||
// Drop frames if the encoder is overloaded
|
||||
videoFrame.close();
|
||||
return;
|
||||
}
|
||||
|
||||
void this._encoder.add(new VideoSample(videoFrame), true)
|
||||
const sample = new VideoSample(videoFrame, {
|
||||
timestamp: currentTimestamp + this._pauseOffset,
|
||||
});
|
||||
|
||||
void this._encoder.add(sample, true)
|
||||
.catch((error) => {
|
||||
errored = true;
|
||||
|
||||
@@ -1182,6 +1220,19 @@ export class MediaStreamVideoTrackSource extends VideoSource {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Pauses the capture of video frames - any video frames emitted by the underlying media stream will be ignored
|
||||
* while paused. This does *not* close the underlying `MediaStreamVideoTrack`, it just ignores its output.
|
||||
*/
|
||||
pause() {
|
||||
this._paused = true;
|
||||
}
|
||||
|
||||
/** Resumes the capture of video frames after being paused. */
|
||||
resume() {
|
||||
this._paused = false;
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
override async _flushAndClose(forceClose: boolean) {
|
||||
if (this._abortController) {
|
||||
@@ -1356,21 +1407,27 @@ class AudioEncoderWrapper {
|
||||
(audioSample.timestamp + audioSample.duration) * audioSample.sampleRate,
|
||||
);
|
||||
|
||||
if (this.lastEndSampleIndex !== null && startSampleIndex > this.lastEndSampleIndex) {
|
||||
const sampleCount = startSampleIndex - this.lastEndSampleIndex;
|
||||
const fillSample = new AudioSample({
|
||||
data: new Float32Array(sampleCount * audioSample.numberOfChannels),
|
||||
format: 'f32-planar',
|
||||
sampleRate: audioSample.sampleRate,
|
||||
numberOfChannels: audioSample.numberOfChannels,
|
||||
numberOfFrames: sampleCount,
|
||||
timestamp: this.lastEndSampleIndex / audioSample.sampleRate,
|
||||
});
|
||||
if (this.lastEndSampleIndex === null) {
|
||||
this.lastEndSampleIndex = endSampleIndex;
|
||||
} else {
|
||||
const sampleDiff = startSampleIndex - this.lastEndSampleIndex;
|
||||
|
||||
await this.add(fillSample, true); // Recursive call
|
||||
if (sampleDiff >= 64) {
|
||||
// The gap is big enough, let's add a correction sample
|
||||
const fillSample = new AudioSample({
|
||||
data: new Float32Array(sampleDiff * audioSample.numberOfChannels),
|
||||
format: 'f32-planar',
|
||||
sampleRate: audioSample.sampleRate,
|
||||
numberOfChannels: audioSample.numberOfChannels,
|
||||
numberOfFrames: sampleDiff,
|
||||
timestamp: this.lastEndSampleIndex / audioSample.sampleRate,
|
||||
});
|
||||
|
||||
await this.add(fillSample, true); // Recursive call
|
||||
}
|
||||
|
||||
this.lastEndSampleIndex += audioSample.numberOfFrames;
|
||||
}
|
||||
|
||||
this.lastEndSampleIndex = endSampleIndex;
|
||||
}
|
||||
|
||||
if (this.customEncoder) {
|
||||
@@ -1851,6 +1908,12 @@ export class MediaStreamAudioTrackSource extends AudioSource {
|
||||
private _promiseWithResolvers = promiseWithResolvers();
|
||||
/** @internal */
|
||||
private _errorPromiseAccessed = false;
|
||||
/** @internal */
|
||||
private _paused = false;
|
||||
/** @internal */
|
||||
private _lastSampleTimestamp: number | null = null;
|
||||
/** @internal */
|
||||
private _pauseOffset = 0;
|
||||
|
||||
/** A promise that rejects upon any error within this source. This promise never resolves. */
|
||||
get errorPromise() {
|
||||
@@ -1858,6 +1921,11 @@ export class MediaStreamAudioTrackSource extends AudioSource {
|
||||
return this._promiseWithResolvers.promise;
|
||||
}
|
||||
|
||||
/** Whether this source is currently paused as a result of calling `.pause()`. */
|
||||
get paused() {
|
||||
return this._paused;
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new {@link MediaStreamAudioTrackSource} from a `MediaStreamAudioTrack`, which will pull audio samples
|
||||
* from the stream in real time and encode them according to {@link AudioEncodingConfig}.
|
||||
@@ -1884,38 +1952,74 @@ export class MediaStreamAudioTrackSource extends AudioSource {
|
||||
|
||||
this._abortController = new AbortController();
|
||||
|
||||
let firstAudioDataTimestamp: number | null = null;
|
||||
let errored = false;
|
||||
|
||||
const onAudioSample = (audioSample: AudioSample) => {
|
||||
if (errored) {
|
||||
audioSample.close();
|
||||
return;
|
||||
}
|
||||
|
||||
const currentTimestamp = audioSample.timestamp;
|
||||
|
||||
if (this._paused) {
|
||||
const dataSeen = firstAudioDataTimestamp !== null;
|
||||
if (dataSeen) {
|
||||
if (this._lastSampleTimestamp !== null) {
|
||||
// In addition to dropping this sample, let's also keep track of the time we have lost due to
|
||||
// the pause. Doing it like this instead of simply keeping track of the paused time is better
|
||||
// since it retains the sample rate of the underlying source.
|
||||
const timeDelta = currentTimestamp - this._lastSampleTimestamp;
|
||||
// We modify this field instead of _timestampOffset since we still might have data in flight
|
||||
// in the encoder, with which we don't want to mess.
|
||||
this._pauseOffset -= timeDelta;
|
||||
}
|
||||
this._lastSampleTimestamp = currentTimestamp;
|
||||
}
|
||||
|
||||
audioSample.close();
|
||||
return;
|
||||
}
|
||||
|
||||
if (firstAudioDataTimestamp === null) {
|
||||
firstAudioDataTimestamp = audioSample.timestamp;
|
||||
|
||||
const muxer = this._connectedTrack!.output._muxer;
|
||||
if (muxer.firstMediaStreamTimestamp === null) {
|
||||
muxer.firstMediaStreamTimestamp = performance.now() / 1000;
|
||||
this._timestampOffset = -firstAudioDataTimestamp;
|
||||
} else {
|
||||
this._timestampOffset = (performance.now() / 1000 - muxer.firstMediaStreamTimestamp)
|
||||
- firstAudioDataTimestamp;
|
||||
}
|
||||
}
|
||||
|
||||
this._lastSampleTimestamp = currentTimestamp;
|
||||
|
||||
if (this._encoder.getQueueSize() >= 4) {
|
||||
// Drop data if the encoder is overloaded
|
||||
audioSample.close();
|
||||
return;
|
||||
}
|
||||
|
||||
audioSample.setTimestamp(currentTimestamp + this._pauseOffset);
|
||||
|
||||
void this._encoder.add(audioSample, true)
|
||||
.catch((error) => {
|
||||
errored = true;
|
||||
|
||||
this._abortController?.abort();
|
||||
this._promiseWithResolvers.reject(error);
|
||||
void this._audioContext?.suspend();
|
||||
});
|
||||
};
|
||||
|
||||
if (typeof MediaStreamTrackProcessor !== 'undefined') {
|
||||
// Great, MediaStreamTrackProcessor is supported, this is the preferred way of doing things
|
||||
let firstAudioDataTimestamp: number | null = null;
|
||||
|
||||
const processor = new MediaStreamTrackProcessor({ track: this._track });
|
||||
const consumer = new WritableStream<AudioData>({
|
||||
write: (audioData) => {
|
||||
if (firstAudioDataTimestamp === null) {
|
||||
firstAudioDataTimestamp = audioData.timestamp / 1e6;
|
||||
|
||||
const muxer = this._connectedTrack!.output._muxer;
|
||||
if (muxer.firstMediaStreamTimestamp === null) {
|
||||
muxer.firstMediaStreamTimestamp = performance.now() / 1000;
|
||||
this._timestampOffset = -firstAudioDataTimestamp;
|
||||
} else {
|
||||
this._timestampOffset = (performance.now() / 1000 - muxer.firstMediaStreamTimestamp)
|
||||
- firstAudioDataTimestamp;
|
||||
}
|
||||
}
|
||||
|
||||
if (this._encoder.getQueueSize() >= 4) {
|
||||
// Drop data if the encoder is overloaded
|
||||
audioData.close();
|
||||
return;
|
||||
}
|
||||
|
||||
void this._encoder.add(new AudioSample(audioData), true)
|
||||
.catch((error) => {
|
||||
this._abortController?.abort();
|
||||
this._promiseWithResolvers.reject(error);
|
||||
});
|
||||
},
|
||||
write: audioData => onAudioSample(new AudioSample(audioData)),
|
||||
});
|
||||
|
||||
processor.readable.pipeTo(consumer, {
|
||||
@@ -1943,7 +2047,6 @@ export class MediaStreamAudioTrackSource extends AudioSource {
|
||||
sourceNode.connect(this._scriptProcessorNode);
|
||||
this._scriptProcessorNode.connect(this._audioContext.destination);
|
||||
|
||||
let audioReceived = false;
|
||||
let totalDuration = 0;
|
||||
|
||||
this._scriptProcessorNode.onaudioprocess = (event) => {
|
||||
@@ -1951,33 +2054,25 @@ export class MediaStreamAudioTrackSource extends AudioSource {
|
||||
totalDuration += event.inputBuffer.duration;
|
||||
|
||||
for (const audioSample of iterator) {
|
||||
if (!audioReceived) {
|
||||
audioReceived = true;
|
||||
|
||||
const muxer = this._connectedTrack!.output._muxer;
|
||||
if (muxer.firstMediaStreamTimestamp === null) {
|
||||
muxer.firstMediaStreamTimestamp = performance.now() / 1000;
|
||||
} else {
|
||||
this._timestampOffset = performance.now() / 1000 - muxer.firstMediaStreamTimestamp;
|
||||
}
|
||||
}
|
||||
|
||||
if (this._encoder.getQueueSize() >= 4) {
|
||||
// Drop data if the encoder is overloaded
|
||||
audioSample.close();
|
||||
continue;
|
||||
}
|
||||
|
||||
void this._encoder.add(audioSample, true)
|
||||
.catch((error) => {
|
||||
void this._audioContext!.suspend();
|
||||
this._promiseWithResolvers.reject(error);
|
||||
});
|
||||
onAudioSample(audioSample);
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Pauses the capture of audio data - any audio data emitted by the underlying media stream will be ignored
|
||||
* while paused. This does *not* close the underlying `MediaStreamAudioTrack`, it just ignores its output.
|
||||
*/
|
||||
pause() {
|
||||
this._paused = true;
|
||||
}
|
||||
|
||||
/** Resumes the capture of audio data after being paused. */
|
||||
resume() {
|
||||
this._paused = false;
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
override async _flushAndClose(forceClose: boolean) {
|
||||
if (this._abortController) {
|
||||
|
||||
+31
-2
@@ -685,7 +685,15 @@ export const isWebKit = () => {
|
||||
}
|
||||
|
||||
// This even returns true for WebKit-wrapping browsers such as Chrome on iOS
|
||||
return isWebKitCache = !!(typeof navigator !== 'undefined' && navigator.vendor?.match(/apple/i));
|
||||
return isWebKitCache = !!(
|
||||
typeof navigator !== 'undefined'
|
||||
&& (
|
||||
navigator.vendor?.match(/apple/i)
|
||||
// Or, in workers:
|
||||
|| (/AppleWebKit/.test(navigator.userAgent) && !/Chrome/.test(navigator.userAgent))
|
||||
|| /\b(iPad|iPhone|iPod)\b/.test(navigator.userAgent)
|
||||
)
|
||||
);
|
||||
};
|
||||
|
||||
let isFirefoxCache: boolean | null = null;
|
||||
@@ -703,7 +711,28 @@ export const isChromium = () => {
|
||||
return isChromiumCache;
|
||||
}
|
||||
|
||||
return isChromiumCache = !!(typeof navigator !== 'undefined' && navigator.vendor?.includes('Google Inc'));
|
||||
return isChromiumCache = !!(
|
||||
typeof navigator !== 'undefined'
|
||||
&& (navigator.vendor?.includes('Google Inc') || /Chrome/.test(navigator.userAgent))
|
||||
);
|
||||
};
|
||||
|
||||
let chromiumVersionCache: number | null = null;
|
||||
export const getChromiumVersion = () => {
|
||||
if (chromiumVersionCache !== null) {
|
||||
return chromiumVersionCache;
|
||||
}
|
||||
|
||||
if (typeof navigator === 'undefined') {
|
||||
return null;
|
||||
}
|
||||
|
||||
const match = /\bChrome\/(\d+)/.exec(navigator.userAgent);
|
||||
if (!match) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return chromiumVersionCache = Number(match[1]!);
|
||||
};
|
||||
|
||||
/**
|
||||
|
||||
+61
-3
@@ -21,6 +21,54 @@ import {
|
||||
|
||||
polyfillSymbolDispose();
|
||||
|
||||
type FinalizationRegistryValue = {
|
||||
type: 'video';
|
||||
data: VideoFrame | OffscreenCanvas | Uint8Array;
|
||||
} | {
|
||||
type: 'audio';
|
||||
data: AudioData | Uint8Array;
|
||||
};
|
||||
|
||||
// Let's manually handle logging the garbage collection errors that are typically logged by the browser. This way, they
|
||||
// also kick for audio samples (which is normally not the case), making sure any incorrect code is quickly caught.
|
||||
let lastVideoGcErrorLog = -Infinity;
|
||||
let lastAudioGcErrorLog = -Infinity;
|
||||
let finalizationRegistry: FinalizationRegistry<FinalizationRegistryValue> | null = null;
|
||||
if (typeof FinalizationRegistry !== 'undefined') {
|
||||
finalizationRegistry = new FinalizationRegistry<FinalizationRegistryValue>((value) => {
|
||||
const now = Date.now();
|
||||
|
||||
if (value.type === 'video') {
|
||||
if (now - lastVideoGcErrorLog >= 1000) {
|
||||
// This error is annoying but oh so important
|
||||
console.error(
|
||||
`A VideoSample was garbage collected without first being closed. For proper resource management,`
|
||||
+ ` make sure to call close() on all your VideoSamples as soon as you're done using them.`,
|
||||
);
|
||||
|
||||
lastVideoGcErrorLog = now;
|
||||
}
|
||||
|
||||
if (typeof VideoFrame !== 'undefined' && value.data instanceof VideoFrame) {
|
||||
value.data.close(); // Prevent the browser error since we're logging our own
|
||||
}
|
||||
} else {
|
||||
if (now - lastAudioGcErrorLog >= 1000) {
|
||||
console.error(
|
||||
`An AudioSample was garbage collected without first being closed. For proper resource management,`
|
||||
+ ` make sure to call close() on all your AudioSamples as soon as you're done using them.`,
|
||||
);
|
||||
|
||||
lastAudioGcErrorLog = now;
|
||||
}
|
||||
|
||||
if (typeof AudioData !== 'undefined' && value.data instanceof AudioData) {
|
||||
value.data.close();
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Metadata used for VideoSample initialization.
|
||||
* @group Samples
|
||||
@@ -144,7 +192,11 @@ export class VideoSample implements Disposable {
|
||||
data: VideoFrame | CanvasImageSource | AllowSharedBufferSource,
|
||||
init?: VideoSampleInit,
|
||||
) {
|
||||
if (data instanceof ArrayBuffer || ArrayBuffer.isView(data)) {
|
||||
if (
|
||||
data instanceof ArrayBuffer
|
||||
|| (typeof SharedArrayBuffer !== 'undefined' && data instanceof SharedArrayBuffer)
|
||||
|| ArrayBuffer.isView(data)
|
||||
) {
|
||||
if (!init || typeof init !== 'object') {
|
||||
throw new TypeError('init must be an object.');
|
||||
}
|
||||
@@ -278,6 +330,7 @@ export class VideoSample implements Disposable {
|
||||
}
|
||||
|
||||
VideoSample._openSampleCount++;
|
||||
finalizationRegistry?.register(this, { type: 'video', data: this._data }, this);
|
||||
}
|
||||
|
||||
/** Clones this video sample. */
|
||||
@@ -326,6 +379,8 @@ export class VideoSample implements Disposable {
|
||||
return;
|
||||
}
|
||||
|
||||
finalizationRegistry?.unregister(this);
|
||||
|
||||
if (isVideoFrame(this._data)) {
|
||||
this._data.close();
|
||||
} else {
|
||||
@@ -977,6 +1032,8 @@ export class AudioSample implements Disposable {
|
||||
|
||||
this._data = dataBuffer;
|
||||
}
|
||||
|
||||
finalizationRegistry?.register(this, { type: 'audio', data: this._data }, this);
|
||||
}
|
||||
|
||||
/** Returns the number of bytes required to hold the audio sample's data as specified by the given options. */
|
||||
@@ -1138,9 +1195,8 @@ export class AudioSample implements Disposable {
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Branch for Uint8Array data (non-AudioData)
|
||||
const uint8Data = this._data;
|
||||
const srcView = new DataView(uint8Data.buffer, uint8Data.byteOffset, uint8Data.byteLength);
|
||||
const srcView = toDataView(uint8Data);
|
||||
|
||||
const srcFormat = this.format;
|
||||
const readFn = getReadFunction(srcFormat);
|
||||
@@ -1210,6 +1266,8 @@ export class AudioSample implements Disposable {
|
||||
return;
|
||||
}
|
||||
|
||||
finalizationRegistry?.unregister(this);
|
||||
|
||||
if (isAudioData(this._data)) {
|
||||
this._data.close();
|
||||
} else {
|
||||
|
||||
@@ -19,7 +19,7 @@ test('can decode samples from a FLAC file', async () => {
|
||||
|
||||
const sink = new AudioSampleSink(track);
|
||||
|
||||
const sample = await sink.getSample(1);
|
||||
using sample = await sink.getSample(1);
|
||||
assert(sample);
|
||||
expect(sample.timestamp).toBe(0.9287981859410431);
|
||||
});
|
||||
|
||||
@@ -21,7 +21,7 @@ test('Can decode transparent video', async () => {
|
||||
expect(await videoTrack.canBeTransparent()).toBe(true);
|
||||
|
||||
const sink = new VideoSampleSink(videoTrack);
|
||||
const sample = (await sink.getSample(0.5))!;
|
||||
using sample = (await sink.getSample(0.5))!;
|
||||
|
||||
expect(sample.format).toContain('A'); // Probably RGBA
|
||||
expect(sample.hasAlpha).toBe(true);
|
||||
@@ -47,10 +47,10 @@ test('Can decode faulty transparent video and behaves gracefully', async () => {
|
||||
|
||||
const sink = new VideoSampleSink(videoTrack);
|
||||
|
||||
const startSample = (await sink.getSample(await videoTrack.getFirstTimestamp()))!;
|
||||
using startSample = (await sink.getSample(await videoTrack.getFirstTimestamp()))!;
|
||||
expect(startSample.format).toContain('A');
|
||||
|
||||
const secondSample = (await sink.getSample(secondKeyPacket.timestamp))!;
|
||||
using secondSample = (await sink.getSample(secondKeyPacket.timestamp))!;
|
||||
expect(secondSample.format).not.toContain('A'); // There was no alpha key frame for this one
|
||||
expect(secondSample.hasAlpha).toBe(false);
|
||||
});
|
||||
@@ -164,7 +164,7 @@ test('Can encode transparent video', async () => {
|
||||
|
||||
const sink = new VideoSampleSink(videoTrack);
|
||||
|
||||
const firstSample = (await sink.getSample(0))!;
|
||||
using firstSample = (await sink.getSample(0))!;
|
||||
expect(firstSample.format).toContain('A');
|
||||
|
||||
probeContext.clearRect(0, 0, probeCanvas.width, probeCanvas.height);
|
||||
@@ -201,7 +201,7 @@ test('Can encode video with alternating transparency', async () => {
|
||||
await output.start();
|
||||
|
||||
for (let i = 0; i < 64; i++) {
|
||||
const sample = new VideoSample(new Uint8Array(640 * 480 * 4), {
|
||||
using sample = new VideoSample(new Uint8Array(640 * 480 * 4), {
|
||||
format: i % 2 ? 'RGBX' : 'RGBA',
|
||||
codedWidth: 640,
|
||||
codedHeight: 480,
|
||||
@@ -235,7 +235,7 @@ test('Can encode video with alternating transparency', async () => {
|
||||
const sampleSink = new VideoSampleSink(videoTrack);
|
||||
|
||||
i = 0;
|
||||
for await (const sample of sampleSink.samples()) {
|
||||
for await (using sample of sampleSink.samples()) {
|
||||
if (i % 2) {
|
||||
expect(sample.format).not.toContain('A');
|
||||
} else {
|
||||
@@ -299,7 +299,7 @@ test('Can transmux transparent video, discards alpha by default', async () => {
|
||||
expect(await videoTrack.canBeTransparent()).toBe(false);
|
||||
|
||||
const sink = new VideoSampleSink(videoTrack);
|
||||
const sample = (await sink.getSample(await videoTrack.getFirstTimestamp()))!;
|
||||
using sample = (await sink.getSample(await videoTrack.getFirstTimestamp()))!;
|
||||
expect(sample.hasAlpha).toBe(false);
|
||||
});
|
||||
|
||||
@@ -331,7 +331,7 @@ test('Can transmux transparent video, can keep alpha', async () => {
|
||||
expect(await videoTrack.canBeTransparent()).toBe(true);
|
||||
|
||||
const sink = new VideoSampleSink(videoTrack);
|
||||
const sample = (await sink.getSample(await videoTrack.getFirstTimestamp()))!;
|
||||
using sample = (await sink.getSample(await videoTrack.getFirstTimestamp()))!;
|
||||
expect(sample.format).toContain('A');
|
||||
expect(sample.hasAlpha).toBe(true);
|
||||
});
|
||||
@@ -370,7 +370,7 @@ test('Can reencode transparent video, keeping alpha', async () => {
|
||||
expect(videoTrack.displayWidth).toBe(320);
|
||||
|
||||
const sink = new VideoSampleSink(videoTrack);
|
||||
const sample = (await sink.getSample(await videoTrack.getFirstTimestamp()))!;
|
||||
using sample = (await sink.getSample(await videoTrack.getFirstTimestamp()))!;
|
||||
expect(sample.format).toContain('A');
|
||||
expect(sample.hasAlpha).toBe(true);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user