diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bc40201..b2b2155 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,6 +5,7 @@ on: types: [published] permissions: + id-token: write # Required for OIDC contents: read jobs: @@ -14,7 +15,7 @@ jobs: permissions: write-all steps: - name: Checkout repository - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + uses: actions/checkout@v4 with: fetch-depth: 0 @@ -27,12 +28,15 @@ jobs: git push origin release - name: Set up Node.js - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + uses: actions/setup-node@v4 with: node-version: 22 cache: 'npm' registry-url: https://registry.npmjs.org + - name: Update npm + run: npm install -g npm@latest + - name: Get package.json version id: package-json-version run: echo "version=v$(cat package.json | jq '.version' --raw-output)" >> $GITHUB_OUTPUT @@ -58,11 +62,7 @@ jobs: run: gh release upload ${{ github.event.release.tag_name }} dist/bundles/mediabunny.cjs dist/bundles/mediabunny.min.cjs dist/bundles/mediabunny.mjs dist/bundles/mediabunny.min.mjs dist/mediabunny.d.ts packages/mp3-encoder/dist/bundles/mediabunny-mp3-encoder.js packages/mp3-encoder/dist/bundles/mediabunny-mp3-encoder.min.js packages/mp3-encoder/dist/bundles/mediabunny-mp3-encoder.mjs packages/mp3-encoder/dist/bundles/mediabunny-mp3-encoder.min.mjs packages/mp3-encoder/dist/mediabunny-mp3-encoder.d.ts - name: Publish Mediabunny to npm - run: npm publish --provenance --access public - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: npm publish --access public - name: Publish workspace packages to npm - run: npm publish --provenance --access public --workspaces - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: npm publish --access public --workspaces diff --git a/README.md b/README.md index fe4ce48..738f4fb 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,10 @@ Mediabunny is a JavaScript library for reading, writing, and converting media fi
- Remotion + + + Remotion +      @@ -39,6 +42,10 @@ Mediabunny is a JavaScript library for reading, writing, and converting media fi Pintura Labs +      + + Ponder +
### Bronze sponsors diff --git a/dev/convert.html b/dev/convert.html index 79b7ae3..36a2645 100644 --- a/dev/convert.html +++ b/dev/convert.html @@ -58,6 +58,7 @@ audio: (_, n) => ({ discard: n > 1, //codec: 'opus', + //codec: 'opus', /* process: (sample) => { return sample; @@ -99,7 +100,12 @@ }, */ video: () => ({ + allowRotationMetadata: false, + //width: 720, + //frameRate: 30, + //bitrate: Mediabunny.QUALITY_VERY_LOW, //discard: true, + /* process: (sample) => { if (!ctx) { // Create a canvas for image compositing @@ -118,6 +124,7 @@ return ctx.canvas; }, + */ //width: 300, //alpha: 'keep', //width: 320, @@ -173,8 +180,8 @@ } }, trim: { - //start: 0, - //end: 10 + start: 0, + end: 4 }, }); console.log(conversion); diff --git a/dev/demux.html b/dev/demux.html index fc1e694..197f091 100644 --- a/dev/demux.html +++ b/dev/demux.html @@ -45,7 +45,51 @@ source: new Mediabunny.BlobSource(file), }); - const videoTrack = await input.getPrimaryAudioTrack(); + const videoTrack = await input.getPrimaryVideoTrack(); + const sink = new Mediabunny.EncodedPacketSink(videoTrack); + + for await (const packet of sink.packets()) { + console.log(packet); + } + + + /* + const videoTrack = await input.getPrimaryVideoTrack(); + const sink = new Mediabunny.EncodedPacketSink(videoTrack); + + for await (const packet of sink.packets()) { + console.log(packet); + } + */ + + /* + return; + + const audioTrack = await input.getPrimaryAudioTrack(); + const sink = new Mediabunny.EncodedPacketSink(audioTrack); + + for await (const packet of sink.packets()) { + console.log(packet.timestamp, packet.duration, packet.type) + } + */ + + /* + const audioTrack = await input.getPrimaryAudioTrack(); + const sink = new Mediabunny.AudioSampleSink(audioTrack); + + let lastEnd = 0; + for await (const sample of sink.samples()) { + if (sample.timestamp - lastEnd > 0) { + console.warn(sample.timestamp - lastEnd) + console.log(sample.timestamp, sample.duration, "diffie", sample.timestamp - lastEnd); + } + + lastEnd = sample.timestamp + sample.duration; + sample.close(); + } + */ + + /* const sink = new Mediabunny.EncodedPacketSink(videoTrack); for await (const packet of sink.packets()) { @@ -53,6 +97,7 @@ if (packet.timestamp > 10) break; } + */ /* const sink = new Mediabunny.VideoSampleSink(videoTrack); diff --git a/dev/live.html b/dev/live.html index afcb991..8eef45b 100644 --- a/dev/live.html +++ b/dev/live.html @@ -1,6 +1,7 @@ + diff --git a/docs/public/sponsors/ponder.png b/docs/public/sponsors/ponder.png new file mode 100644 index 0000000..d335a62 Binary files /dev/null and b/docs/public/sponsors/ponder.png differ diff --git a/docs/public/sponsors/remotion-dark.png b/docs/public/sponsors/remotion-dark.png new file mode 100644 index 0000000..04e6ea1 Binary files /dev/null and b/docs/public/sponsors/remotion-dark.png differ diff --git a/docs/public/sponsors/remotion-light.png b/docs/public/sponsors/remotion-light.png new file mode 100644 index 0000000..74b3592 Binary files /dev/null and b/docs/public/sponsors/remotion-light.png differ diff --git a/docs/public/sponsors/remotion.png b/docs/public/sponsors/remotion.png deleted file mode 100644 index f76b061..0000000 Binary files a/docs/public/sponsors/remotion.png and /dev/null differ diff --git a/examples/live-recording/live-recording.ts b/examples/live-recording/live-recording.ts index d52f9bc..d3d97a3 100644 --- a/examples/live-recording/live-recording.ts +++ b/examples/live-recording/live-recording.ts @@ -21,7 +21,7 @@ const context = canvas.getContext('2d', { alpha: false, desynchronized: true })! const frameRate = 30; -const chunks: Uint8Array[] = []; +const chunks: Uint8Array[] = []; let recording = false; let output: Output; let videoSource: CanvasSource; diff --git a/examples/metadata-extraction/metadata-extraction.ts b/examples/metadata-extraction/metadata-extraction.ts index e5cf198..d2bc280 100644 --- a/examples/metadata-extraction/metadata-extraction.ts +++ b/examples/metadata-extraction/metadata-extraction.ts @@ -97,7 +97,7 @@ const extractMetadata = (resource: File | string) => { 'Lyrics': tags.lyrics, 'Comment': tags.comment, 'Images': tags.images?.map((image) => { - const blob = new Blob([image.data], { type: image.mimeType }); + const blob = new Blob([image.data.slice()], { type: image.mimeType }); const element = new Image(); element.src = URL.createObjectURL(blob); diff --git a/package-lock.json b/package-lock.json index 271da7e..1abaef6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "mediabunny", - "version": "1.24.5", + "version": "1.27.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "mediabunny", - "version": "1.24.5", + "version": "1.27.2", "license": "MPL-2.0", "workspaces": [ "packages/*" @@ -19,7 +19,7 @@ "@eslint/js": "^9.22.0", "@fontsource-variable/rubik": "^5.2.6", "@fontsource/dm-mono": "^5.2.6", - "@microsoft/api-extractor": "^7.52.1", + "@microsoft/api-extractor": "^7.55.1", "@stylistic/eslint-plugin": "^4.2.0", "@tailwindcss/vite": "^4.1.7", "@types/markdown-it-footnote": "^3.0.4", @@ -34,8 +34,8 @@ "mermaid": "^11.6.0", "tailwindcss": "^4.1.7", "tsx": "^4.19.4", - "typescript": "^5.8.2", - "typescript-eslint": "^8.26.1", + "typescript": "^5.9.3", + "typescript-eslint": "^8.48.0", "vite": "^6.3.5", "vitepress": "^1.6.3", "vitepress-plugin-llms": "^1.5.1", @@ -923,10 +923,11 @@ } }, "node_modules/@eslint-community/eslint-utils": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.1.tgz", - "integrity": "sha512-s3O3waFUrMV8P/XaF/+ZTp1X9XBZW1a4B97ZnjQF2KYWaFD2A8KyFBsrsfSjEmjn3RGWAIuvlneuZm3CUK3jbA==", + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.9.0.tgz", + "integrity": "sha512-ayVFHdtZ+hsq1t2Dy24wCmGXGe4q9Gu3smhLYALJrr473ZH27MsnSL+LKUlimp4BWJqMDMLmPpx/Q9R3OAlL4g==", "dev": true, + "license": "MIT", "dependencies": { "eslint-visitor-keys": "^3.4.3" }, @@ -1449,21 +1450,22 @@ } }, "node_modules/@microsoft/api-extractor": { - "version": "7.52.1", - "resolved": "https://registry.npmjs.org/@microsoft/api-extractor/-/api-extractor-7.52.1.tgz", - "integrity": "sha512-m3I5uAwE05orsu3D1AGyisX5KxsgVXB+U4bWOOaX/Z7Ftp/2Cy41qsNhO6LPvSxHBaapyser5dVorF1t5M6tig==", + "version": "7.55.1", + "resolved": "https://registry.npmjs.org/@microsoft/api-extractor/-/api-extractor-7.55.1.tgz", + "integrity": "sha512-l8Z+8qrLkZFM3HM95Dbpqs6G39fpCa7O5p8A7AkA6hSevxkgwsOlLrEuPv0ADOyj5dI1Af5WVDiwpKG/ya5G3w==", "dev": true, "license": "MIT", "dependencies": { - "@microsoft/api-extractor-model": "7.30.4", - "@microsoft/tsdoc": "~0.15.1", - "@microsoft/tsdoc-config": "~0.17.1", - "@rushstack/node-core-library": "5.12.0", - "@rushstack/rig-package": "0.5.3", - "@rushstack/terminal": "0.15.1", - "@rushstack/ts-command-line": "4.23.6", + "@microsoft/api-extractor-model": "7.32.1", + "@microsoft/tsdoc": "~0.16.0", + "@microsoft/tsdoc-config": "~0.18.0", + "@rushstack/node-core-library": "5.19.0", + "@rushstack/rig-package": "0.6.0", + "@rushstack/terminal": "0.19.4", + "@rushstack/ts-command-line": "5.1.4", + "diff": "~8.0.2", "lodash": "~4.17.15", - "minimatch": "~3.0.3", + "minimatch": "10.0.3", "resolve": "~1.22.1", "semver": "~7.5.4", "source-map": "~0.6.1", @@ -1474,75 +1476,51 @@ } }, "node_modules/@microsoft/api-extractor-model": { - "version": "7.30.4", - "resolved": "https://registry.npmjs.org/@microsoft/api-extractor-model/-/api-extractor-model-7.30.4.tgz", - "integrity": "sha512-RobC0gyVYsd2Fao9MTKOfTdBm41P/bCMUmzS5mQ7/MoAKEqy0FOBph3JOYdq4X4BsEnMEiSHc+0NUNmdzxCpjA==", + "version": "7.32.1", + "resolved": "https://registry.npmjs.org/@microsoft/api-extractor-model/-/api-extractor-model-7.32.1.tgz", + "integrity": "sha512-u4yJytMYiUAnhcNQcZDTh/tVtlrzKlyKrQnLOV+4Qr/5gV+cpufWzCYAB1Q23URFqD6z2RoL2UYncM9xJVGNKA==", "dev": true, "license": "MIT", "dependencies": { - "@microsoft/tsdoc": "~0.15.1", - "@microsoft/tsdoc-config": "~0.17.1", - "@rushstack/node-core-library": "5.12.0" + "@microsoft/tsdoc": "~0.16.0", + "@microsoft/tsdoc-config": "~0.18.0", + "@rushstack/node-core-library": "5.19.0" + } + }, + "node_modules/@microsoft/api-extractor/node_modules/typescript": { + "version": "5.8.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.2.tgz", + "integrity": "sha512-aJn6wq13/afZp/jT9QZmwEjDqqvSGp1VT5GVg+f/t6/oVyrgXM6BY1h9BRh/O5p3PlUPAe+WuiEZOmb/49RqoQ==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" } }, "node_modules/@microsoft/tsdoc": { - "version": "0.15.1", - "resolved": "https://registry.npmjs.org/@microsoft/tsdoc/-/tsdoc-0.15.1.tgz", - "integrity": "sha512-4aErSrCR/On/e5G2hDP0wjooqDdauzEbIq8hIkIe5pXV0rtWJZvdCEKL0ykZxex+IxIwBp0eGeV48hQN07dXtw==", + "version": "0.16.0", + "resolved": "https://registry.npmjs.org/@microsoft/tsdoc/-/tsdoc-0.16.0.tgz", + "integrity": "sha512-xgAyonlVVS+q7Vc7qLW0UrJU7rSFcETRWsqdXZtjzRU8dF+6CkozTK4V4y1LwOX7j8r/vHphjDeMeGI4tNGeGA==", "dev": true, "license": "MIT" }, "node_modules/@microsoft/tsdoc-config": { - "version": "0.17.1", - "resolved": "https://registry.npmjs.org/@microsoft/tsdoc-config/-/tsdoc-config-0.17.1.tgz", - "integrity": "sha512-UtjIFe0C6oYgTnad4q1QP4qXwLhe6tIpNTRStJ2RZEPIkqQPREAwE5spzVxsdn9UaEMUqhh0AqSx3X4nWAKXWw==", + "version": "0.18.0", + "resolved": "https://registry.npmjs.org/@microsoft/tsdoc-config/-/tsdoc-config-0.18.0.tgz", + "integrity": "sha512-8N/vClYyfOH+l4fLkkr9+myAoR6M7akc8ntBJ4DJdWH2b09uVfr71+LTMpNyG19fNqWDg8KEDZhx5wxuqHyGjw==", "dev": true, "license": "MIT", "dependencies": { - "@microsoft/tsdoc": "0.15.1", + "@microsoft/tsdoc": "0.16.0", "ajv": "~8.12.0", "jju": "~1.4.0", "resolve": "~1.22.2" } }, - "node_modules/@nodelib/fs.scandir": { - "version": "2.1.5", - "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", - "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@nodelib/fs.stat": "2.0.5", - "run-parallel": "^1.1.9" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/@nodelib/fs.stat": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", - "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 8" - } - }, - "node_modules/@nodelib/fs.walk": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", - "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@nodelib/fs.scandir": "2.1.5", - "fastq": "^1.6.0" - }, - "engines": { - "node": ">= 8" - } - }, "node_modules/@pkgjs/parseargs": { "version": "0.11.0", "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", @@ -1897,9 +1875,9 @@ ] }, "node_modules/@rushstack/node-core-library": { - "version": "5.12.0", - "resolved": "https://registry.npmjs.org/@rushstack/node-core-library/-/node-core-library-5.12.0.tgz", - "integrity": "sha512-QSwwzgzWoil1SCQse+yCHwlhRxNv2dX9siPnAb9zR/UmMhac4mjMrlMZpk64BlCeOFi1kJKgXRkihSwRMbboAQ==", + "version": "5.19.0", + "resolved": "https://registry.npmjs.org/@rushstack/node-core-library/-/node-core-library-5.19.0.tgz", + "integrity": "sha512-BxAopbeWBvNJ6VGiUL+5lbJXywTdsnMeOS8j57Cn/xY10r6sV/gbsTlfYKjzVCUBZATX2eRzJHSMCchsMTGN6A==", "dev": true, "license": "MIT", "dependencies": { @@ -1938,24 +1916,41 @@ "url": "https://github.com/sponsors/epoberezkin" } }, - "node_modules/@rushstack/rig-package": { - "version": "0.5.3", - "resolved": "https://registry.npmjs.org/@rushstack/rig-package/-/rig-package-0.5.3.tgz", - "integrity": "sha512-olzSSjYrvCNxUFZowevC3uz8gvKr3WTpHQ7BkpjtRpA3wK+T0ybep/SRUMfr195gBzJm5gaXw0ZMgjIyHqJUow==", + "node_modules/@rushstack/problem-matcher": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@rushstack/problem-matcher/-/problem-matcher-0.1.1.tgz", + "integrity": "sha512-Fm5XtS7+G8HLcJHCWpES5VmeMyjAKaWeyZU5qPzZC+22mPlJzAsOxymHiWIfuirtPckX3aptWws+K2d0BzniJA==", "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/node": "*" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@rushstack/rig-package": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/@rushstack/rig-package/-/rig-package-0.6.0.tgz", + "integrity": "sha512-ZQmfzsLE2+Y91GF15c65L/slMRVhF6Hycq04D4TwtdGaUAbIXXg9c5pKA5KFU7M4QMaihoobp9JJYpYcaY3zOw==", + "dev": true, + "license": "MIT", "dependencies": { "resolve": "~1.22.1", "strip-json-comments": "~3.1.1" } }, "node_modules/@rushstack/terminal": { - "version": "0.15.1", - "resolved": "https://registry.npmjs.org/@rushstack/terminal/-/terminal-0.15.1.tgz", - "integrity": "sha512-3vgJYwumcjoDOXU3IxZfd616lqOdmr8Ezj4OWgJZfhmiBK4Nh7eWcv8sU8N/HdzXcuHDXCRGn/6O2Q75QvaZMA==", + "version": "0.19.4", + "resolved": "https://registry.npmjs.org/@rushstack/terminal/-/terminal-0.19.4.tgz", + "integrity": "sha512-f4XQk02CrKfrMgyOfhYd3qWI944dLC21S4I/LUhrlAP23GTMDNG6EK5effQtFkISwUKCgD9vMBrJZaPSUquxWQ==", "dev": true, "license": "MIT", "dependencies": { - "@rushstack/node-core-library": "5.12.0", + "@rushstack/node-core-library": "5.19.0", + "@rushstack/problem-matcher": "0.1.1", "supports-color": "~8.1.1" }, "peerDependencies": { @@ -1968,13 +1963,13 @@ } }, "node_modules/@rushstack/ts-command-line": { - "version": "4.23.6", - "resolved": "https://registry.npmjs.org/@rushstack/ts-command-line/-/ts-command-line-4.23.6.tgz", - "integrity": "sha512-7WepygaF3YPEoToh4MAL/mmHkiIImQq3/uAkQX46kVoKTNOOlCtFGyNnze6OYuWw2o9rxsyrHVfIBKxq/am2RA==", + "version": "5.1.4", + "resolved": "https://registry.npmjs.org/@rushstack/ts-command-line/-/ts-command-line-5.1.4.tgz", + "integrity": "sha512-H0I6VdJ6sOUbktDFpP2VW5N29w8v4hRoNZOQz02vtEi6ZTYL1Ju8u+TcFiFawUDrUsx/5MQTUhd79uwZZVwVlA==", "dev": true, "license": "MIT", "dependencies": { - "@rushstack/terminal": "0.15.1", + "@rushstack/terminal": "0.19.4", "@types/argparse": "1.0.38", "argparse": "~1.0.9", "string-argv": "~0.3.1" @@ -2087,18 +2082,6 @@ "eslint": ">=9.0.0" } }, - "node_modules/@stylistic/eslint-plugin/node_modules/picomatch": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", - "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", - "dev": true, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, "node_modules/@tailwindcss/node": { "version": "4.1.7", "resolved": "https://registry.npmjs.org/@tailwindcss/node/-/node-4.1.7.tgz", @@ -2902,21 +2885,21 @@ } }, "node_modules/@typescript-eslint/eslint-plugin": { - "version": "8.26.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.26.1.tgz", - "integrity": "sha512-2X3mwqsj9Bd3Ciz508ZUtoQQYpOhU/kWoUqIf49H8Z0+Vbh6UF/y0OEYp0Q0axOGzaBGs7QxRwq0knSQ8khQNA==", + "version": "8.48.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.48.0.tgz", + "integrity": "sha512-XxXP5tL1txl13YFtrECECQYeZjBZad4fyd3cFV4a19LkAY/bIp9fev3US4S5fDVV2JaYFiKAZ/GRTOLer+mbyQ==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/regexpp": "^4.10.0", - "@typescript-eslint/scope-manager": "8.26.1", - "@typescript-eslint/type-utils": "8.26.1", - "@typescript-eslint/utils": "8.26.1", - "@typescript-eslint/visitor-keys": "8.26.1", + "@typescript-eslint/scope-manager": "8.48.0", + "@typescript-eslint/type-utils": "8.48.0", + "@typescript-eslint/utils": "8.48.0", + "@typescript-eslint/visitor-keys": "8.48.0", "graphemer": "^1.4.0", - "ignore": "^5.3.1", + "ignore": "^7.0.0", "natural-compare": "^1.4.0", - "ts-api-utils": "^2.0.1" + "ts-api-utils": "^2.1.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -2926,22 +2909,32 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "@typescript-eslint/parser": "^8.0.0 || ^8.0.0-alpha.0", + "@typescript-eslint/parser": "^8.48.0", "eslint": "^8.57.0 || ^9.0.0", - "typescript": ">=4.8.4 <5.9.0" + "typescript": ">=4.8.4 <6.0.0" + } + }, + "node_modules/@typescript-eslint/eslint-plugin/node_modules/ignore": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.5.tgz", + "integrity": "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" } }, "node_modules/@typescript-eslint/parser": { - "version": "8.26.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.26.1.tgz", - "integrity": "sha512-w6HZUV4NWxqd8BdeFf81t07d7/YV9s7TCWrQQbG5uhuvGUAW+fq1usZ1Hmz9UPNLniFnD8GLSsDpjP0hm1S4lQ==", + "version": "8.48.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.48.0.tgz", + "integrity": "sha512-jCzKdm/QK0Kg4V4IK/oMlRZlY+QOcdjv89U2NgKHZk1CYTj82/RVSx1mV/0gqCVMJ/DA+Zf/S4NBWNF8GQ+eqQ==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/scope-manager": "8.26.1", - "@typescript-eslint/types": "8.26.1", - "@typescript-eslint/typescript-estree": "8.26.1", - "@typescript-eslint/visitor-keys": "8.26.1", + "@typescript-eslint/scope-manager": "8.48.0", + "@typescript-eslint/types": "8.48.0", + "@typescript-eslint/typescript-estree": "8.48.0", + "@typescript-eslint/visitor-keys": "8.48.0", "debug": "^4.3.4" }, "engines": { @@ -2953,18 +2946,40 @@ }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0", - "typescript": ">=4.8.4 <5.9.0" + "typescript": ">=4.8.4 <6.0.0" } }, - "node_modules/@typescript-eslint/scope-manager": { - "version": "8.26.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.26.1.tgz", - "integrity": "sha512-6EIvbE5cNER8sqBu6V7+KeMZIC1664d2Yjt+B9EWUXrsyWpxx4lEZrmvxgSKRC6gX+efDL/UY9OpPZ267io3mg==", + "node_modules/@typescript-eslint/project-service": { + "version": "8.48.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.48.0.tgz", + "integrity": "sha512-Ne4CTZyRh1BecBf84siv42wv5vQvVmgtk8AuiEffKTUo3DrBaGYZueJSxxBZ8fjk/N3DrgChH4TOdIOwOwiqqw==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.26.1", - "@typescript-eslint/visitor-keys": "8.26.1" + "@typescript-eslint/tsconfig-utils": "^8.48.0", + "@typescript-eslint/types": "^8.48.0", + "debug": "^4.3.4" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "typescript": ">=4.8.4 <6.0.0" + } + }, + "node_modules/@typescript-eslint/scope-manager": { + "version": "8.48.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.48.0.tgz", + "integrity": "sha512-uGSSsbrtJrLduti0Q1Q9+BF1/iFKaxGoQwjWOIVNJv0o6omrdyR8ct37m4xIl5Zzpkp69Kkmvom7QFTtue89YQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/types": "8.48.0", + "@typescript-eslint/visitor-keys": "8.48.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -2974,17 +2989,35 @@ "url": "https://opencollective.com/typescript-eslint" } }, + "node_modules/@typescript-eslint/tsconfig-utils": { + "version": "8.48.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.48.0.tgz", + "integrity": "sha512-WNebjBdFdyu10sR1M4OXTt2OkMd5KWIL+LLfeH9KhgP+jzfDV/LI3eXzwJ1s9+Yc0Kzo2fQCdY/OpdusCMmh6w==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "typescript": ">=4.8.4 <6.0.0" + } + }, "node_modules/@typescript-eslint/type-utils": { - "version": "8.26.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.26.1.tgz", - "integrity": "sha512-Kcj/TagJLwoY/5w9JGEFV0dclQdyqw9+VMndxOJKtoFSjfZhLXhYjzsQEeyza03rwHx2vFEGvrJWJBXKleRvZg==", + "version": "8.48.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.48.0.tgz", + "integrity": "sha512-zbeVaVqeXhhab6QNEKfK96Xyc7UQuoFWERhEnj3mLVnUWrQnv15cJNseUni7f3g557gm0e46LZ6IJ4NJVOgOpw==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/typescript-estree": "8.26.1", - "@typescript-eslint/utils": "8.26.1", + "@typescript-eslint/types": "8.48.0", + "@typescript-eslint/typescript-estree": "8.48.0", + "@typescript-eslint/utils": "8.48.0", "debug": "^4.3.4", - "ts-api-utils": "^2.0.1" + "ts-api-utils": "^2.1.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -2995,13 +3028,13 @@ }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0", - "typescript": ">=4.8.4 <5.9.0" + "typescript": ">=4.8.4 <6.0.0" } }, "node_modules/@typescript-eslint/types": { - "version": "8.26.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.26.1.tgz", - "integrity": "sha512-n4THUQW27VmQMx+3P+B0Yptl7ydfceUj4ON/AQILAASwgYdZ/2dhfymRMh5egRUrvK5lSmaOm77Ry+lmXPOgBQ==", + "version": "8.48.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.48.0.tgz", + "integrity": "sha512-cQMcGQQH7kwKoVswD1xdOytxQR60MWKM1di26xSUtxehaDs/32Zpqsu5WJlXTtTTqyAVK8R7hvsUnIXRS+bjvA==", "dev": true, "license": "MIT", "engines": { @@ -3013,20 +3046,21 @@ } }, "node_modules/@typescript-eslint/typescript-estree": { - "version": "8.26.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.26.1.tgz", - "integrity": "sha512-yUwPpUHDgdrv1QJ7YQal3cMVBGWfnuCdKbXw1yyjArax3353rEJP1ZA+4F8nOlQ3RfS2hUN/wze3nlY+ZOhvoA==", + "version": "8.48.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.48.0.tgz", + "integrity": "sha512-ljHab1CSO4rGrQIAyizUS6UGHHCiAYhbfcIZ1zVJr5nMryxlXMVWS3duFPSKvSUbFPwkXMFk1k0EMIjub4sRRQ==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.26.1", - "@typescript-eslint/visitor-keys": "8.26.1", + "@typescript-eslint/project-service": "8.48.0", + "@typescript-eslint/tsconfig-utils": "8.48.0", + "@typescript-eslint/types": "8.48.0", + "@typescript-eslint/visitor-keys": "8.48.0", "debug": "^4.3.4", - "fast-glob": "^3.3.2", - "is-glob": "^4.0.3", "minimatch": "^9.0.4", "semver": "^7.6.0", - "ts-api-utils": "^2.0.1" + "tinyglobby": "^0.2.15", + "ts-api-utils": "^2.1.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -3036,13 +3070,13 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "typescript": ">=4.8.4 <5.9.0" + "typescript": ">=4.8.4 <6.0.0" } }, "node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", "dev": true, "license": "MIT", "dependencies": { @@ -3066,9 +3100,9 @@ } }, "node_modules/@typescript-eslint/typescript-estree/node_modules/semver": { - "version": "7.7.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.1.tgz", - "integrity": "sha512-hlq8tAfn0m/61p4BVRcPzIGr6LKiMwo4VM6dGi6pt4qcRkmNzTcWq6eCEjEh+qXjkMDvPlOFFSGwQjoEa6gyMA==", + "version": "7.7.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", + "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", "dev": true, "license": "ISC", "bin": { @@ -3079,16 +3113,16 @@ } }, "node_modules/@typescript-eslint/utils": { - "version": "8.26.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.26.1.tgz", - "integrity": "sha512-V4Urxa/XtSUroUrnI7q6yUTD3hDtfJ2jzVfeT3VK0ciizfK2q/zGC0iDh1lFMUZR8cImRrep6/q0xd/1ZGPQpg==", + "version": "8.48.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.48.0.tgz", + "integrity": "sha512-yTJO1XuGxCsSfIVt1+1UrLHtue8xz16V8apzPYI06W0HbEbEWHxHXgZaAgavIkoh+GeV6hKKd5jm0sS6OYxWXQ==", "dev": true, "license": "MIT", "dependencies": { - "@eslint-community/eslint-utils": "^4.4.0", - "@typescript-eslint/scope-manager": "8.26.1", - "@typescript-eslint/types": "8.26.1", - "@typescript-eslint/typescript-estree": "8.26.1" + "@eslint-community/eslint-utils": "^4.7.0", + "@typescript-eslint/scope-manager": "8.48.0", + "@typescript-eslint/types": "8.48.0", + "@typescript-eslint/typescript-estree": "8.48.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -3099,18 +3133,18 @@ }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0", - "typescript": ">=4.8.4 <5.9.0" + "typescript": ">=4.8.4 <6.0.0" } }, "node_modules/@typescript-eslint/visitor-keys": { - "version": "8.26.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.26.1.tgz", - "integrity": "sha512-AjOC3zfnxd6S4Eiy3jwktJPclqhFHNyd8L6Gycf9WUPoKZpgM5PjkxY1X7uSy61xVpiJDhhk7XT2NVsN3ALTWg==", + "version": "8.48.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.48.0.tgz", + "integrity": "sha512-T0XJMaRPOH3+LBbAfzR2jalckP1MSG/L9eUtY0DEzUyVaXJ/t6zN0nR7co5kz0Jko/nkSYCBRkz1djvjajVTTg==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.26.1", - "eslint-visitor-keys": "^4.2.0" + "@typescript-eslint/types": "8.48.0", + "eslint-visitor-keys": "^4.2.1" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -4120,19 +4154,6 @@ "concat-map": "0.0.1" } }, - "node_modules/braces": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", - "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", - "dev": true, - "license": "MIT", - "dependencies": { - "fill-range": "^7.1.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/buffer": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", @@ -5241,6 +5262,16 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/diff": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/diff/-/diff-8.0.2.tgz", + "integrity": "sha512-sSuxWU5j5SR9QQji/o2qMvqRNYRDOcBTgsJ/DeCf4iSN4gW+gNMXM7wFIP+fdXZxoNiAnHUTGjCr+TSWXdRDKg==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.3.1" + } + }, "node_modules/dom-accessibility-api": { "version": "0.5.16", "resolved": "https://registry.npmjs.org/dom-accessibility-api/-/dom-accessibility-api-0.5.16.tgz", @@ -5680,10 +5711,11 @@ } }, "node_modules/eslint-visitor-keys": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.0.tgz", - "integrity": "sha512-UyLnSehNt62FFhSwjZlHmeokpRK59rcz29j+F1/aDgbkbRTk7wIc9XzdoasMUbRNKDM0qQt/+BJ4BrpFeABemw==", + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz", + "integrity": "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==", "dev": true, + "license": "Apache-2.0", "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" }, @@ -5905,36 +5937,6 @@ "dev": true, "license": "MIT" }, - "node_modules/fast-glob": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", - "integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@nodelib/fs.stat": "^2.0.2", - "@nodelib/fs.walk": "^1.2.3", - "glob-parent": "^5.1.2", - "merge2": "^1.3.0", - "micromatch": "^4.0.8" - }, - "engines": { - "node": ">=8.6.0" - } - }, - "node_modules/fast-glob/node_modules/glob-parent": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", - "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", - "dev": true, - "license": "ISC", - "dependencies": { - "is-glob": "^4.0.1" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/fast-json-stable-stringify": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", @@ -5966,16 +5968,6 @@ "fxparser": "src/cli/cli.js" } }, - "node_modules/fastq": { - "version": "1.19.1", - "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.19.1.tgz", - "integrity": "sha512-GwLTyxkCXjXbxqIhTsMI2Nui8huMPtnxg7krajPJAjnEG/iiOS7i+zCtWGZR9G0NBKbXKh6X9m9UIsYX/N6vvQ==", - "dev": true, - "license": "ISC", - "dependencies": { - "reusify": "^1.0.4" - } - }, "node_modules/fault": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/fault/-/fault-2.0.1.tgz", @@ -6000,6 +5992,24 @@ "pend": "~1.2.0" } }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, "node_modules/fetch-blob": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", @@ -6036,19 +6046,6 @@ "node": ">=16.0.0" } }, - "node_modules/fill-range": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", - "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", - "dev": true, - "license": "MIT", - "dependencies": { - "to-regex-range": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/find-cache-dir": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-5.0.0.tgz", @@ -6151,9 +6148,9 @@ } }, "node_modules/fs-extra": { - "version": "11.3.0", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.0.tgz", - "integrity": "sha512-Z4XaCL6dUDHfP/jT25jJKMmtxvuwbkrD1vNSMFlo9lNLY2c5FHYSQgHPRZUjAB26TpDEoW9HCOgplrdbaPV/ew==", + "version": "11.3.2", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.2.tgz", + "integrity": "sha512-Xr9F6z6up6Ws+NjzMCZc6WXg2YFRlrLP9NQDO3VQrWrfiojdhS56TzueT88ze0uBdCTwEIhQ3ptnmKeWGFAe0A==", "dev": true, "license": "MIT", "dependencies": { @@ -6185,6 +6182,7 @@ "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", "dev": true, + "license": "MIT", "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -6478,6 +6476,7 @@ "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", "dev": true, + "license": "MIT", "dependencies": { "function-bind": "^1.1.2" }, @@ -6727,10 +6726,11 @@ } }, "node_modules/is-core-module": { - "version": "2.15.1", - "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.15.1.tgz", - "integrity": "sha512-z0vtXSwucUJtANQWldhbtbt7BnL0vxiFjIdDLAatwhDYty2bad6s+rijD6Ri4YuYJubLzIJLUidCh09e1djEVQ==", + "version": "2.16.1", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.1.tgz", + "integrity": "sha512-UfoeMA6fIJ8wTYFEUjelnaGI67v6+N7qXJEvQuIGa99l4xsCruSYOVSQ0uPANn4dAzm8lkYPaKLrrijLq7x23w==", "dev": true, + "license": "MIT", "dependencies": { "hasown": "^2.0.2" }, @@ -6782,16 +6782,6 @@ "node": ">=0.10.0" } }, - "node_modules/is-number": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", - "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.12.0" - } - }, "node_modules/is-plain-obj": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-4.1.0.tgz", @@ -6924,9 +6914,9 @@ "dev": true }, "node_modules/jsonfile": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", - "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.2.0.tgz", + "integrity": "sha512-FGuPw30AdOIUTRMC2OMRtQV+jkVj2cfPqSeWXv1NEAJ1qZ5zb1X6z1mFhbfOB/iy3ssJCD+3KuZ8r8C3uVFlAg==", "dev": true, "license": "MIT", "dependencies": { @@ -7749,9 +7739,9 @@ } }, "node_modules/mediabunny": { - "version": "1.24.4", - "resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.24.4.tgz", - "integrity": "sha512-dpWYBPTtMg152yNLXZQ7xb6hsXdYbKp9EuK8qq4npS+SZ08FVc1XHlXYhrOm31T+tUVJKgm95Yaqy69wTpZP9Q==", + "version": "1.27.1", + "resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.27.1.tgz", + "integrity": "sha512-XdgXNox1hx2ynQy3jcCbzX1YsJt9en9VnZVidNXTNnQUdl6q2twSoV1GjgTb7Wlw9evCYXcKkgjE8KNU7K5trg==", "license": "MPL-2.0", "peer": true, "workspaces": [ @@ -7772,16 +7762,6 @@ "integrity": "sha512-IAeFvcOnV9V0Yk+bFhYR07O3yNina9ANIN5MoXBKYJ/RLYPurd2d0yw14MDhpr9/momp0WofT1bPUh3hkzdi/g==", "dev": true }, - "node_modules/merge2": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", - "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 8" - } - }, "node_modules/mermaid": { "version": "11.6.0", "resolved": "https://registry.npmjs.org/mermaid/-/mermaid-11.6.0.tgz", @@ -8296,20 +8276,6 @@ ], "license": "MIT" }, - "node_modules/micromatch": { - "version": "4.0.8", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", - "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", - "dev": true, - "license": "MIT", - "dependencies": { - "braces": "^3.0.3", - "picomatch": "^2.3.1" - }, - "engines": { - "node": ">=8.6" - } - }, "node_modules/millify": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/millify/-/millify-6.1.0.tgz", @@ -8336,15 +8302,19 @@ } }, "node_modules/minimatch": { - "version": "3.0.8", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.8.tgz", - "integrity": "sha512-6FsRAQsxQ61mw+qP1ZzbL9Bc78x2p5OqNgNpnoAFLTrX8n5Kxph0CsnhmKKNXTWjXqU5L0pGPR7hYk+XWZr60Q==", + "version": "10.0.3", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.0.3.tgz", + "integrity": "sha512-IPZ167aShDZZUMdRk66cyQAW3qr0WzbHkPdMYa8bzZhlHhO3jALbKdxcaak7W9FfT2rZNpQuUu4Od7ILEpXSaw==", "dev": true, + "license": "ISC", "dependencies": { - "brace-expansion": "^1.1.7" + "@isaacs/brace-expansion": "^5.0.0" }, "engines": { - "node": "*" + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" } }, "node_modules/minipass": { @@ -8777,7 +8747,8 @@ "version": "1.0.7", "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/path-scurry": { "version": "1.11.1", @@ -8851,13 +8822,13 @@ "license": "ISC" }, "node_modules/picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", + "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", "engines": { - "node": ">=8.6" + "node": ">=12" }, "funding": { "url": "https://github.com/sponsors/jonschlinkert" @@ -9189,27 +9160,6 @@ "dev": true, "license": "MIT" }, - "node_modules/queue-microtask": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", - "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" - }, "node_modules/react": { "version": "18.3.1", "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz", @@ -9397,18 +9347,22 @@ } }, "node_modules/resolve": { - "version": "1.22.8", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.8.tgz", - "integrity": "sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==", + "version": "1.22.11", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.11.tgz", + "integrity": "sha512-RfqAvLnMl313r7c9oclB1HhUEAezcpLjz95wFH4LVuhk9JF/r22qmVP9AMmOU4vMX7Q8pN8jwNg/CSpdFnMjTQ==", "dev": true, + "license": "MIT", "dependencies": { - "is-core-module": "^2.13.0", + "is-core-module": "^2.16.1", "path-parse": "^1.0.7", "supports-preserve-symlinks-flag": "^1.0.0" }, "bin": { "resolve": "bin/resolve" }, + "engines": { + "node": ">= 0.4" + }, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -9460,17 +9414,6 @@ "node": ">=10" } }, - "node_modules/reusify": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz", - "integrity": "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==", - "dev": true, - "license": "MIT", - "engines": { - "iojs": ">=1.0.0", - "node": ">=0.10.0" - } - }, "node_modules/rfdc": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.4.1.tgz", @@ -9543,30 +9486,6 @@ "points-on-path": "^0.2.1" } }, - "node_modules/run-parallel": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", - "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT", - "dependencies": { - "queue-microtask": "^1.2.2" - } - }, "node_modules/rw": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/rw/-/rw-1.3.3.tgz", @@ -10140,6 +10059,7 @@ "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", "dev": true, + "license": "MIT", "engines": { "node": ">= 0.4" }, @@ -10250,14 +10170,14 @@ "dev": true }, "node_modules/tinyglobby": { - "version": "0.2.14", - "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.14.tgz", - "integrity": "sha512-tX5e7OM1HnYr2+a2C/4V0htOcSQcoSTH9KgJnVvNm5zm/cyEWKJ7j7YutsH9CxMdtOkkLFy2AHrMci9IM8IPZQ==", + "version": "0.2.15", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.15.tgz", + "integrity": "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==", "dev": true, "license": "MIT", "dependencies": { - "fdir": "^6.4.4", - "picomatch": "^4.0.2" + "fdir": "^6.5.0", + "picomatch": "^4.0.3" }, "engines": { "node": ">=12.0.0" @@ -10266,34 +10186,6 @@ "url": "https://github.com/sponsors/SuperchupuDev" } }, - "node_modules/tinyglobby/node_modules/fdir": { - "version": "6.4.4", - "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.4.4.tgz", - "integrity": "sha512-1NZP+GK4GfuAv3PqKvxQRDMjdSRZjnkq7KfhlNrCNNlZ0ygQFpebfrnfnq/W7fpUnAv9aGWmY1zKx7FYL3gwhg==", - "dev": true, - "license": "MIT", - "peerDependencies": { - "picomatch": "^3 || ^4" - }, - "peerDependenciesMeta": { - "picomatch": { - "optional": true - } - } - }, - "node_modules/tinyglobby/node_modules/picomatch": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", - "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, "node_modules/tinypool": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/tinypool/-/tinypool-1.1.1.tgz", @@ -10324,19 +10216,6 @@ "node": ">=14.0.0" } }, - "node_modules/to-regex-range": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", - "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "is-number": "^7.0.0" - }, - "engines": { - "node": ">=8.0" - } - }, "node_modules/tokenx": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/tokenx/-/tokenx-1.1.0.tgz", @@ -10383,9 +10262,9 @@ } }, "node_modules/ts-api-utils": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.0.1.tgz", - "integrity": "sha512-dnlgjFSVetynI8nzgJ+qF62efpglpWRk8isUEWZGWlJYySCTD6aKvbUDu+zbPeDakk3bg5H4XpitHukgfL1m9w==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.1.0.tgz", + "integrity": "sha512-CUgTZL1irw8u29bzrOD/nH85jqyc74D6SshFgujOIA7osm2Rz7dYH77agkx7H4FBNxDq7Cjf+IjaX/8zwFW+ZQ==", "dev": true, "license": "MIT", "engines": { @@ -10456,9 +10335,9 @@ } }, "node_modules/typescript": { - "version": "5.8.2", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.2.tgz", - "integrity": "sha512-aJn6wq13/afZp/jT9QZmwEjDqqvSGp1VT5GVg+f/t6/oVyrgXM6BY1h9BRh/O5p3PlUPAe+WuiEZOmb/49RqoQ==", + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", "bin": { @@ -10470,15 +10349,16 @@ } }, "node_modules/typescript-eslint": { - "version": "8.26.1", - "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.26.1.tgz", - "integrity": "sha512-t/oIs9mYyrwZGRpDv3g+3K6nZ5uhKEMt2oNmAPwaY4/ye0+EH4nXIPYNtkYFS6QHm+1DFg34DbglYBz5P9Xysg==", + "version": "8.48.0", + "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.48.0.tgz", + "integrity": "sha512-fcKOvQD9GUn3Xw63EgiDqhvWJ5jsyZUaekl3KVpGsDJnN46WJTe3jWxtQP9lMZm1LJNkFLlTaWAxK2vUQR+cqw==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/eslint-plugin": "8.26.1", - "@typescript-eslint/parser": "8.26.1", - "@typescript-eslint/utils": "8.26.1" + "@typescript-eslint/eslint-plugin": "8.48.0", + "@typescript-eslint/parser": "8.48.0", + "@typescript-eslint/typescript-estree": "8.48.0", + "@typescript-eslint/utils": "8.48.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -10489,7 +10369,7 @@ }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0", - "typescript": ">=4.8.4 <5.9.0" + "typescript": ">=4.8.4 <6.0.0" } }, "node_modules/ufo": { @@ -10817,34 +10697,6 @@ "url": "https://opencollective.com/vitest" } }, - "node_modules/vite/node_modules/fdir": { - "version": "6.4.4", - "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.4.4.tgz", - "integrity": "sha512-1NZP+GK4GfuAv3PqKvxQRDMjdSRZjnkq7KfhlNrCNNlZ0ygQFpebfrnfnq/W7fpUnAv9aGWmY1zKx7FYL3gwhg==", - "dev": true, - "license": "MIT", - "peerDependencies": { - "picomatch": "^3 || ^4" - }, - "peerDependenciesMeta": { - "picomatch": { - "optional": true - } - } - }, - "node_modules/vite/node_modules/picomatch": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", - "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, "node_modules/vitepress": { "version": "1.6.3", "resolved": "https://registry.npmjs.org/vitepress/-/vitepress-1.6.3.tgz", @@ -10911,22 +10763,6 @@ "url": "https://github.com/okineadev/vitepress-plugin-llms?sponsor=1" } }, - "node_modules/vitepress-plugin-llms/node_modules/minimatch": { - "version": "10.0.3", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.0.3.tgz", - "integrity": "sha512-IPZ167aShDZZUMdRk66cyQAW3qr0WzbHkPdMYa8bzZhlHhO3jALbKdxcaak7W9FfT2rZNpQuUu4Od7ILEpXSaw==", - "dev": true, - "license": "ISC", - "dependencies": { - "@isaacs/brace-expansion": "^5.0.0" - }, - "engines": { - "node": "20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/vitepress-plugin-mermaid": { "version": "2.0.17", "resolved": "https://registry.npmjs.org/vitepress-plugin-mermaid/-/vitepress-plugin-mermaid-2.0.17.tgz", @@ -11503,19 +11339,6 @@ } } }, - "node_modules/vitest/node_modules/picomatch": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", - "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, "node_modules/vscode-jsonrpc": { "version": "8.2.0", "resolved": "https://registry.npmjs.org/vscode-jsonrpc/-/vscode-jsonrpc-8.2.0.tgz", @@ -12242,7 +12065,7 @@ }, "packages/mp3-encoder": { "name": "@mediabunny/mp3-encoder", - "version": "1.24.5", + "version": "1.27.2", "license": "MPL-2.0", "devDependencies": { "@types/emscripten": "^1.40.1" diff --git a/package.json b/package.json index 3b11949..208366d 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "mediabunny", "author": "Vanilagy", - "version": "1.24.5", + "version": "1.27.2", "description": "Pure TypeScript media toolkit for reading, writing, and converting media files, directly in the browser.", "type": "module", "workspaces": [ @@ -70,7 +70,7 @@ "@eslint/js": "^9.22.0", "@fontsource-variable/rubik": "^5.2.6", "@fontsource/dm-mono": "^5.2.6", - "@microsoft/api-extractor": "^7.52.1", + "@microsoft/api-extractor": "^7.55.1", "@stylistic/eslint-plugin": "^4.2.0", "@tailwindcss/vite": "^4.1.7", "@types/markdown-it-footnote": "^3.0.4", @@ -84,8 +84,8 @@ "mermaid": "^11.6.0", "tailwindcss": "^4.1.7", "tsx": "^4.19.4", - "typescript": "^5.8.2", - "typescript-eslint": "^8.26.1", + "typescript": "^5.9.3", + "typescript-eslint": "^8.48.0", "vite": "^6.3.5", "vitepress": "^1.6.3", "vitepress-plugin-llms": "^1.5.1", diff --git a/packages/mp3-encoder/package.json b/packages/mp3-encoder/package.json index b7d133b..d4f239b 100644 --- a/packages/mp3-encoder/package.json +++ b/packages/mp3-encoder/package.json @@ -1,7 +1,7 @@ { "name": "@mediabunny/mp3-encoder", "author": "Vanilagy", - "version": "1.24.5", + "version": "1.27.2", "description": "MP3 encoder extension for Mediabunny, based on LAME.", "main": "./dist/bundles/mediabunny-mp3-encoder.mjs", "module": "./dist/bundles/mediabunny-mp3-encoder.mjs", diff --git a/packages/mp3-encoder/src/index.ts b/packages/mp3-encoder/src/index.ts index 92aedbc..5c99658 100644 --- a/packages/mp3-encoder/src/index.ts +++ b/packages/mp3-encoder/src/index.ts @@ -22,7 +22,7 @@ class Mp3Encoder extends CustomAudioEncoder { private buffer = new Uint8Array(2 ** 16); private currentBufferOffset = 0; - private currentTimestamp = 0; + private currentTimestamp: number | null = null; private chunkMetadata: EncodedAudioChunkMetadata = {}; static override supports(codec: AudioCodec, config: AudioDecoderConfig): boolean { @@ -79,6 +79,11 @@ class Mp3Encoder extends CustomAudioEncoder { } async encode(audioSample: AudioSample) { + if (this.currentTimestamp === null) { + // The first sample's timestamp determines where we start + this.currentTimestamp = audioSample.timestamp; + } + const sizePerChannel = audioSample.allocationSize({ format: 's16-planar', planeIndex: 0, @@ -123,6 +128,8 @@ class Mp3Encoder extends CustomAudioEncoder { * these chunks and extract the MP3 frames only when they're complete. */ private digestOutput(bytes: Uint8Array) { + assert(this.currentTimestamp !== null); + const requiredBufferSize = this.currentBufferOffset + bytes.length; if (requiredBufferSize > this.buffer.length) { // Grow the buffer to the required size diff --git a/src/adts/adts-demuxer.ts b/src/adts/adts-demuxer.ts index 209365e..3295a9a 100644 --- a/src/adts/adts-demuxer.ts +++ b/src/adts/adts-demuxer.ts @@ -21,6 +21,7 @@ import { } from '../misc'; import { EncodedPacket, PLACEHOLDER_DATA } from '../packet'; import { readBytes, Reader } from '../reader'; +import { DEFAULT_TRACK_DISPOSITION } from '../metadata'; import { FrameHeader, MAX_FRAME_HEADER_SIZE, MIN_FRAME_HEADER_SIZE, readFrameHeader } from './adts-reader'; const SAMPLES_PER_AAC_FRAME = 1024; @@ -188,6 +189,12 @@ class AdtsAudioTrackBacking implements InputAudioTrackBacking { return sampleRate; } + getDisposition() { + return { + ...DEFAULT_TRACK_DISPOSITION, + }; + } + async getDecoderConfig(): Promise { assert(this.demuxer.firstFrameHeader); diff --git a/src/codec-data.ts b/src/codec-data.ts index 377fdea..4882752 100644 --- a/src/codec-data.ts +++ b/src/codec-data.ts @@ -22,9 +22,12 @@ import { textEncoder, toDataView, toUint8Array, + getChromiumVersion, + isChromium, + setUint24, } from './misc'; import { PacketType } from './packet'; -import { MetadataTags } from './tags'; +import { MetadataTags } from './metadata'; // References for AVC/HEVC code: // ISO 14496-15 @@ -34,6 +37,7 @@ import { MetadataTags } from './tags'; export enum AvcNalUnitType { IDR = 5, + SEI = 6, SPS = 7, PPS = 8, SPS_EXT = 13, @@ -158,38 +162,54 @@ const removeEmulationPreventionBytes = (data: Uint8Array) => { return new Uint8Array(result); }; -/** Converts an AVC packet in Annex B format to length-prefixed format. */ -export const transformAnnexBToLengthPrefixed = (packetData: Uint8Array) => { - const NAL_UNIT_LENGTH_SIZE = 4; +const ANNEX_B_START_CODE = new Uint8Array([0, 0, 0, 1]); - const nalUnits = findNalUnitsInAnnexB(packetData); - - if (nalUnits.length === 0) { - // If no NAL units were found, it's not valid Annex B data - return null; - } - - let totalSize = 0; - for (const nalUnit of nalUnits) { - totalSize += NAL_UNIT_LENGTH_SIZE + nalUnit.byteLength; - } - - const avccData = new Uint8Array(totalSize); - const dataView = new DataView(avccData.buffer); +export const concatNalUnitsInAnnexB = (nalUnits: Uint8Array[]) => { + const totalLength = nalUnits.reduce((a, b) => a + ANNEX_B_START_CODE.byteLength + b.byteLength, 0); + const result = new Uint8Array(totalLength); let offset = 0; - // Write each NAL unit with its length prefix for (const nalUnit of nalUnits) { - const length = nalUnit.byteLength; + result.set(ANNEX_B_START_CODE, offset); + offset += ANNEX_B_START_CODE.byteLength; - dataView.setUint32(offset, length, false); - offset += 4; - - avccData.set(nalUnit, offset); + result.set(nalUnit, offset); offset += nalUnit.byteLength; } - return avccData; + return result; +}; + +export const concatNalUnitsInLengthPrefixed = (nalUnits: Uint8Array[], lengthSize: 1 | 2 | 3 | 4) => { + const totalLength = nalUnits.reduce((a, b) => a + lengthSize + b.byteLength, 0); + const result = new Uint8Array(totalLength); + let offset = 0; + + for (const nalUnit of nalUnits) { + const dataView = new DataView(result.buffer, result.byteOffset, result.byteLength); + + switch (lengthSize) { + case 1: + dataView.setUint8(offset, nalUnit.byteLength); + break; + case 2: + dataView.setUint16(offset, nalUnit.byteLength, false); + break; + case 3: + setUint24(dataView, offset, nalUnit.byteLength, false); + break; + case 4: + dataView.setUint32(offset, nalUnit.byteLength, false); + break; + } + + offset += lengthSize; + + result.set(nalUnit, offset); + offset += nalUnit.byteLength; + } + + return result; }; // Data specified in ISO 14496-15 @@ -224,12 +244,27 @@ export const extractAvcNalUnits = (packetData: Uint8Array, decoderConfig: VideoD } }; -const extractNalUnitTypeForAvc = (data: Uint8Array) => { +export const concatAvcNalUnits = (nalUnits: Uint8Array[], decoderConfig: VideoDecoderConfig) => { + if (decoderConfig.description) { + // Stream is length-prefixed. Let's extract the size of the length prefix from the decoder config + + const bytes = toUint8Array(decoderConfig.description); + const lengthSizeMinusOne = bytes[4]! & 0b11; + const lengthSize = (lengthSizeMinusOne + 1) as 1 | 2 | 3 | 4; + + return concatNalUnitsInLengthPrefixed(nalUnits, lengthSize); + } else { + // Stream is in Annex B format + return concatNalUnitsInAnnexB(nalUnits); + } +}; + +export const extractNalUnitTypeForAvc = (data: Uint8Array) => { return data[0]! & 0x1F; }; /** Builds an AvcDecoderConfigurationRecord from an AVC packet in Annex B format. */ -export const extractAvcDecoderConfigurationRecord = (packetData: Uint8Array) => { +export const extractAvcDecoderConfigurationRecord = (packetData: Uint8Array): AvcDecoderConfigurationRecord | null => { try { const nalUnits = findNalUnitsInAnnexB(packetData); @@ -247,60 +282,27 @@ export const extractAvcDecoderConfigurationRecord = (packetData: Uint8Array) => // Let's get the first SPS for profile and level information const spsData = spsUnits[0]!; - const bitstream = new Bitstream(removeEmulationPreventionBytes(spsData)); + const spsInfo = parseAvcSps(spsData); + assert(spsInfo !== null); - bitstream.skipBits(1); // forbidden_zero_bit - bitstream.skipBits(2); // nal_ref_idc - const nal_unit_type = bitstream.readBits(5); + const hasExtendedData = spsInfo.profileIdc === 100 + || spsInfo.profileIdc === 110 + || spsInfo.profileIdc === 122 + || spsInfo.profileIdc === 144; - if (nal_unit_type !== 7) { // SPS NAL unit type is 7 - console.error('Invalid SPS NAL unit type'); - return null; - } - - const profile_idc = bitstream.readAlignedByte(); - const constraint_flags = bitstream.readAlignedByte(); - const level_idc = bitstream.readAlignedByte(); - - const record: AvcDecoderConfigurationRecord = { + return { configurationVersion: 1, - avcProfileIndication: profile_idc, - profileCompatibility: constraint_flags, - avcLevelIndication: level_idc, + avcProfileIndication: spsInfo.profileIdc, + profileCompatibility: spsInfo.constraintFlags, + avcLevelIndication: spsInfo.levelIdc, lengthSizeMinusOne: 3, // Typically 4 bytes for length field sequenceParameterSets: spsUnits, pictureParameterSets: ppsUnits, - chromaFormat: null, - bitDepthLumaMinus8: null, - bitDepthChromaMinus8: null, - sequenceParameterSetExt: null, + chromaFormat: hasExtendedData ? spsInfo.chromaFormatIdc : null, + bitDepthLumaMinus8: hasExtendedData ? spsInfo.bitDepthLumaMinus8 : null, + bitDepthChromaMinus8: hasExtendedData ? spsInfo.bitDepthChromaMinus8 : null, + sequenceParameterSetExt: hasExtendedData ? spsExtUnits : null, }; - - if ( - profile_idc === 100 - || profile_idc === 110 - || profile_idc === 122 - || profile_idc === 144 - ) { - readExpGolomb(bitstream); // seq_parameter_set_id - - const chroma_format_idc = readExpGolomb(bitstream); - - if (chroma_format_idc === 3) { - bitstream.skipBits(1); // separate_colour_plane_flag - } - - const bit_depth_luma_minus8 = readExpGolomb(bitstream); - - const bit_depth_chroma_minus8 = readExpGolomb(bitstream); - - record.chromaFormat = chroma_format_idc; - record.bitDepthLumaMinus8 = bit_depth_luma_minus8; - record.bitDepthChromaMinus8 = bit_depth_chroma_minus8; - record.sequenceParameterSetExt = spsExtUnits; - } - - return record; } catch (error) { console.error('Error building AVC Decoder Configuration Record:', error); return null; @@ -377,6 +379,206 @@ export const serializeAvcDecoderConfigurationRecord = (record: AvcDecoderConfigu return new Uint8Array(bytes); }; +/** Deserializes an AvcDecoderConfigurationRecord from the format specified in Section 5.3.3.1 of ISO 14496-15. */ +export const deserializeAvcDecoderConfigurationRecord = (data: Uint8Array): AvcDecoderConfigurationRecord | null => { + try { + const view = toDataView(data); + let offset = 0; + + // Read header + const configurationVersion = view.getUint8(offset++); + const avcProfileIndication = view.getUint8(offset++); + const profileCompatibility = view.getUint8(offset++); + const avcLevelIndication = view.getUint8(offset++); + const lengthSizeMinusOne = view.getUint8(offset++) & 0x03; + + const numOfSequenceParameterSets = view.getUint8(offset++) & 0x1F; + + // Read SPS + const sequenceParameterSets: Uint8Array[] = []; + for (let i = 0; i < numOfSequenceParameterSets; i++) { + const length = view.getUint16(offset, false); + offset += 2; + + sequenceParameterSets.push(data.subarray(offset, offset + length)); + offset += length; + } + + const numOfPictureParameterSets = view.getUint8(offset++); + + // Read PPS + const pictureParameterSets: Uint8Array[] = []; + for (let i = 0; i < numOfPictureParameterSets; i++) { + const length = view.getUint16(offset, false); + offset += 2; + + pictureParameterSets.push(data.subarray(offset, offset + length)); + offset += length; + } + + const record: AvcDecoderConfigurationRecord = { + configurationVersion, + avcProfileIndication, + profileCompatibility, + avcLevelIndication, + lengthSizeMinusOne, + sequenceParameterSets, + pictureParameterSets, + chromaFormat: null, + bitDepthLumaMinus8: null, + bitDepthChromaMinus8: null, + sequenceParameterSetExt: null, + }; + + // Check if there are extended profile fields + if ( + ( + avcProfileIndication === 100 + || avcProfileIndication === 110 + || avcProfileIndication === 122 + || avcProfileIndication === 144 + ) + && offset + 4 <= data.length + ) { + const chromaFormat = view.getUint8(offset++) & 0x03; + const bitDepthLumaMinus8 = view.getUint8(offset++) & 0x07; + const bitDepthChromaMinus8 = view.getUint8(offset++) & 0x07; + const numOfSequenceParameterSetExt = view.getUint8(offset++); + + record.chromaFormat = chromaFormat; + record.bitDepthLumaMinus8 = bitDepthLumaMinus8; + record.bitDepthChromaMinus8 = bitDepthChromaMinus8; + + // Read SPS Ext + const sequenceParameterSetExt: Uint8Array[] = []; + for (let i = 0; i < numOfSequenceParameterSetExt; i++) { + const length = view.getUint16(offset, false); + offset += 2; + + sequenceParameterSetExt.push(data.subarray(offset, offset + length)); + offset += length; + } + + record.sequenceParameterSetExt = sequenceParameterSetExt; + } + + return record; + } catch (error) { + console.error('Error deserializing AVC Decoder Configuration Record:', error); + return null; + } +}; + +export type AvcSpsInfo = { + profileIdc: number; + constraintFlags: number; + levelIdc: number; + frameMbsOnlyFlag: number; + chromaFormatIdc: number | null; + bitDepthLumaMinus8: number | null; + bitDepthChromaMinus8: number | null; +}; + +/** Parses an AVC SPS (Sequence Parameter Set) to extract basic information. */ +export const parseAvcSps = (sps: Uint8Array): AvcSpsInfo | null => { + try { + const bitstream = new Bitstream(removeEmulationPreventionBytes(sps)); + + bitstream.skipBits(1); // forbidden_zero_bit + bitstream.skipBits(2); // nal_ref_idc + const nalUnitType = bitstream.readBits(5); + + if (nalUnitType !== 7) { // SPS NAL unit type is 7 + return null; + } + + const profileIdc = bitstream.readAlignedByte(); + const constraintFlags = bitstream.readAlignedByte(); + const levelIdc = bitstream.readAlignedByte(); + + readExpGolomb(bitstream); // seq_parameter_set_id + + let chromaFormatIdc: number | null = null; + let bitDepthLumaMinus8: number | null = null; + let bitDepthChromaMinus8: number | null = null; + + // Handle high profile chroma_format_idc + if ( + profileIdc === 100 + || profileIdc === 110 + || profileIdc === 122 + || profileIdc === 244 + || profileIdc === 44 + || profileIdc === 83 + || profileIdc === 86 + || profileIdc === 118 + || profileIdc === 128 + ) { + chromaFormatIdc = readExpGolomb(bitstream); + if (chromaFormatIdc === 3) { + bitstream.skipBits(1); // separate_colour_plane_flag + } + bitDepthLumaMinus8 = readExpGolomb(bitstream); + bitDepthChromaMinus8 = readExpGolomb(bitstream); + bitstream.skipBits(1); // qpprime_y_zero_transform_bypass_flag + const seqScalingMatrixPresentFlag = bitstream.readBits(1); + if (seqScalingMatrixPresentFlag) { + for (let i = 0; i < (chromaFormatIdc !== 3 ? 8 : 12); i++) { + const seqScalingListPresentFlag = bitstream.readBits(1); + if (seqScalingListPresentFlag) { + const sizeOfScalingList = i < 6 ? 16 : 64; + let lastScale = 8; + let nextScale = 8; + for (let j = 0; j < sizeOfScalingList; j++) { + if (nextScale !== 0) { + const deltaScale = readSignedExpGolomb(bitstream); + nextScale = (lastScale + deltaScale + 256) % 256; + } + lastScale = nextScale === 0 ? lastScale : nextScale; + } + } + } + } + } + + readExpGolomb(bitstream); // log2_max_frame_num_minus4 + + const picOrderCntType = readExpGolomb(bitstream); + if (picOrderCntType === 0) { + readExpGolomb(bitstream); // log2_max_pic_order_cnt_lsb_minus4 + } else if (picOrderCntType === 1) { + bitstream.skipBits(1); // delta_pic_order_always_zero_flag + readSignedExpGolomb(bitstream); // offset_for_non_ref_pic + readSignedExpGolomb(bitstream); // offset_for_top_to_bottom_field + const numRefFramesInPicOrderCntCycle = readExpGolomb(bitstream); + for (let i = 0; i < numRefFramesInPicOrderCntCycle; i++) { + readSignedExpGolomb(bitstream); // offset_for_ref_frame[i] + } + } + + readExpGolomb(bitstream); // max_num_ref_frames + bitstream.skipBits(1); // gaps_in_frame_num_value_allowed_flag + + readExpGolomb(bitstream); // pic_width_in_mbs_minus1 + readExpGolomb(bitstream); // pic_height_in_map_units_minus1 + + const frameMbsOnlyFlag = bitstream.readBits(1); + + return { + profileIdc, + constraintFlags, + levelIdc, + frameMbsOnlyFlag, + chromaFormatIdc, + bitDepthLumaMinus8, + bitDepthChromaMinus8, + }; + } catch (error) { + console.error('Error parsing AVC SPS:', error); + return null; + } +}; + // Data specified in ISO 14496-15 export type HevcDecoderConfigurationRecord = { configurationVersion: number; @@ -423,9 +625,7 @@ export const extractNalUnitTypeForHevc = (data: Uint8Array) => { }; /** Builds a HevcDecoderConfigurationRecord from an HEVC packet in Annex B format. */ -export const extractHevcDecoderConfigurationRecord = ( - packetData: Uint8Array, -) => { +export const extractHevcDecoderConfigurationRecord = (packetData: Uint8Array) => { try { const nalUnits = findNalUnitsInAnnexB(packetData); @@ -1289,6 +1489,69 @@ export const extractAv1CodecInfoFromPacket = ( } } + // Frame size + const frameWidthBitsMinus1 = bitstream.readBits(4); + const frameHeightBitsMinus1 = bitstream.readBits(4); + const n1 = frameWidthBitsMinus1 + 1; + bitstream.skipBits(n1); // max_frame_width_minus_1 + const n2 = frameHeightBitsMinus1 + 1; + bitstream.skipBits(n2); // max_frame_height_minus_1 + + // Frame IDs + let frameIdNumbersPresentFlag = 0; + if (reducedStillPictureHeader) { + frameIdNumbersPresentFlag = 0; + } else { + frameIdNumbersPresentFlag = bitstream.readBits(1); + } + + if (frameIdNumbersPresentFlag) { + bitstream.skipBits(4); // delta_frame_id_length_minus_2 + bitstream.skipBits(3); // additional_frame_id_length_minus_1 + } + + bitstream.skipBits(1); // use_128x128_superblock + bitstream.skipBits(1); // enable_filter_intra + bitstream.skipBits(1); // enable_intra_edge_filter + + if (!reducedStillPictureHeader) { + bitstream.skipBits(1); // enable_interintra_compound + bitstream.skipBits(1); // enable_masked_compound + bitstream.skipBits(1); // enable_warped_motion + bitstream.skipBits(1); // enable_dual_filter + const enableOrderHint = bitstream.readBits(1); + + if (enableOrderHint) { + bitstream.skipBits(1); // enable_jnt_comp + bitstream.skipBits(1); // enable_ref_frame_mvs + } + + const seqChooseScreenContentTools = bitstream.readBits(1); + let seqForceScreenContentTools = 0; + + if (seqChooseScreenContentTools) { + seqForceScreenContentTools = 2; // SELECT_SCREEN_CONTENT_TOOLS + } else { + seqForceScreenContentTools = bitstream.readBits(1); + } + + if (seqForceScreenContentTools > 0) { + const seqChooseIntegerMv = bitstream.readBits(1); + if (!seqChooseIntegerMv) { + bitstream.skipBits(1); // seq_force_integer_mv + } + } + + if (enableOrderHint) { + bitstream.skipBits(3); // order_hint_bits_minus_1 + } + } + + bitstream.skipBits(1); // enable_superres + bitstream.skipBits(1); // enable_cdef + bitstream.skipBits(1); // enable_restoration + + // color_config() const highBitdepth = bitstream.readBits(1); let bitDepth = 8; @@ -1486,7 +1749,68 @@ export const determineVideoPacketType = ( switch (codec) { case 'avc': { const nalUnits = extractAvcNalUnits(packetData, decoderConfig); - const isKeyframe = nalUnits.some(x => extractNalUnitTypeForAvc(x) === AvcNalUnitType.IDR); + let isKeyframe = nalUnits.some(x => extractNalUnitTypeForAvc(x) === AvcNalUnitType.IDR); + + if (!isKeyframe && (!isChromium() || getChromiumVersion()! >= 144)) { + // In addition to IDR, Recovery Point SEI also counts as a valid H.264 keyframe by current consensus. + // See https://github.com/w3c/webcodecs/issues/650 for the relevant discussion. WebKit and Firefox have + // always supported them, but Chromium hasn't, therefore the (admittedly dirty) version check. + + for (const nalUnit of nalUnits) { + const type = extractNalUnitTypeForAvc(nalUnit); + if (type !== AvcNalUnitType.SEI) { + continue; + } + + const bytes = removeEmulationPreventionBytes(nalUnit); + let pos = 1; // Skip NALU header + + // sei_rbsp() + do { + // sei_message() + let payloadType = 0; + while (true) { + const nextByte = bytes[pos++]; + if (nextByte === undefined) break; + payloadType += nextByte; + + if (nextByte < 255) { + break; + } + } + + let payloadSize = 0; + while (true) { + const nextByte = bytes[pos++]; + if (nextByte === undefined) break; + payloadSize += nextByte; + + if (nextByte < 255) { + break; + } + } + + // sei_payload() + const PAYLOAD_TYPE_RECOVERY_POINT = 6; + if (payloadType === PAYLOAD_TYPE_RECOVERY_POINT) { + const bitstream = new Bitstream(bytes); + bitstream.pos = 8 * pos; + + const recoveryFrameCount = readExpGolomb(bitstream); + const exactMatchFlag = bitstream.readBits(1); + + if (recoveryFrameCount === 0 && exactMatchFlag === 1) { + // https://github.com/w3c/webcodecs/pull/910 + // "recovery_frame_cnt == 0 and exact_match_flag=1 in the SEI recovery payload" + isKeyframe = true; + break; + } + } + + pos += payloadSize; + } while (pos < bytes.length - 1); + } + } return isKeyframe ? 'key' : 'delta'; }; diff --git a/src/codec.ts b/src/codec.ts index a396e70..b9414fd 100644 --- a/src/codec.ts +++ b/src/codec.ts @@ -337,6 +337,7 @@ export const extractVideoCodecString = (trackInfo: { codec: VideoCodec | null; codecDescription: Uint8Array | null; colorSpace: VideoColorSpaceInit | null; + avcType: 1 | 3 | null; avcCodecInfo: AvcDecoderConfigurationRecord | null; hevcCodecInfo: HevcDecoderConfigurationRecord | null; vp9CodecInfo: Vp9CodecInfo | null; @@ -345,6 +346,8 @@ export const extractVideoCodecString = (trackInfo: { const { codec, codecDescription, colorSpace, avcCodecInfo, hevcCodecInfo, vp9CodecInfo, av1CodecInfo } = trackInfo; if (codec === 'avc') { + assert(trackInfo.avcType !== null); + if (avcCodecInfo) { const bytes = new Uint8Array([ avcCodecInfo.avcProfileIndication, @@ -352,14 +355,14 @@ export const extractVideoCodecString = (trackInfo: { avcCodecInfo.avcLevelIndication, ]); - return `avc1.${bytesToHexString(bytes)}`; + return `avc${trackInfo.avcType}.${bytesToHexString(bytes)}`; } if (!codecDescription || codecDescription.byteLength < 4) { throw new TypeError('AVC decoder description is not provided or is not at least 4 bytes long.'); } - return `avc1.${bytesToHexString(codecDescription.subarray(1, 4))}`; + return `avc${trackInfo.avcType}.${bytesToHexString(codecDescription.subarray(1, 4))}`; } else if (codec === 'hevc') { let generalProfileSpace: number; let generalProfileIdc: number; diff --git a/src/conversion.ts b/src/conversion.ts index 0cabe1a..862b6f0 100644 --- a/src/conversion.ts +++ b/src/conversion.ts @@ -48,7 +48,7 @@ import { import { Output, TrackType } from './output'; import { Mp4OutputFormat } from './output-format'; import { AudioSample, clampCropRectangle, validateCropRectangle, VideoSample } from './sample'; -import { MetadataTags, validateMetadataTags } from './tags'; +import { MetadataTags, validateMetadataTags } from './metadata'; import { NullTarget } from './target'; /** @@ -82,10 +82,16 @@ export type ConversionOptions = { /** Options to trim the input file. */ trim?: { - /** The time in the input file in seconds at which the output file should start. Must be less than `end`. */ - start: number; - /** The time in the input file in seconds at which the output file should end. Must be greater than `start`. */ - end: number; + /** + * The time in the input file in seconds at which the output file should start. Must be less than `end`. + * Defaults to 0 when omitted. + */ + start?: number; + /** + * The time in the input file in seconds at which the output file should end. Must be greater than `start`. + * Defaults to the duration of the input when omitted. + */ + end?: number; }; /** @@ -137,6 +143,12 @@ export type ConversionVideoOptions = { * This rotation is _in addition to_ the natural rotation of the input video as specified in input file's metadata. */ rotate?: Rotation; + /** + * Defaults to `true`. When enabaled, Mediabunny will use the rotation metadata in the output file to perform video + * rotation whenever possible. Set this field to `false` if you want to ensure the output file does not make use of + * rotation metadata and that any rotation is baked into the video frames directly. + */ + allowRotationMetadata?: boolean; /** * Specifies the rectangular region of the input video to crop to. The crop region will automatically be clamped to * the dimensions of the input video track. Cropping is performed after rotation but before resizing. @@ -174,6 +186,11 @@ export type ConversionVideoOptions = { * Setting this fields forces a transcode. */ keyFrameInterval?: number; + /** + * A hint that configures the hardware acceleration method used when transcoding. This is best left on + * `'no-preference'`, the default. + */ + hardwareAcceleration?: 'no-preference' | 'prefer-hardware' | 'prefer-software'; /** When `true`, video will always be re-encoded instead of directly copying over the encoded samples. */ forceTranscode?: boolean; /** @@ -298,6 +315,9 @@ const validateVideoOptions = (videoOptions: ConversionVideoOptions | undefined) if (videoOptions?.rotate !== undefined && ![0, 90, 180, 270].includes(videoOptions.rotate)) { throw new TypeError('options.video.rotate, when provided, must be 0, 90, 180 or 270.'); } + if (videoOptions?.allowRotationMetadata !== undefined && typeof videoOptions.allowRotationMetadata !== 'boolean') { + throw new TypeError('options.video.allowRotationMetadata, when provided, must be a boolean.'); + } if (videoOptions?.crop !== undefined) { validateCropRectangle(videoOptions.crop, 'options.video.'); } @@ -331,6 +351,15 @@ const validateVideoOptions = (videoOptions: ConversionVideoOptions | undefined) ) { throw new TypeError('options.video.processedHeight, when provided, must be a positive integer.'); } + if ( + videoOptions?.hardwareAcceleration !== undefined + && !['no-preference', 'prefer-hardware', 'prefer-software'].includes(videoOptions.hardwareAcceleration) + ) { + throw new TypeError( + 'options.video.hardwareAcceleration, when provided, must be \'no-preference\', \'prefer-hardware\' or' + + ' \'prefer-software\'.', + ); + } }; const validateAudioOptions = (audioOptions: ConversionAudioOptions | undefined) => { @@ -832,7 +861,8 @@ export class Conversion { let videoSource: VideoSource; const totalRotation = normalizeRotation(track.rotation + (trackOptions.rotate ?? 0)); - const outputSupportsRotation = this.output.format.supportsVideoRotationMetadata; + const canUseRotationMetadata = this.output.format.supportsVideoRotationMetadata + && (trackOptions.allowRotationMetadata ?? true); const [rotatedWidth, rotatedHeight] = totalRotation % 180 === 0 ? [track.codedWidth, track.codedHeight] @@ -877,7 +907,7 @@ export class Conversion { // TODO This is suboptimal: Forcing a rerender when both rotation and process are set is not // performance-optimal, but right now there's no other way because we can't change the track rotation // metadata after the output has already started. Should be possible with API changes in v2, though! - || (totalRotation !== 0 && (!outputSupportsRotation || trackOptions.process !== undefined)) + || (totalRotation !== 0 && (!canUseRotationMetadata || trackOptions.process !== undefined)) || !!crop; const alpha = trackOptions.alpha ?? 'discard'; @@ -968,6 +998,7 @@ export class Conversion { keyFrameInterval: trackOptions.keyFrameInterval, sizeChangeBehavior: trackOptions.fit ?? 'passThrough', alpha, + hardwareAcceleration: trackOptions.hardwareAcceleration, }; const source = new VideoSampleSource(encodingConfig); @@ -1043,6 +1074,7 @@ export class Conversion { duration: 1 / frameRate, }); await this._registerVideoSample(track, trackOptions, source, sample); + sample.close(); } }; @@ -1079,12 +1111,11 @@ export class Conversion { duration: frameRate !== undefined ? 1 / frameRate : duration, }); await this._registerVideoSample(track, trackOptions, source, sample); + sample.close(); if (frameRate !== undefined) { lastCanvas = canvas; lastCanvasTimestamp = adjustedSampleTimestamp; - } else { - sample.close(); } } @@ -1184,9 +1215,10 @@ export class Conversion { this.output.addVideoTrack(videoSource, { frameRate: trackOptions.frameRate, - // TEMP: This condition can be removed when all demuxers properly homogenize to BCP47 in v2 + // TODO: This condition can be removed when all demuxers properly homogenize to BCP47 in v2 languageCode: isIso639Dash2LanguageCode(track.languageCode) ? track.languageCode : undefined, name: track.name ?? undefined, + disposition: track.disposition, rotation: needsRerender ? 0 : totalRotation, // Rerendering will bake the rotation into the output }); this._addedCounts.video++; @@ -1423,9 +1455,10 @@ export class Conversion { } this.output.addAudioTrack(audioSource, { - // TEMP: This condition can be removed when all demuxers properly homogenize to BCP47 in v2 + // TODO: This condition can be removed when all demuxers properly homogenize to BCP47 in v2 languageCode: isIso639Dash2LanguageCode(track.languageCode) ? track.languageCode : undefined, name: track.name ?? undefined, + disposition: track.disposition, }); this._addedCounts.audio++; this._totalTrackCount++; @@ -1507,7 +1540,10 @@ export class Conversion { targetSampleRate, startTime: this._startTimestamp, endTime: this._endTimestamp, - onSample: sample => this._registerAudioSample(track, trackOptions, source, sample), + onSample: async (sample) => { + await this._registerAudioSample(track, trackOptions, source, sample); + sample.close(); + }, }); const sink = new AudioSampleSink(track); @@ -1519,6 +1555,7 @@ export class Conversion { } await resampler.add(sample); + sample.close(); } await resampler.finalize(); diff --git a/src/demuxer.ts b/src/demuxer.ts index f654dab..5b53fb2 100644 --- a/src/demuxer.ts +++ b/src/demuxer.ts @@ -8,7 +8,7 @@ import { Input } from './input'; import { InputTrack } from './input-track'; -import { MetadataTags } from './tags'; +import { MetadataTags } from './metadata'; export abstract class Demuxer { input: Input; diff --git a/src/encode.ts b/src/encode.ts index bf7834b..9f07a69 100644 --- a/src/encode.ts +++ b/src/encode.ts @@ -22,6 +22,7 @@ import { VideoCodec, } from './codec'; import { customAudioEncoders, customVideoEncoders } from './custom-coder'; +import { isFirefox } from './misc'; import { EncodedPacket } from './packet'; /** @@ -115,9 +116,15 @@ export type VideoEncodingAdditionalOptions = { * format that supports transparency (such as WebM or Matroska). */ alpha?: 'discard' | 'keep'; - /** Configures the bitrate mode. */ + /** Configures the bitrate mode; defaults to `'variable'`. */ bitrateMode?: 'constant' | 'variable'; - /** The latency mode used by the encoder; controls the performance-quality tradeoff. */ + /** + * The latency mode used by the encoder; controls the performance-quality tradeoff. + * + * - `'quality'` (default): The encoder prioritizes quality over latency, and no frames can be dropped. + * - `'realtime'`: The encoder prioritizes low latency over quality, and may drop frames if the encoder becomes + * overloaded to keep up with real-time requirements. + */ latencyMode?: 'quality' | 'realtime'; /** * The full codec string as specified in the WebCodecs Codec Registry. This string must match the codec @@ -125,7 +132,8 @@ export type VideoEncodingAdditionalOptions = { */ fullCodecString?: string; /** - * A hint that configures the hardware acceleration method of this codec. This is best left on `'no-preference'`. + * A hint that configures the hardware acceleration method of this codec. This is best left on `'no-preference'`, + * the default. */ hardwareAcceleration?: 'no-preference' | 'prefer-hardware' | 'prefer-software'; /** @@ -403,31 +411,31 @@ export class Quality { * @group Encoding * @public */ -export const QUALITY_VERY_LOW = new Quality(0.3); +export const QUALITY_VERY_LOW = /* #__PURE__ */ new Quality(0.3); /** * Represents a low media quality. * @group Encoding * @public */ -export const QUALITY_LOW = new Quality(0.6); +export const QUALITY_LOW = /* #__PURE__ */ new Quality(0.6); /** * Represents a medium media quality. * @group Encoding * @public */ -export const QUALITY_MEDIUM = new Quality(1); +export const QUALITY_MEDIUM = /* #__PURE__ */ new Quality(1); /** * Represents a high media quality. * @group Encoding * @public */ -export const QUALITY_HIGH = new Quality(2); +export const QUALITY_HIGH = /* #__PURE__ */ new Quality(2); /** * Represents a very high media quality. * @group Encoding * @public */ -export const QUALITY_VERY_HIGH = new Quality(4); +export const QUALITY_VERY_HIGH = /* #__PURE__ */ new Quality(4); /** * Checks if the browser is able to encode the given codec. @@ -522,7 +530,45 @@ export const canEncodeVideo = async ( }); const support = await VideoEncoder.isConfigSupported(encoderConfig); - return support.supported === true; + if (!support.supported) { + return false; + } + + if (isFirefox()) { + // isConfigSupported on Firefox appears to unreliably indicate if encoding will actually succeed. Therefore, we + // just try encoding a frame to see if it actually works. + // https://github.com/Vanilagy/mediabunny/issues/222 + + // eslint-disable-next-line @typescript-eslint/no-misused-promises, no-async-promise-executor + return new Promise(async (resolve) => { + try { + const encoder = new VideoEncoder({ + output: () => {}, + error: () => resolve(false), + }); + encoder.configure(encoderConfig); + + const frameData = new Uint8Array(width * height * 4); + const frame = new VideoFrame(frameData, { + format: 'RGBA', + codedWidth: width, + codedHeight: height, + timestamp: 0, + }); + + encoder.encode(frame); + frame.close(); + + await encoder.flush(); + + resolve(true); + } catch { + resolve(false); + } + }); + } else { + return true; + } }; /** diff --git a/src/flac/flac-demuxer.ts b/src/flac/flac-demuxer.ts index 5248394..e362f44 100644 --- a/src/flac/flac-demuxer.ts +++ b/src/flac/flac-demuxer.ts @@ -28,7 +28,7 @@ import { readU32Be, readU8, } from '../reader'; -import { MetadataTags } from '../tags'; +import { DEFAULT_TRACK_DISPOSITION, MetadataTags } from '../metadata'; import { calculateCrc8, readBlockSize, @@ -328,11 +328,13 @@ export class FlacDemuxer extends Demuxer { const nextByte = readU8(slice); if (nextByte === 0xff) { + const positionBeforeReading = slice.filePos; + const byteAfterNextByte = readU8(slice); const expected = this.blockingBit === 1 ? 0b1111_1001 : 0b1111_1000; if (byteAfterNextByte !== expected) { - slice.skip(-1); + slice.filePos = positionBeforeReading; continue; } @@ -345,7 +347,7 @@ export class FlacDemuxer extends Demuxer { }); if (!nextFrameHeader) { - slice.skip(-1); + slice.filePos = positionBeforeReading; continue; } @@ -355,14 +357,14 @@ export class FlacDemuxer extends Demuxer { if (this.blockingBit === 0) { // Case A: If the stream is fixed block size, this is the frame number, which increments by 1 if (nextFrameHeader.num - frameHeader.num !== 1) { - slice.skip(-1); + slice.filePos = positionBeforeReading; continue; } } else { // Case B: If the stream is variable block size, this is the sample number, which increments by // amount of samples in a frame. if (nextFrameHeader.num - frameHeader.num !== frameHeader.blockSize) { - slice.skip(-1); + slice.filePos = positionBeforeReading; continue; } } @@ -563,6 +565,12 @@ class FlacAudioTrackBacking implements InputAudioTrackBacking { return this.demuxer.audioInfo.sampleRate; } + getDisposition() { + return { + ...DEFAULT_TRACK_DISPOSITION, + }; + } + async getFirstTimestamp() { return 0; } diff --git a/src/flac/flac-muxer.ts b/src/flac/flac-muxer.ts index fdda1d0..b444f0b 100644 --- a/src/flac/flac-muxer.ts +++ b/src/flac/flac-muxer.ts @@ -20,7 +20,7 @@ import { Output, OutputAudioTrack } from '../output'; import { FlacOutputFormat } from '../output-format'; import { EncodedPacket } from '../packet'; import { FileSlice, readBytes } from '../reader'; -import { AttachedImage, metadataTagsAreEmpty } from '../tags'; +import { AttachedImage, metadataTagsAreEmpty } from '../metadata'; import { Writer } from '../writer'; import { readBlockSize, @@ -28,7 +28,7 @@ import { readCodedNumber, } from './flac-misc'; -const FLAC_HEADER = new Uint8Array([0x66, 0x4c, 0x61, 0x43]); // 'fLaC' +const FLAC_HEADER = /* #__PURE__ */ new Uint8Array([0x66, 0x4c, 0x61, 0x43]); // 'fLaC' const STREAMINFO_SIZE = 38; const STREAMINFO_BLOCK_SIZE = 34; diff --git a/src/id3.ts b/src/id3.ts index f231176..15db50b 100644 --- a/src/id3.ts +++ b/src/id3.ts @@ -7,7 +7,7 @@ */ import { decodeSynchsafe, encodeSynchsafe } from '../shared/mp3-misc'; -import { MetadataTags } from './tags'; +import { MetadataTags } from './metadata'; import { coalesceIndex, textDecoder, diff --git a/src/index.ts b/src/index.ts index e3abe7a..6cd723f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -168,7 +168,10 @@ export { AudioSampleCopyToOptions, VideoSample, VideoSampleInit, + VideoSamplePixelFormat, + VideoSampleColorSpace, CropRectangle, + VIDEO_SAMPLE_PIXEL_FORMATS, } from './sample'; export { AudioBufferSink, @@ -202,6 +205,7 @@ export { AttachedImage, RichImageData, AttachedFile, -} from './tags'; + TrackDisposition, +} from './metadata'; // 🐡🦔 diff --git a/src/input-format.ts b/src/input-format.ts index ed57825..1748f27 100644 --- a/src/input-format.ts +++ b/src/input-format.ts @@ -481,56 +481,56 @@ export class AdtsInputFormat extends InputFormat { * @group Input formats * @public */ -export const MP4 = new Mp4InputFormat(); +export const MP4 = /* #__PURE__ */ new Mp4InputFormat(); /** * QuickTime File Format input format singleton. * @group Input formats * @public */ -export const QTFF = new QuickTimeInputFormat(); +export const QTFF = /* #__PURE__ */ new QuickTimeInputFormat(); /** * Matroska input format singleton. * @group Input formats * @public */ -export const MATROSKA = new MatroskaInputFormat(); +export const MATROSKA = /* #__PURE__ */ new MatroskaInputFormat(); /** * WebM input format singleton. * @group Input formats * @public */ -export const WEBM = new WebMInputFormat(); +export const WEBM = /* #__PURE__ */ new WebMInputFormat(); /** * MP3 input format singleton. * @group Input formats * @public */ -export const MP3 = new Mp3InputFormat(); +export const MP3 = /* #__PURE__ */ new Mp3InputFormat(); /** * WAVE input format singleton. * @group Input formats * @public */ -export const WAVE = new WaveInputFormat(); +export const WAVE = /* #__PURE__ */ new WaveInputFormat(); /** * Ogg input format singleton. * @group Input formats * @public */ -export const OGG = new OggInputFormat(); +export const OGG = /* #__PURE__ */ new OggInputFormat(); /** * ADTS input format singleton. * @group Input formats * @public */ -export const ADTS = new AdtsInputFormat(); +export const ADTS = /* #__PURE__ */ new AdtsInputFormat(); /** * FLAC input format singleton. * @group Input formats * @public */ -export const FLAC = new FlacInputFormat(); +export const FLAC = /* #__PURE__ */ new FlacInputFormat(); /** * List of all input format singletons. If you don't need to support all input formats, you should specify the diff --git a/src/input-track.ts b/src/input-track.ts index 25b5100..8c137fc 100644 --- a/src/input-track.ts +++ b/src/input-track.ts @@ -14,6 +14,7 @@ import { EncodedPacketSink, PacketRetrievalOptions } from './media-sink'; import { assert, Rotation } from './misc'; import { TrackType } from './output'; import { EncodedPacket, PacketType } from './packet'; +import { TrackDisposition } from './metadata'; /** * Contains aggregate statistics about the encoded packets of a track. @@ -36,6 +37,7 @@ export interface InputTrackBacking { getName(): string | null; getLanguageCode(): string; getTimeResolution(): number; + getDisposition(): TrackDisposition; getFirstTimestamp(): Promise; computeDuration(): Promise; @@ -128,6 +130,11 @@ export abstract class InputTrack { return this._backing.getTimeResolution(); } + /** The track's disposition, i.e. information about its intended usage. */ + get disposition() { + return this._backing.getDisposition(); + } + /** * Returns the start timestamp of the first packet of this track, in seconds. While often near zero, this value * may be positive or even negative. A negative starting timestamp means the track's timing has been offset. Samples diff --git a/src/isobmff/isobmff-boxes.ts b/src/isobmff/isobmff-boxes.ts index 96900d0..0079451 100644 --- a/src/isobmff/isobmff-boxes.ts +++ b/src/isobmff/isobmff-boxes.ts @@ -44,7 +44,7 @@ import { Sample, } from './isobmff-muxer'; import { parseOpusIdentificationHeader } from '../codec-data'; -import { MetadataTags, RichImageData } from '../tags'; +import { MetadataTags, RichImageData } from '../metadata'; export class IsobmffBoxWriter { private helper = new Uint8Array(8); @@ -135,8 +135,8 @@ export class IsobmffBoxWriter { } } -const bytes = new Uint8Array(8); -const view = new DataView(bytes.buffer); +const bytes = /* #__PURE__ */ new Uint8Array(8); +const view = /* #__PURE__ */ new DataView(bytes.buffer); const u8 = (value: number) => { return [(value % 0x100 + 0x100) % 0x100]; @@ -243,7 +243,7 @@ const rotationMatrix = (rotationInDegrees: number): TransformationMatrix => { 0, 0, 1, ]; }; -const IDENTITY_MATRIX = rotationMatrix(0); +const IDENTITY_MATRIX = /* #__PURE__ */ rotationMatrix(0); const matrixToBytes = (matrix: TransformationMatrix) => { return [ @@ -421,7 +421,12 @@ export const tkhd = ( matrix = IDENTITY_MATRIX; } - return fullBox('tkhd', +needsU64, 3, [ + let flags = 0x2; // Track in movie + if (trackData.track.metadata.disposition?.default !== false) { + flags |= 0x1; // Track enabled + } + + return fullBox('tkhd', +needsU64, flags, [ u32OrU64(creationTime), // Creation time u32OrU64(creationTime), // Modification time u32(trackData.track.id), // Track ID @@ -577,7 +582,7 @@ export const stsd = (trackData: IsobmffTrackData) => { if (trackData.type === 'video') { sampleDescription = videoSampleDescription( - VIDEO_CODEC_TO_BOX_NAME[trackData.track.source._codec], + videoCodecToBoxName(trackData.track.source._codec, trackData.info.decoderConfig.codec), trackData, ); } else if (trackData.type === 'audio') { @@ -845,7 +850,7 @@ export const dOps = (trackData: IsobmffAudioTrackData) => { let inputSampleRate = trackData.info.sampleRate; let outputGain = 0; let channelMappingFamily = 0; - let channelMappingTable = new Uint8Array(0); + let channelMappingTable: Uint8Array = new Uint8Array(0); // Read preskip and from codec private data from the encoder // https://www.rfc-editor.org/rfc/rfc7845#section-5 @@ -1572,12 +1577,14 @@ const dataStringBoxLong = (value: string) => { ]); }; -const VIDEO_CODEC_TO_BOX_NAME: Record = { - avc: 'avc1', - hevc: 'hvc1', - vp8: 'vp08', - vp9: 'vp09', - av1: 'av01', +const videoCodecToBoxName = (codec: VideoCodec, fullCodecString: string) => { + switch (codec) { + case 'avc': return fullCodecString.startsWith('avc3') ? 'avc3' : 'avc1'; + case 'hevc': return 'hvc1'; + case 'vp8': return 'vp08'; + case 'vp9': return 'vp09'; + case 'av1': return 'av01'; + } }; const VIDEO_CODEC_TO_CONFIGURATION_BOX: Record Box | null> = { diff --git a/src/isobmff/isobmff-demuxer.ts b/src/isobmff/isobmff-demuxer.ts index e1d047c..b524bb2 100644 --- a/src/isobmff/isobmff-demuxer.ts +++ b/src/isobmff/isobmff-demuxer.ts @@ -51,13 +51,13 @@ import { MATRIX_COEFFICIENTS_MAP_INVERSE, normalizeRotation, roundToMultiple, - roundToPrecision, Rotation, textDecoder, TransformationMatrix, TRANSFER_CHARACTERISTICS_MAP_INVERSE, UNDETERMINED_LANGUAGE, toDataView, + roundIfAlmostInteger, } from '../misc'; import { EncodedPacket, PLACEHOLDER_DATA } from '../packet'; import { buildIsobmffMimeType } from './isobmff-misc'; @@ -86,12 +86,13 @@ import { readU8, readAscii, } from '../reader'; -import { MetadataTags, RichImageData } from '../tags'; +import { DEFAULT_TRACK_DISPOSITION, MetadataTags, RichImageData, TrackDisposition } from '../metadata'; type InternalTrack = { id: number; demuxer: IsobmffDemuxer; inputTrack: InputTrack | null; + disposition: TrackDisposition; timescale: number; durationInMovieTimescale: number; durationInMediaTimescale: number; @@ -126,6 +127,7 @@ type InternalTrack = { codec: VideoCodec | null; codecDescription: Uint8Array | null; colorSpace: VideoColorSpaceInit | null; + avcType: 1 | 3 | null; avcCodecInfo: AvcDecoderConfigurationRecord | null; hevcCodecInfo: HevcDecoderConfigurationRecord | null; vp9CodecInfo: Vp9CodecInfo | null; @@ -314,6 +316,9 @@ export class IsobmffDemuxer extends Demuxer { this.moovSlice = moovSlice; this.readContiguousBoxes(this.moovSlice); + // Put default tracks first + this.tracks.sort((a, b) => Number(b.disposition.default) - Number(a.disposition.default)); + for (const track of this.tracks) { // Modify the edit list offset based on the previous segment durations. They are in different // timescales, so we first convert to seconds and then into the track timescale. @@ -650,6 +655,9 @@ export class IsobmffDemuxer extends Demuxer { id: -1, demuxer: this, inputTrack: null, + disposition: { + ...DEFAULT_TRACK_DISPOSITION, + }, info: null, timescale: -1, durationInMovieTimescale: -1, @@ -694,10 +702,10 @@ export class IsobmffDemuxer extends Demuxer { const version = readU8(slice); const flags = readU24Be(slice); - const trackEnabled = (flags & 0x1) !== 0; - if (!trackEnabled) { - break; - } + // Spec says disabled tracks are to be treated like they don't exist, but in practice, they are treated + // more like non-default tracks. + const trackEnabled = !!(flags & 0x1); + track.disposition.default = trackEnabled; // Skip over creation & modification time to reach the track ID if (version === 0) { @@ -836,6 +844,7 @@ export class IsobmffDemuxer extends Demuxer { codec: null, codecDescription: null, colorSpace: null, + avcType: null, avcCodecInfo: null, hevcCodecInfo: null, vp9CodecInfo: null, @@ -890,8 +899,9 @@ export class IsobmffDemuxer extends Demuxer { const lowercaseBoxName = sampleBoxInfo.name.toLowerCase(); if (track.info.type === 'video') { - if (lowercaseBoxName === 'avc1') { + if (lowercaseBoxName === 'avc1' || lowercaseBoxName === 'avc3') { track.info.codec = 'avc'; + track.info.avcType = lowercaseBoxName === 'avc1' ? 1 : 3; } else if (lowercaseBoxName === 'hvc1' || lowercaseBoxName === 'hev1') { track.info.codec = 'hevc'; } else if (lowercaseBoxName === 'vp08') { @@ -2342,6 +2352,10 @@ abstract class IsobmffTrackBacking implements InputTrackBacking { return this.internalTrack.timescale; } + getDisposition() { + return this.internalTrack.disposition; + } + async computeDuration() { const lastPacket = await this.getPacket(Infinity, { metadataOnly: true }); return (lastPacket?.timestamp ?? 0) + (lastPacket?.duration ?? 0); @@ -2385,7 +2399,7 @@ abstract class IsobmffTrackBacking implements InputTrackBacking { // Do a little rounding to catch cases where the result is very close to an integer. If it is, it's likely // that the number was originally an integer divided by the timescale. For stability, it's best // to return the integer in this case. - return roundToPrecision(timestamp * this.internalTrack.timescale, 14) + this.internalTrack.editListOffset; + return roundIfAlmostInteger(timestamp * this.internalTrack.timescale) + this.internalTrack.editListOffset; } async getPacket(timestamp: number, options: PacketRetrievalOptions) { @@ -2475,11 +2489,8 @@ abstract class IsobmffTrackBacking implements InputTrackBacking { const timestampInTimescale = this.mapTimestampIntoTimescale(timestamp); const sampleTable = this.internalTrack.demuxer.getSampleTableForTrack(this.internalTrack); - const sampleIndex = getSampleIndexForTimestamp(sampleTable, timestampInTimescale); - const keyFrameSampleIndex = sampleIndex === -1 - ? -1 - : getRelevantKeyframeIndexForSample(sampleTable, sampleIndex); - const regularPacket = await this.fetchPacketForSampleIndex(keyFrameSampleIndex, options); + const sampleIndex = getKeyframeSampleIndexForTimestamp(sampleTable, timestampInTimescale); + const regularPacket = await this.fetchPacketForSampleIndex(sampleIndex, options); if (!sampleTableIsEmpty(sampleTable) || !this.internalTrack.demuxer.isFragmented) { // Prefer the non-fragmented packet @@ -2899,7 +2910,45 @@ const getSampleIndexForTimestamp = (sampleTable: SampleTable, timescaleUnits: nu const entry = sampleTable.sampleTimingEntries[index]!; return entry.startIndex - + Math.min(Math.floor((timescaleUnits - entry.startDecodeTimestamp) / entry.delta), entry.count - 1); + + Math.min( + Math.floor((timescaleUnits - entry.startDecodeTimestamp) / entry.delta), + entry.count - 1, + ); + } +}; + +const getKeyframeSampleIndexForTimestamp = (sampleTable: SampleTable, timescaleUnits: number) => { + if (!sampleTable.keySampleIndices) { + // Every sample is a keyframe + return getSampleIndexForTimestamp(sampleTable, timescaleUnits); + } + + if (sampleTable.presentationTimestamps) { + const index = binarySearchLessOrEqual( + sampleTable.presentationTimestamps, + timescaleUnits, + x => x.presentationTimestamp, + ); + if (index === -1) { + return -1; + } + + // Walk the samples in presentation order until we find one that's a keyframe + for (let i = index; i >= 0; i--) { + const sampleIndex = sampleTable.presentationTimestamps[i]!.sampleIndex; + const isKeyFrame = binarySearchExact(sampleTable.keySampleIndices, sampleIndex, x => x) !== -1; + + if (isKeyFrame) { + return sampleIndex; + } + } + + return -1; + } else { + const sampleIndex = getSampleIndexForTimestamp(sampleTable, timescaleUnits); + + const index = binarySearchLessOrEqual(sampleTable.keySampleIndices, sampleIndex, x => x); + return sampleTable.keySampleIndices[index] ?? -1; } }; @@ -2988,15 +3037,6 @@ const getSampleInfo = (sampleTable: SampleTable, sampleIndex: number): SampleInf }; }; -const getRelevantKeyframeIndexForSample = (sampleTable: SampleTable, sampleIndex: number) => { - if (!sampleTable.keySampleIndices) { - return sampleIndex; - } - - const index = binarySearchLessOrEqual(sampleTable.keySampleIndices, sampleIndex, x => x); - return sampleTable.keySampleIndices[index] ?? -1; -}; - const getNextKeyframeIndexForSample = (sampleTable: SampleTable, sampleIndex: number) => { if (!sampleTable.keySampleIndices) { return sampleIndex + 1; diff --git a/src/isobmff/isobmff-muxer.ts b/src/isobmff/isobmff-muxer.ts index e866f9d..fd73b50 100644 --- a/src/isobmff/isobmff-muxer.ts +++ b/src/isobmff/isobmff-muxer.ts @@ -25,11 +25,12 @@ import { import { BufferTarget } from '../target'; import { EncodedPacket, PacketType } from '../packet'; import { + concatNalUnitsInLengthPrefixed, extractAvcDecoderConfigurationRecord, extractHevcDecoderConfigurationRecord, + findNalUnitsInAnnexB, serializeAvcDecoderConfigurationRecord, serializeHevcDecoderConfigurationRecord, - transformAnnexBToLengthPrefixed, } from '../codec-data'; import { buildIsobmffMimeType } from './isobmff-misc'; import { MAX_BOX_HEADER_SIZE, MIN_BOX_HEADER_SIZE } from './isobmff-reader'; @@ -464,15 +465,18 @@ export class IsobmffMuxer extends Muxer { let packetData = packet.data; if (trackData.info.requiresAnnexBTransformation) { - const transformedData = transformAnnexBToLengthPrefixed(packetData); - if (!transformedData) { + const nalUnits = findNalUnitsInAnnexB(packetData); + if (nalUnits.length === 0) { + // It's not valid Annex B data throw new Error( 'Failed to transform packet data. Make sure all packets are provided in Annex B format, as' + ' specified in ITU-T-REC-H.264 and ITU-T-REC-H.265.', ); } - packetData = transformedData; + // We don't strip things like SPS or PPS NALUs here, mainly because they can also appear in the middle + // of a stream and potentially modify the parameters of it. So, let's just leave them in to be sure. + packetData = concatNalUnitsInLengthPrefixed(nalUnits, 4); } const timestamp = this.validateAndNormalizeTimestamp( diff --git a/src/isobmff/isobmff-reader.ts b/src/isobmff/isobmff-reader.ts index 67e0ab8..2579fc9 100644 --- a/src/isobmff/isobmff-reader.ts +++ b/src/isobmff/isobmff-reader.ts @@ -6,7 +6,7 @@ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -import { RichImageData } from '../tags'; +import { RichImageData } from '../metadata'; import { textDecoder } from '../misc'; import { FileSlice, readAscii, readBytes, readI32Be, readU16Be, readU32Be, readU64Be, readU8 } from '../reader'; diff --git a/src/matroska/ebml.ts b/src/matroska/ebml.ts index 4fc70bd..eb27c40 100644 --- a/src/matroska/ebml.ts +++ b/src/matroska/ebml.ts @@ -88,6 +88,10 @@ export enum EBMLId { FlagEnabled = 0xb9, FlagDefault = 0x88, FlagForced = 0x55aa, + FlagOriginal = 0x55ae, + FlagHearingImpaired = 0x55ab, + FlagVisualImpaired = 0x55ac, + FlagCommentary = 0x55af, FlagLacing = 0x9c, Name = 0x536e, Language = 0x22b59c, diff --git a/src/matroska/matroska-demuxer.ts b/src/matroska/matroska-demuxer.ts index e964bef..db421b6 100644 --- a/src/matroska/matroska-demuxer.ts +++ b/src/matroska/matroska-demuxer.ts @@ -31,7 +31,7 @@ import { InputVideoTrack, InputVideoTrackBacking, } from '../input-track'; -import { AttachedFile, MetadataTags } from '../tags'; +import { AttachedFile, DEFAULT_TRACK_DISPOSITION, MetadataTags, TrackDisposition } from '../metadata'; import { PacketRetrievalOptions } from '../media-sink'; import { assert, @@ -43,7 +43,7 @@ import { MATRIX_COEFFICIENTS_MAP_INVERSE, normalizeRotation, Rotation, - roundToPrecision, + roundIfAlmostInteger, TRANSFER_CHARACTERISTICS_MAP_INVERSE, UNDETERMINED_LANGUAGE, } from '../misc'; @@ -61,7 +61,6 @@ import { readElementHeader, readElementId, readFloat, - readSignedInt, readUnsignedInt, readVarInt, resync, @@ -137,7 +136,6 @@ type ClusterBlock = { timestamp: number; duration: number; isKeyFrame: boolean; - referencedTimestamps: number[]; data: Uint8Array; lacing: BlockLacing; decoded: boolean; @@ -190,7 +188,7 @@ type InternalTrack = { }[]; cuePoints: CuePoint[]; - isDefault: boolean; + disposition: TrackDisposition; inputTrack: InputTrack | null; codecId: string | null; codecPrivate: Uint8Array | null; @@ -539,7 +537,7 @@ export class MatroskaDemuxer extends Demuxer { } // Put default tracks first - this.currentSegment.tracks.sort((a, b) => Number(b.isDefault) - Number(a.isDefault)); + this.currentSegment.tracks.sort((a, b) => Number(b.disposition.default) - Number(a.disposition.default)); // Now, let's distribute the cue points to the tracks const idToTrack = new Map(this.currentSegment.tracks.map(x => [x.id, x])); @@ -649,21 +647,15 @@ export class MatroskaDemuxer extends Demuxer { // This must hold, as track datas only get created if a block for that track is encountered assert(trackData.blocks.length > 0); - let blockReferencesExist = false; let hasLacedBlocks = false; for (let i = 0; i < trackData.blocks.length; i++) { const block = trackData.blocks[i]!; block.timestamp += cluster.timestamp; - blockReferencesExist ||= block.referencedTimestamps.length > 0; hasLacedBlocks ||= block.lacing !== BlockLacing.None; } - if (blockReferencesExist) { - trackData.blocks = sortBlocksByReferences(trackData.blocks); - } - trackData.presentationTimestamps = trackData.blocks .map((block, i) => ({ timestamp: block.timestamp, blockIndex: i })) .sort((a, b) => a.timestamp - b.timestamp); @@ -859,7 +851,6 @@ export class MatroskaDemuxer extends Demuxer { timestamp: frameTimestamp, duration: frameDuration, isKeyFrame: originalBlock.isKeyFrame, - referencedTimestamps: originalBlock.referencedTimestamps, data: frameData, lacing: BlockLacing.None, decoded: true, @@ -998,7 +989,9 @@ export class MatroskaDemuxer extends Demuxer { clusterPositionCache: [], cuePoints: [], - isDefault: false, + disposition: { + ...DEFAULT_TRACK_DISPOSITION, + }, inputTrack: null, codecId: null, codecPrivate: null, @@ -1163,7 +1156,37 @@ export class MatroskaDemuxer extends Demuxer { case EBMLId.FlagDefault: { if (!this.currentTrack) break; - this.currentTrack.isDefault = !!readUnsignedInt(slice, size); + this.currentTrack.disposition.default = !!readUnsignedInt(slice, size); + }; break; + + case EBMLId.FlagForced: { + if (!this.currentTrack) break; + + this.currentTrack.disposition.forced = !!readUnsignedInt(slice, size); + }; break; + + case EBMLId.FlagOriginal: { + if (!this.currentTrack) break; + + this.currentTrack.disposition.original = !!readUnsignedInt(slice, size); + }; break; + + case EBMLId.FlagHearingImpaired: { + if (!this.currentTrack) break; + + this.currentTrack.disposition.hearingImpaired = !!readUnsignedInt(slice, size); + }; break; + + case EBMLId.FlagVisualImpaired: { + if (!this.currentTrack) break; + + this.currentTrack.disposition.visuallyImpaired = !!readUnsignedInt(slice, size); + }; break; + + case EBMLId.FlagCommentary: { + if (!this.currentTrack) break; + + this.currentTrack.disposition.commentary = !!readUnsignedInt(slice, size); }; break; case EBMLId.CodecID: { @@ -1383,9 +1406,17 @@ export class MatroskaDemuxer extends Demuxer { const relativeTimestamp = readI16Be(slice); const flags = readU8(slice); - const isKeyFrame = !!(flags & 0x80); const lacing = (flags >> 1) & 0x3 as BlockLacing; // If the block is laced, we'll expand it later + let isKeyFrame = !!(flags & 0x80); + if (trackData.track.info?.type === 'audio' && trackData.track.info.codec) { + // Some files don't mark their audio packets as key packets (I'm looking at you, Firefox). But, we + // can fix this in most cases: if we recognize the codec of the track, then we know every packet is + // necessarily a key packet, no matter what the container says. + // https://github.com/Vanilagy/mediabunny/issues/192 + isKeyFrame = true; + } + const blockData = readBytes(slice, size - (slice.filePos - dataStartPos)); const hasDecodingInstructions = trackData.track.decodingInstructions.length > 0; @@ -1393,7 +1424,6 @@ export class MatroskaDemuxer extends Demuxer { timestamp: relativeTimestamp, // We'll add the cluster's timestamp to this later duration: 0, // Will set later isKeyFrame, - referencedTimestamps: [], data: blockData, lacing, decoded: !hasDecodingInstructions, @@ -1406,13 +1436,7 @@ export class MatroskaDemuxer extends Demuxer { this.readContiguousElements(slice.slice(dataStartPos, size)); - if (this.currentBlock) { - for (let i = 0; i < this.currentBlock.referencedTimestamps.length; i++) { - this.currentBlock.referencedTimestamps[i]! += this.currentBlock.timestamp; - } - - this.currentBlock = null; - } + this.currentBlock = null; }; break; case EBMLId.Block: { @@ -1436,7 +1460,6 @@ export class MatroskaDemuxer extends Demuxer { timestamp: relativeTimestamp, // We'll add the cluster's timestamp to this later duration: 0, // Will set later isKeyFrame: true, - referencedTimestamps: [], data: blockData, lacing, decoded: !hasDecodingInstructions, @@ -1487,11 +1510,8 @@ export class MatroskaDemuxer extends Demuxer { if (!this.currentBlock) break; this.currentBlock.isKeyFrame = false; - - const relativeTimestamp = readSignedInt(slice, size); - - // We'll offset this by the block's timestamp later - this.currentBlock.referencedTimestamps.push(relativeTimestamp); + // We ignore the actual value here, we just use the reference as an indicator for "not a key frame". + // This is in line with FFmpeg's behavior. }; break; case EBMLId.Tag: { @@ -1859,6 +1879,10 @@ abstract class MatroskaTrackBacking implements InputTrackBacking { return this.internalTrack.segment.timestampFactor; } + getDisposition() { + return this.internalTrack.disposition; + } + async getFirstPacket(options: PacketRetrievalOptions) { return this.performClusterLookup( null, @@ -1886,7 +1910,7 @@ abstract class MatroskaTrackBacking implements InputTrackBacking { // Do a little rounding to catch cases where the result is very close to an integer. If it is, it's likely // that the number was originally an integer divided by the timescale. For stability, it's best // to return the integer in this case. - return roundToPrecision(timestamp * this.internalTrack.segment.timestampFactor, 14); + return roundIfAlmostInteger(timestamp * this.internalTrack.segment.timestampFactor); } async getPacket(timestamp: number, options: PacketRetrievalOptions) { @@ -2319,6 +2343,7 @@ class MatroskaVideoTrackBacking extends MatroskaTrackBacking implements InputVid codec: this.internalTrack.info.codec, codecDescription: this.internalTrack.info.codecDescription, colorSpace: this.internalTrack.info.colorSpace, + avcType: 1, // We don't know better (or do we?) so just assume 'avc1' avcCodecInfo: this.internalTrack.info.codec === 'avc' && firstPacket ? extractAvcDecoderConfigurationRecord(firstPacket.data) : null, @@ -2379,43 +2404,3 @@ class MatroskaAudioTrackBacking extends MatroskaTrackBacking implements InputAud }; } } - -/** Sorts blocks such that referenced blocks come before the blocks that reference them. */ -const sortBlocksByReferences = (blocks: ClusterBlock[]) => { - const timestampToBlock = new Map(); - - for (let i = 0; i < blocks.length; i++) { - const block = blocks[i]!; - timestampToBlock.set(block.timestamp, block); - } - - const processedBlocks = new Set(); - const result: ClusterBlock[] = []; - - const processBlock = (block: ClusterBlock) => { - if (processedBlocks.has(block)) { - return; - } - - // Marking the block as processed here already; prevents this algorithm from dying on cycles - processedBlocks.add(block); - - for (let j = 0; j < block.referencedTimestamps.length; j++) { - const timestamp = block.referencedTimestamps[j]!; - const otherBlock = timestampToBlock.get(timestamp); - if (!otherBlock) { - continue; - } - - processBlock(otherBlock); - } - - result.push(block); - }; - - for (let i = 0; i < blocks.length; i++) { - processBlock(blocks[i]!); - } - - return result; -}; diff --git a/src/matroska/matroska-muxer.ts b/src/matroska/matroska-muxer.ts index b12b95a..73af407 100644 --- a/src/matroska/matroska-muxer.ts +++ b/src/matroska/matroska-muxer.ts @@ -63,7 +63,7 @@ import { Muxer } from '../muxer'; import { Writer } from '../writer'; import { EncodedPacket } from '../packet'; import { parseOpusIdentificationHeader } from '../codec-data'; -import { AttachedFile } from '../tags'; +import { AttachedFile } from '../metadata'; const MIN_CLUSTER_TIMESTAMP_MS = -(2 ** 15); const MAX_CLUSTER_TIMESTAMP_MS = 2 ** 15 - 1; @@ -304,6 +304,24 @@ export class MatroskaMuxer extends Muxer { { id: EBMLId.TrackNumber, data: trackData.track.id }, { id: EBMLId.TrackUID, data: trackData.track.id }, { id: EBMLId.TrackType, data: TRACK_TYPE_MAP[trackData.type] }, + trackData.track.metadata.disposition?.default === false + ? { id: EBMLId.FlagDefault, data: 0 } + : null, + trackData.track.metadata.disposition?.forced + ? { id: EBMLId.FlagForced, data: 1 } + : null, + trackData.track.metadata.disposition?.hearingImpaired + ? { id: EBMLId.FlagHearingImpaired, data: 1 } + : null, + trackData.track.metadata.disposition?.visuallyImpaired + ? { id: EBMLId.FlagVisualImpaired, data: 1 } + : null, + trackData.track.metadata.disposition?.original + ? { id: EBMLId.FlagOriginal, data: 1 } + : null, + trackData.track.metadata.disposition?.commentary + ? { id: EBMLId.FlagCommentary, data: 1 } + : null, { id: EBMLId.FlagLacing, data: 0 }, { id: EBMLId.Language, data: trackData.track.metadata.languageCode ?? UNDETERMINED_LANGUAGE }, { id: EBMLId.CodecID, data: codecId }, diff --git a/src/media-sink.ts b/src/media-sink.ts index 195ff2a..539606e 100644 --- a/src/media-sink.ts +++ b/src/media-sink.ts @@ -8,10 +8,15 @@ import { parsePcmCodec, PCM_AUDIO_CODECS, PcmAudioCodec, VideoCodec, AudioCodec } from './codec'; import { + concatAvcNalUnits, + deserializeAvcDecoderConfigurationRecord, determineVideoPacketType, + extractAvcNalUnits, extractHevcNalUnits, + extractNalUnitTypeForAvc, extractNalUnitTypeForHevc, HevcNalUnitType, + parseAvcSps, } from './codec-data'; import { CustomVideoDecoder, customVideoDecoders, CustomAudioDecoder, customAudioDecoders } from './custom-coder'; import { InputDisposedError } from './input'; @@ -24,6 +29,7 @@ import { getInt24, getUint24, insertSorted, + isChromium, isFirefox, isNumber, isWebKit, @@ -33,6 +39,7 @@ import { Rotation, toAsyncIterator, toDataView, + toUint8Array, validateAnyIterable, } from './misc'; import { EncodedPacket } from './packet'; @@ -197,9 +204,10 @@ export class EncodedPacketSink { } const packet = await this._track._backing.getKeyPacket(timestamp, options); - if (!packet || packet.type === 'delta') { + if (!packet) { return packet; } + assert(packet.type === 'key'); const determinedType = await this._track.determinePacketType(packet); if (determinedType === 'delta') { @@ -231,9 +239,10 @@ export class EncodedPacketSink { } const nextPacket = await this._track._backing.getNextKeyPacket(packet, options); - if (!nextPacket || nextPacket.type === 'delta') { + if (!nextPacket) { return nextPacket; } + assert(nextPacket.type === 'key'); const determinedType = await this._track.determinePacketType(nextPacket); if (determinedType === 'delta') { @@ -423,7 +432,6 @@ export abstract class BaseMediaSampleSink< // The following is the "pump" process that keeps pumping packets into the decoder (async () => { - const decoderError = new Error(); const decoder = await this._createDecoder((sample) => { onQueueDequeue(); if (sample.timestamp >= endTimestamp) { @@ -461,7 +469,6 @@ export abstract class BaseMediaSampleSink< } }, (error) => { if (!outOfBandError) { - error.stack = decoderError.stack; // Provide a more useful stack trace outOfBandError = error; onQueueNotEmpty(); } @@ -470,9 +477,6 @@ export abstract class BaseMediaSampleSink< const packetSink = this._createPacketSink(); const keyPacket = await packetSink.getKeyPacket(startTimestamp, { verifyKeyPackets: true }) ?? await packetSink.getFirstPacket(); - if (!keyPacket) { - return; - } let currentPacket: EncodedPacket | null = keyPacket; @@ -494,7 +498,7 @@ export abstract class BaseMediaSampleSink< } } - const packets = packetSink.packets(keyPacket, endPacket); + const packets = packetSink.packets(keyPacket ?? undefined, endPacket); await packets.next(); // Skip the start packet as we already have it while (currentPacket && !ended && !this._track.input._disposed) { @@ -610,7 +614,6 @@ export abstract class BaseMediaSampleSink< // The following is the "pump" process that keeps pumping packets into the decoder (async () => { - const decoderError = new Error(); const decoder = await this._createDecoder((sample) => { onQueueDequeue(); @@ -638,7 +641,6 @@ export abstract class BaseMediaSampleSink< } }, (error) => { if (!outOfBandError) { - error.stack = decoderError.stack; // Provide a more useful stack trace outOfBandError = error; onQueueNotEmpty(); } @@ -872,6 +874,24 @@ class VideoDecoderWrapper extends DecoderWrapper { } }; + if (codec === 'avc' && this.decoderConfig.description && isChromium()) { + // Chromium has/had a bug with playing interlaced AVC (https://issues.chromium.org/issues/456919096) + // which can be worked around by requesting that software decoding be used. So, here we peek into the + // AVC description, if present, and switch to software decoding if we find interlaced content. + const record = deserializeAvcDecoderConfigurationRecord(toUint8Array(this.decoderConfig.description)); + if (record && record.sequenceParameterSets.length > 0) { + const sps = parseAvcSps(record.sequenceParameterSets[0]!); + if (sps && sps.frameMbsOnlyFlag === 0) { + this.decoderConfig = { + ...this.decoderConfig, + hardwareAcceleration: 'prefer-software', + }; + } + } + } + + const stack = new Error('Decoding error').stack; + this.decoder = new VideoDecoder({ output: (frame) => { try { @@ -880,9 +900,12 @@ class VideoDecoderWrapper extends DecoderWrapper { this.onError(error as Error); } }, - error: onError, + error: (error) => { + error.stack = stack; // Provide a more useful stack trace, the default one sucks + this.onError(error); + }, }); - this.decoder.configure(decoderConfig); + this.decoder.configure(this.decoderConfig); } } @@ -908,8 +931,6 @@ class VideoDecoderWrapper extends DecoderWrapper { this.raslSkipped = true; } - this.currentPacketIndex++; - if (this.customDecoder) { this.customDecoderQueueSize++; void this.customDecoderCallSerializer @@ -922,9 +943,24 @@ class VideoDecoderWrapper extends DecoderWrapper { insertSorted(this.inputTimestamps, packet.timestamp, x => x); } + // Workaround for https://issues.chromium.org/issues/470109459 + if (isChromium() && this.currentPacketIndex === 0 && this.codec === 'avc') { + const nalUnits = extractAvcNalUnits(packet.data, this.decoderConfig); + const filteredNalUnits = nalUnits.filter((x) => { + const type = extractNalUnitTypeForAvc(x); + // These trip up Chromium's key frame detection, so let's strip them + return !(type >= 20 && type <= 31); + }); + + const newData = concatAvcNalUnits(filteredNalUnits, this.decoderConfig); + packet = new EncodedPacket(newData, packet.type, packet.timestamp, packet.duration); + } + this.decoder.decode(packet.toEncodedVideoChunk()); this.decodeAlphaData(packet); } + + this.currentPacketIndex++; } decodeAlphaData(packet: EncodedPacket) { @@ -980,6 +1016,8 @@ class VideoDecoderWrapper extends DecoderWrapper { } }; + const stack = new Error('Decoding error').stack; + this.alphaDecoder = new VideoDecoder({ output: (frame) => { try { @@ -988,7 +1026,10 @@ class VideoDecoderWrapper extends DecoderWrapper { this.onError(error as Error); } }, - error: this.onError, + error: (error) => { + error.stack = stack; // Provide a more useful stack trace, the default one sucks + this.onError(error); + }, }); this.alphaDecoder.configure(this.decoderConfig); } @@ -1744,6 +1785,8 @@ class AudioDecoderWrapper extends DecoderWrapper { void this.customDecoderCallSerializer.call(() => this.customDecoder!.init()); } else { + const stack = new Error('Decoding error').stack; + this.decoder = new AudioDecoder({ output: (data) => { try { @@ -1752,7 +1795,10 @@ class AudioDecoderWrapper extends DecoderWrapper { this.onError(error as Error); } }, - error: onError, + error: (error) => { + error.stack = stack; // Provide a more useful stack trace, the default one sucks + this.onError(error); + }, }); this.decoder.configure(decoderConfig); } @@ -2107,11 +2153,14 @@ export class AudioBufferSink { /** @internal */ _audioSampleToWrappedArrayBuffer(sample: AudioSample): WrappedAudioBuffer { - return { + const result: WrappedAudioBuffer = { buffer: sample.toAudioBuffer(), timestamp: sample.timestamp, duration: sample.duration, }; + + sample.close(); + return result; } /** diff --git a/src/media-source.ts b/src/media-source.ts index 86989a2..7b5cae6 100644 --- a/src/media-source.ts +++ b/src/media-source.ts @@ -1055,6 +1055,12 @@ export class MediaStreamVideoTrackSource extends VideoSource { private _promiseWithResolvers = promiseWithResolvers(); /** @internal */ private _errorPromiseAccessed = false; + /** @internal */ + private _paused = false; + /** @internal */ + private _lastSampleTimestamp: number | null = null; + /** @internal */ + private _pauseOffset = 0; /** A promise that rejects upon any error within this source. This promise never resolves. */ get errorPromise() { @@ -1062,6 +1068,11 @@ export class MediaStreamVideoTrackSource extends VideoSource { return this._promiseWithResolvers.promise; } + /** Whether this source is currently paused as a result of calling `.pause()`. */ + get paused() { + return this._paused; + } + /** * Creates a new {@link MediaStreamVideoTrackSource} from a * [`MediaStreamVideoTrack`](https://developer.mozilla.org/en-US/docs/Web/API/MediaStreamTrack), which will pull @@ -1103,8 +1114,29 @@ export class MediaStreamVideoTrackSource extends VideoSource { return; } + const currentTimestamp = videoFrame.timestamp / 1e6; + + if (this._paused) { + const frameSeen = firstVideoFrameTimestamp !== null; + if (frameSeen) { + if (this._lastSampleTimestamp !== null) { + // In addition to dropping this frame, let's also keep track of the time we have lost due to the + // pause. Doing it like this instead of simply keeping track of the paused time is better since + // it retains the frame rate of the underlying source. + const timeDelta = currentTimestamp - this._lastSampleTimestamp; + // We modify this field instead of _timestampOffset since we still might have data in flight + // in the encoder, with which we don't want to mess. + this._pauseOffset -= timeDelta; + } + this._lastSampleTimestamp = currentTimestamp; + } + + videoFrame.close(); + return; + } + if (firstVideoFrameTimestamp === null) { - firstVideoFrameTimestamp = videoFrame.timestamp / 1e6; + firstVideoFrameTimestamp = currentTimestamp; const muxer = this._connectedTrack!.output._muxer; if (muxer.firstMediaStreamTimestamp === null) { @@ -1116,13 +1148,19 @@ export class MediaStreamVideoTrackSource extends VideoSource { } } + this._lastSampleTimestamp = currentTimestamp; + if (this._encoder.getQueueSize() >= 4) { // Drop frames if the encoder is overloaded videoFrame.close(); return; } - void this._encoder.add(new VideoSample(videoFrame), true) + const sample = new VideoSample(videoFrame, { + timestamp: currentTimestamp + this._pauseOffset, + }); + + void this._encoder.add(sample, true) .catch((error) => { errored = true; @@ -1182,6 +1220,19 @@ export class MediaStreamVideoTrackSource extends VideoSource { } } + /** + * Pauses the capture of video frames - any video frames emitted by the underlying media stream will be ignored + * while paused. This does *not* close the underlying `MediaStreamVideoTrack`, it just ignores its output. + */ + pause() { + this._paused = true; + } + + /** Resumes the capture of video frames after being paused. */ + resume() { + this._paused = false; + } + /** @internal */ override async _flushAndClose(forceClose: boolean) { if (this._abortController) { @@ -1294,6 +1345,8 @@ class AudioEncoderWrapper { private customEncoderCallSerializer = new CallSerializer(); private customEncoderQueueSize = 0; + private lastEndSampleIndex: number | null = null; + /** * Encoders typically throw their errors "out of band", meaning asynchronously in some other execution context. * However, we want to surface these errors to the user within the normal control flow, so they don't go uncaught. @@ -1342,6 +1395,41 @@ class AudioEncoderWrapper { } assert(this.encoderInitialized); + // Handle padding of gaps with silence to avoid audio drift over time, like in + // https://github.com/Vanilagy/mediabunny/issues/176 + // TODO An open question is how encoders deal with the first AudioData having a non-zero timestamp, and with + // AudioDatas that have an overlapping timestamp range. + { + const startSampleIndex = Math.round( + audioSample.timestamp * audioSample.sampleRate, + ); + const endSampleIndex = Math.round( + (audioSample.timestamp + audioSample.duration) * audioSample.sampleRate, + ); + + if (this.lastEndSampleIndex === null) { + this.lastEndSampleIndex = endSampleIndex; + } else { + const sampleDiff = startSampleIndex - this.lastEndSampleIndex; + + if (sampleDiff >= 64) { + // The gap is big enough, let's add a correction sample + const fillSample = new AudioSample({ + data: new Float32Array(sampleDiff * audioSample.numberOfChannels), + format: 'f32-planar', + sampleRate: audioSample.sampleRate, + numberOfChannels: audioSample.numberOfChannels, + numberOfFrames: sampleDiff, + timestamp: this.lastEndSampleIndex / audioSample.sampleRate, + }); + + await this.add(fillSample, true); // Recursive call + } + + this.lastEndSampleIndex += audioSample.numberOfFrames; + } + } + if (this.customEncoder) { this.customEncoderQueueSize++; @@ -1820,6 +1908,12 @@ export class MediaStreamAudioTrackSource extends AudioSource { private _promiseWithResolvers = promiseWithResolvers(); /** @internal */ private _errorPromiseAccessed = false; + /** @internal */ + private _paused = false; + /** @internal */ + private _lastSampleTimestamp: number | null = null; + /** @internal */ + private _pauseOffset = 0; /** A promise that rejects upon any error within this source. This promise never resolves. */ get errorPromise() { @@ -1827,6 +1921,11 @@ export class MediaStreamAudioTrackSource extends AudioSource { return this._promiseWithResolvers.promise; } + /** Whether this source is currently paused as a result of calling `.pause()`. */ + get paused() { + return this._paused; + } + /** * Creates a new {@link MediaStreamAudioTrackSource} from a `MediaStreamAudioTrack`, which will pull audio samples * from the stream in real time and encode them according to {@link AudioEncodingConfig}. @@ -1853,38 +1952,74 @@ export class MediaStreamAudioTrackSource extends AudioSource { this._abortController = new AbortController(); + let firstAudioDataTimestamp: number | null = null; + let errored = false; + + const onAudioSample = (audioSample: AudioSample) => { + if (errored) { + audioSample.close(); + return; + } + + const currentTimestamp = audioSample.timestamp; + + if (this._paused) { + const dataSeen = firstAudioDataTimestamp !== null; + if (dataSeen) { + if (this._lastSampleTimestamp !== null) { + // In addition to dropping this sample, let's also keep track of the time we have lost due to + // the pause. Doing it like this instead of simply keeping track of the paused time is better + // since it retains the sample rate of the underlying source. + const timeDelta = currentTimestamp - this._lastSampleTimestamp; + // We modify this field instead of _timestampOffset since we still might have data in flight + // in the encoder, with which we don't want to mess. + this._pauseOffset -= timeDelta; + } + this._lastSampleTimestamp = currentTimestamp; + } + + audioSample.close(); + return; + } + + if (firstAudioDataTimestamp === null) { + firstAudioDataTimestamp = audioSample.timestamp; + + const muxer = this._connectedTrack!.output._muxer; + if (muxer.firstMediaStreamTimestamp === null) { + muxer.firstMediaStreamTimestamp = performance.now() / 1000; + this._timestampOffset = -firstAudioDataTimestamp; + } else { + this._timestampOffset = (performance.now() / 1000 - muxer.firstMediaStreamTimestamp) + - firstAudioDataTimestamp; + } + } + + this._lastSampleTimestamp = currentTimestamp; + + if (this._encoder.getQueueSize() >= 4) { + // Drop data if the encoder is overloaded + audioSample.close(); + return; + } + + audioSample.setTimestamp(currentTimestamp + this._pauseOffset); + + void this._encoder.add(audioSample, true) + .catch((error) => { + errored = true; + + this._abortController?.abort(); + this._promiseWithResolvers.reject(error); + void this._audioContext?.suspend(); + }); + }; + if (typeof MediaStreamTrackProcessor !== 'undefined') { // Great, MediaStreamTrackProcessor is supported, this is the preferred way of doing things - let firstAudioDataTimestamp: number | null = null; - const processor = new MediaStreamTrackProcessor({ track: this._track }); const consumer = new WritableStream({ - write: (audioData) => { - if (firstAudioDataTimestamp === null) { - firstAudioDataTimestamp = audioData.timestamp / 1e6; - - const muxer = this._connectedTrack!.output._muxer; - if (muxer.firstMediaStreamTimestamp === null) { - muxer.firstMediaStreamTimestamp = performance.now() / 1000; - this._timestampOffset = -firstAudioDataTimestamp; - } else { - this._timestampOffset = (performance.now() / 1000 - muxer.firstMediaStreamTimestamp) - - firstAudioDataTimestamp; - } - } - - if (this._encoder.getQueueSize() >= 4) { - // Drop data if the encoder is overloaded - audioData.close(); - return; - } - - void this._encoder.add(new AudioSample(audioData), true) - .catch((error) => { - this._abortController?.abort(); - this._promiseWithResolvers.reject(error); - }); - }, + write: audioData => onAudioSample(new AudioSample(audioData)), }); processor.readable.pipeTo(consumer, { @@ -1912,7 +2047,6 @@ export class MediaStreamAudioTrackSource extends AudioSource { sourceNode.connect(this._scriptProcessorNode); this._scriptProcessorNode.connect(this._audioContext.destination); - let audioReceived = false; let totalDuration = 0; this._scriptProcessorNode.onaudioprocess = (event) => { @@ -1920,33 +2054,25 @@ export class MediaStreamAudioTrackSource extends AudioSource { totalDuration += event.inputBuffer.duration; for (const audioSample of iterator) { - if (!audioReceived) { - audioReceived = true; - - const muxer = this._connectedTrack!.output._muxer; - if (muxer.firstMediaStreamTimestamp === null) { - muxer.firstMediaStreamTimestamp = performance.now() / 1000; - } else { - this._timestampOffset = performance.now() / 1000 - muxer.firstMediaStreamTimestamp; - } - } - - if (this._encoder.getQueueSize() >= 4) { - // Drop data if the encoder is overloaded - audioSample.close(); - continue; - } - - void this._encoder.add(audioSample, true) - .catch((error) => { - void this._audioContext!.suspend(); - this._promiseWithResolvers.reject(error); - }); + onAudioSample(audioSample); } }; } } + /** + * Pauses the capture of audio data - any audio data emitted by the underlying media stream will be ignored + * while paused. This does *not* close the underlying `MediaStreamAudioTrack`, it just ignores its output. + */ + pause() { + this._paused = true; + } + + /** Resumes the capture of audio data after being paused. */ + resume() { + this._paused = false; + } + /** @internal */ override async _flushAndClose(forceClose: boolean) { if (this._abortController) { diff --git a/src/tags.ts b/src/metadata.ts similarity index 81% rename from src/tags.ts rename to src/metadata.ts index 9055efc..dc1d166 100644 --- a/src/tags.ts +++ b/src/metadata.ts @@ -259,3 +259,62 @@ export const metadataTagsAreEmpty = (tags: MetadataTags) => { && tags.comment === undefined && (tags.raw === undefined || Object.keys(tags.raw).length === 0); }; + +/** + * Specifies a track's disposition, i.e. information about its intended usage. + * @public + * @group Miscellaneous + */ +export type TrackDisposition = { + /** + * Indicates that this track is eligible for automatic selection by a player; that it is the main track among other, + * non-default tracks of the same type. + */ + default: boolean; + /** + * Indicates that players should always display this track by default, even if it goes against the user's default + * preferences. For example, a subtitle track only containing translations of foreign-language audio. + */ + forced: boolean; + /** Indicates that this track is in the content's original language. */ + original: boolean; + /** Indicates that this track contains commentary. */ + commentary: boolean; + /** Indicates that this track is intended for hearing-impaired users. */ + hearingImpaired: boolean; + /** Indicates that this track is intended for visually-impaired users. */ + visuallyImpaired: boolean; +}; + +export const DEFAULT_TRACK_DISPOSITION: TrackDisposition = { + default: true, + forced: false, + original: false, + commentary: false, + hearingImpaired: false, + visuallyImpaired: false, +}; + +export const validateTrackDisposition = (disposition: Partial) => { + if (!disposition || typeof disposition !== 'object') { + throw new TypeError('disposition must be an object.'); + } + if (disposition.default !== undefined && typeof disposition.default !== 'boolean') { + throw new TypeError('disposition.default must be a boolean.'); + } + if (disposition.forced !== undefined && typeof disposition.forced !== 'boolean') { + throw new TypeError('disposition.forced must be a boolean.'); + } + if (disposition.original !== undefined && typeof disposition.original !== 'boolean') { + throw new TypeError('disposition.original must be a boolean.'); + } + if (disposition.commentary !== undefined && typeof disposition.commentary !== 'boolean') { + throw new TypeError('disposition.commentary must be a boolean.'); + } + if (disposition.hearingImpaired !== undefined && typeof disposition.hearingImpaired !== 'boolean') { + throw new TypeError('disposition.hearingImpaired must be a boolean.'); + } + if (disposition.visuallyImpaired !== undefined && typeof disposition.visuallyImpaired !== 'boolean') { + throw new TypeError('disposition.visuallyImpaired must be a boolean.'); + } +}; diff --git a/src/misc.ts b/src/misc.ts index c6f5119..3274e7f 100644 --- a/src/misc.ts +++ b/src/misc.ts @@ -157,25 +157,25 @@ export const writeBits = (bytes: Uint8Array, start: number, end: number, value: export const toUint8Array = (source: AllowSharedBufferSource): Uint8Array => { if (source.constructor === Uint8Array) { // We want a true Uint8Array, not something that extends it like Buffer return source; - } else if (source instanceof ArrayBuffer) { - return new Uint8Array(source); - } else { + } else if (ArrayBuffer.isView(source)) { return new Uint8Array(source.buffer, source.byteOffset, source.byteLength); + } else { + return new Uint8Array(source); } }; -export const toDataView = (source: AllowSharedBufferSource) => { +export const toDataView = (source: AllowSharedBufferSource): DataView => { if (source.constructor === DataView) { return source; - } else if (source instanceof ArrayBuffer) { - return new DataView(source); - } else { + } else if (ArrayBuffer.isView(source)) { return new DataView(source.buffer, source.byteOffset, source.byteLength); + } else { + return new DataView(source); } }; -export const textDecoder = new TextDecoder(); -export const textEncoder = new TextEncoder(); +export const textDecoder = /* #__PURE__ */ new TextDecoder(); +export const textEncoder = /* #__PURE__ */ new TextEncoder(); export const isIso88591Compatible = (text: string) => { for (let i = 0; i < text.length; i++) { @@ -201,7 +201,7 @@ export const COLOR_PRIMARIES_MAP = { bt2020: 9, // ITU-R BT.202 smpte432: 12, // SMPTE EG 432-1 }; -export const COLOR_PRIMARIES_MAP_INVERSE = invertObject(COLOR_PRIMARIES_MAP); +export const COLOR_PRIMARIES_MAP_INVERSE = /* #__PURE__ */ invertObject(COLOR_PRIMARIES_MAP); export const TRANSFER_CHARACTERISTICS_MAP = { 'bt709': 1, // ITU-R BT.709 @@ -211,7 +211,7 @@ export const TRANSFER_CHARACTERISTICS_MAP = { 'pq': 16, // Rec. ITU-R BT.2100-2 perceptual quantization (PQ) system 'hlg': 18, // Rec. ITU-R BT.2100-2 hybrid loggamma (HLG) system }; -export const TRANSFER_CHARACTERISTICS_MAP_INVERSE = invertObject(TRANSFER_CHARACTERISTICS_MAP); +export const TRANSFER_CHARACTERISTICS_MAP_INVERSE = /* #__PURE__ */ invertObject(TRANSFER_CHARACTERISTICS_MAP); export const MATRIX_COEFFICIENTS_MAP = { 'rgb': 0, // Identity @@ -220,7 +220,7 @@ export const MATRIX_COEFFICIENTS_MAP = { 'smpte170m': 6, // SMPTE 170M 'bt2020-ncl': 9, // ITU-R BT.2020-2 (non-constant luminance) }; -export const MATRIX_COEFFICIENTS_MAP_INVERSE = invertObject(MATRIX_COEFFICIENTS_MAP); +export const MATRIX_COEFFICIENTS_MAP_INVERSE = /* #__PURE__ */ invertObject(MATRIX_COEFFICIENTS_MAP); export type RequiredNonNull = { [K in keyof T]-?: NonNullable; @@ -486,9 +486,14 @@ export const clamp = (value: number, min: number, max: number) => { export const UNDETERMINED_LANGUAGE = 'und'; -export const roundToPrecision = (value: number, digits: number) => { - const factor = 10 ** digits; - return Math.round(value * factor) / factor; +export const roundIfAlmostInteger = (value: number) => { + const rounded = Math.round(value); + + if (Math.abs(value / rounded - 1) < 10 * Number.EPSILON) { + return rounded; + } else { + return value; + } }; export const roundToMultiple = (value: number, multiple: number) => { @@ -579,6 +584,7 @@ export const retriedFetch = async ( url: string | URL | Request, requestInit: RequestInit, getRetryDelay: (previousAttempts: number, error: unknown, url: string | URL | Request) => number | null, + shouldStop: () => boolean, ) => { let attempts = 0; @@ -586,6 +592,10 @@ export const retriedFetch = async ( try { return await fetchFn(url, requestInit); } catch (error) { + if (shouldStop()) { + throw error; + } + attempts++; const retryDelayInSeconds = getRetryDelay(attempts, error, url); @@ -602,6 +612,10 @@ export const retriedFetch = async ( if (retryDelayInSeconds > 0) { await new Promise(resolve => setTimeout(resolve, 1000 * retryDelayInSeconds)); } + + if (shouldStop()) { + throw error; + } } } }; @@ -665,10 +679,15 @@ export const isWebKit = () => { } // This even returns true for WebKit-wrapping browsers such as Chrome on iOS - const result = !!(typeof navigator !== 'undefined' && navigator.vendor?.match(/apple/i)); - - isWebKitCache = result; - return result; + return isWebKitCache = !!( + typeof navigator !== 'undefined' + && ( + navigator.vendor?.match(/apple/i) + // Or, in workers: + || (/AppleWebKit/.test(navigator.userAgent) && !/Chrome/.test(navigator.userAgent)) + || /\b(iPad|iPhone|iPod)\b/.test(navigator.userAgent) + ) + ); }; let isFirefoxCache: boolean | null = null; @@ -680,6 +699,36 @@ export const isFirefox = () => { return isFirefoxCache = typeof navigator !== 'undefined' && navigator.userAgent?.includes('Firefox'); }; +let isChromiumCache: boolean | null = null; +export const isChromium = () => { + if (isChromiumCache !== null) { + return isChromiumCache; + } + + return isChromiumCache = !!( + typeof navigator !== 'undefined' + && (navigator.vendor?.includes('Google Inc') || /Chrome/.test(navigator.userAgent)) + ); +}; + +let chromiumVersionCache: number | null = null; +export const getChromiumVersion = () => { + if (chromiumVersionCache !== null) { + return chromiumVersionCache; + } + + if (typeof navigator === 'undefined') { + return null; + } + + const match = /\bChrome\/(\d+)/.exec(navigator.userAgent); + if (!match) { + return null; + } + + return chromiumVersionCache = Number(match[1]!); +}; + /** * T or a promise that resolves to T. * @group Miscellaneous diff --git a/src/mp3/mp3-demuxer.ts b/src/mp3/mp3-demuxer.ts index a8333b5..fe654d8 100644 --- a/src/mp3/mp3-demuxer.ts +++ b/src/mp3/mp3-demuxer.ts @@ -10,7 +10,7 @@ import { AudioCodec } from '../codec'; import { Demuxer } from '../demuxer'; import { Input } from '../input'; import { InputAudioTrack, InputAudioTrackBacking } from '../input-track'; -import { MetadataTags } from '../tags'; +import { DEFAULT_TRACK_DISPOSITION, MetadataTags } from '../metadata'; import { PacketRetrievalOptions } from '../media-sink'; import { assert, AsyncMutex, binarySearchExact, binarySearchLessOrEqual, UNDETERMINED_LANGUAGE } from '../misc'; import { EncodedPacket, PLACEHOLDER_DATA } from '../packet'; @@ -257,6 +257,12 @@ class Mp3AudioTrackBacking implements InputAudioTrackBacking { return this.demuxer.firstFrameHeader.sampleRate; } + getDisposition() { + return { + ...DEFAULT_TRACK_DISPOSITION, + }; + } + async getDecoderConfig(): Promise { assert(this.demuxer.firstFrameHeader); diff --git a/src/mp3/mp3-muxer.ts b/src/mp3/mp3-muxer.ts index dd310da..f2c86e8 100644 --- a/src/mp3/mp3-muxer.ts +++ b/src/mp3/mp3-muxer.ts @@ -7,7 +7,7 @@ */ import { assert, toDataView } from '../misc'; -import { metadataTagsAreEmpty } from '../tags'; +import { metadataTagsAreEmpty } from '../metadata'; import { Muxer } from '../muxer'; import { Output, OutputAudioTrack } from '../output'; import { Mp3OutputFormat } from '../output-format'; diff --git a/src/muxer.ts b/src/muxer.ts index b23356f..de2a36e 100644 --- a/src/muxer.ts +++ b/src/muxer.ts @@ -46,21 +46,21 @@ export abstract class Muxer { private trackTimestampInfo = new WeakMap(); - protected validateAndNormalizeTimestamp(track: OutputTrack, timestampInSeconds: number, isKeyFrame: boolean) { + protected validateAndNormalizeTimestamp(track: OutputTrack, timestampInSeconds: number, isKeyPacket: boolean) { timestampInSeconds += track.source._timestampOffset; let timestampInfo = this.trackTimestampInfo.get(track); if (!timestampInfo) { - if (!isKeyFrame) { - throw new Error('First frame must be a key frame.'); + if (!isKeyPacket) { + throw new Error('First packet must be a key packet.'); } timestampInfo = { maxTimestamp: timestampInSeconds, - maxTimestampBeforeLastKeyFrame: timestampInSeconds, + maxTimestampBeforeLastKeyPacket: timestampInSeconds, }; this.trackTimestampInfo.set(track, timestampInfo); } @@ -69,15 +69,15 @@ export abstract class Muxer { throw new Error(`Timestamps must be non-negative (got ${timestampInSeconds}s).`); } - if (isKeyFrame) { - timestampInfo.maxTimestampBeforeLastKeyFrame = timestampInfo.maxTimestamp; + if (isKeyPacket) { + timestampInfo.maxTimestampBeforeLastKeyPacket = timestampInfo.maxTimestamp; } - if (timestampInSeconds < timestampInfo.maxTimestampBeforeLastKeyFrame) { + if (timestampInSeconds < timestampInfo.maxTimestampBeforeLastKeyPacket) { throw new Error( - `Timestamps cannot be smaller than the highest timestamp of the previous GOP (a GOP begins with a key` - + ` frame and ends right before the next key frame). Got ${timestampInSeconds}s, but highest timestamp` - + ` is ${timestampInfo.maxTimestampBeforeLastKeyFrame}s.`, + `Timestamps cannot be smaller than the largest timestamp of the previous GOP (a GOP begins with a key` + + ` packet and ends right before the next key packet). Got ${timestampInSeconds}s, but largest` + + ` timestamp is ${timestampInfo.maxTimestampBeforeLastKeyPacket}s.`, ); } diff --git a/src/ogg/ogg-demuxer.ts b/src/ogg/ogg-demuxer.ts index 495865d..c5164b9 100644 --- a/src/ogg/ogg-demuxer.ts +++ b/src/ogg/ogg-demuxer.ts @@ -12,14 +12,14 @@ import { Demuxer } from '../demuxer'; import { Input } from '../input'; import { InputAudioTrack, InputAudioTrackBacking } from '../input-track'; import { PacketRetrievalOptions } from '../media-sink'; -import { MetadataTags } from '../tags'; +import { DEFAULT_TRACK_DISPOSITION, MetadataTags } from '../metadata'; import { assert, AsyncMutex, binarySearchLessOrEqual, findLast, last, - roundToPrecision, + roundIfAlmostInteger, toDataView, UNDETERMINED_LANGUAGE, } from '../misc'; @@ -463,6 +463,12 @@ class OggAudioTrackBacking implements InputAudioTrackBacking { return UNDETERMINED_LANGUAGE; } + getDisposition() { + return { + ...DEFAULT_TRACK_DISPOSITION, + }; + } + async getFirstTimestamp() { return 0; } @@ -577,7 +583,7 @@ class OggAudioTrackBacking implements InputAudioTrackBacking { return this.getPacketSequential(timestamp, options); } - const timestampInSamples = roundToPrecision(timestamp * this.internalSampleRate, 14); + const timestampInSamples = roundIfAlmostInteger(timestamp * this.internalSampleRate); if (timestampInSamples === 0) { // Fast path for timestamp 0 - avoids binary search when playing back from the start return this.getFirstPacket(options); @@ -910,7 +916,7 @@ class OggAudioTrackBacking implements InputAudioTrackBacking { const release = await this.sequentialScanMutex.acquire(); // Requires exclusivity because we write to a cache try { - const timestampInSamples = roundToPrecision(timestamp * this.internalSampleRate, 14); + const timestampInSamples = roundIfAlmostInteger(timestamp * this.internalSampleRate); timestamp = timestampInSamples / this.internalSampleRate; const index = binarySearchLessOrEqual( diff --git a/src/output.ts b/src/output.ts index 6379388..4046c1f 100644 --- a/src/output.ts +++ b/src/output.ts @@ -7,7 +7,7 @@ */ import { AsyncMutex, isIso639Dash2LanguageCode, Rotation } from './misc'; -import { MetadataTags, validateMetadataTags } from './tags'; +import { MetadataTags, TrackDisposition, validateMetadataTags, validateTrackDisposition } from './metadata'; import { Muxer } from './muxer'; import { OutputFormat } from './output-format'; import { AudioSource, MediaSource, SubtitleSource, VideoSource } from './media-source'; @@ -74,6 +74,8 @@ export type BaseTrackMetadata = { languageCode?: string; /** A user-defined name for this track, like "English" or "Director Commentary". */ name?: string; + /** The track's disposition, i.e. information about its intended usage. */ + disposition?: Partial; /** * The maximum amount of encoded packets that will be added to this track. Setting this field provides the muxer * with an additional signal that it can use to preallocate space in the file. @@ -129,6 +131,9 @@ const validateBaseTrackMetadata = (metadata: BaseTrackMetadata) => { if (metadata.name !== undefined && typeof metadata.name !== 'string') { throw new TypeError('metadata.name, when provided, must be a string.'); } + if (metadata.disposition !== undefined) { + validateTrackDisposition(metadata.disposition); + } if ( metadata.maximumPacketCount !== undefined && (!Number.isInteger(metadata.maximumPacketCount) || metadata.maximumPacketCount < 0) diff --git a/src/packet.ts b/src/packet.ts index f5de60e..4f9b749 100644 --- a/src/packet.ts +++ b/src/packet.ts @@ -8,7 +8,7 @@ import { SECOND_TO_MICROSECOND_FACTOR } from './misc'; -export const PLACEHOLDER_DATA = new Uint8Array(0); +export const PLACEHOLDER_DATA = /* #__PURE__ */ new Uint8Array(0); /** * The type of a packet. Key packets can be decoded without previous packets, while delta packets depend on previous diff --git a/src/sample.ts b/src/sample.ts index 770f450..b7952bc 100644 --- a/src/sample.ts +++ b/src/sample.ts @@ -18,6 +18,7 @@ import { isFirefox, polyfillSymbolDispose, normalizeRotation, + assertNever, } from './misc'; polyfillSymbolDispose(); @@ -37,6 +38,105 @@ declare global { } } +type FinalizationRegistryValue = { + type: 'video'; + data: VideoFrame | OffscreenCanvas | Uint8Array; +} | { + type: 'audio'; + data: AudioData | Uint8Array; +}; + +// Let's manually handle logging the garbage collection errors that are typically logged by the browser. This way, they +// also kick for audio samples (which is normally not the case), making sure any incorrect code is quickly caught. +let lastVideoGcErrorLog = -Infinity; +let lastAudioGcErrorLog = -Infinity; +let finalizationRegistry: FinalizationRegistry | null = null; +if (typeof FinalizationRegistry !== 'undefined') { + finalizationRegistry = new FinalizationRegistry((value) => { + const now = Date.now(); + + if (value.type === 'video') { + if (now - lastVideoGcErrorLog >= 1000) { + // This error is annoying but oh so important + console.error( + `A VideoSample was garbage collected without first being closed. For proper resource management,` + + ` make sure to call close() on all your VideoSamples as soon as you're done using them.`, + ); + + lastVideoGcErrorLog = now; + } + + if (typeof VideoFrame !== 'undefined' && value.data instanceof VideoFrame) { + value.data.close(); // Prevent the browser error since we're logging our own + } + } else { + if (now - lastAudioGcErrorLog >= 1000) { + console.error( + `An AudioSample was garbage collected without first being closed. For proper resource management,` + + ` make sure to call close() on all your AudioSamples as soon as you're done using them.`, + ); + + lastAudioGcErrorLog = now; + } + + if (typeof AudioData !== 'undefined' && value.data instanceof AudioData) { + value.data.close(); + } + } + }); +} + +/** + * The list of {@link VideoSample} pixel formats. + * @group Samples + * @public + */ +export const VIDEO_SAMPLE_PIXEL_FORMATS = [ + // 4:2:0 Y, U, V + 'I420', + 'I420P10', + 'I420P12', + // 4:2:0 Y, U, V, A + 'I420A', + 'I420AP10', + 'I420AP12', + // 4:2:2 Y, U, V + 'I422', + 'I422P10', + 'I422P12', + // 4:2:2 Y, U, V, A + 'I422A', + 'I422AP10', + 'I422AP12', + // 4:4:4 Y, U, V + 'I444', + 'I444P10', + 'I444P12', + // 4:4:4 Y, U, V, A + 'I444A', + 'I444AP10', + 'I444AP12', + // 4:2:0 Y, UV + 'NV12', + // 4:4:4 RGBA + 'RGBA', + // 4:4:4 RGBX (opaque) + 'RGBX', + // 4:4:4 BGRA + 'BGRA', + // 4:4:4 BGRX (opaque) + 'BGRX', +] as const; +const VIDEO_SAMPLE_PIXEL_FORMATS_SET = new Set(VIDEO_SAMPLE_PIXEL_FORMATS); + +/** + * The internal pixel format with which a {@link VideoSample} is stored. + * [See pixel formats](https://www.w3.org/TR/webcodecs/#pixel-format) for more. + * @group Samples + * @public + */ +export type VideoSamplePixelFormat = typeof VIDEO_SAMPLE_PIXEL_FORMATS[number]; + /** * Metadata used for VideoSample initialization. * @group Samples @@ -45,9 +145,9 @@ declare global { export type VideoSampleInit = { /** * The internal pixel format in which the frame is stored. - * [See pixel formats](https://developer.mozilla.org/en-US/docs/Web/API/VideoFrame/format) + * [See pixel formats](https://www.w3.org/TR/webcodecs/#pixel-format) */ - format?: VideoPixelFormat; + format?: VideoSamplePixelFormat; /** The width of the frame in pixels. */ codedWidth?: number; /** The height of the frame in pixels. */ @@ -60,6 +160,8 @@ export type VideoSampleInit = { duration?: number; /** The color space of the frame. */ colorSpace?: VideoColorSpaceInit; + /** The byte layout of the planes of the frame. */ + layout?: PlaneLayout[]; }; /** @@ -71,14 +173,20 @@ export type VideoSampleInit = { export class VideoSample implements Disposable { /** @internal */ _data!: VideoFrame | OffscreenCanvas | Uint8Array | null; + /** + * Used for the ArrayBuffer-backed case. + * @internal + */ + _layout!: PlaneLayout[] | null; /** @internal */ _closed: boolean = false; /** - * The internal pixel format in which the frame is stored. - * [See pixel formats](https://developer.mozilla.org/en-US/docs/Web/API/VideoFrame/format) + * The internal pixel format in which the frame is stored. Will be `null` if it's using an arbitrary internal + * format not representable by `VideoPixelFormat`. + * [See pixel formats](https://www.w3.org/TR/webcodecs/#pixel-format) */ - readonly format!: VideoPixelFormat | null; + readonly format!: VideoSamplePixelFormat | null; /** The width of the frame in pixels. */ readonly codedWidth!: number; /** The height of the frame in pixels. */ @@ -93,7 +201,7 @@ export class VideoSample implements Disposable { /** The duration of the frame in seconds. */ readonly duration!: number; /** The color space of the frame. */ - readonly colorSpace!: VideoColorSpace; + readonly colorSpace!: VideoSampleColorSpace; /** The width of the frame in pixels after rotation. */ get displayWidth() { @@ -149,12 +257,16 @@ export class VideoSample implements Disposable { data: VideoFrame | CanvasImageSource | AllowSharedBufferSource, init?: VideoSampleInit, ) { - if (data instanceof ArrayBuffer || ArrayBuffer.isView(data)) { + if ( + data instanceof ArrayBuffer + || (typeof SharedArrayBuffer !== 'undefined' && data instanceof SharedArrayBuffer) + || ArrayBuffer.isView(data) + ) { if (!init || typeof init !== 'object') { throw new TypeError('init must be an object.'); } - if (!('format' in init) || typeof init.format !== 'string') { - throw new TypeError('init.format must be a string.'); + if (init.format === undefined || !VIDEO_SAMPLE_PIXEL_FORMATS_SET.has(init.format)) { + throw new TypeError('init.format must be one of: ' + VIDEO_SAMPLE_PIXEL_FORMATS.join(', ')); } if (!Number.isInteger(init.codedWidth) || init.codedWidth! <= 0) { throw new TypeError('init.codedWidth must be a positive integer.'); @@ -173,6 +285,7 @@ export class VideoSample implements Disposable { } this._data = toUint8Array(data).slice(); // Copy it + this._layout = init.layout ?? createDefaultPlaneLayout(init.format, init.codedWidth!, init.codedHeight!); this.format = init.format; this.codedWidth = init.codedWidth!; @@ -180,7 +293,7 @@ export class VideoSample implements Disposable { this.rotation = init.rotation ?? 0; this.timestamp = init.timestamp!; this.duration = init.duration ?? 0; - this.colorSpace = new VideoColorSpace(init.colorSpace); + this.colorSpace = new VideoSampleColorSpace(init.colorSpace); } else if (typeof VideoFrame !== 'undefined' && data instanceof VideoFrame) { if (init?.rotation !== undefined && ![0, 90, 180, 270].includes(init.rotation)) { throw new TypeError('init.rotation, when provided, must be 0, 90, 180, or 270.'); @@ -193,6 +306,7 @@ export class VideoSample implements Disposable { } this._data = data; + this._layout = null; this.format = data.format; @@ -212,7 +326,7 @@ export class VideoSample implements Disposable { this.rotation = init?.rotation ?? frameRotation; this.timestamp = init?.timestamp ?? data.timestamp / 1e6; this.duration = init?.duration ?? (data.duration ?? 0) / 1e6; - this.colorSpace = data.colorSpace; + this.colorSpace = new VideoSampleColorSpace(data.colorSpace); } else if ( (typeof HTMLImageElement !== 'undefined' && data instanceof HTMLImageElement) || (typeof SVGImageElement !== 'undefined' && data instanceof SVGImageElement) @@ -274,6 +388,7 @@ export class VideoSample implements Disposable { // Draw it to a canvas context.drawImage(data, 0, 0); this._data = canvas; + this._layout = null; this.format = 'RGBX'; this.codedWidth = width; @@ -281,7 +396,7 @@ export class VideoSample implements Disposable { this.rotation = init.rotation ?? 0; this.timestamp = init.timestamp!; this.duration = init.duration ?? 0; - this.colorSpace = new VideoColorSpace({ + this.colorSpace = new VideoSampleColorSpace({ matrix: 'rgb', primaries: 'bt709', transfer: 'iec61966-2-1', @@ -290,6 +405,8 @@ export class VideoSample implements Disposable { } else { throw new TypeError('Invalid data type: Must be a BufferSource or CanvasImageSource.'); } + + finalizationRegistry?.register(this, { type: 'video', data: this._data }, this); } /** Clones this video sample. */ @@ -307,8 +424,11 @@ export class VideoSample implements Disposable { rotation: this.rotation, }); } else if (this._data instanceof Uint8Array) { - return new VideoSample(this._data.slice(), { + assert(this._layout); + + return new VideoSample(this._data, { format: this.format!, + layout: this._layout, codedWidth: this.codedWidth, codedHeight: this.codedHeight, timestamp: this.timestamp, @@ -338,6 +458,8 @@ export class VideoSample implements Disposable { return; } + finalizationRegistry?.unregister(this); + if (isVideoFrame(this._data)) { this._data.close(); } else { @@ -347,16 +469,43 @@ export class VideoSample implements Disposable { this._closed = true; } - /** Returns the number of bytes required to hold this video sample's pixel data. */ - allocationSize() { + /** + * Returns the number of bytes required to hold this video sample's pixel data. Throws if `format` is `null`; + * specify an explicit RGB format in the options in this case. + */ + allocationSize(options: VideoFrameCopyToOptions = {}): number { + validateVideoFrameCopyToOptions(options); + if (this._closed) { throw new Error('VideoSample is closed.'); } + if ((options.format ?? this.format) === null) { + throw new Error( + 'Cannot get allocation size when format is null. Please manually provide an RGB pixel format in the' + + ' options instead.', + ); + } assert(this._data !== null); + if (!isVideoFrame(this._data)) { + if ( + options.colorSpace + || (options.format && options.format !== this.format) + || options.layout + || options.rect + ) { + // Temporarily convert to VideoFrame to get it done + const videoFrame = this.toVideoFrame(); + const size = videoFrame.allocationSize(options); + videoFrame.close(); + + return size; + } + } + if (isVideoFrame(this._data)) { - return this._data.allocationSize(); + return this._data.allocationSize(options); } else if (this._data instanceof Uint8Array) { return this._data.byteLength; } else { @@ -364,23 +513,54 @@ export class VideoSample implements Disposable { } } - /** Copies this video sample's pixel data to an ArrayBuffer or ArrayBufferView. */ - async copyTo(destination: AllowSharedBufferSource) { + /** + * Copies this video sample's pixel data to an ArrayBuffer or ArrayBufferView. Throws if `format` is `null`; + * specify an explicit RGB format in the options in this case. + * @returns The byte layout of the planes of the copied data. + */ + async copyTo(destination: AllowSharedBufferSource, options: VideoFrameCopyToOptions = {}): Promise { if (!isAllowSharedBufferSource(destination)) { throw new TypeError('destination must be an ArrayBuffer or an ArrayBuffer view.'); } + validateVideoFrameCopyToOptions(options); if (this._closed) { throw new Error('VideoSample is closed.'); } + if ((options.format ?? this.format) === null) { + throw new Error( + 'Cannot copy video sample data when format is null. Please manually provide an RGB pixel format in the' + + ' options instead.', + ); + } assert(this._data !== null); + if (!isVideoFrame(this._data)) { + if ( + options.colorSpace + || (options.format && options.format !== this.format) + || options.layout + || options.rect + ) { + // Temporarily convert to VideoFrame to get it done + const videoFrame = this.toVideoFrame(); + const layout = await videoFrame.copyTo(destination, options); + videoFrame.close(); + + return layout; + } + } + if (isVideoFrame(this._data)) { - await this._data.copyTo(destination); + return this._data.copyTo(destination, options); } else if (this._data instanceof Uint8Array) { + assert(this._layout); + const dest = toUint8Array(destination); dest.set(this._data); + + return this._layout; } else { const canvas = this._data; const context = canvas.getContext('2d'); @@ -389,6 +569,11 @@ export class VideoSample implements Disposable { const imageData = context.getImageData(0, 0, this.codedWidth, this.codedHeight); const dest = toUint8Array(destination); dest.set(imageData.data); + + return [{ + offset: 0, + stride: 4 * this.codedWidth, + }]; } } @@ -413,17 +598,15 @@ export class VideoSample implements Disposable { }; return new VideoFrame(this._data, init); } else if (this._data instanceof Uint8Array) { - const init: VideoFrameBufferInit = { - format: this.format!, + return new VideoFrame(this._data, { + format: this.format! as VideoPixelFormat, codedWidth: this.codedWidth, codedHeight: this.codedHeight, timestamp: this.microsecondTimestamp, duration: this.microsecondDuration || undefined, colorSpace: this.colorSpace, rotation: this.rotation, - }; - - return new VideoFrame(this._data, init); + }); } else { const init: VideoFrameInit = { timestamp: this.microsecondTimestamp, @@ -804,6 +987,40 @@ export class VideoSample implements Disposable { } } +/** + * Describes the color space of a {@link VideoSample}. Corresponds to the WebCodecs API's VideoColorSpace. + * @group Samples + * @public + */ +export class VideoSampleColorSpace { + /** The color primaries standard used. */ + readonly primaries: VideoColorPrimaries | null; + /** The transfer characteristics used. */ + readonly transfer: VideoTransferCharacteristics | null; + /** The color matrix coefficients used. */ + readonly matrix: VideoMatrixCoefficients | null; + /** Whether the color values use the full range or limited range. */ + readonly fullRange: boolean | null; + + /** Creates a new VideoSampleColorSpace. */ + constructor(init?: VideoColorSpaceInit) { + this.primaries = init?.primaries ?? null; + this.transfer = init?.transfer ?? null; + this.matrix = init?.matrix ?? null; + this.fullRange = init?.fullRange ?? null; + } + + /** Serializes the color space to a JSON object. */ + toJSON(): VideoColorSpaceInit { + return { + primaries: this.primaries, + transfer: this.transfer, + matrix: this.matrix, + fullRange: this.fullRange, + }; + } +} + const isVideoFrame = (x: unknown): x is VideoFrame => { return typeof VideoFrame !== 'undefined' && x instanceof VideoFrame; }; @@ -852,7 +1069,168 @@ export const validateCropRectangle = (crop: CropRectangle, prefix: string) => { } }; -const AUDIO_SAMPLE_FORMATS = new Set( +const validateVideoFrameCopyToOptions = (options: VideoFrameCopyToOptions) => { + if (!options || typeof options !== 'object') { + throw new TypeError('options must be an object.'); + } + if (options.colorSpace !== undefined && !['display-p3', 'srgb'].includes(options.colorSpace)) { + throw new TypeError('options.colorSpace, when provided, must be \'display-p3\' or \'srgb\'.'); + } + if (options.format !== undefined && typeof options.format !== 'string') { + throw new TypeError('options.format, when provided, must be a string.'); + } + if (options.layout !== undefined) { + if (!Array.isArray(options.layout)) { + throw new TypeError('options.layout, when provided, must be an array.'); + } + + for (const plane of options.layout) { + if (!plane || typeof plane !== 'object') { + throw new TypeError('Each entry in options.layout must be an object.'); + } + if (!Number.isInteger(plane.offset) || plane.offset < 0) { + throw new TypeError('plane.offset must be a non-negative integer.'); + } + if (!Number.isInteger(plane.stride) || plane.stride < 0) { + throw new TypeError('plane.stride must be a non-negative integer.'); + } + } + } + if (options.rect !== undefined) { + if (!options.rect || typeof options.rect !== 'object') { + throw new TypeError('options.rect, when provided, must be an object.'); + } + if (options.rect.x !== undefined && (!Number.isInteger(options.rect.x) || options.rect.x < 0)) { + throw new TypeError('options.rect.x, when provided, must be a non-negative integer.'); + } + if (options.rect.y !== undefined && (!Number.isInteger(options.rect.y) || options.rect.y < 0)) { + throw new TypeError('options.rect.y, when provided, must be a non-negative integer.'); + } + if (options.rect.width !== undefined && (!Number.isInteger(options.rect.width) || options.rect.width < 0)) { + throw new TypeError('options.rect.width, when provided, must be a non-negative integer.'); + } + if (options.rect.height !== undefined && (!Number.isInteger(options.rect.height) || options.rect.height < 0)) { + throw new TypeError('options.rect.height, when provided, must be a non-negative integer.'); + } + } +}; + +/** Implements logic from WebCodecs § 9.4.6 "Compute Layout and Allocation Size" */ +const createDefaultPlaneLayout = ( + format: VideoSamplePixelFormat, + codedWidth: number, + codedHeight: number, +): PlaneLayout[] => { + const planes = getPlaneConfigs(format); + const layouts: PlaneLayout[] = []; + let currentOffset = 0; + + for (const plane of planes) { + // Per § 9.8, dimensions are usually "rounded up to the nearest integer". + const planeWidth = Math.ceil(codedWidth / plane.widthDivisor); + const planeHeight = Math.ceil(codedHeight / plane.heightDivisor); + + const stride = planeWidth * plane.sampleBytes; + + // Tight packing + const planeSize = stride * planeHeight; + + layouts.push({ + offset: currentOffset, + stride: stride, + }); + + currentOffset += planeSize; + } + + return layouts; +}; + +type PlaneConfig = { + sampleBytes: number; + widthDivisor: number; // Horizontal sub-sampling factor + heightDivisor: number; // Vertical sub-sampling factor +}; + +/** Helper to retrieve plane configurations based on WebCodecs § 9.8 Pixel Format definitions. */ +const getPlaneConfigs = (format: VideoSamplePixelFormat): PlaneConfig[] => { + // Helper for standard YUV planes + const yuv = ( + yBytes: number, + uvBytes: number, + subX: number, + subY: number, + hasAlpha: boolean, + ): PlaneConfig[] => { + const configs: PlaneConfig[] = [ + { sampleBytes: yBytes, widthDivisor: 1, heightDivisor: 1 }, + { sampleBytes: uvBytes, widthDivisor: subX, heightDivisor: subY }, + { sampleBytes: uvBytes, widthDivisor: subX, heightDivisor: subY }, + ]; + + if (hasAlpha) { + // Match luma dimensions + configs.push({ sampleBytes: yBytes, widthDivisor: 1, heightDivisor: 1 }); + } + + return configs; + }; + + switch (format) { + case 'I420': + return yuv(1, 1, 2, 2, false); + case 'I420P10': + case 'I420P12': + return yuv(2, 2, 2, 2, false); + case 'I420A': + return yuv(1, 1, 2, 2, true); + case 'I420AP10': + case 'I420AP12': + return yuv(2, 2, 2, 2, true); + + case 'I422': + return yuv(1, 1, 2, 1, false); + case 'I422P10': + case 'I422P12': + return yuv(2, 2, 2, 1, false); + case 'I422A': + return yuv(1, 1, 2, 1, true); + case 'I422AP10': + case 'I422AP12': + return yuv(2, 2, 2, 1, true); + + case 'I444': + return yuv(1, 1, 1, 1, false); + case 'I444P10': + case 'I444P12': + return yuv(2, 2, 1, 1, false); + case 'I444A': + return yuv(1, 1, 1, 1, true); + case 'I444AP10': + case 'I444AP12': + return yuv(2, 2, 1, 1, true); + + case 'NV12': + return [ + { sampleBytes: 1, widthDivisor: 1, heightDivisor: 1 }, + { sampleBytes: 2, widthDivisor: 2, heightDivisor: 2 }, // Interleaved U and V + ]; + + case 'RGBA': + case 'RGBX': + case 'BGRA': + case 'BGRX': + return [ + { sampleBytes: 4, widthDivisor: 1, heightDivisor: 1 }, + ]; + + default: + assertNever(format); + assert(false); + } +}; + +const AUDIO_SAMPLE_FORMATS = new Set( ['f32', 'f32-planar', 's16', 's16-planar', 's32', 's32-planar', 'u8', 'u8-planar'], ); @@ -1010,6 +1388,8 @@ export class AudioSample implements Disposable { this._data = dataBuffer; } + + finalizationRegistry?.register(this, { type: 'audio', data: this._data }, this); } /** Returns the number of bytes required to hold the audio sample's data as specified by the given options. */ @@ -1171,9 +1551,8 @@ export class AudioSample implements Disposable { } } } else { - // Branch for Uint8Array data (non-AudioData) const uint8Data = this._data; - const srcView = new DataView(uint8Data.buffer, uint8Data.byteOffset, uint8Data.byteLength); + const srcView = toDataView(uint8Data); const srcFormat = this.format; const readFn = getReadFunction(srcFormat); @@ -1243,6 +1622,8 @@ export class AudioSample implements Disposable { return; } + finalizationRegistry?.unregister(this); + if (isAudioData(this._data)) { this._data.close(); } else { @@ -1305,7 +1686,9 @@ export class AudioSample implements Disposable { numberOfFrames: this.numberOfFrames, numberOfChannels: this.numberOfChannels, timestamp: this.microsecondTimestamp, - data: this._data, + data: this._data.buffer instanceof ArrayBuffer + ? this._data.buffer + : this._data.slice(), // In the case of SharedArrayBuffer, convert to ArrayBuffer }); } } diff --git a/src/source.ts b/src/source.ts index 68ebac1..0059189 100644 --- a/src/source.ts +++ b/src/source.ts @@ -102,10 +102,17 @@ export class BufferSource extends Source { /** @internal */ _onreadCalled = false; - /** Creates a new {@link BufferSource} backed the specified `ArrayBuffer` or `ArrayBufferView`. */ - constructor(buffer: ArrayBuffer | ArrayBufferView) { - if (!(buffer instanceof ArrayBuffer) && !ArrayBuffer.isView(buffer)) { - throw new TypeError('buffer must be an ArrayBuffer or ArrayBufferView.'); + /** + * Creates a new {@link BufferSource} backed by the specified `ArrayBuffer`, `SharedArrayBuffer`, + * or `ArrayBufferView`. + */ + constructor(buffer: AllowSharedBufferSource) { + if ( + !(buffer instanceof ArrayBuffer) + && !(typeof SharedArrayBuffer !== 'undefined' && buffer instanceof SharedArrayBuffer) + && !ArrayBuffer.isView(buffer) + ) { + throw new TypeError('buffer must be an ArrayBuffer, SharedArrayBuffer, or ArrayBufferView.'); } super(); @@ -234,12 +241,7 @@ export class BlobSource extends Source { const { done, value } = await reader.read(); if (done) { this._orchestrator.forgetWorker(worker); - - if (worker.currentPos < worker.targetPos) { // I think this `if` should always hit? - throw new Error('Blob reader stopped unexpectedly before all requested data was read.'); - } - - break; + throw new Error('Blob reader stopped unexpectedly before all requested data was read.'); } if (worker.aborted) { @@ -261,6 +263,11 @@ export class BlobSource extends Source { } worker.running = false; + + if (worker.aborted) { + // MDN: "Calling this method signals a loss of interest in the stream by a consumer." + await reader?.cancel(); + } } /** @internal */ @@ -297,6 +304,10 @@ const DEFAULT_RETRY_DELAY = typeof navigator !== 'undefined' && typeof navigator.onLine === 'boolean' ? navigator.onLine : true; if (isOnline && originOfSrc !== null && originOfSrc !== window.location.origin) { + console.warn( + `Request will not be retried because a CORS error was suspected due to different origins. You can` + + ` modify this behavior by providing your own function for the 'getRetryDelay' option.`, + ); return null; } } @@ -425,6 +436,7 @@ export class UrlSource extends Source { signal: abortController.signal, }), this._getRetryDelay, + () => this._disposed, ); if (!response.ok) { @@ -436,7 +448,7 @@ export class UrlSource extends Source { let fileSize: number; if (response.status === 206) { - fileSize = this._getPartialLengthFromRangeResponse(response); + fileSize = this._getTotalLengthFromRangeResponse(response); worker = this._orchestrator.createWorker(0, Math.min(fileSize, URL_SOURCE_MIN_LOAD_AMOUNT)); } else { // Server probably returned a 200. @@ -492,6 +504,7 @@ export class UrlSource extends Source { signal: abortController.signal, }), this._getRetryDelay, + () => this._disposed, ); } @@ -509,14 +522,6 @@ export class UrlSource extends Source { ); } - const length = this._getPartialLengthFromRangeResponse(response); - const required = worker.targetPos - worker.currentPos; - if (length < required) { - throw new Error( - `HTTP response unexpectedly too short: Needed at least ${required} bytes, got only ${length}.`, - ); - } - if (!response.body) { throw new Error( 'Missing HTTP response body stream. The used fetch function must provide the response body as a' @@ -539,6 +544,11 @@ export class UrlSource extends Source { try { readResult = await reader.read(); } catch (error) { + if (this._disposed) { + // No need to try to retry + throw error; + } + const retryDelayInSeconds = this._getRetryDelay(1, error, this._url); if (retryDelayInSeconds !== null) { console.error('Error while reading response stream. Attempting to resume.', error); @@ -551,60 +561,53 @@ export class UrlSource extends Source { } if (worker.aborted) { - break; + continue; // Cleanup happens in next iteration } const { done, value } = readResult; if (done) { - this._orchestrator.forgetWorker(worker); - - if (worker.currentPos < worker.targetPos) { - throw new Error( - 'Response stream reader stopped unexpectedly before all requested data was read.', - ); + if (worker.currentPos >= worker.targetPos) { + // All data was delivered, we're good + this._orchestrator.forgetWorker(worker); + worker.running = false; + return; } - worker.running = false; - return; + // The response stopped early, before the target. This can happen if server decides to cap range + // requests arbitrarily, even if the request had an uncapped end. In this case, let's fetch the rest + // of the data using a new request. + break; } this.onread?.(worker.currentPos, worker.currentPos + value.length); this._orchestrator.supplyWorkerData(worker, value); } - - if (worker.aborted) { - break; - } } - worker.running = false; - // The previous UrlSource had logic for circumventing https://issues.chromium.org/issues/436025873; I haven't // been able to observe this bug with the new UrlSource (maybe because we're using response streaming), so the // logic for that has vanished for now. Leaving a comment here if this becomes relevant again. } /** @internal */ - private _getPartialLengthFromRangeResponse(response: Response) { + private _getTotalLengthFromRangeResponse(response: Response) { const contentRange = response.headers.get('Content-Range'); if (contentRange) { const match = /\/(\d+)/.exec(contentRange); if (match) { return Number(match[1]); - } else { - throw new Error(`Invalid Content-Range header: ${contentRange}`); } + } + + const contentLength = response.headers.get('Content-Length'); + if (contentLength) { + return Number(contentLength); } else { - const contentLength = response.headers.get('Content-Length'); - if (contentLength) { - return Number(contentLength); - } else { - throw new Error( - 'Partial HTTP response (status 206) must surface either Content-Range or' - + ' Content-Length header.', - ); - } + throw new Error( + 'Partial HTTP response (status 206) must surface either Content-Range or' + + ' Content-Length header.', + ); } } @@ -639,7 +642,7 @@ export class FilePathSource extends Source { _fileHandle: FileHandle | null = null; /** Creates a new {@link FilePathSource} backed by the file at the specified file path. */ - constructor(filePath: string, options: BlobSourceOptions = {}) { + constructor(filePath: string, options: FilePathSourceOptions = {}) { if (typeof filePath !== 'string') { throw new TypeError('filePath must be a string.'); } diff --git a/src/target.ts b/src/target.ts index 168e803..5cb78e3 100644 --- a/src/target.ts +++ b/src/target.ts @@ -62,7 +62,7 @@ export type StreamTargetChunk = { /** The operation type. */ type: 'write'; // This ensures automatic compatibility with FileSystemWritableFileStream /** The data to write. */ - data: Uint8Array; + data: Uint8Array; /** The byte offset in the output file at which to write the data. */ position: number; }; diff --git a/src/wave/wave-demuxer.ts b/src/wave/wave-demuxer.ts index c552827..56b5db5 100644 --- a/src/wave/wave-demuxer.ts +++ b/src/wave/wave-demuxer.ts @@ -11,7 +11,7 @@ import { Demuxer } from '../demuxer'; import { Input } from '../input'; import { InputAudioTrack, InputAudioTrackBacking } from '../input-track'; import { PacketRetrievalOptions } from '../media-sink'; -import { MetadataTags } from '../tags'; +import { DEFAULT_TRACK_DISPOSITION, MetadataTags } from '../metadata'; import { assert, UNDETERMINED_LANGUAGE } from '../misc'; import { EncodedPacket, PLACEHOLDER_DATA } from '../packet'; import { readAscii, readBytes, Reader, readU16, readU32, readU64 } from '../reader'; @@ -407,6 +407,12 @@ class WaveAudioTrackBacking implements InputAudioTrackBacking { return UNDETERMINED_LANGUAGE; } + getDisposition() { + return { + ...DEFAULT_TRACK_DISPOSITION, + }; + } + async getFirstTimestamp() { return 0; } diff --git a/src/wave/wave-muxer.ts b/src/wave/wave-muxer.ts index 63c8f96..74b6abf 100644 --- a/src/wave/wave-muxer.ts +++ b/src/wave/wave-muxer.ts @@ -15,7 +15,7 @@ import { Writer } from '../writer'; import { EncodedPacket } from '../packet'; import { WavOutputFormat } from '../output-format'; import { assert, assertNever, isIso88591Compatible, keyValueIterator } from '../misc'; -import { MetadataTags, metadataTagsAreEmpty } from '../tags'; +import { MetadataTags, metadataTagsAreEmpty } from '../metadata'; import { Id3V2Writer } from '../id3'; export class WaveMuxer extends Muxer { diff --git a/src/writer.ts b/src/writer.ts index 30de003..8f8aa86 100644 --- a/src/writer.ts +++ b/src/writer.ts @@ -190,7 +190,7 @@ const MAX_CHUNKS_AT_ONCE = 2; interface Chunk { start: number; written: ChunkSection[]; - data: Uint8Array; + data: Uint8Array; shouldFlush: boolean; } @@ -281,7 +281,7 @@ export class StreamTargetWriter extends Writer { const chunks: { start: number; size: number; - data?: Uint8Array; + data?: Uint8Array; }[] = []; const sorted = [...this.sections].sort((a, b) => a.start - b.start); diff --git a/test/browser/flac.test.ts b/test/browser/flac.test.ts index 2aed0e3..ddc065b 100644 --- a/test/browser/flac.test.ts +++ b/test/browser/flac.test.ts @@ -19,7 +19,7 @@ test('can decode samples from a FLAC file', async () => { const sink = new AudioSampleSink(track); - const sample = await sink.getSample(1); + using sample = await sink.getSample(1); assert(sample); expect(sample.timestamp).toBe(0.9287981859410431); }); diff --git a/test/browser/transparency.test.ts b/test/browser/transparency.test.ts index 35527ae..394c827 100644 --- a/test/browser/transparency.test.ts +++ b/test/browser/transparency.test.ts @@ -21,7 +21,7 @@ test('Can decode transparent video', async () => { expect(await videoTrack.canBeTransparent()).toBe(true); const sink = new VideoSampleSink(videoTrack); - const sample = (await sink.getSample(0.5))!; + using sample = (await sink.getSample(0.5))!; expect(sample.format).toContain('A'); // Probably RGBA expect(sample.hasAlpha).toBe(true); @@ -47,10 +47,10 @@ test('Can decode faulty transparent video and behaves gracefully', async () => { const sink = new VideoSampleSink(videoTrack); - const startSample = (await sink.getSample(await videoTrack.getFirstTimestamp()))!; + using startSample = (await sink.getSample(await videoTrack.getFirstTimestamp()))!; expect(startSample.format).toContain('A'); - const secondSample = (await sink.getSample(secondKeyPacket.timestamp))!; + using secondSample = (await sink.getSample(secondKeyPacket.timestamp))!; expect(secondSample.format).not.toContain('A'); // There was no alpha key frame for this one expect(secondSample.hasAlpha).toBe(false); }); @@ -164,7 +164,7 @@ test('Can encode transparent video', async () => { const sink = new VideoSampleSink(videoTrack); - const firstSample = (await sink.getSample(0))!; + using firstSample = (await sink.getSample(0))!; expect(firstSample.format).toContain('A'); probeContext.clearRect(0, 0, probeCanvas.width, probeCanvas.height); @@ -201,7 +201,7 @@ test('Can encode video with alternating transparency', async () => { await output.start(); for (let i = 0; i < 64; i++) { - const sample = new VideoSample(new Uint8Array(640 * 480 * 4), { + using sample = new VideoSample(new Uint8Array(640 * 480 * 4), { format: i % 2 ? 'RGBX' : 'RGBA', codedWidth: 640, codedHeight: 480, @@ -235,7 +235,7 @@ test('Can encode video with alternating transparency', async () => { const sampleSink = new VideoSampleSink(videoTrack); i = 0; - for await (const sample of sampleSink.samples()) { + for await (using sample of sampleSink.samples()) { if (i % 2) { expect(sample.format).not.toContain('A'); } else { @@ -299,7 +299,7 @@ test('Can transmux transparent video, discards alpha by default', async () => { expect(await videoTrack.canBeTransparent()).toBe(false); const sink = new VideoSampleSink(videoTrack); - const sample = (await sink.getSample(await videoTrack.getFirstTimestamp()))!; + using sample = (await sink.getSample(await videoTrack.getFirstTimestamp()))!; expect(sample.hasAlpha).toBe(false); }); @@ -331,7 +331,7 @@ test('Can transmux transparent video, can keep alpha', async () => { expect(await videoTrack.canBeTransparent()).toBe(true); const sink = new VideoSampleSink(videoTrack); - const sample = (await sink.getSample(await videoTrack.getFirstTimestamp()))!; + using sample = (await sink.getSample(await videoTrack.getFirstTimestamp()))!; expect(sample.format).toContain('A'); expect(sample.hasAlpha).toBe(true); }); @@ -370,7 +370,7 @@ test('Can reencode transparent video, keeping alpha', async () => { expect(videoTrack.displayWidth).toBe(320); const sink = new VideoSampleSink(videoTrack); - const sample = (await sink.getSample(await videoTrack.getFirstTimestamp()))!; + using sample = (await sink.getSample(await videoTrack.getFirstTimestamp()))!; expect(sample.format).toContain('A'); expect(sample.hasAlpha).toBe(true); }); diff --git a/test/browser/video-samples.test.ts b/test/browser/video-samples.test.ts new file mode 100644 index 0000000..b0c12e6 --- /dev/null +++ b/test/browser/video-samples.test.ts @@ -0,0 +1,220 @@ +import { expect, test } from 'vitest'; +import { VideoSample } from '../../src/sample.js'; + +test('allocationSize', async () => { + { + const canvas = new OffscreenCanvas(1280, 720); + canvas.getContext('2d'); + + using sample = new VideoSample(canvas, { timestamp: 0 }); + + const size1 = sample.allocationSize(); + expect(size1).toBe(1280 * 720 * 4); + + const size2 = sample.allocationSize({ + layout: [{ + offset: 0, + stride: 1300 * 4, + }], + }); + expect(size2).toBe(1300 * 720 * 4); + } + + { + const data = new Uint8Array(1280 * 720 * 4); + using sample = new VideoSample(data, { + timestamp: 0, + codedWidth: 1280, + codedHeight: 720, + format: 'RGBA', + }); + + const size1 = sample.allocationSize(); + expect(size1).toBe(1280 * 720 * 4); + + const size2 = sample.allocationSize({ + layout: [{ + offset: 0, + stride: 1300 * 4, + }], + }); + expect(size2).toBe(1300 * 720 * 4); + } + + { + const data = new Uint8Array(1280 * 720 * 1.5); + using sample = new VideoSample(data, { + timestamp: 0, + codedWidth: 1280, + codedHeight: 720, + format: 'I420', + }); + + const size1 = sample.allocationSize(); + expect(size1).toBe(1280 * 720 * 1.5); + + const size2 = sample.allocationSize({ format: 'RGBA' }); + expect(size2).toBe(1280 * 720 * 4); + + const size3 = sample.allocationSize({ + format: 'RGBA', + layout: [{ + offset: 0, + stride: 1300 * 4, + }], + }); + expect(size3).toBe(1300 * 720 * 4); + } +}); + +test('copyTo and plane layouts', async () => { + const buffer = new ArrayBuffer(1e7); + + { + const canvas = new OffscreenCanvas(1280, 720); + canvas.getContext('2d'); + + using sample = new VideoSample(canvas, { timestamp: 0 }); + const layout = await sample.copyTo(buffer); + + expect(layout).toEqual([{ + offset: 0, + stride: 1280 * 4, + }]); + } + + { + const canvas = new OffscreenCanvas(1280, 720); + canvas.getContext('2d'); + + using sample = new VideoSample(canvas, { timestamp: 0 }); + const layout = await sample.copyTo(buffer, { + layout: [{ + offset: 0, + stride: 1300 * 4, + }], + }); + + expect(layout).toEqual([{ + offset: 0, + stride: 1300 * 4, + }]); + } + + { + const data = new Uint8Array(1280 * 720 * 4); + using sample = new VideoSample(data, { + timestamp: 0, + codedWidth: 1280, + codedHeight: 720, + format: 'RGBA', + }); + const layout = await sample.copyTo(buffer); + + expect(layout).toEqual([{ + offset: 0, + stride: 1280 * 4, + }]); + } + + { + const data = new Uint8Array(1280 * 720 * 4); + using sample = new VideoSample(data, { + timestamp: 0, + codedWidth: 1280, + codedHeight: 720, + format: 'RGBA', + }); + const layout = await sample.copyTo(buffer, { + layout: [{ + offset: 0, + stride: 1300 * 4, + }], + }); + + expect(layout).toEqual([{ + offset: 0, + stride: 1300 * 4, + }]); + } + + { + const data = new Uint8Array(1300 * 720 * 4); + using sample = new VideoSample(data, { + timestamp: 0, + codedWidth: 1280, + codedHeight: 720, + format: 'RGBA', + layout: [{ + offset: 0, + stride: 1300 * 4, + }], + }); + const layout = await sample.copyTo(buffer); + + expect(layout).toEqual([{ + offset: 0, + stride: 1300 * 4, + }]); + + using clone = sample.clone(); + const clonedLayout = await clone.copyTo(buffer); + + expect(clonedLayout).toEqual([{ + offset: 0, + stride: 1300 * 4, + }]); + } + + { + const data = new Uint8Array(1280 * 720 * 1.5); + using sample = new VideoSample(data, { + timestamp: 0, + codedWidth: 1280, + codedHeight: 720, + format: 'I420', + }); + const layout = await sample.copyTo(buffer); + + expect(layout).toEqual([{ + offset: 0, + stride: 1280, + }, { + offset: 1280 * 720, + stride: 1280 / 2, + }, { + offset: 1280 * 720 + (1280 / 2) * (720 / 2), + stride: 1280 / 2, + }]); + } +}); + +test('null format', async () => { + const canvas = new OffscreenCanvas(1280, 720); + canvas.getContext('2d'); + + const frame = new VideoFrame(canvas, { timestamp: 0 }); + using sample = new VideoSample(frame); + // @ts-expect-error Sybau + sample.format = null; + + expect(() => sample.allocationSize()).toThrow('when format is null'); + await expect(async () => sample.copyTo(new ArrayBuffer())).rejects.toThrow('when format is null'); + + const size = sample.allocationSize({ format: 'RGBA' }); + expect(size).toBe(1280 * 720 * 4); + const buffer = new ArrayBuffer(size); + const layout = await sample.copyTo(buffer, { format: 'RGBA' }); + + expect(layout).toEqual([{ + offset: 0, + stride: 1280 * 4, + }]); + + sample.allocationSize({ format: 'RGBX' }); + sample.allocationSize({ format: 'BGRA' }); + sample.allocationSize({ format: 'BGRX' }); + await sample.copyTo(buffer, { format: 'RGBX' }); + await sample.copyTo(buffer, { format: 'BGRA' }); + await sample.copyTo(buffer, { format: 'BGRX' }); +}); diff --git a/test/node/annex-b-conversion.test.ts b/test/node/annex-b-conversion.test.ts new file mode 100644 index 0000000..3e86555 --- /dev/null +++ b/test/node/annex-b-conversion.test.ts @@ -0,0 +1,54 @@ +import { expect, test } from 'vitest'; +import { Input } from '../../src/input.js'; +import { BufferSource, FilePathSource } from '../../src/source.js'; +import path from 'node:path'; +import { ALL_FORMATS } from '../../src/input-format.js'; +import { Output } from '../../src/output.js'; +import { Mp4OutputFormat } from '../../src/output-format.js'; +import { BufferTarget } from '../../src/target.js'; +import { Conversion } from '../../src/conversion.js'; +import { EncodedPacketSink } from '../../src/media-sink.js'; +import { extractAvcNalUnits } from '../../src/codec-data.js'; + +const __dirname = new URL('.', import.meta.url).pathname; + +test('Annex B to length-prefixed conversion, MP4', async () => { + using originalInput = new Input({ + source: new FilePathSource(path.join(__dirname, '..', 'public/annex-b-avc.mkv')), + formats: ALL_FORMATS, + }); + const originalVideoTrack = (await originalInput.getPrimaryVideoTrack())!; + const originalDecoderConfig = (await originalVideoTrack.getDecoderConfig())!; + expect(originalDecoderConfig.description).toBeUndefined(); + expect(originalVideoTrack.codec).toBe('avc'); + + const originalSink = new EncodedPacketSink(originalVideoTrack); + const originalFirstPacket = await originalSink.getFirstPacket(); + expect([...originalFirstPacket!.data.slice(0, 4)]).toEqual([0, 0, 0, 1]); + + const originalNalUnits = extractAvcNalUnits(originalFirstPacket!.data, originalDecoderConfig); + + const output = new Output({ + format: new Mp4OutputFormat(), + target: new BufferTarget(), + }); + + const conversion = await Conversion.init({ input: originalInput, output }); + await conversion.execute(); + + using newInput = new Input({ + source: new BufferSource(output.target.buffer!), + formats: ALL_FORMATS, + }); + const newVideoTrack = (await newInput.getPrimaryVideoTrack())!; + const newDecoderConfig = (await newVideoTrack.getDecoderConfig())!; + expect(newDecoderConfig.description).toBeDefined(); + expect(newVideoTrack.codec).toBe('avc'); + + const newSink = new EncodedPacketSink(newVideoTrack); + const newFirstPacket = await newSink.getFirstPacket(); + expect([...newFirstPacket!.data.slice(0, 4)]).not.toEqual([0, 0, 0, 1]); // Successfully converted + + const newNalUnits = extractAvcNalUnits(newFirstPacket!.data, newDecoderConfig); + expect(newNalUnits).toEqual(originalNalUnits); // Content is the same though +}); diff --git a/test/node/disposition.test.ts b/test/node/disposition.test.ts new file mode 100644 index 0000000..18e0b2e --- /dev/null +++ b/test/node/disposition.test.ts @@ -0,0 +1,92 @@ +import { expect, test } from 'vitest'; +import { Output } from '../../src/output.js'; +import { MkvOutputFormat } from '../../src/output-format.js'; +import { BufferTarget } from '../../src/target.js'; +import { EncodedVideoPacketSource } from '../../src/media-source.js'; +import { EncodedPacket } from '../../src/packet.js'; +import { Input } from '../../src/input.js'; +import { BufferSource } from '../../src/source.js'; +import { ALL_FORMATS } from '../../src/input-format.js'; + +test('Default track disposition', async () => { + const output = new Output({ + format: new MkvOutputFormat(), + target: new BufferTarget(), + }); + + const source = new EncodedVideoPacketSource('avc'); + output.addVideoTrack(source); + + await output.start(); + await source.add(new EncodedPacket(new Uint8Array(1024), 'key', 0, 1), { + decoderConfig: { + codec: 'avc1.123456', + codedWidth: 1920, + codedHeight: 1080, + }, + }); + + await output.finalize(); + + using input = new Input({ + source: new BufferSource(output.target.buffer!), + formats: ALL_FORMATS, + }); + + const track = (await input.getPrimaryVideoTrack())!; + + expect(track.disposition).toEqual({ + default: true, + forced: false, + original: false, + hearingImpaired: false, + visuallyImpaired: false, + commentary: false, + }); +}); + +test('Customized track disposition', async () => { + const output = new Output({ + format: new MkvOutputFormat(), + target: new BufferTarget(), + }); + + const source = new EncodedVideoPacketSource('avc'); + output.addVideoTrack(source, { + disposition: { + default: false, + forced: true, + original: true, + hearingImpaired: true, + visuallyImpaired: true, + commentary: true, + }, + }); + + await output.start(); + await source.add(new EncodedPacket(new Uint8Array(1024), 'key', 0, 1), { + decoderConfig: { + codec: 'avc1.123456', + codedWidth: 1920, + codedHeight: 1080, + }, + }); + + await output.finalize(); + + using input = new Input({ + source: new BufferSource(output.target.buffer!), + formats: ALL_FORMATS, + }); + + const track = (await input.getPrimaryVideoTrack())!; + + expect(track.disposition).toEqual({ + default: false, + forced: true, + original: true, + hearingImpaired: true, + visuallyImpaired: true, + commentary: true, + }); +}); diff --git a/test/node/metadata-tags.test.ts b/test/node/metadata-tags.test.ts index ae7ceb2..d9e50c2 100644 --- a/test/node/metadata-tags.test.ts +++ b/test/node/metadata-tags.test.ts @@ -15,7 +15,7 @@ import { EncodedPacket } from '../../src/packet.js'; import { Input } from '../../src/input.js'; import { BufferSource, FilePathSource } from '../../src/source.js'; import { ALL_FORMATS } from '../../src/input-format.js'; -import { AttachedFile, MetadataTags } from '../../src/tags.js'; +import { AttachedFile, MetadataTags } from '../../src/metadata.js'; import path from 'node:path'; import { AudioCodec, buildAudioCodecString } from '../../src/codec.js'; import { Conversion } from '../../src/conversion.js'; diff --git a/test/node/node-sample-creation.test.ts b/test/node/node-sample-creation.test.ts new file mode 100644 index 0000000..0cf4064 --- /dev/null +++ b/test/node/node-sample-creation.test.ts @@ -0,0 +1,27 @@ +import { test } from 'vitest'; +import { AudioSample, VideoSample } from '../../src/sample.js'; + +test('VideoSample creation from bytes', async () => { + const bytes = new Uint8Array(1024); + const sample = new VideoSample(bytes, { + codedWidth: 1280, + codedHeight: 720, + format: 'RGBA', + timestamp: 0, + }); + + sample.close(); +}); + +test('AudioSample creation from bytes', async () => { + const bytes = new Uint8Array(1024); + const sample = new AudioSample({ + data: bytes, + numberOfChannels: 2, + format: 's16', + sampleRate: 48000, + timestamp: 0, + }); + + sample.close(); +}); diff --git a/test/public/annex-b-avc.mkv b/test/public/annex-b-avc.mkv new file mode 100644 index 0000000..699dfd2 Binary files /dev/null and b/test/public/annex-b-avc.mkv differ