diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 744c4a1..550b77c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -75,6 +75,11 @@ jobs: packages/ac3/dist/bundles/mediabunny-ac3.mjs packages/ac3/dist/bundles/mediabunny-ac3.min.mjs packages/ac3/dist/mediabunny-ac3.d.ts + packages/dts/dist/bundles/mediabunny-dts.js + packages/dts/dist/bundles/mediabunny-dts.min.js + packages/dts/dist/bundles/mediabunny-dts.mjs + packages/dts/dist/bundles/mediabunny-dts.min.mjs + packages/dts/dist/mediabunny-dts.d.ts packages/aac-encoder/dist/bundles/mediabunny-aac-encoder.js packages/aac-encoder/dist/bundles/mediabunny-aac-encoder.min.js packages/aac-encoder/dist/bundles/mediabunny-aac-encoder.mjs diff --git a/.gitignore b/.gitignore index 30c66a0..8546073 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ node_modules packages/mp3-encoder/dist packages/ac3/dist +packages/dts/dist packages/aac-encoder/dist packages/flac-encoder/dist packages/server/dist diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 9f6a58e..24dbe67 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -125,6 +125,7 @@ export default withMermaid({ { text: 'mp3-encoder', link: '/guide/extensions/mp3-encoder' }, { text: 'aac-encoder', link: '/guide/extensions/aac-encoder' }, { text: 'ac3', link: '/guide/extensions/ac3' }, + { text: 'dts', link: '/guide/extensions/dts' }, { text: 'flac-encoder', link: '/guide/extensions/flac-encoder' }, { text: 'prores', link: '/guide/extensions/prores' }, ], @@ -162,6 +163,7 @@ export default withMermaid({ { text: 'FLAC', link: '/codec-registry/flac' }, { text: 'AC-3', link: '/codec-registry/ac3' }, { text: 'E-AC-3', link: '/codec-registry/eac3' }, + { text: 'DTS', link: '/codec-registry/dts' }, { text: 'Linear PCM', link: '/codec-registry/pcm' }, { text: 'μ-law PCM', link: '/codec-registry/ulaw' }, { text: 'A-law PCM', link: '/codec-registry/alaw' }, diff --git a/docs/api-config.json b/docs/api-config.json index b56656e..39b9878 100644 --- a/docs/api-config.json +++ b/docs/api-config.json @@ -25,6 +25,7 @@ "@mediabunny/server": "Adds full video/audio decoder and encoder support to Mediabunny running in server-side environments such as Node, Bun, or Deno.", "@mediabunny/mp3-encoder": "Adds MP3 encoder support to Mediabunny.", "@mediabunny/ac3": "Adds AC-3/E-AC-3 decoder and encoder support to Mediabunny.", + "@mediabunny/dts": "Adds DTS decoder and encoder support to Mediabunny.", "@mediabunny/aac-encoder": "Polyfills AAC encoder support to Mediabunny.", "@mediabunny/flac-encoder": "Adds FLAC encoder support to Mediabunny.", "@mediabunny/prores": "Adds Apple ProRes decoder support to Mediabunny." diff --git a/docs/codec-registry/dts.md b/docs/codec-registry/dts.md new file mode 100644 index 0000000..88ef8b2 --- /dev/null +++ b/docs/codec-registry/dts.md @@ -0,0 +1,44 @@ +--- +description: DTS audio codec definition, defining legal codec strings, decoder configs, and packet data formats. +--- + + + + + +# DTS codec registration + +## Description + +The DTS audio codec (DTS Coherent Acoustics), specified in [ETSI TS 102 114](https://www.etsi.org/deliver/etsi_ts/102100_102199/102114/01.06.01_60/ts_102114v010601p.pdf). + +## Codec ID + +```ts +'dts' +``` + +## `EncodedPacket` data + +The packet's data must be a core substream frame as defined in Section 5 of [ETSI TS 102 114](https://www.etsi.org/deliver/etsi_ts/102100_102199/102114/01.06.01_60/ts_102114v010601p.pdf), beginning with the sync word `0x7FFE8001`, followed by any number of extension substreams as defined in Section 7.4.1 of [ETSI TS 102 114](https://www.etsi.org/deliver/etsi_ts/102100_102199/102114/01.06.01_60/ts_102114v010601p.pdf), each beginning with the sync word `0x64582025`. The core substream frame may be omitted. + +The bitstream must use 16-bit big-endian packing. + +## `EncodedPacket` type + +The packet's type is always `'key'`. + +## `AudioDecoderConfig` codec string + +The codec string must be one of the four four-character codes identifying which substreams the bitstream is built from: + +- `'dtsc'` - a core substream only +- `'dtsh'` - a core substream with extension substreams +- `'dtsl'` - extension substreams carrying lossless audio, with no core substream +- `'dtse'` - DTS Express + +## `AudioDecoderConfig` description + +`description` is not used for this codec. diff --git a/docs/codec-registry/overview.md b/docs/codec-registry/overview.md index bae0fb4..85e5b27 100644 --- a/docs/codec-registry/overview.md +++ b/docs/codec-registry/overview.md @@ -26,6 +26,7 @@ The registry is an extension of the [WebCodecs Codec Registry](https://www.w3.or - [FLAC](./flac) - [AC-3](./ac3) - [E-AC-3](./eac3) +- [DTS](./dts) - [Linear PCM](./pcm) - [μ-law PCM](./ulaw) - [A-law PCM](./alaw) diff --git a/docs/guide/extensions/dts.md b/docs/guide/extensions/dts.md new file mode 100644 index 0000000..90f0161 --- /dev/null +++ b/docs/guide/extensions/dts.md @@ -0,0 +1,37 @@ +--- +description: The @mediabunny/dts extension provides fast DTS decoders and encoders for both browser and server environments. +--- + +# @mediabunny/dts + +Browsers have no support for the DTS audio codec in their WebCodecs implementations. This extension package provides both a decoder and encoder for use with Mediabunny, allowing you to decode and encode this codec directly in the browser. It is implemented using Mediabunny's [custom coder API](../supported-formats-and-codecs#custom-coders) and uses a fast, size-optimized WASM build of [FFmpeg](https://ffmpeg.org/)'s DTS coders under the hood. + + + GitHub page + + + +## Installation + +This library peer-depends on Mediabunny. Install both using npm: +```bash +npm install mediabunny @mediabunny/dts +``` + +Alternatively, directly include them using a script tag: +```html + + +``` + +This will expose the global objects `Mediabunny` and `MediabunnyDts`. Use `mediabunny-dts.d.ts` to provide types for these globals. You can download the built distribution files from the [releases page](https://github.com/Vanilagy/mediabunny/releases). + +## Usage + +```ts +import { registerDtsDecoder, registerDtsEncoder } from '@mediabunny/dts'; + +registerDtsDecoder(); +registerDtsEncoder(); +``` +That's it - Mediabunny now uses the registered DTS decoder and encoder automatically. diff --git a/docs/guide/extensions/server.md b/docs/guide/extensions/server.md index 9d3754a..aec0681 100644 --- a/docs/guide/extensions/server.md +++ b/docs/guide/extensions/server.md @@ -8,7 +8,7 @@ By default, Mediabunny requires a browser environment for full access to decoder Features added by this package include: - Video decoders and encoders for AVC (H.264), HEVC (H.265), VP8, VP9, AV1, and ProRes. Supports both length-prefixed and Annex B AVC/HEVC as well as transparent video via VP9 and ProRes. -- Audio decoders and encoders for AAC, MP3, Vorbis, Opus, FLAC, AC-3 and E-AC-3. Supports AAC in both AAC and ADTS formats. +- Audio decoders and encoders for AAC, MP3, Vorbis, Opus, FLAC, AC-3, E-AC-3 and DTS. Supports AAC in both AAC and ADTS formats. - Video frame transformation support (resize, rotate, crop) - Automatic hardware acceleration on all platforms (macOS, Linux, Windows) - Built-in multithreading diff --git a/docs/guide/introduction.md b/docs/guide/introduction.md index 0e66c84..d2ec32a 100644 --- a/docs/guide/introduction.md +++ b/docs/guide/introduction.md @@ -64,7 +64,7 @@ Mediabunny's simple yet flexible API provides a modern alternative to traditiona The extension enables: - Video decoders and encoders for AVC (H.264), HEVC (H.265), VP8, VP9, and AV1. Supports both length-prefixed and Annex B AVC/HEVC as well as transparent video via VP9. -- Audio decoders and encoders for AAC, MP3, Vorbis, Opus, FLAC, AC-3 and E-AC-3. Supports AAC in both AAC and ADTS formats. +- Audio decoders and encoders for AAC, MP3, Vorbis, Opus, FLAC, AC-3, E-AC-3 and DTS. Supports AAC in both AAC and ADTS formats. - Video frame transformation support (resize, rotate, crop) - Automatic hardware acceleration on all platforms (macOS, Linux, Windows) - Built-in multithreading diff --git a/docs/guide/supported-formats-and-codecs.md b/docs/guide/supported-formats-and-codecs.md index d420bf0..bc4c537 100644 --- a/docs/guide/supported-formats-and-codecs.md +++ b/docs/guide/supported-formats-and-codecs.md @@ -51,6 +51,7 @@ Mediabunny ships with built-in decoders and encoders for all audio PCM codecs, m - `'flac'` - Free Lossless Audio Codec (FLAC) [^flac] - `'ac3'` - Dolby Digital (AC-3) [^ac3] - `'eac3'` - Dolby Digital Plus (E-AC-3) [^ac3] +- `'dts'` - DTS Coherent Acoustics [^dts] - `'pcm-u8'` - 8-bit unsigned PCM - `'pcm-s8'` - 8-bit signed PCM - `'pcm-s16'` - 16-bit little-endian signed PCM @@ -89,6 +90,7 @@ Not all codecs can be used with all containers. The following table specifies th | `'flac'` | ✓ | ✓ | ✓ | | | | | | ✓ | | | `'ac3'` | ✓ | ✓ | ✓ | | | | | | | ✓ | | `'eac3'` | ✓ | ✓ | ✓ | | | | | | | ✓ | +| `'dts'` | ✓ | ✓ | ✓ | | | | | | | ✓ | | `'pcm-u8'` | | ✓ | ✓ | | | | ✓ | | | | | `'pcm-s8'` | | ✓ | | | | | | | | | | `'pcm-s16'` | ✓ | ✓ | ✓ | | | | ✓ | | | | @@ -112,6 +114,7 @@ For HLS, the supported codecs depend on the segment format chosen. [^mp3]: MP3 encoding is not supported by WebCodecs. You can polyfill it with the [`@mediabunny/mp3-encoder`](./extensions/mp3-encoder) extension package. [^flac]: FLAC encoding is not supported by WebCodecs. You can polyfill it with the [`@mediabunny/flac-encoder`](./extensions/flac-encoder) extension package. [^ac3]: AC-3 and E-AC-3 are not natively supported by WebCodecs. To encode or decode these codecs, you can use the [`@mediabunny/ac3`](./extensions/ac3) extension package. +[^dts]: DTS is not natively supported by WebCodecs. To encode or decode it, you can use the [`@mediabunny/dts`](./extensions/dts) extension package. [^webm]: WebM only supports a small subset of the codecs supported by Matroska. However, this library can technically read all codecs from a WebM that are supported by Matroska. [^webvtt]: WebVTT can only be written, not read. @@ -127,7 +130,7 @@ import { canEncode } from 'mediabunny'; canEncode('avc'); // => Promise canEncode('opus'); // => Promise ``` -Video codecs are checked using 1280x720 @1Mbps, while audio codecs are checked using 2 channels, 48 kHz @128kbps. +Video codecs are checked using 1280x720, while audio codecs are checked using 2 channels at 48 kHz. You can also check encodability using specific configurations: ```ts diff --git a/eslint.config.mjs b/eslint.config.mjs index 6a9caaf..3c24022 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -43,6 +43,8 @@ export default tseslint.config( 'packages/mp3-encoder/build', 'packages/ac3/dist', 'packages/ac3/build', + 'packages/dts/dist', + 'packages/dts/build', 'packages/aac-encoder/dist', 'packages/aac-encoder/build', 'packages/flac-encoder/dist', diff --git a/examples/file-compression/file-compression.ts b/examples/file-compression/file-compression.ts index 5f9370b..cba16e8 100644 --- a/examples/file-compression/file-compression.ts +++ b/examples/file-compression/file-compression.ts @@ -10,12 +10,14 @@ import { UrlSource, } from 'mediabunny'; import { registerAc3Decoder } from '@mediabunny/ac3'; +import { registerDtsDecoder } from '@mediabunny/dts'; import { registerProresDecoder } from '@mediabunny/prores'; import SampleFileUrl from '../../docs/assets/big-buck-bunny-trimmed.mp4'; // Enable codecs that aren't natively supported by WebCodecs. registerAc3Decoder(); +registerDtsDecoder(); registerProresDecoder(); (document.querySelector('#sample-file-download') as HTMLAnchorElement).href = SampleFileUrl; diff --git a/examples/hls-transcoding/hls-transcoding.ts b/examples/hls-transcoding/hls-transcoding.ts index fc13c1d..a85fdde 100644 --- a/examples/hls-transcoding/hls-transcoding.ts +++ b/examples/hls-transcoding/hls-transcoding.ts @@ -12,12 +12,14 @@ import { UrlSource, } from 'mediabunny'; import { registerAc3Decoder } from '@mediabunny/ac3'; +import { registerDtsDecoder } from '@mediabunny/dts'; import { registerProresDecoder } from '@mediabunny/prores'; import SampleFileUrl from '../../docs/assets/big-buck-bunny-trimmed.mp4'; // Enable codecs that aren't natively supported by WebCodecs. registerAc3Decoder(); +registerDtsDecoder(); registerProresDecoder(); (document.querySelector('#sample-file-download') as HTMLAnchorElement).href = SampleFileUrl; diff --git a/examples/media-player/media-player.ts b/examples/media-player/media-player.ts index 6d851e2..1572163 100644 --- a/examples/media-player/media-player.ts +++ b/examples/media-player/media-player.ts @@ -9,12 +9,14 @@ import { WrappedCanvas, } from 'mediabunny'; import { registerAc3Decoder } from '@mediabunny/ac3'; +import { registerDtsDecoder } from '@mediabunny/dts'; import { registerProresDecoder } from '@mediabunny/prores'; import SampleFileUrl from '../../docs/assets/big-buck-bunny-trimmed.mp4'; // Enable codecs that aren't natively supported by WebCodecs. registerAc3Decoder(); +registerDtsDecoder(); registerProresDecoder(); (document.querySelector('#sample-file-download') as HTMLAnchorElement).href = SampleFileUrl; diff --git a/examples/thumbnail-generation/thumbnail-generation.ts b/examples/thumbnail-generation/thumbnail-generation.ts index 69a995a..7add9bc 100644 --- a/examples/thumbnail-generation/thumbnail-generation.ts +++ b/examples/thumbnail-generation/thumbnail-generation.ts @@ -1,11 +1,13 @@ import { Input, ALL_FORMATS, BlobSource, UrlSource, CanvasSink } from 'mediabunny'; import { registerAc3Decoder } from '@mediabunny/ac3'; +import { registerDtsDecoder } from '@mediabunny/dts'; import { registerProresDecoder } from '@mediabunny/prores'; import SampleFileUrl from '../../docs/assets/big-buck-bunny-trimmed.mp4'; // Enable codecs that aren't natively supported by WebCodecs. registerAc3Decoder(); +registerDtsDecoder(); registerProresDecoder(); (document.querySelector('#sample-file-download') as HTMLAnchorElement).href = SampleFileUrl; diff --git a/package-lock.json b/package-lock.json index e8a6e4e..63832d0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1453,6 +1453,10 @@ "resolved": "packages/ac3", "link": true }, + "node_modules/@mediabunny/dts": { + "resolved": "packages/dts", + "link": true + }, "node_modules/@mediabunny/flac-encoder": { "resolved": "packages/flac-encoder", "link": true @@ -12982,6 +12986,21 @@ "mediabunny": "^1.0.0" } }, + "packages/dts": { + "name": "@mediabunny/dts", + "version": "1.54.0", + "license": "MPL-2.0", + "devDependencies": { + "@types/emscripten": "^1.40.1" + }, + "funding": { + "type": "individual", + "url": "https://github.com/sponsors/Vanilagy" + }, + "peerDependencies": { + "mediabunny": "^1.0.0" + } + }, "packages/flac-encoder": { "name": "@mediabunny/flac-encoder", "version": "1.54.0", diff --git a/package.json b/package.json index b3ea4ae..651137d 100644 --- a/package.json +++ b/package.json @@ -51,7 +51,7 @@ "docs:dev": "vitepress dev docs", "docs:build": "npm run build && npm run docs:generate && vitepress build docs && npm run examples:build && cp dist/mediabunny.d.ts dist-docs/", "docs:preview": "vitepress preview docs", - "docs:generate": "tsx scripts/generate-api-docs.ts src/index.ts packages/mp3-encoder/src/index.ts packages/ac3/src/index.ts packages/aac-encoder/src/index.ts packages/flac-encoder/src/index.ts packages/prores/src/index.ts packages/server/src/index.ts docs/api-config.json", + "docs:generate": "tsx scripts/generate-api-docs.ts src/index.ts packages/mp3-encoder/src/index.ts packages/ac3/src/index.ts packages/dts/src/index.ts packages/aac-encoder/src/index.ts packages/flac-encoder/src/index.ts packages/prores/src/index.ts packages/server/src/index.ts docs/api-config.json", "dev": "vite", "examples:build": "vite build", "fix-build-import-paths": "tsx scripts/add-import-extensions.ts", diff --git a/packages/aac-encoder/package.json b/packages/aac-encoder/package.json index 0efade3..4cf829a 100644 --- a/packages/aac-encoder/package.json +++ b/packages/aac-encoder/package.json @@ -2,7 +2,7 @@ "name": "@mediabunny/aac-encoder", "author": "Vanilagy", "version": "1.54.0", - "description": "AAC encoder extension for Mediabunny, based on FFmpeg.", + "description": "AAC encoder extension for Mediabunny, based on libavcodec.", "main": "./dist/bundles/mediabunny-aac-encoder.mjs", "module": "./dist/bundles/mediabunny-aac-encoder.mjs", "types": "./dist/modules/src/index.d.ts", diff --git a/packages/ac3/package.json b/packages/ac3/package.json index eb7be18..2b2fbe6 100644 --- a/packages/ac3/package.json +++ b/packages/ac3/package.json @@ -2,7 +2,7 @@ "name": "@mediabunny/ac3", "author": "Vanilagy", "version": "1.54.0", - "description": "AC-3 and E-AC-3 (Dolby Digital) decoder and encoder extension for Mediabunny, based on FFmpeg.", + "description": "AC-3 and E-AC-3 (Dolby Digital) decoder and encoder extension for Mediabunny, based on libavcodec.", "main": "./dist/bundles/mediabunny-ac3.mjs", "module": "./dist/bundles/mediabunny-ac3.mjs", "types": "./dist/modules/src/index.d.ts", diff --git a/packages/dts/.gitattributes b/packages/dts/.gitattributes new file mode 100644 index 0000000..72ef19a --- /dev/null +++ b/packages/dts/.gitattributes @@ -0,0 +1 @@ +build/* linguist-generated diff --git a/packages/dts/LICENSE b/packages/dts/LICENSE new file mode 100644 index 0000000..d0a1fa1 --- /dev/null +++ b/packages/dts/LICENSE @@ -0,0 +1,373 @@ +Mozilla Public License Version 2.0 +================================== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at https://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. diff --git a/packages/dts/README.md b/packages/dts/README.md new file mode 100644 index 0000000..7f74a3e --- /dev/null +++ b/packages/dts/README.md @@ -0,0 +1,106 @@ +# @mediabunny/dts + +[![](https://img.shields.io/npm/v/@mediabunny/dts)](https://www.npmjs.com/package/@mediabunny/dts) +[![](https://img.shields.io/bundlephobia/minzip/@mediabunny/dts)](https://bundlephobia.com/package/@mediabunny/dts) +[![](https://img.shields.io/npm/dm/@mediabunny/dts)](https://www.npmjs.com/package/@mediabunny/dts) +[![](https://img.shields.io/discord/1390044844285497344?logo=discord&label=Discord)](https://discord.gg/hmpkyYuS4U) + +
+ +
+ +Browsers have no support for the DTS audio codec in their WebCodecs implementations. This extension package provides both a decoder and encoder for use with [Mediabunny](https://github.com/Vanilagy/mediabunny), allowing you to decode and encode this codec directly in the browser. It is implemented using Mediabunny's [custom coder API](https://mediabunny.dev/guide/supported-formats-and-codecs#custom-coders) and uses a fast, size-optimized WASM build of [FFmpeg](https://ffmpeg.org/)'s DTS coders under the hood. + +> This package, like the rest of Mediabunny, is enabled by its [sponsors](https://mediabunny.dev/#sponsors) and their donations. If you've derived value from this package, please consider [leaving a donation](https://github.com/sponsors/Vanilagy)! 💘 + +## Installation + +This library peer-depends on Mediabunny. Install both using npm: +```bash +npm install mediabunny @mediabunny/dts +``` + +Alternatively, directly include them using a script tag: +```html + + +``` + +This will expose the global objects `Mediabunny` and `MediabunnyDts`. Use `mediabunny-dts.d.ts` to provide types for these globals. You can download the built distribution files from the [releases page](https://github.com/Vanilagy/mediabunny/releases). + +## Usage + +```ts +import { registerDtsDecoder, registerDtsEncoder } from '@mediabunny/dts'; + +registerDtsDecoder(); +registerDtsEncoder(); +``` +That's it - Mediabunny now uses the registered DTS decoder and encoder automatically. + +## Building and development + +For simplicity, all built WASM artifacts are included in the repo, since these rarely change. However, here are the instructions for building them from scratch: + +[Install Emscripten](https://emscripten.org/docs/getting_started/downloads.html) and clone [FFmpeg](https://github.com/FFmpeg/FFmpeg). Then, from the Mediabunny root and with Emscripten sourced in: + +```bash +export FFMPEG_PATH=/path/to/ffmpeg +export MEDIABUNNY_ROOT=$PWD + +# Build FFmpeg +cd $FFMPEG_PATH +emmake make distclean +emconfigure ./configure \ + --target-os=none \ + --arch=x86_32 \ + --enable-cross-compile \ + --disable-asm \ + --disable-x86asm \ + --disable-inline-asm \ + --disable-programs \ + --disable-doc \ + --disable-debug \ + --disable-all \ + --disable-everything \ + --disable-autodetect \ + --disable-pthreads \ + --disable-runtime-cpudetect \ + --enable-avcodec \ + --enable-decoder=dca \ + --enable-encoder=dca \ + --cc="emcc" \ + --cxx=em++ \ + --ar=emar \ + --ranlib=emranlib \ + --extra-cflags="-DNDEBUG -Oz -flto -msimd128" \ + --extra-ldflags="-Oz -flto" +emmake make + +# Compile the bridge between JavaScript and FFmpeg's API +cd $MEDIABUNNY_ROOT/packages/dts +emcc src/bridge.c \ + $FFMPEG_PATH/libavcodec/libavcodec.a \ + $FFMPEG_PATH/libavutil/libavutil.a \ + -I$FFMPEG_PATH \ + -s MODULARIZE=1 \ + -s EXPORT_ES6=1 \ + -s SINGLE_FILE=1 \ + -s ALLOW_MEMORY_GROWTH=1 \ + -s ENVIRONMENT=web,worker \ + -s FILESYSTEM=0 \ + -s MALLOC=emmalloc \ + -s SUPPORT_LONGJMP=0 \ + -s EXPORTED_RUNTIME_METHODS=cwrap,HEAPU8 \ + -s EXPORTED_FUNCTIONS=_malloc,_free \ + -msimd128 \ + -flto \ + -Oz \ + -o build/dts.js +``` + +This generates `build/dts.js`, which contains both the JavaScript "glue code" as well as the compiled WASM inlined. + +### Building the package + +Then, the complete JavaScript package can be built alongside the rest of Mediabunny by running `npm run build` in Mediabunny's root. diff --git a/packages/dts/api-extractor.json b/packages/dts/api-extractor.json new file mode 100644 index 0000000..8cb42c0 --- /dev/null +++ b/packages/dts/api-extractor.json @@ -0,0 +1,37 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", + "mainEntryPointFilePath": "dist/modules/src/index.d.ts", + "bundledPackages": [], + "compiler": {}, + "apiReport": { + "enabled": false + }, + "docModel": { + "enabled": false + }, + "dtsRollup": { + "enabled": true, + "untrimmedFilePath": "dist/mediabunny-dts.d.ts" + }, + "tsdocMetadata": { + "enabled": false + }, + "messages": { + "compilerMessageReporting": { + "default": { + "logLevel": "warning" + } + }, + "extractorMessageReporting": { + "default": { + "logLevel": "warning" + } + }, + "tsdocMessageReporting": { + "default": { + "logLevel": "warning" + } + } + }, + "newlineKind": "lf" +} diff --git a/packages/dts/build/dts.js b/packages/dts/build/dts.js new file mode 100644 index 0000000..ee43507 Binary files /dev/null and b/packages/dts/build/dts.js differ diff --git a/packages/dts/package.json b/packages/dts/package.json new file mode 100644 index 0000000..f59f771 --- /dev/null +++ b/packages/dts/package.json @@ -0,0 +1,59 @@ +{ + "name": "@mediabunny/dts", + "author": "Vanilagy", + "version": "1.54.0", + "description": "DTS decoder and encoder extension for Mediabunny, based on libavcodec.", + "main": "./dist/bundles/mediabunny-dts.mjs", + "module": "./dist/bundles/mediabunny-dts.mjs", + "types": "./dist/modules/src/index.d.ts", + "exports": { + "types": "./dist/modules/src/index.d.ts", + "import": "./dist/bundles/mediabunny-dts.mjs", + "require": "./dist/bundles/mediabunny-dts.mjs" + }, + "files": [ + "README.md", + "package.json", + "LICENSE", + "dist", + "src" + ], + "browser": { + "worker_threads": false + }, + "sideEffects": false, + "license": "MPL-2.0", + "repository": { + "type": "git", + "url": "git+https://github.com/Vanilagy/mediabunny.git", + "directory": "packages/dts" + }, + "bugs": { + "url": "https://github.com/Vanilagy/mediabunny/issues" + }, + "homepage": "https://mediabunny.dev/guide/extensions/dts", + "funding": { + "type": "individual", + "url": "https://github.com/sponsors/Vanilagy" + }, + "peerDependencies": { + "mediabunny": "^1.0.0" + }, + "devDependencies": { + "@types/emscripten": "^1.40.1" + }, + "keywords": [ + "dts", + "dca", + "dts-hd", + "digital-theater-systems", + "encoding", + "decoding", + "codec", + "mediabunny", + "ffmpeg", + "browser", + "wasm", + "polyfill" + ] +} diff --git a/packages/dts/src/bridge.c b/packages/dts/src/bridge.c new file mode 100644 index 0000000..b7446c6 --- /dev/null +++ b/packages/dts/src/bridge.c @@ -0,0 +1,321 @@ +/*! + * Copyright (c) 2026-present, Vanilagy and contributors + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +#include +#include +#include +#include +#include "libavcodec/avcodec.h" +#include "libavutil/opt.h" +#include "libavutil/channel_layout.h" + +typedef struct { + AVCodecContext *codec_ctx; + AVPacket *packet; + AVFrame *frame; +} DecoderContext; + +EMSCRIPTEN_KEEPALIVE +DecoderContext *init_decoder() { + const AVCodec *codec = avcodec_find_decoder(AV_CODEC_ID_DTS); + if (!codec) return NULL; + + AVCodecContext *codec_ctx = avcodec_alloc_context3(codec); + if (!codec_ctx) return NULL; + + if (avcodec_open2(codec_ctx, codec, NULL) < 0) { + avcodec_free_context(&codec_ctx); + return NULL; + } + + AVPacket *packet = av_packet_alloc(); + if (!packet) { + avcodec_free_context(&codec_ctx); + return NULL; + } + + AVFrame *frame = av_frame_alloc(); + if (!frame) { + av_packet_free(&packet); + avcodec_free_context(&codec_ctx); + return NULL; + } + + DecoderContext *ctx = malloc(sizeof(DecoderContext)); + if (!ctx) { + av_frame_free(&frame); + av_packet_free(&packet); + avcodec_free_context(&codec_ctx); + return NULL; + } + + ctx->codec_ctx = codec_ctx; + ctx->packet = packet; + ctx->frame = frame; + + return ctx; +} + +EMSCRIPTEN_KEEPALIVE +uint8_t *configure_decode_packet(DecoderContext *ctx, int size) { + if (av_new_packet(ctx->packet, size) < 0) { + return NULL; + } + + return ctx->packet->data; +} + +EMSCRIPTEN_KEEPALIVE +int decode_packet(DecoderContext *ctx, int64_t pts) { + ctx->packet->pts = pts; + int ret = avcodec_send_packet(ctx->codec_ctx, ctx->packet); + av_packet_unref(ctx->packet); + if (ret < 0) return ret; + + ret = avcodec_receive_frame(ctx->codec_ctx, ctx->frame); + if (ret < 0) return ret; + + return 0; +} + +EMSCRIPTEN_KEEPALIVE +int get_decoded_format(DecoderContext *ctx) { + return ctx->frame->format; +} + +EMSCRIPTEN_KEEPALIVE +uint8_t *get_decoded_plane_ptr(DecoderContext *ctx, int plane) { + return ctx->frame->data[plane]; +} + +EMSCRIPTEN_KEEPALIVE +int get_decoded_channels(DecoderContext *ctx) { + return ctx->frame->ch_layout.nb_channels; +} + +EMSCRIPTEN_KEEPALIVE +int get_decoded_sample_rate(DecoderContext *ctx) { + return ctx->frame->sample_rate; +} + +EMSCRIPTEN_KEEPALIVE +int get_decoded_sample_count(DecoderContext *ctx) { + return ctx->frame->nb_samples; +} + +EMSCRIPTEN_KEEPALIVE +int64_t get_decoded_pts(DecoderContext *ctx) { + return ctx->frame->pts; +} + +EMSCRIPTEN_KEEPALIVE +void flush_decoder(DecoderContext *ctx) { + avcodec_send_packet(ctx->codec_ctx, NULL); + while (avcodec_receive_frame(ctx->codec_ctx, ctx->frame) == 0) {} + avcodec_flush_buffers(ctx->codec_ctx); +} + +EMSCRIPTEN_KEEPALIVE +void close_decoder(DecoderContext *ctx) { + av_frame_free(&ctx->frame); + av_packet_free(&ctx->packet); + avcodec_free_context(&ctx->codec_ctx); + free(ctx); +} + +typedef struct { + AVCodecContext *codec_ctx; + AVPacket *packet; + AVFrame *frame; + float *input_buffer; + int input_buffer_size; + int64_t encoded_pts; + int encoded_duration; +} EncoderContext; + +/** + * DTS insists on the side-based surround layouts and rejects the back-based ones that av_channel_layout_default hands + * out for 4, 5 and 6 channels. + */ +static int set_dts_channel_layout(AVChannelLayout *layout, int channels) { + switch (channels) { + case 1: { + AVChannelLayout mono = AV_CHANNEL_LAYOUT_MONO; + return av_channel_layout_copy(layout, &mono); + } + case 2: { + AVChannelLayout stereo = AV_CHANNEL_LAYOUT_STEREO; + return av_channel_layout_copy(layout, &stereo); + } + case 4: { + AVChannelLayout quad_side = AV_CHANNEL_LAYOUT_2_2; + return av_channel_layout_copy(layout, &quad_side); + } + case 5: { + AVChannelLayout five_zero = AV_CHANNEL_LAYOUT_5POINT0; + return av_channel_layout_copy(layout, &five_zero); + } + case 6: { + AVChannelLayout five_one = AV_CHANNEL_LAYOUT_5POINT1; + return av_channel_layout_copy(layout, &five_one); + } + default: + return -1; + } +} + +EMSCRIPTEN_KEEPALIVE +EncoderContext *init_encoder(int channels, int sample_rate, int bitrate) { + const AVCodec *codec = avcodec_find_encoder(AV_CODEC_ID_DTS); + if (!codec) return NULL; + + AVCodecContext *codec_ctx = avcodec_alloc_context3(codec); + if (!codec_ctx) return NULL; + + codec_ctx->sample_fmt = AV_SAMPLE_FMT_S32; + codec_ctx->sample_rate = sample_rate; + codec_ctx->bit_rate = bitrate; + codec_ctx->time_base = (AVRational){1, sample_rate}; + + // FFmpeg marks its DTS encoder experimental, so it refuses to open at the default compliance level + codec_ctx->strict_std_compliance = FF_COMPLIANCE_EXPERIMENTAL; + + if (set_dts_channel_layout(&codec_ctx->ch_layout, channels) < 0) { + avcodec_free_context(&codec_ctx); + return NULL; + } + + if (avcodec_open2(codec_ctx, codec, NULL) < 0) { + avcodec_free_context(&codec_ctx); + return NULL; + } + + AVPacket *packet = av_packet_alloc(); + if (!packet) { + avcodec_free_context(&codec_ctx); + return NULL; + } + + AVFrame *frame = av_frame_alloc(); + if (!frame) { + av_packet_free(&packet); + avcodec_free_context(&codec_ctx); + return NULL; + } + + // The frame has a fixed format, so let's create it now: + frame->format = AV_SAMPLE_FMT_S32; + frame->sample_rate = sample_rate; + frame->nb_samples = codec_ctx->frame_size; + av_channel_layout_copy(&frame->ch_layout, &codec_ctx->ch_layout); + + if (av_frame_get_buffer(frame, 0) < 0) { + av_frame_free(&frame); + av_packet_free(&packet); + avcodec_free_context(&codec_ctx); + return NULL; + } + + EncoderContext *ctx = malloc(sizeof(EncoderContext)); + if (!ctx) { + av_frame_free(&frame); + av_packet_free(&packet); + avcodec_free_context(&codec_ctx); + return NULL; + } + + ctx->codec_ctx = codec_ctx; + ctx->packet = packet; + ctx->frame = frame; + ctx->input_buffer = NULL; + ctx->input_buffer_size = 0; + ctx->encoded_pts = 0; + ctx->encoded_duration = 0; + + return ctx; +} + +EMSCRIPTEN_KEEPALIVE +int get_encoder_frame_size(EncoderContext *ctx) { + return ctx->codec_ctx->frame_size; +} + +EMSCRIPTEN_KEEPALIVE +float *get_encode_input_ptr(EncoderContext *ctx, int size) { + if (ctx->input_buffer_size < size) { + free(ctx->input_buffer); + ctx->input_buffer = malloc(size); + if (!ctx->input_buffer) { + ctx->input_buffer_size = 0; + return NULL; + } + ctx->input_buffer_size = size; + } + return ctx->input_buffer; +} + +EMSCRIPTEN_KEEPALIVE +int encode_frame(EncoderContext *ctx, int64_t pts) { + int channels = ctx->codec_ctx->ch_layout.nb_channels; + int frame_size = ctx->frame->nb_samples; + + ctx->frame->pts = pts; + + // DTS encodes from s32, which is a packed format, so the samples stay interleaved and all land in data[0] + float *input = ctx->input_buffer; + int32_t *output = (int32_t *)ctx->frame->data[0]; + for (int i = 0; i < frame_size * channels; i++) { + float sample = input[i]; + if (sample > 1.0f) sample = 1.0f; + if (sample < -1.0f) sample = -1.0f; + output[i] = (int32_t)(sample * 2147483647.0f); + } + + int ret = avcodec_send_frame(ctx->codec_ctx, ctx->frame); + if (ret < 0) return ret; + + ret = avcodec_receive_packet(ctx->codec_ctx, ctx->packet); + if (ret < 0) return ret; + + ctx->encoded_pts = ctx->packet->pts; + ctx->encoded_duration = ctx->packet->duration; + + return ctx->packet->size; +} + +EMSCRIPTEN_KEEPALIVE +void flush_encoder(EncoderContext *ctx) { + avcodec_send_frame(ctx->codec_ctx, NULL); + while (avcodec_receive_packet(ctx->codec_ctx, ctx->packet) == 0) { + av_packet_unref(ctx->packet); + } +} + +EMSCRIPTEN_KEEPALIVE +uint8_t *get_encoded_data(EncoderContext *ctx) { + return ctx->packet->data; +} + +EMSCRIPTEN_KEEPALIVE +int64_t get_encoded_pts(EncoderContext *ctx) { + return ctx->encoded_pts; +} + +EMSCRIPTEN_KEEPALIVE +int get_encoded_duration(EncoderContext *ctx) { + return ctx->encoded_duration; +} + +EMSCRIPTEN_KEEPALIVE +void close_encoder(EncoderContext *ctx) { + free(ctx->input_buffer); + av_frame_free(&ctx->frame); + av_packet_free(&ctx->packet); + avcodec_free_context(&ctx->codec_ctx); + free(ctx); +} diff --git a/packages/dts/src/codec.worker.ts b/packages/dts/src/codec.worker.ts new file mode 100644 index 0000000..6cf9c80 --- /dev/null +++ b/packages/dts/src/codec.worker.ts @@ -0,0 +1,306 @@ +/*! + * Copyright (c) 2026-present, Vanilagy and contributors + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +import createModule from '../build/dts'; +import type { WorkerCommand, WorkerResponse, WorkerResponseData } from './shared'; + +type ExtendedEmscriptenModule = EmscriptenModule & { + cwrap: typeof cwrap; +}; + +let module: ExtendedEmscriptenModule; +let modulePromise: Promise | null = null; + +let initDecoderFn: () => number; +let configureDecodePacket: (ctx: number, size: number) => number; +let decodePacket: (ctx: number, pts: bigint) => number; +let getDecodedFormat: (ctx: number) => number; +let getDecodedPlanePtr: (ctx: number, plane: number) => number; +let getDecodedChannels: (ctx: number) => number; +let getDecodedSampleRate: (ctx: number) => number; +let getDecodedSampleCount: (ctx: number) => number; +let getDecodedPts: (ctx: number) => bigint; +let flushDecoderFn: (ctx: number) => void; +let closeDecoderFn: (ctx: number) => void; + +let initEncoderFn: (channels: number, sampleRate: number, bitrate: number) => number; +let getEncoderFrameSize: (ctx: number) => number; +let getEncodeInputPtr: (ctx: number, size: number) => number; +let encodeFrameFn: (ctx: number, pts: bigint) => number; +let flushEncoderFn: (ctx: number) => void; +let getEncodedData: (ctx: number) => number; +let getEncodedPts: (ctx: number) => bigint; +let getEncodedDuration: (ctx: number) => number; +let closeEncoderFn: (ctx: number) => void; + +const ensureModule = async () => { + if (!module) { + if (modulePromise) { + // If we don't do this we can have a race condition + return modulePromise; + } + + modulePromise = createModule() as Promise; + module = await modulePromise; + modulePromise = null; + + initDecoderFn = module.cwrap('init_decoder', 'number', []); + configureDecodePacket = module.cwrap('configure_decode_packet', 'number', ['number', 'number']); + decodePacket = module.cwrap('decode_packet', 'number', ['number', 'number']) as unknown as typeof decodePacket; + getDecodedFormat = module.cwrap('get_decoded_format', 'number', ['number']); + getDecodedPlanePtr = module.cwrap('get_decoded_plane_ptr', 'number', ['number', 'number']); + getDecodedChannels = module.cwrap('get_decoded_channels', 'number', ['number']); + getDecodedSampleRate = module.cwrap('get_decoded_sample_rate', 'number', ['number']); + getDecodedSampleCount = module.cwrap('get_decoded_sample_count', 'number', ['number']); + getDecodedPts = module.cwrap('get_decoded_pts', 'number', ['number']) as unknown as typeof getDecodedPts; + flushDecoderFn = module.cwrap('flush_decoder', null, ['number']); + closeDecoderFn = module.cwrap('close_decoder', null, ['number']); + + initEncoderFn = module.cwrap('init_encoder', 'number', ['number', 'number', 'number']); + getEncoderFrameSize = module.cwrap('get_encoder_frame_size', 'number', ['number']); + getEncodeInputPtr = module.cwrap('get_encode_input_ptr', 'number', ['number', 'number']); + encodeFrameFn = module.cwrap('encode_frame', 'number', ['number', 'number']) as unknown as typeof encodeFrameFn; + flushEncoderFn = module.cwrap('flush_encoder', null, ['number']); + getEncodedData = module.cwrap('get_encoded_data', 'number', ['number']); + getEncodedPts = module.cwrap('get_encoded_pts', 'number', ['number']) as unknown as typeof getEncodedPts; + getEncodedDuration = module.cwrap('get_encoded_duration', 'number', ['number']); + closeEncoderFn = module.cwrap('close_encoder', null, ['number']); + } +}; + +const initDecoder = async () => { + await ensureModule(); + + const ctx = initDecoderFn(); + if (ctx === 0) { + throw new Error('Failed to initialize DTS decoder.'); + } + + return { ctx, frameSize: 0 }; +}; + +// Keys are AVSampleFormat enum values +const AV_FORMAT_MAP: Record = { + 0: { format: 'u8', bytesPerSample: 1, planar: false }, + 1: { format: 's16', bytesPerSample: 2, planar: false }, + 2: { format: 's32', bytesPerSample: 4, planar: false }, + 3: { format: 'f32', bytesPerSample: 4, planar: false }, + 5: { format: 'u8-planar', bytesPerSample: 1, planar: true }, + 6: { format: 's16-planar', bytesPerSample: 2, planar: true }, + 7: { format: 's32-planar', bytesPerSample: 4, planar: true }, + 8: { format: 'f32-planar', bytesPerSample: 4, planar: true }, +}; + +const decode = (ctx: number, encodedData: ArrayBuffer, timestamp: number) => { + const bytes = new Uint8Array(encodedData); + + const dataPtr = configureDecodePacket(ctx, bytes.length); + if (dataPtr === 0) { + throw new Error('Failed to configure decode packet.'); + } + + module.HEAPU8.set(bytes, dataPtr); + + const ret = decodePacket(ctx, BigInt(timestamp)); + if (ret < 0) { + throw new Error(`Decode failed with error code ${ret}.`); + } + + const avFormat = getDecodedFormat(ctx); + const info = AV_FORMAT_MAP[avFormat]; + if (!info) { + throw new Error(`Unsupported AVSampleFormat: ${avFormat}`); + } + + const channels = getDecodedChannels(ctx); + const sampleRate = getDecodedSampleRate(ctx); + const sampleCount = getDecodedSampleCount(ctx); + const pts = Number(getDecodedPts(ctx)); + + let pcmData: ArrayBuffer; + if (info.planar) { + const planeSize = sampleCount * info.bytesPerSample; + const buffer = new Uint8Array(planeSize * channels); + + for (let ch = 0; ch < channels; ch++) { + const ptr = getDecodedPlanePtr(ctx, ch); + buffer.set(module.HEAPU8.subarray(ptr, ptr + planeSize), ch * planeSize); + } + + pcmData = buffer.buffer; + } else { + const totalSize = sampleCount * channels * info.bytesPerSample; + const ptr = getDecodedPlanePtr(ctx, 0); + pcmData = module.HEAPU8.slice(ptr, ptr + totalSize).buffer; + } + + return { pcmData, format: info.format, channels, sampleRate, sampleCount, pts }; +}; + +const initEncoder = async ( + numberOfChannels: number, + sampleRate: number, + bitrate: number, +) => { + await ensureModule(); + + const ctx = initEncoderFn(numberOfChannels, sampleRate, bitrate); + if (ctx === 0) { + throw new Error('Failed to initialize DTS encoder.'); + } + + return { ctx, frameSize: getEncoderFrameSize(ctx) }; +}; + +const encode = (ctx: number, audioData: ArrayBuffer, timestamp: number) => { + const audioBytes = new Uint8Array(audioData); + + const inputPtr = getEncodeInputPtr(ctx, audioBytes.length); + if (inputPtr === 0) { + throw new Error('Failed to allocate encoder input buffer.'); + } + module.HEAPU8.set(audioBytes, inputPtr); + + const bytesWritten = encodeFrameFn(ctx, BigInt(timestamp)); + if (bytesWritten < 0) { + throw new Error(`Encode failed with error code ${bytesWritten}.`); + } + + const ptr = getEncodedData(ctx); + const encodedData = module.HEAPU8.slice(ptr, ptr + bytesWritten).buffer; + const pts = Number(getEncodedPts(ctx)); + const duration = getEncodedDuration(ctx); + + return { encodedData, pts, duration }; +}; + +const flushEncoder = (ctx: number) => { + flushEncoderFn(ctx); +}; + +const onMessage = (data: { id: number; command: WorkerCommand }) => { + const { id, command } = data; + + const handleCommand = async (): Promise => { + try { + let result: WorkerResponseData; + const transferables: Transferable[] = []; + + switch (command.type) { + case 'init-decoder': { + const { ctx, frameSize } = await initDecoder(); + result = { type: command.type, ctx, frameSize }; + }; break; + + case 'decode': { + const decoded = decode(command.data.ctx, command.data.encodedData, command.data.timestamp); + result = { + type: command.type, + pcmData: decoded.pcmData, + format: decoded.format, + channels: decoded.channels, + sampleRate: decoded.sampleRate, + sampleCount: decoded.sampleCount, + pts: decoded.pts, + }; + transferables.push(decoded.pcmData); + }; break; + + case 'flush-decoder': { + flushDecoderFn(command.data.ctx); + result = { type: command.type }; + }; break; + + case 'close-decoder': { + closeDecoderFn(command.data.ctx); + result = { type: command.type }; + }; break; + + case 'init-encoder': { + const { ctx, frameSize } = await initEncoder( + command.data.numberOfChannels, + command.data.sampleRate, + command.data.bitrate, + ); + result = { type: command.type, ctx, frameSize }; + }; break; + + case 'encode': { + const encoded = encode( + command.data.ctx, + command.data.audioData, + command.data.timestamp, + ); + result = { + type: command.type, + encodedData: encoded.encodedData, + pts: encoded.pts, + duration: encoded.duration, + }; + transferables.push(encoded.encodedData); + }; break; + + case 'flush-encoder': { + flushEncoder(command.data.ctx); + result = { type: command.type }; + }; break; + + case 'close-encoder': { + closeEncoderFn(command.data.ctx); + result = { type: command.type }; + }; break; + } + + const response: WorkerResponse = { + id, + success: true, + data: result, + }; + sendMessage(response, transferables); + } catch (error: unknown) { + const response: WorkerResponse = { + id, + success: false, + error, + }; + sendMessage(response); + } + }; + + void handleCommand(); +}; + +const sendMessage = (data: unknown, transferables?: Transferable[]) => { + if (parentPort) { + parentPort.postMessage(data, transferables ?? []); + } else { + self.postMessage(data, { transfer: transferables ?? [] }); + } +}; + +let parentPort: { + postMessage: (data: unknown, transferables?: Transferable[]) => void; + on: (event: string, listener: (data: never) => void) => void; +} | null = null; + +if (typeof self === 'undefined') { + const workerModule = 'worker_threads'; + // eslint-disable-next-line @stylistic/max-len + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-require-imports, @typescript-eslint/no-unsafe-member-access + parentPort = require(workerModule).parentPort; +} + +if (parentPort) { + parentPort.on('message', onMessage); +} else { + self.addEventListener('message', event => onMessage(event.data as { id: number; command: WorkerCommand })); +} + +// Prevents the worker for being randomly closed by Firefox +// https://github.com/Vanilagy/mediabunny/issues/435 +setInterval(() => {}, 1000); diff --git a/packages/dts/src/decoder.ts b/packages/dts/src/decoder.ts new file mode 100644 index 0000000..0b5dfdb --- /dev/null +++ b/packages/dts/src/decoder.ts @@ -0,0 +1,80 @@ +/*! + * Copyright (c) 2026-present, Vanilagy and contributors + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +import { + CustomAudioDecoder, + AudioCodec, + AudioSample, + EncodedPacket, + registerDecoder, +} from 'mediabunny'; +import { sendCommand, refWorker, unrefWorker } from './worker-client'; + +class DtsDecoder extends CustomAudioDecoder { + private ctx = 0; + + static override supports(codec: AudioCodec): boolean { + return codec === 'dts'; + } + + async init() { + await refWorker(); + + const result = await sendCommand({ + type: 'init-decoder', + data: {}, + }); + this.ctx = result.ctx; + } + + async decode(packet: EncodedPacket) { + const encodedData = packet.data.slice().buffer; + const timestamp = Math.round(packet.timestamp * this.config.sampleRate); + + const result = await sendCommand({ + type: 'decode', + data: { ctx: this.ctx, encodedData, timestamp }, + }, [encodedData]); + + const sample = new AudioSample({ + data: result.pcmData, + format: result.format, + numberOfChannels: result.channels, + sampleRate: result.sampleRate, + timestamp: result.pts / result.sampleRate, + }); + this.onSample(sample); + } + + async flush() { + await sendCommand({ type: 'flush-decoder', data: { ctx: this.ctx } }); + } + + async close() { + void sendCommand({ type: 'close-decoder', data: { ctx: this.ctx } }); + await unrefWorker(); + } +} + +let registered = false; + +/** + * Registers a DTS audio decoder, which Mediabunny will then use automatically when applicable. Make sure to call this + * function before starting any decoding task. + * + * @group \@mediabunny/dts + * @public + */ +export const registerDtsDecoder = () => { + if (registered) { + return; + } + registered = true; + + registerDecoder(DtsDecoder); +}; diff --git a/packages/dts/src/encoder.ts b/packages/dts/src/encoder.ts new file mode 100644 index 0000000..7b8ecd2 --- /dev/null +++ b/packages/dts/src/encoder.ts @@ -0,0 +1,197 @@ +/*! + * Copyright (c) 2026-present, Vanilagy and contributors + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +import { + CustomAudioEncoder, + AudioCodec, + AudioSample, + EncodedPacket, + registerEncoder, +} from 'mediabunny'; +import { sendCommand, refWorker, unrefWorker } from './worker-client'; +import { assert } from './shared'; +import { DTS_CHANNEL_COUNTS, DTS_SAMPLE_RATES, dtsBitrateFits } from '../../../shared/dts-misc'; + +class DtsEncoder extends CustomAudioEncoder { + private ctx = 0; + private encoderFrameSize = 0; + private sampleRate = 0; + private numberOfChannels = 0; + private chunkMetadata: EncodedAudioChunkMetadata = {}; + + // Accumulate interleaved f32 samples until we have a full frame + private pendingBuffer = new Float32Array(2 ** 16); + private pendingFrames = 0; + private nextSampleTimestampInSamples: number | null = null; + private nextPacketTimestampInSamples: number | null = null; + + static override supports(codec: AudioCodec, config: AudioEncoderConfig): boolean { + return codec === 'dts' + && DTS_CHANNEL_COUNTS.includes(config.numberOfChannels) + && DTS_SAMPLE_RATES.includes(config.sampleRate) + && config.bitrate !== undefined + && dtsBitrateFits(config.bitrate, config.sampleRate, config.numberOfChannels); + } + + async init() { + await refWorker(); + + assert(this.config.bitrate !== undefined); + this.sampleRate = this.config.sampleRate; + this.numberOfChannels = this.config.numberOfChannels; + + const result = await sendCommand({ + type: 'init-encoder', + data: { + numberOfChannels: this.config.numberOfChannels, + sampleRate: this.config.sampleRate, + bitrate: this.config.bitrate, + }, + }); + + this.ctx = result.ctx; + this.encoderFrameSize = result.frameSize; + + this.resetInternalState(); + } + + private resetInternalState() { + this.pendingFrames = 0; + this.nextSampleTimestampInSamples = null; + this.nextPacketTimestampInSamples = null; + + this.chunkMetadata = { + decoderConfig: { + codec: 'dtsc', + numberOfChannels: this.config.numberOfChannels, + sampleRate: this.config.sampleRate, + }, + }; + } + + async encode(audioSample: AudioSample) { + if (this.nextSampleTimestampInSamples === null) { + this.nextSampleTimestampInSamples = Math.round(audioSample.timestamp * this.sampleRate); + this.nextPacketTimestampInSamples = this.nextSampleTimestampInSamples; + } + + const channels = this.numberOfChannels; + const incomingFrames = audioSample.numberOfFrames; + + // Extract interleaved f32 data + const totalBytes = audioSample.allocationSize({ format: 'f32', planeIndex: 0 }); + const audioBytes = new Uint8Array(totalBytes); + audioSample.copyTo(audioBytes, { format: 'f32', planeIndex: 0 }); + const incomingData = new Float32Array(audioBytes.buffer); + + const requiredSamples = (this.pendingFrames + incomingFrames) * channels; + if (requiredSamples > this.pendingBuffer.length) { + let newSize = this.pendingBuffer.length; + while (newSize < requiredSamples) { + newSize *= 2; + } + const newBuffer = new Float32Array(newSize); + newBuffer.set(this.pendingBuffer.subarray(0, this.pendingFrames * channels)); + this.pendingBuffer = newBuffer; + } + this.pendingBuffer.set(incomingData, this.pendingFrames * channels); + this.pendingFrames += incomingFrames; + + while (this.pendingFrames >= this.encoderFrameSize) { + await this.encodeOneFrame(); + } + } + + async flush() { + // Pad remaining samples with silence to fill a full frame + if (this.pendingFrames > 0) { + const channels = this.numberOfChannels; + const frameSize = this.encoderFrameSize; + const usedSamples = this.pendingFrames * channels; + const frameSamples = frameSize * channels; + + this.pendingBuffer.fill(0, usedSamples, frameSamples); + this.pendingFrames = frameSize; + + await this.encodeOneFrame(); + } + + await sendCommand({ type: 'flush-encoder', data: { ctx: this.ctx } }); + + this.resetInternalState(); + } + + close() { + void sendCommand({ type: 'close-encoder', data: { ctx: this.ctx } }); + void unrefWorker(); + } + + private async encodeOneFrame() { + assert(this.nextSampleTimestampInSamples !== null); + assert(this.nextPacketTimestampInSamples !== null); + + const channels = this.numberOfChannels; + const frameSize = this.encoderFrameSize; + const frameSamples = frameSize * channels; + + const frameData = this.pendingBuffer.slice(0, frameSamples); + + // Shift remaining using copyWithin + this.pendingFrames -= frameSize; + if (this.pendingFrames > 0) { + this.pendingBuffer.copyWithin(0, frameSamples, frameSamples + this.pendingFrames * channels); + } + + const audioData = frameData.buffer; + const result = await sendCommand({ + type: 'encode', + data: { + ctx: this.ctx, + audioData, + timestamp: this.nextSampleTimestampInSamples, + }, + }, [audioData]); + + this.nextSampleTimestampInSamples += frameSize; + + // We always get exactly one packet because we encode the correct frame size + const packet = new EncodedPacket( + new Uint8Array(result.encodedData), + 'key', + this.nextPacketTimestampInSamples / this.sampleRate, + result.duration / this.sampleRate, + ); + + this.nextPacketTimestampInSamples += result.duration; + + this.onPacket( + packet, + this.chunkMetadata, + ); + + this.chunkMetadata = {}; + } +} + +let registered = false; + +/** + * Registers a DTS audio encoder, which Mediabunny will then use automatically when applicable. Make sure to call this + * function before starting any encoding task. + * + * @group \@mediabunny/dts + * @public + */ +export const registerDtsEncoder = () => { + if (registered) { + return; + } + registered = true; + + registerEncoder(DtsEncoder); +}; diff --git a/packages/dts/src/index.ts b/packages/dts/src/index.ts new file mode 100644 index 0000000..3344b00 --- /dev/null +++ b/packages/dts/src/index.ts @@ -0,0 +1,23 @@ +/*! + * Copyright (c) 2026-present, Vanilagy and contributors + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +import { Logging } from 'mediabunny'; + +const DTS_LOADED_SYMBOL = Symbol.for('@mediabunny/dts loaded'); +if ((globalThis as Record)[DTS_LOADED_SYMBOL]) { + Logging._error( + '[WARNING]\n@mediabunny/dts was loaded twice.' + + ' This will likely cause the encoder/decoder not to work correctly.' + + ' Check if multiple dependencies are importing different versions of @mediabunny/dts,' + + ' or if something is being bundled incorrectly.', + ); +} +(globalThis as Record)[DTS_LOADED_SYMBOL] = true; + +export { registerDtsDecoder } from './decoder'; +export { registerDtsEncoder } from './encoder'; diff --git a/packages/dts/src/shared.ts b/packages/dts/src/shared.ts new file mode 100644 index 0000000..27757c1 --- /dev/null +++ b/packages/dts/src/shared.ts @@ -0,0 +1,100 @@ +/*! + * Copyright (c) 2026-present, Vanilagy and contributors + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +export type WorkerCommand = { + type: 'init-decoder'; + data: Record; +} | { + type: 'decode'; + data: { + ctx: number; + encodedData: ArrayBuffer; + timestamp: number; + }; +} | { + type: 'flush-decoder'; + data: { + ctx: number; + }; +} | { + type: 'close-decoder'; + data: { + ctx: number; + }; +} | { + type: 'init-encoder'; + data: { + numberOfChannels: number; + sampleRate: number; + bitrate: number; + }; +} | { + type: 'encode'; + data: { + ctx: number; + audioData: ArrayBuffer; + timestamp: number; + }; +} | { + type: 'flush-encoder'; + data: { + ctx: number; + }; +} | { + type: 'close-encoder'; + data: { + ctx: number; + }; +}; + +export type WorkerResponseData = { + type: 'init-decoder'; + ctx: number; + frameSize: number; +} | { + type: 'decode'; + pcmData: ArrayBuffer; + format: AudioSampleFormat; + channels: number; + sampleRate: number; + sampleCount: number; + pts: number; +} | { + type: 'flush-decoder'; +} | { + type: 'close-decoder'; +} | { + type: 'init-encoder'; + ctx: number; + frameSize: number; +} | { + type: 'encode'; + encodedData: ArrayBuffer; + pts: number; + duration: number; +} | { + type: 'flush-encoder'; +} | { + type: 'close-encoder'; +}; + +export type WorkerResponse = { + id: number; +} & ({ + success: true; + data: WorkerResponseData; +} | { + success: false; + error: unknown; +}); + +export function assert(x: unknown): asserts x { + if (!x) { + throw new Error('Assertion failed.'); + } +} diff --git a/packages/dts/src/worker-client.ts b/packages/dts/src/worker-client.ts new file mode 100644 index 0000000..15c6b8f --- /dev/null +++ b/packages/dts/src/worker-client.ts @@ -0,0 +1,109 @@ +/*! + * Copyright (c) 2026-present, Vanilagy and contributors + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +import { assert, type WorkerCommand, type WorkerResponse, type WorkerResponseData } from './shared'; +// @ts-expect-error An esbuild plugin handles this, TypeScript doesn't need to understand +import createWorker from './codec.worker'; + +type ExtendedWorker = Worker & { + ref?: () => void; + unref?: () => void; +}; + +let workerPromise: Promise | null; +let nextMessageId = 0; +const pendingMessages = new Map void; + reject: (reason?: unknown) => void; +}>(); + +let refCount = 0; +let keepAliveInterval: ReturnType | null = null; + +export const refWorker = async () => { + refCount++; + if (refCount === 1) { + keepAliveInterval = setInterval(() => {}, 2 ** 31 - 1); + const worker = await ensureWorker(); + worker.ref?.(); + } +}; + +export const unrefWorker = async () => { + refCount--; + if (refCount === 0) { + if (keepAliveInterval !== null) { + clearInterval(keepAliveInterval); + keepAliveInterval = null; + } + + const worker = await workerPromise; + if (worker) { + if (worker.unref) { + worker.unref(); // If we don't do this, then the Node process never terminates by itself + // Keep the worker around tho + } else if (typeof window === 'undefined') { + // Non-browser environment without unref - terminate instead + worker.terminate(); + workerPromise = null; + } + } + } +}; + +export const sendCommand = async ( + command: WorkerCommand & { type: T }, + transferables?: Transferable[], +) => { + const worker = await ensureWorker(); + + return new Promise((resolve, reject) => { + const id = nextMessageId++; + pendingMessages.set(id, { + resolve: resolve as (value: WorkerResponseData) => void, + reject, + }); + + if (transferables) { + worker.postMessage({ id, command }, transferables); + } else { + worker.postMessage({ id, command }); + } + }); +}; + +const ensureWorker = () => { + return workerPromise ??= (async () => { + // eslint-disable-next-line @typescript-eslint/no-unsafe-call + const worker = (await createWorker()) as ExtendedWorker; + worker.unref?.(); // Start unreffed + + const onMessage = (data: WorkerResponse) => { + const pending = pendingMessages.get(data.id); + assert(pending !== undefined); + + pendingMessages.delete(data.id); + if (data.success) { + pending.resolve(data.data); + } else { + pending.reject(data.error); + } + }; + + if (worker.addEventListener) { + worker.addEventListener('message', event => onMessage(event.data as WorkerResponse)); + } else { + const nodeWorker = worker as unknown as { + on: (event: string, listener: (data: never) => void) => void; + }; + nodeWorker.on('message', onMessage); + } + + return worker; + })(); +}; diff --git a/packages/dts/tsconfig.json b/packages/dts/tsconfig.json new file mode 100644 index 0000000..27a7ef1 --- /dev/null +++ b/packages/dts/tsconfig.json @@ -0,0 +1,24 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "composite": true, + "outDir": "./dist/modules", + "declaration": true, + "declarationMap": true, + "stripInternal": true, + "noEmit": false, + "moduleResolution": "nodenext", + "module": "nodenext", + "allowJs": true, + "paths": { + "mediabunny": ["../../src/index.ts"], + }, + }, + "include": [ + "./src/**/*", + "./build/**/*", + ], + "references": [ + { "path": "../../src" } + ] +} diff --git a/packages/dts/tsdoc.json b/packages/dts/tsdoc.json new file mode 100644 index 0000000..51d29d3 --- /dev/null +++ b/packages/dts/tsdoc.json @@ -0,0 +1,4 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/tsdoc/v0/tsdoc.schema.json", + "extends": ["../../tsdoc.json"] +} diff --git a/packages/server/src/audio-decoder.ts b/packages/server/src/audio-decoder.ts index 22bb420..247d16b 100644 --- a/packages/server/src/audio-decoder.ts +++ b/packages/server/src/audio-decoder.ts @@ -25,7 +25,8 @@ export class NodeAvAudioDecoder extends CustomAudioDecoder { || codec === 'vorbis' || codec === 'flac' || codec === 'ac3' - || codec === 'eac3'; + || codec === 'eac3' + || codec === 'dts'; } async init(): Promise { diff --git a/packages/server/src/audio-encoder.ts b/packages/server/src/audio-encoder.ts index 9b94106..06ec912 100644 --- a/packages/server/src/audio-encoder.ts +++ b/packages/server/src/audio-encoder.ts @@ -15,7 +15,7 @@ import { EncodedPacket, } from 'mediabunny'; import * as NodeAv from 'node-av'; -import { CODEC_TO_CODEC_ID, getChannelLayout } from './misc'; +import { CODEC_TO_CODEC_ID, getChannelLayout, getDtsChannelLayout } from './misc'; import { assert, toUint8Array } from '../../../src/misc'; import { copyAudioSampleToAvFrame, AvFrameAudioSampleResource } from './audio-sample'; import { @@ -23,6 +23,7 @@ import { buildAdtsHeaderTemplate, parseAacAudioSpecificConfig, } from '../../../shared/aac-misc'; +import { DTS_CHANNEL_COUNTS, DTS_SAMPLE_RATES, dtsBitrateFits } from '../../../shared/dts-misc'; const AAC_SAMPLE_RATES = [96000, 88200, 64000, 48000, 44100, 32000, 24000, 22050, 16000, 12000, 11025, 8000, 7350]; @@ -72,6 +73,10 @@ export class NodeAvAudioEncoder extends CustomAudioEncoder { ) || ( codec === 'eac3' && numberOfChannels >= 1 && numberOfChannels <= 16 && AC3_SAMPLE_RATES.includes(sampleRate) + ) || ( + codec === 'dts' && DTS_CHANNEL_COUNTS.includes(numberOfChannels) + && DTS_SAMPLE_RATES.includes(sampleRate) + && dtsBitrateFits(resolveBitrate(config, codec), sampleRate, numberOfChannels) ); } @@ -107,21 +112,26 @@ export class NodeAvAudioEncoder extends CustomAudioEncoder { } codecContext.sampleRate = this.config.sampleRate; - codecContext.channelLayout = getChannelLayout(this.config.numberOfChannels); + codecContext.channelLayout = this.codec === 'dts' + ? getDtsChannelLayout(this.config.numberOfChannels) + : getChannelLayout(this.config.numberOfChannels); codecContext.codecType = NodeAv.AVMEDIA_TYPE_AUDIO; codecContext.codecId = CODEC_TO_CODEC_ID[this.codec]!; codecContext.sampleFormat = sampleFormat; codecContext.timeBase = new NodeAv.Rational(1, this.config.sampleRate); - codecContext.bitRate = BigInt( - this.config.bitrate ?? new Quality('medium')._toAudioBitrate(this.codec) ?? 0, - ); + codecContext.bitRate = BigInt(resolveBitrate(this.config, this.codec)); if (this.config.bitrateMode === 'constant') { codecContext.rcMinRate = codecContext.bitRate; codecContext.rcMaxRate = codecContext.bitRate; } - const ret = await codecContext.open2(); + // libav's DTS encoder is marked experimental, so it refuses to open at the default compliance level + const options = this.codec === 'dts' + ? NodeAv.Dictionary.fromObject({ strict: -2 }) + : null; + + const ret = await codecContext.open2(this.avCodec, options); NodeAv.FFmpegError.throwIfError(ret, 'Open codec context'); this.codecContext = codecContext; @@ -172,7 +182,9 @@ export class NodeAvAudioEncoder extends CustomAudioEncoder { this.resampler = new NodeAv.SoftwareResampleContext(); this.resamplerInputSampleRate = this.frame.sampleRate; - const outLayout = getChannelLayout(this.codecContext.channels); + // Resample straight to whatever layout the encoder was opened with, since not every codec accepts + // the layout that getChannelLayout hands out for a given channel count + const outLayout = this.codecContext.channelLayout; const inLayout = getChannelLayout(this.frame.channels); const ret = this.resampler.allocSetOpts2( @@ -400,3 +412,7 @@ export class NodeAvAudioEncoder extends CustomAudioEncoder { this.resampler?.free(); } } + +const resolveBitrate = (config: AudioEncoderConfig, codec: AudioCodec) => { + return config.bitrate ?? new Quality('medium')._toAudioBitrate(codec) ?? 0; +}; diff --git a/packages/server/src/misc.ts b/packages/server/src/misc.ts index 3c152ae..ab4998a 100644 --- a/packages/server/src/misc.ts +++ b/packages/server/src/misc.ts @@ -25,6 +25,7 @@ export const CODEC_TO_CODEC_ID: Partial> = flac: NodeAv.AV_CODEC_ID_FLAC, ac3: NodeAv.AV_CODEC_ID_AC3, eac3: NodeAv.AV_CODEC_ID_EAC3, + dts: NodeAv.AV_CODEC_ID_DTS, }; let cachedHardwareContext: NodeAv.HardwareContext | null | undefined = undefined; @@ -267,3 +268,21 @@ export const getChannelLayout = (numChannels: number): NodeAv.ChannelLayout => { default: return { nbChannels: numChannels, order: NodeAv.AV_CHANNEL_ORDER_UNSPEC, mask: 0n }; } }; + +/** FL + FR + SL + SR, which libav has no constant for. */ +const QUAD_SIDE_MASK = 0x603n; + +/** + * DTS insists on the side-based surround layouts and rejects the back-based ones that {@link getChannelLayout} + * hands out, so it gets its own mapping. + */ +export const getDtsChannelLayout = (numChannels: number): NodeAv.ChannelLayout => { + switch (numChannels) { + case 1: return NodeAv.AV_CHANNEL_LAYOUT_MONO; + case 2: return NodeAv.AV_CHANNEL_LAYOUT_STEREO; + case 4: return { nbChannels: 4, order: NodeAv.AV_CHANNEL_ORDER_NATIVE, mask: QUAD_SIDE_MASK }; + case 5: return NodeAv.AV_CHANNEL_LAYOUT_5POINT0; + case 6: return NodeAv.AV_CHANNEL_LAYOUT_5POINT1; + default: return { nbChannels: numChannels, order: NodeAv.AV_CHANNEL_ORDER_UNSPEC, mask: 0n }; + } +}; diff --git a/scripts/build.sh b/scripts/build.sh index 0ffad0a..dec1389 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -7,6 +7,7 @@ set -e rm -rf dist rm -rf packages/mp3-encoder/dist rm -rf packages/ac3/dist +rm -rf packages/dts/dist rm -rf packages/aac-encoder/dist rm -rf packages/flac-encoder/dist rm -rf packages/prores/dist @@ -19,6 +20,7 @@ tsx scripts/ensure-license-headers.ts tsc -p src --stripInternal false # Don't strip internals since the packages may use them tsc -p packages/mp3-encoder tsc -p packages/ac3 +tsc -p packages/dts tsc -p packages/aac-encoder tsc -p packages/flac-encoder tsc -p packages/prores @@ -39,6 +41,7 @@ tsx scripts/bundle.ts api-extractor run api-extractor run -c packages/mp3-encoder/api-extractor.json api-extractor run -c packages/ac3/api-extractor.json +api-extractor run -c packages/dts/api-extractor.json api-extractor run -c packages/aac-encoder/api-extractor.json api-extractor run -c packages/flac-encoder/api-extractor.json api-extractor run -c packages/prores/api-extractor.json @@ -48,6 +51,7 @@ api-extractor run -c packages/server/api-extractor.json tsx scripts/check-docblocks.ts dist/mediabunny.d.ts tsx scripts/check-docblocks.ts packages/mp3-encoder/dist/mediabunny-mp3-encoder.d.ts tsx scripts/check-docblocks.ts packages/ac3/dist/mediabunny-ac3.d.ts +tsx scripts/check-docblocks.ts packages/dts/dist/mediabunny-dts.d.ts tsx scripts/check-docblocks.ts packages/aac-encoder/dist/mediabunny-aac-encoder.d.ts tsx scripts/check-docblocks.ts packages/flac-encoder/dist/mediabunny-flac-encoder.d.ts tsx scripts/check-docblocks.ts packages/prores/dist/mediabunny-prores.d.ts @@ -60,6 +64,7 @@ npm run docs:generate -- --dry echo 'export as namespace Mediabunny;' >> dist/mediabunny.d.ts echo 'export as namespace MediabunnyMp3Encoder;' >> packages/mp3-encoder/dist/mediabunny-mp3-encoder.d.ts echo 'export as namespace MediabunnyAc3;' >> packages/ac3/dist/mediabunny-ac3.d.ts +echo 'export as namespace MediabunnyDts;' >> packages/dts/dist/mediabunny-dts.d.ts echo 'export as namespace MediabunnyAacEncoder;' >> packages/aac-encoder/dist/mediabunny-aac-encoder.d.ts echo 'export as namespace MediabunnyFlacEncoder;' >> packages/flac-encoder/dist/mediabunny-flac-encoder.d.ts echo 'export as namespace MediabunnyProres;' >> packages/prores/dist/mediabunny-prores.d.ts diff --git a/scripts/bundle.ts b/scripts/bundle.ts index 1b2c29e..43633fd 100644 --- a/scripts/bundle.ts +++ b/scripts/bundle.ts @@ -161,6 +161,37 @@ const ac3Variants = await createVariants( }, ); +const dtsVariants = await createVariants( + 'packages/dts/src/index.ts', + 'MediabunnyDts', + 'packages/dts/dist/bundles/mediabunny-dts', + 'js', // The bundles are purely for the browser, not for Node (due to the peer dependency) + { + plugins: [ + PluginExternalGlobal.externalGlobalPlugin({ + mediabunny: 'Mediabunny', + }), + inlineWorkerPlugin({ + define: { + 'import.meta.url': '""', + }, + legalComments: 'none', + }), + ], + }, + { + external: ['mediabunny'], + plugins: [ + inlineWorkerPlugin({ + define: { + 'import.meta.url': '""', + }, + legalComments: 'none', + }), + ], + }, +); + const aacEncoderVariants = await createVariants( 'packages/aac-encoder/src/index.ts', 'MediabunnyAacEncoder', @@ -262,6 +293,7 @@ const contexts = [ ...mediabunnyVariants, ...mp3EncoderVariants, ...ac3Variants, + ...dtsVariants, ...aacEncoderVariants, ...flacEncoderVariants, ...proresVariants, diff --git a/scripts/check.sh b/scripts/check.sh index ed2d185..40eef51 100755 --- a/scripts/check.sh +++ b/scripts/check.sh @@ -10,6 +10,9 @@ tsc -p packages/mp3-encoder rm -rf packages/ac3/dist/modules tsc -p packages/ac3 +rm -rf packages/dts/dist/modules +tsc -p packages/dts + rm -rf packages/aac-encoder/dist/modules tsc -p packages/aac-encoder diff --git a/scripts/ensure-license-headers.ts b/scripts/ensure-license-headers.ts index 44b2dbe..bcb93e2 100644 --- a/scripts/ensure-license-headers.ts +++ b/scripts/ensure-license-headers.ts @@ -38,6 +38,7 @@ checkDirectory(path.join(__dirname, '..', 'src')); checkDirectory(path.join(__dirname, '..', 'shared')); checkDirectory(path.join(__dirname, '..', 'packages', 'mp3-encoder', 'src')); checkDirectory(path.join(__dirname, '..', 'packages', 'ac3', 'src')); +checkDirectory(path.join(__dirname, '..', 'packages', 'dts', 'src')); checkDirectory(path.join(__dirname, '..', 'packages', 'flac-encoder', 'src')); checkDirectory(path.join(__dirname, '..', 'packages', 'aac-encoder', 'src')); checkDirectory(path.join(__dirname, '..', 'packages', 'prores', 'src')); diff --git a/shared/dts-misc.ts b/shared/dts-misc.ts new file mode 100644 index 0000000..d85cf8d --- /dev/null +++ b/shared/dts-misc.ts @@ -0,0 +1,33 @@ +/*! + * Copyright (c) 2026-present, Vanilagy and contributors + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +/** Sample rates the core can be encoded at (Table 5-5) */ +export const DTS_SAMPLE_RATES = [8000, 11025, 12000, 16000, 22050, 24000, 32000, 44100, 48000]; + +/** Channel counts DTS has a speaker layout for; 3 and 7 have no mapping */ +export const DTS_CHANNEL_COUNTS = [1, 2, 4, 5, 6]; + +const DTS_MAX_FRAME_SIZE = 16384; + +/** + * The DTS encoder needs each frame to be big enough to hold the per-channel side info and rejects the whole + * configuration when it isn't, so this mirrors the check from FFmpeg's dcaenc.c. + */ +export const dtsBitrateFits = (bitrate: number, sampleRate: number, numberOfChannels: number) => { + if (bitrate < 32000 || bitrate > 3840000) { + return false; + } + + const hasLfe = numberOfChannels === 6; + const fullbandChannels = numberOfChannels - (hasLfe ? 1 : 0); + + const frameBits = 32 * Math.ceil(Math.ceil(bitrate * 512 / sampleRate) / 32); + const minFrameBits = 132 + (493 + 28 * 32) * fullbandChannels + (hasLfe ? 72 : 0); + + return frameBits >= minFrameBits && frameBits <= 8 * DTS_MAX_FRAME_SIZE; +}; diff --git a/src/codec-data.ts b/src/codec-data.ts index 8319af3..071720d 100644 --- a/src/codec-data.ts +++ b/src/codec-data.ts @@ -6,7 +6,7 @@ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -import { AVC_LEVEL_TABLE, VideoCodec, VP9_LEVEL_TABLE } from './codec'; +import { AVC_LEVEL_TABLE, DtsFourCc, VideoCodec, VP9_LEVEL_TABLE } from './codec'; import { assert, assertNever, @@ -24,6 +24,7 @@ import { toUint8Array, getChromiumVersion, isChromium, + popcount, setUint24, } from './misc'; import { Logging } from './logging'; @@ -3176,3 +3177,495 @@ export const getEac3ChannelCount = (config: Eac3FrameInfo): number => { return channels; }; + +// ============================================================================ +// DTS Parsing +// Reference: ETSI TS 102 114 V1.6.1 +// ============================================================================ + +/** Core substream sync word, in the 16-bit big-endian packing that the registry mandates. */ +export const DTS_CORE_SYNC_WORD = 0x7ffe8001; + +/** Extension substream sync word. Section 7.4.1 */ +export const DTS_EXSS_SYNC_WORD = 0x64582025; + +/** The core frame header never reaches beyond this many bytes. */ +export const DTS_CORE_FRAME_HEADER_SIZE = 18; + +/** An extension substream always declares its own size within this many bytes. */ +export const DTS_EXSS_HEADER_PREFIX_SIZE = 10; + +/** The largest nuExtSSHeaderSize can get, and therefore how far the asset descriptors can reach. */ +export const DTS_EXSS_MAX_HEADER_SIZE = 4096; + +/** Number of PCM samples in one core PCM block; the core codes its length as a count of these. */ +export const DTS_PCM_BLOCK_SAMPLES = 32; + +/** Size of the DTSSpecificBox (ddts) payload in bytes. */ +export const DTS_SPECIFIC_BOX_SIZE = 20; + +/** Number of PCM blocks that a core frame's length must be a multiple of. */ +const DTS_SUBBAND_SAMPLES = 8; + +/** Core sample rates indexed by SFREQ. Zeroes mark invalid codes. Table 5-4 */ +const DTS_CORE_SAMPLE_RATES = [ + 0, 8000, 16000, 32000, 0, 0, 11025, 22050, + 44100, 0, 0, 12000, 24000, 48000, 96000, 192000, +]; + +/** + * Core bit rates in bps indexed by RATE, where a zero means the code isn't a constant rate. Table 5-7 + * + * Note that FFmpeg's ff_dca_bit_rates has 896000 where the spec has 960, and defines rates for codes 25 to 28 + * which this revision of the spec calls invalid. We keep the latter, since they cost nothing and some content + * predating the spec revision uses them. + */ +const DTS_CORE_BIT_RATES = [ + 32000, 56000, 64000, 96000, 112000, 128000, 192000, 224000, + 256000, 320000, 384000, 448000, 512000, 576000, 640000, 768000, + 960000, 1024000, 1152000, 1280000, 1344000, 1408000, 1411200, 1472000, + 1536000, 1920000, 2048000, 3072000, 3840000, 0, 0, 0, +]; + +/** Source PCM resolutions in bits indexed by PCMR. Zeroes mark invalid codes. */ +const DTS_PCM_RESOLUTIONS = [16, 16, 20, 20, 0, 24, 24, 0]; + +/** Channel counts indexed by AMODE, not counting LFE. */ +const DTS_AMODE_CHANNEL_COUNTS = [1, 2, 2, 2, 2, 3, 3, 4, 4, 5, 6, 6, 6, 7, 8, 8]; + +/** + * Speaker layout masks indexed by AMODE, expressed with the same bits as `ChannelLayout` in the DTSSpecificBox + * and `nuSpkrActivityMask` in an extension substream asset descriptor. + */ +const DTS_AMODE_CHANNEL_LAYOUTS = [ + 0x0001, 0x0002, 0x0002, 0x0002, 0x0002, 0x0003, 0x0012, 0x0013, + 0x0006, 0x0007, 0x0206, 0x0143, 0x0053, 0x0207, 0x0246, 0x0217, +]; + +/** The LFE1 speaker bit in a channel layout mask. */ +const DTS_CHANNEL_LAYOUT_LFE1 = 0x0008; + +/** The channel layout bits that stand for a pair of speakers rather than a single one. */ +const DTS_CHANNEL_LAYOUT_PAIR_MASK = 0xae66; + +/** Reference clock rates indexed by nuRefClockCode. The last code is unused. Table 7-3 */ +const DTS_EXSS_REF_CLOCKS = [32000, 44100, 48000, 0]; + +/** Sample rates used by extension substream assets, indexed by nuMaxSampleRate. */ +const DTS_EXSS_SAMPLE_RATES = [ + 8000, 16000, 32000, 64000, 128000, 22050, 44100, 88200, + 176400, 352800, 12000, 24000, 48000, 96000, 192000, 384000, +]; + +/** Frame durations that the DTSSpecificBox can express, indexed by FrameDuration. */ +const DTS_SPECIFIC_BOX_FRAME_DURATIONS = [512, 1024, 2048, 4096]; + +export type DtsCoreFrameInfo = { + /** Size of the core substream frame in bytes */ + frameSize: number; + sampleRate: number; + numberOfChannels: number; + /** Number of PCM samples the frame decodes to */ + sampleCount: number; + /** Speaker layout mask */ + channelLayout: number; + /** Audio channel arrangement (AMODE) */ + amode: number; + /** Whether an LFE channel is present */ + lfePresent: boolean; + /** Constant bit rate in bps, or 0 when the stream doesn't run at one */ + bitRate: number; + /** Source PCM resolution in bits */ + pcmResolution: number; +}; + +export type DtsExssAssetInfo = { + sampleRate: number; + numberOfChannels: number; + /** Number of PCM samples the frame decodes to */ + sampleCount: number; + /** Speaker layout mask, or 0 when the asset doesn't declare one */ + channelLayout: number; + /** Source PCM resolution in bits */ + pcmResolution: number; +}; + +export type DtsExssInfo = { + /** Size of the extension substream in bytes */ + frameSize: number; + /** Null when this substream omits the static fields that describe the stream */ + asset: DtsExssAssetInfo | null; +}; + +export type DtsFrameInfo = { + /** Size of the entire frame in bytes, core substream plus any extension substreams */ + frameSize: number; + sampleRate: number; + numberOfChannels: number; + /** Number of PCM samples the frame decodes to */ + sampleCount: number; + /** Speaker layout mask */ + channelLayout: number; + /** Source PCM resolution in bits */ + pcmResolution: number; + /** Constant bit rate in bps, or 0 when the stream doesn't run at one */ + bitRate: number; + /** The leading core substream, or null for extension-only streams such as DTS Express */ + core: DtsCoreFrameInfo | null; + /** Whether the frame carries extension substreams on top of the core */ + hasExtensions: boolean; +}; + +/** + * Parse one complete DTS frame, being a core substream frame followed by any number of extension substreams, + * or an extension substream on its own. Section 5 and Section 7.4.1 + */ +export const parseDtsFrame = (data: Uint8Array): DtsFrameInfo | null => { + const core = parseDtsCoreFrameHeader(data); + const view = toDataView(data); + + // The core substream is padded out to a 4-byte boundary before the first extension substream starts + let offset = core ? Math.ceil(core.frameSize / 4) * 4 : 0; + let firstExss: DtsExssInfo | null = null; + + while (offset + 4 <= data.length && view.getUint32(offset) === DTS_EXSS_SYNC_WORD) { + const exss = parseDtsExssHeader(data.subarray(offset)); + if (!exss) { + break; + } + + firstExss ??= exss; + offset += exss.frameSize; + } + + if (core) { + // The core describes what every DTS decoder can play back; the extension substreams only build on top of + // it, so the core's parameters are the ones we report. + return { + frameSize: firstExss ? offset : core.frameSize, + sampleRate: core.sampleRate, + numberOfChannels: core.numberOfChannels, + sampleCount: core.sampleCount, + channelLayout: core.channelLayout, + pcmResolution: core.pcmResolution, + bitRate: core.bitRate, + core, + hasExtensions: firstExss !== null, + }; + } + + if (!firstExss?.asset) { + return null; + } + + const { asset } = firstExss; + + return { + frameSize: offset, + sampleRate: asset.sampleRate, + numberOfChannels: asset.numberOfChannels, + sampleCount: asset.sampleCount, + channelLayout: asset.channelLayout, + pcmResolution: asset.pcmResolution, + bitRate: 0, + core: null, + hasExtensions: true, + }; +}; + +/** + * Works out which four-character code describes a packet, or null when the packet doesn't say. Telling 'dtsl' + * from 'dtse' would mean working out whether the asset holds XLL or LBR data, which sits behind the speaker + * remapping and mixing metadata deep in the asset descriptor, so we don't do it. + */ +export const extractDtsFourCcFromPacket = (data: Uint8Array): DtsFourCc | null => { + const frameInfo = parseDtsFrame(data); + if (!frameInfo?.core) { + return null; + } + + return frameInfo.hasExtensions ? 'dtsh' : 'dtsc'; +}; + +/** Parse the header of a core substream frame. Section 5.3 */ +export const parseDtsCoreFrameHeader = (data: Uint8Array): DtsCoreFrameInfo | null => { + if (data.length < DTS_CORE_FRAME_HEADER_SIZE) { + return null; + } + if (data[0] !== 0x7f || data[1] !== 0xfe || data[2] !== 0x80 || data[3] !== 0x01) { + return null; + } + + const bitstream = new Bitstream(data); + bitstream.skipBits(32); // SYNC + + bitstream.skipBits(1); // FTYPE + + // Terminating frames carry fewer samples than a full PCM block; we don't handle those + if (bitstream.readBits(5) !== DTS_PCM_BLOCK_SAMPLES - 1) { + return null; + } + + const cpf = bitstream.readBits(1); + const npcmblocks = bitstream.readBits(7) + 1; + if (npcmblocks % DTS_SUBBAND_SAMPLES !== 0) { + return null; + } + + const frameSize = bitstream.readBits(14) + 1; + if (frameSize < 96) { + return null; + } + + const amode = bitstream.readBits(6); + if (amode >= DTS_AMODE_CHANNEL_COUNTS.length) { + return null; + } + + const sampleRate = DTS_CORE_SAMPLE_RATES[bitstream.readBits(4)]!; + if (sampleRate === 0) { + return null; + } + + const bitRate = DTS_CORE_BIT_RATES[bitstream.readBits(5)]!; + + if (bitstream.readBits(1) !== 0) { + return null; // A reserved bit that must be zero, so a cheap way to reject false sync words + } + + bitstream.skipBits(1 + 1 + 1 + 1); // DYNF, TIMEF, AUXF, HDCD + bitstream.skipBits(3 + 1 + 1); // EXT_AUDIO_ID, EXT_AUDIO, ASPF + + const lff = bitstream.readBits(2); + if (lff === 3) { + return null; + } + + bitstream.skipBits(1); // HFLAG + if (cpf) { + bitstream.skipBits(16); // HCRC + } + + bitstream.skipBits(1 + 4 + 2); // FILTS, VERNUM, CHIST + + const pcmResolution = DTS_PCM_RESOLUTIONS[bitstream.readBits(3)]!; + if (pcmResolution === 0) { + return null; + } + + const lfePresent = lff !== 0; + + return { + frameSize, + sampleRate, + numberOfChannels: DTS_AMODE_CHANNEL_COUNTS[amode]! + (lfePresent ? 1 : 0), + sampleCount: npcmblocks * DTS_PCM_BLOCK_SAMPLES, + channelLayout: DTS_AMODE_CHANNEL_LAYOUTS[amode]! | (lfePresent ? DTS_CHANNEL_LAYOUT_LFE1 : 0), + amode, + lfePresent, + bitRate, + pcmResolution, + }; +}; + +/** Parse the header of an extension substream, along with its first audio asset descriptor. Section 7.4.1 */ +export const parseDtsExssHeader = (data: Uint8Array): DtsExssInfo | null => { + if (data.length < DTS_EXSS_HEADER_PREFIX_SIZE) { + return null; + } + if (data[0] !== 0x64 || data[1] !== 0x58 || data[2] !== 0x20 || data[3] !== 0x25) { + return null; + } + + const bitstream = new Bitstream(data); + bitstream.skipBits(32); // SYNC + bitstream.skipBits(8); // nuUserDefinedBits + + const extSsIndex = bitstream.readBits(2); + const wideHeader = bitstream.readBits(1); + const headerSizeBits = 8 + 4 * wideHeader; + const frameSizeBits = 16 + 4 * wideHeader; + + bitstream.skipBits(headerSizeBits); // nuExtSSHeaderSize + const frameSize = bitstream.readBits(frameSizeBits) + 1; + + // Everything past this point can run off the end of what we were given, in which case the Bitstream keeps + // handing out zeroes; the bounds check further down catches that + const incomplete: DtsExssInfo = { frameSize, asset: null }; + + if (!bitstream.readBits(1)) { // bStaticFieldsPresent + return incomplete; + } + + const refClock = DTS_EXSS_REF_CLOCKS[bitstream.readBits(2)]!; // nuRefClockCode + // The frame duration is a count of reference clock cycles, not of samples + const frameDurationCycles = 512 * (bitstream.readBits(3) + 1); // nuExSSFrameDurationCode + + if (bitstream.readBits(1)) { // bTimeStampFlag + bitstream.skipBits(32 + 4); // nuTimeStamp, nLSB + } + + const numAudioPresentations = bitstream.readBits(3) + 1; + const numAssets = bitstream.readBits(3) + 1; + + const activeExssMasks: number[] = []; + for (let i = 0; i < numAudioPresentations; i++) { + activeExssMasks.push(bitstream.readBits(extSsIndex + 1)); + } + for (const mask of activeExssMasks) { + bitstream.skipBits(8 * popcount(mask)); // nuActiveAssetMask + } + + if (bitstream.readBits(1)) { // bMixMetadataEnbl + bitstream.skipBits(2); // nuMixMetadataAdjLevel + const spkrMaskBits = (bitstream.readBits(2) + 1) << 2; + const numMixOutConfigs = bitstream.readBits(2) + 1; + bitstream.skipBits(numMixOutConfigs * spkrMaskBits); // nuMixOutChMask + } + + for (let i = 0; i < numAssets; i++) { + bitstream.skipBits(frameSizeBits); // nuAssetFsize + } + + // From here on we're inside the first audio asset descriptor + bitstream.skipBits(9); // nuAssetDescriptFsize + bitstream.skipBits(3); // nuAssetIndex + + if (bitstream.readBits(1)) { // bAssetTypeDescrPresent + bitstream.skipBits(4); // nuAssetTypeDescriptor + } + if (bitstream.readBits(1)) { // bLanguageDescrPresent + bitstream.skipBits(24); // LanguageDescriptor + } + if (bitstream.readBits(1)) { // bInfoTextPresent + bitstream.skipBits(8 * (bitstream.readBits(10) + 1)); // nuInfoTextByteSize, InfoTextString + } + + const pcmResolution = bitstream.readBits(5) + 1; + const sampleRate = DTS_EXSS_SAMPLE_RATES[bitstream.readBits(4)]!; + const numberOfChannels = bitstream.readBits(8) + 1; + + let channelLayout = 0; + + if (bitstream.readBits(1)) { // bOne2OneMapChannels2Speakers + if (numberOfChannels > 2) { + bitstream.skipBits(1); // bEmbeddedStereoFlag + } + if (numberOfChannels > 6) { + bitstream.skipBits(1); // bEmbeddedSixChFlag + } + + if (bitstream.readBits(1)) { // bSpkrMaskEnabled + const spkrMaskBits = (bitstream.readBits(2) + 1) << 2; + channelLayout = bitstream.readBits(spkrMaskBits); // nuSpkrActivityMask + } + } + + if (refClock === 0 || bitstream.getBitsLeft() < 0) { + return incomplete; + } + + return { + frameSize, + asset: { + sampleRate, + numberOfChannels, + sampleCount: Math.round(frameDurationCycles * sampleRate / refClock), + channelLayout, + pcmResolution, + }, + }; +}; + +export type DtsSpecificBoxInfo = { + sampleRate: number; + maxBitrate: number; + avgBitrate: number; + pcmSampleDepth: number; + /** Number of PCM samples one frame decodes to */ + sampleCount: number; + /** Speaker layout mask, or 0 when the box doesn't declare one */ + channelLayout: number; + /** Null when the box carries nothing we can derive a channel count from */ + numberOfChannels: number | null; +}; + +/** Parse a DTSSpecificBox (ddts). */ +export const parseDtsSpecificBox = (data: Uint8Array): DtsSpecificBoxInfo | null => { + if (data.length < DTS_SPECIFIC_BOX_SIZE) { + return null; + } + + const view = toDataView(data); + const sampleRate = view.getUint32(0); + if (sampleRate === 0) { + return null; + } + + const bitstream = new Bitstream(data); + bitstream.seekToByte(13); + + const frameDuration = bitstream.readBits(2); + bitstream.skipBits(5); // StreamConstruction + const coreLfePresent = bitstream.readBits(1); + const coreLayout = bitstream.readBits(6); + bitstream.skipBits(14); // CoreSize + bitstream.skipBits(1); // StereoDownmix + bitstream.skipBits(3); // RepresentationType + const channelLayout = bitstream.readBits(16); + + let numberOfChannels: number | null = null; + if (channelLayout !== 0) { + numberOfChannels = getDtsChannelCount(channelLayout); + } else if (coreLayout < DTS_AMODE_CHANNEL_COUNTS.length) { + numberOfChannels = DTS_AMODE_CHANNEL_COUNTS[coreLayout]! + coreLfePresent; + } + + return { + sampleRate, + maxBitrate: view.getUint32(4), + avgBitrate: view.getUint32(8), + pcmSampleDepth: data[12]!, + sampleCount: DTS_SPECIFIC_BOX_FRAME_DURATIONS[frameDuration]!, + channelLayout, + numberOfChannels, + }; +}; + +/** Build the payload of a DTSSpecificBox (ddts) from a frame of the stream it describes. */ +export const buildDtsSpecificBox = (frameInfo: DtsFrameInfo) => { + const bytes = new Uint8Array(DTS_SPECIFIC_BOX_SIZE); + const view = toDataView(bytes); + + view.setUint32(0, frameInfo.sampleRate); + view.setUint32(4, frameInfo.bitRate); + view.setUint32(8, frameInfo.bitRate); + bytes[12] = frameInfo.pcmResolution; + + // The spec only defines codes for streams built out of a known set of substreams. Anything else is required + // to be signaled as 0, meaning the construction is left unspecified. + const streamConstruction = frameInfo.core && !frameInfo.hasExtensions ? 1 : 0; + + const bitstream = new Bitstream(bytes); + bitstream.seekToByte(13); + + bitstream.writeBits(2, Math.max(DTS_SPECIFIC_BOX_FRAME_DURATIONS.indexOf(frameInfo.sampleCount), 0)); + bitstream.writeBits(5, streamConstruction); + bitstream.writeBits(1, frameInfo.core?.lfePresent ? 1 : 0); + bitstream.writeBits(6, frameInfo.core?.amode ?? 0); + bitstream.writeBits(14, frameInfo.core ? frameInfo.core.frameSize - 1 : 0); + bitstream.writeBits(1, 0); // StereoDownmix + bitstream.writeBits(3, 0); // RepresentationType + bitstream.writeBits(16, frameInfo.channelLayout); + bitstream.writeBits(1, 0); // MultiAssetFlag + bitstream.writeBits(1, 0); // LBRDurationMod + bitstream.writeBits(1, 0); // ReservedBoxPresent + bitstream.writeBits(5, 0); // Reserved + + return bytes; +}; + +/** Count the channels in a DTS speaker layout mask, where some bits stand for a pair of speakers. */ +const getDtsChannelCount = (channelLayout: number) => { + return popcount(channelLayout) + popcount(channelLayout & DTS_CHANNEL_LAYOUT_PAIR_MASK); +}; diff --git a/src/codec.ts b/src/codec.ts index 08f1b52..63c41d1 100644 --- a/src/codec.ts +++ b/src/codec.ts @@ -75,6 +75,7 @@ export const NON_PCM_AUDIO_CODECS = [ 'flac', 'ac3', 'eac3', + 'dts', ] as const; /** * List of known audio codecs, ordered by encoding preference. @@ -227,6 +228,14 @@ export const PRORES_FOURCCS = [ ] as const; export type ProresFourCc = typeof PRORES_FOURCCS[number]; +export const DTS_FOURCCS = [ + 'dtsc', // DTS core + 'dtsh', // DTS-HD, core plus extension substreams + 'dtsl', // DTS-HD Lossless, no core + 'dtse', // DTS Express +] as const; +export type DtsFourCc = typeof DTS_FOURCCS[number]; + // Target data rates of the ProRes profiles at 1920x1080 ~30fps, as published by Apple const PRORES_PROFILE_TARGET_BITRATES: { fourCc: ProresFourCc; bitrate: number; alpha: boolean }[] = [ { fourCc: 'apco', bitrate: 45_000_000, alpha: false }, // 422 Proxy @@ -595,6 +604,8 @@ export const buildAudioCodecString = (codec: AudioCodec, numberOfChannels: numbe return 'ac-3'; } else if (codec === 'eac3') { return 'ec-3'; + } else if (codec === 'dts') { + return 'dtsc'; } else if ((PCM_AUDIO_CODECS as readonly string[]).includes(codec)) { return codec; } @@ -611,8 +622,9 @@ export const extractAudioCodecString = (trackInfo: { codec: AudioCodec | null; codecDescription: Uint8Array | null; aacCodecInfo: AacCodecInfo | null; + dtsFormat: DtsFourCc | null; }) => { - const { codec, codecDescription, aacCodecInfo } = trackInfo; + const { codec, codecDescription, aacCodecInfo, dtsFormat } = trackInfo; if (codec === 'aac') { if (!aacCodecInfo) { @@ -644,6 +656,8 @@ export const extractAudioCodecString = (trackInfo: { return 'ac-3'; } else if (codec === 'eac3') { return 'ec-3'; + } else if (codec === 'dts') { + return dtsFormat ?? 'dtsc'; } else if (codec && (PCM_AUDIO_CODECS as readonly string[]).includes(codec)) { return codec; } @@ -756,6 +770,8 @@ export const inferCodecFromCodecString = (codecString: string): MediaCodec | nul return 'ac3'; } else if (codecString === 'ec-3' || codecString === 'eac3') { return 'eac3'; + } else if ((DTS_FOURCCS as readonly string[]).includes(codecString)) { + return 'dts'; } else if (codecString === 'ulaw') { return 'ulaw'; } else if (codecString === 'alaw') { @@ -998,7 +1014,7 @@ export const validateVideoChunkMetadata = ( }; const VALID_AUDIO_CODEC_STRING_PREFIXES = [ - 'mp4a', 'mp3', 'opus', 'vorbis', 'flac', 'ulaw', 'alaw', 'pcm', 'ac-3', 'ec-3', + 'mp4a', 'mp3', 'opus', 'vorbis', 'flac', 'ulaw', 'alaw', 'pcm', 'ac-3', 'ec-3', 'dts', ]; export const validateAudioChunkMetadata = ( @@ -1132,6 +1148,15 @@ export const validateAudioChunkMetadata = ( if (metadata.decoderConfig.codec !== 'ec-3') { throw new TypeError('Audio chunk metadata decoder configuration codec string for EC-3 must be "ec-3".'); } + } else if (metadata.decoderConfig.codec.startsWith('dts')) { + // DTS-specific validation + + if (!(DTS_FOURCCS as readonly string[]).includes(metadata.decoderConfig.codec)) { + throw new TypeError( + 'Audio chunk metadata decoder configuration codec string for DTS must be one of the following' + + ` four-character codes: ${DTS_FOURCCS.join(', ')}.`, + ); + } } else if ( metadata.decoderConfig.codec.startsWith('pcm') || metadata.decoderConfig.codec.startsWith('ulaw') diff --git a/src/encode.ts b/src/encode.ts index 656122d..a30fd9f 100644 --- a/src/encode.ts +++ b/src/encode.ts @@ -858,6 +858,7 @@ export class Quality { vorbis: 64000, // 64kbps base for Vorbis ac3: 384000, // 384kbps base for AC-3 eac3: 192000, // 192kbps base for E-AC-3 + dts: 768000, // 768kbps base for DTS }; const baseBitrate = baseRates[codec as keyof typeof baseRates]; @@ -1071,7 +1072,7 @@ export const canEncodeVideo = async ( } validateVideoEncodingAdditionalOptions(codec, restOptions); - const resolvedQuality = resolveQuality(quality, bitrate) ?? new Quality({ bitrate: 1e6 }); + const resolvedQuality = resolveQuality(quality, bitrate) ?? new Quality('medium'); let candidates: VideoEncoderConfigCandidate[]; try { @@ -1222,7 +1223,7 @@ export const canEncodeAudio = async ( } validateAudioEncodingAdditionalOptions(codec, restOptions); - const resolvedQuality = resolveQuality(quality, bitrate) ?? new Quality({ bitrate: 128e3 }); + const resolvedQuality = resolveQuality(quality, bitrate) ?? new Quality('medium'); const encoderConfig = buildAudioEncoderConfig({ codec, diff --git a/src/isobmff/isobmff-boxes.ts b/src/isobmff/isobmff-boxes.ts index 3867fc4..ca3ba8b 100644 --- a/src/isobmff/isobmff-boxes.ts +++ b/src/isobmff/isobmff-boxes.ts @@ -43,7 +43,13 @@ import { IsobmffVideoTrackData, Sample, } from './isobmff-muxer'; -import { parseAc3SyncFrame, parseEac3SyncFrame, parseOpusIdentificationHeader } from '../codec-data'; +import { + buildDtsSpecificBox, + parseAc3SyncFrame, + parseDtsFrame, + parseEac3SyncFrame, + parseOpusIdentificationHeader, +} from '../codec-data'; import { MetadataTags, RichImageData } from '../metadata'; import { Bitstream } from '../../shared/bitstream'; @@ -690,7 +696,11 @@ export const stsd = (trackData: IsobmffTrackData) => { trackData, ); } else if (trackData.type === 'audio') { - const boxName = audioCodecToBoxName(trackData.track.source._codec, trackData.muxer.isQuickTime); + const boxName = audioCodecToBoxName( + trackData.track.source._codec, + trackData.info.decoderConfig.codec, + trackData.muxer.isQuickTime, + ); assert(boxName); sampleDescription = soundSampleDescription( @@ -968,7 +978,11 @@ export const wave = (trackData: IsobmffAudioTrackData) => { export const frma = (trackData: IsobmffAudioTrackData) => { return box('frma', [ - ascii(audioCodecToBoxName(trackData.track.source._codec, trackData.muxer.isQuickTime)), + ascii(audioCodecToBoxName( + trackData.track.source._codec, + trackData.info.decoderConfig.codec, + trackData.muxer.isQuickTime, + )), ]); }; @@ -1123,6 +1137,21 @@ const dec3 = (trackData: IsobmffAudioTrackData) => { return box('dec3', [...bytes]); }; +/** DTSSpecificBox */ +const ddts = (trackData: IsobmffAudioTrackData) => { + assert(trackData.info.primingPacket); + + const frameInfo = parseDtsFrame(trackData.info.primingPacket.data); + if (!frameInfo) { + throw new Error( + 'Couldn\'t extract DTS frame info from the audio packet. ' + + 'Ensure the packets contain valid DTS frames as specified in ETSI TS 102 114.', + ); + } + + return box('ddts', [...buildDtsSpecificBox(frameInfo)]); +}; + export const subtitleSampleDescription = ( compressionType: string, trackData: IsobmffSubtitleTrackData, @@ -1816,7 +1845,7 @@ const VIDEO_CODEC_TO_CONFIGURATION_BOX: Record< prores: null, }; -const audioCodecToBoxName = (codec: AudioCodec, isQuickTime: boolean): string => { +const audioCodecToBoxName = (codec: AudioCodec, fullCodecString: string, isQuickTime: boolean): string => { switch (codec) { case 'aac': return 'mp4a'; case 'mp3': return 'mp4a'; @@ -1829,6 +1858,7 @@ const audioCodecToBoxName = (codec: AudioCodec, isQuickTime: boolean): string => case 'pcm-s8': return 'sowt'; case 'ac3': return 'ac-3'; case 'eac3': return 'ec-3'; + case 'dts': return fullCodecString; } // Logic diverges here @@ -1870,6 +1900,7 @@ const audioCodecToConfigurationBox = (codec: AudioCodec, isQuickTime: boolean) = case 'flac': return dfLa; case 'ac3': return dac3; case 'eac3': return dec3; + case 'dts': return ddts; } // Logic diverges here diff --git a/src/isobmff/isobmff-demuxer.ts b/src/isobmff/isobmff-demuxer.ts index 98ab435..8419780 100644 --- a/src/isobmff/isobmff-demuxer.ts +++ b/src/isobmff/isobmff-demuxer.ts @@ -11,6 +11,8 @@ import { parseAacAudioSpecificConfig } from '../../shared/aac-misc'; import { AacCodecInfo, AudioCodec, + DTS_FOURCCS, + DtsFourCc, extractAudioCodecString, extractVideoCodecString, MediaCodec, @@ -33,6 +35,9 @@ import { parseEac3Config, getEac3SampleRate, getEac3ChannelCount, + extractDtsFourCcFromPacket, + parseDtsSpecificBox, + DTS_SPECIFIC_BOX_SIZE, AC3_ACMOD_CHANNEL_COUNTS, } from '../codec-data'; import { Demuxer } from '../demuxer'; @@ -159,6 +164,7 @@ type InternalTrack = { codec: AudioCodec | null; codecDescription: Uint8Array | null; aacCodecInfo: AacCodecInfo | null; + dtsFormat: DtsFourCc | null; pcmLittleEndian: boolean; pcmSampleSize: number | null; }; @@ -1034,6 +1040,7 @@ export class IsobmffDemuxer extends Demuxer { codec: null, codecDescription: null, aacCodecInfo: null, + dtsFormat: null, pcmLittleEndian: false, pcmSampleSize: null, }; @@ -1185,6 +1192,9 @@ export class IsobmffDemuxer extends Demuxer { track.info.codec = 'ac3'; } else if (codecName === 'ec-3') { track.info.codec = 'eac3'; + } else if ((DTS_FOURCCS as readonly string[]).includes(codecName!)) { + track.info.codec = 'dts'; + track.info.dtsFormat = codecName as DtsFourCc; } else if (codecName === 'twos') { if (sampleSize === 8) { track.info.codec = 'pcm-s8'; @@ -1564,6 +1574,8 @@ export class IsobmffDemuxer extends Demuxer { track.info.codec = 'mp3'; } else if (objectTypeIndication === 0xdd) { track.info.codec = 'vorbis'; // "nonstandard, gpac uses it" - FFmpeg + } else if (objectTypeIndication === 0xa9) { + track.info.codec = 'dts'; } else { Logging._warn( `Unsupported audio codec (objectTypeIndication ${objectTypeIndication}) - discarding track.`, @@ -1763,6 +1775,28 @@ export class IsobmffDemuxer extends Demuxer { track.info.numberOfChannels = getEac3ChannelCount(config); }; break; + case 'ddts': { // DTSSpecificBox + const track = this.currentTrack; + if (!track) { + break; + } + assert(track.info?.type === 'audio'); + + const bytes = readBytes(slice, Math.min(boxInfo.contentSize, DTS_SPECIFIC_BOX_SIZE)); + const config = parseDtsSpecificBox(bytes); + + if (!config) { + Logging._warn('Invalid ddts box contents, ignoring.'); + break; + } + + track.info.sampleRate = config.sampleRate; + + if (config.numberOfChannels !== null) { + track.info.numberOfChannels = config.numberOfChannels; + } + }; break; + case 'stts': { const track = this.currentTrack; if (!track) { @@ -3404,7 +3438,7 @@ class IsobmffVideoTrackBacking extends IsobmffTrackBacking implements InputVideo class IsobmffAudioTrackBacking extends IsobmffTrackBacking implements InputAudioTrackBacking { override internalTrack: InternalAudioTrack; - decoderConfig: AudioDecoderConfig | null = null; + decoderConfigPromise: Promise | null = null; constructor(internalTrack: InternalAudioTrack) { super(internalTrack); @@ -3432,12 +3466,20 @@ class IsobmffAudioTrackBacking extends IsobmffTrackBacking implements InputAudio return null; } - return this.decoderConfig ??= { - codec: extractAudioCodecString(this.internalTrack.info), - numberOfChannels: this.internalTrack.info.numberOfChannels, - sampleRate: this.internalTrack.info.sampleRate, - description: this.internalTrack.info.codecDescription ?? undefined, - }; + return this.decoderConfigPromise ??= (async (): Promise => { + if (this.internalTrack.info.codec === 'dts' && !this.internalTrack.info.dtsFormat) { + // Gotta check the packet to determine the DTS variant + const firstPacket = await this.getFirstPacket({}); + this.internalTrack.info.dtsFormat = firstPacket && extractDtsFourCcFromPacket(firstPacket.data); + } + + return { + codec: extractAudioCodecString(this.internalTrack.info), + numberOfChannels: this.internalTrack.info.numberOfChannels, + sampleRate: this.internalTrack.info.sampleRate, + description: this.internalTrack.info.codecDescription ?? undefined, + }; + })(); } } diff --git a/src/isobmff/isobmff-muxer.ts b/src/isobmff/isobmff-muxer.ts index 06b5f15..4d6c8c5 100644 --- a/src/isobmff/isobmff-muxer.ts +++ b/src/isobmff/isobmff-muxer.ts @@ -531,10 +531,14 @@ export class IsobmffMuxer extends Muxer { requiresAdtsStripping = true; } - if (track.source._codec === 'ac3' || track.source._codec === 'eac3') { - if (!packet) { + if (!packet) { + if (track.source._codec === 'ac3' || track.source._codec === 'eac3') { throw new Error('AC-3/E-AC-3 require a priming packet.'); } + + if (track.source._codec === 'dts') { + throw new Error('DTS requires a priming packet.'); + } } const newTrackData: IsobmffAudioTrackData = { diff --git a/src/matroska/ebml.ts b/src/matroska/ebml.ts index 08a07c7..6cc5a88 100644 --- a/src/matroska/ebml.ts +++ b/src/matroska/ebml.ts @@ -750,6 +750,7 @@ export const CODEC_STRING_MAP: Partial> = { 'flac': 'A_FLAC', 'ac3': 'A_AC3', 'eac3': 'A_EAC3', + 'dts': 'A_DTS', 'pcm-u8': 'A_PCM/INT/LIT', 'pcm-s16': 'A_PCM/INT/LIT', 'pcm-s16be': 'A_PCM/INT/BIG', diff --git a/src/matroska/matroska-demuxer.ts b/src/matroska/matroska-demuxer.ts index 26d538b..9fbbc3f 100644 --- a/src/matroska/matroska-demuxer.ts +++ b/src/matroska/matroska-demuxer.ts @@ -9,6 +9,7 @@ import { TrackType } from '../output'; import { extractAv1CodecInfoFromPacket, + extractDtsFourCcFromPacket, extractAvcDecoderConfigurationRecord, extractHevcDecoderConfigurationRecord, extractVp9CodecInfoFromPacket, @@ -16,6 +17,7 @@ import { import { AacCodecInfo, AudioCodec, + DtsFourCc, extractAudioCodecString, extractVideoCodecString, MediaCodec, @@ -229,6 +231,7 @@ type InternalTrack = { codec: AudioCodec | null; codecDescription: Uint8Array | null; aacCodecInfo: AacCodecInfo | null; + dtsFormat: DtsFourCc | null; }; }; type InternalVideoTrack = InternalTrack & { info: { type: 'video' } }; @@ -1127,6 +1130,14 @@ export class MatroskaDemuxer extends Demuxer { } else if (codecIdWithoutSuffix === CODEC_STRING_MAP.eac3) { this.currentTrack.info.codec = 'eac3'; this.currentTrack.info.codecDescription = this.currentTrack.codecPrivate; + } else if (codecIdWithoutSuffix === CODEC_STRING_MAP.dts) { + this.currentTrack.info.codec = 'dts'; + + if (this.currentTrack.codecId === 'A_DTS/EXPRESS') { + this.currentTrack.info.dtsFormat = 'dtse'; + } else if (this.currentTrack.codecId === 'A_DTS/LOSSLESS') { + this.currentTrack.info.dtsFormat = 'dtsl'; + } } else if (this.currentTrack.codecId === 'A_PCM/INT/LIT') { if (this.currentTrack.info.bitDepth === 8) { this.currentTrack.info.codec = 'pcm-u8'; @@ -1200,6 +1211,7 @@ export class MatroskaDemuxer extends Demuxer { codec: null, codecDescription: null, aacCodecInfo: null, + dtsFormat: null, }; } }; break; @@ -2556,7 +2568,7 @@ class MatroskaVideoTrackBacking extends MatroskaTrackBacking implements InputVid class MatroskaAudioTrackBacking extends MatroskaTrackBacking implements InputAudioTrackBacking { override internalTrack: InternalAudioTrack; - decoderConfig: AudioDecoderConfig | null = null; + decoderConfigPromise: Promise | null = null; constructor(internalTrack: InternalAudioTrack) { super(internalTrack); @@ -2584,15 +2596,24 @@ class MatroskaAudioTrackBacking extends MatroskaTrackBacking implements InputAud return null; } - return this.decoderConfig ??= { - codec: extractAudioCodecString({ - codec: this.internalTrack.info.codec, - codecDescription: this.internalTrack.info.codecDescription, - aacCodecInfo: this.internalTrack.info.aacCodecInfo, - }), - numberOfChannels: this.internalTrack.info.numberOfChannels, - sampleRate: this.internalTrack.info.sampleRate, - description: this.internalTrack.info.codecDescription ?? undefined, - }; + return this.decoderConfigPromise ??= (async (): Promise => { + if (this.internalTrack.info.codec === 'dts' && !this.internalTrack.info.dtsFormat) { + // Gotta check the packet to determine the DTS variant + const firstPacket = await this.getFirstPacket({}); + this.internalTrack.info.dtsFormat = firstPacket && extractDtsFourCcFromPacket(firstPacket.data); + } + + return { + codec: extractAudioCodecString({ + codec: this.internalTrack.info.codec, + codecDescription: this.internalTrack.info.codecDescription, + aacCodecInfo: this.internalTrack.info.aacCodecInfo, + dtsFormat: this.internalTrack.info.dtsFormat, + }), + numberOfChannels: this.internalTrack.info.numberOfChannels, + sampleRate: this.internalTrack.info.sampleRate, + description: this.internalTrack.info.codecDescription ?? undefined, + }; + })(); } } diff --git a/src/matroska/matroska-muxer.ts b/src/matroska/matroska-muxer.ts index 41b0525..7763df3 100644 --- a/src/matroska/matroska-muxer.ts +++ b/src/matroska/matroska-muxer.ts @@ -310,9 +310,18 @@ export class MatroskaMuxer extends Muxer { this.tracksElement = tracksElement; for (const trackData of this.trackDatas) { - const codecId = CODEC_STRING_MAP[trackData.track.source._codec]; + let codecId = CODEC_STRING_MAP[trackData.track.source._codec]; assert(codecId); + if (trackData.type === 'audio' && trackData.track.source._codec === 'dts') { + // We can further refine the Codec ID + if (trackData.info.decoderConfig.codec === 'dtse') { + codecId = 'A_DTS/EXPRESS'; + } else if (trackData.info.decoderConfig.codec === 'dtsl') { + codecId = 'A_DTS/LOSSLESS'; + } + } + let seekPreRollNs = 0; if (trackData.type === 'audio' && trackData.track.source._codec === 'opus') { seekPreRollNs = 1e6 * 80; // In "Matroska ticks" (nanoseconds) diff --git a/src/media-sink.ts b/src/media-sink.ts index d67148d..01d82cf 100644 --- a/src/media-sink.ts +++ b/src/media-sink.ts @@ -1297,8 +1297,13 @@ class VideoDecoderWrapper extends DecoderWrapper { void this.customDecoderCallSerializer.call(() => this.customDecoder!.close()); } else { assert(this.decoder); - this.decoder.close(); - this.alphaDecoder?.close(); + + if (this.decoder.state !== 'closed') { + this.decoder.close(); + } + if (this.alphaDecoder && this.alphaDecoder.state !== 'closed') { + this.alphaDecoder.close(); + } this.colorQueue.forEach(x => x.close()); this.colorQueue.length = 0; @@ -2202,7 +2207,10 @@ class AudioDecoderWrapper extends DecoderWrapper { void this.customDecoderCallSerializer.call(() => this.customDecoder!.close()); } else { assert(this.decoder); - this.decoder.close(); + + if (this.decoder.state !== 'closed') { + this.decoder.close(); + } } } } diff --git a/src/misc.ts b/src/misc.ts index bd6979d..60fea06 100644 --- a/src/misc.ts +++ b/src/misc.ts @@ -472,6 +472,17 @@ export const ilog = (x: number) => { return ret; }; +export const popcount = (value: number) => { + let count = 0; + + while (value !== 0) { + value &= value - 1; + count++; + } + + return count; +}; + const ISO_639_2_REGEX = /^[a-z]{3}$/; export const isIso639Dash2LanguageCode = (x: string) => { return ISO_639_2_REGEX.test(x); diff --git a/src/mpeg-ts/mpeg-ts-demuxer.ts b/src/mpeg-ts/mpeg-ts-demuxer.ts index f82a09c..54635cd 100644 --- a/src/mpeg-ts/mpeg-ts-demuxer.ts +++ b/src/mpeg-ts/mpeg-ts-demuxer.ts @@ -13,6 +13,7 @@ import { aacChannelMap, aacFrequencyTable } from '../../shared/aac-misc'; import { AacCodecInfo, AudioCodec, + DtsFourCc, extractAudioCodecString, extractVideoCodecString, MediaCodec, @@ -38,6 +39,12 @@ import { AC3_FRAME_SIZES, extractNalUnitTypeForAvc, extractNalUnitTypeForHevc, + parseDtsFrame, + parseDtsCoreFrameHeader, + parseDtsExssHeader, + DTS_CORE_FRAME_HEADER_SIZE, + DTS_EXSS_HEADER_PREFIX_SIZE, + DTS_EXSS_MAX_HEADER_SIZE, } from '../codec-data'; import { Demuxer } from '../demuxer'; import { Input } from '../input'; @@ -82,6 +89,22 @@ import { Bitstream } from '../../shared/bitstream'; const MISSING_PTS_ERROR_MESSAGE = 'PES packet is missing PTS where it was expected. PES packets without PTS are not' + ' currently supported. If you think this file should be supported, please report it.'; +const REGISTRATION_DESCRIPTOR_TAG = 0x05; + +// The 'HDMV' and 'HDPR' format identifiers, which mark a program as Blu-ray-derived +const HDMV_FORMAT_IDENTIFIER = 0x48444d56; +const HDPR_FORMAT_IDENTIFIER = 0x48445052; + +// 'DTS1', 'DTS2' and 'DTS3' all share this prefix +const DTS_FORMAT_IDENTIFIER_PREFIX = 0x44545300; + +// Stream types that only mean DTS within a Blu-ray-derived program +const BLU_RAY_DTS_STREAM_TYPES = new Set([ + MpegTsStreamType.BLU_RAY_DTS_HD, + MpegTsStreamType.BLU_RAY_DTS_HD_MASTER, + MpegTsStreamType.BLU_RAY_DTS_EXPRESS_SECONDARY, +]); + type ElementaryStream = { demuxer: MpegTsDemuxer; pid: number; @@ -110,6 +133,7 @@ type ElementaryStream = { codec: AudioCodec; decoderConfig: AudioDecoderConfig | null; aacCodecInfo: AacCodecInfo | null; + dtsFormat: DtsFourCc | null; numberOfChannels: number; sampleRate: number; }; @@ -294,7 +318,27 @@ export class MpegTsDemuxer extends Demuxer { // "The remaining 10 bits specify the number of bytes of the descriptors immediately following the // program_info_length field" const programInfoLength = bitstream.readBits(10); - bitstream.skipBits(8 * programInfoLength); + + // A few stream types only mean what Blu-ray says they mean, and the program's registration + // descriptor is what tells us we're looking at a Blu-ray-derived stream. + const programInfoEndPos = bitstream.pos + 8 * programInfoLength; + let isBluRayProgram = false; + + while (bitstream.pos < programInfoEndPos) { + const descriptorTag = bitstream.readBits(8); + const descriptorLength = bitstream.readBits(8); + const descriptorEndPos = bitstream.pos + 8 * descriptorLength; + + if (descriptorTag === REGISTRATION_DESCRIPTOR_TAG && descriptorLength >= 4) { + const formatIdentifier = bitstream.readBits(32); + isBluRayProgram ||= formatIdentifier === HDMV_FORMAT_IDENTIFIER + || formatIdentifier === HDPR_FORMAT_IDENTIFIER; + } + + bitstream.pos = descriptorEndPos; + } + + bitstream.pos = programInfoEndPos; while (8 * (sectionLength + BYTES_BEFORE_SECTION_LENGTH) - bitstream.pos > BITS_IN_CRC_32) { const streamType = bitstream.readBits(8); @@ -304,24 +348,40 @@ export class MpegTsDemuxer extends Demuxer { bitstream.skipBits(6); const esInfoLength = bitstream.readBits(10); - // Check ES descriptors to detect AC-3/E-AC-3 in System B + // Check ES descriptors to detect AC-3/E-AC-3/DTS in System B const esInfoEndPos = bitstream.pos + 8 * esInfoLength; let hasAc3Descriptor = false; let hasEac3Descriptor = false; + let hasDtsDescriptor = false; while (bitstream.pos < esInfoEndPos) { const descriptorTag = bitstream.readBits(8); const descriptorLength = bitstream.readBits(8); + const descriptorEndPos = bitstream.pos + 8 * descriptorLength; + if (descriptorTag === 0x6a) { hasAc3Descriptor = true; } else if (descriptorTag === 0x7a || descriptorTag === 0xcc) { hasEac3Descriptor = true; + } else if (descriptorTag === 0x7b) { + hasDtsDescriptor = true; + } else if (descriptorTag === REGISTRATION_DESCRIPTOR_TAG && descriptorLength >= 4) { + // DTS uses 'DTS1', 'DTS2' and 'DTS3' to also signal the stream's frame size, + // which we work out from the bitstream anyway + const formatIdentifier = bitstream.readBits(32); + hasDtsDescriptor ||= (formatIdentifier & 0xffffff00) === DTS_FORMAT_IDENTIFIER_PREFIX; } - bitstream.skipBits(8 * descriptorLength); + + bitstream.pos = descriptorEndPos; } let info: ElementaryStream['info'] | null = null; - switch (streamType) { + // Blu-ray has its own stream types for the DTS extensions + const effectiveStreamType = isBluRayProgram && BLU_RAY_DTS_STREAM_TYPES.has(streamType) + ? MpegTsStreamType.DTS + : streamType; + + switch (effectiveStreamType) { case MpegTsStreamType.AVC: case MpegTsStreamType.HEVC: { const codec = streamType === MpegTsStreamType.AVC ? 'avc' : 'hevc'; @@ -350,21 +410,23 @@ export class MpegTsDemuxer extends Demuxer { case MpegTsStreamType.MP3_MPEG2: case MpegTsStreamType.AAC: case MpegTsStreamType.AC3_SYSTEM_A: - case MpegTsStreamType.EAC3_SYSTEM_A: { + case MpegTsStreamType.EAC3_SYSTEM_A: + case MpegTsStreamType.DTS: + case MpegTsStreamType.DTS_ATSC: { let codec: AudioCodec; if ( - streamType === MpegTsStreamType.MP3_MPEG1 - || streamType === MpegTsStreamType.MP3_MPEG2 + effectiveStreamType === MpegTsStreamType.MP3_MPEG1 + || effectiveStreamType === MpegTsStreamType.MP3_MPEG2 ) { codec = 'mp3'; - } else if (streamType === MpegTsStreamType.AAC) { + } else if (effectiveStreamType === MpegTsStreamType.AAC) { codec = 'aac'; - } else if (streamType === MpegTsStreamType.AC3_SYSTEM_A) { + } else if (effectiveStreamType === MpegTsStreamType.AC3_SYSTEM_A) { codec = 'ac3'; - } else if (streamType === MpegTsStreamType.EAC3_SYSTEM_A) { + } else if (effectiveStreamType === MpegTsStreamType.EAC3_SYSTEM_A) { codec = 'eac3'; } else { - throw new Error('Unreachable.'); + codec = 'dts'; } info = { @@ -372,6 +434,7 @@ export class MpegTsDemuxer extends Demuxer { codec, decoderConfig: null, aacCodecInfo: null, + dtsFormat: null, numberOfChannels: -1, sampleRate: -1, }; @@ -384,6 +447,7 @@ export class MpegTsDemuxer extends Demuxer { codec: 'eac3', decoderConfig: null, aacCodecInfo: null, + dtsFormat: null, numberOfChannels: -1, sampleRate: -1, }; @@ -393,6 +457,17 @@ export class MpegTsDemuxer extends Demuxer { codec: 'ac3', decoderConfig: null, aacCodecInfo: null, + dtsFormat: null, + numberOfChannels: -1, + sampleRate: -1, + }; + } else if (hasDtsDescriptor) { + info = { + type: 'audio', + codec: 'dts', + decoderConfig: null, + aacCodecInfo: null, + dtsFormat: null, numberOfChannels: -1, sampleRate: -1, }; @@ -678,6 +753,22 @@ export class MpegTsDemuxer extends Demuxer { elementaryStream.info.numberOfChannels = getEac3ChannelCount(frameInfo); elementaryStream.info.sampleRate = sampleRate; + } else if (elementaryStream.info.codec === 'dts') { + const frameInfo = parseDtsFrame(context.suppliedPacket.data); + if (!frameInfo) { + throw new Error( + 'Invalid DTS audio stream; could not read frame header from first packet.', + ); + } + + elementaryStream.info.numberOfChannels = frameInfo.numberOfChannels; + elementaryStream.info.sampleRate = frameInfo.sampleRate; + + if (frameInfo.core) { + // Telling the two extension-only variants apart would need us to work out + // whether the asset holds XLL or LBR data, so we leave those unlabeled + elementaryStream.info.dtsFormat = frameInfo.hasExtensions ? 'dtsh' : 'dtsc'; + } } else { throw new Error('Unhandled.'); } @@ -687,6 +778,7 @@ export class MpegTsDemuxer extends Demuxer { codec: elementaryStream.info.codec, codecDescription: null, aacCodecInfo: elementaryStream.info.aacCodecInfo, + dtsFormat: elementaryStream.info.dtsFormat, }), numberOfChannels: elementaryStream.info.numberOfChannels, sampleRate: elementaryStream.info.sampleRate, @@ -2315,6 +2407,89 @@ class PacketReadingContext { samplesPerFrame * TIMESCALE / elementaryStream.info.sampleRate, ); return this.supplyPacket(remaining, duration); + } else if (codec === 'dts') { + if (byte !== 0x7f && byte !== 0x64) { + continue; + } + + this.skip(-1); + const possibleSyncPos = this.currentPos; + + let remaining = this.ensureBuffered(DTS_CORE_FRAME_HEADER_SIZE); + if (remaining instanceof Promise) remaining = await remaining; + + if (remaining < DTS_CORE_FRAME_HEADER_SIZE) { + return; + } + + const headerBytes = this.readBytes(DTS_CORE_FRAME_HEADER_SIZE); + const core = parseDtsCoreFrameHeader(headerBytes); + let leadingExss = core ? null : parseDtsExssHeader(headerBytes); + + if (!core && !leadingExss) { + this.seekTo(possibleSyncPos + 1); + continue; + } + + if (leadingExss && !leadingExss.asset) { + // The static fields and the asset descriptor reach past the bytes we read for the + // core header, so take another look with the substream's whole header available + this.seekTo(possibleSyncPos); + + const headerBound = Math.min(leadingExss.frameSize, DTS_EXSS_MAX_HEADER_SIZE); + let remaining = this.ensureBuffered(headerBound); + if (remaining instanceof Promise) remaining = await remaining; + + leadingExss = parseDtsExssHeader(this.readBytes(remaining)) ?? leadingExss; + } + + let frameSize = core ? core.frameSize : leadingExss!.frameSize; + + // Extension-only streams put exactly one substream in each frame, so it's only a core + // frame that needs us to go looking for the extensions riding along with it. The core is + // padded out to a 4-byte boundary before the first of them starts. + if (core) { + let nextSubstreamPos = Math.ceil(core.frameSize / 4) * 4; + + while (true) { + this.seekTo(possibleSyncPos); + + const neededBytes = nextSubstreamPos + DTS_EXSS_HEADER_PREFIX_SIZE; + let remaining = this.ensureBuffered(neededBytes); + if (remaining instanceof Promise) remaining = await remaining; + + if (remaining < neededBytes) { + break; + } + + this.seekTo(possibleSyncPos + nextSubstreamPos); + + const exss = parseDtsExssHeader(this.readBytes(DTS_EXSS_HEADER_PREFIX_SIZE)); + if (!exss) { + break; + } + + nextSubstreamPos += exss.frameSize; + frameSize = nextSubstreamPos; + } + } + + // Only the substream leading the frame declares how many samples the frame holds + const sampleCount = core?.sampleCount ?? leadingExss!.asset?.sampleCount; + if (sampleCount === undefined) { + this.seekTo(possibleSyncPos + 1); + continue; + } + + this.seekTo(possibleSyncPos); + + remaining = this.ensureBuffered(frameSize); + if (remaining instanceof Promise) remaining = await remaining; + + const duration = Math.round( + sampleCount * TIMESCALE / elementaryStream.info.sampleRate, + ); + return this.supplyPacket(remaining, duration); } else { throw new Error('Unhandled.'); } diff --git a/src/mpeg-ts/mpeg-ts-misc.ts b/src/mpeg-ts/mpeg-ts-misc.ts index d19d062..23b2d4d 100644 --- a/src/mpeg-ts/mpeg-ts-misc.ts +++ b/src/mpeg-ts/mpeg-ts-misc.ts @@ -15,6 +15,11 @@ export const enum MpegTsStreamType { AAC = 0x0f, AC3_SYSTEM_A = 0x81, EAC3_SYSTEM_A = 0x87, + DTS = 0x82, + DTS_ATSC = 0x8a, + BLU_RAY_DTS_HD = 0x85, + BLU_RAY_DTS_HD_MASTER = 0x86, + BLU_RAY_DTS_EXPRESS_SECONDARY = 0xa2, PRIVATE_DATA = 0x06, AVC = 0x1b, HEVC = 0x24, diff --git a/src/mpeg-ts/mpeg-ts-muxer.ts b/src/mpeg-ts/mpeg-ts-muxer.ts index d48f99e..8320a7f 100644 --- a/src/mpeg-ts/mpeg-ts-muxer.ts +++ b/src/mpeg-ts/mpeg-ts-muxer.ts @@ -161,7 +161,7 @@ export class MpegTsMuxer extends Muxer { assert(meta?.decoderConfig); const codec = track.source._codec; - assert(codec === 'aac' || codec === 'mp3' || codec === 'ac3' || codec === 'eac3'); + assert(codec === 'aac' || codec === 'mp3' || codec === 'ac3' || codec === 'eac3' || codec === 'dts'); let streamType: MpegTsStreamType; let streamId: number; @@ -186,6 +186,11 @@ export class MpegTsMuxer extends Muxer { streamType = MpegTsStreamType.EAC3_SYSTEM_A; streamId = 0xbd; }; break; + + case 'dts': { + streamType = MpegTsStreamType.DTS; + streamId = 0xbd; + }; break; } const pid = FIRST_TRACK_PID + this.trackDatas.length; @@ -414,7 +419,7 @@ export class MpegTsMuxer extends Muxer { ): Uint8Array { const codec = (trackData.track as OutputAudioTrack).source._codec; - if (codec === 'mp3' || codec === 'ac3' || codec === 'eac3') { + if (codec === 'mp3' || codec === 'ac3' || codec === 'eac3' || codec === 'dts') { // We're good return packet.data; } diff --git a/src/output-format.ts b/src/output-format.ts index 9098282..96186a9 100644 --- a/src/output-format.ts +++ b/src/output-format.ts @@ -1153,7 +1153,7 @@ export class MpegTsOutputFormat extends OutputFormat { getSupportedCodecs(): MediaCodec[] { return [ ...VIDEO_CODECS.filter(codec => ['avc', 'hevc'].includes(codec)), - ...AUDIO_CODECS.filter(codec => ['aac', 'mp3', 'ac3', 'eac3'].includes(codec)), + ...AUDIO_CODECS.filter(codec => ['aac', 'mp3', 'ac3', 'eac3', 'dts'].includes(codec)), ]; } diff --git a/test/browser/mpeg-ts-muxing.test.ts b/test/browser/mpeg-ts-muxing.test.ts index ac841f7..362453e 100644 --- a/test/browser/mpeg-ts-muxing.test.ts +++ b/test/browser/mpeg-ts-muxing.test.ts @@ -16,7 +16,7 @@ test('MPEG-TS output format', async () => { expect(format.mimeType).toBe('video/MP2T'); expect(format.fileExtension).toBe('.ts'); expect(format.supportsVideoRotationMetadata).toBe(false); - expect(format.getSupportedCodecs()).toEqual(['avc', 'hevc', 'aac', 'mp3', 'ac3', 'eac3']); + expect(format.getSupportedCodecs()).toEqual(['avc', 'hevc', 'aac', 'mp3', 'ac3', 'eac3', 'dts']); expect(format.getSupportedTrackCounts()).toEqual({ video: { min: 0, max: 16 }, audio: { min: 0, max: 32 }, diff --git a/test/node/dts.test.ts b/test/node/dts.test.ts new file mode 100644 index 0000000..656e321 --- /dev/null +++ b/test/node/dts.test.ts @@ -0,0 +1,407 @@ +import { expect, test } from 'vitest'; +import path from 'node:path'; +import { Input } from '../../src/input.js'; +import { BufferSource, FilePathSource } from '../../src/source.js'; +import { ALL_FORMATS } from '../../src/input-format.js'; +import { Output } from '../../src/output.js'; +import { MkvOutputFormat, Mp4OutputFormat, MpegTsOutputFormat, OutputFormat } from '../../src/output-format.js'; +import { BufferTarget } from '../../src/target.js'; +import { Conversion } from '../../src/conversion.js'; +import { EncodedPacketSink } from '../../src/media-sink.js'; +import { EncodedPacket } from '../../src/packet.js'; +import { assert, uint8ArraysAreEqual } from '../../src/misc.js'; +import { AudioSampleSink } from '../../src/media-sink.js'; +import { AudioSampleSource } from '../../src/media-source.js'; +import { AudioSample } from '../../src/sample.js'; +import { canEncode, Quality } from '../../src/encode.js'; +import { registerDtsDecoder, registerDtsEncoder } from '@mediabunny/dts'; + +const __dirname = new URL('.', import.meta.url).pathname; + +const DTSC_FILE = 'toothsome-dts.mp4'; +const ESDS_FILE = 'toothsome-dts-esds.mp4'; + +const MP4_TIMESTAMP_TOLERANCE = 0; +const MATROSKA_TIMESTAMP_TOLERANCE = 1 / 1000; +const MPEG_TS_TIMESTAMP_TOLERANCE = 1 / 90_000; + +test('Read from MP4 with a dtsc sample entry', async () => { + using input = new Input({ + source: new FilePathSource(path.join(__dirname, '..', 'public', DTSC_FILE)), + formats: ALL_FORMATS, + }); + + const track = await input.getPrimaryAudioTrack(); + assert(track); + + const decoderConfig = await track.getDecoderConfig(); + assert(decoderConfig); + + expect(await track.getCodec()).toBe('dts'); + expect(decoderConfig.codec).toBe('dtsc'); + expect(decoderConfig.description).toBeUndefined(); + + // This file declares 48000 in its sample entry, which the ddts box then corrects to the real rate + expect(await track.getSampleRate()).toBe(24000); + expect(await track.getNumberOfChannels()).toBe(2); + + await expectStreamShape(input); +}); + +test('Read from MP4 with an esds sample entry', async () => { + using input = new Input({ + source: new FilePathSource(path.join(__dirname, '..', 'public', ESDS_FILE)), + formats: ALL_FORMATS, + }); + + const track = await input.getPrimaryAudioTrack(); + assert(track); + + const decoderConfig = await track.getDecoderConfig(); + assert(decoderConfig); + + expect(await track.getCodec()).toBe('dts'); + expect(decoderConfig.codec).toBe('dtsc'); + expect(decoderConfig.description).toBeUndefined(); + + expect(await track.getSampleRate()).toBe(24000); + expect(await track.getNumberOfChannels()).toBe(2); + + await expectStreamShape(input); +}); + +test('Transmux dtsc MP4 into MP4', async () => { + await expectTransmuxToPreservePackets(DTSC_FILE, new Mp4OutputFormat(), MP4_TIMESTAMP_TOLERANCE); +}); + +test('Transmux dtsc MP4 into Matroska', async () => { + await expectTransmuxToPreservePackets(DTSC_FILE, new MkvOutputFormat(), MATROSKA_TIMESTAMP_TOLERANCE); +}); + +test('Transmux dtsc MP4 into MPEG-TS', async () => { + await expectTransmuxToPreservePackets(DTSC_FILE, new MpegTsOutputFormat(), MPEG_TS_TIMESTAMP_TOLERANCE); +}); + +test('Transmux esds MP4 into MP4', async () => { + await expectTransmuxToPreservePackets(ESDS_FILE, new Mp4OutputFormat(), MP4_TIMESTAMP_TOLERANCE); +}); + +test('Transmux esds MP4 into Matroska', async () => { + await expectTransmuxToPreservePackets(ESDS_FILE, new MkvOutputFormat(), MATROSKA_TIMESTAMP_TOLERANCE); +}); + +test('Transmux esds MP4 into MPEG-TS', async () => { + await expectTransmuxToPreservePackets(ESDS_FILE, new MpegTsOutputFormat(), MPEG_TS_TIMESTAMP_TOLERANCE); +}); + +test('Custom coder registration', async () => { + using input = new Input({ + source: new FilePathSource(path.join(__dirname, '..', 'public', DTSC_FILE)), + formats: ALL_FORMATS, + }); + + const track = await input.getPrimaryAudioTrack(); + assert(track); + + expect(await track.canDecode()).toBe(false); + expect(await canEncode('dts')).toBe(false); + + registerDtsDecoder(); + registerDtsEncoder(); + + expect(await track.canDecode()).toBe(true); + expect(await canEncode('dts')).toBe(true); +}); + +test('Decode with the extension', async () => { + registerDtsDecoder(); + + using input = new Input({ + source: new FilePathSource(path.join(__dirname, '..', 'public', DTSC_FILE)), + formats: ALL_FORMATS, + }); + + const track = await input.getPrimaryAudioTrack(); + assert(track); + + const { packetCount } = await track.computePacketStats(); + const sink = new AudioSampleSink(track); + + let sampleCount = 0; + let nextTimestamp = 0; + + for await (using sample of sink.samples()) { + expect(sample.timestamp).toBeCloseTo(nextTimestamp); + expect(sample.duration).toBeCloseTo(512 / 24000); + expect(sample.format).toBe('f32-planar'); + expect(sample.numberOfChannels).toBe(2); + expect(sample.sampleRate).toBe(24000); + + nextTimestamp += sample.duration; + sampleCount++; + } + + expect(sampleCount).toBe(packetCount); +}); + +test('Encode with the extension', async () => { + registerDtsEncoder(); + + const output = await encodeSineWaveToMp4(0, new Mp4OutputFormat()); + + using input = new Input({ + source: new BufferSource(output), + formats: ALL_FORMATS, + }); + + const track = await input.getPrimaryAudioTrack(); + assert(track); + + expect(await track.getCodec()).toBe('dts'); + expect(await track.getSampleRate()).toBe(ENCODE_SAMPLE_RATE); + expect(await track.getNumberOfChannels()).toBe(ENCODE_CHANNELS); + + const decoderConfig = await track.getDecoderConfig(); + assert(decoderConfig); + expect(decoderConfig.codec).toBe('dtsc'); + expect(decoderConfig.description).toBeUndefined(); + + const sink = new EncodedPacketSink(track); + let packetCount = 0; + + for await (const packet of sink.packets()) { + expect(packet.type).toBe('key'); + packetCount++; + } + + expect(packetCount).toBeGreaterThan(0); + expect(await track.computeDuration()).toBeCloseTo(ENCODE_DURATION, 1); +}); + +test('Round-trip through the extension', async () => { + registerDtsDecoder(); + registerDtsEncoder(); + + const output = await encodeSineWaveToMp4(0, new Mp4OutputFormat()); + + using input = new Input({ + source: new BufferSource(output), + formats: ALL_FORMATS, + }); + + const track = await input.getPrimaryAudioTrack(); + assert(track); + + const sink = new AudioSampleSink(track); + const chunks: Float32Array[] = []; + let decodedFrames = 0; + + for await (using sample of sink.samples()) { + expect(sample.numberOfChannels).toBe(ENCODE_CHANNELS); + expect(sample.sampleRate).toBe(ENCODE_SAMPLE_RATE); + decodedFrames += sample.numberOfFrames; + + const chunk = new Float32Array( + new ArrayBuffer(sample.allocationSize({ format: 'f32-planar', planeIndex: 0 })), + ); + sample.copyTo(chunk, { format: 'f32-planar', planeIndex: 0 }); + chunks.push(chunk); + } + + expect(decodedFrames).toBeGreaterThanOrEqual(ENCODE_SAMPLE_RATE * ENCODE_DURATION); + + const signal = new Float32Array(chunks.reduce((sum, chunk) => sum + chunk.length, 0)); + let offset = 0; + for (const chunk of chunks) { + signal.set(chunk, offset); + offset += chunk.length; + } + + expect(sine440Score(signal, ENCODE_SAMPLE_RATE, 440)).toBeGreaterThan(0.98); +}); + +test('Encode with huge timestamps', async () => { + registerDtsDecoder(); + registerDtsEncoder(); + + const timestamp = 1e9; + const output = await encodeSineWaveToMp4(timestamp, new Mp4OutputFormat({ fastStart: 'fragmented' })); + + using input = new Input({ + source: new BufferSource(output), + formats: ALL_FORMATS, + }); + + const track = await input.getPrimaryAudioTrack(); + assert(track); + + const packetSink = new EncodedPacketSink(track); + const firstPacket = await packetSink.getFirstPacket(); + assert(firstPacket); + + expect(firstPacket.timestamp).toBe(timestamp); + + const sampleSink = new AudioSampleSink(track); + const firstSample = (await sampleSink.samples(timestamp).next()).value; + assert(firstSample); + + expect(firstSample.timestamp).toBe(timestamp); +}); + +const expectStreamShape = async (input: Input) => { + const packets = await readAllPackets(input); + + expect(packets.length).toBe(1805); + expect(packets.every(packet => packet.type === 'key')).toBe(true); + expect(packets.every(packet => packet.data.byteLength === 512)).toBe(true); +}; + +/** Converts the file without re-encoding and checks that every packet comes back out unchanged. */ +const expectTransmuxToPreservePackets = async ( + fileName: string, + outputFormat: OutputFormat, + timestampTolerance: number, +) => { + using originalInput = new Input({ + source: new FilePathSource(path.join(__dirname, '..', 'public', fileName)), + formats: ALL_FORMATS, + }); + + const originalPackets = await readAllPackets(originalInput); + + const output = new Output({ + format: outputFormat, + target: new BufferTarget(), + }); + + const conversion = await Conversion.init({ input: originalInput, output }); + await conversion.execute(); + + const { buffer } = output.target; + assert(buffer); + + using newInput = new Input({ + source: new BufferSource(buffer), + formats: ALL_FORMATS, + }); + + const newTrack = await newInput.getPrimaryAudioTrack(); + assert(newTrack); + + const newDecoderConfig = await newTrack.getDecoderConfig(); + assert(newDecoderConfig); + + expect(await newTrack.getCodec()).toBe('dts'); + expect(await newTrack.getSampleRate()).toBe(24000); + expect(await newTrack.getNumberOfChannels()).toBe(2); + + // Only the MP4 sample entry can carry this, so for the other formats it comes back out of the bitstream + expect(newDecoderConfig.codec).toBe('dtsc'); + + const newPackets = await readAllPackets(newInput); + expectPacketsToMatch(newPackets, originalPackets, timestampTolerance); +}; + +const expectPacketsToMatch = (actual: EncodedPacket[], expected: EncodedPacket[], timestampTolerance: number) => { + expect(actual.length).toBe(expected.length); + + for (let i = 0; i < expected.length; i++) { + const actualPacket = actual[i]!; + const expectedPacket = expected[i]!; + + expect(actualPacket.type).toBe(expectedPacket.type); + expect(uint8ArraysAreEqual(actualPacket.data, expectedPacket.data)).toBe(true); + expect(Math.abs(actualPacket.timestamp - expectedPacket.timestamp)).toBeLessThanOrEqual(timestampTolerance); + } +}; + +const readAllPackets = async (input: Input) => { + const track = await input.getPrimaryAudioTrack(); + assert(track); + + const sink = new EncodedPacketSink(track); + const packets: EncodedPacket[] = []; + + for await (const packet of sink.packets()) { + packets.push(packet); + } + + return packets; +}; + +const ENCODE_SAMPLE_RATE = 48000; +const ENCODE_CHANNELS = 2; +const ENCODE_DURATION = 2; +const ENCODE_BITRATE = 768000; + +const encodeSineWaveToMp4 = async (startTimestamp: number, format: Mp4OutputFormat) => { + const totalFrames = ENCODE_SAMPLE_RATE * ENCODE_DURATION; + const data = new Float32Array(totalFrames * ENCODE_CHANNELS); + + for (let i = 0; i < totalFrames; i++) { + const value = Math.sin(2 * Math.PI * 440 * i / ENCODE_SAMPLE_RATE); + for (let channel = 0; channel < ENCODE_CHANNELS; channel++) { + data[i * ENCODE_CHANNELS + channel] = value; + } + } + + const output = new Output({ format, target: new BufferTarget() }); + const source = new AudioSampleSource({ codec: 'dts', quality: new Quality({ bitrate: ENCODE_BITRATE }) }); + output.addAudioTrack(source); + + await output.start(); + + const sample = new AudioSample({ + data, + format: 'f32', + numberOfChannels: ENCODE_CHANNELS, + sampleRate: ENCODE_SAMPLE_RATE, + timestamp: startTimestamp, + }); + await source.add(sample); + sample.close(); + source.close(); + + await output.finalize(); + + const { buffer } = output.target; + assert(buffer); + + return buffer; +}; + +/** Returns the fraction of the signal's energy explained by a pure sine at `freq`. */ +const sine440Score = (x: Float32Array, sampleRate: number, freq: number) => { + const w = 2 * Math.PI * freq / sampleRate; + + let ss = 0, cc = 0, sc = 0; + let xs = 0, xc = 0; + let xx = 0; + + for (let n = 0; n < x.length; n++) { + const s = Math.sin(w * n); + const c = Math.cos(w * n); + + ss += s * s; + cc += c * c; + sc += s * c; + + xs += x[n]! * s; + xc += x[n]! * c; + xx += x[n]! * x[n]!; + } + + const det = ss * cc - sc * sc; + + const a = (xs * cc - xc * sc) / det; + const b = (xc * ss - xs * sc) / det; + + let fitEnergy = 0; + + for (let n = 0; n < x.length; n++) { + const y = a * Math.sin(w * n) + b * Math.cos(w * n); + fitEnergy += y * y; + } + + return fitEnergy / xx; +}; diff --git a/test/node/server-extension.test.ts b/test/node/server-extension.test.ts index 88033d2..bc28490 100644 --- a/test/node/server-extension.test.ts +++ b/test/node/server-extension.test.ts @@ -1400,6 +1400,9 @@ describe('Video', async () => { }); describe('Audio', async () => { + /** A DTS bitrate that clears the encoder's per-frame minimum at the 48 kHz stereo the tests here use. */ + const DTS_BITRATE = 768000; + test('Decoder lifecycle', async () => { using input = new Input({ source: new FilePathSource('./test/public/trim-buck-bunny-ffmpeg.ts'), @@ -1726,6 +1729,25 @@ describe('Audio', async () => { }); }); + test('DTS encode & decode', async () => { + await encodeDecodeTest('dts', { bitrate: DTS_BITRATE }, async (packet, meta, i) => { + expect(packet.type).toBe('key'); + expect(packet.duration).toBeCloseTo(512 / 48000); + + if (i === 0) { + expect(meta.decoderConfig).toBeDefined(); + expect(meta.decoderConfig!.codec).toBe('dtsc'); + expect(meta.decoderConfig!.numberOfChannels).toBe(2); + expect(meta.decoderConfig!.sampleRate).toBe(48000); + expect(meta.decoderConfig!.description).toBeUndefined(); + } + }, async (sample) => { + expect(sample.numberOfChannels).toBe(2); + expect(sample.sampleRate).toBe(48000); + expect(sample.duration).toBeCloseTo(512 / 48000); + }); + }); + for (const codec of NON_PCM_AUDIO_CODECS) { test(`${codec} encode & decode, negative timestamps`, async () => { await timestampTest(codec, -1); @@ -1747,7 +1769,10 @@ describe('Audio', async () => { const testDuration = codec !== 'opus'; const testSampleStart = codec !== 'vorbis'; - await encodeDecodeTest(codec, {}, async (packet, _meta, i) => { + // The harness' default bitrate is below what DTS needs to fit its per-channel side info into a frame + const extraConfig = codec === 'dts' ? { bitrate: DTS_BITRATE } : {}; + + await encodeDecodeTest(codec, extraConfig, async (packet, _meta, i) => { if (i === 0) { expect(packet.timestamp).toBe(startTimestamp); } else if (testDuration) { diff --git a/test/public/toothsome-dts-esds.mp4 b/test/public/toothsome-dts-esds.mp4 new file mode 100644 index 0000000..ab704ed Binary files /dev/null and b/test/public/toothsome-dts-esds.mp4 differ diff --git a/test/public/toothsome-dts.mp4 b/test/public/toothsome-dts.mp4 new file mode 100644 index 0000000..f7e1ade Binary files /dev/null and b/test/public/toothsome-dts.mp4 differ diff --git a/vite.config.ts b/vite.config.ts index 576ccc9..a9978e0 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -23,6 +23,8 @@ export default defineConfig({ 'mediabunny': path.resolve(__dirname, './dist/bundles/mediabunny.mjs'), '@mediabunny/ac3': path.resolve(__dirname, './packages/ac3/dist/bundles/mediabunny-ac3.mjs'), + '@mediabunny/dts': + path.resolve(__dirname, './packages/dts/dist/bundles/mediabunny-dts.mjs'), '@mediabunny/aac-encoder': path.resolve(__dirname, './packages/aac-encoder/dist/bundles/mediabunny-aac-encoder.mjs'), '@mediabunny/flac-encoder':