arcodange/mediabunny
1
0
Fork 0
mirror of https://github.com/arcodange-org/mediabunny.git synced 2026-09-27 10:53:50 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add support for PCM codecs in MP4, add 64-bit float PCM support

Browse Source
This commit is contained in:
Vanilagy
2025-06-16 10:49:55 +02:00
parent 0ca1425036
commit 05881ff1f5
12 changed files with 254 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dev/convert.html
+3 -2
View File
@@ -21,7 +21,7 @@
chunked: true,
chunkSize: 2**20
});
const outputFormat = new Metamuxer.OggOutputFormat();
const outputFormat = new Metamuxer.MovOutputFormat();
const button = document.createElement('button');
button.textContent = 'Cancel';
@@ -38,6 +38,7 @@
target
}),
audio: {
codec: 'pcm-f64'
//numberOfChannels: 1,
//sampleRate: 4000
//discard: true
@@ -85,7 +86,7 @@
},
trim: {
start: 0,
end: 3
end: 10
},
});
console.log(conversion);
docs/guide/supported-formats-and-codecs.md
+16 -13
View File
@@ -47,6 +47,8 @@ Mediakit ships with built-in decoders and encoders for all audio PCM codecs, mea
- `'pcm-s32be'` - 32-bit big-endian signed PCM
- `'pcm-f32'` - 32-bit little-endian float PCM
- `'pcm-f32be'` - 32-bit big-endian float PCM
- `'pcm-f64'` - 64-bit little-endian float PCM
- `'pcm-f64be'` - 64-bit big-endian float PCM
- `'ulaw'` - μ-law PCM
- `'alaw'` - A-law PCM
@@ -58,7 +60,7 @@ Mediakit ships with built-in decoders and encoders for all audio PCM codecs, mea
Not all codecs can be used with all containers. The following table specifies the supported codec-container combinations:
| | .mp4[^1] | .mov | .mkv | .webm[^2] | .ogg | .mp3 | .wav |
| | .mp4 | .mov | .mkv | .webm[^1] | .ogg | .mp3 | .wav |
|:--------------:|:--------:|:-----:|:-----:|:---------:|:-----:|:-----:|:-----:|
| `'avc'` | ✓ | ✓ | ✓ | | | | |
| `'hevc'` | ✓ | ✓ | ✓ | | | | |
@@ -72,22 +74,23 @@ Not all codecs can be used with all containers. The following table specifies th
| `'flac'` | ✓ | ✓ | ✓ | | | | |
| `'pcm-u8'` | | ✓ | ✓ | | | | ✓ |
| `'pcm-s8'` | | ✓ | | | | | |
| `'pcm-s16'` | | ✓ | ✓ | | | | ✓ |
| `'pcm-s16be'` | | ✓ | ✓ | | | | |
| `'pcm-s24'` | | ✓ | ✓ | | | | ✓ |
| `'pcm-s24be'` | | ✓ | ✓ | | | | |
| `'pcm-s32'` | | ✓ | ✓ | | | | ✓ |
| `'pcm-s32be'` | | ✓ | ✓ | | | | |
| `'pcm-f32'` | | ✓ | ✓ | | | | ✓ |
| `'pcm-f32be'` | | ✓ | | | | | |
| `'pcm-s16'` | ✓ | ✓ | ✓ | | | | ✓ |
| `'pcm-s16be'` | ✓ | ✓ | ✓ | | | | |
| `'pcm-s24'` | ✓ | ✓ | ✓ | | | | ✓ |
| `'pcm-s24be'` | ✓ | ✓ | ✓ | | | | |
| `'pcm-s32'` | ✓ | ✓ | ✓ | | | | ✓ |
| `'pcm-s32be'` | ✓ | ✓ | ✓ | | | | |
| `'pcm-f32'` | ✓ | ✓ | ✓ | | | | ✓ |
| `'pcm-f32be'` | ✓ | ✓ | | | | | |
| `'pcm-f64'` | ✓ | ✓ | ✓ | | | | |
| `'pcm-f64be'` | ✓ | ✓ | | | | | |
| `'ulaw'` | | ✓ | | | | | ✓ |
| `'alaw'` | | ✓ | | | | | ✓ |
| `'webvtt'`[^3] | (✓) | | (✓) | (✓) | | | |
| `'webvtt'`[^2] | (✓) | | (✓) | (✓) | | | |
[^1]: PCM audio codecs are not supported by MP4. However, if PCM audio is included in an MP4 nonetheless, this library would still be able to read it.
[^2]: WebM only supports a small subset of the codecs supported by Matroska. However, this library can technically read all codecs from a WebM that are supported by Matroska.
[^3]: WebVTT can only be written, not read.
[^1]: WebM only supports a small subset of the codecs supported by Matroska. However, this library can technically read all codecs from a WebM that are supported by Matroska.
[^2]: WebVTT can only be written, not read.
## Querying codec encodability
src/codec.ts
+3 -1
View File
@@ -43,6 +43,8 @@ export const PCM_AUDIO_CODECS = [
'pcm-s32be',
'pcm-f32',
'pcm-f32be',
'pcm-f64',
'pcm-f64be',
'pcm-u8',
'pcm-s8',
'ulaw',
@@ -626,7 +628,7 @@ export const parsePcmCodec = (codec: PcmAudioCodec) => {
dataType = 'float';
}
const sampleSize = (Number(match[2]) / 8) as 1 | 2 | 3 | 4;
const sampleSize = (Number(match[2]) / 8) as 1 | 2 | 3 | 4 | 8;
const littleEndian = match[3] !== 'be';
const silentValue = codec === 'pcm-u8' ? 2 ** 7 : 0;
src/isobmff/isobmff-boxes.ts
+94 -37
View File
@@ -274,7 +274,7 @@ export const fullBox = (
* reader understands.
*/
export const ftyp = (details: {
isMov: boolean;
isQuickTime: boolean;
holdsAvc: boolean;
fragmented: boolean;
}) => {
@@ -284,7 +284,7 @@ export const ftyp = (details: {
const minorVersion = 0x200;
if (details.isMov) {
if (details.isQuickTime) {
return box('ftyp', [
ascii('qt '), // Major brand
u32(minorVersion), // Minor version
@@ -558,7 +558,7 @@ export const stsd = (trackData: IsobmffTrackData) => {
trackData,
);
} else if (trackData.type === 'audio') {
const boxName = AUDIO_CODEC_TO_BOX_NAME[trackData.track.source._codec];
const boxName = audioCodecToBoxName(trackData.track.source._codec, trackData.muxer.isQuickTime);
assert(boxName);
sampleDescription = soundSampleDescription(
@@ -731,7 +731,7 @@ export const soundSampleDescription = (
}
return box(compressionType, contents, [
AUDIO_CODEC_TO_CONFIGURATION_BOX[trackData.track.source._codec]?.(trackData) ?? null,
audioCodecToConfigurationBox(trackData.track.source._codec, trackData.muxer.isQuickTime)?.(trackData) ?? null,
]);
};
@@ -801,7 +801,7 @@ export const wave = (trackData: IsobmffAudioTrackData) => {
export const frma = (trackData: IsobmffAudioTrackData) => {
return box('frma', [
ascii(AUDIO_CODEC_TO_BOX_NAME[trackData.track.source._codec]!),
ascii(audioCodecToBoxName(trackData.track.source._codec, trackData.muxer.isQuickTime)),
]);
};
@@ -868,6 +868,17 @@ export const dfLa = (trackData: IsobmffAudioTrackData) => {
]);
};
/** PCM Configuration Box, ISO/IEC 23003-5. */
const pcmC = (trackData: IsobmffAudioTrackData) => {
const { littleEndian, sampleSize } = parsePcmCodec(trackData.track.source._codec as PcmAudioCodec);
const formatFlags = +littleEndian;
return fullBox('pcmC', 0, 0, [
u8(formatFlags),
u8(8 * sampleSize),
]);
};
export const subtitleSampleDescription = (
compressionType: string,
trackData: IsobmffSubtitleTrackData,
@@ -1252,40 +1263,86 @@ const VIDEO_CODEC_TO_CONFIGURATION_BOX: Record<VideoCodec, (trackData: IsobmffVi
av1: av1C,
};
const AUDIO_CODEC_TO_BOX_NAME: Partial<Record<AudioCodec, string>> = {
'aac': 'mp4a',
'mp3': 'mp4a',
'opus': 'Opus',
'vorbis': 'mp4a',
'flac': 'fLaC',
'ulaw': 'ulaw',
'alaw': 'alaw',
'pcm-u8': 'raw ',
'pcm-s8': 'sowt',
'pcm-s16': 'sowt',
'pcm-s16be': 'twos',
'pcm-s24': 'in24',
'pcm-s24be': 'in24',
'pcm-s32': 'in32',
'pcm-s32be': 'in32',
'pcm-f32': 'fl32',
'pcm-f32be': 'fl32',
const audioCodecToBoxName = (codec: AudioCodec, isQuickTime: boolean): string => {
switch (codec) {
case 'aac': return 'mp4a';
case 'mp3': return 'mp4a';
case 'opus': return 'Opus';
case 'vorbis': return 'mp4a';
case 'flac': return 'fLaC';
case 'ulaw': return 'ulaw';
case 'alaw': return 'alaw';
case 'pcm-u8': return 'raw ';
case 'pcm-s8': return 'sowt';
}
// Logic diverges here
if (isQuickTime) {
switch (codec) {
case 'pcm-s16': return 'sowt';
case 'pcm-s16be': return 'twos';
case 'pcm-s24': return 'in24';
case 'pcm-s24be': return 'in24';
case 'pcm-s32': return 'in32';
case 'pcm-s32be': return 'in32';
case 'pcm-f32': return 'fl32';
case 'pcm-f32be': return 'fl32';
case 'pcm-f64': return 'fl64';
case 'pcm-f64be': return 'fl64';
}
} else {
switch (codec) {
case 'pcm-s16': return 'ipcm';
case 'pcm-s16be': return 'ipcm';
case 'pcm-s24': return 'ipcm';
case 'pcm-s24be': return 'ipcm';
case 'pcm-s32': return 'ipcm';
case 'pcm-s32be': return 'ipcm';
case 'pcm-f32': return 'fpcm';
case 'pcm-f32be': return 'fpcm';
case 'pcm-f64': return 'fpcm';
case 'pcm-f64be': return 'fpcm';
}
}
};
const AUDIO_CODEC_TO_CONFIGURATION_BOX: Partial<
Record<AudioCodec, (trackData: IsobmffAudioTrackData) => Box | null>
> = {
'aac': esds,
'mp3': esds,
'opus': dOps,
'vorbis': esds,
'flac': dfLa,
'pcm-s24': wave,
'pcm-s24be': wave,
'pcm-s32': wave,
'pcm-s32be': wave,
'pcm-f32': wave,
'pcm-f32be': wave,
const audioCodecToConfigurationBox = (codec: AudioCodec, isQuickTime: boolean) => {
switch (codec) {
case 'aac': return esds;
case 'mp3': return esds;
case 'opus': return dOps;
case 'vorbis': return esds;
case 'flac': return dfLa;
}
// Logic diverges here
if (isQuickTime) {
switch (codec) {
case 'pcm-s24': return wave;
case 'pcm-s24be': return wave;
case 'pcm-s32': return wave;
case 'pcm-s32be': return wave;
case 'pcm-f32': return wave;
case 'pcm-f32be': return wave;
case 'pcm-f64': return wave;
case 'pcm-f64be': return wave;
}
} else {
switch (codec) {
case 'pcm-s16': return pcmC;
case 'pcm-s16be': return pcmC;
case 'pcm-s24': return pcmC;
case 'pcm-s24be': return pcmC;
case 'pcm-s32': return pcmC;
case 'pcm-s32be': return pcmC;
case 'pcm-f32': return pcmC;
case 'pcm-f32be': return pcmC;
case 'pcm-f64': return pcmC;
case 'pcm-f64be': return pcmC;
}
}
return null;
};
const SUBTITLE_CODEC_TO_BOX_NAME: Record<SubtitleCodec, string> = {
src/isobmff/isobmff-demuxer.ts
+75 -1
View File
@@ -227,7 +227,7 @@ export class IsobmffDemuxer extends Demuxer {
const codecStrings = await Promise.all(this.tracks.map(x => x.inputTrack!.getCodecParameterString()));
return buildIsobmffMimeType({
isQuicktime: this.isQuickTime,
isQuickTime: this.isQuickTime,
hasVideo: this.tracks.some(x => x.info?.type === 'video'),
hasAudio: this.tracks.some(x => x.info?.type === 'audio'),
codecStrings: codecStrings.filter(Boolean) as string[],
@@ -800,6 +800,7 @@ export class IsobmffDemuxer extends Demuxer {
const entries = this.metadataReader.readU32();
for (let i = 0; i < entries; i++) {
const startPos = this.metadataReader.pos;
const sampleBoxInfo = this.metadataReader.readBoxHeader();
const lowercaseBoxName = sampleBoxInfo.name.toLowerCase();
@@ -840,7 +841,10 @@ export class IsobmffDemuxer extends Demuxer {
|| lowercaseBoxName === 'in24'
|| lowercaseBoxName === 'in32'
|| lowercaseBoxName === 'fl32'
|| lowercaseBoxName === 'fl64'
|| lowercaseBoxName === 'lpcm'
|| lowercaseBoxName === 'ipcm' // ISO/IEC 23003-5
|| lowercaseBoxName === 'fpcm' // "
) {
// It's PCM
// developer.apple.com/documentation/quicktime-file-format/sound_sample_descriptions/
@@ -923,6 +927,7 @@ export class IsobmffDemuxer extends Demuxer {
track.info.numberOfChannels = channelCount;
track.info.sampleRate = sampleRate;
// PCM codec assignments
if (lowercaseBoxName === 'twos') {
if (sampleSize === 8) {
track.info.codec = 'pcm-s8';
@@ -947,6 +952,12 @@ export class IsobmffDemuxer extends Demuxer {
track.info.codec = 'pcm-s32be';
} else if (lowercaseBoxName === 'fl32') {
track.info.codec = 'pcm-f32be';
} else if (lowercaseBoxName === 'fl64') {
track.info.codec = 'pcm-f64be';
} else if (lowercaseBoxName === 'ipcm') {
track.info.codec = 'pcm-s16be'; // Placeholder, will be adjusted by the pcmC box
} else if (lowercaseBoxName === 'fpcm') {
track.info.codec = 'pcm-f32be'; // Placeholder, will be adjusted by the pcmC box
}
this.readContiguousBoxes(startPos + sampleBoxInfo.totalSize - this.metadataReader.pos);
@@ -1147,10 +1158,73 @@ export class IsobmffDemuxer extends Demuxer {
track.info.codec = 'pcm-s32';
} else if (track.info.codec === 'pcm-f32be') {
track.info.codec = 'pcm-f32';
} else if (track.info.codec === 'pcm-f64be') {
track.info.codec = 'pcm-f64';
}
}
}; break;
case 'pcmC': {
const track = this.currentTrack;
assert(track && track.info?.type === 'audio');
this.metadataReader.pos += 1 + 3; // Version + flags
// ISO/IEC 23003-5
const formatFlags = this.metadataReader.readU8();
const isLittleEndian = Boolean(formatFlags & 0x01);
const pcmSampleSize = this.metadataReader.readU8();
if (track.info.codec === 'pcm-s16be') {
// ipcm
if (isLittleEndian) {
if (pcmSampleSize === 16) {
track.info.codec = 'pcm-s16';
} else if (pcmSampleSize === 24) {
track.info.codec = 'pcm-s24';
} else if (pcmSampleSize === 32) {
track.info.codec = 'pcm-s32';
} else {
throw new Error(`Invalid ipcm sample size ${pcmSampleSize}.`);
}
} else {
if (pcmSampleSize === 16) {
track.info.codec = 'pcm-s16be';
} else if (pcmSampleSize === 24) {
track.info.codec = 'pcm-s24be';
} else if (pcmSampleSize === 32) {
track.info.codec = 'pcm-s32be';
} else {
throw new Error(`Invalid ipcm sample size ${pcmSampleSize}.`);
}
}
} else if (track.info.codec === 'pcm-f32be') {
// fpcm
if (isLittleEndian) {
if (pcmSampleSize === 32) {
track.info.codec = 'pcm-f32';
} else if (pcmSampleSize === 64) {
track.info.codec = 'pcm-f64';
} else {
throw new Error(`Invalid fpcm sample size ${pcmSampleSize}.`);
}
} else {
if (pcmSampleSize === 32) {
track.info.codec = 'pcm-f32be';
} else if (pcmSampleSize === 64) {
track.info.codec = 'pcm-f64be';
} else {
throw new Error(`Invalid fpcm sample size ${pcmSampleSize}.`);
}
}
}
break;
};
case 'dOps': { // Used for Opus audio
const track = this.currentTrack;
assert(track && track.info?.type === 'audio');
src/isobmff/isobmff-misc.ts
+2 -2
View File
@@ -1,5 +1,5 @@
export const buildIsobmffMimeType = (info: {
isQuicktime: boolean;
isQuickTime: boolean;
hasVideo: boolean;
hasAudio: boolean;
codecStrings: string[];
@@ -10,7 +10,7 @@ export const buildIsobmffMimeType = (info: {
? 'audio/'
: 'application/';
let string = base + (info.isQuicktime ? 'quicktime' : 'mp4');
let string = base + (info.isQuickTime ? 'quicktime' : 'mp4');
if (info.codecStrings.length > 0) {
const uniqueCodecMimeTypes = [...new Set(info.codecStrings)];
src/isobmff/isobmff-muxer.ts
+9 -4
View File
@@ -48,6 +48,7 @@ type Chunk = {
};
export type IsobmffTrackData = {
muxer: IsobmffMuxer;
timescale: number;
samples: Sample[];
sampleQueue: Sample[]; // For fragmented files
@@ -116,10 +117,11 @@ export class IsobmffMuxer extends Muxer {
private format: IsobmffOutputFormat;
private writer: Writer;
private boxWriter: IsobmffBoxWriter;
private isMov: boolean;
private fastStart: NonNullable<IsobmffOutputFormatOptions['fastStart']>;
private isFragmented: boolean;
isQuickTime: boolean;
private auxTarget = new BufferTarget();
private auxWriter = this.auxTarget._createWriter();
private auxBoxWriter = new IsobmffBoxWriter(this.auxWriter);
@@ -144,7 +146,7 @@ export class IsobmffMuxer extends Muxer {
this.writer = output._writer;
this.boxWriter = new IsobmffBoxWriter(this.writer);
this.isMov = format instanceof MovOutputFormat;
this.isQuickTime = format instanceof MovOutputFormat;
// If the fastStart option isn't defined, enable in-memory fast start if the target is an ArrayBuffer, as the
// memory usage remains identical
@@ -171,7 +173,7 @@ export class IsobmffMuxer extends Muxer {
}
this.boxWriter.writeBox(ftyp({
isMov: this.isMov,
isQuickTime: this.isQuickTime,
holdsAvc: holdsAvc,
fragmented: this.isFragmented,
}));
@@ -227,7 +229,7 @@ export class IsobmffMuxer extends Muxer {
});
return buildIsobmffMimeType({
isQuicktime: this.isMov,
isQuickTime: this.isQuickTime,
hasVideo: this.trackDatas.some(x => x.type === 'video'),
hasAudio: this.trackDatas.some(x => x.type === 'audio'),
codecStrings,
@@ -291,6 +293,7 @@ export class IsobmffMuxer extends Muxer {
const timescale = computeRationalApproximation(1 / (track.metadata.frameRate ?? 57600), 1e6).denominator;
const newTrackData: IsobmffVideoTrackData = {
muxer: this,
track,
type: 'video',
info: {
@@ -334,6 +337,7 @@ export class IsobmffMuxer extends Muxer {
assert(meta.decoderConfig);
const newTrackData: IsobmffAudioTrackData = {
muxer: this,
track,
type: 'audio',
info: {
@@ -379,6 +383,7 @@ export class IsobmffMuxer extends Muxer {
assert(meta.config);
const newTrackData: IsobmffSubtitleTrackData = {
muxer: this,
track,
type: 'subtitle',
info: {
src/matroska/ebml.ts
+1
View File
@@ -575,6 +575,7 @@ export const CODEC_STRING_MAP: Partial<Record<MediaCodec, string>> = {
'pcm-s32': 'A_PCM/INT/LIT',
'pcm-s32be': 'A_PCM/INT/BIG',
'pcm-f32': 'A_PCM/FLOAT/IEEE',
'pcm-f64': 'A_PCM/FLOAT/IEEE',
'webvtt': 'S_TEXT/WEBVTT',
};
src/matroska/matroska-demuxer.ts
+2
View File
@@ -707,6 +707,8 @@ export class MatroskaDemuxer extends Demuxer {
} else if (this.currentTrack.codecId === 'A_PCM/FLOAT/IEEE') {
if (this.currentTrack.info.bitDepth === 32) {
this.currentTrack.info.codec = 'pcm-f32';
} else if (this.currentTrack.info.bitDepth === 64) {
this.currentTrack.info.codec = 'pcm-f64';
}
}
src/media-sink.ts
+23 -1
View File
@@ -4,6 +4,7 @@ import { InputAudioTrack, InputTrack, InputVideoTrack } from './input-track';
import {
AnyIterable,
assert,
assertNever,
binarySearchLessOrEqual,
CallSerializer,
getInt24,
@@ -1222,7 +1223,7 @@ class AudioDecoderWrapper extends DecoderWrapper<AudioSample> {
class PcmAudioDecoderWrapper extends DecoderWrapper<AudioSample> {
codec: PcmAudioCodec;
inputSampleSize: 1 | 2 | 3 | 4;
inputSampleSize: 1 | 2 | 3 | 4 | 8;
readInputValue: (view: DataView, byteOffset: number) => number;
outputSampleSize: 1 | 2 | 4;
@@ -1289,6 +1290,17 @@ class PcmAudioDecoderWrapper extends DecoderWrapper<AudioSample> {
assert(false);
}
}; break;
case 8: {
if (dataType === 'float') {
this.readInputValue = (view, byteOffset) => view.getFloat64(byteOffset, littleEndian);
} else {
assert(false);
}
}; break;
default: {
assertNever(sampleSize);
assert(false);
};
}
switch (sampleSize) {
@@ -1327,6 +1339,16 @@ class PcmAudioDecoderWrapper extends DecoderWrapper<AudioSample> {
this.writeOutputValue = (view, byteOffset, value) => view.setInt32(byteOffset, value, true);
}
}; break;
case 8: {
this.outputSampleSize = 4;
this.outputFormat = 'f32';
this.writeOutputValue = (view, byteOffset, value) => view.setFloat32(byteOffset, value, true);
}; break;
default: {
assertNever(sampleSize);
assert(false);
};
};
}
src/media-source.ts
+13 -1
View File
@@ -16,7 +16,7 @@ import {
VideoCodec,
} from './codec';
import { OutputAudioTrack, OutputSubtitleTrack, OutputTrack, OutputVideoTrack } from './output';
import { assert, CallSerializer, clamp, setInt24, setUint24 } from './misc';
import { assert, assertNever, CallSerializer, clamp, setInt24, setUint24 } from './misc';
import { Muxer } from './muxer';
import { SubtitleParser } from './subtitles';
import { toAlaw, toUlaw } from './pcm';
@@ -1083,7 +1083,19 @@ class AudioEncoderWrapper {
} else {
assert(false);
}
}; break;
case 8: {
if (dataType === 'float') {
this.writeOutputValue = (view, byteOffset, value) =>
view.setFloat64(byteOffset, value, littleEndian);
} else {
assert(false);
}
}; break;
default: {
assertNever(sampleSize);
assert(false);
};
}
}
src/output-format.ts
+12 -1
View File
@@ -237,6 +237,17 @@ export class Mp4OutputFormat extends IsobmffOutputFormat {
return [
...VIDEO_CODECS,
...NON_PCM_AUDIO_CODECS,
// These are supported via ISO/IEC 23003-5
'pcm-s16',
'pcm-s16be',
'pcm-s24',
'pcm-s24be',
'pcm-s32',
'pcm-s32be',
'pcm-f32',
'pcm-f32be',
'pcm-f64',
'pcm-f64be',
...SUBTITLE_CODECS,
];
}
@@ -398,7 +409,7 @@ export class MkvOutputFormat extends OutputFormat {
return [
...VIDEO_CODECS,
...NON_PCM_AUDIO_CODECS,
...PCM_AUDIO_CODECS.filter(codec => !['pcm-s8', 'pcm-f32be', 'ulaw', 'alaw'].includes(codec)),
...PCM_AUDIO_CODECS.filter(codec => !['pcm-s8', 'pcm-f32be', 'pcm-f64be', 'ulaw', 'alaw'].includes(codec)),
...SUBTITLE_CODECS,
];
}