arcodange/mediabunny
1
0
Fork 0
mirror of https://github.com/arcodange-org/mediabunny.git synced 2026-10-02 05:13:50 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add support for PCM codecs in MP4, add 64-bit float PCM support

Browse Source
This commit is contained in:
Vanilagy
2025-06-16 10:49:55 +02:00
parent 0ca1425036
commit 05881ff1f5
12 changed files with 254 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/isobmff/isobmff-boxes.ts
+94 -37
View File
@@ -274,7 +274,7 @@ export const fullBox = (
* reader understands.
*/
export const ftyp = (details: {
isMov: boolean;
isQuickTime: boolean;
holdsAvc: boolean;
fragmented: boolean;
}) => {
@@ -284,7 +284,7 @@ export const ftyp = (details: {
const minorVersion = 0x200;
if (details.isMov) {
if (details.isQuickTime) {
return box('ftyp', [
ascii('qt '), // Major brand
u32(minorVersion), // Minor version
@@ -558,7 +558,7 @@ export const stsd = (trackData: IsobmffTrackData) => {
trackData,
);
} else if (trackData.type === 'audio') {
const boxName = AUDIO_CODEC_TO_BOX_NAME[trackData.track.source._codec];
const boxName = audioCodecToBoxName(trackData.track.source._codec, trackData.muxer.isQuickTime);
assert(boxName);
sampleDescription = soundSampleDescription(
@@ -731,7 +731,7 @@ export const soundSampleDescription = (
}
return box(compressionType, contents, [
AUDIO_CODEC_TO_CONFIGURATION_BOX[trackData.track.source._codec]?.(trackData) ?? null,
audioCodecToConfigurationBox(trackData.track.source._codec, trackData.muxer.isQuickTime)?.(trackData) ?? null,
]);
};
@@ -801,7 +801,7 @@ export const wave = (trackData: IsobmffAudioTrackData) => {
export const frma = (trackData: IsobmffAudioTrackData) => {
return box('frma', [
ascii(AUDIO_CODEC_TO_BOX_NAME[trackData.track.source._codec]!),
ascii(audioCodecToBoxName(trackData.track.source._codec, trackData.muxer.isQuickTime)),
]);
};
@@ -868,6 +868,17 @@ export const dfLa = (trackData: IsobmffAudioTrackData) => {
]);
};
/** PCM Configuration Box, ISO/IEC 23003-5. */
const pcmC = (trackData: IsobmffAudioTrackData) => {
const { littleEndian, sampleSize } = parsePcmCodec(trackData.track.source._codec as PcmAudioCodec);
const formatFlags = +littleEndian;
return fullBox('pcmC', 0, 0, [
u8(formatFlags),
u8(8 * sampleSize),
]);
};
export const subtitleSampleDescription = (
compressionType: string,
trackData: IsobmffSubtitleTrackData,
@@ -1252,40 +1263,86 @@ const VIDEO_CODEC_TO_CONFIGURATION_BOX: Record<VideoCodec, (trackData: IsobmffVi
av1: av1C,
};
const AUDIO_CODEC_TO_BOX_NAME: Partial<Record<AudioCodec, string>> = {
'aac': 'mp4a',
'mp3': 'mp4a',
'opus': 'Opus',
'vorbis': 'mp4a',
'flac': 'fLaC',
'ulaw': 'ulaw',
'alaw': 'alaw',
'pcm-u8': 'raw ',
'pcm-s8': 'sowt',
'pcm-s16': 'sowt',
'pcm-s16be': 'twos',
'pcm-s24': 'in24',
'pcm-s24be': 'in24',
'pcm-s32': 'in32',
'pcm-s32be': 'in32',
'pcm-f32': 'fl32',
'pcm-f32be': 'fl32',
const audioCodecToBoxName = (codec: AudioCodec, isQuickTime: boolean): string => {
switch (codec) {
case 'aac': return 'mp4a';
case 'mp3': return 'mp4a';
case 'opus': return 'Opus';
case 'vorbis': return 'mp4a';
case 'flac': return 'fLaC';
case 'ulaw': return 'ulaw';
case 'alaw': return 'alaw';
case 'pcm-u8': return 'raw ';
case 'pcm-s8': return 'sowt';
}
// Logic diverges here
if (isQuickTime) {
switch (codec) {
case 'pcm-s16': return 'sowt';
case 'pcm-s16be': return 'twos';
case 'pcm-s24': return 'in24';
case 'pcm-s24be': return 'in24';
case 'pcm-s32': return 'in32';
case 'pcm-s32be': return 'in32';
case 'pcm-f32': return 'fl32';
case 'pcm-f32be': return 'fl32';
case 'pcm-f64': return 'fl64';
case 'pcm-f64be': return 'fl64';
}
} else {
switch (codec) {
case 'pcm-s16': return 'ipcm';
case 'pcm-s16be': return 'ipcm';
case 'pcm-s24': return 'ipcm';
case 'pcm-s24be': return 'ipcm';
case 'pcm-s32': return 'ipcm';
case 'pcm-s32be': return 'ipcm';
case 'pcm-f32': return 'fpcm';
case 'pcm-f32be': return 'fpcm';
case 'pcm-f64': return 'fpcm';
case 'pcm-f64be': return 'fpcm';
}
}
};
const AUDIO_CODEC_TO_CONFIGURATION_BOX: Partial<
Record<AudioCodec, (trackData: IsobmffAudioTrackData) => Box | null>
> = {
'aac': esds,
'mp3': esds,
'opus': dOps,
'vorbis': esds,
'flac': dfLa,
'pcm-s24': wave,
'pcm-s24be': wave,
'pcm-s32': wave,
'pcm-s32be': wave,
'pcm-f32': wave,
'pcm-f32be': wave,
const audioCodecToConfigurationBox = (codec: AudioCodec, isQuickTime: boolean) => {
switch (codec) {
case 'aac': return esds;
case 'mp3': return esds;
case 'opus': return dOps;
case 'vorbis': return esds;
case 'flac': return dfLa;
}
// Logic diverges here
if (isQuickTime) {
switch (codec) {
case 'pcm-s24': return wave;
case 'pcm-s24be': return wave;
case 'pcm-s32': return wave;
case 'pcm-s32be': return wave;
case 'pcm-f32': return wave;
case 'pcm-f32be': return wave;
case 'pcm-f64': return wave;
case 'pcm-f64be': return wave;
}
} else {
switch (codec) {
case 'pcm-s16': return pcmC;
case 'pcm-s16be': return pcmC;
case 'pcm-s24': return pcmC;
case 'pcm-s24be': return pcmC;
case 'pcm-s32': return pcmC;
case 'pcm-s32be': return pcmC;
case 'pcm-f32': return pcmC;
case 'pcm-f32be': return pcmC;
case 'pcm-f64': return pcmC;
case 'pcm-f64be': return pcmC;
}
}
return null;
};
const SUBTITLE_CODEC_TO_BOX_NAME: Record<SubtitleCodec, string> = {
src/isobmff/isobmff-demuxer.ts
+75 -1
View File
@@ -227,7 +227,7 @@ export class IsobmffDemuxer extends Demuxer {
const codecStrings = await Promise.all(this.tracks.map(x => x.inputTrack!.getCodecParameterString()));
return buildIsobmffMimeType({
isQuicktime: this.isQuickTime,
isQuickTime: this.isQuickTime,
hasVideo: this.tracks.some(x => x.info?.type === 'video'),
hasAudio: this.tracks.some(x => x.info?.type === 'audio'),
codecStrings: codecStrings.filter(Boolean) as string[],
@@ -800,6 +800,7 @@ export class IsobmffDemuxer extends Demuxer {
const entries = this.metadataReader.readU32();
for (let i = 0; i < entries; i++) {
const startPos = this.metadataReader.pos;
const sampleBoxInfo = this.metadataReader.readBoxHeader();
const lowercaseBoxName = sampleBoxInfo.name.toLowerCase();
@@ -840,7 +841,10 @@ export class IsobmffDemuxer extends Demuxer {
|| lowercaseBoxName === 'in24'
|| lowercaseBoxName === 'in32'
|| lowercaseBoxName === 'fl32'
|| lowercaseBoxName === 'fl64'
|| lowercaseBoxName === 'lpcm'
|| lowercaseBoxName === 'ipcm' // ISO/IEC 23003-5
|| lowercaseBoxName === 'fpcm' // "
) {
// It's PCM
// developer.apple.com/documentation/quicktime-file-format/sound_sample_descriptions/
@@ -923,6 +927,7 @@ export class IsobmffDemuxer extends Demuxer {
track.info.numberOfChannels = channelCount;
track.info.sampleRate = sampleRate;
// PCM codec assignments
if (lowercaseBoxName === 'twos') {
if (sampleSize === 8) {
track.info.codec = 'pcm-s8';
@@ -947,6 +952,12 @@ export class IsobmffDemuxer extends Demuxer {
track.info.codec = 'pcm-s32be';
} else if (lowercaseBoxName === 'fl32') {
track.info.codec = 'pcm-f32be';
} else if (lowercaseBoxName === 'fl64') {
track.info.codec = 'pcm-f64be';
} else if (lowercaseBoxName === 'ipcm') {
track.info.codec = 'pcm-s16be'; // Placeholder, will be adjusted by the pcmC box
} else if (lowercaseBoxName === 'fpcm') {
track.info.codec = 'pcm-f32be'; // Placeholder, will be adjusted by the pcmC box
}
this.readContiguousBoxes(startPos + sampleBoxInfo.totalSize - this.metadataReader.pos);
@@ -1147,10 +1158,73 @@ export class IsobmffDemuxer extends Demuxer {
track.info.codec = 'pcm-s32';
} else if (track.info.codec === 'pcm-f32be') {
track.info.codec = 'pcm-f32';
} else if (track.info.codec === 'pcm-f64be') {
track.info.codec = 'pcm-f64';
}
}
}; break;
case 'pcmC': {
const track = this.currentTrack;
assert(track && track.info?.type === 'audio');
this.metadataReader.pos += 1 + 3; // Version + flags
// ISO/IEC 23003-5
const formatFlags = this.metadataReader.readU8();
const isLittleEndian = Boolean(formatFlags & 0x01);
const pcmSampleSize = this.metadataReader.readU8();
if (track.info.codec === 'pcm-s16be') {
// ipcm
if (isLittleEndian) {
if (pcmSampleSize === 16) {
track.info.codec = 'pcm-s16';
} else if (pcmSampleSize === 24) {
track.info.codec = 'pcm-s24';
} else if (pcmSampleSize === 32) {
track.info.codec = 'pcm-s32';
} else {
throw new Error(`Invalid ipcm sample size ${pcmSampleSize}.`);
}
} else {
if (pcmSampleSize === 16) {
track.info.codec = 'pcm-s16be';
} else if (pcmSampleSize === 24) {
track.info.codec = 'pcm-s24be';
} else if (pcmSampleSize === 32) {
track.info.codec = 'pcm-s32be';
} else {
throw new Error(`Invalid ipcm sample size ${pcmSampleSize}.`);
}
}
} else if (track.info.codec === 'pcm-f32be') {
// fpcm
if (isLittleEndian) {
if (pcmSampleSize === 32) {
track.info.codec = 'pcm-f32';
} else if (pcmSampleSize === 64) {
track.info.codec = 'pcm-f64';
} else {
throw new Error(`Invalid fpcm sample size ${pcmSampleSize}.`);
}
} else {
if (pcmSampleSize === 32) {
track.info.codec = 'pcm-f32be';
} else if (pcmSampleSize === 64) {
track.info.codec = 'pcm-f64be';
} else {
throw new Error(`Invalid fpcm sample size ${pcmSampleSize}.`);
}
}
}
break;
};
case 'dOps': { // Used for Opus audio
const track = this.currentTrack;
assert(track && track.info?.type === 'audio');
src/isobmff/isobmff-misc.ts
+2 -2
View File
@@ -1,5 +1,5 @@
export const buildIsobmffMimeType = (info: {
isQuicktime: boolean;
isQuickTime: boolean;
hasVideo: boolean;
hasAudio: boolean;
codecStrings: string[];
@@ -10,7 +10,7 @@ export const buildIsobmffMimeType = (info: {
? 'audio/'
: 'application/';
let string = base + (info.isQuicktime ? 'quicktime' : 'mp4');
let string = base + (info.isQuickTime ? 'quicktime' : 'mp4');
if (info.codecStrings.length > 0) {
const uniqueCodecMimeTypes = [...new Set(info.codecStrings)];
src/isobmff/isobmff-muxer.ts
+9 -4
View File
@@ -48,6 +48,7 @@ type Chunk = {
};
export type IsobmffTrackData = {
muxer: IsobmffMuxer;
timescale: number;
samples: Sample[];
sampleQueue: Sample[]; // For fragmented files
@@ -116,10 +117,11 @@ export class IsobmffMuxer extends Muxer {
private format: IsobmffOutputFormat;
private writer: Writer;
private boxWriter: IsobmffBoxWriter;
private isMov: boolean;
private fastStart: NonNullable<IsobmffOutputFormatOptions['fastStart']>;
private isFragmented: boolean;
isQuickTime: boolean;
private auxTarget = new BufferTarget();
private auxWriter = this.auxTarget._createWriter();
private auxBoxWriter = new IsobmffBoxWriter(this.auxWriter);
@@ -144,7 +146,7 @@ export class IsobmffMuxer extends Muxer {
this.writer = output._writer;
this.boxWriter = new IsobmffBoxWriter(this.writer);
this.isMov = format instanceof MovOutputFormat;
this.isQuickTime = format instanceof MovOutputFormat;
// If the fastStart option isn't defined, enable in-memory fast start if the target is an ArrayBuffer, as the
// memory usage remains identical
@@ -171,7 +173,7 @@ export class IsobmffMuxer extends Muxer {
}
this.boxWriter.writeBox(ftyp({
isMov: this.isMov,
isQuickTime: this.isQuickTime,
holdsAvc: holdsAvc,
fragmented: this.isFragmented,
}));
@@ -227,7 +229,7 @@ export class IsobmffMuxer extends Muxer {
});
return buildIsobmffMimeType({
isQuicktime: this.isMov,
isQuickTime: this.isQuickTime,
hasVideo: this.trackDatas.some(x => x.type === 'video'),
hasAudio: this.trackDatas.some(x => x.type === 'audio'),
codecStrings,
@@ -291,6 +293,7 @@ export class IsobmffMuxer extends Muxer {
const timescale = computeRationalApproximation(1 / (track.metadata.frameRate ?? 57600), 1e6).denominator;
const newTrackData: IsobmffVideoTrackData = {
muxer: this,
track,
type: 'video',
info: {
@@ -334,6 +337,7 @@ export class IsobmffMuxer extends Muxer {
assert(meta.decoderConfig);
const newTrackData: IsobmffAudioTrackData = {
muxer: this,
track,
type: 'audio',
info: {
@@ -379,6 +383,7 @@ export class IsobmffMuxer extends Muxer {
assert(meta.config);
const newTrackData: IsobmffSubtitleTrackData = {
muxer: this,
track,
type: 'subtitle',
info: {