Add support for all audio codecs in the ISOBMFF muxer (& some fixes)

This commit is contained in:
Vanilagy
2025-01-05 13:50:50 +01:00
parent 316e16211b
commit ee81e1b674
10 changed files with 560 additions and 155 deletions
+198 -48
View File
@@ -12,7 +12,7 @@ import {
IDENTITY_MATRIX,
rotationMatrix,
} from '../misc';
import { AudioCodec, SubtitleCodec, VideoCodec } from '../codec';
import { AudioCodec, parsePcmCodec, PCM_CODECS, PcmAudioCodec, SubtitleCodec, VideoCodec } from '../codec';
import { formatSubtitleTimestamp } from '../subtitles';
import { Writer } from '../writer';
import {
@@ -258,6 +258,7 @@ export const fullBox = (
* reader understands.
*/
export const ftyp = (details: {
isMov: boolean;
holdsAvc: boolean;
fragmented: boolean;
}) => {
@@ -267,6 +268,15 @@ export const ftyp = (details: {
const minorVersion = 0x200;
if (details.isMov) {
return box('ftyp', [
ascii('qt '), // Major brand
u32(minorVersion), // Minor version
// Compatible brands
ascii('qt '),
]);
}
if (details.fragmented) return box('ftyp', [
ascii('iso5'), // Major brand
u32(minorVersion), // Minor version
@@ -688,45 +698,101 @@ export const av1C = (trackData: IsobmffVideoTrackData) => {
export const soundSampleDescription = (
compressionType: string,
trackData: IsobmffAudioTrackData,
) => box(compressionType, [
Array(6).fill(0), // Reserved
u16(1), // Data reference index
u16(0), // Version (AudioSampleEntry, not AudioSampleEntryV1)
u16(0), // Revision level
u32(0), // Vendor
u16(trackData.info.numberOfChannels), // Number of channels
u16(16), // Sample size (bits)
u16(0), // Compression ID
u16(0), // Packet size
u32(2 ** 16 * trackData.info.sampleRate), // Sample rate
], [
AUDIO_CODEC_TO_CONFIGURATION_BOX[trackData.track.source._codec]!(trackData),
]);
) => {
let version = 0;
let contents: NestedNumberArray;
let sampleSizeInBits = 16;
if ((PCM_CODECS as readonly AudioCodec[]).includes(trackData.track.source._codec)) {
const codec = trackData.track.source._codec as PcmAudioCodec;
const { sampleSize } = parsePcmCodec(codec);
sampleSizeInBits = 8 * sampleSize;
if (sampleSizeInBits > 16) {
version = 1;
}
}
if (version === 0) {
contents = [
Array(6).fill(0), // Reserved
u16(1), // Data reference index
u16(version), // Version
u16(0), // Revision level
u32(0), // Vendor
u16(trackData.info.numberOfChannels), // Number of channels
u16(sampleSizeInBits), // Sample size (bits)
u16(0), // Compression ID
u16(0), // Packet size
u16(trackData.info.sampleRate < 2 ** 16 ? trackData.info.sampleRate : 0), // Sample rate (upper)
u16(0), // Sample rate (lower)
];
} else {
contents = [
Array(6).fill(0), // Reserved
u16(1), // Data reference index
u16(version), // Version
u16(0), // Revision level
u32(0), // Vendor
u16(trackData.info.numberOfChannels), // Number of channels
u16(Math.min(sampleSizeInBits, 16)), // Sample size (bits)
u16(0), // Compression ID
u16(0), // Packet size
u16(trackData.info.sampleRate < 2 ** 16 ? trackData.info.sampleRate : 0), // Sample rate (upper)
u16(0), // Sample rate (lower)
u32(1), // Samples per packet (must be 1 for uncompressed formats)
u32(sampleSizeInBits / 8), // Bytes per packet
u32(trackData.info.numberOfChannels * sampleSizeInBits / 8), // Bytes per frame
u32(2), // Bytes per sample (constant in FFmpeg)
];
}
return box(compressionType, contents, [
AUDIO_CODEC_TO_CONFIGURATION_BOX[trackData.track.source._codec]?.(trackData) ?? null,
]);
};
/** MPEG-4 Elementary Stream Descriptor Box. */
export const esds = (trackData: IsobmffAudioTrackData) => {
const description = toUint8Array(trackData.info.decoderConfig.description ?? new ArrayBuffer(0));
// Adapted from https://stackoverflow.com/a/54803118
// We build up the bytes in a layered way which reflects the nested structure
let objectTypeIndication: number;
switch (trackData.track.source._codec) {
case 'aac': {
objectTypeIndication = 0x40;
}; break;
case 'mp3': {
objectTypeIndication = 0x6b;
}; break;
case 'vorbis': {
objectTypeIndication = 0xdd;
}; break;
default: throw new Error(`Unhandled audio codec: ${trackData.track.source._codec}`);
}
let bytes = [
...description,
];
bytes = [
...u8(0x40), // MPEG-4 Audio
...u8(objectTypeIndication), // Object type indication
...u8(0x15), // stream type(6bits)=5 audio, flags(2bits)=1
...u24(0), // 24bit buffer size
...u32(0), // max bitrate
...u32(0), // avg bitrate
...u8(0x05), // TAG(5) = ASC ([2],[3]) embedded in above OD
...variableUnsignedInt(bytes.length),
...bytes,
];
if (trackData.info.decoderConfig.description) {
const description = toUint8Array(trackData.info.decoderConfig.description);
// Add the decoder description to the end
bytes = [
...bytes,
...u8(0x05), // TAG(5) = DecoderSpecificInfo
...variableUnsignedInt(description.byteLength),
...description,
];
}
bytes = [
...u16(1), // ES_ID = 1
...u8(0x00), // flags etc = 0
...u8(0x04), // TAG(4) = ES Descriptor ([2]) embedded in above OD
...u8(0x04), // TAG(4) = ES Descriptor
...variableUnsignedInt(bytes.length),
...bytes,
...u8(0x06), // TAG(6)
@@ -734,7 +800,7 @@ export const esds = (trackData: IsobmffAudioTrackData) => {
...u8(0x02), // data
];
bytes = [
...u8(0x03), // TAG(3) = Object Descriptor ([2])
...u8(0x03), // TAG(3) = Object Descriptor
...variableUnsignedInt(bytes.length),
...bytes,
];
@@ -742,35 +808,82 @@ export const esds = (trackData: IsobmffAudioTrackData) => {
return fullBox('esds', 0, 0, bytes);
};
export const wave = (trackData: IsobmffAudioTrackData) => {
return box('wave', undefined, [
frma(trackData),
enda(trackData),
box('\x00\x00\x00\x00'), // NULL tag at the end
]);
};
export const frma = (trackData: IsobmffAudioTrackData) => {
return box('frma', [
ascii(AUDIO_CODEC_TO_BOX_NAME[trackData.track.source._codec]!),
]);
};
// This box specifies PCM endianness
export const enda = (trackData: IsobmffAudioTrackData) => {
const { littleEndian } = parsePcmCodec(trackData.track.source._codec as PcmAudioCodec);
return box('enda', [
u16(+littleEndian),
]);
};
/** Opus Specific Box. */
export const dOps = (trackData: IsobmffAudioTrackData) => {
let outputChannelCount = trackData.info.numberOfChannels;
// Default PreSkip, should be at least 80 milliseconds worth of playback, measured in 48000 Hz samples
let preskip = 3840;
let gain = 0;
let preSkip = 3840;
let inputSampleRate = trackData.info.sampleRate;
let outputGain = 0;
let channelMappingFamily = 0;
let channelMappingTable = new Uint8Array(0);
// Read preskip and from codec private data from the encoder
// https://www.rfc-editor.org/rfc/rfc7845#section-5
const description = trackData.info.decoderConfig?.description;
if (description) {
if (description.byteLength < 18) {
throw new Error('Opus decoder description is too short (must be at least 18 bytes).');
}
assert(description.byteLength >= 18);
const view = ArrayBuffer.isView(description)
? new DataView(description.buffer, description.byteOffset, description.byteLength)
: new DataView(description);
preskip = view.getUint16(10, true);
gain = view.getInt16(14, true);
outputChannelCount = view.getUint8(9);
preSkip = view.getUint16(10, true);
inputSampleRate = view.getUint32(12, true);
outputGain = view.getInt16(16, true);
channelMappingFamily = view.getUint8(18);
if (channelMappingFamily) {
const bytes = toUint8Array(description);
channelMappingTable = bytes.subarray(19, 19 + 2 + outputChannelCount);
}
}
// https://www.opus-codec.org/docs/opus_in_isobmff.html
return box('dOps', [
u8(0), // Version
u8(trackData.info.numberOfChannels), // OutputChannelCount
u16(preskip),
u32(trackData.info.sampleRate), // InputSampleRate
fixed_8_8(gain), // OutputGain
u8(0), // ChannelMappingFamily
u8(outputChannelCount), // OutputChannelCount
u16(preSkip), // PreSkip
u32(inputSampleRate), // InputSampleRate
i16(outputGain), // OutputGain
u8(channelMappingFamily), // ChannelMappingFamily
...channelMappingTable,
]);
};
/** FLAC specific box. */
export const dfLa = (trackData: IsobmffAudioTrackData) => {
const description = trackData.info.decoderConfig?.description;
assert(description);
const bytes = toUint8Array(description);
return fullBox('dfLa', 0, 0, [
...bytes.subarray(4),
]);
};
@@ -836,11 +949,24 @@ export const stsc = (trackData: IsobmffTrackData) => {
};
/** Sample Size Box: Specifies the byte size of each sample in the media. */
export const stsz = (trackData: IsobmffTrackData) => fullBox('stsz', 0, 0, [
u32(0), // Sample size (0 means non-constant size)
u32(trackData.samples.length), // Number of entries
trackData.samples.map(x => u32(x.size)), // Sample size table
]);
export const stsz = (trackData: IsobmffTrackData) => {
if (trackData.requiresPcmTransformation) {
assert(trackData.type === 'audio');
const { sampleSize } = parsePcmCodec(trackData.track.source._codec as PcmAudioCodec);
// With PCM, every sample has the same size
return fullBox('stsz', 0, 0, [
u32(sampleSize * trackData.info.numberOfChannels), // Sample size
u32(trackData.samples.reduce((acc, x) => acc + intoTimescale(x.duration, trackData.timescale), 0)),
]);
}
return fullBox('stsz', 0, 0, [
u32(0), // Sample size (0 means non-constant size)
u32(trackData.samples.length), // Number of entries
trackData.samples.map(x => u32(x.size)), // Sample size table
]);
};
/** Chunk Offset Box: Identifies the location of each chunk of data in the media's data stream, relative to the file. */
export const stco = (trackData: IsobmffTrackData) => {
@@ -1134,15 +1260,39 @@ const VIDEO_CODEC_TO_CONFIGURATION_BOX: Record<VideoCodec, (trackData: IsobmffVi
};
const AUDIO_CODEC_TO_BOX_NAME: Partial<Record<AudioCodec, string>> = {
aac: 'mp4a',
opus: 'Opus',
'aac': 'mp4a',
'mp3': 'mp4a',
'opus': 'Opus',
'vorbis': 'mp4a',
'flac': 'fLaC',
'ulaw': 'ulaw',
'alaw': 'alaw',
'pcm-u8': 'raw ',
'pcm-s8': 'sowt',
'pcm-s16': 'sowt',
'pcm-s16be': 'twos',
'pcm-s24': 'in24',
'pcm-s24be': 'in24',
'pcm-s32': 'in32',
'pcm-s32be': 'in32',
'pcm-f32': 'fl32',
'pcm-f32be': 'fl32',
};
const AUDIO_CODEC_TO_CONFIGURATION_BOX: Partial<
Record<AudioCodec, (trackData: IsobmffAudioTrackData) => Box | null>
> = {
aac: esds,
opus: dOps,
'aac': esds,
'mp3': esds,
'opus': dOps,
'vorbis': esds,
'flac': dfLa,
'pcm-s24': wave,
'pcm-s24be': wave,
'pcm-s32': wave,
'pcm-s32be': wave,
'pcm-f32': wave,
'pcm-f32be': wave,
};
const SUBTITLE_CODEC_TO_BOX_NAME: Record<SubtitleCodec, string> = {
+12 -11
View File
@@ -6,6 +6,7 @@ import {
extractVideoCodecString,
MediaCodec,
parseAacAudioSpecificConfig,
PCM_CODECS,
VideoCodec,
Vp9CodecInfo,
} from '../codec';
@@ -286,11 +287,9 @@ export class IsobmffDemuxer extends Demuxer {
this.traverseBox();
this.currentTrack = null;
const isPcmCodec = internalTrack.info?.type === 'audio' && (
internalTrack.info.codec?.startsWith('pcm-')
|| internalTrack.info.codec === 'ulaw'
|| internalTrack.info.codec === 'alaw'
);
const isPcmCodec = internalTrack.info?.type === 'audio'
&& internalTrack.info.codec
&& (PCM_CODECS as readonly string[]).includes(internalTrack.info.codec);
if (isPcmCodec && sampleTable.sampleCompositionTimeOffsets.length === 0) {
// If the audio has PCM samples, the way the samples are defined in the sample table is somewhat
@@ -316,7 +315,7 @@ export class IsobmffDemuxer extends Demuxer {
const startTimingEntryIndex = binarySearchLessOrEqual(
sampleTable.sampleTimingEntries,
chunkEntry.startSampleIndex,
startSampleIndex,
x => x.startIndex,
);
const startTimingEntry = sampleTable.sampleTimingEntries[startTimingEntryIndex]!;
@@ -339,7 +338,7 @@ export class IsobmffDemuxer extends Demuxer {
} else {
// One sample for the entire chunk
newSampleTimingEntries.push({
startIndex: chunkEntry.startChunkIndex,
startIndex: chunkEntry.startChunkIndex + j,
startDecodeTimestamp: firstSampleTimestamp,
count: 1,
delta,
@@ -756,12 +755,14 @@ export class IsobmffDemuxer extends Demuxer {
if (stsdVersion === 0 && version > 0) {
// Additional QuickTime fields
if (version === 1) {
this.isobmffReader.pos += 4 * 4;
this.isobmffReader.pos += 4;
sampleSize = 8 * this.isobmffReader.readU32();
this.isobmffReader.pos += 2 * 4;
} else if (version === 2) {
this.isobmffReader.pos += 4;
sampleRate = this.isobmffReader.readF64();
channelCount = this.isobmffReader.readU32();
this.isobmffReader.pos += 4; // Always 0x7F000000
this.isobmffReader.pos += 4; // Always 0x7f000000
sampleSize = this.isobmffReader.readU32();
@@ -777,7 +778,7 @@ export class IsobmffDemuxer extends Demuxer {
if (sampleSize > 0 && sampleSize <= 64) {
if (isFloat) {
if (sampleSize === 32 && !isBigEndian) {
if (sampleSize === 32) {
track.info.codec = isBigEndian ? 'pcm-f32be' : 'pcm-f32';
}
} else {
@@ -800,7 +801,7 @@ export class IsobmffDemuxer extends Demuxer {
}
if (track.info.codec === null) {
console.warn('Unsupportedd PCM format.');
console.warn('Unsupported PCM format.');
}
}
}
+95 -6
View File
@@ -3,10 +3,17 @@ import { Muxer } from '../muxer';
import { Output, OutputAudioTrack, OutputSubtitleTrack, OutputTrack, OutputVideoTrack } from '../output';
import { ArrayBufferTargetWriter, Writer } from '../writer';
import { assert, last } from '../misc';
import { Mp4OutputFormat, Mp4OutputFormatOptions } from '../output-format';
import { IsobmffOutputFormatOptions, IsobmffOutputFormat, MovOutputFormat } from '../output-format';
import { inlineTimestampRegex, SubtitleConfig, SubtitleCue, SubtitleMetadata } from '../subtitles';
import { ArrayBufferTarget } from '../target';
import { validateAudioChunkMetadata, validateSubtitleMetadata, validateVideoChunkMetadata } from '../codec';
import {
parsePcmCodec,
PCM_CODECS,
PcmAudioCodec,
validateAudioChunkMetadata,
validateSubtitleMetadata,
validateVideoChunkMetadata,
} from '../codec';
import { EncodedAudioSample, EncodedVideoSample } from '../sample';
export const GLOBAL_TIMESCALE = 1000;
@@ -40,6 +47,11 @@ export type IsobmffTrackData = {
compositionTimeOffsetTable: { sampleCount: number; sampleCompositionTimeOffset: number }[];
lastTimescaleUnits: number | null;
lastSample: Sample | null;
/**
* The "PCM transformation" is making every sample in the sample table be exactly one PCM audio sample long.
* Some players expect this for PCM audio.
*/
requiresPcmTransformation: boolean;
finalizedChunks: Chunk[];
currentChunk: Chunk | null;
@@ -87,7 +99,8 @@ export const intoTimescale = (timeInSeconds: number, timescale: number, round =
export class IsobmffMuxer extends Muxer {
private writer: Writer;
private boxWriter: IsobmffBoxWriter;
private fastStart: NonNullable<Mp4OutputFormatOptions['fastStart']>;
private isMov: boolean;
private fastStart: NonNullable<IsobmffOutputFormatOptions['fastStart']>;
private auxTarget = new ArrayBufferTarget();
private auxWriter = this.auxTarget._createWriter();
@@ -103,12 +116,14 @@ export class IsobmffMuxer extends Muxer {
private nextFragmentNumber = 1;
constructor(output: Output, format: Mp4OutputFormat) {
constructor(output: Output, format: IsobmffOutputFormat) {
super(output);
this.writer = output._writer;
this.boxWriter = new IsobmffBoxWriter(this.writer);
this.isMov = format instanceof MovOutputFormat;
// If the fastStart option isn't defined, enable in-memory fast start if the target is an ArrayBuffer, as the
// memory usage remains identical
const fastStartDefault = this.writer instanceof ArrayBufferTargetWriter ? 'in-memory' : false;
@@ -126,6 +141,7 @@ export class IsobmffMuxer extends Muxer {
// Write the header
this.boxWriter.writeBox(ftyp({
isMov: this.isMov,
holdsAvc: holdsAvc,
fragmented: this.fastStart === 'fragmented',
}));
@@ -178,6 +194,7 @@ export class IsobmffMuxer extends Muxer {
finalizedChunks: [],
currentChunk: null,
compactlyCodedChunkTable: [],
requiresPcmTransformation: false,
};
this.trackDatas.push(newTrackData);
@@ -216,6 +233,9 @@ export class IsobmffMuxer extends Muxer {
finalizedChunks: [],
currentChunk: null,
compactlyCodedChunkTable: [],
requiresPcmTransformation:
this.fastStart !== 'fragmented'
&& (PCM_CODECS as readonly string[]).includes(track.source._codec),
};
this.trackDatas.push(newTrackData);
@@ -252,6 +272,8 @@ export class IsobmffMuxer extends Muxer {
finalizedChunks: [],
currentChunk: null,
compactlyCodedChunkTable: [],
requiresPcmTransformation: false,
lastCueEndTimestamp: 0,
cueQueue: [],
nextSourceId: 0,
@@ -308,12 +330,47 @@ export class IsobmffMuxer extends Muxer {
sample.type,
);
if (trackData.requiresPcmTransformation) {
await this.maybePadWithSilence(trackData, timestamp);
}
await this.registerSample(trackData, internalSample);
} finally {
release();
}
}
private async maybePadWithSilence(trackData: IsobmffAudioTrackData, untilTimestamp: number) {
// The PCM transformation assumes that all samples are contiguous. This is not something that is enforced, so
// we need to pad the "holes" in between samples (and before the first sample) with additional
// "silence samples".
const lastSample = last(trackData.samples);
const lastEndTimestamp = lastSample
? lastSample.timestamp + lastSample.duration
: 0;
const delta = untilTimestamp - lastEndTimestamp;
const deltaInTimescale = intoTimescale(delta, trackData.timescale);
if (deltaInTimescale > 0) {
const { sampleSize, silentValue } = parsePcmCodec(
trackData.info.decoderConfig.codec as PcmAudioCodec,
);
const samplesNeeded = deltaInTimescale * trackData.info.numberOfChannels;
const data = new Uint8Array(sampleSize * samplesNeeded).fill(silentValue);
const paddingSample = this.createSampleForTrack(
trackData,
new Uint8Array(data.buffer),
lastEndTimestamp,
delta,
'key',
);
await this.registerSample(trackData, paddingSample);
}
}
async addSubtitleCue(track: OutputSubtitleTrack, cue: SubtitleCue, meta?: SubtitleMetadata) {
const release = await this.mutex.acquire();
@@ -451,6 +508,32 @@ export class IsobmffMuxer extends Muxer {
return;
}
if (trackData.requiresPcmTransformation) {
let totalDuration = 0;
// Compute the total duration in the track timescale (which is equal to the amount of PCM audio samples)
// and simply say that's how many new samples there are.
for (let i = 0; i < trackData.timestampProcessingQueue.length; i++) {
const sample = trackData.timestampProcessingQueue[i]!;
const duration = intoTimescale(sample.duration, trackData.timescale);
totalDuration += duration;
}
if (trackData.timeToSampleTable.length === 0) {
trackData.timeToSampleTable.push({
sampleCount: totalDuration,
sampleDelta: 1,
});
} else {
const lastEntry = last(trackData.timeToSampleTable)!;
lastEntry.sampleCount += totalDuration;
}
trackData.timestampProcessingQueue.length = 0;
return;
}
const sortedTimestamps = trackData.timestampProcessingQueue.map(x => x.timestamp).sort((a, b) => a - b);
for (let i = 0; i < trackData.timestampProcessingQueue.length; i++) {
@@ -628,13 +711,19 @@ export class IsobmffMuxer extends Muxer {
trackData.finalizedChunks.push(trackData.currentChunk);
this.finalizedChunks.push(trackData.currentChunk);
let sampleCount = trackData.currentChunk.samples.length;
if (trackData.requiresPcmTransformation) {
sampleCount = trackData.currentChunk.samples
.reduce((acc, sample) => acc + intoTimescale(sample.duration, trackData.timescale), 0);
}
if (
trackData.compactlyCodedChunkTable.length === 0
|| last(trackData.compactlyCodedChunkTable)!.samplesPerChunk !== trackData.currentChunk.samples.length
|| last(trackData.compactlyCodedChunkTable)!.samplesPerChunk !== sampleCount
) {
trackData.compactlyCodedChunkTable.push({
firstChunk: trackData.finalizedChunks.length, // 1-indexed
samplesPerChunk: trackData.currentChunk.samples.length,
samplesPerChunk: sampleCount,
});
}