Add support for all audio codecs in the ISOBMFF muxer (& some fixes)

This commit is contained in:
Vanilagy
2025-01-05 13:50:50 +01:00
parent 316e16211b
commit ee81e1b674
10 changed files with 560 additions and 155 deletions
+34 -16
View File
@@ -40,35 +40,49 @@
source source
}); });
//const videoTrack = await input.getPrimaryVideoTrack();
//console.log(await videoTrack.computeSampleStats());
//return;
const target = new Metamuxer.ArrayBufferTarget(); const target = new Metamuxer.ArrayBufferTarget();
const output = new Metamuxer.Output({ const output = new Metamuxer.Output({
format: new Metamuxer.Mp4OutputFormat({ fastStart: undefined }), format: new Metamuxer.MovOutputFormat({ fastStart: undefined }),
target target
}); });
const audioTrack = await input.getPrimaryAudioTrack();
const videoTrack = await input.getPrimaryVideoTrack(); const videoTrack = await input.getPrimaryVideoTrack();
const drain = new Metamuxer.EncodedVideoSampleDrain(videoTrack);
const drain = new Metamuxer.EncodedAudioSampleDrain(audioTrack);
console.log(await videoTrack.computeSampleStats()) const decoderConfig = await audioTrack.getDecoderConfig();
document.body.textContent = performance.now() - start; const sampleSource = new Metamuxer.EncodedAudioSampleSource(await audioTrack.getCodec());
const audioDataSource = new Metamuxer.AudioDataSource({ codec: 'ulaw' });
const decoderConfig = await videoTrack.getDecoderConfig(); const videoSampleSource = new Metamuxer.EncodedVideoSampleSource(await videoTrack.getCodec());
const sampleSource = new Metamuxer.EncodedVideoSampleSource(await videoTrack.getCodec()); output.addAudioTrack(audioDataSource ?? sampleSource);
output.addVideoTrack(sampleSource); output.addVideoTrack(videoSampleSource);
output.start(); output.start();
let j = 0; const videoDecoderConfig = await videoTrack.getDecoderConfig();
for await (const sample of drain.samples(undefined, undefined)) {
await sampleSource.digest(sample, { decoderConfig });
if (j++ < 12) { /*
console.log(sample); for await (const sample of drain.samples()) {
} //sample.timestamp *= 2;
//console.log(sample);
//console.log(sample)
await sampleSource.digest(sample, { decoderConfig });
}*/
for await (const { data } of new Metamuxer.AudioDataDrain(audioTrack).data()) {
//console.log(sample)
await audioDataSource.digest(data);
}
for await (const sample of new Metamuxer.EncodedVideoSampleDrain(videoTrack).samples()) {
//console.log(sample)
await videoSampleSource.digest(sample, { decoderConfig: videoDecoderConfig });
} }
await output.finalize(); await output.finalize();
console.log(target) console.log(target)
@@ -80,8 +94,11 @@
a.click(); a.click();
URL.revokeObjectURL(url); URL.revokeObjectURL(url);
} }
download(new Blob([target.buffer]), 'converted.mp4'); download(new Blob([target.buffer]), 'converted.mov');
document.body.textContent = performance.now() - start;
/*
const input2 = new Metamuxer.Input({ const input2 = new Metamuxer.Input({
formats: Metamuxer.ALL_FORMATS, formats: Metamuxer.ALL_FORMATS,
source: new Metamuxer.BufferSource(target.buffer) source: new Metamuxer.BufferSource(target.buffer)
@@ -97,6 +114,7 @@
break; break;
} }
} }
*/
+42 -21
View File
@@ -30,6 +30,8 @@ export const PCM_CODECS = [
'pcm-s32be', 'pcm-s32be',
'pcm-f32', 'pcm-f32',
'pcm-f32be', 'pcm-f32be',
'ulaw',
'alaw',
] as const; ] as const;
/** @public */ /** @public */
export const AUDIO_CODECS = [ export const AUDIO_CODECS = [
@@ -38,8 +40,6 @@ export const AUDIO_CODECS = [
'opus', 'opus',
'vorbis', 'vorbis',
'flac', 'flac',
'ulaw',
'alaw',
...PCM_CODECS, ...PCM_CODECS,
] as const; // TODO add the rest ] as const; // TODO add the rest
/** @public */ /** @public */
@@ -495,6 +495,35 @@ export const parseAacAudioSpecificConfig = (bytes: Uint8Array | null) => {
}; };
}; };
const PCM_CODEC_REGEX = /^pcm-([usf])(\d+)+(be)?$/;
export const parsePcmCodec = (codec: PcmAudioCodec) => {
assert(PCM_CODECS.includes(codec));
if (codec === 'ulaw') {
return { dataType: 'ulaw' as const, sampleSize: 1 as const, littleEndian: true, silentValue: 255 };
} else if (codec === 'alaw') {
return { dataType: 'alaw' as const, sampleSize: 1 as const, littleEndian: true, silentValue: 213 };
}
const match = PCM_CODEC_REGEX.exec(codec);
assert(match);
let dataType: 'unsigned' | 'signed' | 'float' | 'ulaw' | 'alaw';
if (match[1] === 'u') {
dataType = 'unsigned';
} else if (match[1] === 's') {
dataType = 'signed';
} else {
dataType = 'float';
}
const sampleSize = (Number(match[2]) / 8) as 1 | 2 | 3 | 4;
const littleEndian = match[3] !== 'be';
const silentValue = codec === 'pcm-u8' ? 2 ** 7 : 0;
return { dataType, sampleSize, littleEndian, silentValue };
};
export const getVideoEncoderConfigExtension = (codec: VideoCodec) => { export const getVideoEncoderConfigExtension = (codec: VideoCodec) => {
if (codec === 'avc') { if (codec === 'avc') {
return { return {
@@ -753,10 +782,11 @@ export const validateAudioChunkMetadata = (metadata: EncodedAudioChunkMetadata |
throw new TypeError('Audio chunk metadata decoder configuration codec string for Opus must be "opus".'); throw new TypeError('Audio chunk metadata decoder configuration codec string for Opus must be "opus".');
} }
if (!metadata.decoderConfig.description) { if (metadata.decoderConfig.description && metadata.decoderConfig.description.byteLength < 18) {
// Description is optional for Opus per-spec, so we shouldn't enforce it
throw new TypeError( throw new TypeError(
'Audio chunk metadata decoder configuration for Opus must include a description, which is expected to' 'Audio chunk metadata decoder configuration description, when specified, is expected to be an'
+ ' be an Identification Header as specified in Section 5.1 of RFC 7845.', + ' Identification Header as specified in Section 5.1 of RFC 7845.',
); );
} }
} }
@@ -781,7 +811,8 @@ export const validateAudioChunkMetadata = (metadata: EncodedAudioChunkMetadata |
throw new TypeError('Audio chunk metadata decoder configuration codec string for FLAC must be "flac".'); throw new TypeError('Audio chunk metadata decoder configuration codec string for FLAC must be "flac".');
} }
if (!metadata.decoderConfig.description) { const minDescriptionSize = 4 + 4 + 34; // 'fLaC' + metadata block header + STREAMINFO block
if (!metadata.decoderConfig.description || metadata.decoderConfig.description.byteLength < minDescriptionSize) {
throw new TypeError( throw new TypeError(
'Audio chunk metadata decoder configuration for FLAC must include a description, which is expected to' 'Audio chunk metadata decoder configuration for FLAC must include a description, which is expected to'
+ ' adhere to the format described in https://www.w3.org/TR/webcodecs-flac-codec-registration/.', + ' adhere to the format described in https://www.w3.org/TR/webcodecs-flac-codec-registration/.',
@@ -789,22 +820,12 @@ export const validateAudioChunkMetadata = (metadata: EncodedAudioChunkMetadata |
} }
} }
// ulaw-specific validation
if (metadata.decoderConfig.codec === 'ulaw') {
if (metadata.decoderConfig.codec !== 'ulaw') {
throw new TypeError('Audio chunk metadata decoder configuration codec string for uLaw must be "ulaw".');
}
}
// alaw-specific validation
if (metadata.decoderConfig.codec === 'alaw') {
if (metadata.decoderConfig.codec !== 'alaw') {
throw new TypeError('Audio chunk metadata decoder configuration codec string for A-law must be "alaw".');
}
}
// PCM-specific validation // PCM-specific validation
if (metadata.decoderConfig.codec.startsWith('pcm')) { if (
metadata.decoderConfig.codec.startsWith('pcm')
|| metadata.decoderConfig.codec.startsWith('ulaw')
|| metadata.decoderConfig.codec.startsWith('alaw')
) {
if (!(PCM_CODECS as readonly string[]).includes(metadata.decoderConfig.codec)) { if (!(PCM_CODECS as readonly string[]).includes(metadata.decoderConfig.codec)) {
throw new TypeError( throw new TypeError(
'Audio chunk metadata decoder configuration codec string for PCM must be one of the supported PCM' 'Audio chunk metadata decoder configuration codec string for PCM must be one of the supported PCM'
+4 -1
View File
@@ -1,8 +1,10 @@
export { Output, OutputOptions, VideoTrackMetadata, AudioTrackMetadata, SubtitleTrackMetadata } from './output'; export { Output, OutputOptions, VideoTrackMetadata, AudioTrackMetadata, SubtitleTrackMetadata } from './output';
export { export {
OutputFormat, OutputFormat,
IsobmffOutputFormat,
Mp4OutputFormat, Mp4OutputFormat,
Mp4OutputFormatOptions, MovOutputFormat,
IsobmffOutputFormatOptions,
MkvOutputFormat, MkvOutputFormat,
MkvOutputFormatOptions, MkvOutputFormatOptions,
WebMOutputFormat, WebMOutputFormat,
@@ -75,6 +77,7 @@ export {
CanvasDrain, CanvasDrain,
WrappedCanvas, WrappedCanvas,
AudioDataDrain, AudioDataDrain,
WrappedAudioData,
AudioBufferDrain, AudioBufferDrain,
WrappedAudioBuffer, WrappedAudioBuffer,
} from './media-drain'; } from './media-drain';
+6 -2
View File
@@ -212,7 +212,11 @@ const computeSampleStats = async (drain: EncodedVideoSampleDrain | EncodedAudioS
return { return {
sampleCount, sampleCount,
averageSampleRate: sampleCount ? Math.fround(sampleCount / (endTimestamp - startTimestamp)) : 0, averageSampleRate: sampleCount
averageBitrate: sampleCount ? Math.fround(8 * totalSampleBytes / (endTimestamp - startTimestamp)) : 0, ? Number((sampleCount / (endTimestamp - startTimestamp)).toPrecision(16))
: 0,
averageBitrate: sampleCount
? Number((8 * totalSampleBytes / (endTimestamp - startTimestamp)).toPrecision(16))
: 0,
}; };
}; };
+198 -48
View File
@@ -12,7 +12,7 @@ import {
IDENTITY_MATRIX, IDENTITY_MATRIX,
rotationMatrix, rotationMatrix,
} from '../misc'; } from '../misc';
import { AudioCodec, SubtitleCodec, VideoCodec } from '../codec'; import { AudioCodec, parsePcmCodec, PCM_CODECS, PcmAudioCodec, SubtitleCodec, VideoCodec } from '../codec';
import { formatSubtitleTimestamp } from '../subtitles'; import { formatSubtitleTimestamp } from '../subtitles';
import { Writer } from '../writer'; import { Writer } from '../writer';
import { import {
@@ -258,6 +258,7 @@ export const fullBox = (
* reader understands. * reader understands.
*/ */
export const ftyp = (details: { export const ftyp = (details: {
isMov: boolean;
holdsAvc: boolean; holdsAvc: boolean;
fragmented: boolean; fragmented: boolean;
}) => { }) => {
@@ -267,6 +268,15 @@ export const ftyp = (details: {
const minorVersion = 0x200; const minorVersion = 0x200;
if (details.isMov) {
return box('ftyp', [
ascii('qt '), // Major brand
u32(minorVersion), // Minor version
// Compatible brands
ascii('qt '),
]);
}
if (details.fragmented) return box('ftyp', [ if (details.fragmented) return box('ftyp', [
ascii('iso5'), // Major brand ascii('iso5'), // Major brand
u32(minorVersion), // Minor version u32(minorVersion), // Minor version
@@ -688,45 +698,101 @@ export const av1C = (trackData: IsobmffVideoTrackData) => {
export const soundSampleDescription = ( export const soundSampleDescription = (
compressionType: string, compressionType: string,
trackData: IsobmffAudioTrackData, trackData: IsobmffAudioTrackData,
) => box(compressionType, [ ) => {
Array(6).fill(0), // Reserved let version = 0;
u16(1), // Data reference index let contents: NestedNumberArray;
u16(0), // Version (AudioSampleEntry, not AudioSampleEntryV1)
u16(0), // Revision level let sampleSizeInBits = 16;
u32(0), // Vendor if ((PCM_CODECS as readonly AudioCodec[]).includes(trackData.track.source._codec)) {
u16(trackData.info.numberOfChannels), // Number of channels const codec = trackData.track.source._codec as PcmAudioCodec;
u16(16), // Sample size (bits) const { sampleSize } = parsePcmCodec(codec);
u16(0), // Compression ID sampleSizeInBits = 8 * sampleSize;
u16(0), // Packet size
u32(2 ** 16 * trackData.info.sampleRate), // Sample rate if (sampleSizeInBits > 16) {
], [ version = 1;
AUDIO_CODEC_TO_CONFIGURATION_BOX[trackData.track.source._codec]!(trackData), }
]); }
if (version === 0) {
contents = [
Array(6).fill(0), // Reserved
u16(1), // Data reference index
u16(version), // Version
u16(0), // Revision level
u32(0), // Vendor
u16(trackData.info.numberOfChannels), // Number of channels
u16(sampleSizeInBits), // Sample size (bits)
u16(0), // Compression ID
u16(0), // Packet size
u16(trackData.info.sampleRate < 2 ** 16 ? trackData.info.sampleRate : 0), // Sample rate (upper)
u16(0), // Sample rate (lower)
];
} else {
contents = [
Array(6).fill(0), // Reserved
u16(1), // Data reference index
u16(version), // Version
u16(0), // Revision level
u32(0), // Vendor
u16(trackData.info.numberOfChannels), // Number of channels
u16(Math.min(sampleSizeInBits, 16)), // Sample size (bits)
u16(0), // Compression ID
u16(0), // Packet size
u16(trackData.info.sampleRate < 2 ** 16 ? trackData.info.sampleRate : 0), // Sample rate (upper)
u16(0), // Sample rate (lower)
u32(1), // Samples per packet (must be 1 for uncompressed formats)
u32(sampleSizeInBits / 8), // Bytes per packet
u32(trackData.info.numberOfChannels * sampleSizeInBits / 8), // Bytes per frame
u32(2), // Bytes per sample (constant in FFmpeg)
];
}
return box(compressionType, contents, [
AUDIO_CODEC_TO_CONFIGURATION_BOX[trackData.track.source._codec]?.(trackData) ?? null,
]);
};
/** MPEG-4 Elementary Stream Descriptor Box. */ /** MPEG-4 Elementary Stream Descriptor Box. */
export const esds = (trackData: IsobmffAudioTrackData) => { export const esds = (trackData: IsobmffAudioTrackData) => {
const description = toUint8Array(trackData.info.decoderConfig.description ?? new ArrayBuffer(0));
// Adapted from https://stackoverflow.com/a/54803118
// We build up the bytes in a layered way which reflects the nested structure // We build up the bytes in a layered way which reflects the nested structure
let objectTypeIndication: number;
switch (trackData.track.source._codec) {
case 'aac': {
objectTypeIndication = 0x40;
}; break;
case 'mp3': {
objectTypeIndication = 0x6b;
}; break;
case 'vorbis': {
objectTypeIndication = 0xdd;
}; break;
default: throw new Error(`Unhandled audio codec: ${trackData.track.source._codec}`);
}
let bytes = [ let bytes = [
...description, ...u8(objectTypeIndication), // Object type indication
];
bytes = [
...u8(0x40), // MPEG-4 Audio
...u8(0x15), // stream type(6bits)=5 audio, flags(2bits)=1 ...u8(0x15), // stream type(6bits)=5 audio, flags(2bits)=1
...u24(0), // 24bit buffer size ...u24(0), // 24bit buffer size
...u32(0), // max bitrate ...u32(0), // max bitrate
...u32(0), // avg bitrate ...u32(0), // avg bitrate
...u8(0x05), // TAG(5) = ASC ([2],[3]) embedded in above OD
...variableUnsignedInt(bytes.length),
...bytes,
]; ];
if (trackData.info.decoderConfig.description) {
const description = toUint8Array(trackData.info.decoderConfig.description);
// Add the decoder description to the end
bytes = [
...bytes,
...u8(0x05), // TAG(5) = DecoderSpecificInfo
...variableUnsignedInt(description.byteLength),
...description,
];
}
bytes = [ bytes = [
...u16(1), // ES_ID = 1 ...u16(1), // ES_ID = 1
...u8(0x00), // flags etc = 0 ...u8(0x00), // flags etc = 0
...u8(0x04), // TAG(4) = ES Descriptor ([2]) embedded in above OD ...u8(0x04), // TAG(4) = ES Descriptor
...variableUnsignedInt(bytes.length), ...variableUnsignedInt(bytes.length),
...bytes, ...bytes,
...u8(0x06), // TAG(6) ...u8(0x06), // TAG(6)
@@ -734,7 +800,7 @@ export const esds = (trackData: IsobmffAudioTrackData) => {
...u8(0x02), // data ...u8(0x02), // data
]; ];
bytes = [ bytes = [
...u8(0x03), // TAG(3) = Object Descriptor ([2]) ...u8(0x03), // TAG(3) = Object Descriptor
...variableUnsignedInt(bytes.length), ...variableUnsignedInt(bytes.length),
...bytes, ...bytes,
]; ];
@@ -742,35 +808,82 @@ export const esds = (trackData: IsobmffAudioTrackData) => {
return fullBox('esds', 0, 0, bytes); return fullBox('esds', 0, 0, bytes);
}; };
export const wave = (trackData: IsobmffAudioTrackData) => {
return box('wave', undefined, [
frma(trackData),
enda(trackData),
box('\x00\x00\x00\x00'), // NULL tag at the end
]);
};
export const frma = (trackData: IsobmffAudioTrackData) => {
return box('frma', [
ascii(AUDIO_CODEC_TO_BOX_NAME[trackData.track.source._codec]!),
]);
};
// This box specifies PCM endianness
export const enda = (trackData: IsobmffAudioTrackData) => {
const { littleEndian } = parsePcmCodec(trackData.track.source._codec as PcmAudioCodec);
return box('enda', [
u16(+littleEndian),
]);
};
/** Opus Specific Box. */ /** Opus Specific Box. */
export const dOps = (trackData: IsobmffAudioTrackData) => { export const dOps = (trackData: IsobmffAudioTrackData) => {
let outputChannelCount = trackData.info.numberOfChannels;
// Default PreSkip, should be at least 80 milliseconds worth of playback, measured in 48000 Hz samples // Default PreSkip, should be at least 80 milliseconds worth of playback, measured in 48000 Hz samples
let preskip = 3840; let preSkip = 3840;
let gain = 0; let inputSampleRate = trackData.info.sampleRate;
let outputGain = 0;
let channelMappingFamily = 0;
let channelMappingTable = new Uint8Array(0);
// Read preskip and from codec private data from the encoder // Read preskip and from codec private data from the encoder
// https://www.rfc-editor.org/rfc/rfc7845#section-5 // https://www.rfc-editor.org/rfc/rfc7845#section-5
const description = trackData.info.decoderConfig?.description; const description = trackData.info.decoderConfig?.description;
if (description) { if (description) {
if (description.byteLength < 18) { assert(description.byteLength >= 18);
throw new Error('Opus decoder description is too short (must be at least 18 bytes).');
}
const view = ArrayBuffer.isView(description) const view = ArrayBuffer.isView(description)
? new DataView(description.buffer, description.byteOffset, description.byteLength) ? new DataView(description.buffer, description.byteOffset, description.byteLength)
: new DataView(description); : new DataView(description);
preskip = view.getUint16(10, true);
gain = view.getInt16(14, true); outputChannelCount = view.getUint8(9);
preSkip = view.getUint16(10, true);
inputSampleRate = view.getUint32(12, true);
outputGain = view.getInt16(16, true);
channelMappingFamily = view.getUint8(18);
if (channelMappingFamily) {
const bytes = toUint8Array(description);
channelMappingTable = bytes.subarray(19, 19 + 2 + outputChannelCount);
}
} }
// https://www.opus-codec.org/docs/opus_in_isobmff.html // https://www.opus-codec.org/docs/opus_in_isobmff.html
return box('dOps', [ return box('dOps', [
u8(0), // Version u8(0), // Version
u8(trackData.info.numberOfChannels), // OutputChannelCount u8(outputChannelCount), // OutputChannelCount
u16(preskip), u16(preSkip), // PreSkip
u32(trackData.info.sampleRate), // InputSampleRate u32(inputSampleRate), // InputSampleRate
fixed_8_8(gain), // OutputGain i16(outputGain), // OutputGain
u8(0), // ChannelMappingFamily u8(channelMappingFamily), // ChannelMappingFamily
...channelMappingTable,
]);
};
/** FLAC specific box. */
export const dfLa = (trackData: IsobmffAudioTrackData) => {
const description = trackData.info.decoderConfig?.description;
assert(description);
const bytes = toUint8Array(description);
return fullBox('dfLa', 0, 0, [
...bytes.subarray(4),
]); ]);
}; };
@@ -836,11 +949,24 @@ export const stsc = (trackData: IsobmffTrackData) => {
}; };
/** Sample Size Box: Specifies the byte size of each sample in the media. */ /** Sample Size Box: Specifies the byte size of each sample in the media. */
export const stsz = (trackData: IsobmffTrackData) => fullBox('stsz', 0, 0, [ export const stsz = (trackData: IsobmffTrackData) => {
u32(0), // Sample size (0 means non-constant size) if (trackData.requiresPcmTransformation) {
u32(trackData.samples.length), // Number of entries assert(trackData.type === 'audio');
trackData.samples.map(x => u32(x.size)), // Sample size table const { sampleSize } = parsePcmCodec(trackData.track.source._codec as PcmAudioCodec);
]);
// With PCM, every sample has the same size
return fullBox('stsz', 0, 0, [
u32(sampleSize * trackData.info.numberOfChannels), // Sample size
u32(trackData.samples.reduce((acc, x) => acc + intoTimescale(x.duration, trackData.timescale), 0)),
]);
}
return fullBox('stsz', 0, 0, [
u32(0), // Sample size (0 means non-constant size)
u32(trackData.samples.length), // Number of entries
trackData.samples.map(x => u32(x.size)), // Sample size table
]);
};
/** Chunk Offset Box: Identifies the location of each chunk of data in the media's data stream, relative to the file. */ /** Chunk Offset Box: Identifies the location of each chunk of data in the media's data stream, relative to the file. */
export const stco = (trackData: IsobmffTrackData) => { export const stco = (trackData: IsobmffTrackData) => {
@@ -1134,15 +1260,39 @@ const VIDEO_CODEC_TO_CONFIGURATION_BOX: Record<VideoCodec, (trackData: IsobmffVi
}; };
const AUDIO_CODEC_TO_BOX_NAME: Partial<Record<AudioCodec, string>> = { const AUDIO_CODEC_TO_BOX_NAME: Partial<Record<AudioCodec, string>> = {
aac: 'mp4a', 'aac': 'mp4a',
opus: 'Opus', 'mp3': 'mp4a',
'opus': 'Opus',
'vorbis': 'mp4a',
'flac': 'fLaC',
'ulaw': 'ulaw',
'alaw': 'alaw',
'pcm-u8': 'raw ',
'pcm-s8': 'sowt',
'pcm-s16': 'sowt',
'pcm-s16be': 'twos',
'pcm-s24': 'in24',
'pcm-s24be': 'in24',
'pcm-s32': 'in32',
'pcm-s32be': 'in32',
'pcm-f32': 'fl32',
'pcm-f32be': 'fl32',
}; };
const AUDIO_CODEC_TO_CONFIGURATION_BOX: Partial< const AUDIO_CODEC_TO_CONFIGURATION_BOX: Partial<
Record<AudioCodec, (trackData: IsobmffAudioTrackData) => Box | null> Record<AudioCodec, (trackData: IsobmffAudioTrackData) => Box | null>
> = { > = {
aac: esds, 'aac': esds,
opus: dOps, 'mp3': esds,
'opus': dOps,
'vorbis': esds,
'flac': dfLa,
'pcm-s24': wave,
'pcm-s24be': wave,
'pcm-s32': wave,
'pcm-s32be': wave,
'pcm-f32': wave,
'pcm-f32be': wave,
}; };
const SUBTITLE_CODEC_TO_BOX_NAME: Record<SubtitleCodec, string> = { const SUBTITLE_CODEC_TO_BOX_NAME: Record<SubtitleCodec, string> = {
+12 -11
View File
@@ -6,6 +6,7 @@ import {
extractVideoCodecString, extractVideoCodecString,
MediaCodec, MediaCodec,
parseAacAudioSpecificConfig, parseAacAudioSpecificConfig,
PCM_CODECS,
VideoCodec, VideoCodec,
Vp9CodecInfo, Vp9CodecInfo,
} from '../codec'; } from '../codec';
@@ -286,11 +287,9 @@ export class IsobmffDemuxer extends Demuxer {
this.traverseBox(); this.traverseBox();
this.currentTrack = null; this.currentTrack = null;
const isPcmCodec = internalTrack.info?.type === 'audio' && ( const isPcmCodec = internalTrack.info?.type === 'audio'
internalTrack.info.codec?.startsWith('pcm-') && internalTrack.info.codec
|| internalTrack.info.codec === 'ulaw' && (PCM_CODECS as readonly string[]).includes(internalTrack.info.codec);
|| internalTrack.info.codec === 'alaw'
);
if (isPcmCodec && sampleTable.sampleCompositionTimeOffsets.length === 0) { if (isPcmCodec && sampleTable.sampleCompositionTimeOffsets.length === 0) {
// If the audio has PCM samples, the way the samples are defined in the sample table is somewhat // If the audio has PCM samples, the way the samples are defined in the sample table is somewhat
@@ -316,7 +315,7 @@ export class IsobmffDemuxer extends Demuxer {
const startTimingEntryIndex = binarySearchLessOrEqual( const startTimingEntryIndex = binarySearchLessOrEqual(
sampleTable.sampleTimingEntries, sampleTable.sampleTimingEntries,
chunkEntry.startSampleIndex, startSampleIndex,
x => x.startIndex, x => x.startIndex,
); );
const startTimingEntry = sampleTable.sampleTimingEntries[startTimingEntryIndex]!; const startTimingEntry = sampleTable.sampleTimingEntries[startTimingEntryIndex]!;
@@ -339,7 +338,7 @@ export class IsobmffDemuxer extends Demuxer {
} else { } else {
// One sample for the entire chunk // One sample for the entire chunk
newSampleTimingEntries.push({ newSampleTimingEntries.push({
startIndex: chunkEntry.startChunkIndex, startIndex: chunkEntry.startChunkIndex + j,
startDecodeTimestamp: firstSampleTimestamp, startDecodeTimestamp: firstSampleTimestamp,
count: 1, count: 1,
delta, delta,
@@ -756,12 +755,14 @@ export class IsobmffDemuxer extends Demuxer {
if (stsdVersion === 0 && version > 0) { if (stsdVersion === 0 && version > 0) {
// Additional QuickTime fields // Additional QuickTime fields
if (version === 1) { if (version === 1) {
this.isobmffReader.pos += 4 * 4; this.isobmffReader.pos += 4;
sampleSize = 8 * this.isobmffReader.readU32();
this.isobmffReader.pos += 2 * 4;
} else if (version === 2) { } else if (version === 2) {
this.isobmffReader.pos += 4; this.isobmffReader.pos += 4;
sampleRate = this.isobmffReader.readF64(); sampleRate = this.isobmffReader.readF64();
channelCount = this.isobmffReader.readU32(); channelCount = this.isobmffReader.readU32();
this.isobmffReader.pos += 4; // Always 0x7F000000 this.isobmffReader.pos += 4; // Always 0x7f000000
sampleSize = this.isobmffReader.readU32(); sampleSize = this.isobmffReader.readU32();
@@ -777,7 +778,7 @@ export class IsobmffDemuxer extends Demuxer {
if (sampleSize > 0 && sampleSize <= 64) { if (sampleSize > 0 && sampleSize <= 64) {
if (isFloat) { if (isFloat) {
if (sampleSize === 32 && !isBigEndian) { if (sampleSize === 32) {
track.info.codec = isBigEndian ? 'pcm-f32be' : 'pcm-f32'; track.info.codec = isBigEndian ? 'pcm-f32be' : 'pcm-f32';
} }
} else { } else {
@@ -800,7 +801,7 @@ export class IsobmffDemuxer extends Demuxer {
} }
if (track.info.codec === null) { if (track.info.codec === null) {
console.warn('Unsupportedd PCM format.'); console.warn('Unsupported PCM format.');
} }
} }
} }
+95 -6
View File
@@ -3,10 +3,17 @@ import { Muxer } from '../muxer';
import { Output, OutputAudioTrack, OutputSubtitleTrack, OutputTrack, OutputVideoTrack } from '../output'; import { Output, OutputAudioTrack, OutputSubtitleTrack, OutputTrack, OutputVideoTrack } from '../output';
import { ArrayBufferTargetWriter, Writer } from '../writer'; import { ArrayBufferTargetWriter, Writer } from '../writer';
import { assert, last } from '../misc'; import { assert, last } from '../misc';
import { Mp4OutputFormat, Mp4OutputFormatOptions } from '../output-format'; import { IsobmffOutputFormatOptions, IsobmffOutputFormat, MovOutputFormat } from '../output-format';
import { inlineTimestampRegex, SubtitleConfig, SubtitleCue, SubtitleMetadata } from '../subtitles'; import { inlineTimestampRegex, SubtitleConfig, SubtitleCue, SubtitleMetadata } from '../subtitles';
import { ArrayBufferTarget } from '../target'; import { ArrayBufferTarget } from '../target';
import { validateAudioChunkMetadata, validateSubtitleMetadata, validateVideoChunkMetadata } from '../codec'; import {
parsePcmCodec,
PCM_CODECS,
PcmAudioCodec,
validateAudioChunkMetadata,
validateSubtitleMetadata,
validateVideoChunkMetadata,
} from '../codec';
import { EncodedAudioSample, EncodedVideoSample } from '../sample'; import { EncodedAudioSample, EncodedVideoSample } from '../sample';
export const GLOBAL_TIMESCALE = 1000; export const GLOBAL_TIMESCALE = 1000;
@@ -40,6 +47,11 @@ export type IsobmffTrackData = {
compositionTimeOffsetTable: { sampleCount: number; sampleCompositionTimeOffset: number }[]; compositionTimeOffsetTable: { sampleCount: number; sampleCompositionTimeOffset: number }[];
lastTimescaleUnits: number | null; lastTimescaleUnits: number | null;
lastSample: Sample | null; lastSample: Sample | null;
/**
* The "PCM transformation" is making every sample in the sample table be exactly one PCM audio sample long.
* Some players expect this for PCM audio.
*/
requiresPcmTransformation: boolean;
finalizedChunks: Chunk[]; finalizedChunks: Chunk[];
currentChunk: Chunk | null; currentChunk: Chunk | null;
@@ -87,7 +99,8 @@ export const intoTimescale = (timeInSeconds: number, timescale: number, round =
export class IsobmffMuxer extends Muxer { export class IsobmffMuxer extends Muxer {
private writer: Writer; private writer: Writer;
private boxWriter: IsobmffBoxWriter; private boxWriter: IsobmffBoxWriter;
private fastStart: NonNullable<Mp4OutputFormatOptions['fastStart']>; private isMov: boolean;
private fastStart: NonNullable<IsobmffOutputFormatOptions['fastStart']>;
private auxTarget = new ArrayBufferTarget(); private auxTarget = new ArrayBufferTarget();
private auxWriter = this.auxTarget._createWriter(); private auxWriter = this.auxTarget._createWriter();
@@ -103,12 +116,14 @@ export class IsobmffMuxer extends Muxer {
private nextFragmentNumber = 1; private nextFragmentNumber = 1;
constructor(output: Output, format: Mp4OutputFormat) { constructor(output: Output, format: IsobmffOutputFormat) {
super(output); super(output);
this.writer = output._writer; this.writer = output._writer;
this.boxWriter = new IsobmffBoxWriter(this.writer); this.boxWriter = new IsobmffBoxWriter(this.writer);
this.isMov = format instanceof MovOutputFormat;
// If the fastStart option isn't defined, enable in-memory fast start if the target is an ArrayBuffer, as the // If the fastStart option isn't defined, enable in-memory fast start if the target is an ArrayBuffer, as the
// memory usage remains identical // memory usage remains identical
const fastStartDefault = this.writer instanceof ArrayBufferTargetWriter ? 'in-memory' : false; const fastStartDefault = this.writer instanceof ArrayBufferTargetWriter ? 'in-memory' : false;
@@ -126,6 +141,7 @@ export class IsobmffMuxer extends Muxer {
// Write the header // Write the header
this.boxWriter.writeBox(ftyp({ this.boxWriter.writeBox(ftyp({
isMov: this.isMov,
holdsAvc: holdsAvc, holdsAvc: holdsAvc,
fragmented: this.fastStart === 'fragmented', fragmented: this.fastStart === 'fragmented',
})); }));
@@ -178,6 +194,7 @@ export class IsobmffMuxer extends Muxer {
finalizedChunks: [], finalizedChunks: [],
currentChunk: null, currentChunk: null,
compactlyCodedChunkTable: [], compactlyCodedChunkTable: [],
requiresPcmTransformation: false,
}; };
this.trackDatas.push(newTrackData); this.trackDatas.push(newTrackData);
@@ -216,6 +233,9 @@ export class IsobmffMuxer extends Muxer {
finalizedChunks: [], finalizedChunks: [],
currentChunk: null, currentChunk: null,
compactlyCodedChunkTable: [], compactlyCodedChunkTable: [],
requiresPcmTransformation:
this.fastStart !== 'fragmented'
&& (PCM_CODECS as readonly string[]).includes(track.source._codec),
}; };
this.trackDatas.push(newTrackData); this.trackDatas.push(newTrackData);
@@ -252,6 +272,8 @@ export class IsobmffMuxer extends Muxer {
finalizedChunks: [], finalizedChunks: [],
currentChunk: null, currentChunk: null,
compactlyCodedChunkTable: [], compactlyCodedChunkTable: [],
requiresPcmTransformation: false,
lastCueEndTimestamp: 0, lastCueEndTimestamp: 0,
cueQueue: [], cueQueue: [],
nextSourceId: 0, nextSourceId: 0,
@@ -308,12 +330,47 @@ export class IsobmffMuxer extends Muxer {
sample.type, sample.type,
); );
if (trackData.requiresPcmTransformation) {
await this.maybePadWithSilence(trackData, timestamp);
}
await this.registerSample(trackData, internalSample); await this.registerSample(trackData, internalSample);
} finally { } finally {
release(); release();
} }
} }
private async maybePadWithSilence(trackData: IsobmffAudioTrackData, untilTimestamp: number) {
// The PCM transformation assumes that all samples are contiguous. This is not something that is enforced, so
// we need to pad the "holes" in between samples (and before the first sample) with additional
// "silence samples".
const lastSample = last(trackData.samples);
const lastEndTimestamp = lastSample
? lastSample.timestamp + lastSample.duration
: 0;
const delta = untilTimestamp - lastEndTimestamp;
const deltaInTimescale = intoTimescale(delta, trackData.timescale);
if (deltaInTimescale > 0) {
const { sampleSize, silentValue } = parsePcmCodec(
trackData.info.decoderConfig.codec as PcmAudioCodec,
);
const samplesNeeded = deltaInTimescale * trackData.info.numberOfChannels;
const data = new Uint8Array(sampleSize * samplesNeeded).fill(silentValue);
const paddingSample = this.createSampleForTrack(
trackData,
new Uint8Array(data.buffer),
lastEndTimestamp,
delta,
'key',
);
await this.registerSample(trackData, paddingSample);
}
}
async addSubtitleCue(track: OutputSubtitleTrack, cue: SubtitleCue, meta?: SubtitleMetadata) { async addSubtitleCue(track: OutputSubtitleTrack, cue: SubtitleCue, meta?: SubtitleMetadata) {
const release = await this.mutex.acquire(); const release = await this.mutex.acquire();
@@ -451,6 +508,32 @@ export class IsobmffMuxer extends Muxer {
return; return;
} }
if (trackData.requiresPcmTransformation) {
let totalDuration = 0;
// Compute the total duration in the track timescale (which is equal to the amount of PCM audio samples)
// and simply say that's how many new samples there are.
for (let i = 0; i < trackData.timestampProcessingQueue.length; i++) {
const sample = trackData.timestampProcessingQueue[i]!;
const duration = intoTimescale(sample.duration, trackData.timescale);
totalDuration += duration;
}
if (trackData.timeToSampleTable.length === 0) {
trackData.timeToSampleTable.push({
sampleCount: totalDuration,
sampleDelta: 1,
});
} else {
const lastEntry = last(trackData.timeToSampleTable)!;
lastEntry.sampleCount += totalDuration;
}
trackData.timestampProcessingQueue.length = 0;
return;
}
const sortedTimestamps = trackData.timestampProcessingQueue.map(x => x.timestamp).sort((a, b) => a - b); const sortedTimestamps = trackData.timestampProcessingQueue.map(x => x.timestamp).sort((a, b) => a - b);
for (let i = 0; i < trackData.timestampProcessingQueue.length; i++) { for (let i = 0; i < trackData.timestampProcessingQueue.length; i++) {
@@ -628,13 +711,19 @@ export class IsobmffMuxer extends Muxer {
trackData.finalizedChunks.push(trackData.currentChunk); trackData.finalizedChunks.push(trackData.currentChunk);
this.finalizedChunks.push(trackData.currentChunk); this.finalizedChunks.push(trackData.currentChunk);
let sampleCount = trackData.currentChunk.samples.length;
if (trackData.requiresPcmTransformation) {
sampleCount = trackData.currentChunk.samples
.reduce((acc, sample) => acc + intoTimescale(sample.duration, trackData.timescale), 0);
}
if ( if (
trackData.compactlyCodedChunkTable.length === 0 trackData.compactlyCodedChunkTable.length === 0
|| last(trackData.compactlyCodedChunkTable)!.samplesPerChunk !== trackData.currentChunk.samples.length || last(trackData.compactlyCodedChunkTable)!.samplesPerChunk !== sampleCount
) { ) {
trackData.compactlyCodedChunkTable.push({ trackData.compactlyCodedChunkTable.push({
firstChunk: trackData.finalizedChunks.length, // 1-indexed firstChunk: trackData.finalizedChunks.length, // 1-indexed
samplesPerChunk: trackData.currentChunk.samples.length, samplesPerChunk: sampleCount,
}); });
} }
+113 -38
View File
@@ -1,4 +1,4 @@
import { PCM_CODECS, PcmAudioCodec } from './codec'; import { parsePcmCodec, PCM_CODECS, PcmAudioCodec } from './codec';
import { InputAudioTrack, InputVideoTrack } from './input-track'; import { InputAudioTrack, InputVideoTrack } from './input-track';
import { import {
AnyIterable, AnyIterable,
@@ -858,8 +858,6 @@ class AudioDecoderWrapper extends DecoderWrapper<EncodedAudioSample, AudioData>
} }
} }
const PCM_CODEC_REGEX = /^pcm-([usf])(\d+)+(be)?$/;
// There are a lot of PCM variants not natively supported by the browser and by AudioData. Therefore we need a simple // There are a lot of PCM variants not natively supported by the browser and by AudioData. Therefore we need a simple
// decoder that maps any input PCM format into a PCM format supported by the browser. // decoder that maps any input PCM format into a PCM format supported by the browser.
class PcmAudioDecoderWrapper extends DecoderWrapper<EncodedAudioSample, AudioData> { class PcmAudioDecoderWrapper extends DecoderWrapper<EncodedAudioSample, AudioData> {
@@ -882,41 +880,88 @@ class PcmAudioDecoderWrapper extends DecoderWrapper<EncodedAudioSample, AudioDat
assert((PCM_CODECS as readonly string[]).includes(decoderConfig.codec)); assert((PCM_CODECS as readonly string[]).includes(decoderConfig.codec));
this.codec = decoderConfig.codec as PcmAudioCodec; this.codec = decoderConfig.codec as PcmAudioCodec;
const match = this.codec.match(PCM_CODEC_REGEX); const { dataType, sampleSize, littleEndian } = parsePcmCodec(this.codec);
assert(match); this.inputSampleSize = sampleSize;
let dataType: 'unsigned' | 'signed' | 'float'; switch (sampleSize) {
if (match[1] === 'u') {
dataType = 'unsigned';
} else if (match[1] === 's') {
dataType = 'signed';
} else {
dataType = 'float';
}
this.inputSampleSize = (Number(match[2]) / 8) as 1 | 2 | 3 | 4;
const littleEndian = match[3] !== 'be';
switch (this.inputSampleSize) {
case 1: { case 1: {
if (dataType === 'unsigned') { if (dataType === 'unsigned') {
this.readInputValue = (view, byteOffset) => view.getUint8(byteOffset) - 2 ** 7; this.readInputValue = (view, byteOffset) => view.getUint8(byteOffset) - 2 ** 7;
} else { } else if (dataType === 'signed') {
this.readInputValue = (view, byteOffset) => view.getInt8(byteOffset); this.readInputValue = (view, byteOffset) => view.getInt8(byteOffset);
} else if (dataType === 'ulaw') {
// https://github.com/dystopiancode/pcm-g711/blob/master/pcm-g711/g711.c
this.readInputValue = (view, byteOffset) => {
const MULAW_BIAS = 33;
let sign = 0;
let position = 0;
// Get byte and invert
let number = ~view.getUint8(byteOffset);
// Handle sign
if (number & 0x80) {
number &= ~(1 << 7);
sign = -1;
}
// Calculate position
position = ((number & 0xF0) >> 4) + 5;
// Reconstruct linear value
const decoded = ((1 << position) | ((number & 0x0F) << (position - 4))
| (1 << (position - 5))) - MULAW_BIAS;
return (sign === 0) ? decoded : -decoded;
};
} else if (dataType === 'alaw') {
this.readInputValue = (view, byteOffset) => {
let sign = 0x00;
let position = 0;
// Get byte and XOR with 0x55
let number = view.getUint8(byteOffset) ^ 0x55;
// Handle sign
if (number & 0x80) {
number &= ~(1 << 7);
sign = -1;
}
// Calculate position
position = ((number & 0xF0) >> 4) + 4;
// Reconstruct linear value
let decoded = 0;
if (position !== 4) {
decoded = ((1 << position) | ((number & 0x0F) << (position - 4))
| (1 << (position - 5)));
} else {
decoded = (number << 1) | 1;
}
return (sign === 0) ? decoded : -decoded;
};
} else {
assert(false);
} }
}; break; }; break;
case 2: { case 2: {
if (dataType === 'unsigned') { if (dataType === 'unsigned') {
this.readInputValue = (view, byteOffset) => view.getUint16(byteOffset, littleEndian) - 2 ** 15; this.readInputValue = (view, byteOffset) => view.getUint16(byteOffset, littleEndian) - 2 ** 15;
} else { } else if (dataType === 'signed') {
this.readInputValue = (view, byteOffset) => view.getInt16(byteOffset, littleEndian); this.readInputValue = (view, byteOffset) => view.getInt16(byteOffset, littleEndian);
} else {
assert(false);
} }
}; break; }; break;
case 3: { case 3: {
if (dataType === 'unsigned') { if (dataType === 'unsigned') {
this.readInputValue = (view, byteOffset) => getUint24(view, byteOffset, littleEndian) - 2 ** 23; this.readInputValue = (view, byteOffset) => getUint24(view, byteOffset, littleEndian) - 2 ** 23;
} else { } else if (dataType === 'signed') {
this.readInputValue = (view, byteOffset) => getInt24(view, byteOffset, littleEndian); this.readInputValue = (view, byteOffset) => getInt24(view, byteOffset, littleEndian);
} else {
assert(false);
} }
}; break; }; break;
case 4: { case 4: {
@@ -924,17 +969,25 @@ class PcmAudioDecoderWrapper extends DecoderWrapper<EncodedAudioSample, AudioDat
this.readInputValue = (view, byteOffset) => view.getUint32(byteOffset, littleEndian) - 2 ** 31; this.readInputValue = (view, byteOffset) => view.getUint32(byteOffset, littleEndian) - 2 ** 31;
} else if (dataType === 'signed') { } else if (dataType === 'signed') {
this.readInputValue = (view, byteOffset) => view.getInt32(byteOffset, littleEndian); this.readInputValue = (view, byteOffset) => view.getInt32(byteOffset, littleEndian);
} else { } else if (dataType === 'float') {
this.readInputValue = (view, byteOffset) => view.getFloat32(byteOffset, littleEndian); this.readInputValue = (view, byteOffset) => view.getFloat32(byteOffset, littleEndian);
} else {
assert(false);
} }
}; break; }; break;
} }
switch (this.inputSampleSize) { switch (sampleSize) {
case 1: { case 1: {
this.outputSampleSize = 1; if (dataType === 'ulaw' || dataType === 'alaw') {
this.outputFormat = 'u8'; this.outputSampleSize = 2;
this.writeOutputValue = (view, byteOffset, value) => view.setUint8(byteOffset, value + 2 ** 7); this.outputFormat = 's16';
this.writeOutputValue = (view, byteOffset, value) => view.setInt16(byteOffset, value, true);
} else {
this.outputSampleSize = 1;
this.outputFormat = 'u8';
this.writeOutputValue = (view, byteOffset, value) => view.setUint8(byteOffset, value + 2 ** 7);
}
}; break; }; break;
case 2: { case 2: {
this.outputSampleSize = 2; this.outputSampleSize = 2;
@@ -1010,6 +1063,13 @@ class PcmAudioDecoderWrapper extends DecoderWrapper<EncodedAudioSample, AudioDat
} }
} }
/** @public */
export type WrappedAudioData = {
data: AudioData;
timestamp: number;
duration: number;
};
/** @public */ /** @public */
export class AudioDataDrain extends BaseMediaFrameDrain<EncodedAudioSample, AudioData> { export class AudioDataDrain extends BaseMediaFrameDrain<EncodedAudioSample, AudioData> {
/** @internal */ /** @internal */
@@ -1040,13 +1100,22 @@ export class AudioDataDrain extends BaseMediaFrameDrain<EncodedAudioSample, Audi
const decoderConfig = await this._audioTrack.getDecoderConfig(); const decoderConfig = await this._audioTrack.getDecoderConfig();
assert(decoderConfig); assert(decoderConfig);
if (decoderConfig.codec.startsWith('pcm-')) { if ((PCM_CODECS as readonly string[]).includes(decoderConfig.codec)) {
return new PcmAudioDecoderWrapper(onData, onError, decoderConfig); return new PcmAudioDecoderWrapper(onData, onError, decoderConfig);
} else { } else {
return new AudioDecoderWrapper(onData, onError, decoderConfig); return new AudioDecoderWrapper(onData, onError, decoderConfig);
} }
} }
/** @internal */
_wrappedFrameToWrappedAudioData(frame: WrappedMediaFrame<AudioData>): WrappedAudioData {
return {
data: frame.frame,
timestamp: frame.timestamp,
duration: frame.duration,
};
}
/** @internal */ /** @internal */
_createSampleDrain() { _createSampleDrain() {
return new EncodedAudioSampleDrain(this._audioTrack); return new EncodedAudioSampleDrain(this._audioTrack);
@@ -1056,17 +1125,23 @@ export class AudioDataDrain extends BaseMediaFrameDrain<EncodedAudioSample, Audi
validateTimestamp(timestamp); validateTimestamp(timestamp);
for await (const data of this.mediaFramesAtTimestamps([timestamp])) { for await (const data of this.mediaFramesAtTimestamps([timestamp])) {
return data; return data && this._wrappedFrameToWrappedAudioData(data);
} }
throw new Error('Internal error: Iterator returned nothing.'); throw new Error('Internal error: Iterator returned nothing.');
} }
data(startTimestamp = 0, endTimestamp = Infinity) { data(startTimestamp = 0, endTimestamp = Infinity) {
return this.mediaFramesInRange(startTimestamp, endTimestamp); return mapAsyncGenerator(
this.mediaFramesInRange(startTimestamp, endTimestamp),
async data => this._wrappedFrameToWrappedAudioData(data),
);
} }
dataAtTimestamps(timestamps: AnyIterable<number>) { dataAtTimestamps(timestamps: AnyIterable<number>) {
return this.mediaFramesAtTimestamps(timestamps); return mapAsyncGenerator(
this.mediaFramesAtTimestamps(timestamps),
async data => data && this._wrappedFrameToWrappedAudioData(data),
);
} }
} }
@@ -1091,18 +1166,18 @@ export class AudioBufferDrain {
} }
/** @internal */ /** @internal */
_audioDataToWrappedArrayBuffer(data: WrappedMediaFrame<AudioData>): WrappedAudioBuffer { _audioDataToWrappedArrayBuffer(data: WrappedAudioData): WrappedAudioBuffer {
const audioBuffer = new AudioBuffer({ const audioBuffer = new AudioBuffer({
numberOfChannels: data.frame.numberOfChannels, numberOfChannels: data.data.numberOfChannels,
length: data.frame.numberOfFrames, length: data.data.numberOfFrames,
sampleRate: data.frame.sampleRate, sampleRate: data.data.sampleRate,
}); });
// All user agents are required to support conversion to f32-planar // All user agents are required to support conversion to f32-planar
const dataBytes = new Float32Array(data.frame.allocationSize({ planeIndex: 0, format: 'f32-planar' }) / 4); const dataBytes = new Float32Array(data.data.allocationSize({ planeIndex: 0, format: 'f32-planar' }) / 4);
for (let i = 0; i < data.frame.numberOfChannels; i++) { for (let i = 0; i < data.data.numberOfChannels; i++) {
data.frame.copyTo(dataBytes, { planeIndex: i, format: 'f32-planar' }); data.data.copyTo(dataBytes, { planeIndex: i, format: 'f32-planar' });
audioBuffer.copyToChannel(dataBytes, i); audioBuffer.copyToChannel(dataBytes, i);
} }
@@ -1112,7 +1187,7 @@ export class AudioBufferDrain {
duration: data.duration, duration: data.duration,
}; };
data.frame.close(); data.data.close();
return result; return result;
} }
+6 -2
View File
@@ -5,6 +5,7 @@ import {
buildVideoCodecString, buildVideoCodecString,
getAudioEncoderConfigExtension, getAudioEncoderConfigExtension,
getVideoEncoderConfigExtension, getVideoEncoderConfigExtension,
PCM_CODECS,
SUBTITLE_CODECS, SUBTITLE_CODECS,
SubtitleCodec, SubtitleCodec,
VIDEO_CODECS, VIDEO_CODECS,
@@ -418,7 +419,7 @@ export class EncodedAudioSampleSource extends AudioSource {
/** @public */ /** @public */
export type AudioEncodingConfig = { export type AudioEncodingConfig = {
codec: AudioCodec; codec: AudioCodec;
bitrate: number; bitrate?: number;
onEncodedChunk?: (chunk: EncodedAudioChunk, meta: EncodedAudioChunkMetadata | undefined) => unknown; onEncodedChunk?: (chunk: EncodedAudioChunk, meta: EncodedAudioChunkMetadata | undefined) => unknown;
onEncodingError?: (error: Error) => unknown; onEncodingError?: (error: Error) => unknown;
}; };
@@ -430,7 +431,10 @@ const validateAudioEncodingConfig = (config: AudioEncodingConfig) => {
if (!AUDIO_CODECS.includes(config.codec)) { if (!AUDIO_CODECS.includes(config.codec)) {
throw new TypeError(`Invalid audio codec '${config.codec}'. Must be one of: ${AUDIO_CODECS.join(', ')}.`); throw new TypeError(`Invalid audio codec '${config.codec}'. Must be one of: ${AUDIO_CODECS.join(', ')}.`);
} }
if (!Number.isInteger(config.bitrate) || config.bitrate <= 0) { if (config.bitrate === undefined && !(PCM_CODECS as readonly string[]).includes(config.codec)) {
throw new TypeError('config.bitrate must be provided for compressed audio codecs.');
}
if (config.bitrate !== undefined && (!Number.isInteger(config.bitrate) || config.bitrate <= 0)) {
throw new TypeError('config.bitrate must be a positive integer.'); throw new TypeError('config.bitrate must be a positive integer.');
} }
if (config.onEncodingError !== undefined && typeof config.onEncodingError !== 'function') { if (config.onEncodingError !== undefined && typeof config.onEncodingError !== 'function') {
+50 -10
View File
@@ -1,4 +1,4 @@
import { AUDIO_CODECS, MediaCodec, SUBTITLE_CODECS, VIDEO_CODECS } from './codec'; import { AUDIO_CODECS, MediaCodec, PCM_CODECS, SUBTITLE_CODECS, VIDEO_CODECS } from './codec';
import { IsobmffMuxer } from './isobmff/isobmff-muxer'; import { IsobmffMuxer } from './isobmff/isobmff-muxer';
import { MatroskaMuxer } from './matroska/matroska-muxer'; import { MatroskaMuxer } from './matroska/matroska-muxer';
import { Muxer } from './muxer'; import { Muxer } from './muxer';
@@ -33,16 +33,16 @@ export abstract class OutputFormat {
} }
/** @public */ /** @public */
export type Mp4OutputFormatOptions = { export type IsobmffOutputFormatOptions = {
fastStart?: false | 'in-memory' | 'fragmented'; fastStart?: false | 'in-memory' | 'fragmented';
}; };
/** @public */ /** @public */
export class Mp4OutputFormat extends OutputFormat { export abstract class IsobmffOutputFormat extends OutputFormat {
/** @internal */ /** @internal */
_options: Mp4OutputFormatOptions; _options: IsobmffOutputFormatOptions;
constructor(options: Mp4OutputFormatOptions = {}) { constructor(options: IsobmffOutputFormatOptions = {}) {
if (!options || typeof options !== 'object') { if (!options || typeof options !== 'object') {
throw new TypeError('options must be an object.'); throw new TypeError('options must be an object.');
} }
@@ -56,10 +56,13 @@ export class Mp4OutputFormat extends OutputFormat {
} }
/** @internal */ /** @internal */
override _createMuxer(output: Output) { _createMuxer(output: Output) {
return new IsobmffMuxer(output, this); return new IsobmffMuxer(output, this);
} }
}
/** @public */
export class Mp4OutputFormat extends IsobmffOutputFormat {
/** @internal */ /** @internal */
_getName() { _getName() {
return 'MP4'; return 'MP4';
@@ -72,10 +75,47 @@ export class Mp4OutputFormat extends OutputFormat {
static getSupportedCodecs(): MediaCodec[] { static getSupportedCodecs(): MediaCodec[] {
return [ return [
'avc', 'hevc', 'vp8', 'vp9', 'av1', 'avc', 'hevc', 'vp8', 'vp9', 'av1',
'aac', 'opus', 'aac', 'mp3', 'opus', 'vorbis', 'flac', // No PCM codecs
'webvtt', 'webvtt',
]; ];
} }
/** @internal */
override _codecUnsupportedHint(codec: MediaCodec) {
if (MovOutputFormat.getSupportedCodecs().includes(codec)) {
return ' Switching to MOV will grant support for this codec.';
}
return '';
}
}
/** @public */
export class MovOutputFormat extends IsobmffOutputFormat {
/** @internal */
_getName() {
return 'MOV';
}
getSupportedCodecs() {
return MovOutputFormat.getSupportedCodecs();
}
static getSupportedCodecs(): MediaCodec[] {
return [
'avc', 'hevc', 'vp8', 'vp9', 'av1',
'aac', 'mp3', 'opus', 'vorbis', 'flac', ...PCM_CODECS,
];
}
/** @internal */
override _codecUnsupportedHint(codec: MediaCodec) {
if (Mp4OutputFormat.getSupportedCodecs().includes(codec)) {
return ' Switching to MP4 will grant support for this codec.';
}
return '';
}
} }
/** @public */ /** @public */
@@ -102,7 +142,7 @@ export class MkvOutputFormat extends OutputFormat {
} }
/** @internal */ /** @internal */
override _createMuxer(output: Output) { _createMuxer(output: Output) {
return new MatroskaMuxer(output, this); return new MatroskaMuxer(output, this);
} }
@@ -150,8 +190,8 @@ export class WebMOutputFormat extends MkvOutputFormat {
override _codecUnsupportedHint(codec: MediaCodec) { override _codecUnsupportedHint(codec: MediaCodec) {
if (MkvOutputFormat.getSupportedCodecs().includes(codec)) { if (MkvOutputFormat.getSupportedCodecs().includes(codec)) {
return ' Switching to MKV will grant support for this codec.'; return ' Switching to MKV will grant support for this codec.';
} else {
return '';
} }
return '';
} }
} }