diff --git a/dev/demux.html b/dev/demux.html index 46a85ea..8e00dc2 100644 --- a/dev/demux.html +++ b/dev/demux.html @@ -40,35 +40,49 @@ source }); + //const videoTrack = await input.getPrimaryVideoTrack(); + //console.log(await videoTrack.computeSampleStats()); + //return; + const target = new Metamuxer.ArrayBufferTarget(); const output = new Metamuxer.Output({ - format: new Metamuxer.Mp4OutputFormat({ fastStart: undefined }), + format: new Metamuxer.MovOutputFormat({ fastStart: undefined }), target }); + const audioTrack = await input.getPrimaryAudioTrack(); const videoTrack = await input.getPrimaryVideoTrack(); - const drain = new Metamuxer.EncodedVideoSampleDrain(videoTrack); + + const drain = new Metamuxer.EncodedAudioSampleDrain(audioTrack); - console.log(await videoTrack.computeSampleStats()) - document.body.textContent = performance.now() - start; - - const decoderConfig = await videoTrack.getDecoderConfig(); - const sampleSource = new Metamuxer.EncodedVideoSampleSource(await videoTrack.getCodec()); - output.addVideoTrack(sampleSource); + const decoderConfig = await audioTrack.getDecoderConfig(); + const sampleSource = new Metamuxer.EncodedAudioSampleSource(await audioTrack.getCodec()); + const audioDataSource = new Metamuxer.AudioDataSource({ codec: 'ulaw' }); + const videoSampleSource = new Metamuxer.EncodedVideoSampleSource(await videoTrack.getCodec()); + output.addAudioTrack(audioDataSource ?? sampleSource); + output.addVideoTrack(videoSampleSource); output.start(); - let j = 0; - for await (const sample of drain.samples(undefined, undefined)) { - await sampleSource.digest(sample, { decoderConfig }); + const videoDecoderConfig = await videoTrack.getDecoderConfig(); - if (j++ < 12) { - console.log(sample); - } + /* + for await (const sample of drain.samples()) { + //sample.timestamp *= 2; + //console.log(sample); + //console.log(sample) + await sampleSource.digest(sample, { decoderConfig }); + }*/ + for await (const { data } of new Metamuxer.AudioDataDrain(audioTrack).data()) { + //console.log(sample) + await audioDataSource.digest(data); + } + for await (const sample of new Metamuxer.EncodedVideoSampleDrain(videoTrack).samples()) { + //console.log(sample) + await videoSampleSource.digest(sample, { decoderConfig: videoDecoderConfig }); } await output.finalize(); - console.log(target) @@ -80,8 +94,11 @@ a.click(); URL.revokeObjectURL(url); } - download(new Blob([target.buffer]), 'converted.mp4'); + download(new Blob([target.buffer]), 'converted.mov'); + document.body.textContent = performance.now() - start; + + /* const input2 = new Metamuxer.Input({ formats: Metamuxer.ALL_FORMATS, source: new Metamuxer.BufferSource(target.buffer) @@ -97,6 +114,7 @@ break; } } + */ diff --git a/src/codec.ts b/src/codec.ts index 2fef35f..a91abb3 100644 --- a/src/codec.ts +++ b/src/codec.ts @@ -30,6 +30,8 @@ export const PCM_CODECS = [ 'pcm-s32be', 'pcm-f32', 'pcm-f32be', + 'ulaw', + 'alaw', ] as const; /** @public */ export const AUDIO_CODECS = [ @@ -38,8 +40,6 @@ export const AUDIO_CODECS = [ 'opus', 'vorbis', 'flac', - 'ulaw', - 'alaw', ...PCM_CODECS, ] as const; // TODO add the rest /** @public */ @@ -495,6 +495,35 @@ export const parseAacAudioSpecificConfig = (bytes: Uint8Array | null) => { }; }; +const PCM_CODEC_REGEX = /^pcm-([usf])(\d+)+(be)?$/; +export const parsePcmCodec = (codec: PcmAudioCodec) => { + assert(PCM_CODECS.includes(codec)); + + if (codec === 'ulaw') { + return { dataType: 'ulaw' as const, sampleSize: 1 as const, littleEndian: true, silentValue: 255 }; + } else if (codec === 'alaw') { + return { dataType: 'alaw' as const, sampleSize: 1 as const, littleEndian: true, silentValue: 213 }; + } + + const match = PCM_CODEC_REGEX.exec(codec); + assert(match); + + let dataType: 'unsigned' | 'signed' | 'float' | 'ulaw' | 'alaw'; + if (match[1] === 'u') { + dataType = 'unsigned'; + } else if (match[1] === 's') { + dataType = 'signed'; + } else { + dataType = 'float'; + } + + const sampleSize = (Number(match[2]) / 8) as 1 | 2 | 3 | 4; + const littleEndian = match[3] !== 'be'; + const silentValue = codec === 'pcm-u8' ? 2 ** 7 : 0; + + return { dataType, sampleSize, littleEndian, silentValue }; +}; + export const getVideoEncoderConfigExtension = (codec: VideoCodec) => { if (codec === 'avc') { return { @@ -753,10 +782,11 @@ export const validateAudioChunkMetadata = (metadata: EncodedAudioChunkMetadata | throw new TypeError('Audio chunk metadata decoder configuration codec string for Opus must be "opus".'); } - if (!metadata.decoderConfig.description) { + if (metadata.decoderConfig.description && metadata.decoderConfig.description.byteLength < 18) { + // Description is optional for Opus per-spec, so we shouldn't enforce it throw new TypeError( - 'Audio chunk metadata decoder configuration for Opus must include a description, which is expected to' - + ' be an Identification Header as specified in Section 5.1 of RFC 7845.', + 'Audio chunk metadata decoder configuration description, when specified, is expected to be an' + + ' Identification Header as specified in Section 5.1 of RFC 7845.', ); } } @@ -781,7 +811,8 @@ export const validateAudioChunkMetadata = (metadata: EncodedAudioChunkMetadata | throw new TypeError('Audio chunk metadata decoder configuration codec string for FLAC must be "flac".'); } - if (!metadata.decoderConfig.description) { + const minDescriptionSize = 4 + 4 + 34; // 'fLaC' + metadata block header + STREAMINFO block + if (!metadata.decoderConfig.description || metadata.decoderConfig.description.byteLength < minDescriptionSize) { throw new TypeError( 'Audio chunk metadata decoder configuration for FLAC must include a description, which is expected to' + ' adhere to the format described in https://www.w3.org/TR/webcodecs-flac-codec-registration/.', @@ -789,22 +820,12 @@ export const validateAudioChunkMetadata = (metadata: EncodedAudioChunkMetadata | } } - // ulaw-specific validation - if (metadata.decoderConfig.codec === 'ulaw') { - if (metadata.decoderConfig.codec !== 'ulaw') { - throw new TypeError('Audio chunk metadata decoder configuration codec string for uLaw must be "ulaw".'); - } - } - - // alaw-specific validation - if (metadata.decoderConfig.codec === 'alaw') { - if (metadata.decoderConfig.codec !== 'alaw') { - throw new TypeError('Audio chunk metadata decoder configuration codec string for A-law must be "alaw".'); - } - } - // PCM-specific validation - if (metadata.decoderConfig.codec.startsWith('pcm')) { + if ( + metadata.decoderConfig.codec.startsWith('pcm') + || metadata.decoderConfig.codec.startsWith('ulaw') + || metadata.decoderConfig.codec.startsWith('alaw') + ) { if (!(PCM_CODECS as readonly string[]).includes(metadata.decoderConfig.codec)) { throw new TypeError( 'Audio chunk metadata decoder configuration codec string for PCM must be one of the supported PCM' diff --git a/src/index.ts b/src/index.ts index 5d6ef4e..857fe05 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,8 +1,10 @@ export { Output, OutputOptions, VideoTrackMetadata, AudioTrackMetadata, SubtitleTrackMetadata } from './output'; export { OutputFormat, + IsobmffOutputFormat, Mp4OutputFormat, - Mp4OutputFormatOptions, + MovOutputFormat, + IsobmffOutputFormatOptions, MkvOutputFormat, MkvOutputFormatOptions, WebMOutputFormat, @@ -75,6 +77,7 @@ export { CanvasDrain, WrappedCanvas, AudioDataDrain, + WrappedAudioData, AudioBufferDrain, WrappedAudioBuffer, } from './media-drain'; diff --git a/src/input-track.ts b/src/input-track.ts index 8522ebb..157d492 100644 --- a/src/input-track.ts +++ b/src/input-track.ts @@ -212,7 +212,11 @@ const computeSampleStats = async (drain: EncodedVideoSampleDrain | EncodedAudioS return { sampleCount, - averageSampleRate: sampleCount ? Math.fround(sampleCount / (endTimestamp - startTimestamp)) : 0, - averageBitrate: sampleCount ? Math.fround(8 * totalSampleBytes / (endTimestamp - startTimestamp)) : 0, + averageSampleRate: sampleCount + ? Number((sampleCount / (endTimestamp - startTimestamp)).toPrecision(16)) + : 0, + averageBitrate: sampleCount + ? Number((8 * totalSampleBytes / (endTimestamp - startTimestamp)).toPrecision(16)) + : 0, }; }; diff --git a/src/isobmff/isobmff-boxes.ts b/src/isobmff/isobmff-boxes.ts index 5dead68..48ef072 100644 --- a/src/isobmff/isobmff-boxes.ts +++ b/src/isobmff/isobmff-boxes.ts @@ -12,7 +12,7 @@ import { IDENTITY_MATRIX, rotationMatrix, } from '../misc'; -import { AudioCodec, SubtitleCodec, VideoCodec } from '../codec'; +import { AudioCodec, parsePcmCodec, PCM_CODECS, PcmAudioCodec, SubtitleCodec, VideoCodec } from '../codec'; import { formatSubtitleTimestamp } from '../subtitles'; import { Writer } from '../writer'; import { @@ -258,6 +258,7 @@ export const fullBox = ( * reader understands. */ export const ftyp = (details: { + isMov: boolean; holdsAvc: boolean; fragmented: boolean; }) => { @@ -267,6 +268,15 @@ export const ftyp = (details: { const minorVersion = 0x200; + if (details.isMov) { + return box('ftyp', [ + ascii('qt '), // Major brand + u32(minorVersion), // Minor version + // Compatible brands + ascii('qt '), + ]); + } + if (details.fragmented) return box('ftyp', [ ascii('iso5'), // Major brand u32(minorVersion), // Minor version @@ -688,45 +698,101 @@ export const av1C = (trackData: IsobmffVideoTrackData) => { export const soundSampleDescription = ( compressionType: string, trackData: IsobmffAudioTrackData, -) => box(compressionType, [ - Array(6).fill(0), // Reserved - u16(1), // Data reference index - u16(0), // Version (AudioSampleEntry, not AudioSampleEntryV1) - u16(0), // Revision level - u32(0), // Vendor - u16(trackData.info.numberOfChannels), // Number of channels - u16(16), // Sample size (bits) - u16(0), // Compression ID - u16(0), // Packet size - u32(2 ** 16 * trackData.info.sampleRate), // Sample rate -], [ - AUDIO_CODEC_TO_CONFIGURATION_BOX[trackData.track.source._codec]!(trackData), -]); +) => { + let version = 0; + let contents: NestedNumberArray; + + let sampleSizeInBits = 16; + if ((PCM_CODECS as readonly AudioCodec[]).includes(trackData.track.source._codec)) { + const codec = trackData.track.source._codec as PcmAudioCodec; + const { sampleSize } = parsePcmCodec(codec); + sampleSizeInBits = 8 * sampleSize; + + if (sampleSizeInBits > 16) { + version = 1; + } + } + + if (version === 0) { + contents = [ + Array(6).fill(0), // Reserved + u16(1), // Data reference index + u16(version), // Version + u16(0), // Revision level + u32(0), // Vendor + u16(trackData.info.numberOfChannels), // Number of channels + u16(sampleSizeInBits), // Sample size (bits) + u16(0), // Compression ID + u16(0), // Packet size + u16(trackData.info.sampleRate < 2 ** 16 ? trackData.info.sampleRate : 0), // Sample rate (upper) + u16(0), // Sample rate (lower) + ]; + } else { + contents = [ + Array(6).fill(0), // Reserved + u16(1), // Data reference index + u16(version), // Version + u16(0), // Revision level + u32(0), // Vendor + u16(trackData.info.numberOfChannels), // Number of channels + u16(Math.min(sampleSizeInBits, 16)), // Sample size (bits) + u16(0), // Compression ID + u16(0), // Packet size + u16(trackData.info.sampleRate < 2 ** 16 ? trackData.info.sampleRate : 0), // Sample rate (upper) + u16(0), // Sample rate (lower) + u32(1), // Samples per packet (must be 1 for uncompressed formats) + u32(sampleSizeInBits / 8), // Bytes per packet + u32(trackData.info.numberOfChannels * sampleSizeInBits / 8), // Bytes per frame + u32(2), // Bytes per sample (constant in FFmpeg) + ]; + } + + return box(compressionType, contents, [ + AUDIO_CODEC_TO_CONFIGURATION_BOX[trackData.track.source._codec]?.(trackData) ?? null, + ]); +}; /** MPEG-4 Elementary Stream Descriptor Box. */ export const esds = (trackData: IsobmffAudioTrackData) => { - const description = toUint8Array(trackData.info.decoderConfig.description ?? new ArrayBuffer(0)); - - // Adapted from https://stackoverflow.com/a/54803118 // We build up the bytes in a layered way which reflects the nested structure + let objectTypeIndication: number; + switch (trackData.track.source._codec) { + case 'aac': { + objectTypeIndication = 0x40; + }; break; + case 'mp3': { + objectTypeIndication = 0x6b; + }; break; + case 'vorbis': { + objectTypeIndication = 0xdd; + }; break; + default: throw new Error(`Unhandled audio codec: ${trackData.track.source._codec}`); + } + let bytes = [ - ...description, - ]; - bytes = [ - ...u8(0x40), // MPEG-4 Audio + ...u8(objectTypeIndication), // Object type indication ...u8(0x15), // stream type(6bits)=5 audio, flags(2bits)=1 ...u24(0), // 24bit buffer size ...u32(0), // max bitrate ...u32(0), // avg bitrate - ...u8(0x05), // TAG(5) = ASC ([2],[3]) embedded in above OD - ...variableUnsignedInt(bytes.length), - ...bytes, ]; + if (trackData.info.decoderConfig.description) { + const description = toUint8Array(trackData.info.decoderConfig.description); + + // Add the decoder description to the end + bytes = [ + ...bytes, + ...u8(0x05), // TAG(5) = DecoderSpecificInfo + ...variableUnsignedInt(description.byteLength), + ...description, + ]; + } + bytes = [ ...u16(1), // ES_ID = 1 ...u8(0x00), // flags etc = 0 - ...u8(0x04), // TAG(4) = ES Descriptor ([2]) embedded in above OD + ...u8(0x04), // TAG(4) = ES Descriptor ...variableUnsignedInt(bytes.length), ...bytes, ...u8(0x06), // TAG(6) @@ -734,7 +800,7 @@ export const esds = (trackData: IsobmffAudioTrackData) => { ...u8(0x02), // data ]; bytes = [ - ...u8(0x03), // TAG(3) = Object Descriptor ([2]) + ...u8(0x03), // TAG(3) = Object Descriptor ...variableUnsignedInt(bytes.length), ...bytes, ]; @@ -742,35 +808,82 @@ export const esds = (trackData: IsobmffAudioTrackData) => { return fullBox('esds', 0, 0, bytes); }; +export const wave = (trackData: IsobmffAudioTrackData) => { + return box('wave', undefined, [ + frma(trackData), + enda(trackData), + box('\x00\x00\x00\x00'), // NULL tag at the end + ]); +}; + +export const frma = (trackData: IsobmffAudioTrackData) => { + return box('frma', [ + ascii(AUDIO_CODEC_TO_BOX_NAME[trackData.track.source._codec]!), + ]); +}; + +// This box specifies PCM endianness +export const enda = (trackData: IsobmffAudioTrackData) => { + const { littleEndian } = parsePcmCodec(trackData.track.source._codec as PcmAudioCodec); + + return box('enda', [ + u16(+littleEndian), + ]); +}; + /** Opus Specific Box. */ export const dOps = (trackData: IsobmffAudioTrackData) => { + let outputChannelCount = trackData.info.numberOfChannels; // Default PreSkip, should be at least 80 milliseconds worth of playback, measured in 48000 Hz samples - let preskip = 3840; - let gain = 0; + let preSkip = 3840; + let inputSampleRate = trackData.info.sampleRate; + let outputGain = 0; + let channelMappingFamily = 0; + let channelMappingTable = new Uint8Array(0); // Read preskip and from codec private data from the encoder // https://www.rfc-editor.org/rfc/rfc7845#section-5 const description = trackData.info.decoderConfig?.description; if (description) { - if (description.byteLength < 18) { - throw new Error('Opus decoder description is too short (must be at least 18 bytes).'); - } + assert(description.byteLength >= 18); const view = ArrayBuffer.isView(description) ? new DataView(description.buffer, description.byteOffset, description.byteLength) : new DataView(description); - preskip = view.getUint16(10, true); - gain = view.getInt16(14, true); + + outputChannelCount = view.getUint8(9); + preSkip = view.getUint16(10, true); + inputSampleRate = view.getUint32(12, true); + outputGain = view.getInt16(16, true); + channelMappingFamily = view.getUint8(18); + + if (channelMappingFamily) { + const bytes = toUint8Array(description); + channelMappingTable = bytes.subarray(19, 19 + 2 + outputChannelCount); + } } // https://www.opus-codec.org/docs/opus_in_isobmff.html return box('dOps', [ u8(0), // Version - u8(trackData.info.numberOfChannels), // OutputChannelCount - u16(preskip), - u32(trackData.info.sampleRate), // InputSampleRate - fixed_8_8(gain), // OutputGain - u8(0), // ChannelMappingFamily + u8(outputChannelCount), // OutputChannelCount + u16(preSkip), // PreSkip + u32(inputSampleRate), // InputSampleRate + i16(outputGain), // OutputGain + u8(channelMappingFamily), // ChannelMappingFamily + ...channelMappingTable, + ]); +}; + +/** FLAC specific box. */ +export const dfLa = (trackData: IsobmffAudioTrackData) => { + const description = trackData.info.decoderConfig?.description; + assert(description); + + const bytes = toUint8Array(description); + + return fullBox('dfLa', 0, 0, [ + ...bytes.subarray(4), ]); }; @@ -836,11 +949,24 @@ export const stsc = (trackData: IsobmffTrackData) => { }; /** Sample Size Box: Specifies the byte size of each sample in the media. */ -export const stsz = (trackData: IsobmffTrackData) => fullBox('stsz', 0, 0, [ - u32(0), // Sample size (0 means non-constant size) - u32(trackData.samples.length), // Number of entries - trackData.samples.map(x => u32(x.size)), // Sample size table -]); +export const stsz = (trackData: IsobmffTrackData) => { + if (trackData.requiresPcmTransformation) { + assert(trackData.type === 'audio'); + const { sampleSize } = parsePcmCodec(trackData.track.source._codec as PcmAudioCodec); + + // With PCM, every sample has the same size + return fullBox('stsz', 0, 0, [ + u32(sampleSize * trackData.info.numberOfChannels), // Sample size + u32(trackData.samples.reduce((acc, x) => acc + intoTimescale(x.duration, trackData.timescale), 0)), + ]); + } + + return fullBox('stsz', 0, 0, [ + u32(0), // Sample size (0 means non-constant size) + u32(trackData.samples.length), // Number of entries + trackData.samples.map(x => u32(x.size)), // Sample size table + ]); +}; /** Chunk Offset Box: Identifies the location of each chunk of data in the media's data stream, relative to the file. */ export const stco = (trackData: IsobmffTrackData) => { @@ -1134,15 +1260,39 @@ const VIDEO_CODEC_TO_CONFIGURATION_BOX: Record> = { - aac: 'mp4a', - opus: 'Opus', + 'aac': 'mp4a', + 'mp3': 'mp4a', + 'opus': 'Opus', + 'vorbis': 'mp4a', + 'flac': 'fLaC', + 'ulaw': 'ulaw', + 'alaw': 'alaw', + 'pcm-u8': 'raw ', + 'pcm-s8': 'sowt', + 'pcm-s16': 'sowt', + 'pcm-s16be': 'twos', + 'pcm-s24': 'in24', + 'pcm-s24be': 'in24', + 'pcm-s32': 'in32', + 'pcm-s32be': 'in32', + 'pcm-f32': 'fl32', + 'pcm-f32be': 'fl32', }; const AUDIO_CODEC_TO_CONFIGURATION_BOX: Partial< Record Box | null> > = { - aac: esds, - opus: dOps, + 'aac': esds, + 'mp3': esds, + 'opus': dOps, + 'vorbis': esds, + 'flac': dfLa, + 'pcm-s24': wave, + 'pcm-s24be': wave, + 'pcm-s32': wave, + 'pcm-s32be': wave, + 'pcm-f32': wave, + 'pcm-f32be': wave, }; const SUBTITLE_CODEC_TO_BOX_NAME: Record = { diff --git a/src/isobmff/isobmff-demuxer.ts b/src/isobmff/isobmff-demuxer.ts index 4a2c90c..4a9ac11 100644 --- a/src/isobmff/isobmff-demuxer.ts +++ b/src/isobmff/isobmff-demuxer.ts @@ -6,6 +6,7 @@ import { extractVideoCodecString, MediaCodec, parseAacAudioSpecificConfig, + PCM_CODECS, VideoCodec, Vp9CodecInfo, } from '../codec'; @@ -286,11 +287,9 @@ export class IsobmffDemuxer extends Demuxer { this.traverseBox(); this.currentTrack = null; - const isPcmCodec = internalTrack.info?.type === 'audio' && ( - internalTrack.info.codec?.startsWith('pcm-') - || internalTrack.info.codec === 'ulaw' - || internalTrack.info.codec === 'alaw' - ); + const isPcmCodec = internalTrack.info?.type === 'audio' + && internalTrack.info.codec + && (PCM_CODECS as readonly string[]).includes(internalTrack.info.codec); if (isPcmCodec && sampleTable.sampleCompositionTimeOffsets.length === 0) { // If the audio has PCM samples, the way the samples are defined in the sample table is somewhat @@ -316,7 +315,7 @@ export class IsobmffDemuxer extends Demuxer { const startTimingEntryIndex = binarySearchLessOrEqual( sampleTable.sampleTimingEntries, - chunkEntry.startSampleIndex, + startSampleIndex, x => x.startIndex, ); const startTimingEntry = sampleTable.sampleTimingEntries[startTimingEntryIndex]!; @@ -339,7 +338,7 @@ export class IsobmffDemuxer extends Demuxer { } else { // One sample for the entire chunk newSampleTimingEntries.push({ - startIndex: chunkEntry.startChunkIndex, + startIndex: chunkEntry.startChunkIndex + j, startDecodeTimestamp: firstSampleTimestamp, count: 1, delta, @@ -756,12 +755,14 @@ export class IsobmffDemuxer extends Demuxer { if (stsdVersion === 0 && version > 0) { // Additional QuickTime fields if (version === 1) { - this.isobmffReader.pos += 4 * 4; + this.isobmffReader.pos += 4; + sampleSize = 8 * this.isobmffReader.readU32(); + this.isobmffReader.pos += 2 * 4; } else if (version === 2) { this.isobmffReader.pos += 4; sampleRate = this.isobmffReader.readF64(); channelCount = this.isobmffReader.readU32(); - this.isobmffReader.pos += 4; // Always 0x7F000000 + this.isobmffReader.pos += 4; // Always 0x7f000000 sampleSize = this.isobmffReader.readU32(); @@ -777,7 +778,7 @@ export class IsobmffDemuxer extends Demuxer { if (sampleSize > 0 && sampleSize <= 64) { if (isFloat) { - if (sampleSize === 32 && !isBigEndian) { + if (sampleSize === 32) { track.info.codec = isBigEndian ? 'pcm-f32be' : 'pcm-f32'; } } else { @@ -800,7 +801,7 @@ export class IsobmffDemuxer extends Demuxer { } if (track.info.codec === null) { - console.warn('Unsupportedd PCM format.'); + console.warn('Unsupported PCM format.'); } } } diff --git a/src/isobmff/isobmff-muxer.ts b/src/isobmff/isobmff-muxer.ts index dd96480..74d7c6b 100644 --- a/src/isobmff/isobmff-muxer.ts +++ b/src/isobmff/isobmff-muxer.ts @@ -3,10 +3,17 @@ import { Muxer } from '../muxer'; import { Output, OutputAudioTrack, OutputSubtitleTrack, OutputTrack, OutputVideoTrack } from '../output'; import { ArrayBufferTargetWriter, Writer } from '../writer'; import { assert, last } from '../misc'; -import { Mp4OutputFormat, Mp4OutputFormatOptions } from '../output-format'; +import { IsobmffOutputFormatOptions, IsobmffOutputFormat, MovOutputFormat } from '../output-format'; import { inlineTimestampRegex, SubtitleConfig, SubtitleCue, SubtitleMetadata } from '../subtitles'; import { ArrayBufferTarget } from '../target'; -import { validateAudioChunkMetadata, validateSubtitleMetadata, validateVideoChunkMetadata } from '../codec'; +import { + parsePcmCodec, + PCM_CODECS, + PcmAudioCodec, + validateAudioChunkMetadata, + validateSubtitleMetadata, + validateVideoChunkMetadata, +} from '../codec'; import { EncodedAudioSample, EncodedVideoSample } from '../sample'; export const GLOBAL_TIMESCALE = 1000; @@ -40,6 +47,11 @@ export type IsobmffTrackData = { compositionTimeOffsetTable: { sampleCount: number; sampleCompositionTimeOffset: number }[]; lastTimescaleUnits: number | null; lastSample: Sample | null; + /** + * The "PCM transformation" is making every sample in the sample table be exactly one PCM audio sample long. + * Some players expect this for PCM audio. + */ + requiresPcmTransformation: boolean; finalizedChunks: Chunk[]; currentChunk: Chunk | null; @@ -87,7 +99,8 @@ export const intoTimescale = (timeInSeconds: number, timescale: number, round = export class IsobmffMuxer extends Muxer { private writer: Writer; private boxWriter: IsobmffBoxWriter; - private fastStart: NonNullable; + private isMov: boolean; + private fastStart: NonNullable; private auxTarget = new ArrayBufferTarget(); private auxWriter = this.auxTarget._createWriter(); @@ -103,12 +116,14 @@ export class IsobmffMuxer extends Muxer { private nextFragmentNumber = 1; - constructor(output: Output, format: Mp4OutputFormat) { + constructor(output: Output, format: IsobmffOutputFormat) { super(output); this.writer = output._writer; this.boxWriter = new IsobmffBoxWriter(this.writer); + this.isMov = format instanceof MovOutputFormat; + // If the fastStart option isn't defined, enable in-memory fast start if the target is an ArrayBuffer, as the // memory usage remains identical const fastStartDefault = this.writer instanceof ArrayBufferTargetWriter ? 'in-memory' : false; @@ -126,6 +141,7 @@ export class IsobmffMuxer extends Muxer { // Write the header this.boxWriter.writeBox(ftyp({ + isMov: this.isMov, holdsAvc: holdsAvc, fragmented: this.fastStart === 'fragmented', })); @@ -178,6 +194,7 @@ export class IsobmffMuxer extends Muxer { finalizedChunks: [], currentChunk: null, compactlyCodedChunkTable: [], + requiresPcmTransformation: false, }; this.trackDatas.push(newTrackData); @@ -216,6 +233,9 @@ export class IsobmffMuxer extends Muxer { finalizedChunks: [], currentChunk: null, compactlyCodedChunkTable: [], + requiresPcmTransformation: + this.fastStart !== 'fragmented' + && (PCM_CODECS as readonly string[]).includes(track.source._codec), }; this.trackDatas.push(newTrackData); @@ -252,6 +272,8 @@ export class IsobmffMuxer extends Muxer { finalizedChunks: [], currentChunk: null, compactlyCodedChunkTable: [], + requiresPcmTransformation: false, + lastCueEndTimestamp: 0, cueQueue: [], nextSourceId: 0, @@ -308,12 +330,47 @@ export class IsobmffMuxer extends Muxer { sample.type, ); + if (trackData.requiresPcmTransformation) { + await this.maybePadWithSilence(trackData, timestamp); + } + await this.registerSample(trackData, internalSample); } finally { release(); } } + private async maybePadWithSilence(trackData: IsobmffAudioTrackData, untilTimestamp: number) { + // The PCM transformation assumes that all samples are contiguous. This is not something that is enforced, so + // we need to pad the "holes" in between samples (and before the first sample) with additional + // "silence samples". + + const lastSample = last(trackData.samples); + const lastEndTimestamp = lastSample + ? lastSample.timestamp + lastSample.duration + : 0; + + const delta = untilTimestamp - lastEndTimestamp; + const deltaInTimescale = intoTimescale(delta, trackData.timescale); + + if (deltaInTimescale > 0) { + const { sampleSize, silentValue } = parsePcmCodec( + trackData.info.decoderConfig.codec as PcmAudioCodec, + ); + const samplesNeeded = deltaInTimescale * trackData.info.numberOfChannels; + const data = new Uint8Array(sampleSize * samplesNeeded).fill(silentValue); + + const paddingSample = this.createSampleForTrack( + trackData, + new Uint8Array(data.buffer), + lastEndTimestamp, + delta, + 'key', + ); + await this.registerSample(trackData, paddingSample); + } + } + async addSubtitleCue(track: OutputSubtitleTrack, cue: SubtitleCue, meta?: SubtitleMetadata) { const release = await this.mutex.acquire(); @@ -451,6 +508,32 @@ export class IsobmffMuxer extends Muxer { return; } + if (trackData.requiresPcmTransformation) { + let totalDuration = 0; + + // Compute the total duration in the track timescale (which is equal to the amount of PCM audio samples) + // and simply say that's how many new samples there are. + + for (let i = 0; i < trackData.timestampProcessingQueue.length; i++) { + const sample = trackData.timestampProcessingQueue[i]!; + const duration = intoTimescale(sample.duration, trackData.timescale); + totalDuration += duration; + } + + if (trackData.timeToSampleTable.length === 0) { + trackData.timeToSampleTable.push({ + sampleCount: totalDuration, + sampleDelta: 1, + }); + } else { + const lastEntry = last(trackData.timeToSampleTable)!; + lastEntry.sampleCount += totalDuration; + } + + trackData.timestampProcessingQueue.length = 0; + return; + } + const sortedTimestamps = trackData.timestampProcessingQueue.map(x => x.timestamp).sort((a, b) => a - b); for (let i = 0; i < trackData.timestampProcessingQueue.length; i++) { @@ -628,13 +711,19 @@ export class IsobmffMuxer extends Muxer { trackData.finalizedChunks.push(trackData.currentChunk); this.finalizedChunks.push(trackData.currentChunk); + let sampleCount = trackData.currentChunk.samples.length; + if (trackData.requiresPcmTransformation) { + sampleCount = trackData.currentChunk.samples + .reduce((acc, sample) => acc + intoTimescale(sample.duration, trackData.timescale), 0); + } + if ( trackData.compactlyCodedChunkTable.length === 0 - || last(trackData.compactlyCodedChunkTable)!.samplesPerChunk !== trackData.currentChunk.samples.length + || last(trackData.compactlyCodedChunkTable)!.samplesPerChunk !== sampleCount ) { trackData.compactlyCodedChunkTable.push({ firstChunk: trackData.finalizedChunks.length, // 1-indexed - samplesPerChunk: trackData.currentChunk.samples.length, + samplesPerChunk: sampleCount, }); } diff --git a/src/media-drain.ts b/src/media-drain.ts index 282f4dc..1eb5be5 100644 --- a/src/media-drain.ts +++ b/src/media-drain.ts @@ -1,4 +1,4 @@ -import { PCM_CODECS, PcmAudioCodec } from './codec'; +import { parsePcmCodec, PCM_CODECS, PcmAudioCodec } from './codec'; import { InputAudioTrack, InputVideoTrack } from './input-track'; import { AnyIterable, @@ -858,8 +858,6 @@ class AudioDecoderWrapper extends DecoderWrapper } } -const PCM_CODEC_REGEX = /^pcm-([usf])(\d+)+(be)?$/; - // There are a lot of PCM variants not natively supported by the browser and by AudioData. Therefore we need a simple // decoder that maps any input PCM format into a PCM format supported by the browser. class PcmAudioDecoderWrapper extends DecoderWrapper { @@ -882,41 +880,88 @@ class PcmAudioDecoderWrapper extends DecoderWrapper view.getUint8(byteOffset) - 2 ** 7; - } else { + } else if (dataType === 'signed') { this.readInputValue = (view, byteOffset) => view.getInt8(byteOffset); + } else if (dataType === 'ulaw') { + // https://github.com/dystopiancode/pcm-g711/blob/master/pcm-g711/g711.c + this.readInputValue = (view, byteOffset) => { + const MULAW_BIAS = 33; + let sign = 0; + let position = 0; + + // Get byte and invert + let number = ~view.getUint8(byteOffset); + + // Handle sign + if (number & 0x80) { + number &= ~(1 << 7); + sign = -1; + } + + // Calculate position + position = ((number & 0xF0) >> 4) + 5; + + // Reconstruct linear value + const decoded = ((1 << position) | ((number & 0x0F) << (position - 4)) + | (1 << (position - 5))) - MULAW_BIAS; + + return (sign === 0) ? decoded : -decoded; + }; + } else if (dataType === 'alaw') { + this.readInputValue = (view, byteOffset) => { + let sign = 0x00; + let position = 0; + + // Get byte and XOR with 0x55 + let number = view.getUint8(byteOffset) ^ 0x55; + + // Handle sign + if (number & 0x80) { + number &= ~(1 << 7); + sign = -1; + } + + // Calculate position + position = ((number & 0xF0) >> 4) + 4; + + // Reconstruct linear value + let decoded = 0; + if (position !== 4) { + decoded = ((1 << position) | ((number & 0x0F) << (position - 4)) + | (1 << (position - 5))); + } else { + decoded = (number << 1) | 1; + } + + return (sign === 0) ? decoded : -decoded; + }; + } else { + assert(false); } }; break; case 2: { if (dataType === 'unsigned') { this.readInputValue = (view, byteOffset) => view.getUint16(byteOffset, littleEndian) - 2 ** 15; - } else { + } else if (dataType === 'signed') { this.readInputValue = (view, byteOffset) => view.getInt16(byteOffset, littleEndian); + } else { + assert(false); } }; break; case 3: { if (dataType === 'unsigned') { this.readInputValue = (view, byteOffset) => getUint24(view, byteOffset, littleEndian) - 2 ** 23; - } else { + } else if (dataType === 'signed') { this.readInputValue = (view, byteOffset) => getInt24(view, byteOffset, littleEndian); + } else { + assert(false); } }; break; case 4: { @@ -924,17 +969,25 @@ class PcmAudioDecoderWrapper extends DecoderWrapper view.getUint32(byteOffset, littleEndian) - 2 ** 31; } else if (dataType === 'signed') { this.readInputValue = (view, byteOffset) => view.getInt32(byteOffset, littleEndian); - } else { + } else if (dataType === 'float') { this.readInputValue = (view, byteOffset) => view.getFloat32(byteOffset, littleEndian); + } else { + assert(false); } }; break; } - switch (this.inputSampleSize) { + switch (sampleSize) { case 1: { - this.outputSampleSize = 1; - this.outputFormat = 'u8'; - this.writeOutputValue = (view, byteOffset, value) => view.setUint8(byteOffset, value + 2 ** 7); + if (dataType === 'ulaw' || dataType === 'alaw') { + this.outputSampleSize = 2; + this.outputFormat = 's16'; + this.writeOutputValue = (view, byteOffset, value) => view.setInt16(byteOffset, value, true); + } else { + this.outputSampleSize = 1; + this.outputFormat = 'u8'; + this.writeOutputValue = (view, byteOffset, value) => view.setUint8(byteOffset, value + 2 ** 7); + } }; break; case 2: { this.outputSampleSize = 2; @@ -1010,6 +1063,13 @@ class PcmAudioDecoderWrapper extends DecoderWrapper { /** @internal */ @@ -1040,13 +1100,22 @@ export class AudioDataDrain extends BaseMediaFrameDrain): WrappedAudioData { + return { + data: frame.frame, + timestamp: frame.timestamp, + duration: frame.duration, + }; + } + /** @internal */ _createSampleDrain() { return new EncodedAudioSampleDrain(this._audioTrack); @@ -1056,17 +1125,23 @@ export class AudioDataDrain extends BaseMediaFrameDrain this._wrappedFrameToWrappedAudioData(data), + ); } dataAtTimestamps(timestamps: AnyIterable) { - return this.mediaFramesAtTimestamps(timestamps); + return mapAsyncGenerator( + this.mediaFramesAtTimestamps(timestamps), + async data => data && this._wrappedFrameToWrappedAudioData(data), + ); } } @@ -1091,18 +1166,18 @@ export class AudioBufferDrain { } /** @internal */ - _audioDataToWrappedArrayBuffer(data: WrappedMediaFrame): WrappedAudioBuffer { + _audioDataToWrappedArrayBuffer(data: WrappedAudioData): WrappedAudioBuffer { const audioBuffer = new AudioBuffer({ - numberOfChannels: data.frame.numberOfChannels, - length: data.frame.numberOfFrames, - sampleRate: data.frame.sampleRate, + numberOfChannels: data.data.numberOfChannels, + length: data.data.numberOfFrames, + sampleRate: data.data.sampleRate, }); // All user agents are required to support conversion to f32-planar - const dataBytes = new Float32Array(data.frame.allocationSize({ planeIndex: 0, format: 'f32-planar' }) / 4); + const dataBytes = new Float32Array(data.data.allocationSize({ planeIndex: 0, format: 'f32-planar' }) / 4); - for (let i = 0; i < data.frame.numberOfChannels; i++) { - data.frame.copyTo(dataBytes, { planeIndex: i, format: 'f32-planar' }); + for (let i = 0; i < data.data.numberOfChannels; i++) { + data.data.copyTo(dataBytes, { planeIndex: i, format: 'f32-planar' }); audioBuffer.copyToChannel(dataBytes, i); } @@ -1112,7 +1187,7 @@ export class AudioBufferDrain { duration: data.duration, }; - data.frame.close(); + data.data.close(); return result; } diff --git a/src/media-source.ts b/src/media-source.ts index 7d18cbf..72ee77c 100644 --- a/src/media-source.ts +++ b/src/media-source.ts @@ -5,6 +5,7 @@ import { buildVideoCodecString, getAudioEncoderConfigExtension, getVideoEncoderConfigExtension, + PCM_CODECS, SUBTITLE_CODECS, SubtitleCodec, VIDEO_CODECS, @@ -418,7 +419,7 @@ export class EncodedAudioSampleSource extends AudioSource { /** @public */ export type AudioEncodingConfig = { codec: AudioCodec; - bitrate: number; + bitrate?: number; onEncodedChunk?: (chunk: EncodedAudioChunk, meta: EncodedAudioChunkMetadata | undefined) => unknown; onEncodingError?: (error: Error) => unknown; }; @@ -430,7 +431,10 @@ const validateAudioEncodingConfig = (config: AudioEncodingConfig) => { if (!AUDIO_CODECS.includes(config.codec)) { throw new TypeError(`Invalid audio codec '${config.codec}'. Must be one of: ${AUDIO_CODECS.join(', ')}.`); } - if (!Number.isInteger(config.bitrate) || config.bitrate <= 0) { + if (config.bitrate === undefined && !(PCM_CODECS as readonly string[]).includes(config.codec)) { + throw new TypeError('config.bitrate must be provided for compressed audio codecs.'); + } + if (config.bitrate !== undefined && (!Number.isInteger(config.bitrate) || config.bitrate <= 0)) { throw new TypeError('config.bitrate must be a positive integer.'); } if (config.onEncodingError !== undefined && typeof config.onEncodingError !== 'function') { diff --git a/src/output-format.ts b/src/output-format.ts index 68eeccd..e8434d6 100644 --- a/src/output-format.ts +++ b/src/output-format.ts @@ -1,4 +1,4 @@ -import { AUDIO_CODECS, MediaCodec, SUBTITLE_CODECS, VIDEO_CODECS } from './codec'; +import { AUDIO_CODECS, MediaCodec, PCM_CODECS, SUBTITLE_CODECS, VIDEO_CODECS } from './codec'; import { IsobmffMuxer } from './isobmff/isobmff-muxer'; import { MatroskaMuxer } from './matroska/matroska-muxer'; import { Muxer } from './muxer'; @@ -33,16 +33,16 @@ export abstract class OutputFormat { } /** @public */ -export type Mp4OutputFormatOptions = { +export type IsobmffOutputFormatOptions = { fastStart?: false | 'in-memory' | 'fragmented'; }; /** @public */ -export class Mp4OutputFormat extends OutputFormat { +export abstract class IsobmffOutputFormat extends OutputFormat { /** @internal */ - _options: Mp4OutputFormatOptions; + _options: IsobmffOutputFormatOptions; - constructor(options: Mp4OutputFormatOptions = {}) { + constructor(options: IsobmffOutputFormatOptions = {}) { if (!options || typeof options !== 'object') { throw new TypeError('options must be an object.'); } @@ -56,10 +56,13 @@ export class Mp4OutputFormat extends OutputFormat { } /** @internal */ - override _createMuxer(output: Output) { + _createMuxer(output: Output) { return new IsobmffMuxer(output, this); } +} +/** @public */ +export class Mp4OutputFormat extends IsobmffOutputFormat { /** @internal */ _getName() { return 'MP4'; @@ -72,10 +75,47 @@ export class Mp4OutputFormat extends OutputFormat { static getSupportedCodecs(): MediaCodec[] { return [ 'avc', 'hevc', 'vp8', 'vp9', 'av1', - 'aac', 'opus', + 'aac', 'mp3', 'opus', 'vorbis', 'flac', // No PCM codecs 'webvtt', ]; } + + /** @internal */ + override _codecUnsupportedHint(codec: MediaCodec) { + if (MovOutputFormat.getSupportedCodecs().includes(codec)) { + return ' Switching to MOV will grant support for this codec.'; + } + + return ''; + } +} + +/** @public */ +export class MovOutputFormat extends IsobmffOutputFormat { + /** @internal */ + _getName() { + return 'MOV'; + } + + getSupportedCodecs() { + return MovOutputFormat.getSupportedCodecs(); + } + + static getSupportedCodecs(): MediaCodec[] { + return [ + 'avc', 'hevc', 'vp8', 'vp9', 'av1', + 'aac', 'mp3', 'opus', 'vorbis', 'flac', ...PCM_CODECS, + ]; + } + + /** @internal */ + override _codecUnsupportedHint(codec: MediaCodec) { + if (Mp4OutputFormat.getSupportedCodecs().includes(codec)) { + return ' Switching to MP4 will grant support for this codec.'; + } + + return ''; + } } /** @public */ @@ -102,7 +142,7 @@ export class MkvOutputFormat extends OutputFormat { } /** @internal */ - override _createMuxer(output: Output) { + _createMuxer(output: Output) { return new MatroskaMuxer(output, this); } @@ -150,8 +190,8 @@ export class WebMOutputFormat extends MkvOutputFormat { override _codecUnsupportedHint(codec: MediaCodec) { if (MkvOutputFormat.getSupportedCodecs().includes(codec)) { return ' Switching to MKV will grant support for this codec.'; - } else { - return ''; } + + return ''; } }