diff --git a/dev/demux.html b/dev/demux.html index eca56a0..65aea09 100644 --- a/dev/demux.html +++ b/dev/demux.html @@ -17,14 +17,42 @@ source: new Mediabunny.BlobSource(file), }); - const track = await input.getPrimaryVideoTrack(); + const track = await input.getPrimaryAudioTrack(); const sink = new Mediabunny.EncodedPacketSink(track); - const packet = await sink.getFirstKeyPacket(); + + for await (const packet of sink.packets()) { + console.log(packet); + } - console.log(packet.data.join(', ')); + const output = new Mediabunny.Output({ + format: new Mediabunny.Mp4OutputFormat(), + target: new Mediabunny.BufferTarget(), + }); - const config = await track.getDecoderConfig(); - console.log(config.description.join(', ')); + const conversion = await Mediabunny.Conversion.init({ input, output }); + await conversion.execute(); + + return; + + // Download it now + const blob = new Blob([output.target.buffer]); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = file.name.replace(/\.\w+$/, '.mp4'); + a.click(); + URL.revokeObjectURL(url); + + /* + const track = await input.getPrimaryAudioTrack(); + const sink = new Mediabunny.EncodedPacketSink(track); + + for await (const packet of sink.packets()) { + console.log(packet); + } + + console.log("Done") + */ /* const input = new Mediabunny.Input({ diff --git a/src/isobmff/isobmff-boxes.ts b/src/isobmff/isobmff-boxes.ts index 5ee7f5e..f09c170 100644 --- a/src/isobmff/isobmff-boxes.ts +++ b/src/isobmff/isobmff-boxes.ts @@ -174,6 +174,12 @@ const u64 = (value: number) => { return [bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7]] as number[]; }; +const i64 = (value: number) => { + view.setInt32(0, Math.floor(value / 2 ** 32), false); + view.setUint32(4, value, false); + return [bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7]] as number[]; +}; + const fixed_8_8 = (value: number) => { view.setInt16(0, 2 ** 8 * value, false); return [bytes[0], bytes[1]] as number[]; @@ -384,11 +390,14 @@ export const mvhd = ( creationTime: number, trackDatas: IsobmffTrackData[], ) => { - const duration = intoTimescale(Math.max( + const duration = Math.max( 0, ...trackDatas - .map(x => presentationSpan(x)), - ), GLOBAL_TIMESCALE); + .map(trackData => ( + intoTimescale(presentationSpan(trackData), GLOBAL_TIMESCALE) + + intoTimescale(trackData.startTimestampOffset ?? 0, GLOBAL_TIMESCALE) + )), + ); const nextTrackId = Math.max(0, ...trackDatas.map(x => x.track.id)) + 1; // Conditionally use u64 if u32 isn't enough @@ -417,7 +426,9 @@ const presentationSpan = (trackData: IsobmffTrackData) => { let minTimestamp = Infinity; let maxEndTimestamp = -Infinity; - for (const sample of trackData.samples) { + for (let i = 0; i < trackData.samples.length; i++) { + const sample = trackData.samples[i]!; + if (sample.timestamp < minTimestamp) { minTimestamp = sample.timestamp; } @@ -440,9 +451,11 @@ const presentationSpan = (trackData: IsobmffTrackData) => { */ export const trak = (trackData: IsobmffTrackData, creationTime: number) => { const trackMetadata = getTrackMetadata(trackData); + const needsEditList = trackData.startTimestampOffset !== null && trackData.startTimestampOffset > 0; return box('trak', undefined, [ tkhd(trackData, creationTime), + needsEditList ? edts(trackData, trackData.startTimestampOffset!) : null, mdia(trackData, creationTime), trackMetadata.name !== undefined ? box('udta', undefined, [ @@ -459,10 +472,8 @@ export const tkhd = ( trackData: IsobmffTrackData, creationTime: number, ) => { - const durationInGlobalTimescale = intoTimescale( - presentationSpan(trackData), - GLOBAL_TIMESCALE, - ); + const durationInGlobalTimescale = intoTimescale(presentationSpan(trackData), GLOBAL_TIMESCALE) + + intoTimescale(trackData.startTimestampOffset ?? 0, GLOBAL_TIMESCALE); const needsU64 = !isU32(creationTime) || !isU32(durationInGlobalTimescale); const u32OrU64 = needsU64 ? u64 : u32; @@ -497,6 +508,32 @@ export const tkhd = ( ]); }; +/** Edit Box: Specifies edits to the track's media. */ +export const edts = (trackData: IsobmffTrackData, offset: number) => { + const startOffset = intoTimescale(offset, GLOBAL_TIMESCALE); + const mediaDuration = intoTimescale(presentationSpan(trackData), GLOBAL_TIMESCALE); + + const needs64Bits = !isU32(startOffset) || !isU32(mediaDuration); + const u32OrU64 = needs64Bits ? u64 : u32; + const i32OrI64 = needs64Bits ? i64 : i32; + + return box('edts', undefined, [ + fullBox('elst', needs64Bits ? 1 : 0, 0, [ + u32(2), // Entry count + + // #1 + u32OrU64(startOffset), // Segment duration + i32OrI64(-1), // Media time + fixed_16_16(1), // Media rate + + // #2 + u32OrU64(mediaDuration), // Segment duration + i32OrI64(0), // Media time + fixed_16_16(1), // Media rate + ]), + ]); +}; + /** Media Box: Describes and define a track's media type and sample data. */ export const mdia = (trackData: IsobmffTrackData, creationTime: number) => box('mdia', undefined, [ mdhd(trackData, creationTime), @@ -509,6 +546,7 @@ export const mdhd = ( trackData: IsobmffTrackData, creationTime: number, ) => { + // Since the duration represents the raw media duration, edit list offsets are not taken into account here const localDuration = intoTimescale( presentationSpan(trackData), trackData.timescale, diff --git a/src/isobmff/isobmff-muxer.ts b/src/isobmff/isobmff-muxer.ts index e40e21e..5e50127 100644 --- a/src/isobmff/isobmff-muxer.ts +++ b/src/isobmff/isobmff-muxer.ts @@ -52,7 +52,7 @@ import { import { buildIsobmffMimeType } from './isobmff-misc'; import { MAX_BOX_HEADER_SIZE, MIN_BOX_HEADER_SIZE } from './isobmff-reader'; -export const GLOBAL_TIMESCALE = 1000; +export const GLOBAL_TIMESCALE = 57600; // LCM of a bunch of common frame rates (24, 25, 30, 60, 144, ...) const TIMESTAMP_OFFSET = 2_082_844_800; // Seconds between Jan 1 1904 and Jan 1 1970 export type Sample = { @@ -85,6 +85,7 @@ export type IsobmffTrackData = { compositionTimeOffsetTable: { sampleCount: number; sampleCompositionTimeOffset: number }[]; lastTimescaleUnits: number | null; lastSample: Sample | null; + startTimestampOffset: number | null; finalizedChunks: Chunk[]; currentChunk: Chunk | null; @@ -120,6 +121,7 @@ export type IsobmffTrackData = { * Some players expect this for PCM audio. */ requiresPcmTransformation: boolean; + expectedNextPcmPacketTimestamp: number | null; /** * The "ADTS stripping" involves removing the ADTS header from each AAC packet. SOBMFF stores raw AAC data, not * ADTS-wrapped data. @@ -395,7 +397,10 @@ export class IsobmffMuxer extends Muxer { // The frame rate set by the user may not be an integer. Since timescale is an integer, we'll approximate the // frame time (inverse of frame rate) with a rational number, then use that approximation's denominator // as the timescale. - const timescale = computeRationalApproximation(1 / (track.metadata.frameRate ?? 57600), 1e6).denominator; + const timescale = computeRationalApproximation( + 1 / (track.metadata.frameRate ?? GLOBAL_TIMESCALE), + 1e6, + ).denominator; const displayAspectWidth = decoderConfig.displayAspectWidth; const displayAspectHeight = decoderConfig.displayAspectHeight; @@ -425,6 +430,7 @@ export class IsobmffMuxer extends Muxer { compositionTimeOffsetTable: [], lastTimescaleUnits: null, lastSample: null, + startTimestampOffset: null, finalizedChunks: [], currentChunk: null, compactlyCodedChunkTable: [], @@ -494,6 +500,7 @@ export class IsobmffMuxer extends Muxer { requiresPcmTransformation: !this.isFragmented && (PCM_AUDIO_CODECS as readonly string[]).includes(track.source._codec), + expectedNextPcmPacketTimestamp: null, requiresAdtsStripping, firstPacket: packet, }, @@ -505,6 +512,7 @@ export class IsobmffMuxer extends Muxer { compositionTimeOffsetTable: [], lastTimescaleUnits: null, lastSample: null, + startTimestampOffset: null, finalizedChunks: [], currentChunk: null, compactlyCodedChunkTable: [], @@ -547,6 +555,7 @@ export class IsobmffMuxer extends Muxer { compositionTimeOffsetTable: [], lastTimescaleUnits: null, lastSample: null, + startTimestampOffset: null, finalizedChunks: [], currentChunk: null, compactlyCodedChunkTable: [], @@ -629,41 +638,59 @@ export class IsobmffMuxer extends Muxer { packetData = packetData.subarray(headerLength); } - const timestamp = this.validateAndNormalizeTimestamp( + let timestamp = this.validateAndNormalizeTimestamp( trackData.track, packet.timestamp, packet.type === 'key', ); + let duration = packet.duration; + + if (trackData.info.requiresPcmTransformation) { + // Packets may have only approximate timestamp/duration information, but for our PCM logic, we need it + // to be precise. So here, we refine the values. + + const pcmInfo = parsePcmCodec( + trackData.info.decoderConfig.codec as PcmAudioCodec, + ); + const frameSize = pcmInfo.sampleSize * trackData.info.numberOfChannels; + + // Compute the precise duration + duration = packetData.byteLength / frameSize / trackData.info.sampleRate; + + if (trackData.info.expectedNextPcmPacketTimestamp !== null) { + const diff = timestamp - trackData.info.expectedNextPcmPacketTimestamp; + if (diff < 0.01) { + timestamp = trackData.info.expectedNextPcmPacketTimestamp; + } else { + const paddedDuration = await this.padWithSilence( + trackData, + trackData.info.expectedNextPcmPacketTimestamp, + diff, + ); + timestamp = trackData.info.expectedNextPcmPacketTimestamp + paddedDuration; + } + } + + trackData.info.expectedNextPcmPacketTimestamp = timestamp + duration; + } + const internalSample = this.createSampleForTrack( trackData, packetData, timestamp, - packet.duration, + duration, packet.type, ); - if (trackData.info.requiresPcmTransformation) { - await this.maybePadWithSilence(trackData, timestamp); - } - await this.registerSample(trackData, internalSample); } finally { release(); } } - private async maybePadWithSilence(trackData: IsobmffAudioTrackData, untilTimestamp: number) { - // The PCM transformation assumes that all samples are contiguous. This is not something that is enforced, so - // we need to pad the "holes" in between samples (and before the first sample) with additional - // "silence samples". - - const lastSample = last(trackData.samples); - const lastEndTimestamp = lastSample - ? lastSample.timestamp + lastSample.duration - : 0; - - const delta = untilTimestamp - lastEndTimestamp; - const deltaInTimescale = intoTimescale(delta, trackData.timescale); + private async padWithSilence(trackData: IsobmffAudioTrackData, timestamp: number, duration: number) { + const deltaInTimescale = intoTimescale(duration, trackData.timescale); + duration = deltaInTimescale / trackData.timescale; if (deltaInTimescale > 0) { const { sampleSize, silentValue } = parsePcmCodec( @@ -675,12 +702,14 @@ export class IsobmffMuxer extends Muxer { const paddingSample = this.createSampleForTrack( trackData, new Uint8Array(data.buffer), - lastEndTimestamp, - delta, + timestamp, + duration, 'key', ); await this.registerSample(trackData, paddingSample); } + + return duration; } async addSubtitleCue(track: OutputSubtitleTrack, cue: SubtitleCue, meta?: SubtitleMetadata) { @@ -821,6 +850,11 @@ export class IsobmffMuxer extends Muxer { } if (trackData.type === 'audio' && trackData.info.requiresPcmTransformation) { + if (!this.isFragmented) { + // The first timestamp is the lowest + trackData.startTimestampOffset ??= trackData.timestampProcessingQueue[0]!.timestamp; + } + let totalDuration = 0; // Compute the total duration in the track timescale (which is equal to the amount of PCM audio samples) @@ -848,6 +882,10 @@ export class IsobmffMuxer extends Muxer { const sortedTimestamps = trackData.timestampProcessingQueue.map(x => x.timestamp).sort((a, b) => a - b); + if (!this.isFragmented) { + trackData.startTimestampOffset ??= sortedTimestamps[0]!; + } + for (let i = 0; i < trackData.timestampProcessingQueue.length; i++) { const sample = trackData.timestampProcessingQueue[i]!; @@ -857,12 +895,6 @@ export class IsobmffMuxer extends Muxer { // model it. sample.decodeTimestamp = sortedTimestamps[i]!; - if (!this.isFragmented && trackData.lastTimescaleUnits === null) { - // In non-fragmented files, the first decode timestamp is always zero. If the first presentation - // timestamp isn't zero, we'll simply use the composition time offset to achieve it. - sample.decodeTimestamp = 0; - } - const sampleCompositionTimeOffset = intoTimescale(sample.timestamp - sample.decodeTimestamp, trackData.timescale); const durationInTimescale = intoTimescale(sample.duration, trackData.timescale); @@ -1377,6 +1409,17 @@ export class IsobmffMuxer extends Muxer { } else { for (const trackData of this.trackDatas) { await this.finalizeCurrentChunk(trackData); + + // Must hold because we will have processed at least one sample + assert(trackData.startTimestampOffset !== null); + + // Shift all of the samples by the start offset. We'll then write out an edit list that will shift them + // back to their proper spot in the composition. + for (let i = 0; i < trackData.samples.length; i++) { + const sample = trackData.samples[i]!; + sample.timestamp -= trackData.startTimestampOffset; + sample.decodeTimestamp -= trackData.startTimestampOffset; + } } } diff --git a/test/node/isobmff-muxer.test.ts b/test/node/isobmff-muxer.test.ts index b8eec61..8d495e4 100644 --- a/test/node/isobmff-muxer.test.ts +++ b/test/node/isobmff-muxer.test.ts @@ -9,6 +9,8 @@ import { BufferTarget } from '../../src/target.js'; import { Mp4OutputFormat } from '../../src/output-format.js'; import { Conversion } from '../../src/conversion.js'; import { assert } from '../../src/misc.js'; +import { EncodedAudioPacketSource, EncodedVideoPacketSource } from '../../src/media-source.js'; +import { EncodedPacket } from '../../src/packet.js'; const __dirname = new URL('.', import.meta.url).pathname; @@ -104,3 +106,249 @@ test('Fragmented fMP4 with video+audio preserves B-frame CTS', async () => { expect(timestamps).toEqual(originalTimestamps); }); + +test('Zero start timestamp, regular MP4', async () => { + const output = new Output({ + format: new Mp4OutputFormat(), + target: new BufferTarget(), + }); + + const source = new EncodedVideoPacketSource('vp8'); + output.addVideoTrack(source); + + await output.start(); + + const meta = { decoderConfig: { codec: 'vp8', codedWidth: 1280, codedHeight: 720 } }; + + await source.add(new EncodedPacket(new Uint8Array(1024), 'key', 0, 0.1), meta); + await source.add(new EncodedPacket(new Uint8Array(1024), 'delta', 0.1, 0.1)); + await source.add(new EncodedPacket(new Uint8Array(1024), 'delta', 0.2, 0.1)); + await source.add(new EncodedPacket(new Uint8Array(1024), 'delta', 0.3, 0.1)); + + await output.finalize(); + + // Hacky but works + const str = String.fromCharCode(...new Uint8Array(output.target.buffer!)); + expect(str.includes('edts') || str.includes('elst')).toBe(false); +}); + +test('Non-zero start timestamp, regular MP4', async () => { + const output = new Output({ + format: new Mp4OutputFormat(), + target: new BufferTarget(), + }); + + const source = new EncodedVideoPacketSource('vp8'); + output.addVideoTrack(source); + + await output.start(); + + const meta = { decoderConfig: { codec: 'vp8', codedWidth: 1280, codedHeight: 720 } }; + + await source.add(new EncodedPacket(new Uint8Array(1024), 'key', 1, 0.1), meta); + await source.add(new EncodedPacket(new Uint8Array(1024), 'delta', 1.1, 0.1)); + await source.add(new EncodedPacket(new Uint8Array(1024), 'delta', 1.2, 0.1)); + await source.add(new EncodedPacket(new Uint8Array(1024), 'delta', 1.3, 0.1)); + + await output.finalize(); + + // Hacky but works + const str = String.fromCharCode(...new Uint8Array(output.target.buffer!)); + expect(str.includes('edts') && str.includes('elst')).toBe(true); + + using input = new Input({ + source: new BufferSource(output.target.buffer!), + formats: ALL_FORMATS, + }); + + const track = await input.getPrimaryVideoTrack(); + assert(track); + const sink = new EncodedPacketSink(track); + + const timestamps: number[] = []; + const durations: number[] = []; + for await (const packet of sink.packets()) { + timestamps.push(packet.timestamp); + durations.push(packet.duration); + } + + expect(timestamps).toEqual([1, 1.1, 1.2, 1.3]); + expect(durations).toEqual([0.1, 0.1, 0.1, 0.1]); +}); + +test('Non-zero start timestamp, fragmented MP4', async () => { + const output = new Output({ + format: new Mp4OutputFormat({ fastStart: 'fragmented' }), + target: new BufferTarget(), + }); + + const source = new EncodedVideoPacketSource('vp8'); + output.addVideoTrack(source); + + await output.start(); + + const meta = { decoderConfig: { codec: 'vp8', codedWidth: 1280, codedHeight: 720 } }; + + await source.add(new EncodedPacket(new Uint8Array(1024), 'key', 1, 0.1), meta); + await source.add(new EncodedPacket(new Uint8Array(1024), 'delta', 1.1, 0.1)); + await source.add(new EncodedPacket(new Uint8Array(1024), 'delta', 1.2, 0.1)); + await source.add(new EncodedPacket(new Uint8Array(1024), 'delta', 1.3, 0.1)); + + await output.finalize(); + + // Hacky but works + const str = String.fromCharCode(...new Uint8Array(output.target.buffer!)); + expect(str.includes('edts') || str.includes('elst')).toBe(false); + + using input = new Input({ + source: new BufferSource(output.target.buffer!), + formats: ALL_FORMATS, + }); + + const track = await input.getPrimaryVideoTrack(); + assert(track); + const sink = new EncodedPacketSink(track); + + const timestamps: number[] = []; + const durations: number[] = []; + for await (const packet of sink.packets()) { + timestamps.push(packet.timestamp); + durations.push(packet.duration); + } + + expect(timestamps).toEqual([1, 1.1, 1.2, 1.3]); + expect(durations).toEqual([0.1, 0.1, 0.1, 0.1]); +}); + +test('PCM audio', async () => { + const output = new Output({ + format: new Mp4OutputFormat(), + target: new BufferTarget(), + }); + + const source = new EncodedAudioPacketSource('pcm-s16'); + output.addAudioTrack(source); + + await output.start(); + + const meta = { decoderConfig: { codec: 'pcm-s16', numberOfChannels: 2, sampleRate: 48000 } }; + + await source.add(new EncodedPacket(new Uint8Array(1024), 'key', 0, 1024 / 2 / 2 / 48000), meta); + + await output.finalize(); + + const input = new Input({ + source: new BufferSource(output.target.buffer!), + formats: ALL_FORMATS, + }); + const audioTrack = await input.getPrimaryAudioTrack(); + assert(audioTrack); + + expect(await audioTrack.getFirstTimestamp()).toBe(0); +}); + +test('PCM audio with non-zero timestamp', async () => { + const output = new Output({ + format: new Mp4OutputFormat(), + target: new BufferTarget(), + }); + + const source = new EncodedAudioPacketSource('pcm-s16'); + output.addAudioTrack(source); + + await output.start(); + + const meta = { decoderConfig: { codec: 'pcm-s16', numberOfChannels: 2, sampleRate: 48000 } }; + + await source.add(new EncodedPacket(new Uint8Array(1024), 'key', 1, 1024 / 2 / 2 / 48000), meta); + + await output.finalize(); + + const input = new Input({ + source: new BufferSource(output.target.buffer!), + formats: ALL_FORMATS, + }); + const audioTrack = await input.getPrimaryAudioTrack(); + assert(audioTrack); + + expect(await audioTrack.getFirstTimestamp()).toBe(1); +}); + +test('PCM audio, silence padding', async () => { + const output = new Output({ + format: new Mp4OutputFormat(), + target: new BufferTarget(), + }); + + const source = new EncodedAudioPacketSource('pcm-s16'); + output.addAudioTrack(source); + + await output.start(); + + const meta = { decoderConfig: { codec: 'pcm-s16', numberOfChannels: 2, sampleRate: 48000 } }; + + await source.add(new EncodedPacket(new Uint8Array(1024), 'key', 0, 1024 / 2 / 2 / 48000), meta); + await source.add(new EncodedPacket(new Uint8Array(1024), 'key', 1, 1024 / 2 / 2 / 48000), meta); + + await output.finalize(); + + const input = new Input({ + source: new BufferSource(output.target.buffer!), + formats: ALL_FORMATS, + }); + const audioTrack = await input.getPrimaryAudioTrack(); + assert(audioTrack); + + expect(await audioTrack.getCodec()).toBe('pcm-s16'); + const numChannels = await audioTrack.getNumberOfChannels(); + + const expectedFrameCount = 48000 + 256; + const sink = new EncodedPacketSink(audioTrack); + let frameCount = 0; + + for await (const packet of sink.packets()) { + frameCount += packet.byteLength / 2 / numChannels; + } + + expect(frameCount).toBe(expectedFrameCount); +}); + +test('PCM audio, no silence padding with approximate timestamps', async () => { + const output = new Output({ + format: new Mp4OutputFormat(), + target: new BufferTarget(), + }); + + const source = new EncodedAudioPacketSource('pcm-s16'); + output.addAudioTrack(source); + + await output.start(); + + const meta = { decoderConfig: { codec: 'pcm-s16', numberOfChannels: 2, sampleRate: 48000 } }; + + await source.add(new EncodedPacket(new Uint8Array(1024), 'key', 0, 1024 / 2 / 2 / 48000), meta); + // 0.006 is 256/48000 "rounded up", but it's close enough for silence padding not to kick in + await source.add(new EncodedPacket(new Uint8Array(1024), 'key', 0.006, 1024 / 2 / 2 / 48000), meta); + + await output.finalize(); + + const input = new Input({ + source: new BufferSource(output.target.buffer!), + formats: ALL_FORMATS, + }); + const audioTrack = await input.getPrimaryAudioTrack(); + assert(audioTrack); + + expect(await audioTrack.getCodec()).toBe('pcm-s16'); + const numChannels = await audioTrack.getNumberOfChannels(); + + const expectedFrameCount = 256 + 256; + const sink = new EncodedPacketSink(audioTrack); + let frameCount = 0; + + for await (const packet of sink.packets()) { + frameCount += packet.byteLength / 2 / numChannels; + } + + expect(frameCount).toBe(expectedFrameCount); +});