Use edit lists to model non-zero start timestamps for regular MP4 (fixes #336); fix MP4 PCM logic breaking down with approximate packet timestamps

This commit is contained in:
Vanilagy
2026-04-30 10:32:42 +02:00
parent 9224fb886c
commit 782d3e134b
4 changed files with 398 additions and 41 deletions
+33 -5
View File
@@ -17,14 +17,42 @@
source: new Mediabunny.BlobSource(file),
});
const track = await input.getPrimaryVideoTrack();
const track = await input.getPrimaryAudioTrack();
const sink = new Mediabunny.EncodedPacketSink(track);
const packet = await sink.getFirstKeyPacket();
console.log(packet.data.join(', '));
for await (const packet of sink.packets()) {
console.log(packet);
}
const config = await track.getDecoderConfig();
console.log(config.description.join(', '));
const output = new Mediabunny.Output({
format: new Mediabunny.Mp4OutputFormat(),
target: new Mediabunny.BufferTarget(),
});
const conversion = await Mediabunny.Conversion.init({ input, output });
await conversion.execute();
return;
// Download it now
const blob = new Blob([output.target.buffer]);
const url = URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url;
a.download = file.name.replace(/\.\w+$/, '.mp4');
a.click();
URL.revokeObjectURL(url);
/*
const track = await input.getPrimaryAudioTrack();
const sink = new Mediabunny.EncodedPacketSink(track);
for await (const packet of sink.packets()) {
console.log(packet);
}
console.log("Done")
*/
/*
const input = new Mediabunny.Input({
+46 -8
View File
@@ -174,6 +174,12 @@ const u64 = (value: number) => {
return [bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7]] as number[];
};
const i64 = (value: number) => {
view.setInt32(0, Math.floor(value / 2 ** 32), false);
view.setUint32(4, value, false);
return [bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7]] as number[];
};
const fixed_8_8 = (value: number) => {
view.setInt16(0, 2 ** 8 * value, false);
return [bytes[0], bytes[1]] as number[];
@@ -384,11 +390,14 @@ export const mvhd = (
creationTime: number,
trackDatas: IsobmffTrackData[],
) => {
const duration = intoTimescale(Math.max(
const duration = Math.max(
0,
...trackDatas
.map(x => presentationSpan(x)),
), GLOBAL_TIMESCALE);
.map(trackData => (
intoTimescale(presentationSpan(trackData), GLOBAL_TIMESCALE)
+ intoTimescale(trackData.startTimestampOffset ?? 0, GLOBAL_TIMESCALE)
)),
);
const nextTrackId = Math.max(0, ...trackDatas.map(x => x.track.id)) + 1;
// Conditionally use u64 if u32 isn't enough
@@ -417,7 +426,9 @@ const presentationSpan = (trackData: IsobmffTrackData) => {
let minTimestamp = Infinity;
let maxEndTimestamp = -Infinity;
for (const sample of trackData.samples) {
for (let i = 0; i < trackData.samples.length; i++) {
const sample = trackData.samples[i]!;
if (sample.timestamp < minTimestamp) {
minTimestamp = sample.timestamp;
}
@@ -440,9 +451,11 @@ const presentationSpan = (trackData: IsobmffTrackData) => {
*/
export const trak = (trackData: IsobmffTrackData, creationTime: number) => {
const trackMetadata = getTrackMetadata(trackData);
const needsEditList = trackData.startTimestampOffset !== null && trackData.startTimestampOffset > 0;
return box('trak', undefined, [
tkhd(trackData, creationTime),
needsEditList ? edts(trackData, trackData.startTimestampOffset!) : null,
mdia(trackData, creationTime),
trackMetadata.name !== undefined
? box('udta', undefined, [
@@ -459,10 +472,8 @@ export const tkhd = (
trackData: IsobmffTrackData,
creationTime: number,
) => {
const durationInGlobalTimescale = intoTimescale(
presentationSpan(trackData),
GLOBAL_TIMESCALE,
);
const durationInGlobalTimescale = intoTimescale(presentationSpan(trackData), GLOBAL_TIMESCALE)
+ intoTimescale(trackData.startTimestampOffset ?? 0, GLOBAL_TIMESCALE);
const needsU64 = !isU32(creationTime) || !isU32(durationInGlobalTimescale);
const u32OrU64 = needsU64 ? u64 : u32;
@@ -497,6 +508,32 @@ export const tkhd = (
]);
};
/** Edit Box: Specifies edits to the track's media. */
export const edts = (trackData: IsobmffTrackData, offset: number) => {
const startOffset = intoTimescale(offset, GLOBAL_TIMESCALE);
const mediaDuration = intoTimescale(presentationSpan(trackData), GLOBAL_TIMESCALE);
const needs64Bits = !isU32(startOffset) || !isU32(mediaDuration);
const u32OrU64 = needs64Bits ? u64 : u32;
const i32OrI64 = needs64Bits ? i64 : i32;
return box('edts', undefined, [
fullBox('elst', needs64Bits ? 1 : 0, 0, [
u32(2), // Entry count
// #1
u32OrU64(startOffset), // Segment duration
i32OrI64(-1), // Media time
fixed_16_16(1), // Media rate
// #2
u32OrU64(mediaDuration), // Segment duration
i32OrI64(0), // Media time
fixed_16_16(1), // Media rate
]),
]);
};
/** Media Box: Describes and define a track's media type and sample data. */
export const mdia = (trackData: IsobmffTrackData, creationTime: number) => box('mdia', undefined, [
mdhd(trackData, creationTime),
@@ -509,6 +546,7 @@ export const mdhd = (
trackData: IsobmffTrackData,
creationTime: number,
) => {
// Since the duration represents the raw media duration, edit list offsets are not taken into account here
const localDuration = intoTimescale(
presentationSpan(trackData),
trackData.timescale,
+71 -28
View File
@@ -52,7 +52,7 @@ import {
import { buildIsobmffMimeType } from './isobmff-misc';
import { MAX_BOX_HEADER_SIZE, MIN_BOX_HEADER_SIZE } from './isobmff-reader';
export const GLOBAL_TIMESCALE = 1000;
export const GLOBAL_TIMESCALE = 57600; // LCM of a bunch of common frame rates (24, 25, 30, 60, 144, ...)
const TIMESTAMP_OFFSET = 2_082_844_800; // Seconds between Jan 1 1904 and Jan 1 1970
export type Sample = {
@@ -85,6 +85,7 @@ export type IsobmffTrackData = {
compositionTimeOffsetTable: { sampleCount: number; sampleCompositionTimeOffset: number }[];
lastTimescaleUnits: number | null;
lastSample: Sample | null;
startTimestampOffset: number | null;
finalizedChunks: Chunk[];
currentChunk: Chunk | null;
@@ -120,6 +121,7 @@ export type IsobmffTrackData = {
* Some players expect this for PCM audio.
*/
requiresPcmTransformation: boolean;
expectedNextPcmPacketTimestamp: number | null;
/**
* The "ADTS stripping" involves removing the ADTS header from each AAC packet. SOBMFF stores raw AAC data, not
* ADTS-wrapped data.
@@ -395,7 +397,10 @@ export class IsobmffMuxer extends Muxer {
// The frame rate set by the user may not be an integer. Since timescale is an integer, we'll approximate the
// frame time (inverse of frame rate) with a rational number, then use that approximation's denominator
// as the timescale.
const timescale = computeRationalApproximation(1 / (track.metadata.frameRate ?? 57600), 1e6).denominator;
const timescale = computeRationalApproximation(
1 / (track.metadata.frameRate ?? GLOBAL_TIMESCALE),
1e6,
).denominator;
const displayAspectWidth = decoderConfig.displayAspectWidth;
const displayAspectHeight = decoderConfig.displayAspectHeight;
@@ -425,6 +430,7 @@ export class IsobmffMuxer extends Muxer {
compositionTimeOffsetTable: [],
lastTimescaleUnits: null,
lastSample: null,
startTimestampOffset: null,
finalizedChunks: [],
currentChunk: null,
compactlyCodedChunkTable: [],
@@ -494,6 +500,7 @@ export class IsobmffMuxer extends Muxer {
requiresPcmTransformation:
!this.isFragmented
&& (PCM_AUDIO_CODECS as readonly string[]).includes(track.source._codec),
expectedNextPcmPacketTimestamp: null,
requiresAdtsStripping,
firstPacket: packet,
},
@@ -505,6 +512,7 @@ export class IsobmffMuxer extends Muxer {
compositionTimeOffsetTable: [],
lastTimescaleUnits: null,
lastSample: null,
startTimestampOffset: null,
finalizedChunks: [],
currentChunk: null,
compactlyCodedChunkTable: [],
@@ -547,6 +555,7 @@ export class IsobmffMuxer extends Muxer {
compositionTimeOffsetTable: [],
lastTimescaleUnits: null,
lastSample: null,
startTimestampOffset: null,
finalizedChunks: [],
currentChunk: null,
compactlyCodedChunkTable: [],
@@ -629,41 +638,59 @@ export class IsobmffMuxer extends Muxer {
packetData = packetData.subarray(headerLength);
}
const timestamp = this.validateAndNormalizeTimestamp(
let timestamp = this.validateAndNormalizeTimestamp(
trackData.track,
packet.timestamp,
packet.type === 'key',
);
let duration = packet.duration;
if (trackData.info.requiresPcmTransformation) {
// Packets may have only approximate timestamp/duration information, but for our PCM logic, we need it
// to be precise. So here, we refine the values.
const pcmInfo = parsePcmCodec(
trackData.info.decoderConfig.codec as PcmAudioCodec,
);
const frameSize = pcmInfo.sampleSize * trackData.info.numberOfChannels;
// Compute the precise duration
duration = packetData.byteLength / frameSize / trackData.info.sampleRate;
if (trackData.info.expectedNextPcmPacketTimestamp !== null) {
const diff = timestamp - trackData.info.expectedNextPcmPacketTimestamp;
if (diff < 0.01) {
timestamp = trackData.info.expectedNextPcmPacketTimestamp;
} else {
const paddedDuration = await this.padWithSilence(
trackData,
trackData.info.expectedNextPcmPacketTimestamp,
diff,
);
timestamp = trackData.info.expectedNextPcmPacketTimestamp + paddedDuration;
}
}
trackData.info.expectedNextPcmPacketTimestamp = timestamp + duration;
}
const internalSample = this.createSampleForTrack(
trackData,
packetData,
timestamp,
packet.duration,
duration,
packet.type,
);
if (trackData.info.requiresPcmTransformation) {
await this.maybePadWithSilence(trackData, timestamp);
}
await this.registerSample(trackData, internalSample);
} finally {
release();
}
}
private async maybePadWithSilence(trackData: IsobmffAudioTrackData, untilTimestamp: number) {
// The PCM transformation assumes that all samples are contiguous. This is not something that is enforced, so
// we need to pad the "holes" in between samples (and before the first sample) with additional
// "silence samples".
const lastSample = last(trackData.samples);
const lastEndTimestamp = lastSample
? lastSample.timestamp + lastSample.duration
: 0;
const delta = untilTimestamp - lastEndTimestamp;
const deltaInTimescale = intoTimescale(delta, trackData.timescale);
private async padWithSilence(trackData: IsobmffAudioTrackData, timestamp: number, duration: number) {
const deltaInTimescale = intoTimescale(duration, trackData.timescale);
duration = deltaInTimescale / trackData.timescale;
if (deltaInTimescale > 0) {
const { sampleSize, silentValue } = parsePcmCodec(
@@ -675,12 +702,14 @@ export class IsobmffMuxer extends Muxer {
const paddingSample = this.createSampleForTrack(
trackData,
new Uint8Array(data.buffer),
lastEndTimestamp,
delta,
timestamp,
duration,
'key',
);
await this.registerSample(trackData, paddingSample);
}
return duration;
}
async addSubtitleCue(track: OutputSubtitleTrack, cue: SubtitleCue, meta?: SubtitleMetadata) {
@@ -821,6 +850,11 @@ export class IsobmffMuxer extends Muxer {
}
if (trackData.type === 'audio' && trackData.info.requiresPcmTransformation) {
if (!this.isFragmented) {
// The first timestamp is the lowest
trackData.startTimestampOffset ??= trackData.timestampProcessingQueue[0]!.timestamp;
}
let totalDuration = 0;
// Compute the total duration in the track timescale (which is equal to the amount of PCM audio samples)
@@ -848,6 +882,10 @@ export class IsobmffMuxer extends Muxer {
const sortedTimestamps = trackData.timestampProcessingQueue.map(x => x.timestamp).sort((a, b) => a - b);
if (!this.isFragmented) {
trackData.startTimestampOffset ??= sortedTimestamps[0]!;
}
for (let i = 0; i < trackData.timestampProcessingQueue.length; i++) {
const sample = trackData.timestampProcessingQueue[i]!;
@@ -857,12 +895,6 @@ export class IsobmffMuxer extends Muxer {
// model it.
sample.decodeTimestamp = sortedTimestamps[i]!;
if (!this.isFragmented && trackData.lastTimescaleUnits === null) {
// In non-fragmented files, the first decode timestamp is always zero. If the first presentation
// timestamp isn't zero, we'll simply use the composition time offset to achieve it.
sample.decodeTimestamp = 0;
}
const sampleCompositionTimeOffset
= intoTimescale(sample.timestamp - sample.decodeTimestamp, trackData.timescale);
const durationInTimescale = intoTimescale(sample.duration, trackData.timescale);
@@ -1377,6 +1409,17 @@ export class IsobmffMuxer extends Muxer {
} else {
for (const trackData of this.trackDatas) {
await this.finalizeCurrentChunk(trackData);
// Must hold because we will have processed at least one sample
assert(trackData.startTimestampOffset !== null);
// Shift all of the samples by the start offset. We'll then write out an edit list that will shift them
// back to their proper spot in the composition.
for (let i = 0; i < trackData.samples.length; i++) {
const sample = trackData.samples[i]!;
sample.timestamp -= trackData.startTimestampOffset;
sample.decodeTimestamp -= trackData.startTimestampOffset;
}
}
}
+248
View File
@@ -9,6 +9,8 @@ import { BufferTarget } from '../../src/target.js';
import { Mp4OutputFormat } from '../../src/output-format.js';
import { Conversion } from '../../src/conversion.js';
import { assert } from '../../src/misc.js';
import { EncodedAudioPacketSource, EncodedVideoPacketSource } from '../../src/media-source.js';
import { EncodedPacket } from '../../src/packet.js';
const __dirname = new URL('.', import.meta.url).pathname;
@@ -104,3 +106,249 @@ test('Fragmented fMP4 with video+audio preserves B-frame CTS', async () => {
expect(timestamps).toEqual(originalTimestamps);
});
test('Zero start timestamp, regular MP4', async () => {
const output = new Output({
format: new Mp4OutputFormat(),
target: new BufferTarget(),
});
const source = new EncodedVideoPacketSource('vp8');
output.addVideoTrack(source);
await output.start();
const meta = { decoderConfig: { codec: 'vp8', codedWidth: 1280, codedHeight: 720 } };
await source.add(new EncodedPacket(new Uint8Array(1024), 'key', 0, 0.1), meta);
await source.add(new EncodedPacket(new Uint8Array(1024), 'delta', 0.1, 0.1));
await source.add(new EncodedPacket(new Uint8Array(1024), 'delta', 0.2, 0.1));
await source.add(new EncodedPacket(new Uint8Array(1024), 'delta', 0.3, 0.1));
await output.finalize();
// Hacky but works
const str = String.fromCharCode(...new Uint8Array(output.target.buffer!));
expect(str.includes('edts') || str.includes('elst')).toBe(false);
});
test('Non-zero start timestamp, regular MP4', async () => {
const output = new Output({
format: new Mp4OutputFormat(),
target: new BufferTarget(),
});
const source = new EncodedVideoPacketSource('vp8');
output.addVideoTrack(source);
await output.start();
const meta = { decoderConfig: { codec: 'vp8', codedWidth: 1280, codedHeight: 720 } };
await source.add(new EncodedPacket(new Uint8Array(1024), 'key', 1, 0.1), meta);
await source.add(new EncodedPacket(new Uint8Array(1024), 'delta', 1.1, 0.1));
await source.add(new EncodedPacket(new Uint8Array(1024), 'delta', 1.2, 0.1));
await source.add(new EncodedPacket(new Uint8Array(1024), 'delta', 1.3, 0.1));
await output.finalize();
// Hacky but works
const str = String.fromCharCode(...new Uint8Array(output.target.buffer!));
expect(str.includes('edts') && str.includes('elst')).toBe(true);
using input = new Input({
source: new BufferSource(output.target.buffer!),
formats: ALL_FORMATS,
});
const track = await input.getPrimaryVideoTrack();
assert(track);
const sink = new EncodedPacketSink(track);
const timestamps: number[] = [];
const durations: number[] = [];
for await (const packet of sink.packets()) {
timestamps.push(packet.timestamp);
durations.push(packet.duration);
}
expect(timestamps).toEqual([1, 1.1, 1.2, 1.3]);
expect(durations).toEqual([0.1, 0.1, 0.1, 0.1]);
});
test('Non-zero start timestamp, fragmented MP4', async () => {
const output = new Output({
format: new Mp4OutputFormat({ fastStart: 'fragmented' }),
target: new BufferTarget(),
});
const source = new EncodedVideoPacketSource('vp8');
output.addVideoTrack(source);
await output.start();
const meta = { decoderConfig: { codec: 'vp8', codedWidth: 1280, codedHeight: 720 } };
await source.add(new EncodedPacket(new Uint8Array(1024), 'key', 1, 0.1), meta);
await source.add(new EncodedPacket(new Uint8Array(1024), 'delta', 1.1, 0.1));
await source.add(new EncodedPacket(new Uint8Array(1024), 'delta', 1.2, 0.1));
await source.add(new EncodedPacket(new Uint8Array(1024), 'delta', 1.3, 0.1));
await output.finalize();
// Hacky but works
const str = String.fromCharCode(...new Uint8Array(output.target.buffer!));
expect(str.includes('edts') || str.includes('elst')).toBe(false);
using input = new Input({
source: new BufferSource(output.target.buffer!),
formats: ALL_FORMATS,
});
const track = await input.getPrimaryVideoTrack();
assert(track);
const sink = new EncodedPacketSink(track);
const timestamps: number[] = [];
const durations: number[] = [];
for await (const packet of sink.packets()) {
timestamps.push(packet.timestamp);
durations.push(packet.duration);
}
expect(timestamps).toEqual([1, 1.1, 1.2, 1.3]);
expect(durations).toEqual([0.1, 0.1, 0.1, 0.1]);
});
test('PCM audio', async () => {
const output = new Output({
format: new Mp4OutputFormat(),
target: new BufferTarget(),
});
const source = new EncodedAudioPacketSource('pcm-s16');
output.addAudioTrack(source);
await output.start();
const meta = { decoderConfig: { codec: 'pcm-s16', numberOfChannels: 2, sampleRate: 48000 } };
await source.add(new EncodedPacket(new Uint8Array(1024), 'key', 0, 1024 / 2 / 2 / 48000), meta);
await output.finalize();
const input = new Input({
source: new BufferSource(output.target.buffer!),
formats: ALL_FORMATS,
});
const audioTrack = await input.getPrimaryAudioTrack();
assert(audioTrack);
expect(await audioTrack.getFirstTimestamp()).toBe(0);
});
test('PCM audio with non-zero timestamp', async () => {
const output = new Output({
format: new Mp4OutputFormat(),
target: new BufferTarget(),
});
const source = new EncodedAudioPacketSource('pcm-s16');
output.addAudioTrack(source);
await output.start();
const meta = { decoderConfig: { codec: 'pcm-s16', numberOfChannels: 2, sampleRate: 48000 } };
await source.add(new EncodedPacket(new Uint8Array(1024), 'key', 1, 1024 / 2 / 2 / 48000), meta);
await output.finalize();
const input = new Input({
source: new BufferSource(output.target.buffer!),
formats: ALL_FORMATS,
});
const audioTrack = await input.getPrimaryAudioTrack();
assert(audioTrack);
expect(await audioTrack.getFirstTimestamp()).toBe(1);
});
test('PCM audio, silence padding', async () => {
const output = new Output({
format: new Mp4OutputFormat(),
target: new BufferTarget(),
});
const source = new EncodedAudioPacketSource('pcm-s16');
output.addAudioTrack(source);
await output.start();
const meta = { decoderConfig: { codec: 'pcm-s16', numberOfChannels: 2, sampleRate: 48000 } };
await source.add(new EncodedPacket(new Uint8Array(1024), 'key', 0, 1024 / 2 / 2 / 48000), meta);
await source.add(new EncodedPacket(new Uint8Array(1024), 'key', 1, 1024 / 2 / 2 / 48000), meta);
await output.finalize();
const input = new Input({
source: new BufferSource(output.target.buffer!),
formats: ALL_FORMATS,
});
const audioTrack = await input.getPrimaryAudioTrack();
assert(audioTrack);
expect(await audioTrack.getCodec()).toBe('pcm-s16');
const numChannels = await audioTrack.getNumberOfChannels();
const expectedFrameCount = 48000 + 256;
const sink = new EncodedPacketSink(audioTrack);
let frameCount = 0;
for await (const packet of sink.packets()) {
frameCount += packet.byteLength / 2 / numChannels;
}
expect(frameCount).toBe(expectedFrameCount);
});
test('PCM audio, no silence padding with approximate timestamps', async () => {
const output = new Output({
format: new Mp4OutputFormat(),
target: new BufferTarget(),
});
const source = new EncodedAudioPacketSource('pcm-s16');
output.addAudioTrack(source);
await output.start();
const meta = { decoderConfig: { codec: 'pcm-s16', numberOfChannels: 2, sampleRate: 48000 } };
await source.add(new EncodedPacket(new Uint8Array(1024), 'key', 0, 1024 / 2 / 2 / 48000), meta);
// 0.006 is 256/48000 "rounded up", but it's close enough for silence padding not to kick in
await source.add(new EncodedPacket(new Uint8Array(1024), 'key', 0.006, 1024 / 2 / 2 / 48000), meta);
await output.finalize();
const input = new Input({
source: new BufferSource(output.target.buffer!),
formats: ALL_FORMATS,
});
const audioTrack = await input.getPrimaryAudioTrack();
assert(audioTrack);
expect(await audioTrack.getCodec()).toBe('pcm-s16');
const numChannels = await audioTrack.getNumberOfChannels();
const expectedFrameCount = 256 + 256;
const sink = new EncodedPacketSink(audioTrack);
let frameCount = 0;
for await (const packet of sink.packets()) {
frameCount += packet.byteLength / 2 / numChannels;
}
expect(frameCount).toBe(expectedFrameCount);
});