mirror of
https://github.com/arcodange-org/mediabunny.git
synced 2026-10-09 16:53:47 +02:00
Use edit lists to model non-zero start timestamps for regular MP4 (fixes #336); fix MP4 PCM logic breaking down with approximate packet timestamps
This commit is contained in:
+33
-5
@@ -17,14 +17,42 @@
|
|||||||
source: new Mediabunny.BlobSource(file),
|
source: new Mediabunny.BlobSource(file),
|
||||||
});
|
});
|
||||||
|
|
||||||
const track = await input.getPrimaryVideoTrack();
|
const track = await input.getPrimaryAudioTrack();
|
||||||
const sink = new Mediabunny.EncodedPacketSink(track);
|
const sink = new Mediabunny.EncodedPacketSink(track);
|
||||||
const packet = await sink.getFirstKeyPacket();
|
|
||||||
|
for await (const packet of sink.packets()) {
|
||||||
|
console.log(packet);
|
||||||
|
}
|
||||||
|
|
||||||
console.log(packet.data.join(', '));
|
const output = new Mediabunny.Output({
|
||||||
|
format: new Mediabunny.Mp4OutputFormat(),
|
||||||
|
target: new Mediabunny.BufferTarget(),
|
||||||
|
});
|
||||||
|
|
||||||
const config = await track.getDecoderConfig();
|
const conversion = await Mediabunny.Conversion.init({ input, output });
|
||||||
console.log(config.description.join(', '));
|
await conversion.execute();
|
||||||
|
|
||||||
|
return;
|
||||||
|
|
||||||
|
// Download it now
|
||||||
|
const blob = new Blob([output.target.buffer]);
|
||||||
|
const url = URL.createObjectURL(blob);
|
||||||
|
const a = document.createElement('a');
|
||||||
|
a.href = url;
|
||||||
|
a.download = file.name.replace(/\.\w+$/, '.mp4');
|
||||||
|
a.click();
|
||||||
|
URL.revokeObjectURL(url);
|
||||||
|
|
||||||
|
/*
|
||||||
|
const track = await input.getPrimaryAudioTrack();
|
||||||
|
const sink = new Mediabunny.EncodedPacketSink(track);
|
||||||
|
|
||||||
|
for await (const packet of sink.packets()) {
|
||||||
|
console.log(packet);
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log("Done")
|
||||||
|
*/
|
||||||
|
|
||||||
/*
|
/*
|
||||||
const input = new Mediabunny.Input({
|
const input = new Mediabunny.Input({
|
||||||
|
|||||||
@@ -174,6 +174,12 @@ const u64 = (value: number) => {
|
|||||||
return [bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7]] as number[];
|
return [bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7]] as number[];
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const i64 = (value: number) => {
|
||||||
|
view.setInt32(0, Math.floor(value / 2 ** 32), false);
|
||||||
|
view.setUint32(4, value, false);
|
||||||
|
return [bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7]] as number[];
|
||||||
|
};
|
||||||
|
|
||||||
const fixed_8_8 = (value: number) => {
|
const fixed_8_8 = (value: number) => {
|
||||||
view.setInt16(0, 2 ** 8 * value, false);
|
view.setInt16(0, 2 ** 8 * value, false);
|
||||||
return [bytes[0], bytes[1]] as number[];
|
return [bytes[0], bytes[1]] as number[];
|
||||||
@@ -384,11 +390,14 @@ export const mvhd = (
|
|||||||
creationTime: number,
|
creationTime: number,
|
||||||
trackDatas: IsobmffTrackData[],
|
trackDatas: IsobmffTrackData[],
|
||||||
) => {
|
) => {
|
||||||
const duration = intoTimescale(Math.max(
|
const duration = Math.max(
|
||||||
0,
|
0,
|
||||||
...trackDatas
|
...trackDatas
|
||||||
.map(x => presentationSpan(x)),
|
.map(trackData => (
|
||||||
), GLOBAL_TIMESCALE);
|
intoTimescale(presentationSpan(trackData), GLOBAL_TIMESCALE)
|
||||||
|
+ intoTimescale(trackData.startTimestampOffset ?? 0, GLOBAL_TIMESCALE)
|
||||||
|
)),
|
||||||
|
);
|
||||||
const nextTrackId = Math.max(0, ...trackDatas.map(x => x.track.id)) + 1;
|
const nextTrackId = Math.max(0, ...trackDatas.map(x => x.track.id)) + 1;
|
||||||
|
|
||||||
// Conditionally use u64 if u32 isn't enough
|
// Conditionally use u64 if u32 isn't enough
|
||||||
@@ -417,7 +426,9 @@ const presentationSpan = (trackData: IsobmffTrackData) => {
|
|||||||
let minTimestamp = Infinity;
|
let minTimestamp = Infinity;
|
||||||
let maxEndTimestamp = -Infinity;
|
let maxEndTimestamp = -Infinity;
|
||||||
|
|
||||||
for (const sample of trackData.samples) {
|
for (let i = 0; i < trackData.samples.length; i++) {
|
||||||
|
const sample = trackData.samples[i]!;
|
||||||
|
|
||||||
if (sample.timestamp < minTimestamp) {
|
if (sample.timestamp < minTimestamp) {
|
||||||
minTimestamp = sample.timestamp;
|
minTimestamp = sample.timestamp;
|
||||||
}
|
}
|
||||||
@@ -440,9 +451,11 @@ const presentationSpan = (trackData: IsobmffTrackData) => {
|
|||||||
*/
|
*/
|
||||||
export const trak = (trackData: IsobmffTrackData, creationTime: number) => {
|
export const trak = (trackData: IsobmffTrackData, creationTime: number) => {
|
||||||
const trackMetadata = getTrackMetadata(trackData);
|
const trackMetadata = getTrackMetadata(trackData);
|
||||||
|
const needsEditList = trackData.startTimestampOffset !== null && trackData.startTimestampOffset > 0;
|
||||||
|
|
||||||
return box('trak', undefined, [
|
return box('trak', undefined, [
|
||||||
tkhd(trackData, creationTime),
|
tkhd(trackData, creationTime),
|
||||||
|
needsEditList ? edts(trackData, trackData.startTimestampOffset!) : null,
|
||||||
mdia(trackData, creationTime),
|
mdia(trackData, creationTime),
|
||||||
trackMetadata.name !== undefined
|
trackMetadata.name !== undefined
|
||||||
? box('udta', undefined, [
|
? box('udta', undefined, [
|
||||||
@@ -459,10 +472,8 @@ export const tkhd = (
|
|||||||
trackData: IsobmffTrackData,
|
trackData: IsobmffTrackData,
|
||||||
creationTime: number,
|
creationTime: number,
|
||||||
) => {
|
) => {
|
||||||
const durationInGlobalTimescale = intoTimescale(
|
const durationInGlobalTimescale = intoTimescale(presentationSpan(trackData), GLOBAL_TIMESCALE)
|
||||||
presentationSpan(trackData),
|
+ intoTimescale(trackData.startTimestampOffset ?? 0, GLOBAL_TIMESCALE);
|
||||||
GLOBAL_TIMESCALE,
|
|
||||||
);
|
|
||||||
|
|
||||||
const needsU64 = !isU32(creationTime) || !isU32(durationInGlobalTimescale);
|
const needsU64 = !isU32(creationTime) || !isU32(durationInGlobalTimescale);
|
||||||
const u32OrU64 = needsU64 ? u64 : u32;
|
const u32OrU64 = needsU64 ? u64 : u32;
|
||||||
@@ -497,6 +508,32 @@ export const tkhd = (
|
|||||||
]);
|
]);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/** Edit Box: Specifies edits to the track's media. */
|
||||||
|
export const edts = (trackData: IsobmffTrackData, offset: number) => {
|
||||||
|
const startOffset = intoTimescale(offset, GLOBAL_TIMESCALE);
|
||||||
|
const mediaDuration = intoTimescale(presentationSpan(trackData), GLOBAL_TIMESCALE);
|
||||||
|
|
||||||
|
const needs64Bits = !isU32(startOffset) || !isU32(mediaDuration);
|
||||||
|
const u32OrU64 = needs64Bits ? u64 : u32;
|
||||||
|
const i32OrI64 = needs64Bits ? i64 : i32;
|
||||||
|
|
||||||
|
return box('edts', undefined, [
|
||||||
|
fullBox('elst', needs64Bits ? 1 : 0, 0, [
|
||||||
|
u32(2), // Entry count
|
||||||
|
|
||||||
|
// #1
|
||||||
|
u32OrU64(startOffset), // Segment duration
|
||||||
|
i32OrI64(-1), // Media time
|
||||||
|
fixed_16_16(1), // Media rate
|
||||||
|
|
||||||
|
// #2
|
||||||
|
u32OrU64(mediaDuration), // Segment duration
|
||||||
|
i32OrI64(0), // Media time
|
||||||
|
fixed_16_16(1), // Media rate
|
||||||
|
]),
|
||||||
|
]);
|
||||||
|
};
|
||||||
|
|
||||||
/** Media Box: Describes and define a track's media type and sample data. */
|
/** Media Box: Describes and define a track's media type and sample data. */
|
||||||
export const mdia = (trackData: IsobmffTrackData, creationTime: number) => box('mdia', undefined, [
|
export const mdia = (trackData: IsobmffTrackData, creationTime: number) => box('mdia', undefined, [
|
||||||
mdhd(trackData, creationTime),
|
mdhd(trackData, creationTime),
|
||||||
@@ -509,6 +546,7 @@ export const mdhd = (
|
|||||||
trackData: IsobmffTrackData,
|
trackData: IsobmffTrackData,
|
||||||
creationTime: number,
|
creationTime: number,
|
||||||
) => {
|
) => {
|
||||||
|
// Since the duration represents the raw media duration, edit list offsets are not taken into account here
|
||||||
const localDuration = intoTimescale(
|
const localDuration = intoTimescale(
|
||||||
presentationSpan(trackData),
|
presentationSpan(trackData),
|
||||||
trackData.timescale,
|
trackData.timescale,
|
||||||
|
|||||||
@@ -52,7 +52,7 @@ import {
|
|||||||
import { buildIsobmffMimeType } from './isobmff-misc';
|
import { buildIsobmffMimeType } from './isobmff-misc';
|
||||||
import { MAX_BOX_HEADER_SIZE, MIN_BOX_HEADER_SIZE } from './isobmff-reader';
|
import { MAX_BOX_HEADER_SIZE, MIN_BOX_HEADER_SIZE } from './isobmff-reader';
|
||||||
|
|
||||||
export const GLOBAL_TIMESCALE = 1000;
|
export const GLOBAL_TIMESCALE = 57600; // LCM of a bunch of common frame rates (24, 25, 30, 60, 144, ...)
|
||||||
const TIMESTAMP_OFFSET = 2_082_844_800; // Seconds between Jan 1 1904 and Jan 1 1970
|
const TIMESTAMP_OFFSET = 2_082_844_800; // Seconds between Jan 1 1904 and Jan 1 1970
|
||||||
|
|
||||||
export type Sample = {
|
export type Sample = {
|
||||||
@@ -85,6 +85,7 @@ export type IsobmffTrackData = {
|
|||||||
compositionTimeOffsetTable: { sampleCount: number; sampleCompositionTimeOffset: number }[];
|
compositionTimeOffsetTable: { sampleCount: number; sampleCompositionTimeOffset: number }[];
|
||||||
lastTimescaleUnits: number | null;
|
lastTimescaleUnits: number | null;
|
||||||
lastSample: Sample | null;
|
lastSample: Sample | null;
|
||||||
|
startTimestampOffset: number | null;
|
||||||
|
|
||||||
finalizedChunks: Chunk[];
|
finalizedChunks: Chunk[];
|
||||||
currentChunk: Chunk | null;
|
currentChunk: Chunk | null;
|
||||||
@@ -120,6 +121,7 @@ export type IsobmffTrackData = {
|
|||||||
* Some players expect this for PCM audio.
|
* Some players expect this for PCM audio.
|
||||||
*/
|
*/
|
||||||
requiresPcmTransformation: boolean;
|
requiresPcmTransformation: boolean;
|
||||||
|
expectedNextPcmPacketTimestamp: number | null;
|
||||||
/**
|
/**
|
||||||
* The "ADTS stripping" involves removing the ADTS header from each AAC packet. SOBMFF stores raw AAC data, not
|
* The "ADTS stripping" involves removing the ADTS header from each AAC packet. SOBMFF stores raw AAC data, not
|
||||||
* ADTS-wrapped data.
|
* ADTS-wrapped data.
|
||||||
@@ -395,7 +397,10 @@ export class IsobmffMuxer extends Muxer {
|
|||||||
// The frame rate set by the user may not be an integer. Since timescale is an integer, we'll approximate the
|
// The frame rate set by the user may not be an integer. Since timescale is an integer, we'll approximate the
|
||||||
// frame time (inverse of frame rate) with a rational number, then use that approximation's denominator
|
// frame time (inverse of frame rate) with a rational number, then use that approximation's denominator
|
||||||
// as the timescale.
|
// as the timescale.
|
||||||
const timescale = computeRationalApproximation(1 / (track.metadata.frameRate ?? 57600), 1e6).denominator;
|
const timescale = computeRationalApproximation(
|
||||||
|
1 / (track.metadata.frameRate ?? GLOBAL_TIMESCALE),
|
||||||
|
1e6,
|
||||||
|
).denominator;
|
||||||
|
|
||||||
const displayAspectWidth = decoderConfig.displayAspectWidth;
|
const displayAspectWidth = decoderConfig.displayAspectWidth;
|
||||||
const displayAspectHeight = decoderConfig.displayAspectHeight;
|
const displayAspectHeight = decoderConfig.displayAspectHeight;
|
||||||
@@ -425,6 +430,7 @@ export class IsobmffMuxer extends Muxer {
|
|||||||
compositionTimeOffsetTable: [],
|
compositionTimeOffsetTable: [],
|
||||||
lastTimescaleUnits: null,
|
lastTimescaleUnits: null,
|
||||||
lastSample: null,
|
lastSample: null,
|
||||||
|
startTimestampOffset: null,
|
||||||
finalizedChunks: [],
|
finalizedChunks: [],
|
||||||
currentChunk: null,
|
currentChunk: null,
|
||||||
compactlyCodedChunkTable: [],
|
compactlyCodedChunkTable: [],
|
||||||
@@ -494,6 +500,7 @@ export class IsobmffMuxer extends Muxer {
|
|||||||
requiresPcmTransformation:
|
requiresPcmTransformation:
|
||||||
!this.isFragmented
|
!this.isFragmented
|
||||||
&& (PCM_AUDIO_CODECS as readonly string[]).includes(track.source._codec),
|
&& (PCM_AUDIO_CODECS as readonly string[]).includes(track.source._codec),
|
||||||
|
expectedNextPcmPacketTimestamp: null,
|
||||||
requiresAdtsStripping,
|
requiresAdtsStripping,
|
||||||
firstPacket: packet,
|
firstPacket: packet,
|
||||||
},
|
},
|
||||||
@@ -505,6 +512,7 @@ export class IsobmffMuxer extends Muxer {
|
|||||||
compositionTimeOffsetTable: [],
|
compositionTimeOffsetTable: [],
|
||||||
lastTimescaleUnits: null,
|
lastTimescaleUnits: null,
|
||||||
lastSample: null,
|
lastSample: null,
|
||||||
|
startTimestampOffset: null,
|
||||||
finalizedChunks: [],
|
finalizedChunks: [],
|
||||||
currentChunk: null,
|
currentChunk: null,
|
||||||
compactlyCodedChunkTable: [],
|
compactlyCodedChunkTable: [],
|
||||||
@@ -547,6 +555,7 @@ export class IsobmffMuxer extends Muxer {
|
|||||||
compositionTimeOffsetTable: [],
|
compositionTimeOffsetTable: [],
|
||||||
lastTimescaleUnits: null,
|
lastTimescaleUnits: null,
|
||||||
lastSample: null,
|
lastSample: null,
|
||||||
|
startTimestampOffset: null,
|
||||||
finalizedChunks: [],
|
finalizedChunks: [],
|
||||||
currentChunk: null,
|
currentChunk: null,
|
||||||
compactlyCodedChunkTable: [],
|
compactlyCodedChunkTable: [],
|
||||||
@@ -629,41 +638,59 @@ export class IsobmffMuxer extends Muxer {
|
|||||||
packetData = packetData.subarray(headerLength);
|
packetData = packetData.subarray(headerLength);
|
||||||
}
|
}
|
||||||
|
|
||||||
const timestamp = this.validateAndNormalizeTimestamp(
|
let timestamp = this.validateAndNormalizeTimestamp(
|
||||||
trackData.track,
|
trackData.track,
|
||||||
packet.timestamp,
|
packet.timestamp,
|
||||||
packet.type === 'key',
|
packet.type === 'key',
|
||||||
);
|
);
|
||||||
|
let duration = packet.duration;
|
||||||
|
|
||||||
|
if (trackData.info.requiresPcmTransformation) {
|
||||||
|
// Packets may have only approximate timestamp/duration information, but for our PCM logic, we need it
|
||||||
|
// to be precise. So here, we refine the values.
|
||||||
|
|
||||||
|
const pcmInfo = parsePcmCodec(
|
||||||
|
trackData.info.decoderConfig.codec as PcmAudioCodec,
|
||||||
|
);
|
||||||
|
const frameSize = pcmInfo.sampleSize * trackData.info.numberOfChannels;
|
||||||
|
|
||||||
|
// Compute the precise duration
|
||||||
|
duration = packetData.byteLength / frameSize / trackData.info.sampleRate;
|
||||||
|
|
||||||
|
if (trackData.info.expectedNextPcmPacketTimestamp !== null) {
|
||||||
|
const diff = timestamp - trackData.info.expectedNextPcmPacketTimestamp;
|
||||||
|
if (diff < 0.01) {
|
||||||
|
timestamp = trackData.info.expectedNextPcmPacketTimestamp;
|
||||||
|
} else {
|
||||||
|
const paddedDuration = await this.padWithSilence(
|
||||||
|
trackData,
|
||||||
|
trackData.info.expectedNextPcmPacketTimestamp,
|
||||||
|
diff,
|
||||||
|
);
|
||||||
|
timestamp = trackData.info.expectedNextPcmPacketTimestamp + paddedDuration;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
trackData.info.expectedNextPcmPacketTimestamp = timestamp + duration;
|
||||||
|
}
|
||||||
|
|
||||||
const internalSample = this.createSampleForTrack(
|
const internalSample = this.createSampleForTrack(
|
||||||
trackData,
|
trackData,
|
||||||
packetData,
|
packetData,
|
||||||
timestamp,
|
timestamp,
|
||||||
packet.duration,
|
duration,
|
||||||
packet.type,
|
packet.type,
|
||||||
);
|
);
|
||||||
|
|
||||||
if (trackData.info.requiresPcmTransformation) {
|
|
||||||
await this.maybePadWithSilence(trackData, timestamp);
|
|
||||||
}
|
|
||||||
|
|
||||||
await this.registerSample(trackData, internalSample);
|
await this.registerSample(trackData, internalSample);
|
||||||
} finally {
|
} finally {
|
||||||
release();
|
release();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private async maybePadWithSilence(trackData: IsobmffAudioTrackData, untilTimestamp: number) {
|
private async padWithSilence(trackData: IsobmffAudioTrackData, timestamp: number, duration: number) {
|
||||||
// The PCM transformation assumes that all samples are contiguous. This is not something that is enforced, so
|
const deltaInTimescale = intoTimescale(duration, trackData.timescale);
|
||||||
// we need to pad the "holes" in between samples (and before the first sample) with additional
|
duration = deltaInTimescale / trackData.timescale;
|
||||||
// "silence samples".
|
|
||||||
|
|
||||||
const lastSample = last(trackData.samples);
|
|
||||||
const lastEndTimestamp = lastSample
|
|
||||||
? lastSample.timestamp + lastSample.duration
|
|
||||||
: 0;
|
|
||||||
|
|
||||||
const delta = untilTimestamp - lastEndTimestamp;
|
|
||||||
const deltaInTimescale = intoTimescale(delta, trackData.timescale);
|
|
||||||
|
|
||||||
if (deltaInTimescale > 0) {
|
if (deltaInTimescale > 0) {
|
||||||
const { sampleSize, silentValue } = parsePcmCodec(
|
const { sampleSize, silentValue } = parsePcmCodec(
|
||||||
@@ -675,12 +702,14 @@ export class IsobmffMuxer extends Muxer {
|
|||||||
const paddingSample = this.createSampleForTrack(
|
const paddingSample = this.createSampleForTrack(
|
||||||
trackData,
|
trackData,
|
||||||
new Uint8Array(data.buffer),
|
new Uint8Array(data.buffer),
|
||||||
lastEndTimestamp,
|
timestamp,
|
||||||
delta,
|
duration,
|
||||||
'key',
|
'key',
|
||||||
);
|
);
|
||||||
await this.registerSample(trackData, paddingSample);
|
await this.registerSample(trackData, paddingSample);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return duration;
|
||||||
}
|
}
|
||||||
|
|
||||||
async addSubtitleCue(track: OutputSubtitleTrack, cue: SubtitleCue, meta?: SubtitleMetadata) {
|
async addSubtitleCue(track: OutputSubtitleTrack, cue: SubtitleCue, meta?: SubtitleMetadata) {
|
||||||
@@ -821,6 +850,11 @@ export class IsobmffMuxer extends Muxer {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (trackData.type === 'audio' && trackData.info.requiresPcmTransformation) {
|
if (trackData.type === 'audio' && trackData.info.requiresPcmTransformation) {
|
||||||
|
if (!this.isFragmented) {
|
||||||
|
// The first timestamp is the lowest
|
||||||
|
trackData.startTimestampOffset ??= trackData.timestampProcessingQueue[0]!.timestamp;
|
||||||
|
}
|
||||||
|
|
||||||
let totalDuration = 0;
|
let totalDuration = 0;
|
||||||
|
|
||||||
// Compute the total duration in the track timescale (which is equal to the amount of PCM audio samples)
|
// Compute the total duration in the track timescale (which is equal to the amount of PCM audio samples)
|
||||||
@@ -848,6 +882,10 @@ export class IsobmffMuxer extends Muxer {
|
|||||||
|
|
||||||
const sortedTimestamps = trackData.timestampProcessingQueue.map(x => x.timestamp).sort((a, b) => a - b);
|
const sortedTimestamps = trackData.timestampProcessingQueue.map(x => x.timestamp).sort((a, b) => a - b);
|
||||||
|
|
||||||
|
if (!this.isFragmented) {
|
||||||
|
trackData.startTimestampOffset ??= sortedTimestamps[0]!;
|
||||||
|
}
|
||||||
|
|
||||||
for (let i = 0; i < trackData.timestampProcessingQueue.length; i++) {
|
for (let i = 0; i < trackData.timestampProcessingQueue.length; i++) {
|
||||||
const sample = trackData.timestampProcessingQueue[i]!;
|
const sample = trackData.timestampProcessingQueue[i]!;
|
||||||
|
|
||||||
@@ -857,12 +895,6 @@ export class IsobmffMuxer extends Muxer {
|
|||||||
// model it.
|
// model it.
|
||||||
sample.decodeTimestamp = sortedTimestamps[i]!;
|
sample.decodeTimestamp = sortedTimestamps[i]!;
|
||||||
|
|
||||||
if (!this.isFragmented && trackData.lastTimescaleUnits === null) {
|
|
||||||
// In non-fragmented files, the first decode timestamp is always zero. If the first presentation
|
|
||||||
// timestamp isn't zero, we'll simply use the composition time offset to achieve it.
|
|
||||||
sample.decodeTimestamp = 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
const sampleCompositionTimeOffset
|
const sampleCompositionTimeOffset
|
||||||
= intoTimescale(sample.timestamp - sample.decodeTimestamp, trackData.timescale);
|
= intoTimescale(sample.timestamp - sample.decodeTimestamp, trackData.timescale);
|
||||||
const durationInTimescale = intoTimescale(sample.duration, trackData.timescale);
|
const durationInTimescale = intoTimescale(sample.duration, trackData.timescale);
|
||||||
@@ -1377,6 +1409,17 @@ export class IsobmffMuxer extends Muxer {
|
|||||||
} else {
|
} else {
|
||||||
for (const trackData of this.trackDatas) {
|
for (const trackData of this.trackDatas) {
|
||||||
await this.finalizeCurrentChunk(trackData);
|
await this.finalizeCurrentChunk(trackData);
|
||||||
|
|
||||||
|
// Must hold because we will have processed at least one sample
|
||||||
|
assert(trackData.startTimestampOffset !== null);
|
||||||
|
|
||||||
|
// Shift all of the samples by the start offset. We'll then write out an edit list that will shift them
|
||||||
|
// back to their proper spot in the composition.
|
||||||
|
for (let i = 0; i < trackData.samples.length; i++) {
|
||||||
|
const sample = trackData.samples[i]!;
|
||||||
|
sample.timestamp -= trackData.startTimestampOffset;
|
||||||
|
sample.decodeTimestamp -= trackData.startTimestampOffset;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -9,6 +9,8 @@ import { BufferTarget } from '../../src/target.js';
|
|||||||
import { Mp4OutputFormat } from '../../src/output-format.js';
|
import { Mp4OutputFormat } from '../../src/output-format.js';
|
||||||
import { Conversion } from '../../src/conversion.js';
|
import { Conversion } from '../../src/conversion.js';
|
||||||
import { assert } from '../../src/misc.js';
|
import { assert } from '../../src/misc.js';
|
||||||
|
import { EncodedAudioPacketSource, EncodedVideoPacketSource } from '../../src/media-source.js';
|
||||||
|
import { EncodedPacket } from '../../src/packet.js';
|
||||||
|
|
||||||
const __dirname = new URL('.', import.meta.url).pathname;
|
const __dirname = new URL('.', import.meta.url).pathname;
|
||||||
|
|
||||||
@@ -104,3 +106,249 @@ test('Fragmented fMP4 with video+audio preserves B-frame CTS', async () => {
|
|||||||
|
|
||||||
expect(timestamps).toEqual(originalTimestamps);
|
expect(timestamps).toEqual(originalTimestamps);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('Zero start timestamp, regular MP4', async () => {
|
||||||
|
const output = new Output({
|
||||||
|
format: new Mp4OutputFormat(),
|
||||||
|
target: new BufferTarget(),
|
||||||
|
});
|
||||||
|
|
||||||
|
const source = new EncodedVideoPacketSource('vp8');
|
||||||
|
output.addVideoTrack(source);
|
||||||
|
|
||||||
|
await output.start();
|
||||||
|
|
||||||
|
const meta = { decoderConfig: { codec: 'vp8', codedWidth: 1280, codedHeight: 720 } };
|
||||||
|
|
||||||
|
await source.add(new EncodedPacket(new Uint8Array(1024), 'key', 0, 0.1), meta);
|
||||||
|
await source.add(new EncodedPacket(new Uint8Array(1024), 'delta', 0.1, 0.1));
|
||||||
|
await source.add(new EncodedPacket(new Uint8Array(1024), 'delta', 0.2, 0.1));
|
||||||
|
await source.add(new EncodedPacket(new Uint8Array(1024), 'delta', 0.3, 0.1));
|
||||||
|
|
||||||
|
await output.finalize();
|
||||||
|
|
||||||
|
// Hacky but works
|
||||||
|
const str = String.fromCharCode(...new Uint8Array(output.target.buffer!));
|
||||||
|
expect(str.includes('edts') || str.includes('elst')).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('Non-zero start timestamp, regular MP4', async () => {
|
||||||
|
const output = new Output({
|
||||||
|
format: new Mp4OutputFormat(),
|
||||||
|
target: new BufferTarget(),
|
||||||
|
});
|
||||||
|
|
||||||
|
const source = new EncodedVideoPacketSource('vp8');
|
||||||
|
output.addVideoTrack(source);
|
||||||
|
|
||||||
|
await output.start();
|
||||||
|
|
||||||
|
const meta = { decoderConfig: { codec: 'vp8', codedWidth: 1280, codedHeight: 720 } };
|
||||||
|
|
||||||
|
await source.add(new EncodedPacket(new Uint8Array(1024), 'key', 1, 0.1), meta);
|
||||||
|
await source.add(new EncodedPacket(new Uint8Array(1024), 'delta', 1.1, 0.1));
|
||||||
|
await source.add(new EncodedPacket(new Uint8Array(1024), 'delta', 1.2, 0.1));
|
||||||
|
await source.add(new EncodedPacket(new Uint8Array(1024), 'delta', 1.3, 0.1));
|
||||||
|
|
||||||
|
await output.finalize();
|
||||||
|
|
||||||
|
// Hacky but works
|
||||||
|
const str = String.fromCharCode(...new Uint8Array(output.target.buffer!));
|
||||||
|
expect(str.includes('edts') && str.includes('elst')).toBe(true);
|
||||||
|
|
||||||
|
using input = new Input({
|
||||||
|
source: new BufferSource(output.target.buffer!),
|
||||||
|
formats: ALL_FORMATS,
|
||||||
|
});
|
||||||
|
|
||||||
|
const track = await input.getPrimaryVideoTrack();
|
||||||
|
assert(track);
|
||||||
|
const sink = new EncodedPacketSink(track);
|
||||||
|
|
||||||
|
const timestamps: number[] = [];
|
||||||
|
const durations: number[] = [];
|
||||||
|
for await (const packet of sink.packets()) {
|
||||||
|
timestamps.push(packet.timestamp);
|
||||||
|
durations.push(packet.duration);
|
||||||
|
}
|
||||||
|
|
||||||
|
expect(timestamps).toEqual([1, 1.1, 1.2, 1.3]);
|
||||||
|
expect(durations).toEqual([0.1, 0.1, 0.1, 0.1]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('Non-zero start timestamp, fragmented MP4', async () => {
|
||||||
|
const output = new Output({
|
||||||
|
format: new Mp4OutputFormat({ fastStart: 'fragmented' }),
|
||||||
|
target: new BufferTarget(),
|
||||||
|
});
|
||||||
|
|
||||||
|
const source = new EncodedVideoPacketSource('vp8');
|
||||||
|
output.addVideoTrack(source);
|
||||||
|
|
||||||
|
await output.start();
|
||||||
|
|
||||||
|
const meta = { decoderConfig: { codec: 'vp8', codedWidth: 1280, codedHeight: 720 } };
|
||||||
|
|
||||||
|
await source.add(new EncodedPacket(new Uint8Array(1024), 'key', 1, 0.1), meta);
|
||||||
|
await source.add(new EncodedPacket(new Uint8Array(1024), 'delta', 1.1, 0.1));
|
||||||
|
await source.add(new EncodedPacket(new Uint8Array(1024), 'delta', 1.2, 0.1));
|
||||||
|
await source.add(new EncodedPacket(new Uint8Array(1024), 'delta', 1.3, 0.1));
|
||||||
|
|
||||||
|
await output.finalize();
|
||||||
|
|
||||||
|
// Hacky but works
|
||||||
|
const str = String.fromCharCode(...new Uint8Array(output.target.buffer!));
|
||||||
|
expect(str.includes('edts') || str.includes('elst')).toBe(false);
|
||||||
|
|
||||||
|
using input = new Input({
|
||||||
|
source: new BufferSource(output.target.buffer!),
|
||||||
|
formats: ALL_FORMATS,
|
||||||
|
});
|
||||||
|
|
||||||
|
const track = await input.getPrimaryVideoTrack();
|
||||||
|
assert(track);
|
||||||
|
const sink = new EncodedPacketSink(track);
|
||||||
|
|
||||||
|
const timestamps: number[] = [];
|
||||||
|
const durations: number[] = [];
|
||||||
|
for await (const packet of sink.packets()) {
|
||||||
|
timestamps.push(packet.timestamp);
|
||||||
|
durations.push(packet.duration);
|
||||||
|
}
|
||||||
|
|
||||||
|
expect(timestamps).toEqual([1, 1.1, 1.2, 1.3]);
|
||||||
|
expect(durations).toEqual([0.1, 0.1, 0.1, 0.1]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('PCM audio', async () => {
|
||||||
|
const output = new Output({
|
||||||
|
format: new Mp4OutputFormat(),
|
||||||
|
target: new BufferTarget(),
|
||||||
|
});
|
||||||
|
|
||||||
|
const source = new EncodedAudioPacketSource('pcm-s16');
|
||||||
|
output.addAudioTrack(source);
|
||||||
|
|
||||||
|
await output.start();
|
||||||
|
|
||||||
|
const meta = { decoderConfig: { codec: 'pcm-s16', numberOfChannels: 2, sampleRate: 48000 } };
|
||||||
|
|
||||||
|
await source.add(new EncodedPacket(new Uint8Array(1024), 'key', 0, 1024 / 2 / 2 / 48000), meta);
|
||||||
|
|
||||||
|
await output.finalize();
|
||||||
|
|
||||||
|
const input = new Input({
|
||||||
|
source: new BufferSource(output.target.buffer!),
|
||||||
|
formats: ALL_FORMATS,
|
||||||
|
});
|
||||||
|
const audioTrack = await input.getPrimaryAudioTrack();
|
||||||
|
assert(audioTrack);
|
||||||
|
|
||||||
|
expect(await audioTrack.getFirstTimestamp()).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('PCM audio with non-zero timestamp', async () => {
|
||||||
|
const output = new Output({
|
||||||
|
format: new Mp4OutputFormat(),
|
||||||
|
target: new BufferTarget(),
|
||||||
|
});
|
||||||
|
|
||||||
|
const source = new EncodedAudioPacketSource('pcm-s16');
|
||||||
|
output.addAudioTrack(source);
|
||||||
|
|
||||||
|
await output.start();
|
||||||
|
|
||||||
|
const meta = { decoderConfig: { codec: 'pcm-s16', numberOfChannels: 2, sampleRate: 48000 } };
|
||||||
|
|
||||||
|
await source.add(new EncodedPacket(new Uint8Array(1024), 'key', 1, 1024 / 2 / 2 / 48000), meta);
|
||||||
|
|
||||||
|
await output.finalize();
|
||||||
|
|
||||||
|
const input = new Input({
|
||||||
|
source: new BufferSource(output.target.buffer!),
|
||||||
|
formats: ALL_FORMATS,
|
||||||
|
});
|
||||||
|
const audioTrack = await input.getPrimaryAudioTrack();
|
||||||
|
assert(audioTrack);
|
||||||
|
|
||||||
|
expect(await audioTrack.getFirstTimestamp()).toBe(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('PCM audio, silence padding', async () => {
|
||||||
|
const output = new Output({
|
||||||
|
format: new Mp4OutputFormat(),
|
||||||
|
target: new BufferTarget(),
|
||||||
|
});
|
||||||
|
|
||||||
|
const source = new EncodedAudioPacketSource('pcm-s16');
|
||||||
|
output.addAudioTrack(source);
|
||||||
|
|
||||||
|
await output.start();
|
||||||
|
|
||||||
|
const meta = { decoderConfig: { codec: 'pcm-s16', numberOfChannels: 2, sampleRate: 48000 } };
|
||||||
|
|
||||||
|
await source.add(new EncodedPacket(new Uint8Array(1024), 'key', 0, 1024 / 2 / 2 / 48000), meta);
|
||||||
|
await source.add(new EncodedPacket(new Uint8Array(1024), 'key', 1, 1024 / 2 / 2 / 48000), meta);
|
||||||
|
|
||||||
|
await output.finalize();
|
||||||
|
|
||||||
|
const input = new Input({
|
||||||
|
source: new BufferSource(output.target.buffer!),
|
||||||
|
formats: ALL_FORMATS,
|
||||||
|
});
|
||||||
|
const audioTrack = await input.getPrimaryAudioTrack();
|
||||||
|
assert(audioTrack);
|
||||||
|
|
||||||
|
expect(await audioTrack.getCodec()).toBe('pcm-s16');
|
||||||
|
const numChannels = await audioTrack.getNumberOfChannels();
|
||||||
|
|
||||||
|
const expectedFrameCount = 48000 + 256;
|
||||||
|
const sink = new EncodedPacketSink(audioTrack);
|
||||||
|
let frameCount = 0;
|
||||||
|
|
||||||
|
for await (const packet of sink.packets()) {
|
||||||
|
frameCount += packet.byteLength / 2 / numChannels;
|
||||||
|
}
|
||||||
|
|
||||||
|
expect(frameCount).toBe(expectedFrameCount);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('PCM audio, no silence padding with approximate timestamps', async () => {
|
||||||
|
const output = new Output({
|
||||||
|
format: new Mp4OutputFormat(),
|
||||||
|
target: new BufferTarget(),
|
||||||
|
});
|
||||||
|
|
||||||
|
const source = new EncodedAudioPacketSource('pcm-s16');
|
||||||
|
output.addAudioTrack(source);
|
||||||
|
|
||||||
|
await output.start();
|
||||||
|
|
||||||
|
const meta = { decoderConfig: { codec: 'pcm-s16', numberOfChannels: 2, sampleRate: 48000 } };
|
||||||
|
|
||||||
|
await source.add(new EncodedPacket(new Uint8Array(1024), 'key', 0, 1024 / 2 / 2 / 48000), meta);
|
||||||
|
// 0.006 is 256/48000 "rounded up", but it's close enough for silence padding not to kick in
|
||||||
|
await source.add(new EncodedPacket(new Uint8Array(1024), 'key', 0.006, 1024 / 2 / 2 / 48000), meta);
|
||||||
|
|
||||||
|
await output.finalize();
|
||||||
|
|
||||||
|
const input = new Input({
|
||||||
|
source: new BufferSource(output.target.buffer!),
|
||||||
|
formats: ALL_FORMATS,
|
||||||
|
});
|
||||||
|
const audioTrack = await input.getPrimaryAudioTrack();
|
||||||
|
assert(audioTrack);
|
||||||
|
|
||||||
|
expect(await audioTrack.getCodec()).toBe('pcm-s16');
|
||||||
|
const numChannels = await audioTrack.getNumberOfChannels();
|
||||||
|
|
||||||
|
const expectedFrameCount = 256 + 256;
|
||||||
|
const sink = new EncodedPacketSink(audioTrack);
|
||||||
|
let frameCount = 0;
|
||||||
|
|
||||||
|
for await (const packet of sink.packets()) {
|
||||||
|
frameCount += packet.byteLength / 2 / numChannels;
|
||||||
|
}
|
||||||
|
|
||||||
|
expect(frameCount).toBe(expectedFrameCount);
|
||||||
|
});
|
||||||
|
|||||||
Reference in New Issue
Block a user