diff --git a/dev/convert.html b/dev/convert.html
index 5f30d71..825693d 100644
--- a/dev/convert.html
+++ b/dev/convert.html
@@ -25,7 +25,7 @@
chunked: true,
chunkSize: 2**20
});
- const outputFormat = new Mediabunny.MovOutputFormat();
+ const outputFormat = new Mediabunny.Mp4OutputFormat();
const p = document.createElement('p');
p.textContent = 'Capturing...';
@@ -99,6 +99,8 @@
input,
output,
audio: {
+ discard: true,
+ //discard: true,
//codec: 'aac',
//forceTranscode: true,
//forceTranscode: true,
@@ -129,7 +131,9 @@
},
*/
video: {
- height: 320,
+ forceTranscode: true,
+ //forceTranscode: true,
+ //height: 320,
//codec: 'avc',
//bitrate: new Mediabunny.Quality({
// quality: Infinity,
@@ -158,7 +162,8 @@
}
},
trim: {
- end: 5,
+ //start: -60,
+ //end: 60,
//start: -2,
//end: 10,
//start: 300.14984567374756 - 100,
@@ -167,6 +172,12 @@
//start: startTime,
//end: startTime + 2,
},
+ copy: {
+ //mode: 'forced',
+ //mode: 'forced',
+ //shiftTolerance: Infinity,
+ //boundaryPolicy: 'shrink',
+ },
});
//console.log(conversion);
console.log(conversion.discardedTracks);
@@ -213,4 +224,4 @@
conversion = null;
input = null;
}, { once: true });
-
\ No newline at end of file
+
diff --git a/src/conversion.ts b/src/conversion.ts
index b8690bc..14b3663 100644
--- a/src/conversion.ts
+++ b/src/conversion.ts
@@ -44,6 +44,7 @@ import {
ceilToMultipleOfTwo,
clamp,
isIso639Dash2LanguageCode,
+ isNumber,
MaybePromise,
normalizeRotation,
promiseWithResolvers,
@@ -51,6 +52,7 @@ import {
} from './misc';
import { Output, OutputTrackGroup } from './output';
import { Mp4OutputFormat } from './output-format';
+import { EncodedPacket } from './packet';
import {
AudioSample,
clampCropRectangle,
@@ -132,6 +134,13 @@ export type ConversionOptions = {
end?: number;
};
+ /**
+ * Options for controlling when media is copied directly without transcoding it. Set to `false` to always transcode.
+ * Defaults to `{}`, which will copy media whenever possible and otherwise transcode it while retaining precise
+ * timestamps.
+ */
+ copy?: ConversionCopyOptions | false;
+
/**
* An object or a callback that returns or resolves to an object containing the descriptive metadata tags that
* should be written to the output file. If a function is passed, it will be passed the tags of the input file as
@@ -237,7 +246,7 @@ export type ConversionVideoOptions = {
* `'no-preference'`, the default.
*/
hardwareAcceleration?: 'no-preference' | 'prefer-hardware' | 'prefer-software';
- /** When `true`, video will always be re-encoded instead of directly copying over the encoded samples. */
+ /** When `true`, video will always be re-encoded instead of directly copying over the encoded packets. */
forceTranscode?: boolean;
/**
* Allows for custom user-defined processing of video frames, e.g. for applying overlays, color transformations, or
@@ -303,7 +312,7 @@ export type ConversionAudioOptions = {
* @deprecated Use `quality` instead.
*/
bitrate?: number | Quality;
- /** When `true`, audio will always be re-encoded instead of directly copying over the encoded samples. */
+ /** When `true`, audio will always be re-encoded instead of directly copying over the encoded packets. */
forceTranscode?: boolean;
/**
* Allows for custom user-defined processing of audio samples, e.g. for applying audio effects, transformations, or
@@ -338,6 +347,44 @@ export type ConversionAudioOptions = {
group?: OutputTrackGroup | OutputTrackGroup[];
};
+/**
+ * Options for copying encoded media during conversion.
+ * @group Conversion
+ * @public
+ */
+export type ConversionCopyOptions = {
+ /**
+ * Controls whether media copying is preferred or required. Defaults to `'preferred'`.
+ *
+ * - `'forced'`: Copy encoded media where possible and discard tracks that cannot possibly be copied.
+ * - `'preferred'`: Copy encoded media when possible, and transcode tracks that cannot be copied.
+ */
+ mode?: 'forced' | 'preferred';
+ /**
+ * The maximum absolute shift, in seconds, that may be applied to the media to be able to copy it into the output
+ * format. Defaults to `0`, which permits no additional shift. Set to `Infinity` to permit any shift.
+ *
+ * A shift of `0` gives you perfect _timeline sync_: output timestamps will match input timestamps exactly (only
+ * offset by the trim region). Any non-zero shift will break this property but will still, under all circumstances,
+ * maintain perfect cross-track and audio-video sync.
+ */
+ shiftTolerance?: number;
+ /**
+ * Controls which media region will be copied to satisfy the requested trim range. Defaults to `'expand'`.
+ *
+ * - `'expand'`: Include at least all media in the requested range. This may require expanding the media region due
+ * to key frames and packet boundaries, and thus may include media outside of your trim range. The region is always
+ * minimally expanded to satisfy the copy criteria.
+ * - `'shrink'`: Only include media that lies entirely within the requested trim range. This may require shrinking
+ * the media region due to key frames and packet boundaries, and thus may exclude media inside of your trim range.
+ * The region is always minimally shrunk to satisfy the copy criteria.
+ *
+ * Use `expand` if you don't want to lose any media; use `shrink` to never expose any media outside of the
+ * trim region.
+ */
+ boundaryPolicy?: 'expand' | 'shrink';
+};
+
const validateVideoOptions = (videoOptions: ConversionVideoOptions) => {
if (!videoOptions || typeof videoOptions !== 'object') {
throw new TypeError('options.video, when provided, must be an object.');
@@ -546,6 +593,8 @@ export type DiscardedTrack = {
* - `'no_encodable_target_codec'`: We can't find a codec that we are able to encode and that can be contained
* within the output format. This reason can be hit if the environment doesn't support the necessary encoders, or if
* you requested a codec that cannot be contained within the output format.
+ * - `'cannot_copy'`: {@link ConversionCopyOptions.mode} was set to `'forced'` but the track could not be copied
+ * with the given copy configuration because it would require a transcode instead.
*/
reason:
| 'discarded_by_user'
@@ -553,7 +602,8 @@ export type DiscardedTrack = {
| 'max_track_count_of_type_reached'
| 'unknown_source_codec'
| 'undecodable_source_codec'
- | 'no_encodable_target_codec';
+ | 'no_encodable_target_codec'
+ | 'cannot_copy';
/** The options that were provided for this track, or `{}` if none were provided. */
trackOptions: ConversionVideoOptions | ConversionAudioOptions;
};
@@ -597,22 +647,28 @@ export class Conversion {
/** The output file. */
readonly output: Output;
- /**
- * The current state of the conversion.
- *
- * - `'idle'`: The conversion is not currently executing and isn't done; `execute` can be called.
- * - `'executing'`: A call to `execute` is currently running.
- * - `'canceled'`: The conversion has been canceled and can no longer be executed.
- * - `'done'`: The conversion has run to completion. Subsequent calls to `execute` do nothing.
- */
- state: 'idle' | 'executing' | 'canceled' | 'done' = 'idle';
-
+ /** @internal */
+ _state: 'idle' | 'executing' | 'canceled' | 'done' = 'idle';
/** @internal */
_options: ConversionOptions;
/** @internal */
+ _copyMode: false | 'forced' | 'preferred';
+ /** @internal */
+ _copyTimestampShiftTolerance: number;
+ /** @internal */
+ _copyBoundaryPolicy: 'expand' | 'shrink';
+ /** @internal */
_startTimestamp!: number;
/** @internal */
_endTimestamp!: number;
+ /** @internal */
+ _timestampOffset = 0;
+ /** @internal */
+ _timestampOffsetAdjusted = false;
+ /** @internal */
+ _copyTimestampPossible = new Map();
+ /** @internal */
+ _copyStartPackets = new Map();
/** @internal */
_nextOutputTrackId = 0;
@@ -673,6 +729,18 @@ export class Conversion {
/** The list of tracks from the input file that have been discarded, alongside the discard reason. */
readonly discardedTracks: DiscardedTrack[] = [];
+ /**
+ * The current state of the conversion.
+ *
+ * - `'idle'`: The conversion is not currently executing and isn't done; `execute` can be called.
+ * - `'executing'`: A call to `execute` is currently running.
+ * - `'canceled'`: The conversion has been canceled and can no longer be executed.
+ * - `'done'`: The conversion has run to completion. Subsequent calls to `execute` do nothing.
+ */
+ get state() {
+ return this._state;
+ }
+
/** Initializes a new conversion process without starting the conversion. */
static async init(options: ConversionOptions) {
const conversion = new Conversion(options);
@@ -704,6 +772,28 @@ export class Conversion {
if (options.composable !== undefined && typeof options.composable !== 'boolean') {
throw new TypeError('options.composable, when provided, must be a boolean.');
}
+ if (options.copy !== undefined && options.copy !== false) {
+ if (!options.copy || typeof options.copy !== 'object') {
+ throw new TypeError('options.copy, when provided, must be an object or false.');
+ }
+ if (options.copy.mode !== undefined && !['forced', 'preferred'].includes(options.copy.mode)) {
+ throw new TypeError('options.copy.mode, when provided, must be \'forced\' or \'preferred\'.');
+ }
+ if (
+ options.copy.shiftTolerance !== undefined
+ && (!isNumber(options.copy.shiftTolerance) || options.copy.shiftTolerance < 0)
+ ) {
+ throw new TypeError('options.copy.shiftTolerance, when provided, must be a non-negative number.');
+ }
+ if (
+ options.copy.boundaryPolicy !== undefined
+ && !['expand', 'shrink'].includes(options.copy.boundaryPolicy)
+ ) {
+ throw new TypeError(
+ 'options.copy.boundaryPolicy, when provided, must be \'expand\' or \'shrink\'.',
+ );
+ }
+ }
const composable = options.composable ?? false;
if (!composable) {
@@ -757,8 +847,8 @@ export class Conversion {
if (options.trim?.start !== undefined && (!Number.isFinite(options.trim.start))) {
throw new TypeError('options.trim.start, when provided, must be a finite number.');
}
- if (options.trim?.end !== undefined && (!Number.isFinite(options.trim.end))) {
- throw new TypeError('options.trim.end, when provided, must be a finite number.');
+ if (options.trim?.end !== undefined && (!isNumber(options.trim.end))) {
+ throw new TypeError('options.trim.end, when provided, must be a number.');
}
if (
options.trim?.start !== undefined
@@ -781,6 +871,9 @@ export class Conversion {
}
this._options = options;
+ this._copyMode = options.copy === false ? false : options.copy?.mode ?? 'preferred';
+ this._copyTimestampShiftTolerance = options.copy === false ? 0 : options.copy?.shiftTolerance ?? 0;
+ this._copyBoundaryPolicy = options.copy === false ? 'expand' : options.copy?.boundaryPolicy ?? 'expand';
this._composable = composable;
this.input = options.input;
this.output = options.output;
@@ -924,6 +1017,7 @@ export class Conversion {
}
this._endTimestamp = Math.max(this._options.trim?.end ?? Infinity, this._startTimestamp);
+ this._timestampOffset = -this._startTimestamp; // Initial value, may get refined later by track processing
// Run these sequentially so that output tracks have a deterministic order
for (let i = 0; i < filteredTracks.length; i++) {
@@ -1156,15 +1250,15 @@ export class Conversion {
);
}
- if (this.state === 'executing') {
+ if (this._state === 'executing') {
throw new Error('Cannot call execute() while a previous call to execute() is still running.');
}
- if (this.state === 'canceled') {
+ if (this._state === 'canceled') {
throw new ConversionCanceledError();
}
- if (this.state === 'done') {
+ if (this._state === 'done') {
// The conversion already ran to completion, nothing left to do
return;
}
@@ -1176,12 +1270,12 @@ export class Conversion {
);
}
- this.state = 'executing';
+ this._state = 'executing';
this._executionUntil = options.until ?? Infinity;
this._pauseRequested = options.pauseSignal?.aborted ?? false;
const onPause = () => {
- if (this.state !== 'executing') {
+ if (this._state !== 'executing') {
return;
}
@@ -1224,6 +1318,9 @@ export class Conversion {
);
for (const id of this._outputTrackIds) {
+ // Used for progress calculation. We start these at 0 which is technically not always the first
+ // timestamp, but this is how we choose to model what "progress" means: it's how far we are done
+ // with the trim region.
this._maxTimestamps.set(id, 0);
}
@@ -1247,7 +1344,7 @@ export class Conversion {
try {
await Promise.all(this._trackPumps.map(x => x.resolvers.promise));
} catch (error) {
- if ((this.state as Conversion['state']) !== 'canceled') {
+ if ((this._state as Conversion['_state']) !== 'canceled') {
// Make sure to cancel to stop other encoding processes and clean up resources
void this.cancel();
}
@@ -1257,12 +1354,12 @@ export class Conversion {
options.pauseSignal?.removeEventListener('abort', onPause);
}
- if ((this.state as Conversion['state']) === 'canceled') {
+ if ((this._state as Conversion['_state']) === 'canceled') {
throw new ConversionCanceledError();
}
const isDone = this._trackPumps.every(x => x.done);
- this.state = isDone ? 'done' : 'idle';
+ this._state = isDone ? 'done' : 'idle';
if (isDone) {
if (!this._composable) {
@@ -1281,16 +1378,16 @@ export class Conversion {
* Does nothing if the conversion is already complete.
*/
async cancel() {
- if (this.state === 'done') {
+ if (this._state === 'done') {
return;
}
- if (this.state === 'canceled') {
+ if (this._state === 'canceled') {
Logging._warn('Conversion already canceled.');
return;
}
- this.state = 'canceled';
+ this._state = 'canceled';
// Wake all suspended track pumps so they can wind down
for (const pump of this._trackPumps) {
@@ -1355,11 +1452,11 @@ export class Conversion {
height = ceilToMultipleOfTwo(trackOptions.height);
}
- const firstTimestamp = await track.getFirstTimestamp();
let videoCodecs = this.output.format.getSupportedVideoCodecs();
+ const alpha = trackOptions.alpha ?? 'discard';
- const needsTranscode = !!trackOptions.forceTranscode
- || firstTimestamp < this._startTimestamp
+ let needsTranscode = !this._copyMode
+ || !!trackOptions.forceTranscode
|| !!trackOptions.frameRate
|| trackOptions.keyFrameInterval !== undefined
|| trackOptions.process !== undefined
@@ -1376,7 +1473,92 @@ export class Conversion {
|| (totalRotation !== 0 && !canUseRotationMetadata)
|| !!crop;
- const alpha = trackOptions.alpha ?? 'discard';
+ let copyStartPacket: EncodedPacket | null = null;
+
+ if (!needsTranscode) {
+ // Check if we can copy it
+ const sink = new EncodedPacketSink(track);
+ let startPacket = await sink.getKeyPacket(this._startTimestamp, { verifyKeyPackets: true })
+ ?? await sink.getFirstKeyPacket({ verifyKeyPackets: true });
+
+ if (
+ startPacket
+ && startPacket.timestamp < this._startTimestamp
+ && startPacket.timestamp + startPacket.duration <= this._startTimestamp
+ && this._copyBoundaryPolicy === 'shrink'
+ ) {
+ startPacket = await sink.getNextKeyPacket(startPacket, { verifyKeyPackets: true });
+ }
+
+ copyStartPacket = startPacket;
+
+ if (startPacket) {
+ // This clamp mirrors the packet timestamp clamping the copy loop does. The reason this is valid is
+ // because in the shrink case, we've already proven that the packet (at least partially) overlaps the
+ // trim region.
+ const effectiveStartTimestamp = this._copyBoundaryPolicy === 'shrink'
+ ? Math.max(startPacket.timestamp, this._startTimestamp)
+ : startPacket.timestamp;
+
+ if (!this.output.format.supportsTimestampedMediaData) {
+ // Wants zero
+
+ if (this._timestampOffsetAdjusted) {
+ // We've already adjusted, we can't adjust twice
+ const isValid = effectiveStartTimestamp + this._timestampOffset === 0;
+ if (!isValid) {
+ needsTranscode = true;
+ }
+ } else {
+ const correction = clamp(
+ this._startTimestamp - effectiveStartTimestamp,
+ -this._copyTimestampShiftTolerance,
+ this._copyTimestampShiftTolerance,
+ );
+
+ const shiftedStartTimestamp = effectiveStartTimestamp + correction;
+ const isValid = shiftedStartTimestamp === this._startTimestamp;
+
+ if (isValid) {
+ this._timestampOffset = -this._startTimestamp + correction;
+ this._timestampOffsetAdjusted = true;
+ } else {
+ needsTranscode = true;
+ }
+ }
+ } else if (
+ this.output.format.negativeTimestampSupport !== 'full'
+ && effectiveStartTimestamp < this._startTimestamp
+ ) {
+ const correction = Math.min(
+ this._startTimestamp - effectiveStartTimestamp,
+ this._copyTimestampShiftTolerance,
+ );
+
+ const shiftedStartTimestamp = effectiveStartTimestamp + correction;
+ const isValid = shiftedStartTimestamp >= this._startTimestamp
+ || (
+ this.output.format.negativeTimestampSupport === 'prefer-non-negative'
+ && this._copyMode === 'forced'
+ );
+
+ if (isValid) {
+ this._timestampOffset = Math.max(this._timestampOffset, -this._startTimestamp + correction);
+ } else {
+ needsTranscode = true;
+ }
+ }
+ }
+ }
+
+ if (needsTranscode && this._copyMode === 'forced') {
+ this.discardedTracks.push({
+ track,
+ reason: 'cannot_copy',
+ trackOptions,
+ });
+ return;
+ }
if (!needsTranscode) {
// Fast path, we can simply copy over the encoded packets
@@ -1389,22 +1571,66 @@ export class Conversion {
const decoderConfig = await track.getDecoderConfig();
const meta: EncodedVideoChunkMetadata = { decoderConfig: decoderConfig ?? undefined };
- for await (const packet of sink.packets(undefined, undefined, { verifyKeyPackets: true })) {
- if (this.state === 'canceled') {
+ // eslint-disable-next-line curly
+ if (copyStartPacket) for await (const packet of sink.packets(
+ copyStartPacket,
+ undefined,
+ { verifyKeyPackets: true },
+ )) {
+ if (this._state === 'canceled') {
break;
}
if (packet.timestamp >= this._endTimestamp) {
- break;
+ if (this._copyBoundaryPolicy === 'shrink') {
+ break;
+ } else {
+ // Due to B-frames, there might still be packets we care about later on. Do a short
+ // lookahead to find out if there are.
+
+ let current = packet;
+ let found = false;
+ const lookahead = 6; // Heuristic, but should be enough for most streams
+
+ for (let i = 0; i < lookahead; i++) {
+ const next = await sink.getNextPacket(current, { metadataOnly: true });
+ if (!next) {
+ break;
+ }
+
+ if (next.timestamp < this._endTimestamp) {
+ found = true;
+ break;
+ }
+
+ current = next;
+ }
+
+ if (!found) {
+ break;
+ }
+ }
}
+ let packetStartTimestamp = packet.timestamp;
+ let packetEndTimestamp = packet.timestamp + packet.duration;
+
+ if (this._copyBoundaryPolicy === 'shrink') {
+ packetStartTimestamp = Math.max(packetStartTimestamp, this._startTimestamp);
+ packetEndTimestamp = Math.min(packetEndTimestamp, this._endTimestamp);
+ packetEndTimestamp = Math.max(packetEndTimestamp, packetStartTimestamp); // Just in case
+ }
+
+ packetStartTimestamp += this._timestampOffset;
+ packetEndTimestamp += this._timestampOffset;
+
const modifiedPacket = packet.clone({
- timestamp: packet.timestamp - this._startTimestamp,
+ timestamp: packetStartTimestamp,
+ duration: packetEndTimestamp - packetStartTimestamp,
sideData: alpha === 'discard'
? {} // Remove alpha side data
: packet.sideData,
});
- assert(modifiedPacket.timestamp >= 0);
this._reportProgress(outputTrackId, modifiedPacket.timestamp + modifiedPacket.duration);
await source.add(modifiedPacket, meta);
@@ -1496,8 +1722,9 @@ export class Conversion {
await tempOutput.start();
+ // Let's just use the first sample to test
const sink = new VideoSampleSink(track);
- using firstSample = await sink.getSample(firstTimestamp); // Let's just use the first sample
+ using firstSample = await sink.getSample(await track.getFirstTimestamp());
if (firstSample) {
try {
@@ -1550,11 +1777,11 @@ export class Conversion {
const sink = new VideoSampleSink(track);
for await (using sample of sink.samples(this._startTimestamp, this._endTimestamp)) {
- if (this.state === 'canceled') {
+ if (this._state === 'canceled') {
break;
}
- const adjustedSampleTimestamp = Math.max(sample.timestamp - this._startTimestamp, 0);
+ const adjustedSampleTimestamp = Math.max(0, sample.timestamp + this._timestampOffset);
sample.setTimestamp(adjustedSampleTimestamp);
this._reportProgress(outputTrackId, sample.timestamp + sample.duration);
@@ -1583,12 +1810,17 @@ export class Conversion {
}
const videoTrackLanguageCode = await track.getLanguageCode();
+ const trackName = await track.getName();
+ const trackDisposition = await track.getDisposition();
+
this.output.addVideoTrack(videoSource, {
frameRate: trackOptions.frameRate,
// TODO: This condition can be removed when all demuxers properly homogenize to BCP47 in v2
- languageCode: isIso639Dash2LanguageCode(videoTrackLanguageCode) ? videoTrackLanguageCode : undefined,
- name: await track.getName() ?? undefined,
- disposition: await track.getDisposition(),
+ languageCode: isIso639Dash2LanguageCode(videoTrackLanguageCode)
+ ? videoTrackLanguageCode
+ : undefined,
+ name: trackName ?? undefined,
+ disposition: trackDisposition,
rotation: outputTrackRotation,
group: ownGroup ?? trackOptions.group,
});
@@ -1615,29 +1847,115 @@ export class Conversion {
const originalNumberOfChannels = await track.getNumberOfChannels();
const originalSampleRate = await track.getSampleRate();
- const firstTimestamp = await track.getFirstTimestamp();
-
let numberOfChannels = trackOptions.numberOfChannels ?? originalNumberOfChannels;
let sampleRate = trackOptions.sampleRate ?? originalSampleRate;
- const needsTrimming = firstTimestamp < this._startTimestamp;
- let needsPadding = firstTimestamp > this._startTimestamp && !this.output.format.supportsTimestampedMediaData;
-
let audioCodecs = this.output.format.getSupportedAudioCodecs();
- if (
- !trackOptions.forceTranscode
- && !trackOptions.quality
+
+ let needsTranscode = !this._copyMode
+ || !!trackOptions.forceTranscode
+ || !!trackOptions.quality
// eslint-disable-next-line @typescript-eslint/no-deprecated
- && !trackOptions.bitrate
- && numberOfChannels === originalNumberOfChannels
- && sampleRate === originalSampleRate
- && !needsTrimming
- && !needsPadding
- && audioCodecs.includes(sourceCodec)
- && (!trackOptions.codec || trackOptions.codec === sourceCodec)
- && !trackOptions.process
- && trackOptions.sampleFormat === undefined
- ) {
+ || !!trackOptions.bitrate
+ || numberOfChannels !== originalNumberOfChannels
+ || sampleRate !== originalSampleRate
+ || !audioCodecs.includes(sourceCodec)
+ || (!!trackOptions.codec && trackOptions.codec !== sourceCodec)
+ || trackOptions.process !== undefined
+ || trackOptions.sampleFormat !== undefined;
+
+ let copyStartPacket: EncodedPacket | null = null;
+
+ if (!needsTranscode) {
+ // Check if we can copy it
+ const sink = new EncodedPacketSink(track);
+ let startPacket = await sink.getKeyPacket(this._startTimestamp)
+ ?? await sink.getFirstKeyPacket();
+
+ if (
+ startPacket
+ && startPacket.timestamp < this._startTimestamp
+ && this._copyBoundaryPolicy === 'shrink'
+ ) {
+ startPacket = await sink.getNextKeyPacket(startPacket);
+ }
+
+ const hasDecoderWarmup = (NON_PCM_AUDIO_CODECS as readonly AudioCodec[]).includes(sourceCodec)
+ && sourceCodec !== 'flac';
+ if (startPacket && this._copyBoundaryPolicy === 'expand' && hasDecoderWarmup) {
+ // Go one packet back
+ const previousPacket = await sink.getKeyPacket(
+ startPacket.timestamp - 1 / (await track.getTimeResolution()),
+ );
+ if (previousPacket) {
+ startPacket = previousPacket;
+ }
+ }
+
+ copyStartPacket = startPacket;
+
+ if (startPacket) {
+ if (!this.output.format.supportsTimestampedMediaData) {
+ // Wants zero
+
+ if (this._timestampOffsetAdjusted) {
+ // We've already adjusted, we can't adjust twice
+ const isValid = startPacket.timestamp + this._timestampOffset === 0;
+ if (!isValid) {
+ needsTranscode = true;
+ }
+ } else {
+ const correction = clamp(
+ this._startTimestamp - startPacket.timestamp,
+ -this._copyTimestampShiftTolerance,
+ this._copyTimestampShiftTolerance,
+ );
+
+ const shiftedStartTimestamp = startPacket.timestamp + correction;
+ const isValid = shiftedStartTimestamp === this._startTimestamp;
+
+ if (isValid) {
+ this._timestampOffset = -this._startTimestamp + correction;
+ this._timestampOffsetAdjusted = true;
+ } else {
+ needsTranscode = true;
+ }
+ }
+ } else if (
+ this.output.format.negativeTimestampSupport !== 'full'
+ && startPacket.timestamp < this._startTimestamp
+ ) {
+ const correction = Math.min(
+ this._startTimestamp - startPacket.timestamp,
+ this._copyTimestampShiftTolerance,
+ );
+
+ const shiftedStartTimestamp = startPacket.timestamp + correction;
+ const isValid = shiftedStartTimestamp >= this._startTimestamp
+ || (
+ this.output.format.negativeTimestampSupport === 'prefer-non-negative'
+ && this._copyMode === 'forced'
+ );
+
+ if (isValid) {
+ this._timestampOffset = Math.max(this._timestampOffset, -this._startTimestamp + correction);
+ } else {
+ needsTranscode = true;
+ }
+ }
+ }
+ }
+
+ if (needsTranscode && this._copyMode === 'forced') {
+ this.discardedTracks.push({
+ track,
+ reason: 'cannot_copy',
+ trackOptions,
+ });
+ return;
+ }
+
+ if (!needsTranscode) {
// Fast path, we can simply copy over the encoded packets
const source = new EncodedAudioPacketSource(sourceCodec);
@@ -1648,19 +1966,26 @@ export class Conversion {
const decoderConfig = await track.getDecoderConfig();
const meta: EncodedAudioChunkMetadata = { decoderConfig: decoderConfig ?? undefined };
- for await (const packet of sink.packets()) {
- if (this.state === 'canceled') {
+ // eslint-disable-next-line curly
+ if (copyStartPacket) for await (const packet of sink.packets(copyStartPacket)) {
+ if (this._state === 'canceled') {
break;
}
if (packet.timestamp >= this._endTimestamp) {
break;
}
+ if (
+ this._copyBoundaryPolicy === 'shrink'
+ && packet.timestamp + packet.duration > this._endTimestamp
+ ) {
+ break;
+ }
const modifiedPacket = packet.clone({
- timestamp: packet.timestamp - this._startTimestamp,
+ timestamp: packet.timestamp + this._timestampOffset,
+ duration: packet.duration,
});
- assert(modifiedPacket.timestamp >= 0);
this._reportProgress(outputTrackId, modifiedPacket.timestamp + modifiedPacket.duration);
await source.add(modifiedPacket, meta);
@@ -1770,39 +2095,14 @@ export class Conversion {
audioSource = source;
this._registerTrackPump(async (pump) => {
+ let needsPadding: boolean | null = null;
+
const sink = new AudioSampleSink(track);
for await (using sample of sink.samples(this._startTimestamp, this._endTimestamp)) {
- if (this.state === 'canceled') {
+ if (this._state === 'canceled') {
break;
}
- if (needsPadding) {
- // Add one padding sample at the beginning
- const paddingLength = firstTimestamp - this._startTimestamp;
- const paddingLengthSamples = Math.round(paddingLength * originalSampleRate);
-
- const bytesPerSample = getBytesPerSample(sample.format);
- const data = new Uint8Array(bytesPerSample * paddingLengthSamples * originalNumberOfChannels);
- if (sample.format === 'u8' || sample.format === 'u8-planar') {
- data.fill(2 ** 7); // Fill it with the silent value
- }
-
- using silentSample = new AudioSample({
- data,
- // Use the same format the decoder is spitting out. This avoids feeding changing sample
- // formats to the audio encoder.
- format: sample.format,
- numberOfChannels: originalNumberOfChannels,
- sampleRate: originalSampleRate,
- timestamp: 0,
- });
- await this._registerAudioSample(
- pump, silentSample, source, outputTrackId, () => lastSampleTimestamp,
- );
-
- needsPadding = false;
- }
-
let startFrame = 0;
let endFrame = sample.numberOfFrames;
@@ -1832,7 +2132,38 @@ export class Conversion {
using finalSample = finalSampleLet;
// Offset the timestamp as needed
- finalSample.setTimestamp(finalSample.timestamp - this._startTimestamp);
+ finalSample.setTimestamp(finalSample.timestamp + this._timestampOffset);
+
+ if (needsPadding === null) {
+ needsPadding = finalSample.timestamp > 0 && !this.output.format.supportsTimestampedMediaData;
+ }
+
+ if (needsPadding) {
+ // Add one padding sample at the beginning
+ const paddingLength = finalSample.timestamp;
+ const paddingLengthSamples = Math.round(paddingLength * originalSampleRate);
+
+ const bytesPerSample = getBytesPerSample(sample.format);
+ const data = new Uint8Array(bytesPerSample * paddingLengthSamples * originalNumberOfChannels);
+ if (sample.format === 'u8' || sample.format === 'u8-planar') {
+ data.fill(2 ** 7); // Fill it with the silent value
+ }
+
+ using silentSample = new AudioSample({
+ data,
+ // Use the same format the decoder is spitting out. This avoids feeding changing sample
+ // formats to the audio encoder.
+ format: sample.format,
+ numberOfChannels: originalNumberOfChannels,
+ sampleRate: originalSampleRate,
+ timestamp: 0,
+ });
+ await this._registerAudioSample(
+ pump, silentSample, source, outputTrackId, () => lastSampleTimestamp,
+ );
+
+ needsPadding = false;
+ }
await this._registerAudioSample(
pump, finalSample, source, outputTrackId, () => lastSampleTimestamp,
@@ -1852,11 +2183,16 @@ export class Conversion {
}
const audioTrackLanguageCode = await track.getLanguageCode();
+ const trackName = await track.getName();
+ const trackDisposition = await track.getDisposition();
+
this.output.addAudioTrack(audioSource, {
// TODO: This condition can be removed when all demuxers properly homogenize to BCP47 in v2
- languageCode: isIso639Dash2LanguageCode(audioTrackLanguageCode) ? audioTrackLanguageCode : undefined,
- name: await track.getName() ?? undefined,
- disposition: await track.getDisposition(),
+ languageCode: isIso639Dash2LanguageCode(audioTrackLanguageCode)
+ ? audioTrackLanguageCode
+ : undefined,
+ name: trackName ?? undefined,
+ disposition: trackDisposition,
group: ownGroup ?? trackOptions.group,
});
@@ -1909,7 +2245,7 @@ export class Conversion {
/** @internal */
async _checkpoint(pump: TrackPump, timestamp: number) {
- while (this.state !== 'canceled' && (timestamp >= this._executionUntil || this._pauseRequested)) {
+ while (this._state !== 'canceled' && (timestamp >= this._executionUntil || this._pauseRequested)) {
// We've reached the target; signal it and suspend until the next execution wakes us up
pump.resolvers.resolve();
@@ -1975,7 +2311,9 @@ class TrackSynchronizer {
}
declareTrack(trackId: number) {
- this.maxTimestamps.set(trackId, 0);
+ // Using -Infinity will automatically cause all tracks to wait for each other at the start until they figure out
+ // the true min timestamp
+ this.maxTimestamps.set(trackId, -Infinity);
}
shouldWait(trackId: number, timestamp: number) {
@@ -1986,7 +2324,7 @@ class TrackSynchronizer {
const newMin = this.computeMinAndMaybeResolve();
if (
- this.conversion.state === 'canceled'
+ this.conversion._state === 'canceled'
|| this.conversion._pauseRequested
|| timestamp >= this.conversion._executionUntil
) {
diff --git a/src/index.ts b/src/index.ts
index 46f07b0..8ea2e23 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -299,6 +299,7 @@ export {
type ConversionOptions,
type ConversionVideoOptions,
type ConversionAudioOptions,
+ type ConversionCopyOptions,
type ConversionExecuteOptions,
ConversionCanceledError,
type DiscardedTrack,
diff --git a/src/matroska/matroska-demuxer.ts b/src/matroska/matroska-demuxer.ts
index d9e199d..0b8bbf4 100644
--- a/src/matroska/matroska-demuxer.ts
+++ b/src/matroska/matroska-demuxer.ts
@@ -1934,6 +1934,29 @@ export class MatroskaDemuxer extends Demuxer {
}
}
}
+
+ async getDurationFromMetadata(segment: Segment) {
+ if (segment.duration <= 0) {
+ return null;
+ }
+
+ // The kosher definition of the Duration field is "latest end time - earliest start time" across all tracks in
+ // the segment; since we currently mean "end timestamp" with "duration", we need to determine the earliest
+ // start time before we can return a value here.
+ let minTimestamp: number | null = null;
+ for (const track of segment.tracks) {
+ assert(track.trackBacking);
+ const firstPacket = await track.trackBacking.getFirstPacket({ metadataOnly: true });
+ if (firstPacket) {
+ minTimestamp = Math.min(minTimestamp ?? Infinity, firstPacket.timestamp);
+ }
+ }
+
+ let endTimestamp = segment.duration / segment.timestampFactor;
+ endTimestamp += minTimestamp ?? 0;
+
+ return endTimestamp;
+ }
}
abstract class MatroskaTrackBacking implements InputTrackBacking {
@@ -2016,17 +2039,7 @@ abstract class MatroskaTrackBacking implements InputTrackBacking {
}
async getDurationFromMetadata() {
- const segment = this.internalTrack.segment;
- if (segment.duration <= 0) {
- return null;
- }
-
- let endTimestamp = segment.duration / segment.timestampFactor;
-
- const firstPacket = await this.getFirstPacket({ metadataOnly: true });
- endTimestamp += firstPacket?.timestamp ?? 0;
-
- return endTimestamp;
+ return this.internalTrack.demuxer.getDurationFromMetadata(this.internalTrack.segment);
}
async getLiveRefreshInterval() {
diff --git a/src/output-format.ts b/src/output-format.ts
index 5304fab..4a3fb1f 100644
--- a/src/output-format.ts
+++ b/src/output-format.ts
@@ -85,6 +85,14 @@ export abstract class OutputFormat {
* durations of the media data.
*/
abstract get supportsTimestampedMediaData(): boolean;
+ /**
+ * The degree to which this output format supports writing media data with negative timestamps.
+ * - `'full'` - Negative timestamps are fully supported.
+ * - `'prefer-non-negative'` - Negative timestamps are technically supported, but their use is discouraged.
+ * - `'none'` - Negative timestamps are not supported.
+ * - `null` - Not applicable since the container doesn't support timestamped media at all.
+ */
+ abstract get negativeTimestampSupport(): 'full' | 'prefer-non-negative' | 'none' | null;
/** Returns a list of video codecs that this output format can contain. */
getSupportedVideoCodecs(): VideoCodec[] {
@@ -278,6 +286,10 @@ export abstract class IsobmffOutputFormat extends OutputFormat {
return true;
}
+ get negativeTimestampSupport() {
+ return 'full' as const;
+ }
+
/** @internal */
_createMuxer(output: Output) {
return new IsobmffMuxer(output, this);
@@ -579,6 +591,10 @@ export class MkvOutputFormat extends OutputFormat {
get supportsTimestampedMediaData() {
return true;
}
+
+ get negativeTimestampSupport() {
+ return 'prefer-non-negative' as const;
+ }
}
/**
@@ -719,6 +735,10 @@ export class Mp3OutputFormat extends OutputFormat {
get supportsTimestampedMediaData() {
return false;
}
+
+ get negativeTimestampSupport() {
+ return null;
+ }
}
/**
@@ -821,6 +841,10 @@ export class WavOutputFormat extends OutputFormat {
get supportsTimestampedMediaData() {
return false;
}
+
+ get negativeTimestampSupport() {
+ return null;
+ }
}
/**
@@ -916,6 +940,10 @@ export class OggOutputFormat extends OutputFormat {
get supportsTimestampedMediaData() {
return false;
}
+
+ get negativeTimestampSupport() {
+ return null;
+ }
}
/**
@@ -994,6 +1022,10 @@ export class AdtsOutputFormat extends OutputFormat {
get supportsTimestampedMediaData() {
return false;
}
+
+ get negativeTimestampSupport() {
+ return null;
+ }
}
/**
@@ -1079,6 +1111,10 @@ export class FlacOutputFormat extends OutputFormat {
get supportsTimestampedMediaData() {
return false;
}
+
+ get negativeTimestampSupport() {
+ return null;
+ }
}
/**
@@ -1164,6 +1200,10 @@ export class MpegTsOutputFormat extends OutputFormat {
get supportsTimestampedMediaData() {
return true;
}
+
+ get negativeTimestampSupport() {
+ return 'prefer-non-negative' as const;
+ }
}
/**
@@ -1424,7 +1464,24 @@ export class HlsOutputFormat extends OutputFormat {
}
get supportsTimestampedMediaData(): boolean {
- return true; // I guess??
+ return true; // It's only half true, really, but "false" is not correct either
+ }
+
+ get negativeTimestampSupport() {
+ const formats = toArray(this._options.segmentFormat);
+
+ // Return the lowest baseline across all segment formats
+ if (formats.some(format => format.negativeTimestampSupport === 'none')) {
+ return 'none' as const;
+ }
+ if (formats.some(format => format.negativeTimestampSupport === 'prefer-non-negative')) {
+ return 'prefer-non-negative' as const;
+ }
+ if (formats.some(format => format.negativeTimestampSupport === 'full')) {
+ return 'full' as const;
+ }
+
+ return null;
}
/** @internal */
diff --git a/test/browser/conversion.test.ts b/test/browser/conversion.test.ts
index 7ed297a..05e98b9 100644
--- a/test/browser/conversion.test.ts
+++ b/test/browser/conversion.test.ts
@@ -3,20 +3,23 @@ import { Input } from '../../src/input.js';
import {
AdtsOutputFormat,
HlsOutputFormat,
+ MkvOutputFormat,
Mp4OutputFormat,
MpegTsOutputFormat,
+ OutputFormat,
WavOutputFormat,
} from '../../src/output-format.js';
import { Output, OutputTrackGroup } from '../../src/output.js';
import { BufferSource, CustomPathedSource, UrlSource } from '../../src/source.js';
import { expect, test } from 'vitest';
import { BufferTarget, PathedTarget } from '../../src/target.js';
-import { Conversion, ConversionCanceledError } from '../../src/conversion.js';
-import { assert } from '../../src/misc.js';
+import { Conversion, ConversionCanceledError, ConversionOptions } from '../../src/conversion.js';
+import { assert, uint8ArraysAreEqual } from '../../src/misc.js';
import { InputVideoTrack } from '../../src/input-track.js';
import { CanvasSource, EncodedAudioPacketSource } from '../../src/media-source.js';
import { Quality } from '../../src/encode.js';
import { EncodedPacket } from '../../src/packet.js';
+import { EncodedPacketSink } from '../../src/media-sink.js';
test('Rotation is baked in when rerendering', async () => {
using input = new Input({
@@ -783,3 +786,270 @@ test('Resizing at various scale factors', async () => {
expect(await videoTrack!.getDisplayHeight()).toBe(height);
}
});
+
+test('Packet copy, whole file', async () => {
+ await testCopy({
+ conversionOptions: {},
+ expectedTimeOffset: 0,
+ videoStartTimestamp: 0,
+ videoEndTimestamp: 5,
+ audioStartTimestamp: -1024 / 48000,
+ audioEndTimestamp: 5,
+ });
+});
+
+test('Packet copy, keyframe trim', async () => {
+ await testCopy({
+ conversionOptions: {
+ trim: {
+ start: 1,
+ end: 2,
+ },
+ },
+ expectedTimeOffset: 1,
+ videoStartTimestamp: 0,
+ videoEndTimestamp: 1,
+ audioStartTimestamp: -0.04,
+ audioEndTimestamp: 1.0053333333333334,
+ });
+});
+
+test('Packet copy, keyframe trim, shrink', async () => {
+ await testCopy({
+ conversionOptions: {
+ trim: {
+ start: 1,
+ end: 2,
+ },
+ copy: {
+ boundaryPolicy: 'shrink',
+ },
+ },
+ expectedTimeOffset: 1,
+ videoStartTimestamp: 0,
+ videoEndTimestamp: 1,
+ audioStartTimestamp: -0.04 + 2 * 1024 / 48000,
+ audioEndTimestamp: 1.0053333333333334 - 1024 / 48000,
+ });
+});
+
+test('Packet copy, delta frame trim', async () => {
+ await testCopy({
+ conversionOptions: {
+ trim: {
+ start: 1.5,
+ end: 2.5,
+ },
+ },
+ expectedTimeOffset: 1.5,
+ videoStartTimestamp: -0.5,
+ videoEndTimestamp: 1.02,
+ audioStartTimestamp: -0.028,
+ audioEndTimestamp: 1.0173333333333334,
+ });
+});
+
+test('Packet copy, delta frame trim, shrink', async () => {
+ await testCopy({
+ conversionOptions: {
+ trim: {
+ start: 1.5,
+ end: 2.5,
+ },
+ copy: {
+ boundaryPolicy: 'shrink',
+ },
+ },
+ expectedTimeOffset: 1.5,
+ videoStartTimestamp: 0.5,
+ videoEndTimestamp: 1.02, // Due to MP4 not being able to express a different duration for the last packet
+ audioStartTimestamp: -0.028 + 2 * 1024 / 48000,
+ audioEndTimestamp: 1.0173333333333334 - 1024 / 48000,
+ });
+});
+
+test('Packet copy, whole file, Matroska', async () => {
+ await testCopy({
+ outputFormat: new MkvOutputFormat(),
+ conversionOptions: {
+ copy: {
+ shiftTolerance: Infinity,
+ },
+ },
+ expectedTimeOffset: -1024 / 48000,
+ videoStartTimestamp: 0 + 1024 / 48000,
+ videoEndTimestamp: 5 + 1024 / 48000 - 1 / 25,
+ audioStartTimestamp: 0,
+ audioEndTimestamp: 235 * 1024 / 48000,
+ precision: 0.001,
+ });
+});
+
+test('Packet copy, whole file, Matroska, forced', async () => {
+ await testCopy({
+ outputFormat: new MkvOutputFormat(),
+ conversionOptions: {
+ copy: {
+ mode: 'forced',
+ },
+ },
+ expectedTimeOffset: 0,
+ videoStartTimestamp: 0,
+ videoEndTimestamp: 5 - 1 / 25,
+ audioStartTimestamp: -1024 / 48000,
+ audioEndTimestamp: 4.992,
+ precision: 0.001,
+ });
+});
+
+test('Packet copy, delta frame trim, Matroska', async () => {
+ await testCopy({
+ outputFormat: new MkvOutputFormat(),
+ conversionOptions: {
+ trim: {
+ start: 1.5,
+ end: 2.5,
+ },
+ copy: {
+ shiftTolerance: Infinity,
+ },
+ },
+ expectedTimeOffset: 1,
+ videoStartTimestamp: 0,
+ videoEndTimestamp: 1.48,
+ audioStartTimestamp: 0.472,
+ audioEndTimestamp: 1.496,
+ precision: 0.001,
+ });
+});
+
+test('Packet copy, delta frame trim, video transcoded, Matroska', async () => {
+ await testCopy({
+ outputFormat: new MkvOutputFormat(),
+ conversionOptions: {
+ video: {
+ forceTranscode: true,
+ codec: 'vp9',
+ },
+ trim: {
+ start: 1.5,
+ end: 2.5,
+ },
+ copy: {
+ shiftTolerance: Infinity,
+ },
+ },
+ expectedTimeOffset: 1.484,
+ videoStartTimestamp: 0.008,
+ videoEndTimestamp: 1.008,
+ audioStartTimestamp: 0,
+ audioEndTimestamp: 1.024,
+ precision: 0.001,
+ compareVideoPackets: false,
+ });
+});
+
+test('Packet copy, whole file, ADTS', async () => {
+ await testCopy({
+ outputFormat: new AdtsOutputFormat(),
+ conversionOptions: {
+ video: {
+ discard: true,
+ },
+ copy: {
+ mode: 'forced',
+ shiftTolerance: Infinity,
+ },
+ },
+ expectedTimeOffset: -1024 / 48000,
+ audioStartTimestamp: 0,
+ audioEndTimestamp: 5.034666666666666,
+ processNewAudioPacketData: data => data.subarray(7),
+ });
+});
+
+const testCopy = async (options: {
+ outputFormat?: OutputFormat;
+ conversionOptions: Omit;
+ expectedTimeOffset: number;
+ videoStartTimestamp?: number;
+ videoEndTimestamp?: number;
+ audioStartTimestamp: number;
+ audioEndTimestamp: number;
+ precision?: number;
+ processNewAudioPacketData?: (data: Uint8Array) => Uint8Array;
+ compareVideoPackets?: boolean;
+}) => {
+ const precision = options.precision ?? 0.000001;
+
+ const isCloseTo = (a: number, b: number) => {
+ return Math.abs(a - b) <= precision;
+ };
+
+ using input = new Input({
+ source: new UrlSource('/demo.mp4'),
+ formats: ALL_FORMATS,
+ });
+ const output = new Output({
+ format: options.outputFormat ?? new Mp4OutputFormat(),
+ target: new BufferTarget(),
+ });
+
+ const videoTrack = await input.getPrimaryVideoTrack();
+ const audioTrack = await input.getPrimaryAudioTrack();
+ assert(videoTrack);
+ assert(audioTrack);
+ const videoSink = new EncodedPacketSink(videoTrack);
+ const audioSink = new EncodedPacketSink(audioTrack);
+
+ const conversion = await Conversion.init({
+ input,
+ output,
+ ...options.conversionOptions,
+ });
+ await conversion.execute();
+
+ using newInput = new Input({
+ source: new BufferSource(output.target.buffer!),
+ formats: ALL_FORMATS,
+ });
+
+ const newVideoTrack = await newInput.getPrimaryVideoTrack();
+ const newAudioTrack = await newInput.getPrimaryAudioTrack();
+
+ if (newVideoTrack) {
+ const newVideoSink = new EncodedPacketSink(newVideoTrack);
+
+ expect(isCloseTo(await newVideoTrack.getFirstTimestamp(), options.videoStartTimestamp!)).toBe(true);
+ expect(isCloseTo(await newVideoTrack.computeDuration(), options.videoEndTimestamp!)).toBe(true);
+
+ if (options.compareVideoPackets ?? true) {
+ for await (const newPacket of newVideoSink.packets()) {
+ const oldPacket = await videoSink.getPacket(
+ newPacket.timestamp + options.expectedTimeOffset + precision,
+ );
+ assert(oldPacket);
+
+ expect(uint8ArraysAreEqual(oldPacket.data, newPacket.data)).toBe(true);
+ expect(oldPacket.type).toEqual(newPacket.type);
+ }
+ }
+ }
+
+ if (newAudioTrack) {
+ const newAudioSink = new EncodedPacketSink(newAudioTrack);
+
+ expect(isCloseTo(await newAudioTrack.getFirstTimestamp(), options.audioStartTimestamp)).toBe(true);
+ expect(isCloseTo(await newAudioTrack.computeDuration(), options.audioEndTimestamp)).toBe(true);
+
+ for await (const newPacket of newAudioSink.packets()) {
+ const oldPacket = await audioSink.getPacket(newPacket.timestamp + options.expectedTimeOffset + precision);
+ assert(oldPacket);
+
+ const process = options.processNewAudioPacketData ?? (x => x);
+
+ expect(uint8ArraysAreEqual(oldPacket.data, process(newPacket.data))).toBe(true);
+ expect(oldPacket.type).toEqual(newPacket.type);
+ }
+ }
+};
diff --git a/test/public/demo.mp4 b/test/public/demo.mp4
new file mode 100644
index 0000000..24296a8
Binary files /dev/null and b/test/public/demo.mp4 differ