arcodange/mediabunny
1
0
Fork 0
mirror of https://github.com/arcodange-org/mediabunny.git synced 2026-09-27 02:43:48 +02:00
Code Issues Packages Projects Releases Wiki Activity

Implement more powerful copy conversions, add conversion copy config, add OutputFormat.negativeTimestampSupport, fix Matroska duration from metadata computation

Browse Source
This commit is contained in:
Vanilagy
2026-09-08 14:42:03 +02:00
parent e1f3b5252b
commit a1ba54d337
7 changed files with 806 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dev/convert.html
+15 -4
View File
@@ -25,7 +25,7 @@
chunked: true,
chunkSize: 2**20
});
const outputFormat = new Mediabunny.MovOutputFormat();
const outputFormat = new Mediabunny.Mp4OutputFormat();
const p = document.createElement('p');
p.textContent = 'Capturing...';
@@ -99,6 +99,8 @@
input,
output,
audio: {
discard: true,
//discard: true,
//codec: 'aac',
//forceTranscode: true,
//forceTranscode: true,
@@ -129,7 +131,9 @@
},
*/
video: {
height: 320,
forceTranscode: true,
//forceTranscode: true,
//height: 320,
//codec: 'avc',
//bitrate: new Mediabunny.Quality({
// quality: Infinity,
@@ -158,7 +162,8 @@
}
},
trim: {
end: 5,
//start: -60,
//end: 60,
//start: -2,
//end: 10,
//start: 300.14984567374756 - 100,
@@ -167,6 +172,12 @@
//start: startTime,
//end: startTime + 2,
},
copy: {
//mode: 'forced',
//mode: 'forced',
//shiftTolerance: Infinity,
//boundaryPolicy: 'shrink',
},
});
//console.log(conversion);
console.log(conversion.discardedTracks);
@@ -213,4 +224,4 @@
conversion = null;
input = null;
}, { once: true });
</script>
</script>
src/conversion.ts
+436 -98
View File
@@ -44,6 +44,7 @@ import {
ceilToMultipleOfTwo,
clamp,
isIso639Dash2LanguageCode,
isNumber,
MaybePromise,
normalizeRotation,
promiseWithResolvers,
@@ -51,6 +52,7 @@ import {
} from './misc';
import { Output, OutputTrackGroup } from './output';
import { Mp4OutputFormat } from './output-format';
import { EncodedPacket } from './packet';
import {
AudioSample,
clampCropRectangle,
@@ -132,6 +134,13 @@ export type ConversionOptions = {
end?: number;
};
/**
* Options for controlling when media is copied directly without transcoding it. Set to `false` to always transcode.
* Defaults to `{}`, which will copy media whenever possible and otherwise transcode it while retaining precise
* timestamps.
*/
copy?: ConversionCopyOptions | false;
/**
* An object or a callback that returns or resolves to an object containing the descriptive metadata tags that
* should be written to the output file. If a function is passed, it will be passed the tags of the input file as
@@ -237,7 +246,7 @@ export type ConversionVideoOptions = {
* `'no-preference'`, the default.
*/
hardwareAcceleration?: 'no-preference' | 'prefer-hardware' | 'prefer-software';
/** When `true`, video will always be re-encoded instead of directly copying over the encoded samples. */
/** When `true`, video will always be re-encoded instead of directly copying over the encoded packets. */
forceTranscode?: boolean;
/**
* Allows for custom user-defined processing of video frames, e.g. for applying overlays, color transformations, or
@@ -303,7 +312,7 @@ export type ConversionAudioOptions = {
* @deprecated Use `quality` instead.
*/
bitrate?: number | Quality;
/** When `true`, audio will always be re-encoded instead of directly copying over the encoded samples. */
/** When `true`, audio will always be re-encoded instead of directly copying over the encoded packets. */
forceTranscode?: boolean;
/**
* Allows for custom user-defined processing of audio samples, e.g. for applying audio effects, transformations, or
@@ -338,6 +347,44 @@ export type ConversionAudioOptions = {
group?: OutputTrackGroup | OutputTrackGroup[];
};
/**
* Options for copying encoded media during conversion.
* @group Conversion
* @public
*/
export type ConversionCopyOptions = {
/**
* Controls whether media copying is preferred or required. Defaults to `'preferred'`.
*
* - `'forced'`: Copy encoded media where possible and discard tracks that cannot possibly be copied.
* - `'preferred'`: Copy encoded media when possible, and transcode tracks that cannot be copied.
*/
mode?: 'forced' | 'preferred';
/**
* The maximum absolute shift, in seconds, that may be applied to the media to be able to copy it into the output
* format. Defaults to `0`, which permits no additional shift. Set to `Infinity` to permit any shift.
*
* A shift of `0` gives you perfect _timeline sync_: output timestamps will match input timestamps exactly (only
* offset by the trim region). Any non-zero shift will break this property but will still, under all circumstances,
* maintain perfect cross-track and audio-video sync.
*/
shiftTolerance?: number;
/**
* Controls which media region will be copied to satisfy the requested trim range. Defaults to `'expand'`.
*
* - `'expand'`: Include at least all media in the requested range. This may require expanding the media region due
* to key frames and packet boundaries, and thus may include media outside of your trim range. The region is always
* minimally expanded to satisfy the copy criteria.
* - `'shrink'`: Only include media that lies entirely within the requested trim range. This may require shrinking
* the media region due to key frames and packet boundaries, and thus may exclude media inside of your trim range.
* The region is always minimally shrunk to satisfy the copy criteria.
*
* Use `expand` if you don't want to lose any media; use `shrink` to never expose any media outside of the
* trim region.
*/
boundaryPolicy?: 'expand' | 'shrink';
};
const validateVideoOptions = (videoOptions: ConversionVideoOptions) => {
if (!videoOptions || typeof videoOptions !== 'object') {
throw new TypeError('options.video, when provided, must be an object.');
@@ -546,6 +593,8 @@ export type DiscardedTrack = {
* - `'no_encodable_target_codec'`: We can't find a codec that we are able to encode and that can be contained
* within the output format. This reason can be hit if the environment doesn't support the necessary encoders, or if
* you requested a codec that cannot be contained within the output format.
* - `'cannot_copy'`: {@link ConversionCopyOptions.mode} was set to `'forced'` but the track could not be copied
* with the given copy configuration because it would require a transcode instead.
*/
reason:
| 'discarded_by_user'
@@ -553,7 +602,8 @@ export type DiscardedTrack = {
| 'max_track_count_of_type_reached'
| 'unknown_source_codec'
| 'undecodable_source_codec'
| 'no_encodable_target_codec';
| 'no_encodable_target_codec'
| 'cannot_copy';
/** The options that were provided for this track, or `{}` if none were provided. */
trackOptions: ConversionVideoOptions | ConversionAudioOptions;
};
@@ -597,22 +647,28 @@ export class Conversion {
/** The output file. */
readonly output: Output;
/**
* The current state of the conversion.
*
* - `'idle'`: The conversion is not currently executing and isn't done; `execute` can be called.
* - `'executing'`: A call to `execute` is currently running.
* - `'canceled'`: The conversion has been canceled and can no longer be executed.
* - `'done'`: The conversion has run to completion. Subsequent calls to `execute` do nothing.
*/
state: 'idle' | 'executing' | 'canceled' | 'done' = 'idle';
/** @internal */
_state: 'idle' | 'executing' | 'canceled' | 'done' = 'idle';
/** @internal */
_options: ConversionOptions;
/** @internal */
_copyMode: false | 'forced' | 'preferred';
/** @internal */
_copyTimestampShiftTolerance: number;
/** @internal */
_copyBoundaryPolicy: 'expand' | 'shrink';
/** @internal */
_startTimestamp!: number;
/** @internal */
_endTimestamp!: number;
/** @internal */
_timestampOffset = 0;
/** @internal */
_timestampOffsetAdjusted = false;
/** @internal */
_copyTimestampPossible = new Map<InputTrack, boolean>();
/** @internal */
_copyStartPackets = new Map<InputTrack, EncodedPacket | null>();
/** @internal */
_nextOutputTrackId = 0;
@@ -673,6 +729,18 @@ export class Conversion {
/** The list of tracks from the input file that have been discarded, alongside the discard reason. */
readonly discardedTracks: DiscardedTrack[] = [];
/**
* The current state of the conversion.
*
* - `'idle'`: The conversion is not currently executing and isn't done; `execute` can be called.
* - `'executing'`: A call to `execute` is currently running.
* - `'canceled'`: The conversion has been canceled and can no longer be executed.
* - `'done'`: The conversion has run to completion. Subsequent calls to `execute` do nothing.
*/
get state() {
return this._state;
}
/** Initializes a new conversion process without starting the conversion. */
static async init(options: ConversionOptions) {
const conversion = new Conversion(options);
@@ -704,6 +772,28 @@ export class Conversion {
if (options.composable !== undefined && typeof options.composable !== 'boolean') {
throw new TypeError('options.composable, when provided, must be a boolean.');
}
if (options.copy !== undefined && options.copy !== false) {
if (!options.copy || typeof options.copy !== 'object') {
throw new TypeError('options.copy, when provided, must be an object or false.');
}
if (options.copy.mode !== undefined && !['forced', 'preferred'].includes(options.copy.mode)) {
throw new TypeError('options.copy.mode, when provided, must be \'forced\' or \'preferred\'.');
}
if (
options.copy.shiftTolerance !== undefined
&& (!isNumber(options.copy.shiftTolerance) || options.copy.shiftTolerance < 0)
) {
throw new TypeError('options.copy.shiftTolerance, when provided, must be a non-negative number.');
}
if (
options.copy.boundaryPolicy !== undefined
&& !['expand', 'shrink'].includes(options.copy.boundaryPolicy)
) {
throw new TypeError(
'options.copy.boundaryPolicy, when provided, must be \'expand\' or \'shrink\'.',
);
}
}
const composable = options.composable ?? false;
if (!composable) {
@@ -757,8 +847,8 @@ export class Conversion {
if (options.trim?.start !== undefined && (!Number.isFinite(options.trim.start))) {
throw new TypeError('options.trim.start, when provided, must be a finite number.');
}
if (options.trim?.end !== undefined && (!Number.isFinite(options.trim.end))) {
throw new TypeError('options.trim.end, when provided, must be a finite number.');
if (options.trim?.end !== undefined && (!isNumber(options.trim.end))) {
throw new TypeError('options.trim.end, when provided, must be a number.');
}
if (
options.trim?.start !== undefined
@@ -781,6 +871,9 @@ export class Conversion {
}
this._options = options;
this._copyMode = options.copy === false ? false : options.copy?.mode ?? 'preferred';
this._copyTimestampShiftTolerance = options.copy === false ? 0 : options.copy?.shiftTolerance ?? 0;
this._copyBoundaryPolicy = options.copy === false ? 'expand' : options.copy?.boundaryPolicy ?? 'expand';
this._composable = composable;
this.input = options.input;
this.output = options.output;
@@ -924,6 +1017,7 @@ export class Conversion {
}
this._endTimestamp = Math.max(this._options.trim?.end ?? Infinity, this._startTimestamp);
this._timestampOffset = -this._startTimestamp; // Initial value, may get refined later by track processing
// Run these sequentially so that output tracks have a deterministic order
for (let i = 0; i < filteredTracks.length; i++) {
@@ -1156,15 +1250,15 @@ export class Conversion {
);
}
if (this.state === 'executing') {
if (this._state === 'executing') {
throw new Error('Cannot call execute() while a previous call to execute() is still running.');
}
if (this.state === 'canceled') {
if (this._state === 'canceled') {
throw new ConversionCanceledError();
}
if (this.state === 'done') {
if (this._state === 'done') {
// The conversion already ran to completion, nothing left to do
return;
}
@@ -1176,12 +1270,12 @@ export class Conversion {
);
}
this.state = 'executing';
this._state = 'executing';
this._executionUntil = options.until ?? Infinity;
this._pauseRequested = options.pauseSignal?.aborted ?? false;
const onPause = () => {
if (this.state !== 'executing') {
if (this._state !== 'executing') {
return;
}
@@ -1224,6 +1318,9 @@ export class Conversion {
);
for (const id of this._outputTrackIds) {
// Used for progress calculation. We start these at 0 which is technically not always the first
// timestamp, but this is how we choose to model what "progress" means: it's how far we are done
// with the trim region.
this._maxTimestamps.set(id, 0);
}
@@ -1247,7 +1344,7 @@ export class Conversion {
try {
await Promise.all(this._trackPumps.map(x => x.resolvers.promise));
} catch (error) {
if ((this.state as Conversion['state']) !== 'canceled') {
if ((this._state as Conversion['_state']) !== 'canceled') {
// Make sure to cancel to stop other encoding processes and clean up resources
void this.cancel();
}
@@ -1257,12 +1354,12 @@ export class Conversion {
options.pauseSignal?.removeEventListener('abort', onPause);
}
if ((this.state as Conversion['state']) === 'canceled') {
if ((this._state as Conversion['_state']) === 'canceled') {
throw new ConversionCanceledError();
}
const isDone = this._trackPumps.every(x => x.done);
this.state = isDone ? 'done' : 'idle';
this._state = isDone ? 'done' : 'idle';
if (isDone) {
if (!this._composable) {
@@ -1281,16 +1378,16 @@ export class Conversion {
* Does nothing if the conversion is already complete.
*/
async cancel() {
if (this.state === 'done') {
if (this._state === 'done') {
return;
}
if (this.state === 'canceled') {
if (this._state === 'canceled') {
Logging._warn('Conversion already canceled.');
return;
}
this.state = 'canceled';
this._state = 'canceled';
// Wake all suspended track pumps so they can wind down
for (const pump of this._trackPumps) {
@@ -1355,11 +1452,11 @@ export class Conversion {
height = ceilToMultipleOfTwo(trackOptions.height);
}
const firstTimestamp = await track.getFirstTimestamp();
let videoCodecs = this.output.format.getSupportedVideoCodecs();
const alpha = trackOptions.alpha ?? 'discard';
const needsTranscode = !!trackOptions.forceTranscode
|| firstTimestamp < this._startTimestamp
let needsTranscode = !this._copyMode
|| !!trackOptions.forceTranscode
|| !!trackOptions.frameRate
|| trackOptions.keyFrameInterval !== undefined
|| trackOptions.process !== undefined
@@ -1376,7 +1473,92 @@ export class Conversion {
|| (totalRotation !== 0 && !canUseRotationMetadata)
|| !!crop;
const alpha = trackOptions.alpha ?? 'discard';
let copyStartPacket: EncodedPacket | null = null;
if (!needsTranscode) {
// Check if we can copy it
const sink = new EncodedPacketSink(track);
let startPacket = await sink.getKeyPacket(this._startTimestamp, { verifyKeyPackets: true })
?? await sink.getFirstKeyPacket({ verifyKeyPackets: true });
if (
startPacket
&& startPacket.timestamp < this._startTimestamp
&& startPacket.timestamp + startPacket.duration <= this._startTimestamp
&& this._copyBoundaryPolicy === 'shrink'
) {
startPacket = await sink.getNextKeyPacket(startPacket, { verifyKeyPackets: true });
}
copyStartPacket = startPacket;
if (startPacket) {
// This clamp mirrors the packet timestamp clamping the copy loop does. The reason this is valid is
// because in the shrink case, we've already proven that the packet (at least partially) overlaps the
// trim region.
const effectiveStartTimestamp = this._copyBoundaryPolicy === 'shrink'
? Math.max(startPacket.timestamp, this._startTimestamp)
: startPacket.timestamp;
if (!this.output.format.supportsTimestampedMediaData) {
// Wants zero
if (this._timestampOffsetAdjusted) {
// We've already adjusted, we can't adjust twice
const isValid = effectiveStartTimestamp + this._timestampOffset === 0;
if (!isValid) {
needsTranscode = true;
}
} else {
const correction = clamp(
this._startTimestamp - effectiveStartTimestamp,
-this._copyTimestampShiftTolerance,
this._copyTimestampShiftTolerance,
);
const shiftedStartTimestamp = effectiveStartTimestamp + correction;
const isValid = shiftedStartTimestamp === this._startTimestamp;
if (isValid) {
this._timestampOffset = -this._startTimestamp + correction;
this._timestampOffsetAdjusted = true;
} else {
needsTranscode = true;
}
}
} else if (
this.output.format.negativeTimestampSupport !== 'full'
&& effectiveStartTimestamp < this._startTimestamp
) {
const correction = Math.min(
this._startTimestamp - effectiveStartTimestamp,
this._copyTimestampShiftTolerance,
);
const shiftedStartTimestamp = effectiveStartTimestamp + correction;
const isValid = shiftedStartTimestamp >= this._startTimestamp
|| (
this.output.format.negativeTimestampSupport === 'prefer-non-negative'
&& this._copyMode === 'forced'
);
if (isValid) {
this._timestampOffset = Math.max(this._timestampOffset, -this._startTimestamp + correction);
} else {
needsTranscode = true;
}
}
}
}
if (needsTranscode && this._copyMode === 'forced') {
this.discardedTracks.push({
track,
reason: 'cannot_copy',
trackOptions,
});
return;
}
if (!needsTranscode) {
// Fast path, we can simply copy over the encoded packets
@@ -1389,22 +1571,66 @@ export class Conversion {
const decoderConfig = await track.getDecoderConfig();
const meta: EncodedVideoChunkMetadata = { decoderConfig: decoderConfig ?? undefined };
for await (const packet of sink.packets(undefined, undefined, { verifyKeyPackets: true })) {
if (this.state === 'canceled') {
// eslint-disable-next-line curly
if (copyStartPacket) for await (const packet of sink.packets(
copyStartPacket,
undefined,
{ verifyKeyPackets: true },
)) {
if (this._state === 'canceled') {
break;
}
if (packet.timestamp >= this._endTimestamp) {
break;
if (this._copyBoundaryPolicy === 'shrink') {
break;
} else {
// Due to B-frames, there might still be packets we care about later on. Do a short
// lookahead to find out if there are.
let current = packet;
let found = false;
const lookahead = 6; // Heuristic, but should be enough for most streams
for (let i = 0; i < lookahead; i++) {
const next = await sink.getNextPacket(current, { metadataOnly: true });
if (!next) {
break;
}
if (next.timestamp < this._endTimestamp) {
found = true;
break;
}
current = next;
}
if (!found) {
break;
}
}
}
let packetStartTimestamp = packet.timestamp;
let packetEndTimestamp = packet.timestamp + packet.duration;
if (this._copyBoundaryPolicy === 'shrink') {
packetStartTimestamp = Math.max(packetStartTimestamp, this._startTimestamp);
packetEndTimestamp = Math.min(packetEndTimestamp, this._endTimestamp);
packetEndTimestamp = Math.max(packetEndTimestamp, packetStartTimestamp); // Just in case
}
packetStartTimestamp += this._timestampOffset;
packetEndTimestamp += this._timestampOffset;
const modifiedPacket = packet.clone({
timestamp: packet.timestamp - this._startTimestamp,
timestamp: packetStartTimestamp,
duration: packetEndTimestamp - packetStartTimestamp,
sideData: alpha === 'discard'
? {} // Remove alpha side data
: packet.sideData,
});
assert(modifiedPacket.timestamp >= 0);
this._reportProgress(outputTrackId, modifiedPacket.timestamp + modifiedPacket.duration);
await source.add(modifiedPacket, meta);
@@ -1496,8 +1722,9 @@ export class Conversion {
await tempOutput.start();
// Let's just use the first sample to test
const sink = new VideoSampleSink(track);
using firstSample = await sink.getSample(firstTimestamp); // Let's just use the first sample
using firstSample = await sink.getSample(await track.getFirstTimestamp());
if (firstSample) {
try {
@@ -1550,11 +1777,11 @@ export class Conversion {
const sink = new VideoSampleSink(track);
for await (using sample of sink.samples(this._startTimestamp, this._endTimestamp)) {
if (this.state === 'canceled') {
if (this._state === 'canceled') {
break;
}
const adjustedSampleTimestamp = Math.max(sample.timestamp - this._startTimestamp, 0);
const adjustedSampleTimestamp = Math.max(0, sample.timestamp + this._timestampOffset);
sample.setTimestamp(adjustedSampleTimestamp);
this._reportProgress(outputTrackId, sample.timestamp + sample.duration);
@@ -1583,12 +1810,17 @@ export class Conversion {
}
const videoTrackLanguageCode = await track.getLanguageCode();
const trackName = await track.getName();
const trackDisposition = await track.getDisposition();
this.output.addVideoTrack(videoSource, {
frameRate: trackOptions.frameRate,
// TODO: This condition can be removed when all demuxers properly homogenize to BCP47 in v2
languageCode: isIso639Dash2LanguageCode(videoTrackLanguageCode) ? videoTrackLanguageCode : undefined,
name: await track.getName() ?? undefined,
disposition: await track.getDisposition(),
languageCode: isIso639Dash2LanguageCode(videoTrackLanguageCode)
? videoTrackLanguageCode
: undefined,
name: trackName ?? undefined,
disposition: trackDisposition,
rotation: outputTrackRotation,
group: ownGroup ?? trackOptions.group,
});
@@ -1615,29 +1847,115 @@ export class Conversion {
const originalNumberOfChannels = await track.getNumberOfChannels();
const originalSampleRate = await track.getSampleRate();
const firstTimestamp = await track.getFirstTimestamp();
let numberOfChannels = trackOptions.numberOfChannels ?? originalNumberOfChannels;
let sampleRate = trackOptions.sampleRate ?? originalSampleRate;
const needsTrimming = firstTimestamp < this._startTimestamp;
let needsPadding = firstTimestamp > this._startTimestamp && !this.output.format.supportsTimestampedMediaData;
let audioCodecs = this.output.format.getSupportedAudioCodecs();
if (
!trackOptions.forceTranscode
&& !trackOptions.quality
let needsTranscode = !this._copyMode
|| !!trackOptions.forceTranscode
|| !!trackOptions.quality
// eslint-disable-next-line @typescript-eslint/no-deprecated
&& !trackOptions.bitrate
&& numberOfChannels === originalNumberOfChannels
&& sampleRate === originalSampleRate
&& !needsTrimming
&& !needsPadding
&& audioCodecs.includes(sourceCodec)
&& (!trackOptions.codec || trackOptions.codec === sourceCodec)
&& !trackOptions.process
&& trackOptions.sampleFormat === undefined
) {
|| !!trackOptions.bitrate
|| numberOfChannels !== originalNumberOfChannels
|| sampleRate !== originalSampleRate
|| !audioCodecs.includes(sourceCodec)
|| (!!trackOptions.codec && trackOptions.codec !== sourceCodec)
|| trackOptions.process !== undefined
|| trackOptions.sampleFormat !== undefined;
let copyStartPacket: EncodedPacket | null = null;
if (!needsTranscode) {
// Check if we can copy it
const sink = new EncodedPacketSink(track);
let startPacket = await sink.getKeyPacket(this._startTimestamp)
?? await sink.getFirstKeyPacket();
if (
startPacket
&& startPacket.timestamp < this._startTimestamp
&& this._copyBoundaryPolicy === 'shrink'
) {
startPacket = await sink.getNextKeyPacket(startPacket);
}
const hasDecoderWarmup = (NON_PCM_AUDIO_CODECS as readonly AudioCodec[]).includes(sourceCodec)
&& sourceCodec !== 'flac';
if (startPacket && this._copyBoundaryPolicy === 'expand' && hasDecoderWarmup) {
// Go one packet back
const previousPacket = await sink.getKeyPacket(
startPacket.timestamp - 1 / (await track.getTimeResolution()),
);
if (previousPacket) {
startPacket = previousPacket;
}
}
copyStartPacket = startPacket;
if (startPacket) {
if (!this.output.format.supportsTimestampedMediaData) {
// Wants zero
if (this._timestampOffsetAdjusted) {
// We've already adjusted, we can't adjust twice
const isValid = startPacket.timestamp + this._timestampOffset === 0;
if (!isValid) {
needsTranscode = true;
}
} else {
const correction = clamp(
this._startTimestamp - startPacket.timestamp,
-this._copyTimestampShiftTolerance,
this._copyTimestampShiftTolerance,
);
const shiftedStartTimestamp = startPacket.timestamp + correction;
const isValid = shiftedStartTimestamp === this._startTimestamp;
if (isValid) {
this._timestampOffset = -this._startTimestamp + correction;
this._timestampOffsetAdjusted = true;
} else {
needsTranscode = true;
}
}
} else if (
this.output.format.negativeTimestampSupport !== 'full'
&& startPacket.timestamp < this._startTimestamp
) {
const correction = Math.min(
this._startTimestamp - startPacket.timestamp,
this._copyTimestampShiftTolerance,
);
const shiftedStartTimestamp = startPacket.timestamp + correction;
const isValid = shiftedStartTimestamp >= this._startTimestamp
|| (
this.output.format.negativeTimestampSupport === 'prefer-non-negative'
&& this._copyMode === 'forced'
);
if (isValid) {
this._timestampOffset = Math.max(this._timestampOffset, -this._startTimestamp + correction);
} else {
needsTranscode = true;
}
}
}
}
if (needsTranscode && this._copyMode === 'forced') {
this.discardedTracks.push({
track,
reason: 'cannot_copy',
trackOptions,
});
return;
}
if (!needsTranscode) {
// Fast path, we can simply copy over the encoded packets
const source = new EncodedAudioPacketSource(sourceCodec);
@@ -1648,19 +1966,26 @@ export class Conversion {
const decoderConfig = await track.getDecoderConfig();
const meta: EncodedAudioChunkMetadata = { decoderConfig: decoderConfig ?? undefined };
for await (const packet of sink.packets()) {
if (this.state === 'canceled') {
// eslint-disable-next-line curly
if (copyStartPacket) for await (const packet of sink.packets(copyStartPacket)) {
if (this._state === 'canceled') {
break;
}
if (packet.timestamp >= this._endTimestamp) {
break;
}
if (
this._copyBoundaryPolicy === 'shrink'
&& packet.timestamp + packet.duration > this._endTimestamp
) {
break;
}
const modifiedPacket = packet.clone({
timestamp: packet.timestamp - this._startTimestamp,
timestamp: packet.timestamp + this._timestampOffset,
duration: packet.duration,
});
assert(modifiedPacket.timestamp >= 0);
this._reportProgress(outputTrackId, modifiedPacket.timestamp + modifiedPacket.duration);
await source.add(modifiedPacket, meta);
@@ -1770,39 +2095,14 @@ export class Conversion {
audioSource = source;
this._registerTrackPump(async (pump) => {
let needsPadding: boolean | null = null;
const sink = new AudioSampleSink(track);
for await (using sample of sink.samples(this._startTimestamp, this._endTimestamp)) {
if (this.state === 'canceled') {
if (this._state === 'canceled') {
break;
}
if (needsPadding) {
// Add one padding sample at the beginning
const paddingLength = firstTimestamp - this._startTimestamp;
const paddingLengthSamples = Math.round(paddingLength * originalSampleRate);
const bytesPerSample = getBytesPerSample(sample.format);
const data = new Uint8Array(bytesPerSample * paddingLengthSamples * originalNumberOfChannels);
if (sample.format === 'u8' || sample.format === 'u8-planar') {
data.fill(2 ** 7); // Fill it with the silent value
}
using silentSample = new AudioSample({
data,
// Use the same format the decoder is spitting out. This avoids feeding changing sample
// formats to the audio encoder.
format: sample.format,
numberOfChannels: originalNumberOfChannels,
sampleRate: originalSampleRate,
timestamp: 0,
});
await this._registerAudioSample(
pump, silentSample, source, outputTrackId, () => lastSampleTimestamp,
);
needsPadding = false;
}
let startFrame = 0;
let endFrame = sample.numberOfFrames;
@@ -1832,7 +2132,38 @@ export class Conversion {
using finalSample = finalSampleLet;
// Offset the timestamp as needed
finalSample.setTimestamp(finalSample.timestamp - this._startTimestamp);
finalSample.setTimestamp(finalSample.timestamp + this._timestampOffset);
if (needsPadding === null) {
needsPadding = finalSample.timestamp > 0 && !this.output.format.supportsTimestampedMediaData;
}
if (needsPadding) {
// Add one padding sample at the beginning
const paddingLength = finalSample.timestamp;
const paddingLengthSamples = Math.round(paddingLength * originalSampleRate);
const bytesPerSample = getBytesPerSample(sample.format);
const data = new Uint8Array(bytesPerSample * paddingLengthSamples * originalNumberOfChannels);
if (sample.format === 'u8' || sample.format === 'u8-planar') {
data.fill(2 ** 7); // Fill it with the silent value
}
using silentSample = new AudioSample({
data,
// Use the same format the decoder is spitting out. This avoids feeding changing sample
// formats to the audio encoder.
format: sample.format,
numberOfChannels: originalNumberOfChannels,
sampleRate: originalSampleRate,
timestamp: 0,
});
await this._registerAudioSample(
pump, silentSample, source, outputTrackId, () => lastSampleTimestamp,
);
needsPadding = false;
}
await this._registerAudioSample(
pump, finalSample, source, outputTrackId, () => lastSampleTimestamp,
@@ -1852,11 +2183,16 @@ export class Conversion {
}
const audioTrackLanguageCode = await track.getLanguageCode();
const trackName = await track.getName();
const trackDisposition = await track.getDisposition();
this.output.addAudioTrack(audioSource, {
// TODO: This condition can be removed when all demuxers properly homogenize to BCP47 in v2
languageCode: isIso639Dash2LanguageCode(audioTrackLanguageCode) ? audioTrackLanguageCode : undefined,
name: await track.getName() ?? undefined,
disposition: await track.getDisposition(),
languageCode: isIso639Dash2LanguageCode(audioTrackLanguageCode)
? audioTrackLanguageCode
: undefined,
name: trackName ?? undefined,
disposition: trackDisposition,
group: ownGroup ?? trackOptions.group,
});
@@ -1909,7 +2245,7 @@ export class Conversion {
/** @internal */
async _checkpoint(pump: TrackPump, timestamp: number) {
while (this.state !== 'canceled' && (timestamp >= this._executionUntil || this._pauseRequested)) {
while (this._state !== 'canceled' && (timestamp >= this._executionUntil || this._pauseRequested)) {
// We've reached the target; signal it and suspend until the next execution wakes us up
pump.resolvers.resolve();
@@ -1975,7 +2311,9 @@ class TrackSynchronizer {
}
declareTrack(trackId: number) {
this.maxTimestamps.set(trackId, 0);
// Using -Infinity will automatically cause all tracks to wait for each other at the start until they figure out
// the true min timestamp
this.maxTimestamps.set(trackId, -Infinity);
}
shouldWait(trackId: number, timestamp: number) {
@@ -1986,7 +2324,7 @@ class TrackSynchronizer {
const newMin = this.computeMinAndMaybeResolve();
if (
this.conversion.state === 'canceled'
this.conversion._state === 'canceled'
|| this.conversion._pauseRequested
|| timestamp >= this.conversion._executionUntil
) {
src/index.ts
+1
View File
@@ -299,6 +299,7 @@ export {
type ConversionOptions,
type ConversionVideoOptions,
type ConversionAudioOptions,
type ConversionCopyOptions,
type ConversionExecuteOptions,
ConversionCanceledError,
type DiscardedTrack,
src/matroska/matroska-demuxer.ts
+24 -11
View File
@@ -1934,6 +1934,29 @@ export class MatroskaDemuxer extends Demuxer {
}
}
}
async getDurationFromMetadata(segment: Segment) {
if (segment.duration <= 0) {
return null;
}
// The kosher definition of the Duration field is "latest end time - earliest start time" across all tracks in
// the segment; since we currently mean "end timestamp" with "duration", we need to determine the earliest
// start time before we can return a value here.
let minTimestamp: number | null = null;
for (const track of segment.tracks) {
assert(track.trackBacking);
const firstPacket = await track.trackBacking.getFirstPacket({ metadataOnly: true });
if (firstPacket) {
minTimestamp = Math.min(minTimestamp ?? Infinity, firstPacket.timestamp);
}
}
let endTimestamp = segment.duration / segment.timestampFactor;
endTimestamp += minTimestamp ?? 0;
return endTimestamp;
}
}
abstract class MatroskaTrackBacking implements InputTrackBacking {
@@ -2016,17 +2039,7 @@ abstract class MatroskaTrackBacking implements InputTrackBacking {
}
async getDurationFromMetadata() {
const segment = this.internalTrack.segment;
if (segment.duration <= 0) {
return null;
}
let endTimestamp = segment.duration / segment.timestampFactor;
const firstPacket = await this.getFirstPacket({ metadataOnly: true });
endTimestamp += firstPacket?.timestamp ?? 0;
return endTimestamp;
return this.internalTrack.demuxer.getDurationFromMetadata(this.internalTrack.segment);
}
async getLiveRefreshInterval() {
src/output-format.ts
+58 -1
View File
@@ -85,6 +85,14 @@ export abstract class OutputFormat {
* durations of the media data.
*/
abstract get supportsTimestampedMediaData(): boolean;
/**
* The degree to which this output format supports writing media data with negative timestamps.
* - `'full'` - Negative timestamps are fully supported.
* - `'prefer-non-negative'` - Negative timestamps are technically supported, but their use is discouraged.
* - `'none'` - Negative timestamps are not supported.
* - `null` - Not applicable since the container doesn't support timestamped media at all.
*/
abstract get negativeTimestampSupport(): 'full' | 'prefer-non-negative' | 'none' | null;
/** Returns a list of video codecs that this output format can contain. */
getSupportedVideoCodecs(): VideoCodec[] {
@@ -278,6 +286,10 @@ export abstract class IsobmffOutputFormat extends OutputFormat {
return true;
}
get negativeTimestampSupport() {
return 'full' as const;
}
/** @internal */
_createMuxer(output: Output) {
return new IsobmffMuxer(output, this);
@@ -579,6 +591,10 @@ export class MkvOutputFormat extends OutputFormat {
get supportsTimestampedMediaData() {
return true;
}
get negativeTimestampSupport() {
return 'prefer-non-negative' as const;
}
}
/**
@@ -719,6 +735,10 @@ export class Mp3OutputFormat extends OutputFormat {
get supportsTimestampedMediaData() {
return false;
}
get negativeTimestampSupport() {
return null;
}
}
/**
@@ -821,6 +841,10 @@ export class WavOutputFormat extends OutputFormat {
get supportsTimestampedMediaData() {
return false;
}
get negativeTimestampSupport() {
return null;
}
}
/**
@@ -916,6 +940,10 @@ export class OggOutputFormat extends OutputFormat {
get supportsTimestampedMediaData() {
return false;
}
get negativeTimestampSupport() {
return null;
}
}
/**
@@ -994,6 +1022,10 @@ export class AdtsOutputFormat extends OutputFormat {
get supportsTimestampedMediaData() {
return false;
}
get negativeTimestampSupport() {
return null;
}
}
/**
@@ -1079,6 +1111,10 @@ export class FlacOutputFormat extends OutputFormat {
get supportsTimestampedMediaData() {
return false;
}
get negativeTimestampSupport() {
return null;
}
}
/**
@@ -1164,6 +1200,10 @@ export class MpegTsOutputFormat extends OutputFormat {
get supportsTimestampedMediaData() {
return true;
}
get negativeTimestampSupport() {
return 'prefer-non-negative' as const;
}
}
/**
@@ -1424,7 +1464,24 @@ export class HlsOutputFormat extends OutputFormat {
}
get supportsTimestampedMediaData(): boolean {
return true; // I guess??
return true; // It's only half true, really, but "false" is not correct either
}
get negativeTimestampSupport() {
const formats = toArray(this._options.segmentFormat);
// Return the lowest baseline across all segment formats
if (formats.some(format => format.negativeTimestampSupport === 'none')) {
return 'none' as const;
}
if (formats.some(format => format.negativeTimestampSupport === 'prefer-non-negative')) {
return 'prefer-non-negative' as const;
}
if (formats.some(format => format.negativeTimestampSupport === 'full')) {
return 'full' as const;
}
return null;
}
/** @internal */
test/browser/conversion.test.ts
+272 -2
View File
@@ -3,20 +3,23 @@ import { Input } from '../../src/input.js';
import {
AdtsOutputFormat,
HlsOutputFormat,
MkvOutputFormat,
Mp4OutputFormat,
MpegTsOutputFormat,
OutputFormat,
WavOutputFormat,
} from '../../src/output-format.js';
import { Output, OutputTrackGroup } from '../../src/output.js';
import { BufferSource, CustomPathedSource, UrlSource } from '../../src/source.js';
import { expect, test } from 'vitest';
import { BufferTarget, PathedTarget } from '../../src/target.js';
import { Conversion, ConversionCanceledError } from '../../src/conversion.js';
import { assert } from '../../src/misc.js';
import { Conversion, ConversionCanceledError, ConversionOptions } from '../../src/conversion.js';
import { assert, uint8ArraysAreEqual } from '../../src/misc.js';
import { InputVideoTrack } from '../../src/input-track.js';
import { CanvasSource, EncodedAudioPacketSource } from '../../src/media-source.js';
import { Quality } from '../../src/encode.js';
import { EncodedPacket } from '../../src/packet.js';
import { EncodedPacketSink } from '../../src/media-sink.js';
test('Rotation is baked in when rerendering', async () => {
using input = new Input({
@@ -783,3 +786,270 @@ test('Resizing at various scale factors', async () => {
expect(await videoTrack!.getDisplayHeight()).toBe(height);
}
});
test('Packet copy, whole file', async () => {
await testCopy({
conversionOptions: {},
expectedTimeOffset: 0,
videoStartTimestamp: 0,
videoEndTimestamp: 5,
audioStartTimestamp: -1024 / 48000,
audioEndTimestamp: 5,
});
});
test('Packet copy, keyframe trim', async () => {
await testCopy({
conversionOptions: {
trim: {
start: 1,
end: 2,
},
},
expectedTimeOffset: 1,
videoStartTimestamp: 0,
videoEndTimestamp: 1,
audioStartTimestamp: -0.04,
audioEndTimestamp: 1.0053333333333334,
});
});
test('Packet copy, keyframe trim, shrink', async () => {
await testCopy({
conversionOptions: {
trim: {
start: 1,
end: 2,
},
copy: {
boundaryPolicy: 'shrink',
},
},
expectedTimeOffset: 1,
videoStartTimestamp: 0,
videoEndTimestamp: 1,
audioStartTimestamp: -0.04 + 2 * 1024 / 48000,
audioEndTimestamp: 1.0053333333333334 - 1024 / 48000,
});
});
test('Packet copy, delta frame trim', async () => {
await testCopy({
conversionOptions: {
trim: {
start: 1.5,
end: 2.5,
},
},
expectedTimeOffset: 1.5,
videoStartTimestamp: -0.5,
videoEndTimestamp: 1.02,
audioStartTimestamp: -0.028,
audioEndTimestamp: 1.0173333333333334,
});
});
test('Packet copy, delta frame trim, shrink', async () => {
await testCopy({
conversionOptions: {
trim: {
start: 1.5,
end: 2.5,
},
copy: {
boundaryPolicy: 'shrink',
},
},
expectedTimeOffset: 1.5,
videoStartTimestamp: 0.5,
videoEndTimestamp: 1.02, // Due to MP4 not being able to express a different duration for the last packet
audioStartTimestamp: -0.028 + 2 * 1024 / 48000,
audioEndTimestamp: 1.0173333333333334 - 1024 / 48000,
});
});
test('Packet copy, whole file, Matroska', async () => {
await testCopy({
outputFormat: new MkvOutputFormat(),
conversionOptions: {
copy: {
shiftTolerance: Infinity,
},
},
expectedTimeOffset: -1024 / 48000,
videoStartTimestamp: 0 + 1024 / 48000,
videoEndTimestamp: 5 + 1024 / 48000 - 1 / 25,
audioStartTimestamp: 0,
audioEndTimestamp: 235 * 1024 / 48000,
precision: 0.001,
});
});
test('Packet copy, whole file, Matroska, forced', async () => {
await testCopy({
outputFormat: new MkvOutputFormat(),
conversionOptions: {
copy: {
mode: 'forced',
},
},
expectedTimeOffset: 0,
videoStartTimestamp: 0,
videoEndTimestamp: 5 - 1 / 25,
audioStartTimestamp: -1024 / 48000,
audioEndTimestamp: 4.992,
precision: 0.001,
});
});
test('Packet copy, delta frame trim, Matroska', async () => {
await testCopy({
outputFormat: new MkvOutputFormat(),
conversionOptions: {
trim: {
start: 1.5,
end: 2.5,
},
copy: {
shiftTolerance: Infinity,
},
},
expectedTimeOffset: 1,
videoStartTimestamp: 0,
videoEndTimestamp: 1.48,
audioStartTimestamp: 0.472,
audioEndTimestamp: 1.496,
precision: 0.001,
});
});
test('Packet copy, delta frame trim, video transcoded, Matroska', async () => {
await testCopy({
outputFormat: new MkvOutputFormat(),
conversionOptions: {
video: {
forceTranscode: true,
codec: 'vp9',
},
trim: {
start: 1.5,
end: 2.5,
},
copy: {
shiftTolerance: Infinity,
},
},
expectedTimeOffset: 1.484,
videoStartTimestamp: 0.008,
videoEndTimestamp: 1.008,
audioStartTimestamp: 0,
audioEndTimestamp: 1.024,
precision: 0.001,
compareVideoPackets: false,
});
});
test('Packet copy, whole file, ADTS', async () => {
await testCopy({
outputFormat: new AdtsOutputFormat(),
conversionOptions: {
video: {
discard: true,
},
copy: {
mode: 'forced',
shiftTolerance: Infinity,
},
},
expectedTimeOffset: -1024 / 48000,
audioStartTimestamp: 0,
audioEndTimestamp: 5.034666666666666,
processNewAudioPacketData: data => data.subarray(7),
});
});
const testCopy = async (options: {
outputFormat?: OutputFormat;
conversionOptions: Omit<ConversionOptions, 'input' | 'output'>;
expectedTimeOffset: number;
videoStartTimestamp?: number;
videoEndTimestamp?: number;
audioStartTimestamp: number;
audioEndTimestamp: number;
precision?: number;
processNewAudioPacketData?: (data: Uint8Array) => Uint8Array;
compareVideoPackets?: boolean;
}) => {
const precision = options.precision ?? 0.000001;
const isCloseTo = (a: number, b: number) => {
return Math.abs(a - b) <= precision;
};
using input = new Input({
source: new UrlSource('/demo.mp4'),
formats: ALL_FORMATS,
});
const output = new Output({
format: options.outputFormat ?? new Mp4OutputFormat(),
target: new BufferTarget(),
});
const videoTrack = await input.getPrimaryVideoTrack();
const audioTrack = await input.getPrimaryAudioTrack();
assert(videoTrack);
assert(audioTrack);
const videoSink = new EncodedPacketSink(videoTrack);
const audioSink = new EncodedPacketSink(audioTrack);
const conversion = await Conversion.init({
input,
output,
...options.conversionOptions,
});
await conversion.execute();
using newInput = new Input({
source: new BufferSource(output.target.buffer!),
formats: ALL_FORMATS,
});
const newVideoTrack = await newInput.getPrimaryVideoTrack();
const newAudioTrack = await newInput.getPrimaryAudioTrack();
if (newVideoTrack) {
const newVideoSink = new EncodedPacketSink(newVideoTrack);
expect(isCloseTo(await newVideoTrack.getFirstTimestamp(), options.videoStartTimestamp!)).toBe(true);
expect(isCloseTo(await newVideoTrack.computeDuration(), options.videoEndTimestamp!)).toBe(true);
if (options.compareVideoPackets ?? true) {
for await (const newPacket of newVideoSink.packets()) {
const oldPacket = await videoSink.getPacket(
newPacket.timestamp + options.expectedTimeOffset + precision,
);
assert(oldPacket);
expect(uint8ArraysAreEqual(oldPacket.data, newPacket.data)).toBe(true);
expect(oldPacket.type).toEqual(newPacket.type);
}
}
}
if (newAudioTrack) {
const newAudioSink = new EncodedPacketSink(newAudioTrack);
expect(isCloseTo(await newAudioTrack.getFirstTimestamp(), options.audioStartTimestamp)).toBe(true);
expect(isCloseTo(await newAudioTrack.computeDuration(), options.audioEndTimestamp)).toBe(true);
for await (const newPacket of newAudioSink.packets()) {
const oldPacket = await audioSink.getPacket(newPacket.timestamp + options.expectedTimeOffset + precision);
assert(oldPacket);
const process = options.processNewAudioPacketData ?? (x => x);
expect(uint8ArraysAreEqual(oldPacket.data, process(newPacket.data))).toBe(true);
expect(oldPacket.type).toEqual(newPacket.type);
}
}
};
test/public/demo.mp4
BIN
View File
Binary file not shown.