Compare commits

..
22 Commits
Author SHA1 Message Date
Vanilagy cee57d1cdf Bump patch 2026-09-09 20:33:05 +02:00
Vanilagy f767b6f601 Fix AVC software decoder sometimes dropping B-frames (fixes #488) 2026-09-09 16:55:06 +02:00
Vanilagy 4b140fe46c Change FLAC test 2026-09-09 14:07:56 +02:00
Vanilagy b2b408749c Surface repeated keys are string[] for Vorbis-style metadata (fixes #490) 2026-09-09 14:01:31 +02:00
Vanilagy 533c857a31 Bump minor 2026-09-08 17:49:33 +02:00
Vanilagy 4f88d71277 Add handleUnhandledError to most sources (closes #489) 2026-09-08 17:42:23 +02:00
Vanilagy 6e6785e5fc Add more conversion tests for trimming wholly outside of the available media range 2026-09-08 17:20:14 +02:00
Vanilagy 450c5def97 Fix incorrect use of Content-Length when the server compresses the content (fixes #487) 2026-09-08 17:19:32 +02:00
Vanilagy b560977b50 Fix empty audio trim range crashing conversion (fixes #486) 2026-09-08 16:19:56 +02:00
David P.andGitHub 8a25743ee8 Negative timestamps (#485)
* Allow negative timestamps in media sources, add negative timestamp support to ISOBMFF muxer

* Add negative timestamp muxing support to Matroska

* Add negative timestamp support to MPEG-TS

* Add negative timestamp roundtrip tests for HLS

* Implement more powerful copy conversions, add conversion copy config, add OutputFormat.negativeTimestampSupport, fix Matroska duration from metadata computation

* Add copy conversion doc section
2026-09-08 15:39:35 +02:00
Vanilagy 36c72e0168 Add copy conversion doc section 2026-09-08 15:32:14 +02:00
Vanilagy a1ba54d337 Implement more powerful copy conversions, add conversion copy config, add OutputFormat.negativeTimestampSupport, fix Matroska duration from metadata computation 2026-09-08 14:42:03 +02:00
Vanilagy c67c5e4072 Bump patch 2026-09-04 22:56:36 +02:00
Vanilagy e1f3b5252b Add negative timestamp roundtrip tests for HLS 2026-09-04 16:07:10 +02:00
Vanilagy 34733015cb Add negative timestamp support to MPEG-TS 2026-09-04 15:27:19 +02:00
Vanilagy 9726b0cb4e Add negative timestamp muxing support to Matroska 2026-09-04 12:17:33 +02:00
Vanilagy 3c94d062d2 Allow negative timestamps in media sources, add negative timestamp support to ISOBMFF muxer 2026-09-04 11:48:50 +02:00
hi.jack2eeandGitHub 6c88763a5c Terminate sample iterators before throwing so late-decoded samples are closed (#484)
for await never calls return() on an iterator whose next() threw, so the
error branches in mediaSamplesInRange / mediaSamplesAtTimestamps left the
iteration un-terminated: the pump's decoder callback kept queueing decoded
samples that no consumer or cleanup path could ever close, orphaning them
until GC ("A VideoSample was garbage collected without first being closed").

Set terminated (and ended, where the pump gates on it) before throwing in
both the InputDisposedError and out-of-band error branches of both sample
generators - samples emitted after the throw are then closed on arrival by
the existing checks in the decoder callbacks.

Observed on iOS Safari when the hardware VideoDecoder dies mid-iteration
("Decoding task did not complete"): 3-14 samples orphaned per failing
iterator; with this change, at most the single sample already in flight
during the error microtask window.
2026-09-04 10:01:18 +02:00
Vanilagy 5954a472fa Remove console.log 2026-09-04 10:00:33 +02:00
Vanilagy 6319bf2eee Bump patch 2026-09-03 10:48:32 +02:00
Vanilagy 5ffb5a0979 Add sponsor 2026-09-03 10:48:03 +02:00
Vanilagy cf2edfada2 Relax audio track assert for esds box (fixes #481) 2026-09-02 09:32:06 +02:00
49 changed files with 1984 additions and 290 deletions
+17 -4
View File
@@ -25,7 +25,7 @@
chunked: true,
chunkSize: 2**20
});
const outputFormat = new Mediabunny.MovOutputFormat();
const outputFormat = new Mediabunny.Mp4OutputFormat();
const p = document.createElement('p');
p.textContent = 'Capturing...';
@@ -99,6 +99,8 @@
input,
output,
audio: {
forceTranscode: true,
//discard: true,
//codec: 'aac',
//forceTranscode: true,
//forceTranscode: true,
@@ -129,7 +131,9 @@
},
*/
video: {
height: 320,
forceTranscode: true,
//forceTranscode: true,
//height: 320,
//codec: 'avc',
//bitrate: new Mediabunny.Quality({
// quality: Infinity,
@@ -158,7 +162,10 @@
}
},
trim: {
end: 5,
start: 30,
end: 35,
//start: -60,
//end: 60,
//start: -2,
//end: 10,
//start: 300.14984567374756 - 100,
@@ -167,6 +174,12 @@
//start: startTime,
//end: startTime + 2,
},
copy: {
//mode: 'forced',
//mode: 'forced',
//shiftTolerance: Infinity,
//boundaryPolicy: 'shrink',
},
});
//console.log(conversion);
console.log(conversion.discardedTracks);
@@ -213,4 +226,4 @@
conversion = null;
input = null;
}, { once: true });
</script>
</script>
+34
View File
@@ -17,11 +17,45 @@
source: new Mediabunny.BlobSource(file),
});
const track = await input.getPrimaryVideoTrack();
if (!track) {
throw new Error('The synthetic file has no video track.');
}
let encodedPackets = 0;
for await (const packet of new Mediabunny.EncodedPacketSink(track).packets()) {
console.log(packet.timestamp)
encodedPackets++;
}
let decodedFrames = 0;
let lastTimestamp = null;
const sink = new Mediabunny.VideoSampleSink(track, {
hardwareAcceleration: 'prefer-software',
});
for await (const sample of sink.samples()) {
console.log(sample.timestamp)
decodedFrames++;
lastTimestamp = sample.timestamp;
sample.close();
}
input.dispose();
const reproduced = encodedPackets === 48 && decodedFrames === 47;
console.log({
decodedFrames,
encodedPackets,
lastTimestamp,
reproduced,
});
/*
const track = await input.getPrimaryVideoTrack();
const sink = new Mediabunny.EncodedPacketSink(track);
const packet = await sink.getFirstPacket();
console.log(packet);
*/
/*
for await (const packet of packetSink.packets()) {
+5 -5
View File
@@ -47,7 +47,7 @@
format = new Mediabunny.MkvOutputFormat();
format = new Mediabunny.MovOutputFormat();
format = new Mediabunny.Mp4OutputFormat({ fastStart: 'reserve' });
format = new Mediabunny.Mp4OutputFormat();
format = new Mediabunny.MkvOutputFormat({ });
let target = new Mediabunny.BufferTarget();
/*
@@ -191,8 +191,8 @@ Testing... <00:17.350>One... <00:18.125>Two...
9. <b>justify (bottom, right)</b>.
`;
subtitleSource.add(simpleWebvttFile);
subtitleSource.close();
//subtitleSource.add(simpleWebvttFile);
//subtitleSource.close();
const p = document.createElement('p');
document.body.append(p);
@@ -201,7 +201,7 @@ Testing... <00:17.350>One... <00:18.125>Two...
context.fillStyle = ['red', 'green', 'blue', 'yellow'][i % 4];
context.fillRect(canvas.width * Math.random(), canvas.height * Math.random(), canvas.width * Math.random(), canvas.height * Math.random());
await videoSource.add(i / 10, 1 / 10);
await videoSource.add(i / 10 - 1, 1 / 10);
p.textContent = i;
}
@@ -211,7 +211,7 @@ Testing... <00:17.350>One... <00:18.125>Two...
let length = 10;
let slicedAudioBuffer = sliceAudioBuffer(audioBuffer, length, audioContext);
await audioSource.add(slicedAudioBuffer);
//await audioSource.add(slicedAudioBuffer);
await output.finalize();
+48 -1
View File
@@ -8,7 +8,7 @@ The [reading](./reading-media-files) and [writing](./writing-media-files) primit
It has the following features:
- Transmuxing (changing the container format)
- Transmuxing (changing the container format while copying media data)
- Transcoding (changing a track's codec)
- Track removal
- Compression
@@ -446,6 +446,53 @@ Setting `start` to a value other than the default will currently force a *transc
In a future version of Mediabunny, the fast "packet copy path" that is currently reachable by leaving `start` unset, may be made available for any arbitrary trim range.
:::
## Copying media data
In media conversions, media data can either be copied directly from the old file to the new file or go through a transcoding step. If you're coming from FFmpeg, you may know the copy path by `-c copy`. Mediabunny differs from FFmpeg in that it will always try to perform a copy conversion by default, if the config permits. Copy conversions are often desirable because they are fast and lossless.
There are a few major factors that decide if a copy conversion is possible:
- No transcode-forcing operations (resizing, rotation, resampling, codec changes, etc.)
- Output format must be able to contain the input codecs
- Output format must support timestamps flexible enough
If you're converting to MP4, copy conversions are usually possible due to the wide range of codecs supported by that format plus the support for negative timestamps via edit lists which allow you to model any arbitrary trim range. Simpler formats such as MP3 are more restrictive and may require transcoding or timestamp offsetting to realize a copy conversion.
---
You can fully configure Mediabunny's behavior via the `copy` conversion option:
```ts
type ConversionCopyOptions = {
mode?: 'forced' | 'preferred';
shiftTolerance?: number;
boundaryPolicy?: 'expand' | 'shrink';
};
```
- `mode`\
Controls whether media copying is preferred or required. Defaults to `'preferred'`.
- `'forced'`: Copy encoded media where possible and discard tracks that cannot possibly be copied.
- `'preferred'`: Copy encoded media when possible, and transcode tracks that cannot be copied.
- `shiftTolerance`\
The maximum absolute shift, in seconds, that may be applied to the media to be able to copy it into the output format. Defaults to `0`, which permits no additional shift. Set to `Infinity` to permit any shift.
A shift of `0` gives you perfect _timeline sync_: output timestamps will match input timestamps exactly (only offset by the trim region). Any non-zero shift will break this property but will still, under all circumstances, maintain perfect cross-track and audio-video sync.
- `boundaryPolicy`\
Controls which media region will be copied to satisfy the requested trim range. Defaults to `'expand'`.
- `'expand'`: Include at least all media in the requested range. This may require expanding the media region due to key frames and packet boundaries, and thus may include media outside of your trim range. The region is always minimally expanded to satisfy the copy criteria.
- `'shrink'`: Only include media that lies entirely within the requested trim range. This may require shrinking the media region due to key frames and packet boundaries, and thus may exclude media inside of your trim range. The region is always minimally shrunk to satisfy the copy criteria.
Use `expand` if you don't want to lose any media; use `shrink` to never expose any media outside of the trim region.
---
You can disable copy conversions entirely by setting the field to `false`:
```ts
await Conversion.init({
copy: false,
// ...
});
```
## Metadata tags
By default, any [descriptive metadata tags](../api/MetadataTags.md) of the input will be copied to the output. If you want to further control the metadata tags written to the output, you can use the `tags` options:
+16 -1
View File
@@ -607,6 +607,9 @@ type BlobSourceOptions = {
// The maximum number of bytes the cache is allowed to hold
// in memory. Defaults to 8 MiB.
maxCacheSize?: number;
// Handles errors that occur while no read is pending
handleUnhandledError?: (error: unknown) => unknown;
};
```
@@ -640,6 +643,9 @@ type UrlSourceOptions = {
// Used to provide a custom fetch function
fetchFn?: typeof fetch;
// Handles errors that occur while no read is pending
handleUnhandledError?: (error: unknown) => unknown;
};
```
@@ -704,6 +710,9 @@ type FilePathSourceOptions = {
// The maximum number of bytes the cache is allowed to hold
// in memory. Defaults to 8 MiB.
maxCacheSize?: number;
// Handles errors that occur while no read is pending
handleUnhandledError?: (error: unknown) => unknown;
};
```
@@ -743,6 +752,7 @@ type CustomSourceOptions = {
dispose?: () => unknown;
maxCacheSize?: number;
prefetchProfile?: 'none' | 'fileSystem' | 'network';
handleUnhandledError?: (error: unknown) => unknown;
};
type MaybePromise<T> = T | Promise<T>;
@@ -761,6 +771,8 @@ type MaybePromise<T> = T | Promise<T>;
- `'none'` (default): No prefetching; only the data needed in the moment is requested.
- `'fileSystem'`: File system-optimized prefetching: a small amount of data is prefetched bidirectionally, aligned with page boundaries.
- `'network'`: Network-optimized prefetching, or more generally, prefetching optimized for any high-latency environment: tries to minimize the amount of read calls and aggressively prefetches data when sequential access patterns are detected.
- `handleUnhandledError`\
Handles errors that occur while no read is pending. By default, these become unhandled promise rejections.
::: info
`CustomSource` was previously known as `StreamSource` and is still available under that alias, but usage of `StreamSource` is deprecated.
@@ -790,6 +802,9 @@ type ReadableStreamSourceOptions = {
// The maximum number of bytes the cache is allowed to hold
// in memory. Defaults to 16 MiB.
maxCacheSize?: number;
// Handles errors that occur while no read is pending
handleUnhandledError?: (error: unknown) => unknown;
};
```
@@ -896,4 +911,4 @@ const input = new Input({
formats: ALL_FORMATS,
initInput,
});
```
```
+1
View File
@@ -127,6 +127,7 @@ const sponsors = {
{ image: 'https://avatars.githubusercontent.com/u/2565549', name: 'MatthewNorton', url: 'https://github.com/MatthewNorton' },
{ image: 'https://avatars.githubusercontent.com/u/272247752', name: 'Motionik', url: 'https://github.com/Motionik' },
{ image: 'https://avatars.githubusercontent.com/u/2698271', name: 'Matthew Gardner', url: 'https://github.com/spheric' },
{ image: 'https://avatars.githubusercontent.com/u/41021374', name: 'arth', url: 'https://github.com/arthtyagi' },
{ image: 'https://avatars.githubusercontent.com/u/5475819', name: 'AJ Funk', url: 'https://github.com/AJFunk' },
{ image: 'https://avatars.githubusercontent.com/u/30229596', name: 'Pablo Bonilla', url: 'https://github.com/devPablo' },
{ image: 'https://avatars.githubusercontent.com/u/536008', name: 'Satish Goda', url: 'https://github.com/satishgoda' },
+3
View File
@@ -128,6 +128,9 @@ const initMediaPlayer = async (resource: File | string) => {
isRelativeToUnixEpoch = (await Promise.all(tracks.map(t => t.isRelativeToUnixEpoch()))).some(Boolean);
playbackTimeAtStart = firstTimestamp;
// For degenerate cases where the end timestamp is less than 0
endTimestamp = Math.max(firstTimestamp, endTimestamp);
// Configure the time display elements accordingly
const timestampFontSize = isRelativeToUnixEpoch ? '12px' : '';
const timestampWhiteSpace = isRelativeToUnixEpoch ? 'pre' : '';
+10 -10
View File
@@ -1,12 +1,12 @@
{
"name": "mediabunny",
"version": "1.55.5",
"version": "1.56.1",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "mediabunny",
"version": "1.55.5",
"version": "1.56.1",
"license": "MPL-2.0",
"workspaces": [
".",
@@ -12958,7 +12958,7 @@
},
"packages/aac-encoder": {
"name": "@mediabunny/aac-encoder",
"version": "1.55.5",
"version": "1.56.1",
"license": "MPL-2.0",
"devDependencies": {
"@types/emscripten": "^1.40.1"
@@ -12973,7 +12973,7 @@
},
"packages/ac3": {
"name": "@mediabunny/ac3",
"version": "1.55.5",
"version": "1.56.1",
"license": "MPL-2.0",
"devDependencies": {
"@types/emscripten": "^1.40.1"
@@ -12988,7 +12988,7 @@
},
"packages/dts": {
"name": "@mediabunny/dts",
"version": "1.55.5",
"version": "1.56.1",
"license": "MPL-2.0",
"devDependencies": {
"@types/emscripten": "^1.40.1"
@@ -13003,7 +13003,7 @@
},
"packages/flac-encoder": {
"name": "@mediabunny/flac-encoder",
"version": "1.55.5",
"version": "1.56.1",
"license": "MPL-2.0",
"devDependencies": {
"@types/emscripten": "^1.40.1"
@@ -13018,7 +13018,7 @@
},
"packages/mp3-encoder": {
"name": "@mediabunny/mp3-encoder",
"version": "1.55.5",
"version": "1.56.1",
"license": "MPL-2.0",
"devDependencies": {
"@types/emscripten": "^1.40.1"
@@ -13033,7 +13033,7 @@
},
"packages/prores": {
"name": "@mediabunny/prores",
"version": "1.55.5",
"version": "1.56.1",
"license": "MPL-2.0",
"dependencies": {
"turbores": "^1.2.2"
@@ -13048,10 +13048,10 @@
},
"packages/server": {
"name": "@mediabunny/server",
"version": "1.55.5",
"version": "1.56.1",
"license": "MPL-2.0",
"dependencies": {
"@mediabunny/prores": "^1.55.5",
"@mediabunny/prores": "^1.56.1",
"node-av": "^6.0.0"
},
"funding": {
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "mediabunny",
"author": "Vanilagy",
"version": "1.55.5",
"version": "1.56.1",
"description": "Pure TypeScript media toolkit for reading, writing, and converting media files, directly in the browser.",
"type": "module",
"workspaces": [
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@mediabunny/aac-encoder",
"author": "Vanilagy",
"version": "1.55.5",
"version": "1.56.1",
"description": "AAC encoder extension for Mediabunny, based on libavcodec.",
"main": "./dist/bundles/mediabunny-aac-encoder.mjs",
"module": "./dist/bundles/mediabunny-aac-encoder.mjs",
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@mediabunny/ac3",
"author": "Vanilagy",
"version": "1.55.5",
"version": "1.56.1",
"description": "AC-3 and E-AC-3 (Dolby Digital) decoder and encoder extension for Mediabunny, based on libavcodec.",
"main": "./dist/bundles/mediabunny-ac3.mjs",
"module": "./dist/bundles/mediabunny-ac3.mjs",
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@mediabunny/dts",
"author": "Vanilagy",
"version": "1.55.5",
"version": "1.56.1",
"description": "DTS decoder and encoder extension for Mediabunny, based on libavcodec.",
"main": "./dist/bundles/mediabunny-dts.mjs",
"module": "./dist/bundles/mediabunny-dts.mjs",
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@mediabunny/flac-encoder",
"author": "Vanilagy",
"version": "1.55.5",
"version": "1.56.1",
"description": "FLAC encoder extension for Mediabunny, based on libFLAC.",
"main": "./dist/bundles/mediabunny-flac-encoder.mjs",
"module": "./dist/bundles/mediabunny-flac-encoder.mjs",
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@mediabunny/mp3-encoder",
"author": "Vanilagy",
"version": "1.55.5",
"version": "1.56.1",
"description": "MP3 encoder extension for Mediabunny, based on LAME.",
"main": "./dist/bundles/mediabunny-mp3-encoder.mjs",
"module": "./dist/bundles/mediabunny-mp3-encoder.mjs",
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@mediabunny/prores",
"author": "Vanilagy",
"version": "1.55.5",
"version": "1.56.1",
"description": "Apple ProRes decoder extension for Mediabunny, based on TurboRes.",
"main": "./dist/bundles/mediabunny-prores.mjs",
"module": "./dist/bundles/mediabunny-prores.mjs",
+2 -2
View File
@@ -1,7 +1,7 @@
{
"name": "@mediabunny/server",
"author": "Vanilagy",
"version": "1.55.5",
"version": "1.56.1",
"description": "Adds full video and audio decoder and encoder support to Mediabunny for use in server-side environments (Node, Bun, Deno). Based on NodeAV.",
"main": "./dist/bundles/mediabunny-server.cjs",
"module": "./dist/bundles/mediabunny-server.mjs",
@@ -35,7 +35,7 @@
},
"dependencies": {
"node-av": "^6.0.0",
"@mediabunny/prores": "^1.55.5"
"@mediabunny/prores": "^1.56.1"
},
"peerDependencies": {
"mediabunny": "^1.45.0"
+5 -1
View File
@@ -82,7 +82,11 @@ export class NodeAvVideoDecoder extends CustomVideoDecoder {
: null;
codecContext.sampleAspectRatio = new NodeAv.Rational(this.pixelAspectRatio.num, this.pixelAspectRatio.den);
const ret = await codecContext.open2();
// Honor inferred H.264 reorder buffering even when bitstream restrictions aren't explicitly signaled
// See https://github.com/Vanilagy/mediabunny/issues/488
const options = NodeAv.Dictionary.fromObject({ strict: NodeAv.FF_COMPLIANCE_STRICT });
const ret = await codecContext.open2(codec, options);
NodeAv.FFmpegError.throwIfError(ret, 'Open codec context');
this.codecContext = codecContext;
+13 -1
View File
@@ -55,7 +55,19 @@ export class Bitstream {
}
this.pos = end;
};
}
copyBits(n: number, other: Bitstream) {
let i = 0;
for (i; i < n - 7; i += 8) {
this.writeBits(8, other.readBits(8));
}
const leftover = n - i;
if (leftover > 0) {
this.writeBits(leftover, other.readBits(leftover));
}
}
readAlignedByte() {
if (this.pos % 8 !== 0) {
+112 -20
View File
@@ -25,6 +25,7 @@ import {
isChromium,
popcount,
setUint24,
writeExpGolomb,
} from './misc';
import { Logging } from './logging';
import { PacketType } from './packet';
@@ -185,6 +186,23 @@ const removeEmulationPreventionBytes = (data: Uint8Array) => {
return new Uint8Array(result);
};
export const addEmulationPreventionBytes = (data: Uint8Array) => {
const result: number[] = [];
let zeroCount = 0;
for (const byte of data) {
if (zeroCount === 2 && byte <= 0x03) {
result.push(0x03);
zeroCount = 0;
}
result.push(byte);
zeroCount = byte === 0 ? zeroCount + 1 : 0;
}
return new Uint8Array(result);
};
const ANNEX_B_START_CODE = new Uint8Array([0, 0, 0, 1]);
export const concatNalUnitsInAnnexB = (nalUnits: Uint8Array[]) => {
@@ -363,12 +381,14 @@ export const serializeAvcDecoderConfigurationRecord = (record: AvcDecoderConfigu
}
if (
record.avcProfileIndication === 100
|| record.avcProfileIndication === 110
|| record.avcProfileIndication === 122
|| record.avcProfileIndication === 144
(
record.avcProfileIndication === 100
|| record.avcProfileIndication === 110
|| record.avcProfileIndication === 122
|| record.avcProfileIndication === 144
)
&& record.chromaFormat !== null // Can happen if the data was too short
) {
assert(record.chromaFormat !== null);
assert(record.bitDepthLumaMinus8 !== null);
assert(record.bitDepthChromaMinus8 !== null);
assert(record.sequenceParameterSetExt !== null);
@@ -485,6 +505,7 @@ export const deserializeAvcDecoderConfigurationRecord = (data: Uint8Array): AvcD
};
export type AvcSpsInfo = {
emulationUnpreventedBytes: Uint8Array;
profileIdc: number;
constraintFlags: number;
levelIdc: number;
@@ -503,6 +524,9 @@ export type AvcSpsInfo = {
fullRangeFlag: number;
numReorderFrames: number;
maxDecFrameBuffering: number;
vuiParametersFlagBitOffset: number;
bitstreamRestrictionFlagBitOffset: number | null;
bitstreamRestrictionFlag: number | null;
};
const AVC_HEVC_ASPECT_RATIO_IDC_TABLE: Partial<Record<number, Rational>> = {
@@ -527,7 +551,8 @@ const AVC_HEVC_ASPECT_RATIO_IDC_TABLE: Partial<Record<number, Rational>> = {
/** Parses an AVC SPS (Sequence Parameter Set) to extract basic information. */
export const parseAvcSps = (sps: Uint8Array): AvcSpsInfo | null => {
try {
const bitstream = new Bitstream(removeEmulationPreventionBytes(sps));
const emulationUnpreventedBytes = removeEmulationPreventionBytes(sps);
const bitstream = new Bitstream(emulationUnpreventedBytes);
bitstream.skipBits(1); // forbidden_zero_bit
bitstream.skipBits(2); // nal_ref_idc
@@ -660,7 +685,10 @@ export const parseAvcSps = (sps: Uint8Array): AvcSpsInfo | null => {
let numReorderFrames: number | null = null;
let maxDecFrameBuffering: number | null = null;
let bitstreamRestrictionFlagBitOffset: number | null = null;
let bitstreamRestrictionFlag: number | null = null;
const vuiParametersFlagBitOffset = bitstream.pos;
const vuiParametersPresentFlag = bitstream.readBits(1);
if (vuiParametersPresentFlag) {
const aspectRatioInfoPresentFlag = bitstream.readBits(1);
@@ -726,7 +754,8 @@ export const parseAvcSps = (sps: Uint8Array): AvcSpsInfo | null => {
bitstream.skipBits(1); // pic_struct_present_flag
const bitstreamRestrictionFlag = bitstream.readBits(1);
bitstreamRestrictionFlagBitOffset = bitstream.pos;
bitstreamRestrictionFlag = bitstream.readBits(1);
if (bitstreamRestrictionFlag) {
bitstream.skipBits(1); // motion_vectors_over_pic_boundaries_flag
readExpGolomb(bitstream); // max_bytes_per_pic_denom
@@ -776,6 +805,7 @@ export const parseAvcSps = (sps: Uint8Array): AvcSpsInfo | null => {
assert(maxDecFrameBuffering !== null);
return {
emulationUnpreventedBytes,
profileIdc,
constraintFlags,
levelIdc,
@@ -794,6 +824,9 @@ export const parseAvcSps = (sps: Uint8Array): AvcSpsInfo | null => {
fullRangeFlag,
numReorderFrames,
maxDecFrameBuffering,
vuiParametersFlagBitOffset,
bitstreamRestrictionFlagBitOffset,
bitstreamRestrictionFlag,
};
} catch (error) {
Logging._error('Error parsing AVC SPS:', error);
@@ -818,6 +851,57 @@ const skipAvcHrdParameters = (bitstream: Bitstream) => {
bitstream.skipBits(5); // time_offset_length
};
/**
* Adds the missing "bitstream restriction" section within the VUI parameter section. This is done to communicate
* frame reorder buffer size to the decoder, which may otherwise, in its absence, assume no B-frames and drop or skip
* them.
* See https://github.com/Vanilagy/mediabunny/issues/488
*/
export const addAvcBitstreamRestriction = (sps: AvcSpsInfo) => {
assert(sps.bitstreamRestrictionFlag !== 1);
const modifiedBytes = new Uint8Array(sps.emulationUnpreventedBytes.byteLength + 64);
const oldBitstream = new Bitstream(sps.emulationUnpreventedBytes);
const newBitstream = new Bitstream(modifiedBytes);
if (sps.bitstreamRestrictionFlag === null) {
// No VUI at all; let's write a minimal one
newBitstream.copyBits(sps.vuiParametersFlagBitOffset, oldBitstream);
newBitstream.writeBits(1, 1); // vui_parameters_present_flag
newBitstream.writeBits(1, 0); // aspect_ratio_info_present_flag
newBitstream.writeBits(1, 0); // overscan_info_present_flag
newBitstream.writeBits(1, 0); // video_signal_type_present_flag
newBitstream.writeBits(1, 0); // chroma_loc_info_present_flag
newBitstream.writeBits(1, 0); // timing_info_present_flag
newBitstream.writeBits(1, 0); // nal_hrd_parameters_present_flag
newBitstream.writeBits(1, 0); // vcl_hrd_parameters_present_flag
newBitstream.writeBits(1, 0); // pic_struct_present_flag
} else {
// We have a VUI but no bitstream restriction info
assert(sps.bitstreamRestrictionFlagBitOffset !== null);
newBitstream.copyBits(sps.bitstreamRestrictionFlagBitOffset, oldBitstream);
}
newBitstream.writeBits(1, 1); // bitstream_restriction_flag
// Defaults from the H.264 spec:
newBitstream.writeBits(1, 1); // motion_vectors_over_pic_boundaries_flag
writeExpGolomb(newBitstream, 2); // max_bytes_per_pic_denom
writeExpGolomb(newBitstream, 1); // max_bits_per_mb_denom
writeExpGolomb(newBitstream, 16); // log2_max_mv_length_horizontal
writeExpGolomb(newBitstream, 16); // log2_max_mv_length_vertical
writeExpGolomb(newBitstream, sps.numReorderFrames);
writeExpGolomb(newBitstream, sps.maxDecFrameBuffering);
// There's nothing after this (VUI is at the end of SPS)
newBitstream.writeBits(1, 1); // rbsp_stop_one_bit
newBitstream.writeBits((8 - newBitstream.pos % 8) % 8, 0);
const byteLength = newBitstream.pos / 8;
assert(Number.isInteger(byteLength));
return addEmulationPreventionBytes(modifiedBytes.subarray(0, byteLength));
};
// Data specified in ISO 14496-15
export type HevcDecoderConfigurationRecord = {
configurationVersion: number;
@@ -2597,7 +2681,13 @@ export const readVorbisComments = (bytes: Uint8Array, metadataTags: MetadataTags
const value = string.slice(separatorIndex + 1);
metadataTags.raw ??= {};
metadataTags.raw[key] ??= value;
if (Array.isArray(metadataTags.raw[key])) {
metadataTags.raw[key] = [...metadataTags.raw[key], value];
} else if (typeof metadataTags.raw[key] === 'string') {
metadataTags.raw[key] = [metadataTags.raw[key], value];
} else {
metadataTags.raw[key] ??= value;
}
switch (key) {
case 'TITLE': {
@@ -2728,7 +2818,7 @@ export const createVorbisComments = (headerBytes: Uint8Array, tags: MetadataTags
commentHeaderParts.push(currentBuffer);
const writtenTags = new Set<string>();
const writtenTags: string[] = [];
const addCommentTag = (key: string, value: string) => {
const joined = `${key}=${value}`;
const encoded = textEncoder.encode(joined);
@@ -2740,7 +2830,7 @@ export const createVorbisComments = (headerBytes: Uint8Array, tags: MetadataTags
currentBuffer.set(encoded, 4);
commentHeaderParts.push(currentBuffer);
writtenTags.add(key);
writtenTags.push(key);
};
for (const { key, value } of keyValueIterator(tags)) {
@@ -2770,12 +2860,7 @@ export const createVorbisComments = (headerBytes: Uint8Array, tags: MetadataTags
}; break;
case 'date': {
const rawVersion = tags.raw?.['DATE'] ?? tags.raw?.['date'];
if (rawVersion && typeof rawVersion === 'string') {
addCommentTag('DATE', rawVersion);
} else {
addCommentTag('DATE', value.toISOString().slice(0, 10));
}
addCommentTag('DATE', value.toISOString().slice(0, 10));
}; break;
case 'comment': {
@@ -2803,8 +2888,7 @@ export const createVorbisComments = (headerBytes: Uint8Array, tags: MetadataTags
}; break;
case 'images': {
// For example, in .flac, we put the pictures in a different section,
// not in the Vorbis comment header.
// For example, in .flac, we put the pictures in a different section, not in the Vorbis comment header.
if (!writeImages) {
break;
}
@@ -2861,18 +2945,26 @@ export const createVorbisComments = (headerBytes: Uint8Array, tags: MetadataTags
if (tags.raw) {
for (const key in tags.raw) {
const value = tags.raw[key] ?? tags.raw[key.toLowerCase()];
if (key === 'vendor' || value == null || writtenTags.has(key)) {
if (key === 'vendor' || value == null || writtenTags.includes(key)) {
continue;
}
if (typeof value === 'string') {
addCommentTag(key, value);
} else if (Array.isArray(value)) {
// String arrays turn into the tag being repeated for each element
const isOnlyStrings = value.every(x => typeof x === 'string');
if (isOnlyStrings) {
for (const elem of value) {
addCommentTag(key, elem);
}
}
}
}
}
const listLengthBuffer = new Uint8Array(4);
toDataView(listLengthBuffer).setUint32(0, writtenTags.size, true);
toDataView(listLengthBuffer).setUint32(0, writtenTags.length, true);
commentHeaderParts.splice(2, 0, listLengthBuffer); // Insert after the header and vendor section
// Merge all comment header parts into a single buffer
+465 -104
View File
@@ -44,6 +44,7 @@ import {
ceilToMultipleOfTwo,
clamp,
isIso639Dash2LanguageCode,
isNumber,
MaybePromise,
normalizeRotation,
promiseWithResolvers,
@@ -51,6 +52,7 @@ import {
} from './misc';
import { Output, OutputTrackGroup } from './output';
import { Mp4OutputFormat } from './output-format';
import { EncodedPacket } from './packet';
import {
AudioSample,
clampCropRectangle,
@@ -132,6 +134,13 @@ export type ConversionOptions = {
end?: number;
};
/**
* Options for controlling when media is copied directly without transcoding it. Set to `false` to always transcode.
* Defaults to `{}`, which will copy media whenever possible and otherwise transcode it while retaining precise
* timestamps.
*/
copy?: ConversionCopyOptions | false;
/**
* An object or a callback that returns or resolves to an object containing the descriptive metadata tags that
* should be written to the output file. If a function is passed, it will be passed the tags of the input file as
@@ -237,7 +246,7 @@ export type ConversionVideoOptions = {
* `'no-preference'`, the default.
*/
hardwareAcceleration?: 'no-preference' | 'prefer-hardware' | 'prefer-software';
/** When `true`, video will always be re-encoded instead of directly copying over the encoded samples. */
/** When `true`, video will always be re-encoded instead of directly copying over the encoded packets. */
forceTranscode?: boolean;
/**
* Allows for custom user-defined processing of video frames, e.g. for applying overlays, color transformations, or
@@ -303,7 +312,7 @@ export type ConversionAudioOptions = {
* @deprecated Use `quality` instead.
*/
bitrate?: number | Quality;
/** When `true`, audio will always be re-encoded instead of directly copying over the encoded samples. */
/** When `true`, audio will always be re-encoded instead of directly copying over the encoded packets. */
forceTranscode?: boolean;
/**
* Allows for custom user-defined processing of audio samples, e.g. for applying audio effects, transformations, or
@@ -338,6 +347,44 @@ export type ConversionAudioOptions = {
group?: OutputTrackGroup | OutputTrackGroup[];
};
/**
* Options for copying encoded media during conversion.
* @group Conversion
* @public
*/
export type ConversionCopyOptions = {
/**
* Controls whether media copying is preferred or required. Defaults to `'preferred'`.
*
* - `'forced'`: Copy encoded media where possible and discard tracks that cannot possibly be copied.
* - `'preferred'`: Copy encoded media when possible, and transcode tracks that cannot be copied.
*/
mode?: 'forced' | 'preferred';
/**
* The maximum absolute shift, in seconds, that may be applied to the media to be able to copy it into the output
* format. Defaults to `0`, which permits no additional shift. Set to `Infinity` to permit any shift.
*
* A shift of `0` gives you perfect _timeline sync_: output timestamps will match input timestamps exactly (only
* offset by the trim region). Any non-zero shift will break this property but will still, under all circumstances,
* maintain perfect cross-track and audio-video sync.
*/
shiftTolerance?: number;
/**
* Controls which media region will be copied to satisfy the requested trim range. Defaults to `'expand'`.
*
* - `'expand'`: Include at least all media in the requested range. This may require expanding the media region due
* to key frames and packet boundaries, and thus may include media outside of your trim range. The region is always
* minimally expanded to satisfy the copy criteria.
* - `'shrink'`: Only include media that lies entirely within the requested trim range. This may require shrinking
* the media region due to key frames and packet boundaries, and thus may exclude media inside of your trim range.
* The region is always minimally shrunk to satisfy the copy criteria.
*
* Use `expand` if you don't want to lose any media; use `shrink` to never expose any media outside of the
* trim region.
*/
boundaryPolicy?: 'expand' | 'shrink';
};
const validateVideoOptions = (videoOptions: ConversionVideoOptions) => {
if (!videoOptions || typeof videoOptions !== 'object') {
throw new TypeError('options.video, when provided, must be an object.');
@@ -546,6 +593,8 @@ export type DiscardedTrack = {
* - `'no_encodable_target_codec'`: We can't find a codec that we are able to encode and that can be contained
* within the output format. This reason can be hit if the environment doesn't support the necessary encoders, or if
* you requested a codec that cannot be contained within the output format.
* - `'cannot_copy'`: {@link ConversionCopyOptions.mode} was set to `'forced'` but the track could not be copied
* with the given copy configuration because it would require a transcode instead.
*/
reason:
| 'discarded_by_user'
@@ -553,7 +602,8 @@ export type DiscardedTrack = {
| 'max_track_count_of_type_reached'
| 'unknown_source_codec'
| 'undecodable_source_codec'
| 'no_encodable_target_codec';
| 'no_encodable_target_codec'
| 'cannot_copy';
/** The options that were provided for this track, or `{}` if none were provided. */
trackOptions: ConversionVideoOptions | ConversionAudioOptions;
};
@@ -597,22 +647,28 @@ export class Conversion {
/** The output file. */
readonly output: Output;
/**
* The current state of the conversion.
*
* - `'idle'`: The conversion is not currently executing and isn't done; `execute` can be called.
* - `'executing'`: A call to `execute` is currently running.
* - `'canceled'`: The conversion has been canceled and can no longer be executed.
* - `'done'`: The conversion has run to completion. Subsequent calls to `execute` do nothing.
*/
state: 'idle' | 'executing' | 'canceled' | 'done' = 'idle';
/** @internal */
_state: 'idle' | 'executing' | 'canceled' | 'done' = 'idle';
/** @internal */
_options: ConversionOptions;
/** @internal */
_copyMode: false | 'forced' | 'preferred';
/** @internal */
_copyTimestampShiftTolerance: number;
/** @internal */
_copyBoundaryPolicy: 'expand' | 'shrink';
/** @internal */
_startTimestamp!: number;
/** @internal */
_endTimestamp!: number;
/** @internal */
_timestampOffset = 0;
/** @internal */
_timestampOffsetAdjusted = false;
/** @internal */
_copyTimestampPossible = new Map<InputTrack, boolean>();
/** @internal */
_copyStartPackets = new Map<InputTrack, EncodedPacket | null>();
/** @internal */
_nextOutputTrackId = 0;
@@ -673,6 +729,18 @@ export class Conversion {
/** The list of tracks from the input file that have been discarded, alongside the discard reason. */
readonly discardedTracks: DiscardedTrack[] = [];
/**
* The current state of the conversion.
*
* - `'idle'`: The conversion is not currently executing and isn't done; `execute` can be called.
* - `'executing'`: A call to `execute` is currently running.
* - `'canceled'`: The conversion has been canceled and can no longer be executed.
* - `'done'`: The conversion has run to completion. Subsequent calls to `execute` do nothing.
*/
get state() {
return this._state;
}
/** Initializes a new conversion process without starting the conversion. */
static async init(options: ConversionOptions) {
const conversion = new Conversion(options);
@@ -704,6 +772,28 @@ export class Conversion {
if (options.composable !== undefined && typeof options.composable !== 'boolean') {
throw new TypeError('options.composable, when provided, must be a boolean.');
}
if (options.copy !== undefined && options.copy !== false) {
if (!options.copy || typeof options.copy !== 'object') {
throw new TypeError('options.copy, when provided, must be an object or false.');
}
if (options.copy.mode !== undefined && !['forced', 'preferred'].includes(options.copy.mode)) {
throw new TypeError('options.copy.mode, when provided, must be \'forced\' or \'preferred\'.');
}
if (
options.copy.shiftTolerance !== undefined
&& (!isNumber(options.copy.shiftTolerance) || options.copy.shiftTolerance < 0)
) {
throw new TypeError('options.copy.shiftTolerance, when provided, must be a non-negative number.');
}
if (
options.copy.boundaryPolicy !== undefined
&& !['expand', 'shrink'].includes(options.copy.boundaryPolicy)
) {
throw new TypeError(
'options.copy.boundaryPolicy, when provided, must be \'expand\' or \'shrink\'.',
);
}
}
const composable = options.composable ?? false;
if (!composable) {
@@ -757,8 +847,8 @@ export class Conversion {
if (options.trim?.start !== undefined && (!Number.isFinite(options.trim.start))) {
throw new TypeError('options.trim.start, when provided, must be a finite number.');
}
if (options.trim?.end !== undefined && (!Number.isFinite(options.trim.end))) {
throw new TypeError('options.trim.end, when provided, must be a finite number.');
if (options.trim?.end !== undefined && (!isNumber(options.trim.end))) {
throw new TypeError('options.trim.end, when provided, must be a number.');
}
if (
options.trim?.start !== undefined
@@ -781,6 +871,9 @@ export class Conversion {
}
this._options = options;
this._copyMode = options.copy === false ? false : options.copy?.mode ?? 'preferred';
this._copyTimestampShiftTolerance = options.copy === false ? 0 : options.copy?.shiftTolerance ?? 0;
this._copyBoundaryPolicy = options.copy === false ? 'expand' : options.copy?.boundaryPolicy ?? 'expand';
this._composable = composable;
this.input = options.input;
this.output = options.output;
@@ -924,6 +1017,7 @@ export class Conversion {
}
this._endTimestamp = Math.max(this._options.trim?.end ?? Infinity, this._startTimestamp);
this._timestampOffset = -this._startTimestamp; // Initial value, may get refined later by track processing
// Run these sequentially so that output tracks have a deterministic order
for (let i = 0; i < filteredTracks.length; i++) {
@@ -1007,7 +1101,7 @@ export class Conversion {
const inputAndOutputFormatMatch = inputFormat.mimeType === this.output.format.mimeType;
const rawTagsAreUnchanged = inputTags.raw === outputTags.raw;
if (inputTags.raw && rawTagsAreUnchanged && !inputAndOutputFormatMatch) {
if (rawTagsAreUnchanged && !inputAndOutputFormatMatch) {
// If the input and output formats aren't the same, copying over raw metadata tags makes no sense and
// only results in junk tags, so let's cut them out.
delete outputTags.raw;
@@ -1156,15 +1250,15 @@ export class Conversion {
);
}
if (this.state === 'executing') {
if (this._state === 'executing') {
throw new Error('Cannot call execute() while a previous call to execute() is still running.');
}
if (this.state === 'canceled') {
if (this._state === 'canceled') {
throw new ConversionCanceledError();
}
if (this.state === 'done') {
if (this._state === 'done') {
// The conversion already ran to completion, nothing left to do
return;
}
@@ -1176,12 +1270,12 @@ export class Conversion {
);
}
this.state = 'executing';
this._state = 'executing';
this._executionUntil = options.until ?? Infinity;
this._pauseRequested = options.pauseSignal?.aborted ?? false;
const onPause = () => {
if (this.state !== 'executing') {
if (this._state !== 'executing') {
return;
}
@@ -1224,6 +1318,9 @@ export class Conversion {
);
for (const id of this._outputTrackIds) {
// Used for progress calculation. We start these at 0 which is technically not always the first
// timestamp, but this is how we choose to model what "progress" means: it's how far we are done
// with the trim region.
this._maxTimestamps.set(id, 0);
}
@@ -1247,7 +1344,7 @@ export class Conversion {
try {
await Promise.all(this._trackPumps.map(x => x.resolvers.promise));
} catch (error) {
if ((this.state as Conversion['state']) !== 'canceled') {
if ((this._state as Conversion['_state']) !== 'canceled') {
// Make sure to cancel to stop other encoding processes and clean up resources
void this.cancel();
}
@@ -1257,12 +1354,12 @@ export class Conversion {
options.pauseSignal?.removeEventListener('abort', onPause);
}
if ((this.state as Conversion['state']) === 'canceled') {
if ((this._state as Conversion['_state']) === 'canceled') {
throw new ConversionCanceledError();
}
const isDone = this._trackPumps.every(x => x.done);
this.state = isDone ? 'done' : 'idle';
this._state = isDone ? 'done' : 'idle';
if (isDone) {
if (!this._composable) {
@@ -1281,16 +1378,16 @@ export class Conversion {
* Does nothing if the conversion is already complete.
*/
async cancel() {
if (this.state === 'done') {
if (this._state === 'done') {
return;
}
if (this.state === 'canceled') {
if (this._state === 'canceled') {
Logging._warn('Conversion already canceled.');
return;
}
this.state = 'canceled';
this._state = 'canceled';
// Wake all suspended track pumps so they can wind down
for (const pump of this._trackPumps) {
@@ -1355,11 +1452,11 @@ export class Conversion {
height = ceilToMultipleOfTwo(trackOptions.height);
}
const firstTimestamp = await track.getFirstTimestamp();
let videoCodecs = this.output.format.getSupportedVideoCodecs();
const alpha = trackOptions.alpha ?? 'discard';
const needsTranscode = !!trackOptions.forceTranscode
|| firstTimestamp < this._startTimestamp
let needsTranscode = !this._copyMode
|| !!trackOptions.forceTranscode
|| !!trackOptions.frameRate
|| trackOptions.keyFrameInterval !== undefined
|| trackOptions.process !== undefined
@@ -1376,7 +1473,92 @@ export class Conversion {
|| (totalRotation !== 0 && !canUseRotationMetadata)
|| !!crop;
const alpha = trackOptions.alpha ?? 'discard';
let copyStartPacket: EncodedPacket | null = null;
if (!needsTranscode) {
// Check if we can copy it
const sink = new EncodedPacketSink(track);
let startPacket = await sink.getKeyPacket(this._startTimestamp, { verifyKeyPackets: true })
?? await sink.getFirstKeyPacket({ verifyKeyPackets: true });
if (
startPacket
&& startPacket.timestamp < this._startTimestamp
&& startPacket.timestamp + startPacket.duration <= this._startTimestamp
&& this._copyBoundaryPolicy === 'shrink'
) {
startPacket = await sink.getNextKeyPacket(startPacket, { verifyKeyPackets: true });
}
copyStartPacket = startPacket;
if (startPacket) {
// This clamp mirrors the packet timestamp clamping the copy loop does. The reason this is valid is
// because in the shrink case, we've already proven that the packet (at least partially) overlaps the
// trim region.
const effectiveStartTimestamp = this._copyBoundaryPolicy === 'shrink'
? Math.max(startPacket.timestamp, this._startTimestamp)
: startPacket.timestamp;
if (!this.output.format.supportsTimestampedMediaData) {
// Wants zero
if (this._timestampOffsetAdjusted) {
// We've already adjusted, we can't adjust twice
const isValid = effectiveStartTimestamp + this._timestampOffset === 0;
if (!isValid) {
needsTranscode = true;
}
} else {
const correction = clamp(
this._startTimestamp - effectiveStartTimestamp,
-this._copyTimestampShiftTolerance,
this._copyTimestampShiftTolerance,
);
const shiftedStartTimestamp = effectiveStartTimestamp + correction;
const isValid = shiftedStartTimestamp === this._startTimestamp;
if (isValid) {
this._timestampOffset = -this._startTimestamp + correction;
this._timestampOffsetAdjusted = true;
} else {
needsTranscode = true;
}
}
} else if (
this.output.format.negativeTimestampSupport !== 'full'
&& effectiveStartTimestamp < this._startTimestamp
) {
const correction = Math.min(
this._startTimestamp - effectiveStartTimestamp,
this._copyTimestampShiftTolerance,
);
const shiftedStartTimestamp = effectiveStartTimestamp + correction;
const isValid = shiftedStartTimestamp >= this._startTimestamp
|| (
this.output.format.negativeTimestampSupport === 'prefer-non-negative'
&& this._copyMode === 'forced'
);
if (isValid) {
this._timestampOffset = Math.max(this._timestampOffset, -this._startTimestamp + correction);
} else {
needsTranscode = true;
}
}
}
}
if (needsTranscode && this._copyMode === 'forced') {
this.discardedTracks.push({
track,
reason: 'cannot_copy',
trackOptions,
});
return;
}
if (!needsTranscode) {
// Fast path, we can simply copy over the encoded packets
@@ -1389,22 +1571,66 @@ export class Conversion {
const decoderConfig = await track.getDecoderConfig();
const meta: EncodedVideoChunkMetadata = { decoderConfig: decoderConfig ?? undefined };
for await (const packet of sink.packets(undefined, undefined, { verifyKeyPackets: true })) {
if (this.state === 'canceled') {
// eslint-disable-next-line curly
if (copyStartPacket) for await (const packet of sink.packets(
copyStartPacket,
undefined,
{ verifyKeyPackets: true },
)) {
if (this._state === 'canceled') {
break;
}
if (packet.timestamp >= this._endTimestamp) {
break;
if (this._copyBoundaryPolicy === 'shrink') {
break;
} else {
// Due to B-frames, there might still be packets we care about later on. Do a short
// lookahead to find out if there are.
let current = packet;
let found = false;
const lookahead = 6; // Heuristic, but should be enough for most streams
for (let i = 0; i < lookahead; i++) {
const next = await sink.getNextPacket(current, { metadataOnly: true });
if (!next) {
break;
}
if (next.timestamp < this._endTimestamp) {
found = true;
break;
}
current = next;
}
if (!found) {
break;
}
}
}
let packetStartTimestamp = packet.timestamp;
let packetEndTimestamp = packet.timestamp + packet.duration;
if (this._copyBoundaryPolicy === 'shrink') {
packetStartTimestamp = Math.max(packetStartTimestamp, this._startTimestamp);
packetEndTimestamp = Math.min(packetEndTimestamp, this._endTimestamp);
packetEndTimestamp = Math.max(packetEndTimestamp, packetStartTimestamp); // Just in case
}
packetStartTimestamp += this._timestampOffset;
packetEndTimestamp += this._timestampOffset;
const modifiedPacket = packet.clone({
timestamp: packet.timestamp - this._startTimestamp,
timestamp: packetStartTimestamp,
duration: packetEndTimestamp - packetStartTimestamp,
sideData: alpha === 'discard'
? {} // Remove alpha side data
: packet.sideData,
});
assert(modifiedPacket.timestamp >= 0);
this._reportProgress(outputTrackId, modifiedPacket.timestamp + modifiedPacket.duration);
await source.add(modifiedPacket, meta);
@@ -1496,8 +1722,9 @@ export class Conversion {
await tempOutput.start();
// Let's just use the first sample to test
const sink = new VideoSampleSink(track);
using firstSample = await sink.getSample(firstTimestamp); // Let's just use the first sample
using firstSample = await sink.getSample(await track.getFirstTimestamp());
if (firstSample) {
try {
@@ -1550,12 +1777,20 @@ export class Conversion {
const sink = new VideoSampleSink(track);
for await (using sample of sink.samples(this._startTimestamp, this._endTimestamp)) {
if (this.state === 'canceled') {
if (this._state === 'canceled') {
break;
}
const adjustedSampleTimestamp = Math.max(sample.timestamp - this._startTimestamp, 0);
sample.setTimestamp(adjustedSampleTimestamp);
const clampedStartTimestamp = Math.max(this._startTimestamp, sample.timestamp);
const clampedEndTimestamp = Math.min(this._endTimestamp, sample.timestamp + sample.duration);
if (clampedStartTimestamp >= clampedEndTimestamp) {
// Wholly out of the trim region
continue;
}
sample.setTimestamp(clampedStartTimestamp + this._timestampOffset);
sample.setDuration(clampedEndTimestamp - clampedStartTimestamp);
this._reportProgress(outputTrackId, sample.timestamp + sample.duration);
await source.add(sample);
@@ -1583,12 +1818,17 @@ export class Conversion {
}
const videoTrackLanguageCode = await track.getLanguageCode();
const trackName = await track.getName();
const trackDisposition = await track.getDisposition();
this.output.addVideoTrack(videoSource, {
frameRate: trackOptions.frameRate,
// TODO: This condition can be removed when all demuxers properly homogenize to BCP47 in v2
languageCode: isIso639Dash2LanguageCode(videoTrackLanguageCode) ? videoTrackLanguageCode : undefined,
name: await track.getName() ?? undefined,
disposition: await track.getDisposition(),
languageCode: isIso639Dash2LanguageCode(videoTrackLanguageCode)
? videoTrackLanguageCode
: undefined,
name: trackName ?? undefined,
disposition: trackDisposition,
rotation: outputTrackRotation,
group: ownGroup ?? trackOptions.group,
});
@@ -1615,29 +1855,124 @@ export class Conversion {
const originalNumberOfChannels = await track.getNumberOfChannels();
const originalSampleRate = await track.getSampleRate();
const firstTimestamp = await track.getFirstTimestamp();
let numberOfChannels = trackOptions.numberOfChannels ?? originalNumberOfChannels;
let sampleRate = trackOptions.sampleRate ?? originalSampleRate;
const needsTrimming = firstTimestamp < this._startTimestamp;
let needsPadding = firstTimestamp > this._startTimestamp && !this.output.format.supportsTimestampedMediaData;
let audioCodecs = this.output.format.getSupportedAudioCodecs();
if (
!trackOptions.forceTranscode
&& !trackOptions.quality
let needsTranscode = !this._copyMode
|| !!trackOptions.forceTranscode
|| !!trackOptions.quality
// eslint-disable-next-line @typescript-eslint/no-deprecated
&& !trackOptions.bitrate
&& numberOfChannels === originalNumberOfChannels
&& sampleRate === originalSampleRate
&& !needsTrimming
&& !needsPadding
&& audioCodecs.includes(sourceCodec)
&& (!trackOptions.codec || trackOptions.codec === sourceCodec)
&& !trackOptions.process
&& trackOptions.sampleFormat === undefined
) {
|| !!trackOptions.bitrate
|| numberOfChannels !== originalNumberOfChannels
|| sampleRate !== originalSampleRate
|| !audioCodecs.includes(sourceCodec)
|| (!!trackOptions.codec && trackOptions.codec !== sourceCodec)
|| trackOptions.process !== undefined
|| trackOptions.sampleFormat !== undefined;
let copyStartPacket: EncodedPacket | null = null;
if (!needsTranscode) {
// Check if we can copy it
const sink = new EncodedPacketSink(track);
let startPacket = await sink.getKeyPacket(this._startTimestamp)
?? await sink.getFirstKeyPacket();
if (
startPacket
&& (
(
this._copyBoundaryPolicy === 'shrink'
&& startPacket.timestamp < this._startTimestamp
)
|| (
this._copyBoundaryPolicy === 'expand'
// Check if packet is wholly before the start
&& startPacket.timestamp + startPacket.duration <= this._startTimestamp
)
)
) {
startPacket = await sink.getNextKeyPacket(startPacket);
}
const hasDecoderWarmup = (NON_PCM_AUDIO_CODECS as readonly AudioCodec[]).includes(sourceCodec)
&& sourceCodec !== 'flac';
if (startPacket && this._copyBoundaryPolicy === 'expand' && hasDecoderWarmup) {
// Go one packet back
const previousPacket = await sink.getKeyPacket(
startPacket.timestamp - 1 / (await track.getTimeResolution()),
);
if (previousPacket) {
startPacket = previousPacket;
}
}
copyStartPacket = startPacket;
if (startPacket) {
if (!this.output.format.supportsTimestampedMediaData) {
// Wants zero
if (this._timestampOffsetAdjusted) {
// We've already adjusted, we can't adjust twice
const isValid = startPacket.timestamp + this._timestampOffset === 0;
if (!isValid) {
needsTranscode = true;
}
} else {
const correction = clamp(
this._startTimestamp - startPacket.timestamp,
-this._copyTimestampShiftTolerance,
this._copyTimestampShiftTolerance,
);
const shiftedStartTimestamp = startPacket.timestamp + correction;
const isValid = shiftedStartTimestamp === this._startTimestamp;
if (isValid) {
this._timestampOffset = -this._startTimestamp + correction;
this._timestampOffsetAdjusted = true;
} else {
needsTranscode = true;
}
}
} else if (
this.output.format.negativeTimestampSupport !== 'full'
&& startPacket.timestamp < this._startTimestamp
) {
const correction = Math.min(
this._startTimestamp - startPacket.timestamp,
this._copyTimestampShiftTolerance,
);
const shiftedStartTimestamp = startPacket.timestamp + correction;
const isValid = shiftedStartTimestamp >= this._startTimestamp
|| (
this.output.format.negativeTimestampSupport === 'prefer-non-negative'
&& this._copyMode === 'forced'
);
if (isValid) {
this._timestampOffset = Math.max(this._timestampOffset, -this._startTimestamp + correction);
} else {
needsTranscode = true;
}
}
}
}
if (needsTranscode && this._copyMode === 'forced') {
this.discardedTracks.push({
track,
reason: 'cannot_copy',
trackOptions,
});
return;
}
if (!needsTranscode) {
// Fast path, we can simply copy over the encoded packets
const source = new EncodedAudioPacketSource(sourceCodec);
@@ -1648,19 +1983,26 @@ export class Conversion {
const decoderConfig = await track.getDecoderConfig();
const meta: EncodedAudioChunkMetadata = { decoderConfig: decoderConfig ?? undefined };
for await (const packet of sink.packets()) {
if (this.state === 'canceled') {
// eslint-disable-next-line curly
if (copyStartPacket) for await (const packet of sink.packets(copyStartPacket)) {
if (this._state === 'canceled') {
break;
}
if (packet.timestamp >= this._endTimestamp) {
break;
}
if (
this._copyBoundaryPolicy === 'shrink'
&& packet.timestamp + packet.duration > this._endTimestamp
) {
break;
}
const modifiedPacket = packet.clone({
timestamp: packet.timestamp - this._startTimestamp,
timestamp: packet.timestamp + this._timestampOffset,
duration: packet.duration,
});
assert(modifiedPacket.timestamp >= 0);
this._reportProgress(outputTrackId, modifiedPacket.timestamp + modifiedPacket.duration);
await source.add(modifiedPacket, meta);
@@ -1770,15 +2112,58 @@ export class Conversion {
audioSource = source;
this._registerTrackPump(async (pump) => {
let needsPadding: boolean | null = null;
const sink = new AudioSampleSink(track);
for await (using sample of sink.samples(this._startTimestamp, this._endTimestamp)) {
if (this.state === 'canceled') {
if (this._state === 'canceled') {
break;
}
let startFrame = 0;
let endFrame = sample.numberOfFrames;
if (sample.timestamp < this._startTimestamp) {
startFrame = Math.round((this._startTimestamp - sample.timestamp) * sample.sampleRate);
}
if (sample.timestamp + sample.duration > this._endTimestamp) {
endFrame = Math.round((this._endTimestamp - sample.timestamp) * sample.sampleRate);
}
if (startFrame >= endFrame) {
// Sample lies wholly out of trim region
sample.close();
continue; // No break since we may be before the start
}
// Can't assign to "using" identifiers so we gotta do this
let finalSampleLet: AudioSample;
if (startFrame > 0 || endFrame < sample.numberOfFrames) {
// Trim the sample if it sticks out of the trim region on either end
const trimmedSample = sample.trim(startFrame, endFrame);
sample.close();
finalSampleLet = trimmedSample;
if (trimmedSample.numberOfFrames === 0) {
trimmedSample.close();
continue;
}
} else {
finalSampleLet = sample;
}
using finalSample = finalSampleLet;
// Offset the timestamp as needed
finalSample.setTimestamp(finalSample.timestamp + this._timestampOffset);
if (needsPadding === null) {
needsPadding = finalSample.timestamp > 0 && !this.output.format.supportsTimestampedMediaData;
}
if (needsPadding) {
// Add one padding sample at the beginning
const paddingLength = firstTimestamp - this._startTimestamp;
const paddingLength = finalSample.timestamp;
const paddingLengthSamples = Math.round(paddingLength * originalSampleRate);
const bytesPerSample = getBytesPerSample(sample.format);
@@ -1803,37 +2188,6 @@ export class Conversion {
needsPadding = false;
}
let startFrame = 0;
let endFrame = sample.numberOfFrames;
if (sample.timestamp < this._startTimestamp) {
startFrame = Math.round((this._startTimestamp - sample.timestamp) * sample.sampleRate);
}
if (sample.timestamp + sample.duration > this._endTimestamp) {
endFrame = Math.round((this._endTimestamp - sample.timestamp) * sample.sampleRate);
}
// Can't assign to "using" identifiers so we gotta do this
let finalSampleLet: AudioSample;
if (startFrame > 0 || endFrame < sample.numberOfFrames) {
// Trim the sample if it sticks out of the trim region on either end
const trimmedSample = sample.trim(startFrame, endFrame);
sample.close();
finalSampleLet = trimmedSample;
if (trimmedSample.numberOfFrames === 0) {
trimmedSample.close();
continue;
}
} else {
finalSampleLet = sample;
}
using finalSample = finalSampleLet;
// Offset the timestamp as needed
finalSample.setTimestamp(finalSample.timestamp - this._startTimestamp);
await this._registerAudioSample(
pump, finalSample, source, outputTrackId, () => lastSampleTimestamp,
);
@@ -1852,11 +2206,16 @@ export class Conversion {
}
const audioTrackLanguageCode = await track.getLanguageCode();
const trackName = await track.getName();
const trackDisposition = await track.getDisposition();
this.output.addAudioTrack(audioSource, {
// TODO: This condition can be removed when all demuxers properly homogenize to BCP47 in v2
languageCode: isIso639Dash2LanguageCode(audioTrackLanguageCode) ? audioTrackLanguageCode : undefined,
name: await track.getName() ?? undefined,
disposition: await track.getDisposition(),
languageCode: isIso639Dash2LanguageCode(audioTrackLanguageCode)
? audioTrackLanguageCode
: undefined,
name: trackName ?? undefined,
disposition: trackDisposition,
group: ownGroup ?? trackOptions.group,
});
@@ -1909,7 +2268,7 @@ export class Conversion {
/** @internal */
async _checkpoint(pump: TrackPump, timestamp: number) {
while (this.state !== 'canceled' && (timestamp >= this._executionUntil || this._pauseRequested)) {
while (this._state !== 'canceled' && (timestamp >= this._executionUntil || this._pauseRequested)) {
// We've reached the target; signal it and suspend until the next execution wakes us up
pump.resolvers.resolve();
@@ -1975,7 +2334,9 @@ class TrackSynchronizer {
}
declareTrack(trackId: number) {
this.maxTimestamps.set(trackId, 0);
// Using -Infinity will automatically cause all tracks to wait for each other at the start until they figure out
// the true min timestamp
this.maxTimestamps.set(trackId, -Infinity);
}
shouldWait(trackId: number, timestamp: number) {
@@ -1986,7 +2347,7 @@ class TrackSynchronizer {
const newMin = this.computeMinAndMaybeResolve();
if (
this.conversion.state === 'canceled'
this.conversion._state === 'canceled'
|| this.conversion._pauseRequested
|| timestamp >= this.conversion._executionUntil
) {
+2
View File
@@ -73,6 +73,7 @@ export {
AudioSource,
SubtitleSource,
AudioBufferSource,
type AudioBufferSourceOptions,
AudioSampleSource,
CanvasSource,
EncodedAudioPacketSource,
@@ -298,6 +299,7 @@ export {
type ConversionOptions,
type ConversionVideoOptions,
type ConversionAudioOptions,
type ConversionCopyOptions,
type ConversionExecuteOptions,
ConversionCanceledError,
type DiscardedTrack,
+72 -28
View File
@@ -9,6 +9,7 @@
import {
toUint8Array,
assert,
isI32,
isU32,
last,
TransformationMatrix,
@@ -356,7 +357,8 @@ export const styp = () => box('styp', [
/** Segment Index Box */
export const sidx = (muxer: IsobmffMuxer, referencedSize: number) => {
let duration = muxer.maxWrittenEndTimestamp - muxer.minWrittenTimestamp;
const earliestPresentationTime = Math.max(0, muxer.minWrittenTimestamp);
let duration = Math.max(0, muxer.maxWrittenEndTimestamp - earliestPresentationTime);
if (!Number.isFinite(duration)) {
duration = 0;
}
@@ -364,7 +366,7 @@ export const sidx = (muxer: IsobmffMuxer, referencedSize: number) => {
return fullBox('sidx', 1, 0, [
u32(1), // Reference ID
u32(GLOBAL_TIMESCALE), // Timescale
u64(intoTimescale(muxer.minWrittenTimestamp, GLOBAL_TIMESCALE)), // Earliest presentation time
u64(intoTimescale(earliestPresentationTime, GLOBAL_TIMESCALE)), // Earliest presentation time
u64(0), // First offset
u16(0), // Reserved
u16(1), // Reference count
@@ -402,8 +404,12 @@ export const mvhd = (
0,
...trackDatas
.map(trackData => (
intoTimescale(presentationSpan(trackData), GLOBAL_TIMESCALE)
+ intoTimescale(trackData.startTimestampOffset ?? 0, GLOBAL_TIMESCALE)
// Round separately to match the edit list
Math.max(
0,
intoTimescale(presentationSpan(trackData), GLOBAL_TIMESCALE)
+ intoTimescale(trackData.startTimestampOffset ?? 0, GLOBAL_TIMESCALE),
)
)),
);
const nextTrackId = Math.max(0, ...trackDatas.map(x => x.track.id)) + 1;
@@ -459,11 +465,11 @@ const presentationSpan = (trackData: IsobmffTrackData) => {
*/
export const trak = (trackData: IsobmffTrackData, creationTime: number) => {
const trackMetadata = getTrackMetadata(trackData);
const needsEditList = trackData.startTimestampOffset !== null && trackData.startTimestampOffset > 0;
const needsEditList = trackData.startTimestampOffset !== null && trackData.startTimestampOffset !== 0;
return box('trak', undefined, [
tkhd(trackData, creationTime),
needsEditList ? edts(trackData, trackData.startTimestampOffset!) : null,
needsEditList ? edts(trackData) : null,
mdia(trackData, creationTime),
trackMetadata.name !== undefined
? box('udta', undefined, [
@@ -480,8 +486,12 @@ export const tkhd = (
trackData: IsobmffTrackData,
creationTime: number,
) => {
const durationInGlobalTimescale = intoTimescale(presentationSpan(trackData), GLOBAL_TIMESCALE)
+ intoTimescale(trackData.startTimestampOffset ?? 0, GLOBAL_TIMESCALE);
// Round separately to match the edit list
const durationInGlobalTimescale = Math.max(
0,
intoTimescale(presentationSpan(trackData), GLOBAL_TIMESCALE)
+ intoTimescale(trackData.startTimestampOffset ?? 0, GLOBAL_TIMESCALE),
);
const needsU64 = !isU32(creationTime) || !isU32(durationInGlobalTimescale);
const u32OrU64 = needsU64 ? u64 : u32;
@@ -528,29 +538,63 @@ export const tkhd = (
};
/** Edit Box: Specifies edits to the track's media. */
export const edts = (trackData: IsobmffTrackData, offset: number) => {
const startOffset = intoTimescale(offset, GLOBAL_TIMESCALE);
const mediaDuration = intoTimescale(presentationSpan(trackData), GLOBAL_TIMESCALE);
export const edts = (trackData: IsobmffTrackData) => {
const offset = trackData.startTimestampOffset;
assert(offset !== null);
const needs64Bits = !isU32(startOffset) || !isU32(mediaDuration);
const u32OrU64 = needs64Bits ? u64 : u32;
const i32OrI64 = needs64Bits ? i64 : i32;
if (offset > 0) {
// Positive offset: empty segment at the start, then the full media afterwards
return box('edts', undefined, [
fullBox('elst', needs64Bits ? 1 : 0, 0, [
u32(2), // Entry count
const startOffset = intoTimescale(offset, GLOBAL_TIMESCALE);
const mediaDuration = intoTimescale(presentationSpan(trackData), GLOBAL_TIMESCALE);
// #1
u32OrU64(startOffset), // Segment duration
i32OrI64(-1), // Media time
fixed_16_16(1), // Media rate
const needs64Bits = !isU32(startOffset) || !isU32(mediaDuration);
const u32OrU64 = needs64Bits ? u64 : u32;
const i32OrI64 = needs64Bits ? i64 : i32;
// #2
u32OrU64(mediaDuration), // Segment duration
i32OrI64(0), // Media time
fixed_16_16(1), // Media rate
]),
]);
return box('edts', undefined, [
fullBox('elst', needs64Bits ? 1 : 0, 0, [
u32(2), // Entry count
// #1
u32OrU64(startOffset), // Segment duration
i32OrI64(-1), // Media time
fixed_16_16(1), // Media rate
// #2
u32OrU64(mediaDuration), // Segment duration
i32OrI64(0), // Media time
fixed_16_16(1), // Media rate
]),
]);
} else {
// Negative offset: the negative section of the media is trimmed off
const mediaTime = intoTimescale(-offset, trackData.timescale);
// Not the entire media is visible.
// For fragmented files, this value is zero, which simply means "unknown duration" in this case. Spec:
// "the segment_duration of this edit may be zero"
const mediaDuration = Math.max(
0,
intoTimescale(presentationSpan(trackData), GLOBAL_TIMESCALE)
+ intoTimescale(offset, GLOBAL_TIMESCALE),
);
const needs64Bits = !isI32(mediaTime) || !isU32(mediaDuration);
const u32OrU64 = needs64Bits ? u64 : u32;
const i32OrI64 = needs64Bits ? i64 : i32;
return box('edts', undefined, [
fullBox('elst', needs64Bits ? 1 : 0, 0, [
u32(1), // Entry count
// #1
u32OrU64(mediaDuration), // Segment duration
i32OrI64(mediaTime), // Media time
fixed_16_16(1), // Media rate
]),
]);
}
};
/** Media Box: Describes and define a track's media type and sample data. */
@@ -565,7 +609,7 @@ export const mdhd = (
trackData: IsobmffTrackData,
creationTime: number,
) => {
// Since the duration represents the raw media duration, edit list offsets are not taken into account here
// Since _this_ duration represents the raw media duration, edit list offsets are not taken into account here
const localDuration = intoTimescale(
presentationSpan(trackData),
trackData.timescale,
+1 -7
View File
@@ -953,11 +953,6 @@ export class IsobmffDemuxer extends Demuxer {
: readI32Be(slice);
const mediaRate = readFixed_16_16(slice);
if (segmentDuration === 0) {
// Don't care
continue;
}
if (relevantEntryFound) {
Logging._warn(
'Unsupported edit list: multiple edits are not currently supported. Only using first edit.',
@@ -1558,10 +1553,9 @@ export class IsobmffDemuxer extends Demuxer {
case 'esds': {
const track = this.currentTrack;
if (!track) {
if (!track || track.info?.type !== 'audio') {
break;
}
assert(track.info?.type === 'audio');
slice.skip(4); // Version + flags
+22 -12
View File
@@ -139,7 +139,7 @@ export type IsobmffTrackData = {
info: {
config: SubtitleConfig;
};
lastCueEndTimestamp: number;
lastCueEndTimestamp: number | null;
cueQueue: SubtitleCue[];
nextSourceId: number;
cueToSourceId: WeakMap<SubtitleCue, number>;
@@ -613,7 +613,7 @@ export class IsobmffMuxer extends Muxer {
compactlyCodedChunkTable: [],
closed: false,
lastCueEndTimestamp: 0,
lastCueEndTimestamp: null,
cueQueue: [],
nextSourceId: 0,
cueToSourceId: new WeakMap(),
@@ -790,6 +790,8 @@ export class IsobmffMuxer extends Muxer {
// overlapping samples require special logic. The algorithm produces the format specified in ISO 14496-30.
while (trackData.cueQueue.length > 0) {
trackData.lastCueEndTimestamp ??= Math.min(0, trackData.cueQueue[0]!.timestamp);
const timestamps = new Set<number>([]);
for (const cue of trackData.cueQueue) {
assert(cue.timestamp <= until);
@@ -904,10 +906,10 @@ export class IsobmffMuxer extends Muxer {
}
if (trackData.type === 'audio' && trackData.info.requiresPcmTransformation) {
if (!this.isFragmented) {
// The first timestamp is the lowest
trackData.startTimestampOffset ??= trackData.timestampProcessingQueue[0]!.timestamp;
}
assert(!this.isFragmented);
// The first timestamp is the lowest
trackData.startTimestampOffset ??= trackData.timestampProcessingQueue[0]!.timestamp;
let totalDuration = 0;
@@ -936,7 +938,9 @@ export class IsobmffMuxer extends Muxer {
const sortedTimestamps = trackData.timestampProcessingQueue.map(x => x.timestamp).sort((a, b) => a - b);
if (!this.isFragmented) {
if (this.isFragmented) {
trackData.startTimestampOffset ??= Math.min(sortedTimestamps[0]!, 0);
} else {
trackData.startTimestampOffset ??= sortedTimestamps[0]!;
}
@@ -1292,16 +1296,22 @@ export class IsobmffMuxer extends Muxer {
let fragmentStartTimestamp = Infinity;
for (let i = 0; i < tracksInFragment.length; i++) {
const trackData = tracksInFragment[i]!;
assert(trackData.currentChunk);
assert(trackData.startTimestampOffset !== null);
trackData.currentChunk!.offset = currentPos;
trackData.currentChunk!.moofOffset = moofOffset;
trackData.currentChunk!.trafIndex = i;
trackData.currentChunk.offset = currentPos;
trackData.currentChunk.moofOffset = moofOffset;
trackData.currentChunk.trafIndex = i;
trackData.currentChunk.startTimestamp -= trackData.startTimestampOffset;
for (const sample of trackData.currentChunk!.samples) {
for (const sample of trackData.currentChunk.samples) {
currentPos += sample.size;
sample.timestamp -= trackData.startTimestampOffset;
sample.decodeTimestamp -= trackData.startTimestampOffset;
}
fragmentStartTimestamp = Math.min(fragmentStartTimestamp, trackData.currentChunk!.startTimestamp);
fragmentStartTimestamp = Math.min(fragmentStartTimestamp, trackData.currentChunk.startTimestamp);
}
const mdatSize = currentPos - mdatStartPos;
+24 -11
View File
@@ -1934,6 +1934,29 @@ export class MatroskaDemuxer extends Demuxer {
}
}
}
async getDurationFromMetadata(segment: Segment) {
if (segment.duration <= 0) {
return null;
}
// The kosher definition of the Duration field is "latest end time - earliest start time" across all tracks in
// the segment; since we currently mean "end timestamp" with "duration", we need to determine the earliest
// start time before we can return a value here.
let minTimestamp: number | null = null;
for (const track of segment.tracks) {
assert(track.trackBacking);
const firstPacket = await track.trackBacking.getFirstPacket({ metadataOnly: true });
if (firstPacket) {
minTimestamp = Math.min(minTimestamp ?? Infinity, firstPacket.timestamp);
}
}
let endTimestamp = segment.duration / segment.timestampFactor;
endTimestamp += minTimestamp ?? 0;
return endTimestamp;
}
}
abstract class MatroskaTrackBacking implements InputTrackBacking {
@@ -2016,17 +2039,7 @@ abstract class MatroskaTrackBacking implements InputTrackBacking {
}
async getDurationFromMetadata() {
const segment = this.internalTrack.segment;
if (segment.duration <= 0) {
return null;
}
let endTimestamp = segment.duration / segment.timestampFactor;
const firstPacket = await this.getFirstPacket({ metadataOnly: true });
endTimestamp += firstPacket?.timestamp ?? 0;
return endTimestamp;
return this.internalTrack.demuxer.getDurationFromMetadata(this.internalTrack.segment);
}
async getLiveRefreshInterval() {
+14 -1
View File
@@ -69,6 +69,7 @@ import { Writer } from '../writer';
import { EncodedPacket } from '../packet';
import { parseOpusIdentificationHeader } from '../codec-data';
import { AttachedFile } from '../metadata';
import { Logging } from '../logging';
const MIN_CLUSTER_TIMESTAMP_MS = -(2 ** 15);
const MAX_CLUSTER_TIMESTAMP_MS = 2 ** 15 - 1;
@@ -157,6 +158,7 @@ export class MatroskaMuxer extends Muxer {
private startTimestamp = Infinity;
private endTimestamp = -Infinity;
private warnedAboutTooNegativeTimestamp = false;
constructor(output: Output, format: MkvOutputFormat) {
super(output);
@@ -1191,6 +1193,15 @@ export class MatroskaMuxer extends Muxer {
const relativeTimestamp = msTimestamp - this.currentClusterStartMsTimestamp!;
if (relativeTimestamp < MIN_CLUSTER_TIMESTAMP_MS) {
// The block lies too far in the past, it's not representable within this cluster
if (!this.warnedAboutTooNegativeTimestamp) {
const formatName = this.format instanceof WebMOutputFormat ? 'WebM' : 'Matroska';
Logging._warn(
`Packets had to be discarded because their timestamp is too negative to represent in`
+ ` ${formatName}.`,
);
this.warnedAboutTooNegativeTimestamp = true;
}
return;
}
@@ -1253,6 +1264,8 @@ export class MatroskaMuxer extends Muxer {
/** Creates a new Cluster element to contain media chunks. */
private createNewCluster(msTimestamp: number) {
msTimestamp = Math.max(0, msTimestamp); // Cluster timestamps cannot be negative
if (this.currentCluster) {
this.finalizeCurrentCluster();
}
@@ -1313,7 +1326,7 @@ export class MatroskaMuxer extends Muxer {
for (const [msTimestamp, trackDatas] of groupedAndSortedByTimestamp) {
assert(this.cues);
(this.cues.data as EBML[]).push({ id: EBMLId.CuePoint, data: [
{ id: EBMLId.CueTime, data: msTimestamp },
{ id: EBMLId.CueTime, data: Math.max(0, msTimestamp) }, // CueTime is unsigned
// Create CueTrackPositions for each track that starts at this timestamp
...trackDatas.map((trackData) => {
return { id: EBMLId.CueTrackPositions, data: [
+51 -8
View File
@@ -8,6 +8,7 @@
import { parsePcmCodec, PCM_AUDIO_CODECS, PcmAudioCodec, VideoCodec, AudioCodec } from './codec';
import {
addAvcBitstreamRestriction,
AvcNalUnitType,
concatAvcNalUnits,
deserializeAvcDecoderConfigurationRecord,
@@ -19,6 +20,7 @@ import {
iterateHevcNalUnits,
parseAvcSps,
sanitizeHevcPacketForChromium,
serializeAvcDecoderConfigurationRecord,
} from './codec-data';
import { CustomVideoDecoder, customVideoDecoders, CustomAudioDecoder, customAudioDecoders } from './custom-coder';
import { InputDisposedError } from './input';
@@ -601,11 +603,18 @@ export abstract class BaseMediaSampleSink<
async next() {
while (true) {
if (track.input._disposed) {
// Once next() throws, the consumer will never call return(), so terminate the
// iteration here - otherwise, the pump keeps queueing decoded samples that
// nothing can ever close.
terminated = true;
ended = true;
closeSamples();
throw new InputDisposedError();
} else if (terminated) {
return { value: undefined, done: true };
} else if (hasOutOfBandError) {
terminated = true;
ended = true;
closeSamples();
throw outOfBandError;
} else if (sampleQueue.length > 0) {
@@ -827,11 +836,16 @@ export abstract class BaseMediaSampleSink<
async next() {
while (true) {
if (track.input._disposed) {
// Once next() throws, the consumer will never call return(), so terminate the
// iteration here - otherwise, the pump keeps queueing decoded samples that
// nothing can ever close.
terminated = true;
closeSamples();
throw new InputDisposedError();
} else if (terminated) {
return { value: undefined, done: true };
} else if (hasOutOfBandError) {
terminated = true;
closeSamples();
throw outOfBandError;
} else if (sampleQueue.length > 0) {
@@ -947,19 +961,31 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
if (isChromium()) {
if (codec === 'avc' && this.decoderConfig.description) {
// Chromium has/had a bug with playing interlaced AVC (https://issues.chromium.org/issues/456919096)
// which can be worked around by requesting that software decoding be used. So, here we peek into
// the AVC description, if present, and switch to software decoding if we find interlaced content.
const record = deserializeAvcDecoderConfigurationRecord(
toUint8Array(this.decoderConfig.description),
);
if (record && record.sequenceParameterSets.length > 0) {
const sps = parseAvcSps(record.sequenceParameterSets[0]!);
if (sps && sps.frameMbsOnlyFlag === 0) {
this.decoderConfig = {
...this.decoderConfig,
hardwareAcceleration: 'prefer-software',
};
if (sps) {
if (sps.frameMbsOnlyFlag === 0) {
// Chromium has/had a bug with playing interlaced AVC
// (https://issues.chromium.org/issues/456919096) which can be worked around by
// requesting that software decoding be used. So, here we peek into the AVC description,
// if present, and switch to software decoding if we find interlaced content.
this.decoderConfig = {
...this.decoderConfig,
hardwareAcceleration: 'prefer-software',
};
}
if (sps.maxDecFrameBuffering !== 0 && sps.bitstreamRestrictionFlag !== 1) {
// Modify the SPS to fix potential loss of B frames
record.sequenceParameterSets[0] = addAvcBitstreamRestriction(sps);
this.decoderConfig = {
...this.decoderConfig,
description: serializeAvcDecoderConfigurationRecord(record),
};
}
}
}
}
@@ -1065,6 +1091,23 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
}
}
if (!this.decoderConfig.description) {
// Do SPS fixups if necessary
for (let i = 0; i < filteredNalUnits.length; i++) {
const nalUnit = filteredNalUnits[i]!;
if (extractNalUnitTypeForAvc(nalUnit[0]!) !== AvcNalUnitType.SPS) {
continue;
}
const sps = parseAvcSps(nalUnit);
if (sps && sps.maxDecFrameBuffering !== 0 && sps.bitstreamRestrictionFlag !== 1) {
filteredNalUnits[i] = addAvcBitstreamRestriction(sps);
}
break;
}
}
const newData = concatAvcNalUnits(filteredNalUnits, this.decoderConfig);
packet = new EncodedPacket(newData, packet.type, packet.timestamp, packet.duration);
} else if (this.codec === 'hevc') {
+28 -8
View File
@@ -1340,8 +1340,8 @@ export class CanvasSource extends VideoSource {
* to respect writer and encoder backpressure.
*/
add(timestamp: number, duration = 0, encodeOptions?: VideoEncoderEncodeOptions) {
if (!Number.isFinite(timestamp) || timestamp < 0) {
throw new TypeError('timestamp must be a non-negative number.');
if (!Number.isFinite(timestamp)) {
throw new TypeError('timestamp must be a finite number.');
}
if (!Number.isFinite(duration) || duration < 0) {
throw new TypeError('duration must be a non-negative number.');
@@ -2486,6 +2486,19 @@ export class AudioSampleSource extends AudioSource {
}
}
/**
* Options for {@link AudioBufferSource}.
* @group Media sources
* @public
*/
export type AudioBufferSourceOptions = {
/**
* The timestamp of the first `AudioBuffer`, in seconds. Subsequent buffers are placed directly after the previous
* one. Defaults to 0.
*/
startTimestamp?: number;
};
/**
* This source can be used to add audio data from an AudioBuffer to the output track. This is useful when working with
* the Web Audio API.
@@ -2496,23 +2509,30 @@ export class AudioBufferSource extends AudioSource {
/** @internal */
private _encoder: AudioEncoderWrapper;
/** @internal */
private _accumulatedTime = 0;
private _accumulatedTime: number;
/**
* Creates a new {@link AudioBufferSource} whose `AudioBuffer` instances are encoded according to the specified
* {@link AudioEncodingConfig}.
* {@link AudioEncodingConfig} and {@link AudioBufferSourceOptions}.
*/
constructor(encodingConfig: AudioEncodingConfig) {
constructor(encodingConfig: AudioEncodingConfig, options: AudioBufferSourceOptions = {}) {
validateAudioEncodingConfig(encodingConfig);
if (typeof options !== 'object' || !options) {
throw new TypeError('options must be an object.');
}
if (options.startTimestamp !== undefined && !Number.isFinite(options.startTimestamp)) {
throw new TypeError('options.startTimestamp, when provided, must be a finite number.');
}
super(encodingConfig.codec);
this._encoder = new AudioEncoderWrapper(this, encodingConfig);
this._accumulatedTime = options.startTimestamp ?? 0;
}
/**
* Converts an AudioBuffer to audio samples, encodes them and adds them to the output. The first AudioBuffer will
* be played at timestamp 0, and any subsequent AudioBuffer will have a timestamp equal to the total duration of
* all previous AudioBuffers.
* Converts an AudioBuffer to audio samples, encodes them and adds them to the output. The first `AudioBuffer` will
* be played at the configured start timestamp (the default is 0), and each subsequent `AudioBuffer` will be placed
* directly after the previous one.
*
* @returns A Promise that resolves once the output is ready to receive more samples. You should await this Promise
* to respect writer and encoder backpressure.
+8 -5
View File
@@ -76,14 +76,16 @@ export type MetadataTags = {
* user-defined text frames are exposed as a `Record<string, string>`.
* - ADTS: The ID3v2 tags, just like in MP3.
* - Ogg: The key-value string pairs from the Vorbis-style comment header (see RFC 7845, Section 5.2).
* Additionally, the `'vendor'` key refers to the vendor string within this header.
* Additionally, the `'vendor'` key refers to the vendor string within this header. If a key exists more than once,
* a string array is used instead.
* - WAVE: The individual metadata chunks within the RIFF INFO chunk. Values are always ISO 8859-1 strings.
* - FLAC: The key-value string pairs from the vorbis metadata block (see RFC 9639, Section D.2.3).
* Additionally, the `'vendor'` key refers to the vendor string within this header. If ID3v2 tags appear at the
* start of the file, their content is stored just like for MP3.
* Additionally, the `'vendor'` key refers to the vendor string within this header. If a key exists more than once,
* a string array is used instead. If ID3v2 tags appear at the start of the file, their content is stored just like
* for MP3.
* - MPEG-TS: Not supported.
*/
raw?: Record<string, string | Uint8Array | RichImageData | AttachedFile | Record<string, string> | null>;
raw?: Record<string, string | string[] | Uint8Array | RichImageData | AttachedFile | Record<string, string> | null>;
};
/**
@@ -238,13 +240,14 @@ export const validateMetadataTags = (tags: MetadataTags) => {
if (
value !== null
&& typeof value !== 'string'
&& !(Array.isArray(value) && value.every(x => typeof x === 'string'))
&& !(value instanceof Uint8Array)
&& !(value instanceof RichImageData)
&& !(value instanceof AttachedFile)
&& !isRecordStringString(value)
) {
throw new TypeError(
'Each value in tags.raw must be a string, Uint8Array, RichImageData, AttachedFile, '
'Each value in tags.raw must be a string, string array, Uint8Array, RichImageData, AttachedFile, '
+ 'Record<string, string>, or null.',
);
}
+16
View File
@@ -42,6 +42,10 @@ export const isU32 = (value: number) => {
return value >= 0 && value < 2 ** 32;
};
export const isI32 = (value: number) => {
return value >= -(2 ** 31) && value < 2 ** 31;
};
/** Reads an exponential-Golomb universal code from a Bitstream. */
export const readExpGolomb = (bitstream: Bitstream) => {
let leadingZeroBits = 0;
@@ -57,6 +61,14 @@ export const readExpGolomb = (bitstream: Bitstream) => {
return result;
};
export const writeExpGolomb = (bitstream: Bitstream, value: number) => {
const codeNum = value + 1;
const leadingZeroBits = Math.floor(Math.log2(codeNum));
bitstream.writeBits(leadingZeroBits, 0);
bitstream.writeBits(1, 1);
bitstream.writeBits(leadingZeroBits, codeNum - 2 ** leadingZeroBits);
};
/** Reads a signed exponential-Golomb universal code from a Bitstream. */
export const readSignedExpGolomb = (bitstream: Bitstream) => {
const codeNum = readExpGolomb(bitstream);
@@ -454,6 +466,10 @@ export const lerp = (from: number, to: number, t: number) => {
return from + (to - from) * t;
};
export const modEuclid = (value: number, modulus: number) => {
return value - Math.floor(value / modulus) * modulus;
};
export const UNDETERMINED_LANGUAGE = 'und';
export const roundIfAlmostInteger = (value: number) => {
+52 -14
View File
@@ -80,7 +80,13 @@ import {
} from '../../shared/mp3-misc';
import { EncodedPacket, PacketType, PLACEHOLDER_DATA } from '../packet';
import { FileSlice, readBytes, Reader, readU16Be, readU32Be, readU8 } from '../reader';
import { buildMpegTsMimeType, MpegTsStreamType, TIMESCALE, TS_PACKET_SIZE } from './mpeg-ts-misc';
import {
buildMpegTsMimeType,
MpegTsStreamType,
TIMESCALE,
TIMESTAMP_MODULUS,
TS_PACKET_SIZE,
} from './mpeg-ts-misc';
import { AC3_SAMPLE_RATES } from '../../shared/ac3-misc';
import { Bitstream } from '../../shared/bitstream';
@@ -180,6 +186,10 @@ export class MpegTsDemuxer extends Demuxer {
sectionEndPositions: number[] = [];
seekChunkSize = 5 * 1024 * 1024; // 5 MiB, picked because most HLS segments are below this size
minReferencePointByteDistance = -1;
timestampWrapInfo: {
reference: number;
offset: number;
} | null = null;
constructor(input: Input) {
super(input);
@@ -508,7 +518,7 @@ export class MpegTsDemuxer extends Demuxer {
const elementaryStream = this.elementaryStreams.find(x => x.pid === section.pid);
outer:
if (elementaryStream && !elementaryStream.initialized) {
const pesPacket = readPesPacket(section, true);
const pesPacket = readPesPacket(this, section, true);
if (!pesPacket) {
throw new Error(
`Couldn't read first PES packet for Elementary Stream with PID ${elementaryStream.pid}`,
@@ -1023,6 +1033,30 @@ export class MpegTsDemuxer extends Demuxer {
body: bytes.subarray(4),
};
}
normalizeTimestamp(timestamp: number) {
if (!this.timestampWrapInfo) {
// No info yet, let's initialize it anchored on this timestamp. Since we process packets sequentially when
// extracting metadata, this means we'll anchor to the first PTS in the file.
const tolerance = 60 * TIMESCALE;
this.timestampWrapInfo = {
reference: timestamp - tolerance,
offset: timestamp >= TIMESTAMP_MODULUS - tolerance
? -TIMESTAMP_MODULUS // Timestamps close to the modulus are treated as negative
: TIMESTAMP_MODULUS,
};
}
const { reference, offset } = this.timestampWrapInfo;
if (
(offset < 0 && timestamp >= reference)
|| (offset > 0 && timestamp < reference)
) {
return timestamp + offset;
}
return timestamp;
}
}
type PesPacketHeader = {
@@ -1045,6 +1079,7 @@ type TimestampedPesPacket = PesPacket & {
};
const readPesPacketHeader = <T extends boolean>(
demuxer: MpegTsDemuxer,
section: Section,
expectPts: T,
): (T extends true ? TimestampedPesPacketHeader : PesPacketHeader) | null => {
@@ -1091,6 +1126,8 @@ const readPesPacketHeader = <T extends boolean>(
pts += bitstream.readBits(15) * (1 << 15);
bitstream.skipBits(1);
pts += bitstream.readBits(15);
pts = demuxer.normalizeTimestamp(pts);
} else {
if (expectPts) {
throw new Error(MISSING_PTS_ERROR_MESSAGE);
@@ -1106,12 +1143,13 @@ const readPesPacketHeader = <T extends boolean>(
};
const readPesPacket = <T extends boolean>(
demuxer: MpegTsDemuxer,
section: Section,
expectPts: T,
): (T extends true ? TimestampedPesPacket : PesPacket) | null => {
assert(section.endPos !== null); // Can only read full PES packets from fully read sections
const header = readPesPacketHeader(section, expectPts);
const header = readPesPacketHeader(demuxer, section, expectPts);
if (!header) {
return null;
}
@@ -1266,7 +1304,7 @@ abstract class MpegTsTrackBacking implements InputTrackBacking {
const section = this.elementaryStream.firstSection;
assert(section);
const pesPacket = readPesPacket(section, true);
const pesPacket = readPesPacket(this.elementaryStream.demuxer, section, true);
assert(pesPacket);
const context = new PacketReadingContext(this.elementaryStream, pesPacket);
@@ -1314,7 +1352,7 @@ abstract class MpegTsTrackBacking implements InputTrackBacking {
const section = await demuxer.readSection(sectionStartPos, true);
assert(section);
const pesPacket = readPesPacket(section, true);
const pesPacket = readPesPacket(demuxer, section, true);
assert(pesPacket);
const context = new PacketReadingContext(this.elementaryStream, pesPacket);
@@ -1398,7 +1436,7 @@ abstract class MpegTsTrackBacking implements InputTrackBacking {
return null;
}
const pesPacketHeader = readPesPacketHeader(section, false);
const pesPacketHeader = readPesPacketHeader(demuxer, section, false);
if (pesPacketHeader && pesPacketHeader.pts !== null) {
return {
pesPacketHeader: pesPacketHeader as TimestampedPesPacketHeader,
@@ -1416,7 +1454,7 @@ abstract class MpegTsTrackBacking implements InputTrackBacking {
// Get the first PES packet of the track
const firstSection = this.elementaryStream.firstSection;
assert(firstSection);
const firstPesPacketHeader = readPesPacketHeader(firstSection, true);
const firstPesPacketHeader = readPesPacketHeader(demuxer, firstSection, true);
assert(firstPesPacketHeader);
if (searchPts < firstPesPacketHeader.pts) {
@@ -1499,7 +1537,7 @@ abstract class MpegTsTrackBacking implements InputTrackBacking {
const section = await demuxer.readSection(sectionStartPos, true);
assert(section);
const pesPacket = readPesPacket(section, true);
const pesPacket = readPesPacket(demuxer, section, true);
assert(pesPacket);
const context = new PacketReadingContext(this.elementaryStream, pesPacket);
@@ -1563,7 +1601,7 @@ abstract class MpegTsTrackBacking implements InputTrackBacking {
if (packetHeader.pid === pid && packetHeader.payloadUnitStartIndicator === 1) {
const section = await demuxer.readSection(currentPos, false);
if (section) {
const nextPesHeader = readPesPacketHeader(section, false);
const nextPesHeader = readPesPacketHeader(demuxer, section, false);
if (nextPesHeader && nextPesHeader.pts !== null) {
if (nextPesHeader.pts > searchPts) {
break outer;
@@ -1595,7 +1633,7 @@ abstract class MpegTsTrackBacking implements InputTrackBacking {
if (packetHeader.pid === pid && packetHeader.payloadUnitStartIndicator === 1) {
const section = await demuxer.readSection(pos, false);
if (section) {
const header = readPesPacketHeader(section, false);
const header = readPesPacketHeader(demuxer, section, false);
if (header && header.pts !== null) {
currentPesHeader = header as TimestampedPesPacketHeader;
break;
@@ -1655,7 +1693,7 @@ abstract class MpegTsTrackBacking implements InputTrackBacking {
isKeyPacket = pesHeader.randomAccessIndicator === 1;
} else {
assert(pesHeaderSection);
const pesPacket = readPesPacket(pesHeaderSection, true);
const pesPacket = readPesPacket(demuxer, pesHeaderSection, true);
assert(pesPacket);
const context = new PacketReadingContext(this.elementaryStream, pesPacket);
@@ -1694,7 +1732,7 @@ abstract class MpegTsTrackBacking implements InputTrackBacking {
if (packetHeader.pid === pid && packetHeader.payloadUnitStartIndicator === 1) {
const section = await demuxer.readSection(currentPos, readSectionsInFull);
if (section) {
const nextPesHeader = readPesPacketHeader(section, false);
const nextPesHeader = readPesPacketHeader(demuxer, section, false);
if (nextPesHeader && nextPesHeader.pts !== null) {
pesHeader = nextPesHeader as TimestampedPesPacketHeader;
@@ -1728,7 +1766,7 @@ abstract class MpegTsTrackBacking implements InputTrackBacking {
if (packetHeader.pid === pid && packetHeader.payloadUnitStartIndicator === 1) {
const section = await demuxer.readSection(pos, readSectionsInFull);
if (section) {
const header = readPesPacketHeader(section, false);
const header = readPesPacketHeader(demuxer, section, false);
if (header && header.pts !== null) {
startPesHeader = header as TimestampedPesPacketHeader;
break;
@@ -1971,7 +2009,7 @@ class PacketReadingContext {
return;
}
const nextPesPacket = readPesPacket(nextSection, false);
const nextPesPacket = readPesPacket(this.demuxer, nextSection, false);
if (nextPesPacket) {
pesPacket = nextPesPacket;
break;
+1
View File
@@ -7,6 +7,7 @@
*/
export const TIMESCALE = 90_000; // MPEG-TS timestamps run on a 90 kHz clock
export const TIMESTAMP_MODULUS = 2 ** 33;
export const TS_PACKET_SIZE = 188;
export const enum MpegTsStreamType {
+20 -10
View File
@@ -24,13 +24,19 @@ import {
iterateNalUnitsInLengthPrefixed,
} from '../codec-data';
import { Bitstream } from '../../shared/bitstream';
import { assert, promiseWithResolvers, setUint24, toDataView, toUint8Array } from '../misc';
import { assert, modEuclid, promiseWithResolvers, setUint24, toDataView, toUint8Array } from '../misc';
import { Muxer } from '../muxer';
import { Output, OutputAudioTrack, OutputTrack, OutputVideoTrack } from '../output';
import { MpegTsOutputFormat } from '../output-format';
import { EncodedPacket } from '../packet';
import { Writer } from '../writer';
import { buildMpegTsMimeType, MpegTsStreamType, TIMESCALE, TS_PACKET_SIZE } from './mpeg-ts-misc';
import {
buildMpegTsMimeType,
MpegTsStreamType,
TIMESCALE,
TIMESTAMP_MODULUS,
TS_PACKET_SIZE,
} from './mpeg-ts-misc';
// Resources:
// ISO/IEC 13818-1
@@ -589,26 +595,30 @@ export class MpegTsMuxer extends Muxer {
pesView.setUint8(7, includeDts ? 0xC0 : 0x80); // PTS_DTS_flags, other flags=0
pesView.setUint8(8, headerDataLength); // PES_header_data_length
const pts = Math.round(queuedPacket.presentationTimestamp * TIMESCALE);
// This logic supports writing negative timestamps by simplying Euclid-modding them. It is then up to the
// demuxer to interpret them as negative, if it wasnts to.
const unwrappedPts = Math.round(queuedPacket.presentationTimestamp * TIMESCALE);
const pts = modEuclid(unwrappedPts, TIMESTAMP_MODULUS);
ptsDtsBitstream.pos = 0;
ptsDtsBitstream.writeBits(4, includeDts ? 0b0011 : 0b0010); // marker
ptsDtsBitstream.writeBits(3, (pts >>> 30) & 0x7); // PTS[32:30]
ptsDtsBitstream.writeBits(3, Math.floor(pts / 2 ** 30)); // PTS[32:30]
ptsDtsBitstream.writeBits(1, 1); // marker_bit
ptsDtsBitstream.writeBits(15, (pts >>> 15) & 0x7FFF); // PTS[29:15]
ptsDtsBitstream.writeBits(15, Math.floor(pts / 2 ** 15) % 2 ** 15); // PTS[29:15]
ptsDtsBitstream.writeBits(1, 1); // marker_bit
ptsDtsBitstream.writeBits(15, pts & 0x7FFF); // PTS[14:0]
ptsDtsBitstream.writeBits(15, pts % 2 ** 15); // PTS[14:0]
ptsDtsBitstream.writeBits(1, 1); // marker_bit
if (includeDts) {
assert(queuedPacket.decodeTimestamp !== null);
const dts = Math.round(queuedPacket.decodeTimestamp * TIMESCALE);
const unwrappedDts = Math.round(queuedPacket.decodeTimestamp * TIMESCALE);
const dts = modEuclid(unwrappedDts, TIMESTAMP_MODULUS);
ptsDtsBitstream.writeBits(4, 0b0001);
ptsDtsBitstream.writeBits(3, (dts >>> 30) & 0x7); // DTS[32:30]
ptsDtsBitstream.writeBits(3, Math.floor(dts / 2 ** 30)); // DTS[32:30]
ptsDtsBitstream.writeBits(1, 1); // marker_bit
ptsDtsBitstream.writeBits(15, (dts >>> 15) & 0x7FFF); // DTS[29:15]
ptsDtsBitstream.writeBits(15, Math.floor(dts / 2 ** 15) % 2 ** 15); // DTS[29:15]
ptsDtsBitstream.writeBits(1, 1); // marker_bit
ptsDtsBitstream.writeBits(15, dts & 0x7FFF); // DTS[14:0]
ptsDtsBitstream.writeBits(15, dts % 2 ** 15); // DTS[14:0]
ptsDtsBitstream.writeBits(1, 1); // marker_bit
}
-4
View File
@@ -43,10 +43,6 @@ export abstract class Muxer {
}>();
protected validateTimestamp(track: OutputTrack, timestampInSeconds: number, isKeyPacket: boolean) {
if (timestampInSeconds < 0) {
throw new Error(`Timestamps must be non-negative (got ${timestampInSeconds}s).`);
}
let timestampInfo = this.trackTimestampInfo.get(track);
if (!timestampInfo) {
if (!isKeyPacket) {
+58 -1
View File
@@ -85,6 +85,14 @@ export abstract class OutputFormat {
* durations of the media data.
*/
abstract get supportsTimestampedMediaData(): boolean;
/**
* The degree to which this output format supports writing media data with negative timestamps.
* - `'full'` - Negative timestamps are fully supported.
* - `'prefer-non-negative'` - Negative timestamps are technically supported, but their use is discouraged.
* - `'none'` - Negative timestamps are not supported.
* - `null` - Not applicable since the container doesn't support timestamped media at all.
*/
abstract get negativeTimestampSupport(): 'full' | 'prefer-non-negative' | 'none' | null;
/** Returns a list of video codecs that this output format can contain. */
getSupportedVideoCodecs(): VideoCodec[] {
@@ -278,6 +286,10 @@ export abstract class IsobmffOutputFormat extends OutputFormat {
return true;
}
get negativeTimestampSupport() {
return 'full' as const;
}
/** @internal */
_createMuxer(output: Output) {
return new IsobmffMuxer(output, this);
@@ -579,6 +591,10 @@ export class MkvOutputFormat extends OutputFormat {
get supportsTimestampedMediaData() {
return true;
}
get negativeTimestampSupport() {
return 'prefer-non-negative' as const;
}
}
/**
@@ -719,6 +735,10 @@ export class Mp3OutputFormat extends OutputFormat {
get supportsTimestampedMediaData() {
return false;
}
get negativeTimestampSupport() {
return null;
}
}
/**
@@ -821,6 +841,10 @@ export class WavOutputFormat extends OutputFormat {
get supportsTimestampedMediaData() {
return false;
}
get negativeTimestampSupport() {
return null;
}
}
/**
@@ -916,6 +940,10 @@ export class OggOutputFormat extends OutputFormat {
get supportsTimestampedMediaData() {
return false;
}
get negativeTimestampSupport() {
return null;
}
}
/**
@@ -994,6 +1022,10 @@ export class AdtsOutputFormat extends OutputFormat {
get supportsTimestampedMediaData() {
return false;
}
get negativeTimestampSupport() {
return null;
}
}
/**
@@ -1079,6 +1111,10 @@ export class FlacOutputFormat extends OutputFormat {
get supportsTimestampedMediaData() {
return false;
}
get negativeTimestampSupport() {
return null;
}
}
/**
@@ -1164,6 +1200,10 @@ export class MpegTsOutputFormat extends OutputFormat {
get supportsTimestampedMediaData() {
return true;
}
get negativeTimestampSupport() {
return 'prefer-non-negative' as const;
}
}
/**
@@ -1424,7 +1464,24 @@ export class HlsOutputFormat extends OutputFormat {
}
get supportsTimestampedMediaData(): boolean {
return true; // I guess??
return true; // It's only half true, really, but "false" is not correct either
}
get negativeTimestampSupport() {
const formats = toArray(this._options.segmentFormat);
// Return the lowest baseline across all segment formats
if (formats.some(format => format.negativeTimestampSupport === 'none')) {
return 'none' as const;
}
if (formats.some(format => format.negativeTimestampSupport === 'prefer-non-negative')) {
return 'prefer-non-negative' as const;
}
if (formats.some(format => format.negativeTimestampSupport === 'full')) {
return 'full' as const;
}
return null;
}
/** @internal */
+16 -16
View File
@@ -2866,32 +2866,32 @@ export class AudioSample implements Disposable {
}
/**
* Returns a new {@link AudioSample} containing only the frames in the range [startSample, endSample). Both bounds
* Returns a new {@link AudioSample} containing only the frames in the range [startFrame, endFrame). Both bounds
* must lie within this sample's range of frames. The returned sample's timestamp is shifted to match the start of
* the trimmed section.
*/
trim(startSample: number, endSample = this.numberOfFrames) {
if (!Number.isInteger(startSample) || startSample < 0) {
throw new TypeError('startSample must be a non-negative integer.');
trim(startFrame: number, endFrame = this.numberOfFrames) {
if (!Number.isInteger(startFrame) || startFrame < 0) {
throw new TypeError('startFrame must be a non-negative integer.');
}
if (!Number.isInteger(endSample) || endSample < 0) {
throw new TypeError('endSample must be a non-negative integer.');
if (!Number.isInteger(endFrame) || endFrame < 0) {
throw new TypeError('endFrame must be a non-negative integer.');
}
if (startSample > this.numberOfFrames) {
throw new RangeError('startSample out of range.');
if (startFrame > this.numberOfFrames) {
throw new RangeError('startFrame out of range.');
}
if (endSample > this.numberOfFrames) {
throw new RangeError('endSample out of range.');
if (endFrame > this.numberOfFrames) {
throw new RangeError('endFrame out of range.');
}
if (endSample < startSample) {
throw new RangeError('endSample must not be less than startSample.');
if (endFrame < startFrame) {
throw new RangeError('endFrame must not be less than startFrame.');
}
if (this._closed) {
throw new Error('AudioSample is closed.');
}
const frameCount = endSample - startSample;
const frameCount = endFrame - startFrame;
const bytesPerSample = getBytesPerSample(this.format);
let data: Uint8Array;
@@ -2905,7 +2905,7 @@ export class AudioSample implements Disposable {
this.copyTo(data.subarray(i * planeSize, (i + 1) * planeSize), {
planeIndex: i,
format: this.format,
frameOffset: startSample,
frameOffset: startFrame,
frameCount,
});
}
@@ -2918,7 +2918,7 @@ export class AudioSample implements Disposable {
this.copyTo(data, {
planeIndex: 0,
format: this.format,
frameOffset: startSample,
frameOffset: startFrame,
frameCount,
});
}
@@ -2929,7 +2929,7 @@ export class AudioSample implements Disposable {
format: this.format,
sampleRate: this.sampleRate,
numberOfChannels: this.numberOfChannels,
timestamp: this.timestamp + startSample / this.sampleRate,
timestamp: this.timestamp + startFrame / this.sampleRate,
});
}
+55 -3
View File
@@ -494,6 +494,9 @@ export type BlobSourceOptions = {
* field to `false` to try a slower but more stable reading method.
*/
useStreamReader?: boolean;
/** Handles errors that occur while no read is pending. By default, these become unhandled rejections. */
handleUnhandledError?: (error: unknown) => unknown;
};
const blobReaderRegistry = typeof FinalizationRegistry !== 'undefined'
@@ -541,6 +544,9 @@ export class BlobSource extends Source {
if (options.useStreamReader !== undefined && typeof options.useStreamReader !== 'boolean') {
throw new TypeError('options.useStreamReader, when provided, must be a boolean.');
}
if (options.handleUnhandledError !== undefined && typeof options.handleUnhandledError !== 'function') {
throw new TypeError('options.handleUnhandledError, when provided, must be a function.');
}
super();
@@ -563,6 +569,7 @@ export class BlobSource extends Source {
}
},
prefetchProfile: PREFETCH_PROFILES.fileSystem,
handleUnhandledError: options.handleUnhandledError,
});
this._orchestrator.fileSize = blob.size;
@@ -732,6 +739,9 @@ export type UrlSourceOptions = {
* features, or use a custom implementation.
*/
fetchFn?: typeof fetch;
/** Handles errors that occur while no read is pending. By default, these become unhandled rejections. */
handleUnhandledError?: (error: unknown) => unknown;
};
/**
@@ -808,6 +818,9 @@ export class UrlSource extends PathedSource {
throw new TypeError('options.fetchFn, when provided, must be a function.');
// Won't bother validating this function beyond this
}
if (options.handleUnhandledError !== undefined && typeof options.handleUnhandledError !== 'function') {
throw new TypeError('options.handleUnhandledError, when provided, must be a function.');
}
const urlString = url instanceof Request
? url.url
@@ -869,6 +882,7 @@ export class UrlSource extends PathedSource {
maxWorkerCount: options.parallelism ?? DEFAULT_PARALLELISM,
runWorker: this._runWorker.bind(this),
prefetchProfile: PREFETCH_PROFILES.network,
handleUnhandledError: options.handleUnhandledError,
});
}
@@ -955,7 +969,13 @@ export class UrlSource extends PathedSource {
}
outer:
if (this._orchestrator.fileSize === null) {
if (
this._orchestrator.fileSize === null
// Content-Range/Length fields are meaningless if Content-Encoding is present. Content-Encoding is
// basically never used for range responses (since the encoding runs *before* the slicing), so we're set
// in that case.
&& (response.status === 206 || (response.type === 'basic' && !response.headers.has('Content-Encoding')))
) {
// See if we can deduce the file size from the response
const contentRange = response.headers.get('Content-Range');
@@ -1185,6 +1205,7 @@ export class UrlSource extends PathedSource {
const backing = new ReadableStreamSource(wrappedStream, {
maxCacheSize: this._orchestrator.options.maxCacheSize,
handleUnhandledError: this._options.handleUnhandledError,
});
backing._endIndex = this._orchestrator.fileSize; // Might still be null
backing._cacheMissErrorMessage = 'Attempted to read data from an already-evicted part of the cache. Because the'
@@ -1280,6 +1301,9 @@ const parseByteRangeHeader = (value: string) => {
export type FilePathSourceOptions = {
/** The maximum number of bytes the cache is allowed to hold in memory. Defaults to 8 MiB. */
maxCacheSize?: number;
/** Handles errors that occur while no read is pending. By default, these become unhandled rejections. */
handleUnhandledError?: (error: unknown) => unknown;
};
/**
@@ -1343,6 +1367,7 @@ export class FilePathSource extends PathedSource {
},
maxCacheSize: options.maxCacheSize,
prefetchProfile: 'fileSystem',
handleUnhandledError: options.handleUnhandledError,
});
}
@@ -1413,6 +1438,9 @@ export type CustomSourceOptions = {
* patterns are detected.
*/
prefetchProfile?: 'none' | 'fileSystem' | 'network';
/** Handles errors that occur while no read is pending. By default, these become unhandled rejections. */
handleUnhandledError?: (error: unknown) => unknown;
};
/**
@@ -1441,6 +1469,9 @@ export class CustomSource extends Source {
if (options.dispose !== undefined && typeof options.dispose !== 'function') {
throw new TypeError('options.dispose, when provided, must be a function.');
}
if (options.handleUnhandledError !== undefined && typeof options.handleUnhandledError !== 'function') {
throw new TypeError('options.handleUnhandledError, when provided, must be a function.');
}
if (
options.maxCacheSize !== undefined
&& (!isNumber(options.maxCacheSize) || options.maxCacheSize < 0)
@@ -1462,6 +1493,7 @@ export class CustomSource extends Source {
maxWorkerCount: 2, // Fixed for now, *should* be fine
prefetchProfile: PREFETCH_PROFILES[options.prefetchProfile ?? 'none'],
runWorker: this._runWorker.bind(this),
handleUnhandledError: options.handleUnhandledError,
});
}
@@ -1611,6 +1643,9 @@ type ReadableStreamSourcePendingSlice = {
export type ReadableStreamSourceOptions = {
/** The maximum number of bytes the cache is allowed to hold in memory. Defaults to 32 MiB. */
maxCacheSize?: number;
/** Handles errors that occur while no read is pending. By default, these become unhandled rejections. */
handleUnhandledError?: (error: unknown) => unknown;
};
/**
@@ -1648,6 +1683,8 @@ export class ReadableStreamSource extends Source {
_endIndex: number | null = null;
/** @internal */
_pulling = false;
/** @internal */
_handleUnhandledError: ((error: unknown) => void) | undefined;
/**
* Overridable for internal use.
* @internal
@@ -1663,6 +1700,9 @@ export class ReadableStreamSource extends Source {
if (!options || typeof options !== 'object') {
throw new TypeError('options must be an object.');
}
if (options.handleUnhandledError !== undefined && typeof options.handleUnhandledError !== 'function') {
throw new TypeError('options.handleUnhandledError, when provided, must be a function.');
}
if (
options.maxCacheSize !== undefined
&& (!isNumber(options.maxCacheSize) || options.maxCacheSize < 0)
@@ -1674,6 +1714,7 @@ export class ReadableStreamSource extends Source {
this._stream = stream;
this._maxCacheSize = options.maxCacheSize ?? (32 * 2 ** 20 /* 32 MiB */);
this._handleUnhandledError = options.handleUnhandledError;
}
/** @internal */
@@ -1768,6 +1809,8 @@ export class ReadableStreamSource extends Source {
if (this._pendingSlices.length > 0) {
this._pendingSlices.forEach(x => x.reject(error)); // Make sure to propagate any errors
this._pendingSlices.length = 0;
} else if (this._handleUnhandledError) {
this._handleUnhandledError(error);
} else {
throw error; // So it doesn't get swallowed
}
@@ -1994,6 +2037,7 @@ class ReadOrchestrator {
prefetchProfile: PrefetchProfile;
maxWorkerCount: number;
onIdleWorkerRemoved?: (worker: ReadWorker) => void;
handleUnhandledError?: (error: unknown) => unknown;
}) {}
read(
@@ -2208,7 +2252,11 @@ class ReadOrchestrator {
}
// Nobody's awaiting this result but an errored read is still notable
throw error;
if (this.options.handleUnhandledError) {
this.options.handleUnhandledError(error);
} else {
throw error;
}
});
}
@@ -2326,7 +2374,11 @@ class ReadOrchestrator {
worker.pendingSlices.forEach(x => x.reject(error)); // Make sure to propagate any errors
worker.pendingSlices.length = 0;
} else if (!worker.aborted && !this.disposed) {
throw error; // So it doesn't get swallowed
if (this.options.handleUnhandledError) {
this.options.handleUnhandledError(error);
} else {
throw error; // So it doesn't get swallowed
}
}
})
.finally(() => {
+399 -2
View File
@@ -3,20 +3,23 @@ import { Input } from '../../src/input.js';
import {
AdtsOutputFormat,
HlsOutputFormat,
MkvOutputFormat,
Mp4OutputFormat,
MpegTsOutputFormat,
OutputFormat,
WavOutputFormat,
} from '../../src/output-format.js';
import { Output, OutputTrackGroup } from '../../src/output.js';
import { BufferSource, CustomPathedSource, UrlSource } from '../../src/source.js';
import { expect, test } from 'vitest';
import { BufferTarget, PathedTarget } from '../../src/target.js';
import { Conversion, ConversionCanceledError } from '../../src/conversion.js';
import { assert } from '../../src/misc.js';
import { Conversion, ConversionCanceledError, ConversionOptions } from '../../src/conversion.js';
import { assert, uint8ArraysAreEqual } from '../../src/misc.js';
import { InputVideoTrack } from '../../src/input-track.js';
import { CanvasSource, EncodedAudioPacketSource } from '../../src/media-source.js';
import { Quality } from '../../src/encode.js';
import { EncodedPacket } from '../../src/packet.js';
import { EncodedPacketSink } from '../../src/media-sink.js';
test('Rotation is baked in when rerendering', async () => {
using input = new Input({
@@ -783,3 +786,397 @@ test('Resizing at various scale factors', async () => {
expect(await videoTrack!.getDisplayHeight()).toBe(height);
}
});
test('Packet copy, whole file', async () => {
await testCopy({
conversionOptions: {},
expectedTimeOffset: 0,
videoStartTimestamp: 0,
videoEndTimestamp: 5,
audioStartTimestamp: -1024 / 48000,
audioEndTimestamp: 5,
});
});
test('Packet copy, keyframe trim', async () => {
await testCopy({
conversionOptions: {
trim: {
start: 1,
end: 2,
},
},
expectedTimeOffset: 1,
videoStartTimestamp: 0,
videoEndTimestamp: 1,
audioStartTimestamp: -0.04,
audioEndTimestamp: 1.0053333333333334,
});
});
test('Packet copy, keyframe trim, shrink', async () => {
await testCopy({
conversionOptions: {
trim: {
start: 1,
end: 2,
},
copy: {
boundaryPolicy: 'shrink',
},
},
expectedTimeOffset: 1,
videoStartTimestamp: 0,
videoEndTimestamp: 1,
audioStartTimestamp: -0.04 + 2 * 1024 / 48000,
audioEndTimestamp: 1.0053333333333334 - 1024 / 48000,
});
});
test('Packet copy, delta frame trim', async () => {
await testCopy({
conversionOptions: {
trim: {
start: 1.5,
end: 2.5,
},
},
expectedTimeOffset: 1.5,
videoStartTimestamp: -0.5,
videoEndTimestamp: 1.02,
audioStartTimestamp: -0.028,
audioEndTimestamp: 1.0173333333333334,
});
});
test('Packet copy, delta frame trim, shrink', async () => {
await testCopy({
conversionOptions: {
trim: {
start: 1.5,
end: 2.5,
},
copy: {
boundaryPolicy: 'shrink',
},
},
expectedTimeOffset: 1.5,
videoStartTimestamp: 0.5,
videoEndTimestamp: 1.02, // Due to MP4 not being able to express a different duration for the last packet
audioStartTimestamp: -0.028 + 2 * 1024 / 48000,
audioEndTimestamp: 1.0173333333333334 - 1024 / 48000,
});
});
test('Packet copy, whole file, Matroska', async () => {
await testCopy({
outputFormat: new MkvOutputFormat(),
conversionOptions: {
copy: {
shiftTolerance: Infinity,
},
},
expectedTimeOffset: -1024 / 48000,
videoStartTimestamp: 0 + 1024 / 48000,
videoEndTimestamp: 5 + 1024 / 48000 - 1 / 25,
audioStartTimestamp: 0,
audioEndTimestamp: 235 * 1024 / 48000,
precision: 0.001,
});
});
test('Packet copy, whole file, Matroska, forced', async () => {
await testCopy({
outputFormat: new MkvOutputFormat(),
conversionOptions: {
copy: {
mode: 'forced',
},
},
expectedTimeOffset: 0,
videoStartTimestamp: 0,
videoEndTimestamp: 5 - 1 / 25,
audioStartTimestamp: -1024 / 48000,
audioEndTimestamp: 4.992,
precision: 0.001,
});
});
test('Packet copy, delta frame trim, Matroska', async () => {
await testCopy({
outputFormat: new MkvOutputFormat(),
conversionOptions: {
trim: {
start: 1.5,
end: 2.5,
},
copy: {
shiftTolerance: Infinity,
},
},
expectedTimeOffset: 1,
videoStartTimestamp: 0,
videoEndTimestamp: 1.48,
audioStartTimestamp: 0.472,
audioEndTimestamp: 1.496,
precision: 0.001,
});
});
test('Packet copy, delta frame trim, video transcoded, Matroska', async () => {
await testCopy({
outputFormat: new MkvOutputFormat(),
conversionOptions: {
video: {
forceTranscode: true,
codec: 'vp9',
},
trim: {
start: 1.5,
end: 2.5,
},
copy: {
shiftTolerance: Infinity,
},
},
expectedTimeOffset: 1.472,
videoStartTimestamp: 0.028,
videoEndTimestamp: 1.008,
audioStartTimestamp: 0,
audioEndTimestamp: 1.024,
precision: 0.001,
compareVideoPackets: false,
});
});
test('Packet copy, whole file, ADTS', async () => {
await testCopy({
outputFormat: new AdtsOutputFormat(),
conversionOptions: {
video: {
discard: true,
},
copy: {
mode: 'forced',
shiftTolerance: Infinity,
},
},
expectedTimeOffset: -1024 / 48000,
audioStartTimestamp: 0,
audioEndTimestamp: 5.034666666666666,
processNewAudioPacketData: data => data.subarray(7),
});
});
const testCopy = async (options: {
outputFormat?: OutputFormat;
conversionOptions: Omit<ConversionOptions, 'input' | 'output'>;
expectedTimeOffset: number;
videoStartTimestamp?: number;
videoEndTimestamp?: number;
audioStartTimestamp: number;
audioEndTimestamp: number;
precision?: number;
processNewAudioPacketData?: (data: Uint8Array) => Uint8Array;
compareVideoPackets?: boolean;
}) => {
const precision = options.precision ?? 0.000001;
const isCloseTo = (a: number, b: number) => {
return Math.abs(a - b) <= precision;
};
using input = new Input({
source: new UrlSource('/demo.mp4'),
formats: ALL_FORMATS,
});
const output = new Output({
format: options.outputFormat ?? new Mp4OutputFormat(),
target: new BufferTarget(),
});
const videoTrack = await input.getPrimaryVideoTrack();
const audioTrack = await input.getPrimaryAudioTrack();
assert(videoTrack);
assert(audioTrack);
const videoSink = new EncodedPacketSink(videoTrack);
const audioSink = new EncodedPacketSink(audioTrack);
const conversion = await Conversion.init({
input,
output,
...options.conversionOptions,
});
await conversion.execute();
using newInput = new Input({
source: new BufferSource(output.target.buffer!),
formats: ALL_FORMATS,
});
const newVideoTrack = await newInput.getPrimaryVideoTrack();
const newAudioTrack = await newInput.getPrimaryAudioTrack();
if (newVideoTrack) {
const newVideoSink = new EncodedPacketSink(newVideoTrack);
expect(isCloseTo(await newVideoTrack.getFirstTimestamp(), options.videoStartTimestamp!)).toBe(true);
expect(isCloseTo(await newVideoTrack.computeDuration(), options.videoEndTimestamp!)).toBe(true);
if (options.compareVideoPackets ?? true) {
for await (const newPacket of newVideoSink.packets()) {
const oldPacket = await videoSink.getPacket(
newPacket.timestamp + options.expectedTimeOffset + precision,
);
assert(oldPacket);
expect(uint8ArraysAreEqual(oldPacket.data, newPacket.data)).toBe(true);
expect(oldPacket.type).toEqual(newPacket.type);
}
}
}
if (newAudioTrack) {
const newAudioSink = new EncodedPacketSink(newAudioTrack);
expect(isCloseTo(await newAudioTrack.getFirstTimestamp(), options.audioStartTimestamp)).toBe(true);
expect(isCloseTo(await newAudioTrack.computeDuration(), options.audioEndTimestamp)).toBe(true);
for await (const newPacket of newAudioSink.packets()) {
const oldPacket = await audioSink.getPacket(newPacket.timestamp + options.expectedTimeOffset + precision);
assert(oldPacket);
const process = options.processNewAudioPacketData ?? (x => x);
expect(uint8ArraysAreEqual(oldPacket.data, process(newPacket.data))).toBe(true);
expect(oldPacket.type).toEqual(newPacket.type);
}
}
};
test('Trim wholly before media data, transcode', async () => {
using input = new Input({
source: new UrlSource('/demo.mp4'),
formats: ALL_FORMATS,
});
const output = new Output({
format: new Mp4OutputFormat(),
target: new BufferTarget(),
});
const conversion = await Conversion.init({
input,
output,
video: {
forceTranscode: true,
},
audio: {
forceTranscode: true,
},
trim: {
start: -10,
end: -5,
},
});
await conversion.execute();
using newInput = new Input({
source: new BufferSource(output.target.buffer!),
formats: ALL_FORMATS,
});
expect(await newInput.getPrimaryVideoTrack()).toBeNull(); // No video data
expect(await newInput.getPrimaryAudioTrack()).toBeNull(); // No audio data
});
test('Trim wholly before media data, copy', async () => {
using input = new Input({
source: new UrlSource('/demo.mp4'),
formats: ALL_FORMATS,
});
const output = new Output({
format: new Mp4OutputFormat(),
target: new BufferTarget(),
});
const conversion = await Conversion.init({
input,
output,
copy: { mode: 'forced' },
trim: {
start: -10,
end: -5,
},
});
await conversion.execute();
using newInput = new Input({
source: new BufferSource(output.target.buffer!),
formats: ALL_FORMATS,
});
expect(await newInput.getPrimaryVideoTrack()).toBeNull(); // No video data
expect(await newInput.getPrimaryAudioTrack()).toBeNull(); // No audio data
});
test('Trim wholly past media data, transcode', async () => {
using input = new Input({
source: new UrlSource('/demo.mp4'),
formats: ALL_FORMATS,
});
const output = new Output({
format: new Mp4OutputFormat(),
target: new BufferTarget(),
});
const conversion = await Conversion.init({
input,
output,
video: {
forceTranscode: true,
},
audio: {
forceTranscode: true,
},
trim: {
start: 10,
end: 15,
},
});
await conversion.execute();
using newInput = new Input({
source: new BufferSource(output.target.buffer!),
formats: ALL_FORMATS,
});
expect(await newInput.getPrimaryVideoTrack()).toBeNull(); // No video data
expect(await newInput.getPrimaryAudioTrack()).toBeNull(); // No audio data
});
test('Trim wholly past media data, copy', async () => {
using input = new Input({
source: new UrlSource('/demo.mp4'),
formats: ALL_FORMATS,
});
const output = new Output({
format: new Mp4OutputFormat(),
target: new BufferTarget(),
});
const conversion = await Conversion.init({
input,
output,
copy: { mode: 'forced' },
trim: {
start: 10,
end: 15,
},
});
await conversion.execute();
using newInput = new Input({
source: new BufferSource(output.target.buffer!),
formats: ALL_FORMATS,
});
// Don't test video yet; requires extensions to packet fetching logic. B-frames!!
// expect(await newInput.getPrimaryVideoTrack()).toBeNull(); // No video data
expect(await newInput.getPrimaryAudioTrack()).toBeNull(); // No audio data
});
@@ -30,8 +30,6 @@ test('FLAC encoder, 24-bit', async () => {
expect(result.size).toBeGreaterThan(90_000);
console.log(result.size);
expect(sample.format).toBe('s32');
});
+64 -2
View File
@@ -1,9 +1,13 @@
import { expect, test } from 'vitest';
import { Input } from '../../src/input.js';
import { UrlSource } from '../../src/source.js';
import { BufferSource, UrlSource } from '../../src/source.js';
import { ALL_FORMATS } from '../../src/input-format.js';
import { assert } from '../../src/misc.js';
import { AudioSampleSink } from '../../src/media-sink.js';
import { AudioSampleSink, VideoSampleSink } from '../../src/media-sink.js';
import { Output } from '../../src/output.js';
import { MpegTsOutputFormat } from '../../src/output-format.js';
import { BufferTarget } from '../../src/target.js';
import { Conversion } from '../../src/conversion.js';
// https://github.com/Vanilagy/mediabunny/issues/370
test('Negative audio timestamps are preserved', async () => {
@@ -24,3 +28,61 @@ test('Negative audio timestamps are preserved', async () => {
break;
}
});
test('No B-frames are skipped when software-decoding AVC', async () => {
using input = new Input({
source: new UrlSource('/missing-reorder-metadata-v1.mp4'),
formats: ALL_FORMATS,
});
const track = await input.getPrimaryVideoTrack();
assert(track);
const sink = new VideoSampleSink(track, {
hardwareAcceleration: 'prefer-software',
});
let count = 0;
// eslint-disable-next-line @typescript-eslint/no-unused-vars
for await (using sample of sink.samples()) {
count++;
}
expect(count).toBe(48);
});
test('No B-frames are skipped when software-decoding AVC, Annex B edition', async () => {
using input = new Input({
source: new UrlSource('/missing-reorder-metadata-v1.mp4'),
formats: ALL_FORMATS,
});
// Force Annex B by converting to MPEG-TS
const output = new Output({
format: new MpegTsOutputFormat(),
target: new BufferTarget(),
});
const conversion = await Conversion.init({ input, output, copy: { mode: 'forced' } });
await conversion.execute();
using newInput = new Input({
source: new BufferSource(output.target.buffer!),
formats: ALL_FORMATS,
});
const track = await newInput.getPrimaryVideoTrack();
assert(track);
const sink = new VideoSampleSink(track, {
hardwareAcceleration: 'prefer-software',
});
let count = 0;
// eslint-disable-next-line @typescript-eslint/no-unused-vars
for await (using sample of sink.samples()) {
count++;
}
expect(count).toBe(48);
});
+64
View File
@@ -738,3 +738,67 @@ test('MPEG-TS muxing with StreamTarget', async () => {
}
expect(videoPacketCount).toBe(frameCount);
});
test('MPEG-TS muxing with negative start timestamps', async () => {
await testNegativeTimestampRoundTrip(Array.from({ length: 50 }, (_, index) => (index - 10) / 10), 0.1);
});
test('MPEG-TS muxing with wholly negative timestamps', async () => {
await testNegativeTimestampRoundTrip([-1, -0.9, -0.8, -0.7, -0.6], 0.1);
});
const testNegativeTimestampRoundTrip = async (timestamps: number[], duration: number) => {
const output = new Output({
format: new MpegTsOutputFormat(),
target: new BufferTarget(),
});
const canvas = new OffscreenCanvas(640, 480);
const context = canvas.getContext('2d')!;
context.fillStyle = '#0000ff';
context.fillRect(0, 0, canvas.width, canvas.height);
const source = new CanvasSource(canvas, {
codec: 'avc',
quality: new Quality('high'),
});
output.addVideoTrack(source, { frameRate: 10 });
await output.start();
for (const timestamp of timestamps) {
await source.add(timestamp, duration);
}
await output.finalize();
using input = new Input({
source: new BufferSource(output.target.buffer!),
formats: ALL_FORMATS,
});
const track = await input.getPrimaryVideoTrack();
assert(track);
const sink = new EncodedPacketSink(track);
const outputPackets = [];
for await (const packet of sink.packets()) {
outputPackets.push(packet);
}
const expectedPackets = timestamps.map(timestamp => ({ timestamp, duration }));
expect(outputPackets.map(packet => ({
timestamp: packet.timestamp,
duration: packet.duration,
}))).toEqual(expectedPackets);
for (const expectedPacket of expectedPackets) {
const outputPacket = await sink.getPacket(expectedPacket.timestamp);
assert(outputPacket);
expect({
timestamp: outputPacket.timestamp,
duration: outputPacket.duration,
}).toEqual(expectedPacket);
}
};
+1
View File
@@ -205,6 +205,7 @@ test('can re-mux a .flac', async () => {
...otherInputMetadataTags,
raw: {
...otherInputMetadataTags.raw,
DATE: '2020-01-01',
vendor: 'Mediabunny',
},
images: inputImages,
+64
View File
@@ -1463,6 +1463,70 @@ segment-1-2.ts
);
});
test('Segmentation, negative start time', async () => {
const env = await setUpSegmentationEnvironment({ video: true });
const timestamps = Array.from({ length: 50 }, (_, index) => (index - 10) / 10);
for (const timestamp of timestamps) {
await env.addVideoPacket(timestamp, 'key', 0.1);
if (timestamp === 1) {
expect(env.segmentCount).toBe(1);
expect(await env.lastSegmentVideoTimestamps).toEqual(timestamps.slice(0, 20));
} else if (timestamp === 3) {
expect(env.segmentCount).toBe(2);
expect(await env.lastSegmentVideoTimestamps).toEqual(timestamps.slice(20, 40));
}
}
await env.output.finalize();
expect(env.segmentCount).toBe(3);
expect(await env.lastSegmentVideoTimestamps).toEqual(timestamps.slice(40));
expect(env.result).toBe(`#EXTM3U
#EXT-X-VERSION:3
#EXT-X-PLAYLIST-TYPE:VOD
#EXT-X-TARGETDURATION:2
#EXT-X-INDEPENDENT-SEGMENTS
#EXTINF:2,
segment-1-1.ts
#EXTINF:2,
segment-1-2.ts
#EXTINF:1,
segment-1-3.ts
#EXT-X-ENDLIST
`,
);
});
test('Segmentation, wholly negative timestamps', async () => {
const env = await setUpSegmentationEnvironment({ video: true });
const timestamps = [-1, -0.9, -0.8, -0.7, -0.6];
for (const timestamp of timestamps) {
await env.addVideoPacket(timestamp, 'key', 0.1);
}
await env.output.finalize();
expect(env.segmentCount).toBe(1);
expect(await env.lastSegmentVideoTimestamps).toEqual(timestamps);
expect(env.result).toBe(`#EXTM3U
#EXT-X-VERSION:3
#EXT-X-PLAYLIST-TYPE:VOD
#EXT-X-TARGETDURATION:2
#EXT-X-INDEPENDENT-SEGMENTS
#EXTINF:0.5,
segment-1-1.ts
#EXT-X-ENDLIST
`,
);
});
test('Segmentation, B-frames before key frame', async () => {
const env = await setUpSegmentationEnvironment({ video: true });
+81
View File
@@ -220,6 +220,87 @@ test('Non-zero start timestamp, fragmented MP4', async () => {
expect(durations).toEqual([0.1, 0.1, 0.1, 0.1]);
});
test('Negative start timestamps, regular MP4', async () => {
await testNegativeTimestampRoundTrip(Array.from({ length: 50 }, (_, index) => (index - 10) / 10), 0.1, false);
});
test('Negative start timestamps, fragmented MP4', async () => {
await testNegativeTimestampRoundTrip(Array.from({ length: 50 }, (_, index) => (index - 10) / 10), 0.1, true);
});
test('Wholly negative timestamps, regular MP4', async () => {
await testNegativeTimestampRoundTrip([-1, -0.9, -0.8, -0.7, -0.6], 0.1, false);
});
test('Wholly negative timestamps, fragmented MP4', async () => {
await testNegativeTimestampRoundTrip([-1, -0.9, -0.8, -0.7, -0.6], 0.1, true);
});
const testNegativeTimestampRoundTrip = async (
timestamps: number[],
duration: number,
fragmented: boolean,
) => {
const output = new Output({
format: new Mp4OutputFormat({ fastStart: fragmented ? 'fragmented' : false }),
target: new BufferTarget(),
});
const source = new EncodedVideoPacketSource('vp8');
output.addVideoTrack(source, { frameRate: 10 });
await output.start();
const meta = { decoderConfig: { codec: 'vp8', codedWidth: 1280, codedHeight: 720 } };
const inputPackets = timestamps.map((timestamp, index) => new EncodedPacket(
new Uint8Array(1024).fill(index),
'key',
timestamp,
duration,
));
for (let i = 0; i < inputPackets.length; i++) {
await source.add(inputPackets[i]!, i === 0 ? meta : undefined);
}
await output.finalize();
using input = new Input({
source: new BufferSource(output.target.buffer!),
formats: ALL_FORMATS,
});
const track = await input.getPrimaryVideoTrack();
assert(track);
const sink = new EncodedPacketSink(track);
const outputPackets: EncodedPacket[] = [];
for await (const packet of sink.packets()) {
outputPackets.push(packet);
}
expect(outputPackets.map(packet => ({
timestamp: packet.timestamp,
duration: packet.duration,
}))).toEqual(inputPackets.map(packet => ({
timestamp: packet.timestamp,
duration: packet.duration,
})));
for (const inputPacket of inputPackets) {
const outputPacket = await sink.getPacket(inputPacket.timestamp);
assert(outputPacket);
expect({
timestamp: outputPacket.timestamp,
duration: outputPacket.duration,
}).toEqual({
timestamp: inputPacket.timestamp,
duration: inputPacket.duration,
});
}
};
test('PCM audio', async () => {
const output = new Output({
format: new Mp4OutputFormat(),
+75
View File
@@ -9,6 +9,8 @@ import { BufferTarget } from '../../src/target.js';
import { MkvOutputFormat } from '../../src/output-format.js';
import { Conversion } from '../../src/conversion.js';
import { assert } from '../../src/misc.js';
import { EncodedVideoPacketSource } from '../../src/media-source.js';
import { EncodedPacket } from '../../src/packet.js';
const __dirname = new URL('.', import.meta.url).pathname;
@@ -61,3 +63,76 @@ test('Matroska muxer internally converts ADTS to AAC', async () => {
expect(count).toBe(4557);
});
test('Negative start timestamps', async () => {
await testNegativeTimestampRoundTrip(
Array.from({ length: 50 }, (_, index) => (index - 10) / 10),
0.1,
10,
);
});
test('Wholly negative timestamps', async () => {
await testNegativeTimestampRoundTrip([-1, -0.9, -0.8, -0.7, -0.6], 0.1, 10);
});
const testNegativeTimestampRoundTrip = async (timestamps: number[], duration: number, frameRate: number) => {
const output = new Output({
format: new MkvOutputFormat(),
target: new BufferTarget(),
});
const source = new EncodedVideoPacketSource('vp8');
output.addVideoTrack(source, { frameRate });
await output.start();
const meta = { decoderConfig: { codec: 'vp8', codedWidth: 1280, codedHeight: 720 } };
const inputPackets = timestamps.map((timestamp, index) => new EncodedPacket(
new Uint8Array(1024).fill(index),
'key',
timestamp,
duration,
));
for (let i = 0; i < inputPackets.length; i++) {
await source.add(inputPackets[i]!, i === 0 ? meta : undefined);
}
await output.finalize();
using input = new Input({
source: new BufferSource(output.target.buffer!),
formats: ALL_FORMATS,
});
const track = await input.getPrimaryVideoTrack();
assert(track);
const sink = new EncodedPacketSink(track);
const outputPackets: EncodedPacket[] = [];
for await (const packet of sink.packets()) {
outputPackets.push(packet);
}
expect(outputPackets.map(packet => ({
timestamp: packet.timestamp,
duration: packet.duration,
}))).toEqual(inputPackets.map(packet => ({
timestamp: packet.timestamp,
duration: packet.duration,
})));
for (const inputPacket of inputPackets) {
const outputPacket = await sink.getPacket(inputPacket.timestamp);
assert(outputPacket);
expect({
timestamp: outputPacket.timestamp,
duration: outputPacket.duration,
}).toEqual({
timestamp: inputPacket.timestamp,
duration: inputPacket.duration,
});
}
};
+22
View File
@@ -1223,6 +1223,28 @@ describe('Video', async () => {
await decoder.close();
});
test('No B-frames are skipped when decoding AVC', async () => {
using input = new Input({
source: new FilePathSource('./test/public/missing-reorder-metadata-v1.mp4'),
formats: ALL_FORMATS,
});
const track = await input.getPrimaryVideoTrack();
assert(track);
const sink = new VideoSampleSink(track, {
hardwareAcceleration: 'prefer-software',
});
let count = 0;
// eslint-disable-next-line @typescript-eslint/no-unused-vars
for await (using sample of sink.samples()) {
count++;
}
expect(count).toBe(48);
});
describe('VideoSample transformation', () => {
// 400x400 image: red everywhere, with a 200x200 blue square filling the bottom-left quadrant.
const TEST_IMAGE = (() => {
+36
View File
@@ -2,6 +2,9 @@ import { expect, test } from 'vitest';
import http from 'node:http';
import fs from 'node:fs';
import path from 'node:path';
import { brotliCompressSync } from 'node:zlib';
import { assert } from '../../src/misc.js';
import { Reader, readBytes } from '../../src/reader.js';
import {
ALL_FORMATS,
EncodedPacket,
@@ -225,6 +228,39 @@ test('UrlSource in sequential mode downloads lazily and aborts the response on d
}
}, 10_000);
test('UrlSource reads the full decoded body of a compressed response', async () => {
const text = 'Some playlist text\n'.repeat(256);
const content = Buffer.from(text);
const compressed = brotliCompressSync(content);
const server = http.createServer((req, res) => {
res.writeHead(200, {
'Content-Type': 'text/plain',
'Content-Encoding': 'br',
'Content-Length': compressed.byteLength,
});
res.end(compressed);
});
await new Promise<void>(resolve => server.listen(0, resolve));
try {
const address = server.address();
assert(address && typeof address !== 'string');
const source = new UrlSource(`http://localhost:${address.port}/playlist.m3u8`);
using ref = source.ref();
const reader = new Reader(ref.source);
const slice = await reader.requestEntireFile();
expect(slice).not.toBeNull();
expect(compressed.byteLength).toBeLessThan(content.byteLength);
expect(slice!.length).toBe(content.byteLength);
expect(Buffer.from(readBytes(slice!, slice!.length)).toString()).toBe(text);
} finally {
server.closeAllConnections();
server.close();
}
});
const startRangelessServer = async (
options: { responseByteLimits?: number[]; trailingPaddingSize?: number } = {},
) => {
Binary file not shown.
Binary file not shown.