diff --git a/dev/convert.html b/dev/convert.html
index 36a2645..df27399 100644
--- a/dev/convert.html
+++ b/dev/convert.html
@@ -56,6 +56,7 @@
}),
output,
audio: (_, n) => ({
+ codec: 'opus',
discard: n > 1,
//codec: 'opus',
//codec: 'opus',
@@ -100,7 +101,12 @@
},
*/
video: () => ({
+ //discard: true,
allowRotationMetadata: false,
+ forceTranscode: true,
+ width: 320,
+ frameRate: 3,
+ //frameRate: 30,
//width: 720,
//frameRate: 30,
//bitrate: Mediabunny.QUALITY_VERY_LOW,
@@ -180,8 +186,8 @@
}
},
trim: {
- start: 0,
- end: 4
+ start: 590,
+ //end: 590
},
});
console.log(conversion);
diff --git a/src/adts/adts-demuxer.ts b/src/adts/adts-demuxer.ts
index d50cd03..793766d 100644
--- a/src/adts/adts-demuxer.ts
+++ b/src/adts/adts-demuxer.ts
@@ -10,7 +10,6 @@ import { aacChannelMap, aacFrequencyTable, AudioCodec } from '../codec';
import { Demuxer } from '../demuxer';
import { Input } from '../input';
import { InputAudioTrack, InputAudioTrackBacking } from '../input-track';
-import { PacketRetrievalOptions } from '../media-sink';
import {
assert,
AsyncMutex4,
@@ -24,6 +23,7 @@ import { EncodedPacket, PLACEHOLDER_DATA } from '../packet';
import { readBytes, Reader } from '../reader';
import { DEFAULT_TRACK_DISPOSITION } from '../metadata';
import { FrameHeader, MAX_FRAME_HEADER_SIZE, MIN_FRAME_HEADER_SIZE, readFrameHeader } from './adts-reader';
+import { PacketRetrievalOptions } from '../cursors';
const SAMPLES_PER_AAC_FRAME = 1024;
diff --git a/src/conversion.ts b/src/conversion.ts
index 433f125..7a78c67 100644
--- a/src/conversion.ts
+++ b/src/conversion.ts
@@ -22,12 +22,6 @@ import {
} from './encode';
import { Input } from './input';
import { InputAudioTrack, InputTrack, InputVideoTrack } from './input-track';
-import {
- AudioSampleSink,
- CanvasSink,
- EncodedPacketSink,
- VideoSampleSink,
-} from './media-sink';
import {
AudioSource,
EncodedVideoPacketSource,
@@ -50,6 +44,7 @@ import { Mp4OutputFormat } from './output-format';
import { AudioSample, clampCropRectangle, validateCropRectangle, VideoSample } from './sample';
import { MetadataTags, validateMetadataTags } from './metadata';
import { NullTarget } from './target';
+import { AudioSampleCursor, canvasTransformer, PacketCursor, VideoSampleCursor, WrappedCanvas } from './cursors';
/**
* The options for media file conversion.
@@ -915,16 +910,13 @@ export class Conversion {
this._trackPromises.push((async () => {
await this._started;
- const sink = new EncodedPacketSink(track);
+ const cursor = new PacketCursor(track, { verifyKeyPackets: true });
const decoderConfig = await track.getDecoderConfig();
const meta: EncodedVideoChunkMetadata = { decoderConfig: decoderConfig ?? undefined };
- const endPacket = Number.isFinite(this._endTimestamp)
- ? await sink.getPacket(this._endTimestamp, { metadataOnly: true }) ?? undefined
- : undefined;
- for await (const packet of sink.packets(undefined, endPacket, { verifyKeyPackets: true })) {
- if (this._canceled) {
- return;
+ await cursor.iterate(async (packet) => {
+ if (packet.timestamp >= this._endTimestamp || this._canceled) {
+ return false;
}
if (alpha === 'discard') {
@@ -933,16 +925,20 @@ export class Conversion {
delete packet.sideData.alphaByteLength;
}
- this._reportProgress(track.id, packet.timestamp);
+ this._updateProgress(track.id, packet.timestamp);
await source.add(packet, meta);
if (this._synchronizer.shouldWait(track.id, packet.timestamp)) {
await this._synchronizer.wait(packet.timestamp);
}
+ });
+
+ if (this._canceled) {
+ return;
}
source.close();
- this._synchronizer.closeTrack(track.id);
+ this._closeTrack(track.id);
})());
} else {
// We need to decode & reencode the video
@@ -1009,13 +1005,13 @@ export class Conversion {
await tempOutput.start();
- const sink = new VideoSampleSink(track);
- const firstSample = await sink.getSample(firstTimestamp); // Let's just use the first sample
+ const cursor = new VideoSampleCursor(track, { autoClose: false });
+ using firstSample = await cursor.seekToFirst(); // Let's just use the first sample
+ await cursor.close();
if (firstSample) {
try {
await tempSource.add(firstSample);
- firstSample.close();
await tempOutput.finalize();
} catch (error) {
console.info('Error when probing encoder support. Falling back to rerender path.', error);
@@ -1027,176 +1023,94 @@ export class Conversion {
}
}
- if (needsRerender) {
- this._trackPromises.push((async () => {
- await this._started;
+ this._trackPromises.push((async () => {
+ await this._started;
- const sink = new CanvasSink(track, {
- width,
- height,
- fit: trackOptions.fit ?? 'fill',
- rotation: totalRotation, // Bake the rotation into the output
- crop: trackOptions.crop,
- poolSize: 1,
- alpha: alpha === 'keep',
- });
- const iterator = sink.canvases(this._startTimestamp, this._endTimestamp);
- const frameRate = trackOptions.frameRate;
+ const transform = needsRerender
+ ? canvasTransformer({
+ width,
+ height,
+ fit: trackOptions.fit ?? 'fill',
+ rotation: totalRotation, // Bake the rotation into the output
+ crop: trackOptions.crop,
+ poolSize: 1,
+ alpha: alpha === 'keep',
+ })
+ : undefined;
- let lastCanvas: HTMLCanvasElement | OffscreenCanvas | null = null;
- let lastCanvasTimestamp: number | null = null;
- let lastCanvasEndTimestamp: number | null = null;
+ await using cursor = new VideoSampleCursor(track, { transform });
+ const frameRate = trackOptions.frameRate;
- /** Repeats the last sample to pad out the time until the specified timestamp. */
- const padFrames = async (until: number) => {
- assert(lastCanvas);
- assert(frameRate !== undefined);
+ if (frameRate === undefined) {
+ await cursor.seekTo(this._startTimestamp);
- const frameDifference = Math.round((until - lastCanvasTimestamp!) * frameRate);
-
- for (let i = 1; i < frameDifference; i++) {
- const sample = new VideoSample(lastCanvas, {
- timestamp: lastCanvasTimestamp! + i / frameRate,
- duration: 1 / frameRate,
- });
- await this._registerVideoSample(track, trackOptions, source, sample);
- sample.close();
- }
- };
-
- for await (const { canvas, timestamp, duration } of iterator) {
+ for await (const it of cursor) {
if (this._canceled) {
return;
}
- let adjustedSampleTimestamp = Math.max(timestamp - this._startTimestamp, 0);
- lastCanvasEndTimestamp = adjustedSampleTimestamp + duration;
-
- if (frameRate !== undefined) {
- // Logic for skipping/repeating frames when a frame rate is set
- const alignedTimestamp = Math.floor(adjustedSampleTimestamp * frameRate) / frameRate;
-
- if (lastCanvas !== null) {
- if (alignedTimestamp <= lastCanvasTimestamp!) {
- lastCanvas = canvas;
- lastCanvasTimestamp = alignedTimestamp;
-
- // Skip this sample, since we already added one for this frame
- continue;
- } else {
- // Check if we may need to repeat the previous frame
- await padFrames(alignedTimestamp);
- }
- }
-
- adjustedSampleTimestamp = alignedTimestamp;
+ if (it.timestamp >= this._endTimestamp) {
+ break;
}
- const sample = new VideoSample(canvas, {
- timestamp: adjustedSampleTimestamp,
- duration: frameRate !== undefined ? 1 / frameRate : duration,
- });
+ const timestamp = Math.max(it.timestamp - this._startTimestamp, 0);
+ const duration = clamp(
+ it.timestamp + it.duration - this._startTimestamp,
+ 0,
+ it.duration,
+ );
+
+ using sample = it instanceof WrappedCanvas
+ ? new VideoSample(it.canvas, { timestamp, duration })
+ : it.clone({ timestamp, duration });
+
await this._registerVideoSample(track, trackOptions, source, sample);
- sample.close();
-
- if (frameRate !== undefined) {
- lastCanvas = canvas;
- lastCanvasTimestamp = adjustedSampleTimestamp;
- }
}
+ } else {
+ let targetFrame = 0;
- if (lastCanvas) {
- assert(lastCanvasEndTimestamp !== null);
- assert(frameRate !== undefined);
-
- // If necessary, pad until the end timestamp of the last sample
- await padFrames(Math.floor(lastCanvasEndTimestamp * frameRate) / frameRate);
- }
-
- source.close();
- this._synchronizer.closeTrack(track.id);
- })());
- } else {
- this._trackPromises.push((async () => {
- await this._started;
-
- const sink = new VideoSampleSink(track);
- const frameRate = trackOptions.frameRate;
-
- let lastSample: VideoSample | null = null;
- let lastSampleTimestamp: number | null = null;
- let lastSampleEndTimestamp: number | null = null;
-
- /** Repeats the last sample to pad out the time until the specified timestamp. */
- const padFrames = async (until: number) => {
- assert(lastSample);
- assert(frameRate !== undefined);
-
- const frameDifference = Math.round((until - lastSampleTimestamp!) * frameRate);
-
- for (let i = 1; i < frameDifference; i++) {
- lastSample.setTimestamp(lastSampleTimestamp! + i / frameRate);
- lastSample.setDuration(1 / frameRate);
- await this._registerVideoSample(track, trackOptions, source, lastSample);
- }
-
- lastSample.close();
- };
-
- for await (const sample of sink.samples(this._startTimestamp, this._endTimestamp)) {
+ while (true) {
if (this._canceled) {
- lastSample?.close();
return;
}
- let adjustedSampleTimestamp = Math.max(sample.timestamp - this._startTimestamp, 0);
- lastSampleEndTimestamp = adjustedSampleTimestamp + sample.duration;
+ const targetTimestamp = targetFrame / frameRate;
+ const sourceTimestamp = this._startTimestamp + targetTimestamp;
- if (frameRate !== undefined) {
- // Logic for skipping/repeating frames when a frame rate is set
- const alignedTimestamp = Math.floor(adjustedSampleTimestamp * frameRate) / frameRate;
-
- if (lastSample !== null) {
- if (alignedTimestamp <= lastSampleTimestamp!) {
- lastSample.close();
- lastSample = sample;
- lastSampleTimestamp = alignedTimestamp;
-
- // Skip this sample, since we already added one for this frame
- continue;
- } else {
- // Check if we may need to repeat the previous frame
- await padFrames(alignedTimestamp);
- }
- }
-
- adjustedSampleTimestamp = alignedTimestamp;
- sample.setDuration(1 / frameRate);
+ if (sourceTimestamp >= this._endTimestamp) {
+ break;
}
- sample.setTimestamp(adjustedSampleTimestamp);
+ let it = await cursor.seekTo(sourceTimestamp);
+ if (!it) {
+ it = await cursor.seekToFirst();
+ }
+
+ if (!it) {
+ break;
+ }
+
+ const timestamp = targetTimestamp;
+ const duration = 1 / frameRate;
+ using sample = it instanceof WrappedCanvas
+ ? new VideoSample(it.canvas, { timestamp, duration })
+ : it.clone({ timestamp, duration });
+
await this._registerVideoSample(track, trackOptions, source, sample);
- if (frameRate !== undefined) {
- lastSample = sample;
- lastSampleTimestamp = adjustedSampleTimestamp;
- } else {
- sample.close();
+ const hasNext = await cursor.hasNext();
+ if (!hasNext && sourceTimestamp >= it.timestamp + it.duration) {
+ // We've reached the end
+ break;
}
+
+ targetFrame++;
}
+ }
- if (lastSample) {
- assert(lastSampleEndTimestamp !== null);
- assert(frameRate !== undefined);
-
- // If necessary, pad until the end timestamp of the last sample
- await padFrames(Math.floor(lastSampleEndTimestamp * frameRate) / frameRate);
- }
-
- source.close();
- this._synchronizer.closeTrack(track.id);
- })());
- }
+ source.close();
+ this._closeTrack(track.id);
+ })());
}
this.output.addVideoTrack(videoSource, {
@@ -1224,7 +1138,7 @@ export class Conversion {
return;
}
- this._reportProgress(track.id, sample.timestamp);
+ this._updateProgress(track.id, sample.timestamp);
let finalSamples: VideoSample[];
if (!trackOptions.process) {
@@ -1316,28 +1230,29 @@ export class Conversion {
this._trackPromises.push((async () => {
await this._started;
- const sink = new EncodedPacketSink(track);
+ const cursor = new PacketCursor(track);
const decoderConfig = await track.getDecoderConfig();
const meta: EncodedAudioChunkMetadata = { decoderConfig: decoderConfig ?? undefined };
- const endPacket = Number.isFinite(this._endTimestamp)
- ? await sink.getPacket(this._endTimestamp, { metadataOnly: true }) ?? undefined
- : undefined;
- for await (const packet of sink.packets(undefined, endPacket)) {
- if (this._canceled) {
- return;
+ await cursor.iterate(async (packet) => {
+ if (packet.timestamp >= this._endTimestamp || this._canceled) {
+ return false;
}
- this._reportProgress(track.id, packet.timestamp);
+ this._updateProgress(track.id, packet.timestamp);
await source.add(packet, meta);
if (this._synchronizer.shouldWait(track.id, packet.timestamp)) {
await this._synchronizer.wait(packet.timestamp);
}
+ });
+
+ if (this._canceled) {
+ return;
}
source.close();
- this._synchronizer.closeTrack(track.id);
+ this._closeTrack(track.id);
})());
} else {
// We need to decode & reencode the audio
@@ -1424,18 +1339,22 @@ export class Conversion {
this._trackPromises.push((async () => {
await this._started;
- const sink = new AudioSampleSink(track);
- for await (const sample of sink.samples(undefined, this._endTimestamp)) {
+ await using cursor = new AudioSampleCursor(track);
+
+ for await (const sample of cursor) {
if (this._canceled) {
return;
}
+ if (sample.timestamp >= this._endTimestamp) {
+ break;
+ }
+
await this._registerAudioSample(track, trackOptions, source, sample);
- sample.close();
}
source.close();
- this._synchronizer.closeTrack(track.id);
+ this._closeTrack(track.id);
})());
}
}
@@ -1463,7 +1382,7 @@ export class Conversion {
return;
}
- this._reportProgress(track.id, sample.timestamp);
+ this._updateProgress(track.id, sample.timestamp);
let finalSamples: AudioSample[];
if (!trackOptions.process) {
@@ -1532,38 +1451,53 @@ export class Conversion {
},
});
- const sink = new AudioSampleSink(track);
- const iterator = sink.samples(this._startTimestamp, this._endTimestamp);
+ await using cursor = new AudioSampleCursor(track);
+ await cursor.seekTo(this._startTimestamp);
- for await (const sample of iterator) {
+ for await (const sample of cursor) {
if (this._canceled) {
return;
}
+ if (sample.timestamp >= this._endTimestamp) {
+ break;
+ }
+
await resampler.add(sample);
- sample.close();
}
await resampler.finalize();
source.close();
- this._synchronizer.closeTrack(track.id);
+ this._closeTrack(track.id);
})());
return source;
}
/** @internal */
- _reportProgress(trackId: number, endTimestamp: number) {
- if (!this._computeProgress) {
- return;
- }
- assert(this._totalDuration !== null);
+ _closeTrack(id: number) {
+ this._synchronizer.closeTrack(id);
+ this._maxTimestamps.delete(id);
+ this._reportProgress();
+ }
+
+ /** @internal */
+ _updateProgress(trackId: number, endTimestamp: number) {
this._maxTimestamps.set(
trackId,
Math.max(endTimestamp, this._maxTimestamps.get(trackId)!),
);
+ this._reportProgress();
+ }
+
+ /** @internal */
+ _reportProgress() {
+ if (!this._computeProgress || this._maxTimestamps.size === 0) {
+ return;
+ }
+ assert(this._totalDuration !== null);
const minTimestamp = Math.min(...this._maxTimestamps.values());
const newProgress = clamp(minTimestamp / this._totalDuration, 0, 1);
diff --git a/src/cursors.ts b/src/cursors.ts
index a416cec..49ddd0a 100644
--- a/src/cursors.ts
+++ b/src/cursors.ts
@@ -1,15 +1,7 @@
import { PCM_AUDIO_CODECS } from './codec';
+import { AudioDecoderWrapper, DecoderWrapper, PcmAudioDecoderWrapper, VideoDecoderWrapper } from './decode';
import { InputDisposedError } from './input';
import { InputAudioTrack, InputTrack, InputVideoTrack } from './input-track';
-import {
- AudioDecoderWrapper,
- DecoderWrapper,
- PacketRetrievalOptions,
- PcmAudioDecoderWrapper,
- validatePacketRetrievalOptions,
- validateTimestamp,
- VideoDecoderWrapper,
-} from './media-sink';
import {
assert,
AsyncMutex4,
@@ -25,12 +17,55 @@ import {
Rotation,
AsyncGate,
Yo,
+ isNumber,
} from './misc';
import { EncodedPacket } from './packet';
import { AudioSample, clampCropRectangle, CropRectangle, validateCropRectangle, VideoSample } from './sample';
polyfillSymbolDispose();
+/**
+ * Additional options for controlling packet retrieval.
+ * @group Media sinks
+ * @public
+ */
+export type PacketRetrievalOptions = {
+ /**
+ * When set to `true`, only packet metadata (like timestamp) will be retrieved - the actual packet data will not
+ * be loaded.
+ */
+ metadataOnly?: boolean;
+
+ /**
+ * When set to true, key packets will be verified upon retrieval by looking into the packet's bitstream.
+ * If not enabled, the packet types will be determined solely by what's stored in the containing file and may be
+ * incorrect, potentially leading to decoder errors. Since determining a packet's actual type requires looking into
+ * its data, this option cannot be enabled together with `metadataOnly`.
+ */
+ verifyKeyPackets?: boolean;
+};
+
+export const validatePacketRetrievalOptions = (options: PacketRetrievalOptions) => {
+ if (!options || typeof options !== 'object') {
+ throw new TypeError('options must be an object.');
+ }
+ if (options.metadataOnly !== undefined && typeof options.metadataOnly !== 'boolean') {
+ throw new TypeError('options.metadataOnly, when defined, must be a boolean.');
+ }
+ if (options.verifyKeyPackets !== undefined && typeof options.verifyKeyPackets !== 'boolean') {
+ throw new TypeError('options.verifyKeyPackets, when defined, must be a boolean.');
+ }
+ if (options.verifyKeyPackets && options.metadataOnly) {
+ throw new TypeError('options.verifyKeyPackets and options.metadataOnly cannot be enabled together.');
+ }
+};
+
+export const validateTimestamp = (timestamp: number) => {
+ if (!isNumber(timestamp)) {
+ throw new TypeError('timestamp must be a number.'); // It can be non-finite, that's fine
+ }
+};
+
export class PacketReader {
track: T;
@@ -331,26 +366,23 @@ export class PacketCursor {
}
async iterate(
- callback: (packet: EncodedPacket, stop: () => void) => MaybePromise,
+ callback: (packet: EncodedPacket) => MaybePromise,
) {
if (typeof callback !== 'function') {
throw new TypeError('callback must be a function.');
}
- let stopped = false;
- const stop = () => stopped = true;
-
const donePromise = this._callSerializer.waitUntilIdle();
if (donePromise) await donePromise;
while (true) {
if (this.current) {
- const result = callback(this.current, stop);
- if (result instanceof Promise) await result;
- }
+ let result = callback(this.current);
+ if (result instanceof Promise) result = await result;
- if (stopped) {
- break;
+ if (result === false) {
+ break;
+ }
}
const result = this.next();
@@ -398,9 +430,9 @@ type PendingRequest = {
successor: PendingRequest | null;
};
-type SampleTransformer = (sample: Sample) => TransformedSample;
+export type SampleTransformer = (sample: Sample) => TransformedSample;
-type SampleCursorOptions = {
+export type SampleCursorOptions = {
autoClose?: boolean;
transform?: SampleTransformer;
};
@@ -440,26 +472,27 @@ export abstract class SampleCursor<
private _currentSampleTimestamp: number | null = null;
/** The queue of samples that have been decoded and are now waiting. */
private _sampleQueue: Sample[] = [];
- /** Used to pause and resume the pump. */
- private _pumpGate = new AsyncGate();
private _pendingRequests: PendingRequest[] = [];
private _lastPendingRequest: PendingRequest | null = null;
- /**
- * When this value is above 0, the pump is instructed to be lazy: that is, only decode packets until the target and
- * not further to increase decoder efficiency.
- */
- private _lazyPump = 0;
/** Whether the next sample is the first sample. */
private _nextIsFirst = true;
private _queuedResets = 0;
/** @internal */
_pumpRunning = false;
+ /** Used to pause and resume the pump. */
+ private _pumpGate = new AsyncGate();
private _pumpStopQueued = false;
private _pumpStopped = new AsyncGate();
/** The minimum target packet until which the pump should decode. */
private _pumpTarget: EncodedPacket | null = null;
private _lastTarget: EncodedPacket | null = null;
+ private _decoderFlushPromise: Promise | null = null;
+ /**
+ * When this value is above 0, the pump is instructed to be lazy: that is, only decode packets until the target and
+ * not further to increase decoder efficiency.
+ */
+ private _lazyPump = 0;
private _closed = false;
private _closePromise: Promise | null = null;
@@ -511,6 +544,27 @@ export abstract class SampleCursor<
.finally(() => lock.release());
}
+ private _getSample(
+ callback: (result: ResultValue) => Promise,
+ ): MaybePromise {
+ this._ensureWillBeOpen();
+
+ try {
+ const result = new ResultValue();
+ const promise = callback(result);
+
+ if (result.pending) {
+ return promise
+ .then(() => result.value)
+ .catch(this._closeWithErrorAndThrow.bind(this));
+ } else {
+ return result.value;
+ }
+ } catch (error) {
+ this._closeWithErrorAndThrow(error);
+ }
+ }
+
seekToFirst(): MaybePromise {
return this._getSample(result => this._seekToPacket(result, this._packetReader.readFirst()));
}
@@ -533,14 +587,12 @@ export abstract class SampleCursor<
return this._getSample(result => this._nextKeyInternal(result));
}
- private _getSample(
- callback: (result: ResultValue) => Promise,
- ): MaybePromise {
+ hasNext(): MaybePromise {
this._ensureWillBeOpen();
try {
- const result = new ResultValue();
- const promise = callback(result);
+ const result = new ResultValue();
+ const promise = this._hasNextInternal(result);
if (result.pending) {
return promise
@@ -555,7 +607,7 @@ export abstract class SampleCursor<
}
async iterate(
- callback: (sample: TransformedSample, stop: () => void) => MaybePromise,
+ callback: (sample: TransformedSample) => MaybePromise,
) {
if (typeof callback !== 'function') {
throw new TypeError('callback must be a function.');
@@ -568,17 +620,14 @@ export abstract class SampleCursor<
this._ensureNotClosed();
- let stopped = false;
- const stop = () => stopped = true;
-
while (true) {
if (this.current) {
- const result = callback(this.current, stop);
- if (result instanceof Promise) await result;
- }
+ let result = callback(this.current);
+ if (result instanceof Promise) result = await result;
- if (stopped) {
- break;
+ if (result === false) {
+ break;
+ }
}
const result = this.next();
@@ -1052,6 +1101,30 @@ export abstract class SampleCursor<
return await this._seekToPacket(res, nextKey, lock);
}
+ private async _hasNextInternal(res: ResultValue): Promise {
+ using lock = this._mutex.lock();
+ if (lock.pending) await lock.ready;
+
+ this._ensureNotClosed();
+
+ if (this._nextIsFirst) {
+ let first = this._packetReader.readFirst();
+ if (first instanceof Promise) first = await first;
+
+ return res.set(!!first);
+ }
+
+ if (!this._pumpRunning) {
+ return res.set(false);
+ }
+
+ if (this._decoderFlushPromise) {
+ await this._decoderFlushPromise;
+ }
+
+ return res.set(this._sampleQueue.length > 0 || this._pumpRunning);
+ }
+
/**
* Starts the "pump process", which handles pushing packets into the decoder. It throttles itself if it is far
* enough ahead and must be woken up again by the outside. It also stops itself when the outside tells it to.
@@ -1122,7 +1195,15 @@ export abstract class SampleCursor<
}
if (!this._closed || this._pendingRequests.length > 0) {
- await this._decoder.flush();
+ const { promise, resolve } = promiseWithResolvers();
+ this._decoderFlushPromise = promise;
+
+ try {
+ await this._decoder.flush();
+ } finally {
+ resolve();
+ this._decoderFlushPromise = null;
+ }
} else {
// We're closed with no pending requests, don't bother flushing what's left
}
@@ -1356,13 +1437,19 @@ export class AudioSampleCursor extends SampleCu
* A canvas with additional timing information (timestamp & duration).
* @public
*/
-export type WrappedCanvas = {
+export class WrappedCanvas {
/** A canvas element or offscreen canvas. */
canvas: HTMLCanvasElement | OffscreenCanvas;
/** The timestamp of the corresponding video sample, in seconds. */
timestamp: number;
/** The duration of the corresponding video sample, in seconds. */
duration: number;
+
+ constructor(canvas: HTMLCanvasElement | OffscreenCanvas, timestamp: number, duration: number) {
+ this.canvas = canvas;
+ this.timestamp = timestamp;
+ this.duration = duration;
+ }
};
/**
@@ -1540,15 +1627,9 @@ export const canvasTransformer = (
}
sample.drawWithFit(context, { fit, rotation, crop });
-
- const result = {
- canvas,
- timestamp: sample.timestamp,
- duration: sample.duration,
- };
-
sample.close();
- return result;
+
+ return new WrappedCanvas(canvas, sample.timestamp, sample.duration);
};
};
diff --git a/src/decode.ts b/src/decode.ts
new file mode 100644
index 0000000..d67d8ed
--- /dev/null
+++ b/src/decode.ts
@@ -0,0 +1,950 @@
+import { AudioCodec, parsePcmCodec, PCM_AUDIO_CODECS, PcmAudioCodec, VideoCodec } from './codec';
+import {
+ deserializeAvcDecoderConfigurationRecord,
+ parseAvcSps,
+ determineVideoPacketType,
+ extractHevcNalUnits,
+ extractNalUnitTypeForHevc,
+ HevcNalUnitType,
+} from './codec-data';
+import { CustomAudioDecoder, customAudioDecoders, CustomVideoDecoder, customVideoDecoders } from './custom-coder';
+import {
+ CallSerializer,
+ Rotation,
+ isChromium,
+ toUint8Array,
+ isWebKit,
+ insertSorted,
+ last,
+ assert,
+ assertNever,
+ getInt24,
+ getUint24,
+ toDataView,
+} from './misc';
+import { EncodedPacket } from './packet';
+import { fromUlaw, fromAlaw } from './pcm';
+import { VideoSample, AudioSample } from './sample';
+
+export abstract class DecoderWrapper<
+ MediaSample extends VideoSample | AudioSample,
+> {
+ constructor(
+ public onSample: (sample: MediaSample) => unknown,
+ public onError: (error: Error) => unknown,
+ ) {}
+
+ abstract getDecodeQueueSize(): number;
+ abstract decode(packet: EncodedPacket): void;
+ abstract flush(): Promise;
+ abstract close(): void;
+
+ abstract get closed(): boolean;
+}
+
+export class VideoDecoderWrapper extends DecoderWrapper {
+ decoder: VideoDecoder | null = null;
+
+ customDecoder: CustomVideoDecoder | null = null;
+ customDecoderCallSerializer = new CallSerializer();
+ customDecoderQueueSize = 0;
+ customDecoderClosed = false;
+
+ inputTimestamps: number[] = []; // Timestamps input into the decoder, sorted.
+ frameQueue: VideoFrame[] = []; // Safari-specific thing, check usage.
+ currentPacketIndex = 0;
+ raslSkipped = false; // For HEVC stuff
+
+ // Alpha stuff
+ alphaDecoder: VideoDecoder | null = null;
+ alphaHadKeyframe = false;
+ colorQueue: VideoFrame[] = [];
+ alphaQueue: (VideoFrame | null)[] = [];
+ merger: ColorAlphaMerger | null = null;
+ mergerCreationFailed = false;
+ decodedAlphaChunkCount = 0;
+ alphaDecoderQueueSize = 0;
+ /** Each value is the number of decoded alpha chunks at which a null alpha frame should be added. */
+ nullAlphaFrameQueue: number[] = [];
+ currentAlphaPacketIndex = 0;
+ alphaRaslSkipped = false; // For HEVC stuff
+
+ onDequeue: (() => unknown) | null = null;
+
+ constructor(
+ onSample: (sample: VideoSample) => unknown,
+ onError: (error: Error) => unknown,
+ public codec: VideoCodec,
+ public decoderConfig: VideoDecoderConfig,
+ public rotation: Rotation,
+ public timeResolution: number,
+ ) {
+ super(onSample, onError);
+
+ const MatchingCustomDecoder = customVideoDecoders.find(x => x.supports(codec, decoderConfig));
+ if (MatchingCustomDecoder) {
+ // @ts-expect-error "Can't create instance of abstract class 🤓"
+ this.customDecoder = new MatchingCustomDecoder() as CustomVideoDecoder;
+ // @ts-expect-error It's technically readonly
+ this.customDecoder.codec = codec;
+ // @ts-expect-error It's technically readonly
+ this.customDecoder.config = decoderConfig;
+ // @ts-expect-error It's technically readonly
+ this.customDecoder.onSample = (sample) => {
+ if (!(sample instanceof VideoSample)) {
+ throw new TypeError('The argument passed to onSample must be a VideoSample.');
+ }
+
+ // @ts-expect-error Readonly
+ sample.rotation = this.rotation;
+
+ this.onSample(sample);
+ };
+
+ void this.customDecoderCallSerializer.call(() => this.customDecoder!.init());
+ } else {
+ const colorHandler = (frame: VideoFrame) => {
+ if (this.alphaQueue.length > 0) {
+ // Even when no alpha data is present (most of the time), there will be nulls in this queue
+ const alphaFrame = this.alphaQueue.shift();
+ assert(alphaFrame !== undefined);
+
+ this.mergeAlpha(frame, alphaFrame);
+ } else {
+ this.colorQueue.push(frame);
+ }
+ };
+
+ if (codec === 'avc' && this.decoderConfig.description && isChromium()) {
+ // Chromium has/had a bug with playing interlaced AVC (https://issues.chromium.org/issues/456919096)
+ // which can be worked around by requesting that software decoding be used. So, here we peek into the
+ // AVC description, if present, and switch to software decoding if we find interlaced content.
+ const record = deserializeAvcDecoderConfigurationRecord(toUint8Array(this.decoderConfig.description));
+ if (record && record.sequenceParameterSets.length > 0) {
+ const sps = parseAvcSps(record.sequenceParameterSets[0]!);
+ if (sps && sps.frameMbsOnlyFlag === 0) {
+ this.decoderConfig = {
+ ...this.decoderConfig,
+ hardwareAcceleration: 'prefer-software',
+ };
+ }
+ }
+ }
+
+ this.decoder = new VideoDecoder({
+ output: (frame) => {
+ try {
+ colorHandler(frame);
+ } catch (error) {
+ this.onError(error as Error);
+ }
+ },
+ error: onError,
+ });
+ this.decoder.configure(this.decoderConfig);
+
+ this.decoder.addEventListener('dequeue', () => {
+ this.onDequeue?.();
+ });
+ }
+ }
+
+ getDecodeQueueSize() {
+ if (this.customDecoder) {
+ return this.customDecoderQueueSize;
+ } else {
+ assert(this.decoder);
+
+ return Math.max(
+ this.decoder.decodeQueueSize,
+ this.alphaDecoder?.decodeQueueSize ?? 0,
+ );
+ }
+ }
+
+ decode(packet: EncodedPacket) {
+ if (this.codec === 'hevc' && this.currentPacketIndex > 0 && !this.raslSkipped) {
+ if (this.hasHevcRaslPicture(packet.data)) {
+ return; // Drop
+ }
+
+ this.raslSkipped = true;
+ }
+
+ this.currentPacketIndex++;
+
+ if (this.customDecoder) {
+ this.customDecoderQueueSize++;
+ void this.customDecoderCallSerializer
+ .call(() => this.customDecoder!.decode(packet))
+ .then(() => {
+ this.customDecoderQueueSize--;
+ this.onDequeue?.();
+ });
+ } else {
+ assert(this.decoder);
+
+ if (!isWebKit()) {
+ insertSorted(this.inputTimestamps, packet.timestamp, x => x);
+ }
+
+ this.decoder.decode(packet.toEncodedVideoChunk());
+ this.decodeAlphaData(packet);
+ }
+ }
+
+ decodeAlphaData(packet: EncodedPacket) {
+ if (!packet.sideData.alpha || this.mergerCreationFailed) {
+ // No alpha side data in the packet, most common case
+ this.pushNullAlphaFrame();
+ return;
+ }
+
+ if (!this.merger) {
+ try {
+ this.merger = new ColorAlphaMerger();
+ } catch (error) {
+ console.error('Due to an error, only color data will be decoded.', error);
+
+ this.mergerCreationFailed = true;
+ this.decodeAlphaData(packet); // Go again
+
+ return;
+ }
+ }
+
+ // Check if we need to set up the alpha decoder
+ if (!this.alphaDecoder) {
+ const alphaHandler = (frame: VideoFrame) => {
+ this.alphaDecoderQueueSize--;
+
+ if (this.colorQueue.length > 0) {
+ const colorFrame = this.colorQueue.shift();
+ assert(colorFrame !== undefined);
+
+ this.mergeAlpha(colorFrame, frame);
+ } else {
+ this.alphaQueue.push(frame);
+ }
+
+ // Check if any null frames have been queued for this point
+ this.decodedAlphaChunkCount++;
+ while (
+ this.nullAlphaFrameQueue.length > 0
+ && this.nullAlphaFrameQueue[0] === this.decodedAlphaChunkCount
+ ) {
+ this.nullAlphaFrameQueue.shift();
+
+ if (this.colorQueue.length > 0) {
+ const colorFrame = this.colorQueue.shift();
+ assert(colorFrame !== undefined);
+
+ this.mergeAlpha(colorFrame, null);
+ } else {
+ this.alphaQueue.push(null);
+ }
+ }
+ };
+
+ this.alphaDecoder = new VideoDecoder({
+ output: (frame) => {
+ try {
+ alphaHandler(frame);
+ } catch (error) {
+ this.onError(error as Error);
+ }
+ },
+ error: this.onError,
+ });
+ this.alphaDecoder.configure(this.decoderConfig);
+
+ this.alphaDecoder.addEventListener('dequeue', () => {
+ this.onDequeue?.();
+ });
+ }
+
+ const type = determineVideoPacketType(this.codec, this.decoderConfig, packet.sideData.alpha);
+
+ // Alpha packets might follow a different key frame rhythm than the main packets. Therefore, before we start
+ // decoding, we must first find a packet that's actually a key frame. Until then, we treat the image as opaque.
+ if (!this.alphaHadKeyframe) {
+ this.alphaHadKeyframe = type === 'key';
+ }
+
+ if (this.alphaHadKeyframe) {
+ // Same RASL skipping logic as for color, unlikely to be hit (since who uses HEVC with separate alpha??) but
+ // here for symmetry.
+ if (this.codec === 'hevc' && this.currentAlphaPacketIndex > 0 && !this.alphaRaslSkipped) {
+ if (this.hasHevcRaslPicture(packet.sideData.alpha)) {
+ this.pushNullAlphaFrame();
+ return;
+ }
+
+ this.alphaRaslSkipped = true;
+ }
+
+ this.currentAlphaPacketIndex++;
+ this.alphaDecoder.decode(packet.alphaToEncodedVideoChunk(type ?? packet.type));
+ this.alphaDecoderQueueSize++;
+ } else {
+ this.pushNullAlphaFrame();
+ }
+ }
+
+ pushNullAlphaFrame() {
+ if (this.alphaDecoderQueueSize === 0) {
+ // Easy
+ this.alphaQueue.push(null);
+ } else {
+ // There are still alpha chunks being decoded, so pushing `null` immediately would result in out-of-order
+ // data and be incorrect. Instead, we need to enqueue a "null frame" for when the current decoder workload
+ // has finished.
+ this.nullAlphaFrameQueue.push(this.decodedAlphaChunkCount + this.alphaDecoderQueueSize);
+ }
+ }
+
+ /**
+ * If we're using HEVC, we need to make sure to skip any RASL slices that follow a non-IDR key frame such as
+ * CRA_NUT. This is because RASL slices cannot be decoded without data before the CRA_NUT. Browsers behave
+ * differently here: Chromium drops the packets, Safari throws a decoder error. Either way, it's not good
+ * and causes bugs upstream. So, let's take the dropping into our own hands.
+ */
+ hasHevcRaslPicture(packetData: Uint8Array) {
+ const nalUnits = extractHevcNalUnits(packetData, this.decoderConfig);
+ return nalUnits.some((x) => {
+ const type = extractNalUnitTypeForHevc(x);
+ return type === HevcNalUnitType.RASL_N || type === HevcNalUnitType.RASL_R;
+ });
+ }
+
+ /** Handler for the WebCodecs VideoDecoder for ironing out browser differences. */
+ frameHandler(frame: VideoFrame) {
+ if (isWebKit()) {
+ // For correct B-frame handling, we don't just hand over the frames directly but instead add them to
+ // a queue, because we want to ensure frames are emitted in presentation order. We flush the queue
+ // each time we receive a frame with a timestamp larger than the highest we've seen so far, as we
+ // can sure that is not a B-frame. Typically, WebCodecs automatically guarantees that frames are
+ // emitted in presentation order, but Safari doesn't always follow this rule.
+ if (this.frameQueue.length > 0 && (frame.timestamp >= last(this.frameQueue)!.timestamp)) {
+ for (const frame of this.frameQueue) {
+ this.finalizeAndEmitSample(frame);
+ }
+
+ this.frameQueue.length = 0;
+ }
+
+ insertSorted(this.frameQueue, frame, x => x.timestamp);
+ } else {
+ // Assign it the next earliest timestamp from the input. We do this because browsers, by spec, are
+ // required to emit decoded frames in presentation order *while* retaining the timestamp of their
+ // originating EncodedVideoChunk. For files with B-frames but no out-of-order timestamps (like a
+ // missing ctts box, for example), this causes a mismatch. We therefore fix the timestamps and
+ // ensure they are sorted by doing this.
+ const timestamp = this.inputTimestamps.shift();
+
+ // There's no way we'd have more decoded frames than encoded packets we passed in. Actually, the
+ // correspondence should be 1:1.
+ assert(timestamp !== undefined);
+ this.finalizeAndEmitSample(frame, timestamp);
+ }
+ }
+
+ finalizeAndEmitSample(frame: VideoFrame, timestampOverride?: number) {
+ const sample = new VideoSample(frame, {
+ // Round the timestamps to the time resolution
+ timestamp: Math.round(
+ (timestampOverride ?? (frame.timestamp / 1e6)) * this.timeResolution,
+ ) / this.timeResolution,
+ duration: Math.round(
+ (frame.duration ?? 0) / 1e6 * this.timeResolution,
+ ) / this.timeResolution,
+ rotation: this.rotation,
+ });
+
+ this.onSample(sample);
+ }
+
+ mergeAlpha(color: VideoFrame, alpha: VideoFrame | null) {
+ if (!alpha) {
+ // Nothing needs to be merged
+ this.frameHandler(color);
+ return;
+ }
+
+ assert(this.merger);
+
+ this.merger.update(color, alpha);
+ color.close();
+ alpha.close();
+
+ const finalFrame = new VideoFrame(this.merger.canvas, {
+ timestamp: color.timestamp,
+ duration: color.duration ?? undefined,
+ });
+ this.frameHandler(finalFrame);
+ }
+
+ async flush() {
+ if (this.customDecoder) {
+ await this.customDecoderCallSerializer.call(() => this.customDecoder!.flush());
+ } else {
+ assert(this.decoder);
+ await Promise.all([
+ this.decoder.flush(),
+ this.alphaDecoder?.flush(),
+ ]);
+
+ this.colorQueue.forEach(x => x.close());
+ this.colorQueue.length = 0;
+ this.alphaQueue.forEach(x => x?.close());
+ this.alphaQueue.length = 0;
+
+ this.alphaHadKeyframe = false;
+ this.decodedAlphaChunkCount = 0;
+ this.alphaDecoderQueueSize = 0;
+ this.nullAlphaFrameQueue.length = 0;
+ this.currentAlphaPacketIndex = 0;
+ this.alphaRaslSkipped = false;
+ }
+
+ if (isWebKit()) {
+ for (const sample of this.frameQueue) {
+ this.finalizeAndEmitSample(sample);
+ }
+
+ this.frameQueue.length = 0;
+ }
+
+ this.currentPacketIndex = 0;
+ this.raslSkipped = false;
+ }
+
+ close() {
+ if (this.customDecoder) {
+ if (!this.customDecoderClosed) {
+ this.customDecoderClosed = true;
+ void this.customDecoderCallSerializer.call(() => this.customDecoder!.close());
+ }
+ } else {
+ assert(this.decoder);
+
+ if (this.decoder.state !== 'closed') {
+ this.decoder.close();
+ }
+ if (this.alphaDecoder && this.alphaDecoder.state !== 'closed') {
+ this.alphaDecoder.close();
+ }
+
+ this.colorQueue.forEach(x => x.close());
+ this.colorQueue.length = 0;
+ this.alphaQueue.forEach(x => x?.close());
+ this.alphaQueue.length = 0;
+
+ this.merger?.close();
+ }
+
+ for (const sample of this.frameQueue) {
+ sample.close();
+ }
+ this.frameQueue.length = 0;
+ }
+
+ get closed() {
+ if (this.customDecoder) {
+ if (this.customDecoderClosed) {
+ return true;
+ }
+
+ return !this.customDecoderCallSerializer.errored;
+ } else {
+ assert(this.decoder);
+ return this.decoder.state === 'closed';
+ }
+ }
+}
+
+/** Utility class that merges together color and alpha information using simple WebGL 2 shaders. */
+class ColorAlphaMerger {
+ canvas: OffscreenCanvas | HTMLCanvasElement;
+ private gl: WebGL2RenderingContext;
+ private program: WebGLProgram;
+ private vao: WebGLVertexArrayObject;
+ private colorTexture: WebGLTexture;
+ private alphaTexture: WebGLTexture;
+
+ constructor() {
+ // Canvas will be resized later
+ if (typeof OffscreenCanvas !== 'undefined') {
+ // Prefer OffscreenCanvas for Worker environments
+ this.canvas = new OffscreenCanvas(300, 150);
+ } else {
+ this.canvas = document.createElement('canvas');
+ }
+
+ const gl = this.canvas.getContext('webgl2', {
+ premultipliedAlpha: false,
+ }) as unknown as WebGL2RenderingContext | null; // Casting because of some TypeScript weirdness
+ if (!gl) {
+ throw new Error('Couldn\'t acquire WebGL 2 context.');
+ }
+
+ this.gl = gl;
+ this.program = this.createProgram();
+ this.vao = this.createVAO();
+ this.colorTexture = this.createTexture();
+ this.alphaTexture = this.createTexture();
+
+ this.gl.useProgram(this.program);
+ this.gl.uniform1i(this.gl.getUniformLocation(this.program, 'u_colorTexture'), 0);
+ this.gl.uniform1i(this.gl.getUniformLocation(this.program, 'u_alphaTexture'), 1);
+ }
+
+ private createProgram(): WebGLProgram {
+ const vertexShader = this.createShader(this.gl.VERTEX_SHADER, `#version 300 es
+ in vec2 a_position;
+ in vec2 a_texCoord;
+ out vec2 v_texCoord;
+
+ void main() {
+ gl_Position = vec4(a_position, 0.0, 1.0);
+ v_texCoord = a_texCoord;
+ }
+ `);
+
+ const fragmentShader = this.createShader(this.gl.FRAGMENT_SHADER, `#version 300 es
+ precision highp float;
+
+ uniform sampler2D u_colorTexture;
+ uniform sampler2D u_alphaTexture;
+ in vec2 v_texCoord;
+ out vec4 fragColor;
+
+ void main() {
+ vec3 color = texture(u_colorTexture, v_texCoord).rgb;
+ float alpha = texture(u_alphaTexture, v_texCoord).r;
+ fragColor = vec4(color, alpha);
+ }
+ `);
+
+ const program = this.gl.createProgram();
+ this.gl.attachShader(program, vertexShader);
+ this.gl.attachShader(program, fragmentShader);
+ this.gl.linkProgram(program);
+
+ return program;
+ }
+
+ private createShader(type: number, source: string): WebGLShader {
+ const shader = this.gl.createShader(type)!;
+ this.gl.shaderSource(shader, source);
+ this.gl.compileShader(shader);
+ return shader;
+ }
+
+ private createVAO(): WebGLVertexArrayObject {
+ const vao = this.gl.createVertexArray();
+ this.gl.bindVertexArray(vao);
+
+ const vertices = new Float32Array([
+ -1, -1, 0, 1,
+ 1, -1, 1, 1,
+ -1, 1, 0, 0,
+ 1, 1, 1, 0,
+ ]);
+
+ const buffer = this.gl.createBuffer();
+ this.gl.bindBuffer(this.gl.ARRAY_BUFFER, buffer);
+ this.gl.bufferData(this.gl.ARRAY_BUFFER, vertices, this.gl.STATIC_DRAW);
+
+ const positionLocation = this.gl.getAttribLocation(this.program, 'a_position');
+ const texCoordLocation = this.gl.getAttribLocation(this.program, 'a_texCoord');
+
+ this.gl.enableVertexAttribArray(positionLocation);
+ this.gl.vertexAttribPointer(positionLocation, 2, this.gl.FLOAT, false, 16, 0);
+
+ this.gl.enableVertexAttribArray(texCoordLocation);
+ this.gl.vertexAttribPointer(texCoordLocation, 2, this.gl.FLOAT, false, 16, 8);
+
+ return vao;
+ }
+
+ private createTexture(): WebGLTexture {
+ const texture = this.gl.createTexture();
+
+ this.gl.bindTexture(this.gl.TEXTURE_2D, texture);
+ this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_S, this.gl.CLAMP_TO_EDGE);
+ this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_T, this.gl.CLAMP_TO_EDGE);
+ this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MIN_FILTER, this.gl.LINEAR);
+ this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MAG_FILTER, this.gl.LINEAR);
+
+ return texture;
+ }
+
+ update(color: VideoFrame, alpha: VideoFrame): void {
+ if (color.displayWidth !== this.canvas.width || color.displayHeight !== this.canvas.height) {
+ this.canvas.width = color.displayWidth;
+ this.canvas.height = color.displayHeight;
+ }
+
+ this.gl.activeTexture(this.gl.TEXTURE0);
+ this.gl.bindTexture(this.gl.TEXTURE_2D, this.colorTexture);
+ this.gl.texImage2D(this.gl.TEXTURE_2D, 0, this.gl.RGBA, this.gl.RGBA, this.gl.UNSIGNED_BYTE, color);
+
+ this.gl.activeTexture(this.gl.TEXTURE1);
+ this.gl.bindTexture(this.gl.TEXTURE_2D, this.alphaTexture);
+ this.gl.texImage2D(this.gl.TEXTURE_2D, 0, this.gl.RGBA, this.gl.RGBA, this.gl.UNSIGNED_BYTE, alpha);
+
+ this.gl.viewport(0, 0, this.canvas.width, this.canvas.height);
+ this.gl.clear(this.gl.COLOR_BUFFER_BIT);
+
+ this.gl.bindVertexArray(this.vao);
+ this.gl.drawArrays(this.gl.TRIANGLE_STRIP, 0, 4);
+ }
+
+ close() {
+ this.gl.getExtension('WEBGL_lose_context')?.loseContext();
+ this.gl = null as unknown as WebGL2RenderingContext;
+ }
+}
+
+export class AudioDecoderWrapper extends DecoderWrapper {
+ decoder: AudioDecoder | null = null;
+
+ customDecoder: CustomAudioDecoder | null = null;
+ customDecoderCallSerializer = new CallSerializer();
+ customDecoderQueueSize = 0;
+ customDecoderClosed = false;
+
+ // Internal state to accumulate a precise current timestamp based on audio durations, not the (potentially
+ // inaccurate) packet timestamps.
+ currentTimestamp: number | null = null;
+
+ onDequeue: (() => unknown) | null = null;
+
+ constructor(
+ onSample: (sample: AudioSample) => unknown,
+ onError: (error: Error) => unknown,
+ codec: AudioCodec,
+ decoderConfig: AudioDecoderConfig,
+ ) {
+ super(onSample, onError);
+
+ const sampleHandler = (sample: AudioSample) => {
+ if (
+ this.currentTimestamp === null
+ || Math.abs(sample.timestamp - this.currentTimestamp) >= sample.duration
+ ) {
+ // We need to sync with the sample timestamp again
+ this.currentTimestamp = sample.timestamp;
+ }
+
+ const preciseTimestamp = this.currentTimestamp;
+ this.currentTimestamp += sample.duration;
+
+ if (sample.numberOfFrames === 0) {
+ // We skip zero-data (empty) AudioSamples. These are sometimes emitted, for example, by Firefox when it
+ // decodes Vorbis (at the start).
+ sample.close();
+ return;
+ }
+
+ // Round the timestamp to the sample rate
+ const sampleRate = decoderConfig.sampleRate;
+ sample.setTimestamp(Math.round(preciseTimestamp * sampleRate) / sampleRate);
+
+ onSample(sample);
+ };
+
+ const MatchingCustomDecoder = customAudioDecoders.find(x => x.supports(codec, decoderConfig));
+ if (MatchingCustomDecoder) {
+ // @ts-expect-error "Can't create instance of abstract class 🤓"
+ this.customDecoder = new MatchingCustomDecoder() as CustomAudioDecoder;
+ // @ts-expect-error It's technically readonly
+ this.customDecoder.codec = codec;
+ // @ts-expect-error It's technically readonly
+ this.customDecoder.config = decoderConfig;
+ // @ts-expect-error It's technically readonly
+ this.customDecoder.onSample = (sample) => {
+ if (!(sample instanceof AudioSample)) {
+ throw new TypeError('The argument passed to onSample must be an AudioSample.');
+ }
+
+ sampleHandler(sample);
+ };
+
+ void this.customDecoderCallSerializer.call(() => this.customDecoder!.init());
+ } else {
+ this.decoder = new AudioDecoder({
+ output: (data) => {
+ try {
+ sampleHandler(new AudioSample(data));
+ } catch (error) {
+ this.onError(error as Error);
+ }
+ },
+ error: onError,
+ });
+ this.decoder.configure(decoderConfig);
+
+ this.decoder.addEventListener('dequeue', () => {
+ this.onDequeue?.();
+ });
+ }
+ }
+
+ getDecodeQueueSize() {
+ if (this.customDecoder) {
+ return this.customDecoderQueueSize;
+ } else {
+ assert(this.decoder);
+ return this.decoder.decodeQueueSize;
+ }
+ }
+
+ decode(packet: EncodedPacket) {
+ if (this.customDecoder) {
+ this.customDecoderQueueSize++;
+ void this.customDecoderCallSerializer
+ .call(() => this.customDecoder!.decode(packet))
+ .then(() => {
+ this.customDecoderQueueSize--;
+ this.onDequeue?.();
+ });
+ } else {
+ assert(this.decoder);
+ this.decoder.decode(packet.toEncodedAudioChunk());
+ }
+ }
+
+ flush() {
+ if (this.customDecoder) {
+ return this.customDecoderCallSerializer.call(() => this.customDecoder!.flush());
+ } else {
+ assert(this.decoder);
+ return this.decoder.flush();
+ }
+ }
+
+ close() {
+ if (this.customDecoder) {
+ if (!this.customDecoderClosed) {
+ this.customDecoderClosed = true;
+ void this.customDecoderCallSerializer.call(() => this.customDecoder!.close());
+ }
+ } else {
+ assert(this.decoder);
+
+ if (this.decoder.state !== 'closed') {
+ this.decoder.close();
+ }
+ }
+ }
+
+ get closed() {
+ if (this.customDecoder) {
+ if (this.customDecoderClosed) {
+ return true;
+ }
+
+ return !this.customDecoderCallSerializer.errored;
+ } else {
+ assert(this.decoder);
+ return this.decoder.state === 'closed';
+ }
+ }
+}
+
+// There are a lot of PCM variants not natively supported by the browser and by AudioData. Therefore we need a simple
+// decoder that maps any input PCM format into a PCM format supported by the browser.
+export class PcmAudioDecoderWrapper extends DecoderWrapper {
+ codec: PcmAudioCodec;
+
+ inputSampleSize: 1 | 2 | 3 | 4 | 8;
+ readInputValue: (view: DataView, byteOffset: number) => number;
+
+ outputSampleSize: 1 | 2 | 4;
+ outputFormat: 'u8' | 's16' | 's32' | 'f32';
+ writeOutputValue: (view: DataView, byteOffset: number, value: number) => void;
+
+ // Internal state to accumulate a precise current timestamp based on audio durations, not the (potentially
+ // inaccurate) packet timestamps.
+ currentTimestamp: number | null = null;
+
+ isClosed = false;
+
+ onDequeue: (() => unknown) | null = null;
+
+ constructor(
+ onSample: (sample: AudioSample) => unknown,
+ onError: (error: Error) => unknown,
+ public decoderConfig: AudioDecoderConfig,
+ ) {
+ super(onSample, onError);
+
+ assert((PCM_AUDIO_CODECS as readonly string[]).includes(decoderConfig.codec));
+ this.codec = decoderConfig.codec as PcmAudioCodec;
+
+ const { dataType, sampleSize, littleEndian } = parsePcmCodec(this.codec);
+ this.inputSampleSize = sampleSize;
+
+ switch (sampleSize) {
+ case 1: {
+ if (dataType === 'unsigned') {
+ this.readInputValue = (view, byteOffset) => view.getUint8(byteOffset) - 2 ** 7;
+ } else if (dataType === 'signed') {
+ this.readInputValue = (view, byteOffset) => view.getInt8(byteOffset);
+ } else if (dataType === 'ulaw') {
+ this.readInputValue = (view, byteOffset) => fromUlaw(view.getUint8(byteOffset));
+ } else if (dataType === 'alaw') {
+ this.readInputValue = (view, byteOffset) => fromAlaw(view.getUint8(byteOffset));
+ } else {
+ assert(false);
+ }
+ }; break;
+ case 2: {
+ if (dataType === 'unsigned') {
+ this.readInputValue = (view, byteOffset) => view.getUint16(byteOffset, littleEndian) - 2 ** 15;
+ } else if (dataType === 'signed') {
+ this.readInputValue = (view, byteOffset) => view.getInt16(byteOffset, littleEndian);
+ } else {
+ assert(false);
+ }
+ }; break;
+ case 3: {
+ if (dataType === 'unsigned') {
+ this.readInputValue = (view, byteOffset) => getUint24(view, byteOffset, littleEndian) - 2 ** 23;
+ } else if (dataType === 'signed') {
+ this.readInputValue = (view, byteOffset) => getInt24(view, byteOffset, littleEndian);
+ } else {
+ assert(false);
+ }
+ }; break;
+ case 4: {
+ if (dataType === 'unsigned') {
+ this.readInputValue = (view, byteOffset) => view.getUint32(byteOffset, littleEndian) - 2 ** 31;
+ } else if (dataType === 'signed') {
+ this.readInputValue = (view, byteOffset) => view.getInt32(byteOffset, littleEndian);
+ } else if (dataType === 'float') {
+ this.readInputValue = (view, byteOffset) => view.getFloat32(byteOffset, littleEndian);
+ } else {
+ assert(false);
+ }
+ }; break;
+ case 8: {
+ if (dataType === 'float') {
+ this.readInputValue = (view, byteOffset) => view.getFloat64(byteOffset, littleEndian);
+ } else {
+ assert(false);
+ }
+ }; break;
+ default: {
+ assertNever(sampleSize);
+ assert(false);
+ };
+ }
+
+ switch (sampleSize) {
+ case 1: {
+ if (dataType === 'ulaw' || dataType === 'alaw') {
+ this.outputSampleSize = 2;
+ this.outputFormat = 's16';
+ this.writeOutputValue = (view, byteOffset, value) => view.setInt16(byteOffset, value, true);
+ } else {
+ this.outputSampleSize = 1;
+ this.outputFormat = 'u8';
+ this.writeOutputValue = (view, byteOffset, value) => view.setUint8(byteOffset, value + 2 ** 7);
+ }
+ }; break;
+ case 2: {
+ this.outputSampleSize = 2;
+ this.outputFormat = 's16';
+ this.writeOutputValue = (view, byteOffset, value) => view.setInt16(byteOffset, value, true);
+ }; break;
+ case 3: {
+ this.outputSampleSize = 4;
+ this.outputFormat = 's32';
+ // From https://www.w3.org/TR/webcodecs:
+ // AudioData containing 24-bit samples SHOULD store those samples in s32 or f32. When samples are
+ // stored in s32, each sample MUST be left-shifted by 8 bits.
+ this.writeOutputValue = (view, byteOffset, value) => view.setInt32(byteOffset, value << 8, true);
+ }; break;
+ case 4: {
+ this.outputSampleSize = 4;
+
+ if (dataType === 'float') {
+ this.outputFormat = 'f32';
+ this.writeOutputValue = (view, byteOffset, value) => view.setFloat32(byteOffset, value, true);
+ } else {
+ this.outputFormat = 's32';
+ this.writeOutputValue = (view, byteOffset, value) => view.setInt32(byteOffset, value, true);
+ }
+ }; break;
+ case 8: {
+ this.outputSampleSize = 4;
+
+ this.outputFormat = 'f32';
+ this.writeOutputValue = (view, byteOffset, value) => view.setFloat32(byteOffset, value, true);
+ }; break;
+ default: {
+ assertNever(sampleSize);
+ assert(false);
+ };
+ };
+ }
+
+ getDecodeQueueSize() {
+ return 0;
+ }
+
+ decode(packet: EncodedPacket) {
+ this.onDequeue?.();
+
+ const inputView = toDataView(packet.data);
+
+ const numberOfFrames = packet.byteLength / this.decoderConfig.numberOfChannels / this.inputSampleSize;
+
+ const outputBufferSize = numberOfFrames * this.decoderConfig.numberOfChannels * this.outputSampleSize;
+ const outputBuffer = new ArrayBuffer(outputBufferSize);
+ const outputView = new DataView(outputBuffer);
+
+ for (let i = 0; i < numberOfFrames * this.decoderConfig.numberOfChannels; i++) {
+ const inputIndex = i * this.inputSampleSize;
+ const outputIndex = i * this.outputSampleSize;
+
+ const value = this.readInputValue(inputView, inputIndex);
+ this.writeOutputValue(outputView, outputIndex, value);
+ }
+
+ const preciseDuration = numberOfFrames / this.decoderConfig.sampleRate;
+ if (this.currentTimestamp === null || Math.abs(packet.timestamp - this.currentTimestamp) >= preciseDuration) {
+ // We need to sync with the packet timestamp again
+ this.currentTimestamp = packet.timestamp;
+ }
+
+ const preciseTimestamp = this.currentTimestamp;
+ this.currentTimestamp += preciseDuration;
+
+ const audioSample = new AudioSample({
+ format: this.outputFormat,
+ data: outputBuffer,
+ numberOfChannels: this.decoderConfig.numberOfChannels,
+ sampleRate: this.decoderConfig.sampleRate,
+ numberOfFrames,
+ timestamp: preciseTimestamp,
+ });
+
+ this.onSample(audioSample);
+ }
+
+ async flush() {
+ // Do nothing
+ }
+
+ close() {
+ this.isClosed = true;
+ }
+
+ get closed() {
+ return this.isClosed;
+ }
+}
diff --git a/src/flac/flac-demuxer.ts b/src/flac/flac-demuxer.ts
index 1c06e73..bf2b51e 100644
--- a/src/flac/flac-demuxer.ts
+++ b/src/flac/flac-demuxer.ts
@@ -10,7 +10,6 @@ import { FlacBlockType, readVorbisComments } from '../codec-data';
import { Demuxer } from '../demuxer';
import { Input } from '../input';
import { InputAudioTrack, InputAudioTrackBacking } from '../input-track';
-import { PacketRetrievalOptions } from '../media-sink';
import {
assert,
AsyncMutex4,
@@ -39,6 +38,7 @@ import {
readSampleRate,
getSampleRateOrUncommon,
} from './flac-misc';
+import { PacketRetrievalOptions } from '../cursors';
type FlacAudioInfo = {
numberOfChannels: number;
diff --git a/src/index.ts b/src/index.ts
index 0a96687..b6953d2 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -170,18 +170,6 @@ export {
VideoSampleInit,
CropRectangle,
} from './sample';
-export {
- AudioBufferSink,
- AudioSampleSink,
- BaseMediaSampleSink,
- CanvasSink,
- CanvasSinkOptions,
- EncodedPacketSink,
- PacketRetrievalOptions,
- VideoSampleSink,
- WrappedAudioBuffer,
- WrappedCanvas,
-} from './media-sink';
export {
PacketReader,
PacketCursor,
diff --git a/src/input-track.ts b/src/input-track.ts
index c9c61ea..f3eb98b 100644
--- a/src/input-track.ts
+++ b/src/input-track.ts
@@ -10,11 +10,11 @@ import { AudioCodec, MediaCodec, VideoCodec } from './codec';
import { determineVideoPacketType } from './codec-data';
import { customAudioDecoders, customVideoDecoders } from './custom-coder';
import { Input } from './input';
-import { EncodedPacketSink, PacketRetrievalOptions } from './media-sink';
import { assert, ResultValue, Rotation, Yo } from './misc';
import { TrackType } from './output';
import { EncodedPacket, PacketType } from './packet';
import { TrackDisposition } from './metadata';
+import { PacketCursor, PacketRetrievalOptions } from './cursors';
/**
* Contains aggregate statistics about the encoded packets of a track.
@@ -183,20 +183,20 @@ export abstract class InputTrack {
* entire file.
*/
async computePacketStats(targetPacketCount = Infinity): Promise {
- const sink = new EncodedPacketSink(this);
+ const cursor = new PacketCursor(this, { metadataOnly: true });
let startTimestamp = Infinity;
let endTimestamp = -Infinity;
let packetCount = 0;
let totalPacketBytes = 0;
- for await (const packet of sink.packets(undefined, undefined, { metadataOnly: true })) {
+ await cursor.iterate((packet) => {
if (
packetCount >= targetPacketCount
// This additional condition is needed to produce correct results with out-of-presentation-order packets
&& packet.timestamp >= endTimestamp
) {
- break;
+ return false;
}
startTimestamp = Math.min(startTimestamp, packet.timestamp);
@@ -204,7 +204,7 @@ export abstract class InputTrack {
packetCount++;
totalPacketBytes += packet.byteLength;
- }
+ });
return {
packetCount,
diff --git a/src/isobmff/isobmff-demuxer.ts b/src/isobmff/isobmff-demuxer.ts
index 2797ac3..d6d429b 100644
--- a/src/isobmff/isobmff-demuxer.ts
+++ b/src/isobmff/isobmff-demuxer.ts
@@ -38,7 +38,6 @@ import {
InputVideoTrack,
InputVideoTrackBacking,
} from '../input-track';
-import { PacketRetrievalOptions } from '../media-sink';
import {
assert,
binarySearchExact,
@@ -89,6 +88,7 @@ import {
readAscii,
} from '../reader';
import { DEFAULT_TRACK_DISPOSITION, MetadataTags, RichImageData, TrackDisposition } from '../metadata';
+import { PacketRetrievalOptions } from '../cursors';
type InternalTrack = {
id: number;
diff --git a/src/matroska/matroska-demuxer.ts b/src/matroska/matroska-demuxer.ts
index 1135cf6..97b617e 100644
--- a/src/matroska/matroska-demuxer.ts
+++ b/src/matroska/matroska-demuxer.ts
@@ -32,7 +32,6 @@ import {
InputVideoTrackBacking,
} from '../input-track';
import { AttachedFile, DEFAULT_TRACK_DISPOSITION, MetadataTags, TrackDisposition } from '../metadata';
-import { PacketRetrievalOptions } from '../media-sink';
import {
assert,
binarySearchLessOrEqual,
@@ -71,6 +70,7 @@ import {
} from './ebml';
import { buildMatroskaMimeType } from './matroska-misc';
import { FileSlice, readBytes, Reader, readI16Be, readU8 } from '../reader';
+import { PacketRetrievalOptions } from '../cursors';
type Segment = {
seekHeadSeen: boolean;
diff --git a/src/media-sink.ts b/src/media-sink.ts
deleted file mode 100644
index 622b39e..0000000
--- a/src/media-sink.ts
+++ /dev/null
@@ -1,2272 +0,0 @@
-/*!
- * Copyright (c) 2025-present, Vanilagy and contributors
- *
- * This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at https://mozilla.org/MPL/2.0/.
- */
-
-import { parsePcmCodec, PCM_AUDIO_CODECS, PcmAudioCodec, VideoCodec, AudioCodec } from './codec';
-import {
- deserializeAvcDecoderConfigurationRecord,
- determineVideoPacketType,
- extractHevcNalUnits,
- extractNalUnitTypeForHevc,
- HevcNalUnitType,
- parseAvcSps,
-} from './codec-data';
-import { CustomVideoDecoder, customVideoDecoders, CustomAudioDecoder, customAudioDecoders } from './custom-coder';
-import { InputDisposedError } from './input';
-import { InputAudioTrack, InputTrack, InputVideoTrack } from './input-track';
-import {
- AnyIterable,
- assert,
- assertNever,
- CallSerializer,
- getInt24,
- getUint24,
- insertSorted,
- isChromium,
- isFirefox,
- isNumber,
- isWebKit,
- last,
- mapAsyncGenerator,
- MaybePromise,
- promiseWithResolvers,
- ResultValue,
- Rotation,
- toAsyncIterator,
- toDataView,
- toUint8Array,
- validateAnyIterable,
- Yo,
-} from './misc';
-import { EncodedPacket } from './packet';
-import { fromAlaw, fromUlaw } from './pcm';
-import { AudioSample, clampCropRectangle, CropRectangle, validateCropRectangle, VideoSample } from './sample';
-
-/**
- * Additional options for controlling packet retrieval.
- * @group Media sinks
- * @public
- */
-export type PacketRetrievalOptions = {
- /**
- * When set to `true`, only packet metadata (like timestamp) will be retrieved - the actual packet data will not
- * be loaded.
- */
- metadataOnly?: boolean;
-
- /**
- * When set to true, key packets will be verified upon retrieval by looking into the packet's bitstream.
- * If not enabled, the packet types will be determined solely by what's stored in the containing file and may be
- * incorrect, potentially leading to decoder errors. Since determining a packet's actual type requires looking into
- * its data, this option cannot be enabled together with `metadataOnly`.
- */
- verifyKeyPackets?: boolean;
-};
-
-export const validatePacketRetrievalOptions = (options: PacketRetrievalOptions) => {
- if (!options || typeof options !== 'object') {
- throw new TypeError('options must be an object.');
- }
- if (options.metadataOnly !== undefined && typeof options.metadataOnly !== 'boolean') {
- throw new TypeError('options.metadataOnly, when defined, must be a boolean.');
- }
- if (options.verifyKeyPackets !== undefined && typeof options.verifyKeyPackets !== 'boolean') {
- throw new TypeError('options.verifyKeyPackets, when defined, must be a boolean.');
- }
- if (options.verifyKeyPackets && options.metadataOnly) {
- throw new TypeError('options.verifyKeyPackets and options.metadataOnly cannot be enabled together.');
- }
-};
-
-export const validateTimestamp = (timestamp: number) => {
- if (!isNumber(timestamp)) {
- throw new TypeError('timestamp must be a number.'); // It can be non-finite, that's fine
- }
-};
-
-const maybeFixPacketType = (
- track: InputTrack,
- promise: Promise,
- options: PacketRetrievalOptions,
-) => {
- if (options.verifyKeyPackets) {
- return promise.then(async (packet) => {
- if (!packet || packet.type === 'delta') {
- return packet;
- }
-
- const determinedType = await track.determinePacketType(packet);
- if (determinedType) {
- // @ts-expect-error Technically readonly
- packet.type = determinedType;
- }
-
- return packet;
- });
- } else {
- return promise;
- }
-};
-
-/**
- * Sink for retrieving encoded packets from an input track.
- * @group Media sinks
- * @public
- */
-export class EncodedPacketSink {
- /** @internal */
- _track: InputTrack;
-
- /** Creates a new {@link EncodedPacketSink} for the given {@link InputTrack}. */
- constructor(track: InputTrack) {
- if (!(track instanceof InputTrack)) {
- throw new TypeError('track must be an InputTrack.');
- }
-
- this._track = track;
- }
-
- /**
- * Retrieves the track's first packet (in decode order), or null if it has no packets. The first packet is very
- * likely to be a key packet.
- */
- getFirstPacket(options: PacketRetrievalOptions = {}): MaybePromise {
- validatePacketRetrievalOptions(options);
-
- if (this._track.input._disposed) {
- throw new InputDisposedError();
- }
-
- const result = new ResultValue();
- const promise = this._track._backing.getFirstPacket(result, options);
- if (result.pending) return (promise as Promise).then(() => result.value);
-
- return result.value;
-
- // return maybeFixPacketType(this._track, this._track._backing.getFirstPacket(options), options);
- }
-
- /**
- * Retrieves the packet corresponding to the given timestamp, in seconds. More specifically, returns the last packet
- * (in presentation order) with a start timestamp less than or equal to the given timestamp. This method can be
- * used to retrieve a track's last packet using `getPacket(Infinity)`. The method returns null if the timestamp
- * is before the first packet in the track.
- *
- * @param timestamp - The timestamp used for retrieval, in seconds.
- */
- getPacket(timestamp: number, options: PacketRetrievalOptions = {}) {
- validateTimestamp(timestamp);
- validatePacketRetrievalOptions(options);
-
- if (this._track.input._disposed) {
- throw new InputDisposedError();
- }
-
- return maybeFixPacketType(this._track, this._track._backing.getPacket(timestamp, options), options);
- }
-
- /**
- * Retrieves the packet following the given packet (in decode order), or null if the given packet is the
- * last packet.
- */
- getNextPacket(packet: EncodedPacket, options: PacketRetrievalOptions = {}) {
- if (!(packet instanceof EncodedPacket)) {
- throw new TypeError('packet must be an EncodedPacket.');
- }
- validatePacketRetrievalOptions(options);
-
- if (this._track.input._disposed) {
- throw new InputDisposedError();
- }
-
- const result = new ResultValue();
- const promise = this._track._backing.getNextPacket(result, packet, options);
- if (result.pending) return (promise as Promise).then(() => result.value);
-
- return result.value;
-
- // return maybeFixPacketType(this._track, this._track._backing.getNextPacket(packet, options), options);
- }
-
- /**
- * Retrieves the key packet corresponding to the given timestamp, in seconds. More specifically, returns the last
- * key packet (in presentation order) with a start timestamp less than or equal to the given timestamp. A key packet
- * is a packet that doesn't require previous packets to be decoded. This method can be used to retrieve a track's
- * last key packet using `getKeyPacket(Infinity)`. The method returns null if the timestamp is before the first
- * key packet in the track.
- *
- * To ensure that the returned packet is guaranteed to be a real key frame, enable `options.verifyKeyPackets`.
- *
- * @param timestamp - The timestamp used for retrieval, in seconds.
- */
- async getKeyPacket(timestamp: number, options: PacketRetrievalOptions = {}): Promise {
- validateTimestamp(timestamp);
- validatePacketRetrievalOptions(options);
-
- if (this._track.input._disposed) {
- throw new InputDisposedError();
- }
-
- if (!options.verifyKeyPackets) {
- return this._track._backing.getKeyPacket(timestamp, options);
- }
-
- const packet = await this._track._backing.getKeyPacket(timestamp, options);
- if (!packet) {
- return packet;
- }
- assert(packet.type === 'key');
-
- const determinedType = await this._track.determinePacketType(packet);
- if (determinedType === 'delta') {
- // Try returning the previous key packet (in hopes that it's actually a key packet)
- return this.getKeyPacket(packet.timestamp - 1 / this._track.timeResolution, options);
- }
-
- return packet;
- }
-
- /**
- * Retrieves the key packet following the given packet (in decode order), or null if the given packet is the last
- * key packet.
- *
- * To ensure that the returned packet is guaranteed to be a real key frame, enable `options.verifyKeyPackets`.
- */
- async getNextKeyPacket(packet: EncodedPacket, options: PacketRetrievalOptions = {}): Promise {
- if (!(packet instanceof EncodedPacket)) {
- throw new TypeError('packet must be an EncodedPacket.');
- }
- validatePacketRetrievalOptions(options);
-
- if (this._track.input._disposed) {
- throw new InputDisposedError();
- }
-
- if (!options.verifyKeyPackets) {
- return this._track._backing.getNextKeyPacket(packet, options);
- }
-
- const nextPacket = await this._track._backing.getNextKeyPacket(packet, options);
- if (!nextPacket) {
- return nextPacket;
- }
- assert(nextPacket.type === 'key');
-
- const determinedType = await this._track.determinePacketType(nextPacket);
- if (determinedType === 'delta') {
- // Try returning the next key packet (in hopes that it's actually a key packet)
- return this.getNextKeyPacket(nextPacket, options);
- }
-
- return nextPacket;
- }
-
- /**
- * Creates an async iterator that yields the packets in this track in decode order. To enable fast iteration, this
- * method will intelligently preload packets based on the speed of the consumer.
- *
- * @param startPacket - (optional) The packet from which iteration should begin. This packet will also be yielded.
- * @param endTimestamp - (optional) The timestamp at which iteration should end. This packet will _not_ be yielded.
- */
- packets(
- startPacket?: EncodedPacket,
- endPacket?: EncodedPacket,
- options: PacketRetrievalOptions = {},
- ): AsyncGenerator {
- if (startPacket !== undefined && !(startPacket instanceof EncodedPacket)) {
- throw new TypeError('startPacket must be an EncodedPacket.');
- }
- if (startPacket !== undefined && startPacket.isMetadataOnly && !options?.metadataOnly) {
- throw new TypeError('startPacket can only be metadata-only if options.metadataOnly is enabled.');
- }
- if (endPacket !== undefined && !(endPacket instanceof EncodedPacket)) {
- throw new TypeError('endPacket must be an EncodedPacket.');
- }
- validatePacketRetrievalOptions(options);
-
- if (this._track.input._disposed) {
- throw new InputDisposedError();
- }
-
- const packetQueue: EncodedPacket[] = [];
-
- let { promise: queueNotEmpty, resolve: onQueueNotEmpty } = promiseWithResolvers();
- let { promise: queueDequeue, resolve: onQueueDequeue } = promiseWithResolvers();
- let ended = false;
- let terminated = false;
-
- // This stores errors that are "out of band" in the sense that they didn't occur in the normal flow of this
- // method but instead in a different context. This error should not go unnoticed and must be bubbled up to
- // the consumer.
- let outOfBandError = null as Error | null;
-
- const timestamps: number[] = [];
- // The queue should always be big enough to hold 1 second worth of packets
- const maxQueueSize = () => Math.max(2, timestamps.length);
-
- // The following is the "pump" process that keeps pumping packets into the queue
- (async () => {
- let packet = startPacket ?? await this.getFirstPacket(options);
-
- while (packet && !terminated && !this._track.input._disposed) {
- if (endPacket && packet.sequenceNumber >= endPacket?.sequenceNumber) {
- break;
- }
-
- if (packetQueue.length > maxQueueSize()) {
- ({ promise: queueDequeue, resolve: onQueueDequeue } = promiseWithResolvers());
- await queueDequeue;
- continue;
- }
-
- packetQueue.push(packet);
-
- onQueueNotEmpty();
- ({ promise: queueNotEmpty, resolve: onQueueNotEmpty } = promiseWithResolvers());
-
- packet = await this.getNextPacket(packet, options);
- }
-
- ended = true;
- onQueueNotEmpty();
- })().catch((error: Error) => {
- if (!outOfBandError) {
- outOfBandError = error;
- onQueueNotEmpty();
- }
- });
-
- const track = this._track;
-
- return {
- async next() {
- while (true) {
- if (track.input._disposed) {
- throw new InputDisposedError();
- } else if (terminated) {
- return { value: undefined, done: true };
- } else if (outOfBandError) {
- throw outOfBandError;
- } else if (packetQueue.length > 0) {
- const value = packetQueue.shift()!;
- const now = performance.now();
- timestamps.push(now);
-
- while (timestamps.length > 0 && now - timestamps[0]! >= 1000) {
- timestamps.shift();
- }
-
- onQueueDequeue();
-
- return { value, done: false };
- } else if (ended) {
- return { value: undefined, done: true };
- } else {
- await queueNotEmpty;
- }
- }
- },
- async return() {
- terminated = true;
- onQueueDequeue();
- onQueueNotEmpty();
-
- return { value: undefined, done: true };
- },
- async throw(error) {
- throw error;
- },
- [Symbol.asyncIterator]() {
- return this;
- },
- };
- }
-}
-
-export abstract class DecoderWrapper<
- MediaSample extends VideoSample | AudioSample,
-> {
- constructor(
- public onSample: (sample: MediaSample) => unknown,
- public onError: (error: Error) => unknown,
- ) {}
-
- abstract getDecodeQueueSize(): number;
- abstract decode(packet: EncodedPacket): void;
- abstract flush(): Promise;
- abstract close(): void;
-
- abstract get closed(): boolean;
-}
-
-/**
- * Base class for decoded media sample sinks.
- * @group Media sinks
- * @public
- */
-export abstract class BaseMediaSampleSink<
- MediaSample extends VideoSample | AudioSample,
-> {
- /** @internal */
- abstract _track: InputTrack;
-
- /** @internal */
- abstract _createDecoder(
- onSample: (sample: MediaSample) => unknown,
- onError: (error: Error) => unknown
- ): Promise>;
- /** @internal */
- abstract _createPacketSink(): EncodedPacketSink;
-
- /** @internal */
- protected mediaSamplesInRange(
- startTimestamp = 0,
- endTimestamp = Infinity,
- ): AsyncGenerator {
- validateTimestamp(startTimestamp);
- validateTimestamp(endTimestamp);
-
- const sampleQueue: MediaSample[] = [];
- let firstSampleQueued = false;
- let lastSample: MediaSample | null = null;
- let { promise: queueNotEmpty, resolve: onQueueNotEmpty } = promiseWithResolvers();
- let { promise: queueDequeue, resolve: onQueueDequeue } = promiseWithResolvers();
- let decoderIsFlushed = false;
- let ended = false;
- let terminated = false;
-
- // This stores errors that are "out of band" in the sense that they didn't occur in the normal flow of this
- // method but instead in a different context. This error should not go unnoticed and must be bubbled up to
- // the consumer.
- let outOfBandError = null as Error | null;
-
- // The following is the "pump" process that keeps pumping packets into the decoder
- (async () => {
- const decoderError = new Error();
- const decoder = await this._createDecoder((sample) => {
- onQueueDequeue();
- if (sample.timestamp >= endTimestamp) {
- ended = true;
- }
-
- if (ended) {
- sample.close();
- return;
- }
-
- if (lastSample) {
- if (sample.timestamp > startTimestamp) {
- // We don't know ahead of time what the first first is. This is because the first first is the
- // last first whose timestamp is less than or equal to the start timestamp. Therefore we need to
- // wait for the first first after the start timestamp, and then we'll know that the previous
- // first was the first first.
- sampleQueue.push(lastSample);
- firstSampleQueued = true;
- } else {
- lastSample.close();
- }
- }
-
- if (sample.timestamp >= startTimestamp) {
- sampleQueue.push(sample);
- firstSampleQueued = true;
- }
-
- lastSample = firstSampleQueued ? null : sample;
-
- if (sampleQueue.length > 0) {
- onQueueNotEmpty();
- ({ promise: queueNotEmpty, resolve: onQueueNotEmpty } = promiseWithResolvers());
- }
- }, (error) => {
- if (!outOfBandError) {
- error.stack = decoderError.stack; // Provide a more useful stack trace
- outOfBandError = error;
- onQueueNotEmpty();
- }
- });
-
- const packetSink = this._createPacketSink();
- const keyPacket = await packetSink.getKeyPacket(startTimestamp, { verifyKeyPackets: true })
- ?? await packetSink.getFirstPacket();
-
- let currentPacket: EncodedPacket | null = keyPacket;
-
- let endPacket: EncodedPacket | undefined = undefined;
- if (endTimestamp < Infinity) {
- // When an end timestamp is set, we cannot simply use that for the packet iterator due to out-of-order
- // frames (B-frames). Instead, we'll need to keep decoding packets until we get a frame that exceeds
- // this end time. However, we can still put a bound on it: Since key frames are by definition never
- // out of order, we can stop at the first key frame after the end timestamp.
- const packet = await packetSink.getPacket(endTimestamp);
- const keyPacket = !packet
- ? null
- : packet.type === 'key' && packet.timestamp === endTimestamp
- ? packet
- : await packetSink.getNextKeyPacket(packet, { verifyKeyPackets: true });
-
- if (keyPacket) {
- endPacket = keyPacket;
- }
- }
-
- const packets = packetSink.packets(keyPacket ?? undefined, endPacket);
- await packets.next(); // Skip the start packet as we already have it
-
- while (currentPacket && !ended && !this._track.input._disposed) {
- const maxQueueSize = computeMaxQueueSize(sampleQueue.length);
- if (sampleQueue.length + decoder.getDecodeQueueSize() > maxQueueSize) {
- ({ promise: queueDequeue, resolve: onQueueDequeue } = promiseWithResolvers());
- await queueDequeue;
- continue;
- }
-
- decoder.decode(currentPacket);
-
- const packetResult = await packets.next();
- if (packetResult.done) {
- break;
- }
-
- currentPacket = packetResult.value;
- }
-
- await packets.return();
-
- if (!terminated && !this._track.input._disposed) {
- await decoder.flush();
- }
- decoder.close();
-
- if (!firstSampleQueued && lastSample) {
- sampleQueue.push(lastSample);
- }
-
- decoderIsFlushed = true;
- onQueueNotEmpty(); // To unstuck the generator
- })().catch((error: Error) => {
- if (!outOfBandError) {
- outOfBandError = error;
- onQueueNotEmpty();
- }
- });
-
- const track = this._track;
- const closeSamples = () => {
- lastSample?.close();
- for (const sample of sampleQueue) {
- sample.close();
- }
- };
-
- return {
- async next() {
- while (true) {
- if (track.input._disposed) {
- closeSamples();
- throw new InputDisposedError();
- } else if (terminated) {
- return { value: undefined, done: true };
- } else if (outOfBandError) {
- closeSamples();
- throw outOfBandError;
- } else if (sampleQueue.length > 0) {
- const value = sampleQueue.shift()!;
- onQueueDequeue();
- return { value, done: false };
- } else if (!decoderIsFlushed) {
- await queueNotEmpty;
- } else {
- return { value: undefined, done: true };
- }
- }
- },
- async return() {
- terminated = true;
- ended = true;
- onQueueDequeue();
- onQueueNotEmpty();
- closeSamples();
-
- return { value: undefined, done: true };
- },
- async throw(error) {
- throw error;
- },
- [Symbol.asyncIterator]() {
- return this;
- },
- };
- }
-
- /** @internal */
- protected mediaSamplesAtTimestamps(
- timestamps: AnyIterable,
- ): AsyncGenerator {
- validateAnyIterable(timestamps);
- const timestampIterator = toAsyncIterator(timestamps);
- const timestampsOfInterest: number[] = [];
-
- const sampleQueue: (MediaSample | null)[] = [];
- let { promise: queueNotEmpty, resolve: onQueueNotEmpty } = promiseWithResolvers();
- let { promise: queueDequeue, resolve: onQueueDequeue } = promiseWithResolvers();
- let decoderIsFlushed = false;
- let terminated = false;
-
- // This stores errors that are "out of band" in the sense that they didn't occur in the normal flow of this
- // method but instead in a different context. This error should not go unnoticed and must be bubbled up to
- // the consumer.
- let outOfBandError = null as Error | null;
-
- const pushToQueue = (sample: MediaSample | null) => {
- sampleQueue.push(sample);
- onQueueNotEmpty();
- ({ promise: queueNotEmpty, resolve: onQueueNotEmpty } = promiseWithResolvers());
- };
-
- // The following is the "pump" process that keeps pumping packets into the decoder
- (async () => {
- const decoderError = new Error();
- const decoder = await this._createDecoder((sample) => {
- onQueueDequeue();
-
- if (terminated) {
- sample.close();
- return;
- }
-
- let sampleUses = 0;
- while (
- timestampsOfInterest.length > 0
- && sample.timestamp - timestampsOfInterest[0]! > -1e-10 // Give it a little epsilon
- ) {
- sampleUses++;
- timestampsOfInterest.shift();
- }
-
- if (sampleUses > 0) {
- for (let i = 0; i < sampleUses; i++) {
- // Clone the sample if we need to emit it multiple times
- pushToQueue((i < sampleUses - 1 ? sample.clone() : sample) as MediaSample);
- }
- } else {
- sample.close();
- }
- }, (error) => {
- if (!outOfBandError) {
- error.stack = decoderError.stack; // Provide a more useful stack trace
- outOfBandError = error;
- onQueueNotEmpty();
- }
- });
-
- const packetSink = this._createPacketSink();
- let lastPacket: EncodedPacket | null = null;
- let lastKeyPacket: EncodedPacket | null = null;
-
- // The end sequence number (inclusive) in the next batch of packets that will be decoded. The batch starts
- // at the last key frame and goes until this sequence number.
- let maxSequenceNumber = -1;
-
- const decodePackets = async () => {
- assert(lastKeyPacket);
-
- // Start at the current key packet
- let currentPacket = lastKeyPacket;
- decoder.decode(currentPacket);
-
- while (currentPacket.sequenceNumber < maxSequenceNumber) {
- const maxQueueSize = computeMaxQueueSize(sampleQueue.length);
- while (sampleQueue.length + decoder.getDecodeQueueSize() > maxQueueSize && !terminated) {
- ({ promise: queueDequeue, resolve: onQueueDequeue } = promiseWithResolvers());
- await queueDequeue;
- }
-
- if (terminated) {
- break;
- }
-
- const nextPacket = await packetSink.getNextPacket(currentPacket);
- assert(nextPacket);
-
- decoder.decode(nextPacket);
- currentPacket = nextPacket;
- }
-
- maxSequenceNumber = -1;
- };
-
- const flushDecoder = async () => {
- await decoder.flush();
-
- // We don't expect this list to have any elements in it anymore, but in case it does, let's emit
- // nulls for every remaining element, then clear it.
- for (let i = 0; i < timestampsOfInterest.length; i++) {
- pushToQueue(null);
- }
- timestampsOfInterest.length = 0;
- };
-
- for await (const timestamp of timestampIterator) {
- validateTimestamp(timestamp);
-
- if (terminated || this._track.input._disposed) {
- break;
- }
-
- const targetPacket = await packetSink.getPacket(timestamp);
- const keyPacket = targetPacket && await packetSink.getKeyPacket(timestamp, { verifyKeyPackets: true });
-
- if (!keyPacket) {
- if (maxSequenceNumber !== -1) {
- await decodePackets();
- await flushDecoder();
- }
-
- pushToQueue(null);
- lastPacket = null;
- continue;
- }
-
- // Check if the key packet has changed or if we're going back in time
- if (
- lastPacket
- && (
- keyPacket.sequenceNumber !== lastKeyPacket!.sequenceNumber
- || targetPacket.timestamp < lastPacket.timestamp
- )
- ) {
- await decodePackets();
- await flushDecoder(); // Always flush here, improves decoder compatibility
- }
-
- timestampsOfInterest.push(targetPacket.timestamp);
- maxSequenceNumber = Math.max(targetPacket.sequenceNumber, maxSequenceNumber);
-
- lastPacket = targetPacket;
- lastKeyPacket = keyPacket;
- }
-
- if (!terminated && !this._track.input._disposed) {
- if (maxSequenceNumber !== -1) {
- // We still need to decode packets
- await decodePackets();
- }
-
- await flushDecoder();
- }
- decoder.close();
-
- decoderIsFlushed = true;
- onQueueNotEmpty(); // To unstuck the generator
- })().catch((error: Error) => {
- if (!outOfBandError) {
- outOfBandError = error;
- onQueueNotEmpty();
- }
- });
-
- const track = this._track;
- const closeSamples = () => {
- for (const sample of sampleQueue) {
- sample?.close();
- }
- };
-
- return {
- async next() {
- while (true) {
- if (track.input._disposed) {
- closeSamples();
- throw new InputDisposedError();
- } else if (terminated) {
- return { value: undefined, done: true };
- } else if (outOfBandError) {
- closeSamples();
- throw outOfBandError;
- } else if (sampleQueue.length > 0) {
- const value = sampleQueue.shift();
- assert(value !== undefined);
- onQueueDequeue();
- return { value, done: false };
- } else if (!decoderIsFlushed) {
- await queueNotEmpty;
- } else {
- return { value: undefined, done: true };
- }
- }
- },
- async return() {
- terminated = true;
- onQueueDequeue();
- onQueueNotEmpty();
- closeSamples();
-
- return { value: undefined, done: true };
- },
- async throw(error) {
- throw error;
- },
- [Symbol.asyncIterator]() {
- return this;
- },
- };
- }
-}
-
-const computeMaxQueueSize = (decodedSampleQueueSize: number) => {
- // If we have decoded samples lying around, limit the total queue size to a small value (decoded samples can use up
- // a lot of memory). If not, we're fine with a much bigger queue of encoded packets waiting to be decoded. In fact,
- // some decoders only start flushing out decoded chunks when the packet queue is large enough.
- return decodedSampleQueueSize === 0 ? 40 : 8;
-};
-
-export class VideoDecoderWrapper extends DecoderWrapper {
- decoder: VideoDecoder | null = null;
-
- customDecoder: CustomVideoDecoder | null = null;
- customDecoderCallSerializer = new CallSerializer();
- customDecoderQueueSize = 0;
- customDecoderClosed = false;
-
- inputTimestamps: number[] = []; // Timestamps input into the decoder, sorted.
- sampleQueue: VideoSample[] = []; // Safari-specific thing, check usage.
- currentPacketIndex = 0;
- raslSkipped = false; // For HEVC stuff
-
- // Alpha stuff
- alphaDecoder: VideoDecoder | null = null;
- alphaHadKeyframe = false;
- colorQueue: VideoFrame[] = [];
- alphaQueue: (VideoFrame | null)[] = [];
- merger: ColorAlphaMerger | null = null;
- mergerCreationFailed = false;
- decodedAlphaChunkCount = 0;
- alphaDecoderQueueSize = 0;
- /** Each value is the number of decoded alpha chunks at which a null alpha frame should be added. */
- nullAlphaFrameQueue: number[] = [];
- currentAlphaPacketIndex = 0;
- alphaRaslSkipped = false; // For HEVC stuff
-
- onDequeue: (() => unknown) | null = null;
-
- constructor(
- onSample: (sample: VideoSample) => unknown,
- onError: (error: Error) => unknown,
- public codec: VideoCodec,
- public decoderConfig: VideoDecoderConfig,
- public rotation: Rotation,
- public timeResolution: number,
- ) {
- super(onSample, onError);
-
- const MatchingCustomDecoder = customVideoDecoders.find(x => x.supports(codec, decoderConfig));
- if (MatchingCustomDecoder) {
- // @ts-expect-error "Can't create instance of abstract class 🤓"
- this.customDecoder = new MatchingCustomDecoder() as CustomVideoDecoder;
- // @ts-expect-error It's technically readonly
- this.customDecoder.codec = codec;
- // @ts-expect-error It's technically readonly
- this.customDecoder.config = decoderConfig;
- // @ts-expect-error It's technically readonly
- this.customDecoder.onSample = (sample) => {
- if (!(sample instanceof VideoSample)) {
- throw new TypeError('The argument passed to onSample must be a VideoSample.');
- }
-
- this.finalizeAndEmitSample(sample);
- };
-
- void this.customDecoderCallSerializer.call(() => this.customDecoder!.init());
- } else {
- const colorHandler = (frame: VideoFrame) => {
- if (this.alphaQueue.length > 0) {
- // Even when no alpha data is present (most of the time), there will be nulls in this queue
- const alphaFrame = this.alphaQueue.shift();
- assert(alphaFrame !== undefined);
-
- this.mergeAlpha(frame, alphaFrame);
- } else {
- this.colorQueue.push(frame);
- }
- };
-
- if (codec === 'avc' && this.decoderConfig.description && isChromium()) {
- // Chromium has/had a bug with playing interlaced AVC (https://issues.chromium.org/issues/456919096)
- // which can be worked around by requesting that software decoding be used. So, here we peek into the
- // AVC description, if present, and switch to software decoding if we find interlaced content.
- const record = deserializeAvcDecoderConfigurationRecord(toUint8Array(this.decoderConfig.description));
- if (record && record.sequenceParameterSets.length > 0) {
- const sps = parseAvcSps(record.sequenceParameterSets[0]!);
- if (sps && sps.frameMbsOnlyFlag === 0) {
- this.decoderConfig = {
- ...this.decoderConfig,
- hardwareAcceleration: 'prefer-software',
- };
- }
- }
- }
-
- this.decoder = new VideoDecoder({
- output: (frame) => {
- try {
- colorHandler(frame);
- } catch (error) {
- this.onError(error as Error);
- }
- },
- error: onError,
- });
- this.decoder.configure(this.decoderConfig);
-
- this.decoder.addEventListener('dequeue', () => {
- this.onDequeue?.();
- });
- }
- }
-
- getDecodeQueueSize() {
- if (this.customDecoder) {
- return this.customDecoderQueueSize;
- } else {
- assert(this.decoder);
-
- return Math.max(
- this.decoder.decodeQueueSize,
- this.alphaDecoder?.decodeQueueSize ?? 0,
- );
- }
- }
-
- decode(packet: EncodedPacket) {
- if (this.codec === 'hevc' && this.currentPacketIndex > 0 && !this.raslSkipped) {
- if (this.hasHevcRaslPicture(packet.data)) {
- return; // Drop
- }
-
- this.raslSkipped = true;
- }
-
- this.currentPacketIndex++;
-
- if (this.customDecoder) {
- this.customDecoderQueueSize++;
- void this.customDecoderCallSerializer
- .call(() => this.customDecoder!.decode(packet))
- .then(() => {
- this.customDecoderQueueSize--;
- this.onDequeue?.();
- });
- } else {
- assert(this.decoder);
-
- if (!isWebKit()) {
- insertSorted(this.inputTimestamps, packet.timestamp, x => x);
- }
-
- this.decoder.decode(packet.toEncodedVideoChunk());
- this.decodeAlphaData(packet);
- }
- }
-
- decodeAlphaData(packet: EncodedPacket) {
- if (!packet.sideData.alpha || this.mergerCreationFailed) {
- // No alpha side data in the packet, most common case
- this.pushNullAlphaFrame();
- return;
- }
-
- if (!this.merger) {
- try {
- this.merger = new ColorAlphaMerger();
- } catch (error) {
- console.error('Due to an error, only color data will be decoded.', error);
-
- this.mergerCreationFailed = true;
- this.decodeAlphaData(packet); // Go again
-
- return;
- }
- }
-
- // Check if we need to set up the alpha decoder
- if (!this.alphaDecoder) {
- const alphaHandler = (frame: VideoFrame) => {
- this.alphaDecoderQueueSize--;
-
- if (this.colorQueue.length > 0) {
- const colorFrame = this.colorQueue.shift();
- assert(colorFrame !== undefined);
-
- this.mergeAlpha(colorFrame, frame);
- } else {
- this.alphaQueue.push(frame);
- }
-
- // Check if any null frames have been queued for this point
- this.decodedAlphaChunkCount++;
- while (
- this.nullAlphaFrameQueue.length > 0
- && this.nullAlphaFrameQueue[0] === this.decodedAlphaChunkCount
- ) {
- this.nullAlphaFrameQueue.shift();
-
- if (this.colorQueue.length > 0) {
- const colorFrame = this.colorQueue.shift();
- assert(colorFrame !== undefined);
-
- this.mergeAlpha(colorFrame, null);
- } else {
- this.alphaQueue.push(null);
- }
- }
- };
-
- this.alphaDecoder = new VideoDecoder({
- output: (frame) => {
- try {
- alphaHandler(frame);
- } catch (error) {
- this.onError(error as Error);
- }
- },
- error: this.onError,
- });
- this.alphaDecoder.configure(this.decoderConfig);
-
- this.alphaDecoder.addEventListener('dequeue', () => {
- this.onDequeue?.();
- });
- }
-
- const type = determineVideoPacketType(this.codec, this.decoderConfig, packet.sideData.alpha);
-
- // Alpha packets might follow a different key frame rhythm than the main packets. Therefore, before we start
- // decoding, we must first find a packet that's actually a key frame. Until then, we treat the image as opaque.
- if (!this.alphaHadKeyframe) {
- this.alphaHadKeyframe = type === 'key';
- }
-
- if (this.alphaHadKeyframe) {
- // Same RASL skipping logic as for color, unlikely to be hit (since who uses HEVC with separate alpha??) but
- // here for symmetry.
- if (this.codec === 'hevc' && this.currentAlphaPacketIndex > 0 && !this.alphaRaslSkipped) {
- if (this.hasHevcRaslPicture(packet.sideData.alpha)) {
- this.pushNullAlphaFrame();
- return;
- }
-
- this.alphaRaslSkipped = true;
- }
-
- this.currentAlphaPacketIndex++;
- this.alphaDecoder.decode(packet.alphaToEncodedVideoChunk(type ?? packet.type));
- this.alphaDecoderQueueSize++;
- } else {
- this.pushNullAlphaFrame();
- }
- }
-
- pushNullAlphaFrame() {
- if (this.alphaDecoderQueueSize === 0) {
- // Easy
- this.alphaQueue.push(null);
- } else {
- // There are still alpha chunks being decoded, so pushing `null` immediately would result in out-of-order
- // data and be incorrect. Instead, we need to enqueue a "null frame" for when the current decoder workload
- // has finished.
- this.nullAlphaFrameQueue.push(this.decodedAlphaChunkCount + this.alphaDecoderQueueSize);
- }
- }
-
- /**
- * If we're using HEVC, we need to make sure to skip any RASL slices that follow a non-IDR key frame such as
- * CRA_NUT. This is because RASL slices cannot be decoded without data before the CRA_NUT. Browsers behave
- * differently here: Chromium drops the packets, Safari throws a decoder error. Either way, it's not good
- * and causes bugs upstream. So, let's take the dropping into our own hands.
- */
- hasHevcRaslPicture(packetData: Uint8Array) {
- const nalUnits = extractHevcNalUnits(packetData, this.decoderConfig);
- return nalUnits.some((x) => {
- const type = extractNalUnitTypeForHevc(x);
- return type === HevcNalUnitType.RASL_N || type === HevcNalUnitType.RASL_R;
- });
- }
-
- /** Handler for the WebCodecs VideoDecoder for ironing out browser differences. */
- sampleHandler(sample: VideoSample) {
- if (isWebKit()) {
- // For correct B-frame handling, we don't just hand over the frames directly but instead add them to
- // a queue, because we want to ensure frames are emitted in presentation order. We flush the queue
- // each time we receive a frame with a timestamp larger than the highest we've seen so far, as we
- // can sure that is not a B-frame. Typically, WebCodecs automatically guarantees that frames are
- // emitted in presentation order, but Safari doesn't always follow this rule.
- if (this.sampleQueue.length > 0 && (sample.timestamp >= last(this.sampleQueue)!.timestamp)) {
- for (const sample of this.sampleQueue) {
- this.finalizeAndEmitSample(sample);
- }
-
- this.sampleQueue.length = 0;
- }
-
- insertSorted(this.sampleQueue, sample, x => x.timestamp);
- } else {
- // Assign it the next earliest timestamp from the input. We do this because browsers, by spec, are
- // required to emit decoded frames in presentation order *while* retaining the timestamp of their
- // originating EncodedVideoChunk. For files with B-frames but no out-of-order timestamps (like a
- // missing ctts box, for example), this causes a mismatch. We therefore fix the timestamps and
- // ensure they are sorted by doing this.
- const timestamp = this.inputTimestamps.shift();
-
- // There's no way we'd have more decoded frames than encoded packets we passed in. Actually, the
- // correspondence should be 1:1.
- assert(timestamp !== undefined);
-
- sample.setTimestamp(timestamp);
- this.finalizeAndEmitSample(sample);
- }
- }
-
- finalizeAndEmitSample(sample: VideoSample) {
- // Round the timestamps to the time resolution
- sample.setTimestamp(Math.round(sample.timestamp * this.timeResolution) / this.timeResolution);
- sample.setDuration(Math.round(sample.duration * this.timeResolution) / this.timeResolution);
- sample.setRotation(this.rotation);
-
- this.onSample(sample);
- }
-
- mergeAlpha(color: VideoFrame, alpha: VideoFrame | null) {
- if (!alpha) {
- // Nothing needs to be merged
- const finalSample = new VideoSample(color);
- this.sampleHandler(finalSample);
-
- return;
- }
-
- assert(this.merger);
-
- this.merger.update(color, alpha);
- color.close();
- alpha.close();
-
- const finalFrame = new VideoFrame(this.merger.canvas, {
- timestamp: color.timestamp,
- duration: color.duration ?? undefined,
- });
-
- const finalSample = new VideoSample(finalFrame);
- this.sampleHandler(finalSample);
- }
-
- async flush() {
- if (this.customDecoder) {
- await this.customDecoderCallSerializer.call(() => this.customDecoder!.flush());
- } else {
- assert(this.decoder);
- await Promise.all([
- this.decoder.flush(),
- this.alphaDecoder?.flush(),
- ]);
-
- this.colorQueue.forEach(x => x.close());
- this.colorQueue.length = 0;
- this.alphaQueue.forEach(x => x?.close());
- this.alphaQueue.length = 0;
-
- this.alphaHadKeyframe = false;
- this.decodedAlphaChunkCount = 0;
- this.alphaDecoderQueueSize = 0;
- this.nullAlphaFrameQueue.length = 0;
- this.currentAlphaPacketIndex = 0;
- this.alphaRaslSkipped = false;
- }
-
- if (isWebKit()) {
- for (const sample of this.sampleQueue) {
- this.finalizeAndEmitSample(sample);
- }
-
- this.sampleQueue.length = 0;
- }
-
- this.currentPacketIndex = 0;
- this.raslSkipped = false;
- }
-
- close() {
- if (this.customDecoder) {
- if (!this.customDecoderClosed) {
- this.customDecoderClosed = true;
- void this.customDecoderCallSerializer.call(() => this.customDecoder!.close());
- }
- } else {
- assert(this.decoder);
-
- if (this.decoder.state !== 'closed') {
- this.decoder.close();
- }
- if (this.alphaDecoder && this.alphaDecoder.state !== 'closed') {
- this.alphaDecoder.close();
- }
-
- this.colorQueue.forEach(x => x.close());
- this.colorQueue.length = 0;
- this.alphaQueue.forEach(x => x?.close());
- this.alphaQueue.length = 0;
-
- this.merger?.close();
- }
-
- for (const sample of this.sampleQueue) {
- sample.close();
- }
- this.sampleQueue.length = 0;
- }
-
- get closed() {
- if (this.customDecoder) {
- if (this.customDecoderClosed) {
- return true;
- }
-
- return !this.customDecoderCallSerializer.errored;
- } else {
- assert(this.decoder);
- return this.decoder.state === 'closed';
- }
- }
-}
-
-/** Utility class that merges together color and alpha information using simple WebGL 2 shaders. */
-class ColorAlphaMerger {
- canvas: OffscreenCanvas | HTMLCanvasElement;
- private gl: WebGL2RenderingContext;
- private program: WebGLProgram;
- private vao: WebGLVertexArrayObject;
- private colorTexture: WebGLTexture;
- private alphaTexture: WebGLTexture;
-
- constructor() {
- // Canvas will be resized later
- if (typeof OffscreenCanvas !== 'undefined') {
- // Prefer OffscreenCanvas for Worker environments
- this.canvas = new OffscreenCanvas(300, 150);
- } else {
- this.canvas = document.createElement('canvas');
- }
-
- const gl = this.canvas.getContext('webgl2', {
- premultipliedAlpha: false,
- }) as unknown as WebGL2RenderingContext | null; // Casting because of some TypeScript weirdness
- if (!gl) {
- throw new Error('Couldn\'t acquire WebGL 2 context.');
- }
-
- this.gl = gl;
- this.program = this.createProgram();
- this.vao = this.createVAO();
- this.colorTexture = this.createTexture();
- this.alphaTexture = this.createTexture();
-
- this.gl.useProgram(this.program);
- this.gl.uniform1i(this.gl.getUniformLocation(this.program, 'u_colorTexture'), 0);
- this.gl.uniform1i(this.gl.getUniformLocation(this.program, 'u_alphaTexture'), 1);
- }
-
- private createProgram(): WebGLProgram {
- const vertexShader = this.createShader(this.gl.VERTEX_SHADER, `#version 300 es
- in vec2 a_position;
- in vec2 a_texCoord;
- out vec2 v_texCoord;
-
- void main() {
- gl_Position = vec4(a_position, 0.0, 1.0);
- v_texCoord = a_texCoord;
- }
- `);
-
- const fragmentShader = this.createShader(this.gl.FRAGMENT_SHADER, `#version 300 es
- precision highp float;
-
- uniform sampler2D u_colorTexture;
- uniform sampler2D u_alphaTexture;
- in vec2 v_texCoord;
- out vec4 fragColor;
-
- void main() {
- vec3 color = texture(u_colorTexture, v_texCoord).rgb;
- float alpha = texture(u_alphaTexture, v_texCoord).r;
- fragColor = vec4(color, alpha);
- }
- `);
-
- const program = this.gl.createProgram();
- this.gl.attachShader(program, vertexShader);
- this.gl.attachShader(program, fragmentShader);
- this.gl.linkProgram(program);
-
- return program;
- }
-
- private createShader(type: number, source: string): WebGLShader {
- const shader = this.gl.createShader(type)!;
- this.gl.shaderSource(shader, source);
- this.gl.compileShader(shader);
- return shader;
- }
-
- private createVAO(): WebGLVertexArrayObject {
- const vao = this.gl.createVertexArray();
- this.gl.bindVertexArray(vao);
-
- const vertices = new Float32Array([
- -1, -1, 0, 1,
- 1, -1, 1, 1,
- -1, 1, 0, 0,
- 1, 1, 1, 0,
- ]);
-
- const buffer = this.gl.createBuffer();
- this.gl.bindBuffer(this.gl.ARRAY_BUFFER, buffer);
- this.gl.bufferData(this.gl.ARRAY_BUFFER, vertices, this.gl.STATIC_DRAW);
-
- const positionLocation = this.gl.getAttribLocation(this.program, 'a_position');
- const texCoordLocation = this.gl.getAttribLocation(this.program, 'a_texCoord');
-
- this.gl.enableVertexAttribArray(positionLocation);
- this.gl.vertexAttribPointer(positionLocation, 2, this.gl.FLOAT, false, 16, 0);
-
- this.gl.enableVertexAttribArray(texCoordLocation);
- this.gl.vertexAttribPointer(texCoordLocation, 2, this.gl.FLOAT, false, 16, 8);
-
- return vao;
- }
-
- private createTexture(): WebGLTexture {
- const texture = this.gl.createTexture();
-
- this.gl.bindTexture(this.gl.TEXTURE_2D, texture);
- this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_S, this.gl.CLAMP_TO_EDGE);
- this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_T, this.gl.CLAMP_TO_EDGE);
- this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MIN_FILTER, this.gl.LINEAR);
- this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MAG_FILTER, this.gl.LINEAR);
-
- return texture;
- }
-
- update(color: VideoFrame, alpha: VideoFrame): void {
- if (color.displayWidth !== this.canvas.width || color.displayHeight !== this.canvas.height) {
- this.canvas.width = color.displayWidth;
- this.canvas.height = color.displayHeight;
- }
-
- this.gl.activeTexture(this.gl.TEXTURE0);
- this.gl.bindTexture(this.gl.TEXTURE_2D, this.colorTexture);
- this.gl.texImage2D(this.gl.TEXTURE_2D, 0, this.gl.RGBA, this.gl.RGBA, this.gl.UNSIGNED_BYTE, color);
-
- this.gl.activeTexture(this.gl.TEXTURE1);
- this.gl.bindTexture(this.gl.TEXTURE_2D, this.alphaTexture);
- this.gl.texImage2D(this.gl.TEXTURE_2D, 0, this.gl.RGBA, this.gl.RGBA, this.gl.UNSIGNED_BYTE, alpha);
-
- this.gl.viewport(0, 0, this.canvas.width, this.canvas.height);
- this.gl.clear(this.gl.COLOR_BUFFER_BIT);
-
- this.gl.bindVertexArray(this.vao);
- this.gl.drawArrays(this.gl.TRIANGLE_STRIP, 0, 4);
- }
-
- close() {
- this.gl.getExtension('WEBGL_lose_context')?.loseContext();
- this.gl = null as unknown as WebGL2RenderingContext;
- }
-}
-
-/**
- * A sink that retrieves decoded video samples (video frames) from a video track.
- * @group Media sinks
- * @public
- */
-export class VideoSampleSink extends BaseMediaSampleSink {
- /** @internal */
- _track: InputVideoTrack;
-
- /** Creates a new {@link VideoSampleSink} for the given {@link InputVideoTrack}. */
- constructor(videoTrack: InputVideoTrack) {
- if (!(videoTrack instanceof InputVideoTrack)) {
- throw new TypeError('videoTrack must be an InputVideoTrack.');
- }
-
- super();
-
- this._track = videoTrack;
- }
-
- /** @internal */
- async _createDecoder(
- onSample: (sample: VideoSample) => unknown,
- onError: (error: Error) => unknown,
- ) {
- if (!(await this._track.canDecode())) {
- throw new Error(
- 'This video track cannot be decoded by this browser. Make sure to check decodability before using'
- + ' a track.',
- );
- }
-
- const codec = this._track.codec;
- const rotation = this._track.rotation;
- const decoderConfig = await this._track.getDecoderConfig();
- const timeResolution = this._track.timeResolution;
- assert(codec && decoderConfig);
-
- return new VideoDecoderWrapper(onSample, onError, codec, decoderConfig, rotation, timeResolution);
- }
-
- /** @internal */
- _createPacketSink() {
- return new EncodedPacketSink(this._track);
- }
-
- /**
- * Retrieves the video sample (frame) corresponding to the given timestamp, in seconds. More specifically, returns
- * the last video sample (in presentation order) with a start timestamp less than or equal to the given timestamp.
- * Returns null if the timestamp is before the track's first timestamp.
- *
- * @param timestamp - The timestamp used for retrieval, in seconds.
- */
- async getSample(timestamp: number) {
- validateTimestamp(timestamp);
-
- for await (const sample of this.mediaSamplesAtTimestamps([timestamp])) {
- return sample;
- }
- throw new Error('Internal error: Iterator returned nothing.');
- }
-
- /**
- * Creates an async iterator that yields the video samples (frames) of this track in presentation order. This method
- * will intelligently pre-decode a few frames ahead to enable fast iteration.
- *
- * @param startTimestamp - The timestamp in seconds at which to start yielding samples (inclusive).
- * @param endTimestamp - The timestamp in seconds at which to stop yielding samples (exclusive).
- */
- samples(startTimestamp = 0, endTimestamp = Infinity) {
- return this.mediaSamplesInRange(startTimestamp, endTimestamp);
- }
-
- /**
- * Creates an async iterator that yields a video sample (frame) for each timestamp in the argument. This method
- * uses an optimized decoding pipeline if these timestamps are monotonically sorted, decoding each packet at most
- * once, and is therefore more efficient than manually getting the sample for every timestamp. The iterator may
- * yield null if no frame is available for a given timestamp.
- *
- * @param timestamps - An iterable or async iterable of timestamps in seconds.
- */
- samplesAtTimestamps(timestamps: AnyIterable) {
- return this.mediaSamplesAtTimestamps(timestamps);
- }
-}
-
-/**
- * A canvas with additional timing information (timestamp & duration).
- * @group Media sinks
- * @public
- */
-export type WrappedCanvas = {
- /** A canvas element or offscreen canvas. */
- canvas: HTMLCanvasElement | OffscreenCanvas;
- /** The timestamp of the corresponding video sample, in seconds. */
- timestamp: number;
- /** The duration of the corresponding video sample, in seconds. */
- duration: number;
-};
-
-/**
- * Options for constructing a CanvasSink.
- * @group Media sinks
- * @public
- */
-export type CanvasSinkOptions = {
- /**
- * Whether the output canvases should have transparency instead of a black background. Defaults to `false`. Set
- * this to `true` when using this sink to read transparent videos.
- */
- alpha?: boolean;
- /**
- * The width of the output canvas in pixels, defaulting to the display width of the video track. If height is not
- * set, it will be deduced automatically based on aspect ratio.
- */
- width?: number;
- /**
- * The height of the output canvas in pixels, defaulting to the display height of the video track. If width is not
- * set, it will be deduced automatically based on aspect ratio.
- */
- height?: number;
- /**
- * The fitting algorithm in case both width and height are set.
- *
- * - `'fill'` will stretch the image to fill the entire box, potentially altering aspect ratio.
- * - `'contain'` will contain the entire image within the box while preserving aspect ratio. This may lead to
- * letterboxing.
- * - `'cover'` will scale the image until the entire box is filled, while preserving aspect ratio.
- */
- fit?: 'fill' | 'contain' | 'cover';
- /**
- * The clockwise rotation by which to rotate the raw video frame. Defaults to the rotation set in the file metadata.
- * Rotation is applied before resizing.
- */
- rotation?: Rotation;
- /**
- * Specifies the rectangular region of the input video to crop to. The crop region will automatically be clamped to
- * the dimensions of the input video track. Cropping is performed after rotation but before resizing.
- */
- crop?: CropRectangle;
- /**
- * When set, specifies the number of canvases in the pool. These canvases will be reused in a ring buffer /
- * round-robin type fashion. This keeps the amount of allocated VRAM constant and relieves the browser from
- * constantly allocating/deallocating canvases. A pool size of 0 or `undefined` disables the pool and means a new
- * canvas is created each time.
- */
- poolSize?: number;
-};
-
-/**
- * A sink that renders video samples (frames) of the given video track to canvases. This is often more useful than
- * directly retrieving frames, as it comes with common preprocessing steps such as resizing or applying rotation
- * metadata.
- *
- * This sink will yield `HTMLCanvasElement`s when in a DOM context, and `OffscreenCanvas`es otherwise.
- *
- * @group Media sinks
- * @public
- */
-export class CanvasSink {
- /** @internal */
- _videoTrack: InputVideoTrack;
- /** @internal */
- _alpha: boolean;
- /** @internal */
- _width: number;
- /** @internal */
- _height: number;
- /** @internal */
- _fit: 'fill' | 'contain' | 'cover';
- /** @internal */
- _rotation: Rotation;
- /** @internal */
- _crop?: { left: number; top: number; width: number; height: number };
- /** @internal */
- _videoSampleSink: VideoSampleSink;
- /** @internal */
- _canvasPool: (HTMLCanvasElement | OffscreenCanvas | null)[];
- /** @internal */
- _nextCanvasIndex = 0;
-
- /** Creates a new {@link CanvasSink} for the given {@link InputVideoTrack}. */
- constructor(videoTrack: InputVideoTrack, options: CanvasSinkOptions = {}) {
- if (!(videoTrack instanceof InputVideoTrack)) {
- throw new TypeError('videoTrack must be an InputVideoTrack.');
- }
- if (options && typeof options !== 'object') {
- throw new TypeError('options must be an object.');
- }
- if (options.alpha !== undefined && typeof options.alpha !== 'boolean') {
- throw new TypeError('options.alpha, when provided, must be a boolean.');
- }
- if (options.width !== undefined && (!Number.isInteger(options.width) || options.width <= 0)) {
- throw new TypeError('options.width, when defined, must be a positive integer.');
- }
- if (options.height !== undefined && (!Number.isInteger(options.height) || options.height <= 0)) {
- throw new TypeError('options.height, when defined, must be a positive integer.');
- }
- if (options.fit !== undefined && !['fill', 'contain', 'cover'].includes(options.fit)) {
- throw new TypeError('options.fit, when provided, must be one of "fill", "contain", or "cover".');
- }
- if (
- options.width !== undefined
- && options.height !== undefined
- && options.fit === undefined
- ) {
- throw new TypeError(
- 'When both options.width and options.height are provided, options.fit must also be provided.',
- );
- }
- if (options.rotation !== undefined && ![0, 90, 180, 270].includes(options.rotation)) {
- throw new TypeError('options.rotation, when provided, must be 0, 90, 180 or 270.');
- }
- if (options.crop !== undefined) {
- validateCropRectangle(options.crop, 'options.');
- }
- if (
- options.poolSize !== undefined
- && (typeof options.poolSize !== 'number' || !Number.isInteger(options.poolSize) || options.poolSize < 0)
- ) {
- throw new TypeError('poolSize must be a non-negative integer.');
- }
-
- const rotation = options.rotation ?? videoTrack.rotation;
-
- const [rotatedWidth, rotatedHeight] = rotation % 180 === 0
- ? [videoTrack.codedWidth, videoTrack.codedHeight]
- : [videoTrack.codedHeight, videoTrack.codedWidth];
-
- const crop = options.crop;
- if (crop) {
- clampCropRectangle(crop, rotatedWidth, rotatedHeight);
- }
-
- let [width, height] = crop
- ? [crop.width, crop.height]
- : [rotatedWidth, rotatedHeight];
- const originalAspectRatio = width / height;
-
- // If width and height aren't defined together, deduce the missing value using the aspect ratio
- if (options.width !== undefined && options.height === undefined) {
- width = options.width;
- height = Math.round(width / originalAspectRatio);
- } else if (options.width === undefined && options.height !== undefined) {
- height = options.height;
- width = Math.round(height * originalAspectRatio);
- } else if (options.width !== undefined && options.height !== undefined) {
- width = options.width;
- height = options.height;
- }
-
- this._videoTrack = videoTrack;
- this._alpha = options.alpha ?? false;
- this._width = width;
- this._height = height;
- this._rotation = rotation;
- this._crop = crop;
- this._fit = options.fit ?? 'fill';
- this._videoSampleSink = new VideoSampleSink(videoTrack);
- this._canvasPool = Array.from({ length: options.poolSize ?? 0 }, () => null);
- }
-
- /** @internal */
- _videoSampleToWrappedCanvas(sample: VideoSample): WrappedCanvas {
- let canvas = this._canvasPool[this._nextCanvasIndex];
- let canvasIsNew = false;
-
- if (!canvas) {
- if (typeof document !== 'undefined') {
- // Prefer an HTMLCanvasElement
- canvas = document.createElement('canvas');
- canvas.width = this._width;
- canvas.height = this._height;
- } else {
- canvas = new OffscreenCanvas(this._width, this._height);
- }
-
- if (this._canvasPool.length > 0) {
- this._canvasPool[this._nextCanvasIndex] = canvas;
- }
-
- canvasIsNew = true;
- }
-
- if (this._canvasPool.length > 0) {
- this._nextCanvasIndex = (this._nextCanvasIndex + 1) % this._canvasPool.length;
- }
-
- const context = canvas.getContext('2d', {
- alpha: this._alpha || isFirefox(), // Firefox has VideoFrame glitches with opaque canvases
- }) as CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;
- assert(context);
-
- context.resetTransform();
-
- if (!canvasIsNew) {
- if (!this._alpha && isFirefox()) {
- context.fillStyle = 'black';
- context.fillRect(0, 0, this._width, this._height);
- } else {
- context.clearRect(0, 0, this._width, this._height);
- }
- }
-
- sample.drawWithFit(context, {
- fit: this._fit,
- rotation: this._rotation,
- crop: this._crop,
- });
-
- const result = {
- canvas,
- timestamp: sample.timestamp,
- duration: sample.duration,
- };
-
- sample.close();
- return result;
- }
-
- /**
- * Retrieves a canvas with the video frame corresponding to the given timestamp, in seconds. More specifically,
- * returns the last video frame (in presentation order) with a start timestamp less than or equal to the given
- * timestamp. Returns null if the timestamp is before the track's first timestamp.
- *
- * @param timestamp - The timestamp used for retrieval, in seconds.
- */
- async getCanvas(timestamp: number) {
- validateTimestamp(timestamp);
-
- const sample = await this._videoSampleSink.getSample(timestamp);
- return sample && this._videoSampleToWrappedCanvas(sample);
- }
-
- /**
- * Creates an async iterator that yields canvases with the video frames of this track in presentation order. This
- * method will intelligently pre-decode a few frames ahead to enable fast iteration.
- *
- * @param startTimestamp - The timestamp in seconds at which to start yielding canvases (inclusive).
- * @param endTimestamp - The timestamp in seconds at which to stop yielding canvases (exclusive).
- */
- canvases(startTimestamp = 0, endTimestamp = Infinity) {
- return mapAsyncGenerator(
- this._videoSampleSink.samples(startTimestamp, endTimestamp),
- sample => this._videoSampleToWrappedCanvas(sample),
- );
- }
-
- /**
- * Creates an async iterator that yields a canvas for each timestamp in the argument. This method uses an optimized
- * decoding pipeline if these timestamps are monotonically sorted, decoding each packet at most once, and is
- * therefore more efficient than manually getting the canvas for every timestamp. The iterator may yield null if
- * no frame is available for a given timestamp.
- *
- * @param timestamps - An iterable or async iterable of timestamps in seconds.
- */
- canvasesAtTimestamps(timestamps: AnyIterable) {
- return mapAsyncGenerator(
- this._videoSampleSink.samplesAtTimestamps(timestamps),
- sample => sample && this._videoSampleToWrappedCanvas(sample),
- );
- }
-}
-
-export class AudioDecoderWrapper extends DecoderWrapper {
- decoder: AudioDecoder | null = null;
-
- customDecoder: CustomAudioDecoder | null = null;
- customDecoderCallSerializer = new CallSerializer();
- customDecoderQueueSize = 0;
- customDecoderClosed = false;
-
- // Internal state to accumulate a precise current timestamp based on audio durations, not the (potentially
- // inaccurate) packet timestamps.
- currentTimestamp: number | null = null;
-
- onDequeue: (() => unknown) | null = null;
-
- constructor(
- onSample: (sample: AudioSample) => unknown,
- onError: (error: Error) => unknown,
- codec: AudioCodec,
- decoderConfig: AudioDecoderConfig,
- ) {
- super(onSample, onError);
-
- const sampleHandler = (sample: AudioSample) => {
- if (
- this.currentTimestamp === null
- || Math.abs(sample.timestamp - this.currentTimestamp) >= sample.duration
- ) {
- // We need to sync with the sample timestamp again
- this.currentTimestamp = sample.timestamp;
- }
-
- const preciseTimestamp = this.currentTimestamp;
- this.currentTimestamp += sample.duration;
-
- if (sample.numberOfFrames === 0) {
- // We skip zero-data (empty) AudioSamples. These are sometimes emitted, for example, by Firefox when it
- // decodes Vorbis (at the start).
- sample.close();
- return;
- }
-
- // Round the timestamp to the sample rate
- const sampleRate = decoderConfig.sampleRate;
- sample.setTimestamp(Math.round(preciseTimestamp * sampleRate) / sampleRate);
-
- onSample(sample);
- };
-
- const MatchingCustomDecoder = customAudioDecoders.find(x => x.supports(codec, decoderConfig));
- if (MatchingCustomDecoder) {
- // @ts-expect-error "Can't create instance of abstract class 🤓"
- this.customDecoder = new MatchingCustomDecoder() as CustomAudioDecoder;
- // @ts-expect-error It's technically readonly
- this.customDecoder.codec = codec;
- // @ts-expect-error It's technically readonly
- this.customDecoder.config = decoderConfig;
- // @ts-expect-error It's technically readonly
- this.customDecoder.onSample = (sample) => {
- if (!(sample instanceof AudioSample)) {
- throw new TypeError('The argument passed to onSample must be an AudioSample.');
- }
-
- sampleHandler(sample);
- };
-
- void this.customDecoderCallSerializer.call(() => this.customDecoder!.init());
- } else {
- this.decoder = new AudioDecoder({
- output: (data) => {
- try {
- sampleHandler(new AudioSample(data));
- } catch (error) {
- this.onError(error as Error);
- }
- },
- error: onError,
- });
- this.decoder.configure(decoderConfig);
-
- this.decoder.addEventListener('dequeue', () => {
- this.onDequeue?.();
- });
- }
- }
-
- getDecodeQueueSize() {
- if (this.customDecoder) {
- return this.customDecoderQueueSize;
- } else {
- assert(this.decoder);
- return this.decoder.decodeQueueSize;
- }
- }
-
- decode(packet: EncodedPacket) {
- if (this.customDecoder) {
- this.customDecoderQueueSize++;
- void this.customDecoderCallSerializer
- .call(() => this.customDecoder!.decode(packet))
- .then(() => {
- this.customDecoderQueueSize--;
- this.onDequeue?.();
- });
- } else {
- assert(this.decoder);
- this.decoder.decode(packet.toEncodedAudioChunk());
- }
- }
-
- flush() {
- if (this.customDecoder) {
- return this.customDecoderCallSerializer.call(() => this.customDecoder!.flush());
- } else {
- assert(this.decoder);
- return this.decoder.flush();
- }
- }
-
- close() {
- if (this.customDecoder) {
- if (!this.customDecoderClosed) {
- this.customDecoderClosed = true;
- void this.customDecoderCallSerializer.call(() => this.customDecoder!.close());
- }
- } else {
- assert(this.decoder);
-
- if (this.decoder.state !== 'closed') {
- this.decoder.close();
- }
- }
- }
-
- get closed() {
- if (this.customDecoder) {
- if (this.customDecoderClosed) {
- return true;
- }
-
- return !this.customDecoderCallSerializer.errored;
- } else {
- assert(this.decoder);
- return this.decoder.state === 'closed';
- }
- }
-}
-
-// There are a lot of PCM variants not natively supported by the browser and by AudioData. Therefore we need a simple
-// decoder that maps any input PCM format into a PCM format supported by the browser.
-export class PcmAudioDecoderWrapper extends DecoderWrapper {
- codec: PcmAudioCodec;
-
- inputSampleSize: 1 | 2 | 3 | 4 | 8;
- readInputValue: (view: DataView, byteOffset: number) => number;
-
- outputSampleSize: 1 | 2 | 4;
- outputFormat: 'u8' | 's16' | 's32' | 'f32';
- writeOutputValue: (view: DataView, byteOffset: number, value: number) => void;
-
- // Internal state to accumulate a precise current timestamp based on audio durations, not the (potentially
- // inaccurate) packet timestamps.
- currentTimestamp: number | null = null;
-
- isClosed = false;
-
- onDequeue: (() => unknown) | null = null;
-
- constructor(
- onSample: (sample: AudioSample) => unknown,
- onError: (error: Error) => unknown,
- public decoderConfig: AudioDecoderConfig,
- ) {
- super(onSample, onError);
-
- assert((PCM_AUDIO_CODECS as readonly string[]).includes(decoderConfig.codec));
- this.codec = decoderConfig.codec as PcmAudioCodec;
-
- const { dataType, sampleSize, littleEndian } = parsePcmCodec(this.codec);
- this.inputSampleSize = sampleSize;
-
- switch (sampleSize) {
- case 1: {
- if (dataType === 'unsigned') {
- this.readInputValue = (view, byteOffset) => view.getUint8(byteOffset) - 2 ** 7;
- } else if (dataType === 'signed') {
- this.readInputValue = (view, byteOffset) => view.getInt8(byteOffset);
- } else if (dataType === 'ulaw') {
- this.readInputValue = (view, byteOffset) => fromUlaw(view.getUint8(byteOffset));
- } else if (dataType === 'alaw') {
- this.readInputValue = (view, byteOffset) => fromAlaw(view.getUint8(byteOffset));
- } else {
- assert(false);
- }
- }; break;
- case 2: {
- if (dataType === 'unsigned') {
- this.readInputValue = (view, byteOffset) => view.getUint16(byteOffset, littleEndian) - 2 ** 15;
- } else if (dataType === 'signed') {
- this.readInputValue = (view, byteOffset) => view.getInt16(byteOffset, littleEndian);
- } else {
- assert(false);
- }
- }; break;
- case 3: {
- if (dataType === 'unsigned') {
- this.readInputValue = (view, byteOffset) => getUint24(view, byteOffset, littleEndian) - 2 ** 23;
- } else if (dataType === 'signed') {
- this.readInputValue = (view, byteOffset) => getInt24(view, byteOffset, littleEndian);
- } else {
- assert(false);
- }
- }; break;
- case 4: {
- if (dataType === 'unsigned') {
- this.readInputValue = (view, byteOffset) => view.getUint32(byteOffset, littleEndian) - 2 ** 31;
- } else if (dataType === 'signed') {
- this.readInputValue = (view, byteOffset) => view.getInt32(byteOffset, littleEndian);
- } else if (dataType === 'float') {
- this.readInputValue = (view, byteOffset) => view.getFloat32(byteOffset, littleEndian);
- } else {
- assert(false);
- }
- }; break;
- case 8: {
- if (dataType === 'float') {
- this.readInputValue = (view, byteOffset) => view.getFloat64(byteOffset, littleEndian);
- } else {
- assert(false);
- }
- }; break;
- default: {
- assertNever(sampleSize);
- assert(false);
- };
- }
-
- switch (sampleSize) {
- case 1: {
- if (dataType === 'ulaw' || dataType === 'alaw') {
- this.outputSampleSize = 2;
- this.outputFormat = 's16';
- this.writeOutputValue = (view, byteOffset, value) => view.setInt16(byteOffset, value, true);
- } else {
- this.outputSampleSize = 1;
- this.outputFormat = 'u8';
- this.writeOutputValue = (view, byteOffset, value) => view.setUint8(byteOffset, value + 2 ** 7);
- }
- }; break;
- case 2: {
- this.outputSampleSize = 2;
- this.outputFormat = 's16';
- this.writeOutputValue = (view, byteOffset, value) => view.setInt16(byteOffset, value, true);
- }; break;
- case 3: {
- this.outputSampleSize = 4;
- this.outputFormat = 's32';
- // From https://www.w3.org/TR/webcodecs:
- // AudioData containing 24-bit samples SHOULD store those samples in s32 or f32. When samples are
- // stored in s32, each sample MUST be left-shifted by 8 bits.
- this.writeOutputValue = (view, byteOffset, value) => view.setInt32(byteOffset, value << 8, true);
- }; break;
- case 4: {
- this.outputSampleSize = 4;
-
- if (dataType === 'float') {
- this.outputFormat = 'f32';
- this.writeOutputValue = (view, byteOffset, value) => view.setFloat32(byteOffset, value, true);
- } else {
- this.outputFormat = 's32';
- this.writeOutputValue = (view, byteOffset, value) => view.setInt32(byteOffset, value, true);
- }
- }; break;
- case 8: {
- this.outputSampleSize = 4;
-
- this.outputFormat = 'f32';
- this.writeOutputValue = (view, byteOffset, value) => view.setFloat32(byteOffset, value, true);
- }; break;
- default: {
- assertNever(sampleSize);
- assert(false);
- };
- };
- }
-
- getDecodeQueueSize() {
- return 0;
- }
-
- decode(packet: EncodedPacket) {
- this.onDequeue?.();
-
- const inputView = toDataView(packet.data);
-
- const numberOfFrames = packet.byteLength / this.decoderConfig.numberOfChannels / this.inputSampleSize;
-
- const outputBufferSize = numberOfFrames * this.decoderConfig.numberOfChannels * this.outputSampleSize;
- const outputBuffer = new ArrayBuffer(outputBufferSize);
- const outputView = new DataView(outputBuffer);
-
- for (let i = 0; i < numberOfFrames * this.decoderConfig.numberOfChannels; i++) {
- const inputIndex = i * this.inputSampleSize;
- const outputIndex = i * this.outputSampleSize;
-
- const value = this.readInputValue(inputView, inputIndex);
- this.writeOutputValue(outputView, outputIndex, value);
- }
-
- const preciseDuration = numberOfFrames / this.decoderConfig.sampleRate;
- if (this.currentTimestamp === null || Math.abs(packet.timestamp - this.currentTimestamp) >= preciseDuration) {
- // We need to sync with the packet timestamp again
- this.currentTimestamp = packet.timestamp;
- }
-
- const preciseTimestamp = this.currentTimestamp;
- this.currentTimestamp += preciseDuration;
-
- const audioSample = new AudioSample({
- format: this.outputFormat,
- data: outputBuffer,
- numberOfChannels: this.decoderConfig.numberOfChannels,
- sampleRate: this.decoderConfig.sampleRate,
- numberOfFrames,
- timestamp: preciseTimestamp,
- });
-
- this.onSample(audioSample);
- }
-
- async flush() {
- // Do nothing
- }
-
- close() {
- this.isClosed = true;
- }
-
- get closed() {
- return this.isClosed;
- }
-}
-
-/**
- * Sink for retrieving decoded audio samples from an audio track.
- * @group Media sinks
- * @public
- */
-export class AudioSampleSink extends BaseMediaSampleSink {
- /** @internal */
- _track: InputAudioTrack;
-
- /** Creates a new {@link AudioSampleSink} for the given {@link InputAudioTrack}. */
- constructor(audioTrack: InputAudioTrack) {
- if (!(audioTrack instanceof InputAudioTrack)) {
- throw new TypeError('audioTrack must be an InputAudioTrack.');
- }
-
- super();
-
- this._track = audioTrack;
- }
-
- /** @internal */
- async _createDecoder(
- onSample: (sample: AudioSample) => unknown,
- onError: (error: Error) => unknown,
- ) {
- if (!(await this._track.canDecode())) {
- throw new Error(
- 'This audio track cannot be decoded by this browser. Make sure to check decodability before using'
- + ' a track.',
- );
- }
-
- const codec = this._track.codec;
- const decoderConfig = await this._track.getDecoderConfig();
- assert(codec && decoderConfig);
-
- if ((PCM_AUDIO_CODECS as readonly string[]).includes(decoderConfig.codec)) {
- return new PcmAudioDecoderWrapper(onSample, onError, decoderConfig);
- } else {
- return new AudioDecoderWrapper(onSample, onError, codec, decoderConfig);
- }
- }
-
- /** @internal */
- _createPacketSink() {
- return new EncodedPacketSink(this._track);
- }
-
- /**
- * Retrieves the audio sample corresponding to the given timestamp, in seconds. More specifically, returns
- * the last audio sample (in presentation order) with a start timestamp less than or equal to the given timestamp.
- * Returns null if the timestamp is before the track's first timestamp.
- *
- * @param timestamp - The timestamp used for retrieval, in seconds.
- */
- async getSample(timestamp: number) {
- validateTimestamp(timestamp);
-
- for await (const sample of this.mediaSamplesAtTimestamps([timestamp])) {
- return sample;
- }
- throw new Error('Internal error: Iterator returned nothing.');
- }
-
- /**
- * Creates an async iterator that yields the audio samples of this track in presentation order. This method
- * will intelligently pre-decode a few samples ahead to enable fast iteration.
- *
- * @param startTimestamp - The timestamp in seconds at which to start yielding samples (inclusive).
- * @param endTimestamp - The timestamp in seconds at which to stop yielding samples (exclusive).
- */
- samples(startTimestamp = 0, endTimestamp = Infinity) {
- return this.mediaSamplesInRange(startTimestamp, endTimestamp);
- }
-
- /**
- * Creates an async iterator that yields an audio sample for each timestamp in the argument. This method
- * uses an optimized decoding pipeline if these timestamps are monotonically sorted, decoding each packet at most
- * once, and is therefore more efficient than manually getting the sample for every timestamp. The iterator may
- * yield null if no sample is available for a given timestamp.
- *
- * @param timestamps - An iterable or async iterable of timestamps in seconds.
- */
- samplesAtTimestamps(timestamps: AnyIterable) {
- return this.mediaSamplesAtTimestamps(timestamps);
- }
-}
-
-/**
- * An AudioBuffer with additional timing information (timestamp & duration).
- * @group Media sinks
- * @public
- */
-export type WrappedAudioBuffer = {
- /** An AudioBuffer. */
- buffer: AudioBuffer;
- /** The timestamp of the corresponding audio sample, in seconds. */
- timestamp: number;
- /** The duration of the corresponding audio sample, in seconds. */
- duration: number;
-};
-
-/**
- * A sink that retrieves decoded audio samples from an audio track and converts them to `AudioBuffer` instances. This is
- * often more useful than directly retrieving audio samples, as audio buffers can be directly used with the
- * Web Audio API.
- * @group Media sinks
- * @public
- */
-export class AudioBufferSink {
- /** @internal */
- _audioSampleSink: AudioSampleSink;
-
- /** Creates a new {@link AudioBufferSink} for the given {@link InputAudioTrack}. */
- constructor(audioTrack: InputAudioTrack) {
- if (!(audioTrack instanceof InputAudioTrack)) {
- throw new TypeError('audioTrack must be an InputAudioTrack.');
- }
-
- this._audioSampleSink = new AudioSampleSink(audioTrack);
- }
-
- /** @internal */
- _audioSampleToWrappedArrayBuffer(sample: AudioSample): WrappedAudioBuffer {
- const result: WrappedAudioBuffer = {
- buffer: sample.toAudioBuffer(),
- timestamp: sample.timestamp,
- duration: sample.duration,
- };
-
- sample.close();
- return result;
- }
-
- /**
- * Retrieves the audio buffer corresponding to the given timestamp, in seconds. More specifically, returns
- * the last audio buffer (in presentation order) with a start timestamp less than or equal to the given timestamp.
- * Returns null if the timestamp is before the track's first timestamp.
- *
- * @param timestamp - The timestamp used for retrieval, in seconds.
- */
- async getBuffer(timestamp: number) {
- validateTimestamp(timestamp);
-
- const data = await this._audioSampleSink.getSample(timestamp);
- return data && this._audioSampleToWrappedArrayBuffer(data);
- }
-
- /**
- * Creates an async iterator that yields audio buffers of this track in presentation order. This method
- * will intelligently pre-decode a few buffers ahead to enable fast iteration.
- *
- * @param startTimestamp - The timestamp in seconds at which to start yielding buffers (inclusive).
- * @param endTimestamp - The timestamp in seconds at which to stop yielding buffers (exclusive).
- */
- buffers(startTimestamp = 0, endTimestamp = Infinity) {
- return mapAsyncGenerator(
- this._audioSampleSink.samples(startTimestamp, endTimestamp),
- data => this._audioSampleToWrappedArrayBuffer(data),
- );
- }
-
- /**
- * Creates an async iterator that yields an audio buffer for each timestamp in the argument. This method
- * uses an optimized decoding pipeline if these timestamps are monotonically sorted, decoding each packet at most
- * once, and is therefore more efficient than manually getting the buffer for every timestamp. The iterator may
- * yield null if no buffer is available for a given timestamp.
- *
- * @param timestamps - An iterable or async iterable of timestamps in seconds.
- */
- buffersAtTimestamps(timestamps: AnyIterable) {
- return mapAsyncGenerator(
- this._audioSampleSink.samplesAtTimestamps(timestamps),
- data => data && this._audioSampleToWrappedArrayBuffer(data),
- );
- }
-}
diff --git a/src/misc.ts b/src/misc.ts
index 0a6377a..d136f84 100644
--- a/src/misc.ts
+++ b/src/misc.ts
@@ -924,12 +924,6 @@ export class AsyncMutex2 {
}
}
-export interface AsyncMutexLock extends Disposable {
- readonly pending: boolean;
- readonly ready: Promise | null;
- release(): void;
-}
-
export class AsyncMutex4 {
locked = false;
private resolverQueue: (() => void)[] = [];
@@ -938,33 +932,16 @@ export class AsyncMutex4 {
if (!this.locked) {
// Fast path
this.locked = true;
- return this.createLock(false, null);
+ return new AsyncMutexLock(this, false, null);
}
const { promise, resolve } = promiseWithResolvers();
this.resolverQueue.push(resolve);
- return this.createLock(true, promise);
+ return new AsyncMutexLock(this, true, promise);
}
- private createLock(pending: boolean, ready: Promise | null): AsyncMutexLock {
- let released = false;
-
- return {
- pending,
- ready,
- release: () => {
- if (released) return;
- released = true;
- this.dispatch();
- },
- [Symbol.dispose]() {
- this.release();
- },
- };
- }
-
- private dispatch() {
+ dispatch() {
if (this.resolverQueue.length > 0) {
const resolve = this.resolverQueue.shift()!;
resolve();
@@ -974,6 +951,29 @@ export class AsyncMutex4 {
}
}
+export class AsyncMutexLock implements Disposable {
+ private released = false;
+
+ constructor(
+ private readonly mutex: AsyncMutex4,
+ public readonly pending: boolean,
+ public readonly ready: Promise | null,
+ ) {}
+
+ release() {
+ if (this.released) {
+ return;
+ }
+
+ this.released = true;
+ this.mutex.dispatch();
+ }
+
+ [Symbol.dispose]() {
+ this.release();
+ }
+}
+
export class CallSerializer2 {
private currentPromise: Promise | null = null;
private queuedCalls = 0;
diff --git a/src/mp3/mp3-demuxer.ts b/src/mp3/mp3-demuxer.ts
index 191902c..52f6403 100644
--- a/src/mp3/mp3-demuxer.ts
+++ b/src/mp3/mp3-demuxer.ts
@@ -11,7 +11,6 @@ import { Demuxer } from '../demuxer';
import { Input } from '../input';
import { InputAudioTrack, InputAudioTrackBacking } from '../input-track';
import { DEFAULT_TRACK_DISPOSITION, MetadataTags } from '../metadata';
-import { PacketRetrievalOptions } from '../media-sink';
import {
assert,
AsyncMutex4,
@@ -31,6 +30,7 @@ import {
} from '../id3';
import { readNextFrameHeader } from './mp3-reader';
import { readAscii, readBytes, Reader, readU32Be } from '../reader';
+import { PacketRetrievalOptions } from '../cursors';
type Sample = {
timestamp: number;
diff --git a/src/ogg/ogg-demuxer.ts b/src/ogg/ogg-demuxer.ts
index ef58c7c..3d44ef3 100644
--- a/src/ogg/ogg-demuxer.ts
+++ b/src/ogg/ogg-demuxer.ts
@@ -8,10 +8,10 @@
import { OPUS_SAMPLE_RATE } from '../codec';
import { parseModesFromVorbisSetupPacket, parseOpusIdentificationHeader, readVorbisComments } from '../codec-data';
+import { PacketRetrievalOptions } from '../cursors';
import { Demuxer } from '../demuxer';
import { Input } from '../input';
import { InputAudioTrack, InputAudioTrackBacking } from '../input-track';
-import { PacketRetrievalOptions } from '../media-sink';
import { DEFAULT_TRACK_DISPOSITION, MetadataTags } from '../metadata';
import {
assert,
diff --git a/src/sample.ts b/src/sample.ts
index 3d0de84..d30e101 100644
--- a/src/sample.ts
+++ b/src/sample.ts
@@ -334,38 +334,46 @@ export class VideoSample implements Disposable {
}
/** Clones this video sample. */
- clone() {
+ clone(override?: {
+ timestamp?: number;
+ duration?: number;
+ rotation?: Rotation;
+ }) {
if (this._closed) {
throw new Error('VideoSample is closed.');
}
assert(this._data !== null);
+ const timestamp = override?.timestamp ?? this.timestamp;
+ const duration = override?.duration ?? this.duration;
+ const rotation = override?.rotation ?? this.rotation;
+
if (isVideoFrame(this._data)) {
return new VideoSample(this._data.clone(), {
- timestamp: this.timestamp,
- duration: this.duration,
- rotation: this.rotation,
+ timestamp,
+ duration,
+ rotation,
});
} else if (this._data instanceof Uint8Array) {
return new VideoSample(this._data.slice(), {
format: this.format!,
codedWidth: this.codedWidth,
codedHeight: this.codedHeight,
- timestamp: this.timestamp,
- duration: this.duration,
+ timestamp,
+ duration,
colorSpace: this.colorSpace,
- rotation: this.rotation,
+ rotation,
});
} else {
return new VideoSample(this._data, {
format: this.format!,
codedWidth: this.codedWidth,
codedHeight: this.codedHeight,
- timestamp: this.timestamp,
- duration: this.duration,
+ timestamp,
+ duration,
colorSpace: this.colorSpace,
- rotation: this.rotation,
+ rotation,
});
}
}
@@ -782,36 +790,6 @@ export class VideoSample implements Disposable {
}
}
- /** Sets the rotation metadata of this video sample. */
- setRotation(newRotation: Rotation) {
- if (![0, 90, 180, 270].includes(newRotation)) {
- throw new TypeError('newRotation must be 0, 90, 180, or 270.');
- }
-
- // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion
- (this.rotation as Rotation) = newRotation;
- }
-
- /** Sets the presentation timestamp of this video sample, in seconds. */
- setTimestamp(newTimestamp: number) {
- if (!Number.isFinite(newTimestamp)) {
- throw new TypeError('newTimestamp must be a number.');
- }
-
- // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion
- (this.timestamp as number) = newTimestamp;
- }
-
- /** Sets the duration of this video sample, in seconds. */
- setDuration(newDuration: number) {
- if (!Number.isFinite(newDuration) || newDuration < 0) {
- throw new TypeError('newDuration must be a non-negative number.');
- }
-
- // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion
- (this.duration as number) = newDuration;
- }
-
/** Calls `.close()`. */
[Symbol.dispose]() {
this.close();
diff --git a/src/wave/wave-demuxer.ts b/src/wave/wave-demuxer.ts
index a63ab92..92be19f 100644
--- a/src/wave/wave-demuxer.ts
+++ b/src/wave/wave-demuxer.ts
@@ -10,12 +10,12 @@ import { AudioCodec } from '../codec';
import { Demuxer } from '../demuxer';
import { Input } from '../input';
import { InputAudioTrack, InputAudioTrackBacking } from '../input-track';
-import { PacketRetrievalOptions } from '../media-sink';
import { DEFAULT_TRACK_DISPOSITION, MetadataTags } from '../metadata';
import { assert, ResultValue, UNDETERMINED_LANGUAGE, Yo } from '../misc';
import { EncodedPacket, PLACEHOLDER_DATA } from '../packet';
import { readAscii, readBytes, Reader, readU16, readU32, readU64 } from '../reader';
import { parseId3V2Tag, readId3V2Header } from '../id3';
+import { PacketRetrievalOptions } from '../cursors';
export enum WaveFormat {
PCM = 0x0001,
diff --git a/test/browser/sample-cursor.test.ts b/test/browser/sample-cursor.test.ts
index 659a3d4..a65ac8e 100644
--- a/test/browser/sample-cursor.test.ts
+++ b/test/browser/sample-cursor.test.ts
@@ -7,6 +7,7 @@ import {
AudioSampleCursor,
canvasTransformer,
VideoSampleCursor,
+ WrappedCanvas,
} from '../../src/cursors.js';
import { AudioSample, VideoSample } from '../../src/sample.js';
import { promiseAllEnsureOrder, promiseIterateAll } from '../../src/misc.js';
@@ -221,15 +222,14 @@ test('Sample cursor advancing', async () => {
expect(cursor.current).toBe(null);
total = 0;
- await cursor.iterate(() => total++);
+ await cursor.iterate(() => void total++);
expect(total).toBe(0); // Since we're at the end
await cursor.seekToFirst(); ;
total = 0;
- await cursor.iterate((sample, stop) => {
+ await cursor.iterate((sample) => {
if (sample.timestamp === 1) {
- stop();
- return;
+ return false;
}
total++;
@@ -296,9 +296,9 @@ test('Sample cursor advancing, cold start', async () => {
break;
}
- await cursor2.iterate((sample, stop) => {
+ await cursor2.iterate((sample) => {
expect(sample.timestamp).toBe(0);
- stop();
+ return false;
});
await cursor2.close();
@@ -1076,6 +1076,7 @@ test('Canvas transformer', async () => {
});
const firstSample = (await cursor1.seekToFirst())!;
+ expect(firstSample).toBeInstanceOf(WrappedCanvas);
expect(firstSample.canvas).toBeInstanceOf(HTMLCanvasElement);
expect(firstSample.canvas.width).toBe(videoTrack.displayWidth);
expect(firstSample.canvas.height).toBe(videoTrack.displayHeight);
@@ -1211,3 +1212,35 @@ test('Unthrottled decoder', async () => {
// Test that all samples in the queue get closed now
await cursor2.close();
});
+
+test('hasNext', async () => {
+ using input = new Input({
+ source: new UrlSource('/trim-buck-bunny.mov'),
+ formats: ALL_FORMATS,
+ });
+
+ const videoTrack = (await input.getPrimaryVideoTrack())!;
+ await using cursor = new VideoSampleCursor(videoTrack);
+
+ expect(cursor.current).toBe(null);
+ expect(await cursor.hasNext()).toBe(true);
+
+ await cursor.seekToFirst();
+ expect(await cursor.hasNext()).toBe(true);
+
+ await cursor.seekTo(Infinity);
+ expect(cursor.current).not.toBe(null);
+ expect(await cursor.hasNext()).toBe(false);
+
+ await cursor.next();
+ expect(cursor.current).toBe(null);
+ expect(await cursor.hasNext()).toBe(false);
+
+ void cursor.seekTo(2.5);
+ expect(await cursor.hasNext()).toBe(true);
+
+ void cursor.seekTo(Infinity);
+ expect(await cursor.hasNext()).toBe(false);
+
+ expect(cursor.hasNext()).not.toBeInstanceOf(Promise);
+});
diff --git a/test/node/packet-reading.test.ts b/test/node/packet-reading.test.ts
index 8bb8845..1f86591 100644
--- a/test/node/packet-reading.test.ts
+++ b/test/node/packet-reading.test.ts
@@ -178,17 +178,16 @@ test('Packet cursor iteration', async () => {
total = 0;
await cursor.seekToFirst();
- await cursor.iterate(() => total++);
+ await cursor.iterate(() => void total++);
expect(total).toBe(121);
expect(cursor.current).toBe(null);
total = 0;
await cursor.seekToFirst();
- await cursor.iterate((packet, stop) => {
+ await cursor.iterate((packet) => {
if (packet.timestamp === 1) {
- stop();
- return;
+ return false;
}
total++;
@@ -205,7 +204,7 @@ test('Packet cursor iteration', async () => {
await cursor.seekTo(-Infinity);
total = 0;
- await cursor.iterate(() => total++);
+ await cursor.iterate(() => void total++);
expect(total).toBe(121);
@@ -300,9 +299,9 @@ test('Command queuing', async () => {
expect(resolved[12]!.type).toBe('key');
void cursor.seekTo(1);
- await cursor.iterate((packet, stop) => {
+ await cursor.iterate((packet) => {
expect(packet.timestamp).toBe(1);
- stop();
+ return false;
});
void cursor.seekTo(3);