Add BufferTarget.onFinalize, OutputOptions.onFinalize, ConcurrentRunner (#349)

* Add BufferTarget.onFinalize option for awaitable async action on finalize

The `finalized` event fires synchronously and its return value is ignored,
making it unsuitable for use cases where the muxer should wait (e.g.
uploading the buffer to S3, R2, or other object stores that require a
known Content-Length and therefore can't stream via StreamTarget).

Adds a new `BufferTargetOptions` type with an `onFinalize` callback that
the muxer awaits before resolving. Matches the existing callback pattern
used by `HlsOutputFormatOptions.onSegment`, `onMaster`, etc.

When used with PathedTarget, this provides proper backpressure: the next
segment won't start being produced until the previous one has finished
uploading, keeping memory bounded regardless of video length.

- Adds `BufferTargetOptions` type, exported from the package root
- `BufferTarget` constructor now accepts optional options (backward compatible)
- `_finalize()` awaits `onFinalize` before emitting the `finalized` event
- Adds runtime validation for non-function callbacks
- Updates "Upload to a server" docs in writing-hls.md with S3 pattern
- Documents `onFinalize` in writing-media-files.md BufferTarget section
- Adds tests for callback invocation, async awaiting, backward compat, and validation

* Add ConcurrentRunner, add OutputOptions.onFinalize, adjust docs accordingly, clean up tests

* give me more control baby

* forgot to tell about it

* polish guide and js docs

* vanilagy says remove this, docs will reveal it

---------

Co-authored-by: Vanilagy <[email protected]>

src/index.ts

@@ -130,6 +130,7 @@ export {
 	TargetEvents,
 	TargetRequest,
 	BufferTarget,
+	BufferTargetOptions,
 	FilePathTarget,
 	FilePathTargetOptions,
 	NullTarget,
@@ -141,6 +142,7 @@ export {
 } from './target';
 export {
 	AnyIterable,
+	ConcurrentRunner,
 	EventEmitter,
 	EventListenerOptions,
 	FilePath,

src/misc.ts

@@ -1235,3 +1235,66 @@ export class EventEmitter<TEvents extends Record<string, unknown>> {
 }
 
 export const ceilToMultipleOfTwo = (value: number) => Math.ceil(value / 2) * 2;
+
+/**
+ * Utility class for running async functions in parallel up to a certain level of parallelism. Can be used to apply
+ * backpressure only if the concurrency level would be exceeded.
+ *
+ * @group Miscellaneous
+ * @public
+*/
+export class ConcurrentRunner {
+	/** @internal */
+	_queue: Promise<unknown>[] = [];
+	/** @internal */
+	_errored = false;
+
+	/**
+	 * The maximum number of in-flight promises. You can also think of it as the "high water mark".
+	 * You can set this value to dynamically change the level of parallelism.
+	 */
+	parallelism: number;
+
+	constructor(parallelism: number) {
+		this.parallelism = parallelism;
+	}
+
+	/** Whether any function has errored. The runner is effectively bricked if this is `true`, by design. */
+	get errored() {
+		return this._errored;
+	}
+
+	/** The number of tasks currently running. */
+	get inFlightCount() {
+		return this._queue.length;
+	}
+
+	/**
+	 * Schedules an async function to be run. If the maximum allowed level of parallelism has not yet been reached,
+	 * the function will be executed immediately and `run()` will resolve immediately. Otherwise, the function will be
+	 * called as soon as any currently-running function finishes, and `run()` will only resolve then.
+	 *
+	 * Throws if the runner is errored.
+	 */
+	async run(fn: () => Promise<unknown>) {
+		if (this._errored) {
+			await Promise.race(this._queue); // Will surface the error
+		}
+
+		while (this._queue.length >= this.parallelism) {
+			await Promise.race(this._queue);
+		}
+
+		const promise = fn();
+		this._queue.push(promise);
+
+		void promise
+			.then(() => removeItem(this._queue, promise))
+			.catch(() => this._errored = true);
+	}
+
+	/** Waits for all currently running functions to finish. Throws if the runner is errored. */
+	async flush() {
+		await Promise.all(this._queue);
+	}
+}

src/output.ts

@@ -324,6 +324,11 @@ export type OutputOptions<
 	 * When this is a function, it will only be called if an init target is needed.
 	 */
 	initTarget?: T | (() => MaybePromise<T>);
+	/**
+	 * Optional; a callback to be called at the end of {@link Output.finalize}. Can be used to run logic once the
+	 * output has completed. If a promise is returned, it will be awaited internally by {@link Output.finalize}.
+	 */
+	onFinalize?: () => MaybePromise<unknown>;
 };
 
 /**
@@ -369,6 +374,8 @@ export class Output<
 	/** @internal */
 	private _initTarget: T | (() => MaybePromise<T>) | null;
 	/** @internal */
+	_onFinalize: (() => MaybePromise<unknown>) | null = null;
+	/** @internal */
 	_muxer: Muxer;
 	/** @internal */
 	_targets = new Set<Target>();
@@ -449,9 +456,13 @@ export class Output<
 				+ ' a Target.',
 			);
 		}
+		if (options.onFinalize !== undefined && typeof options.onFinalize !== 'function') {
+			throw new TypeError('options.onFinalize, when provided, must be a function.');
+		}
 
 		this.format = options.format;
 		this._target = options.target;
+		this._onFinalize = options.onFinalize ?? null;
 
 		this._initTarget = options.initTarget ?? null;
 		if (this._initTarget instanceof Target) {
@@ -881,6 +892,10 @@ export class Output<
 					}
 				}
 
+				if (this._onFinalize) {
+					await this._onFinalize();
+				}
+
 				this.state = 'finalized';
 			} finally {
 				release();

src/target.ts

@@ -89,6 +89,22 @@ export abstract class Target extends EventEmitter<TargetEvents> {
 const ARRAY_BUFFER_INITIAL_SIZE = 2 ** 16;
 const ARRAY_BUFFER_MAX_SIZE = 2 ** 32;
 
+/**
+ * Options for {@link BufferTarget}.
+ * @group Output targets
+ * @public
+ */
+export type BufferTargetOptions = {
+	/**
+	 * Called once the target has been finalized, with the complete output buffer. If you return a promise, it will be
+	 * used to apply backpressure internally.
+	 *
+	 * One use for this callback is for uploading to a server where the full buffer must be known before
+	 * sending (e.g. S3 PutObject) and stream-uploading is not an option.
+	 */
+	onFinalize?: (buffer: ArrayBuffer) => MaybePromise<unknown>;
+};
+
 /**
  * A target that writes data directly into an ArrayBuffer in memory. Great for performance, but not suitable for very
  * large files. The buffer will be available once the output has been finalized.
@@ -107,11 +123,22 @@ export class BufferTarget extends Target {
 	_maxPos = 0;
 	/** @internal */
 	_supportsResize: boolean;
+	/** @internal */
+	_options: BufferTargetOptions;
 
 	/** Creates a new {@link BufferTarget}. The buffer holding the data will be created and managed internally. */
-	constructor() {
+	constructor(options: BufferTargetOptions = {}) {
 		super();
 
+		if (!options || typeof options !== 'object') {
+			throw new TypeError('BufferTarget options, when provided, must be an object.');
+		}
+		if (options.onFinalize !== undefined && typeof options.onFinalize !== 'function') {
+			throw new TypeError('options.onFinalize, when provided, must be a function.');
+		}
+
+		this._options = options;
+
 		this._supportsResize = 'resize' in new ArrayBuffer(0);
 		if (this._supportsResize) {
 			try {
@@ -178,6 +205,11 @@ export class BufferTarget extends Target {
 	/** @internal */
 	async _finalize() {
 		this.buffer = this._buffer.slice(0, this._maxPos);
+
+		if (this._options.onFinalize) {
+			await this._options.onFinalize(this.buffer);
+		}
+
 		this._emit('finalized');
 	}