Merge pull request #292 from fredrikj/parallelism

Add UrlSourceOptions.parallelism
This commit is contained in:
David P.
2026-02-03 16:03:55 +01:00
committed by GitHub
2 changed files with 14 additions and 3 deletions
+3
View File
@@ -506,6 +506,9 @@ type UrlSourceOptions = {
// in memory. Defaults to 8 MiB. // in memory. Defaults to 8 MiB.
maxCacheSize?: number; maxCacheSize?: number;
// The maximum number of parallel requests to use for fetching. Defaults to 2.
parallelism?: number;
// Used to provide a custom fetch function // Used to provide a custom fetch function
fetchFn?: typeof fetch; fetchFn?: typeof fetch;
}; };
+11 -3
View File
@@ -343,6 +343,9 @@ export type UrlSourceOptions = {
/** The maximum number of bytes the cache is allowed to hold in memory. Defaults to 64 MiB. */ /** The maximum number of bytes the cache is allowed to hold in memory. Defaults to 64 MiB. */
maxCacheSize?: number; maxCacheSize?: number;
/** The maximum number of parallel requests to use for fetching. Defaults to 2. */
parallelism?: number;
/** /**
* A WHATWG-compatible fetch function. You can use this field to polyfill the `fetch` function, add missing * A WHATWG-compatible fetch function. You can use this field to polyfill the `fetch` function, add missing
* features, or use a custom implementation. * features, or use a custom implementation.
@@ -403,6 +406,9 @@ export class UrlSource extends Source {
) { ) {
throw new TypeError('options.maxCacheSize, when provided, must be a non-negative number.'); throw new TypeError('options.maxCacheSize, when provided, must be a non-negative number.');
} }
if (options.parallelism !== undefined && (!Number.isInteger(options.parallelism) || options.parallelism < 1)) {
throw new TypeError('options.parallelism, when provided, must be a positive number.');
}
if (options.fetchFn !== undefined && typeof options.fetchFn !== 'function') { if (options.fetchFn !== undefined && typeof options.fetchFn !== 'function') {
throw new TypeError('options.fetchFn, when provided, must be a function.'); throw new TypeError('options.fetchFn, when provided, must be a function.');
// Won't bother validating this function beyond this // Won't bother validating this function beyond this
@@ -414,11 +420,13 @@ export class UrlSource extends Source {
this._options = options; this._options = options;
this._getRetryDelay = options.getRetryDelay ?? DEFAULT_RETRY_DELAY; this._getRetryDelay = options.getRetryDelay ?? DEFAULT_RETRY_DELAY;
this._orchestrator = new ReadOrchestrator({
maxCacheSize: options.maxCacheSize ?? (64 * 2 ** 20 /* 64 MiB */),
// Most files in the real-world have a single sequential access pattern, but having two in parallel can // Most files in the real-world have a single sequential access pattern, but having two in parallel can
// also happen // also happen
maxWorkerCount: 2, const DEFAULT_PARALLELISM = 2;
this._orchestrator = new ReadOrchestrator({
maxCacheSize: options.maxCacheSize ?? (64 * 2 ** 20 /* 64 MiB */),
maxWorkerCount: options.parallelism ?? DEFAULT_PARALLELISM,
runWorker: this._runWorker.bind(this), runWorker: this._runWorker.bind(this),
prefetchProfile: PREFETCH_PROFILES.network, prefetchProfile: PREFETCH_PROFILES.network,
}); });