Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore: Replace ts-private with #private in all methods (#4185)
Jan Buchara256d2321bee
diff --git a/docs/guides/custom-logger/pino.ts b/docs/guides/custom-logger/pino.tsindex 2cf60813a..43927a32a 100644--- a/docs/guides/custom-logger/pino.ts+++ b/docs/guides/custom-logger/pino.ts@@ -13,20 +13,20 @@ const CRAWLEE_TO_PINO: Record<number, string> = {};class PinoAdapter extends BaseCrawleeLogger {- constructor(- private logger: pino.Logger,- options?: Partial<CrawleeLoggerOptions>,- ) {+ #logger: pino.Logger;++ constructor(logger: pino.Logger, options?: Partial<CrawleeLoggerOptions>) {super(options);+ this.#logger = logger;}logWithLevel(level: number, message: string, data?: Record<string, unknown>): void {const pinoLevel = CRAWLEE_TO_PINO[level] ?? 'info';- this.logger[pinoLevel as pino.Level](data ?? {}, message);+ this.#logger[pinoLevel as pino.Level](data ?? {}, message);}protected createChild(options: Partial<CrawleeLoggerOptions>): CrawleeLogger {- return new PinoAdapter(this.logger.child({ prefix: options.prefix }), { ...this.getOptions(), ...options });+ return new PinoAdapter(this.#logger.child({ prefix: options.prefix }), { ...this.getOptions(), ...options });}}diff --git a/docs/guides/custom-logger/winston.ts b/docs/guides/custom-logger/winston.tsindex 9a967988b..fabc5a9a2 100644--- a/docs/guides/custom-logger/winston.ts+++ b/docs/guides/custom-logger/winston.ts@@ -13,20 +13,20 @@ const CRAWLEE_TO_WINSTON: Record<number, string> = {};class WinstonAdapter extends BaseCrawleeLogger {- constructor(- private logger: winston.Logger,- options?: Partial<CrawleeLoggerOptions>,- ) {+ #logger: winston.Logger;++ constructor(logger: winston.Logger, options?: Partial<CrawleeLoggerOptions>) {super(options);+ this.#logger = logger;}logWithLevel(level: number, message: string, data?: Record<string, unknown>): void {const winstonLevel = CRAWLEE_TO_WINSTON[level] ?? 'info';- this.logger.log(winstonLevel, message, data);+ this.#logger.log(winstonLevel, message, data);}protected createChild(options: Partial<CrawleeLoggerOptions>): CrawleeLogger {- return new WinstonAdapter(this.logger.child({ prefix: options.prefix }), { ...this.getOptions(), ...options });+ return new WinstonAdapter(this.#logger.child({ prefix: options.prefix }), { ...this.getOptions(), ...options });}}diff --git a/oxlint.config.ts b/oxlint.config.tsindex 17f342ef9..7128ccac9 100644--- a/oxlint.config.ts+++ b/oxlint.config.ts@@ -57,8 +57,8 @@ export default defineConfig({},],// Counterpart to `no-underscore-dangle` above: private members should be native `#` fields,- // not the TypeScript `private` modifier. `warn` until the existing usages are migrated.- 'crawlee/prefer-private-fields': 'warn',+ // not the TypeScript `private` modifier.+ 'crawlee/prefer-private-fields': 'error','typescript/consistent-type-imports': ['error', { disallowTypeAnnotations: false }],'typescript/consistent-type-definitions': ['error', 'interface'],diff --git a/packages/basic-crawler/src/internals/autoscaling/autoscaled_pool.ts b/packages/basic-crawler/src/internals/autoscaling/autoscaled_pool.tsindex 33263ea97..711f73e5b 100644--- a/packages/basic-crawler/src/internals/autoscaling/autoscaled_pool.ts+++ b/packages/basic-crawler/src/internals/autoscaling/autoscaled_pool.ts@@ -205,7 +205,6 @@ export class AutoscaledPool {this.#isStopped = false;this.#resolve = null;this.#reject = null;- this.maybeRunTask = this.maybeRunTask.bind(this);}/**@@ -259,13 +258,16 @@ export class AutoscaledPool {// This is here because if we scale down to let's say 1, then after each promise is finished// this.maybeRunTask() doesn't trigger another one. So if that 1 instance gets stuck it results// in the crawler getting stuck and even after scaling up it never triggers another promise.- this.#maybeRunInterval = betterSetInterval(this.maybeRunTask, this.#maybeRunIntervalMillis);+ this.#maybeRunInterval = betterSetInterval(+ (cb: () => void) => this.#maybeRunTask(cb),+ this.#maybeRunIntervalMillis,+ );try {await poolPromise;} finally {// If resolve is null, the pool is already destroyed.- if (this.#resolve) await this.destroy();+ if (this.#resolve) await this.#destroy();}}@@ -284,7 +286,7 @@ export class AutoscaledPool {this.#isStopped = true;if (this.#resolve) {this.#resolve();- await this.destroy();+ await this.#destroy();}}@@ -347,7 +349,7 @@ export class AutoscaledPool {* every `maybeRunIntervalSecs` seconds. If you want to trigger the processing immediately, use this method.*/async notify(): Promise<void> {- setImmediate(this.maybeRunTask);+ setImmediate(() => this.#maybeRunTask());}/**@@ -358,7 +360,7 @@ export class AutoscaledPool {** It doesn't allow multiple concurrent runs of this method.*/- private async maybeRunTask(intervalCallback?: () => void): Promise<void> {+ async #maybeRunTask(intervalCallback?: () => void): Promise<void> {this.#log.perf('Attempting to run a task.');// Check if the function was invoked by the maybeRunInterval and use an empty function if not.const done = intervalCallback || (() => {});@@ -381,7 +383,7 @@ export class AutoscaledPool {// to notice that *this* pool has run out of work — `maybeFinish()` is the only thing that ever resolves// `run()`. It no-ops while this pool has tasks of its own in flight, which is every case in which an// unshared governor reports no capacity.- return this.maybeFinish();+ return this.#maybeFinish();}// - a task is ready.this.#queryingIsTaskReady = true;@@ -405,7 +407,7 @@ export class AutoscaledPool {this.#log.perf('Task will not run. No tasks are ready.');done();// No tasks could mean that we're finished with all tasks.- return this.maybeFinish();+ return this.#maybeFinish();}const taskId = String(this.#nextTaskId++);@@ -422,7 +424,7 @@ export class AutoscaledPool {// Everything's fine. Run task.// Try to run next task to build up concurrency,// but defer it so it doesn't create a cycle.- setImmediate(this.maybeRunTask);+ setImmediate(() => this.#maybeRunTask());// We need to restart interval here, so that it doesn't get blocked by a stalled task.done();@@ -443,7 +445,7 @@ export class AutoscaledPool {this.#log.perf('Task finished.');// Run task after the previous one finished. Only on success: a failed task rejects the pool, and// nudging the loop afterwards could start work on an already destroyed pool.- setImmediate(this.maybeRunTask);+ setImmediate(() => this.#maybeRunTask());} catch (e) {const err = e as Error;this.#log.perf('Running a task failed.');@@ -472,7 +474,7 @@ export class AutoscaledPool {** It doesn't allow multiple concurrent runs of this method.*/- private async maybeFinish(): Promise<void> {+ async #maybeFinish(): Promise<void> {if (this.#queryingIsFinished) return;if (this.#ownConcurrency > 0) return;@@ -495,7 +497,7 @@ export class AutoscaledPool {/*** Cleans up resources.*/- private async destroy(): Promise<void> {+ async #destroy(): Promise<void> {this.#resolve = null;this.#reject = null;diff --git a/packages/basic-crawler/src/internals/autoscaling/concurrency_system.ts b/packages/basic-crawler/src/internals/autoscaling/concurrency_system.tsindex 32af4d146..225ca8a58 100644--- a/packages/basic-crawler/src/internals/autoscaling/concurrency_system.ts+++ b/packages/basic-crawler/src/internals/autoscaling/concurrency_system.ts@@ -216,11 +216,11 @@ export interface IConcurrencySystem {* @category Scaling*/export class ConcurrencySystem implements IConcurrencySystem {- private readonly log: CrawleeLogger;+ readonly #log: CrawleeLogger;- private readonly desiredConcurrencyRatio: number;- private readonly scaleUpStepRatio: number;- private readonly scaleDownStepRatio: number;+ readonly #desiredConcurrencyRatio: number;+ readonly #scaleUpStepRatio: number;+ readonly #scaleDownStepRatio: number;readonly #loggingIntervalMillis: number;readonly #autoscaleIntervalMillis: number;/** The cap on tasks started per minute, or `Infinity` when uncapped. */@@ -233,9 +233,9 @@ export class ConcurrencySystem implements IConcurrencySystem {#lastLoggingTime?: number;#tasksPerMinute: number[] = Array.from({ length: 60 }, () => 0);- private readonly snapshotter: Snapshotter;+ readonly #snapshotter: Snapshotter;readonly #loadSignals: LoadSignal[];- private readonly systemStatus: SystemStatus;+ readonly #systemStatus: SystemStatus;#autoscaleInterval?: BetterIntervalID;#tasksDonePerSecondInterval?: BetterIntervalID;@@ -266,11 +266,11 @@ export class ConcurrencySystem implements IConcurrencySystem {maxTasksPerMinute,} = parseArgument(options, concurrencySystemOptionsSchema, 'ConcurrencySystemOptions');- this.log = log.child({ prefix: 'ConcurrencySystem' });+ this.#log = log.child({ prefix: 'ConcurrencySystem' });- this.desiredConcurrencyRatio = desiredConcurrencyRatio;- this.scaleUpStepRatio = scaleUpStepRatio;- this.scaleDownStepRatio = scaleDownStepRatio;+ this.#desiredConcurrencyRatio = desiredConcurrencyRatio;+ this.#scaleUpStepRatio = scaleUpStepRatio;+ this.#scaleDownStepRatio = scaleDownStepRatio;this.#loggingIntervalMillis = (loggingIntervalSecs ?? 0) * 1000;this.#autoscaleIntervalMillis = autoscaleIntervalSecs * 1000;this.maxTasksPerMinute = maxTasksPerMinute;@@ -278,18 +278,15 @@ export class ConcurrencySystem implements IConcurrencySystem {this.#minConcurrency = minConcurrency;this.#maxConcurrency = maxConcurrency;this.#desiredConcurrency = desiredConcurrency ?? minConcurrency;- this.clampDesiredConcurrency();-- this.autoscale = this.autoscale.bind(this);- this.incrementTasksDonePerSecond = this.incrementTasksDonePerSecond.bind(this);+ this.#clampDesiredConcurrency();// The built-in signals are collected by the snapshotter; custom ones are simply evaluated alongside them.const { custom: customLoadSignals = [], ...builtinSignalOptions } = loadSignals;- this.snapshotter = new Snapshotter(builtinSignalOptions);+ this.#snapshotter = new Snapshotter(builtinSignalOptions);this.#loadSignals = customLoadSignals;- this.systemStatus = new SystemStatus({- snapshotter: this.snapshotter,+ this.#systemStatus = new SystemStatus({+ snapshotter: this.#snapshotter,loadSignals: customLoadSignals,currentHistorySecs,// Both windows are requested from the signals explicitly, so a signal's own retention can neither widen@@ -314,7 +311,7 @@ export class ConcurrencySystem implements IConcurrencySystem {set minConcurrency(value: number) {parseArgument(value, concurrencySchema);this.#minConcurrency = value;- this.clampDesiredConcurrency();+ this.#clampDesiredConcurrency();}/**@@ -333,7 +330,7 @@ export class ConcurrencySystem implements IConcurrencySystem {set maxConcurrency(value: number) {parseArgument(value, concurrencySchema);this.#maxConcurrency = value;- this.clampDesiredConcurrency();+ this.#clampDesiredConcurrency();}/**@@ -353,7 +350,7 @@ export class ConcurrencySystem implements IConcurrencySystem {* meaningless. A contradictory pair (`minConcurrency > maxConcurrency`) resolves in favour of the maximum, since* that is the limit callers set in order to protect something.*/- private clampDesiredConcurrency(): void {+ #clampDesiredConcurrency(): void {const atLeastMin = Math.max(this.#desiredConcurrency, this.#minConcurrency);this.#desiredConcurrency = Math.min(atLeastMin, this.#maxConcurrency);}@@ -375,16 +372,16 @@ export class ConcurrencySystem implements IConcurrencySystem {async start(): Promise<void> {// Unwound and dropped again on failure, so a later `start()` retries instead of resolving instantly against// a system that is down.- this.#startPromise ??= this.boot().catch(async (error) => {+ this.#startPromise ??= this.#boot().catch(async (error) => {this.#startPromise = undefined;- await this.shutDown();+ await this.#shutDown();throw error;});await this.#startPromise;}- private async boot(): Promise<void> {+ async #boot(): Promise<void> {// Per-session measurement state, reset so a restarted system isn't judged on the previous session. The// per-minute window matters most: its ageing interval is cleared while we are down, so starts from before an// arbitrarily long stop would otherwise still count against "this minute" and trip the cap immediately.@@ -394,14 +391,20 @@ export class ConcurrencySystem implements IConcurrencySystem {// Signals are told how much history to keep when they start: exactly the longest window they will be sampled// over, so nobody has to guess a retention value that matches this system's configuration.- const startContext = { maxSampleWindowMillis: this.systemStatus.maxSampleWindowMillis };- await this.snapshotter.start(startContext);+ const startContext = { maxSampleWindowMillis: this.#systemStatus.maxSampleWindowMillis };+ await this.#snapshotter.start(startContext);await Promise.all(this.#loadSignals.map(async (s) => s.start(startContext)));- this.#autoscaleInterval = betterSetInterval(this.autoscale, this.#autoscaleIntervalMillis);+ this.#autoscaleInterval = betterSetInterval(+ (cb: () => void) => this.#autoscale(cb),+ this.#autoscaleIntervalMillis,+ );if (this.maxTasksPerMinute !== Infinity) {- this.#tasksDonePerSecondInterval = betterSetInterval(this.incrementTasksDonePerSecond, 1000);+ this.#tasksDonePerSecondInterval = betterSetInterval(+ (cb: () => void) => this.#incrementTasksDonePerSecond(cb),+ 1000,+ );}// Last, so `isRunning` never claims a system whose signals aren't collecting yet.@@ -425,13 +428,13 @@ export class ConcurrencySystem implements IConcurrencySystem {this.#startPromise = undefined;this.#running = false;- await this.shutDown();+ await this.#shutDown();}- private async shutDown(): Promise<void> {+ async #shutDown(): Promise<void> {if (this.#autoscaleInterval) betterClearInterval(this.#autoscaleInterval);if (this.#tasksDonePerSecondInterval) betterClearInterval(this.#tasksDonePerSecondInterval);- await this.snapshotter.stop();+ await this.#snapshotter.stop();await Promise.all(this.#loadSignals.map(async (s) => s.stop()));}@@ -441,13 +444,13 @@ export class ConcurrencySystem implements IConcurrencySystem {* {@apilink ConcurrencySystem.isRunning|`isRunning`} on the way in. Both the overload verdict and* `desiredConcurrency` are frozen at that point, so the borrowing pool would otherwise just quietly mis-scale.*/- private warnIfNotRunning(): void {+ #warnIfNotRunning(): void {if (this.#running || this.#warnedAboutQueryWhileStopped) {return;}this.#warnedAboutQueryWhileStopped = true;- this.log.warning(+ this.#log.warning('Capacity is being queried on a ConcurrencySystem that is not running, so system load is no longer being ' +'monitored and the concurrency will no longer be adjusted. Whoever creates a ConcurrencySystem owns ' +'its lifecycle: call `await concurrencySystem.stop()` only once every pool and crawler borrowing it ' +@@ -463,17 +466,17 @@ export class ConcurrencySystem implements IConcurrencySystem {* interface, letting the answer be queried directly.*/hasCapacityForTask(_consumer?: ConcurrencyConsumer): boolean {- this.warnIfNotRunning();+ this.#warnIfNotRunning();if (this.#currentConcurrency >= this.#desiredConcurrency) {- this.log.perf('Task will not run. Desired concurrency achieved.');+ this.#log.perf('Task will not run. Desired concurrency achieved.');return false;}- const currentStatus = this.systemStatus.getCurrentStatus();+ const currentStatus = this.#systemStatus.getCurrentStatus();const { isSystemIdle } = currentStatus;if (!isSystemIdle && this.#currentConcurrency >= this.#minConcurrency) {- this.log.perf(+ this.#log.perf('Task will not be run. System is overloaded.',currentStatus as unknown as Record<string, unknown>,);@@ -484,7 +487,7 @@ export class ConcurrencySystem implements IConcurrencySystem {}/** Whether the per-minute task cap has been reached. */- private get isOverMaxRequestLimit(): boolean {+ get #isOverMaxRequestLimit(): boolean {if (this.maxTasksPerMinute === Infinity) {return false;}@@ -506,8 +509,8 @@ export class ConcurrencySystem implements IConcurrencySystem {return false;}- if (this.isOverMaxRequestLimit) {- this.log.perf('Task will not run. Maximum tasks per minute reached.');+ if (this.#isOverMaxRequestLimit) {+ this.#log.perf('Task will not run. Maximum tasks per minute reached.');return false;}@@ -527,28 +530,28 @@ export class ConcurrencySystem implements IConcurrencySystem {* The one public window into load monitoring — useful for answering *why* a crawl is not scaling up.*/getCurrentStatus(): SystemInfo {- return this.systemStatus.getCurrentStatus();+ return this.#systemStatus.getCurrentStatus();}/*** Evaluates the historical system status and scales the shared desired concurrency up or down accordingly. Driven* by the autoscaling interval started in {@apilink ConcurrencySystem.start|`start()`}.*/- private autoscale(intervalCallback: () => void): void {- if (this.isOverMaxRequestLimit) return intervalCallback();+ #autoscale(intervalCallback: () => void): void {+ if (this.#isOverMaxRequestLimit) return intervalCallback();- const systemStatus = this.systemStatus.getHistoricalStatus();+ const systemStatus = this.#systemStatus.getHistoricalStatus();const { isSystemIdle } = systemStatus;const weAreNotAtMax = this.#desiredConcurrency < this.#maxConcurrency;- const minCurrentConcurrency = Math.floor(this.#desiredConcurrency * this.desiredConcurrencyRatio);+ const minCurrentConcurrency = Math.floor(this.#desiredConcurrency * this.#desiredConcurrencyRatio);const weAreReachingDesiredConcurrency = this.#currentConcurrency >= minCurrentConcurrency;- if (isSystemIdle && weAreNotAtMax && weAreReachingDesiredConcurrency) this.scaleUp(systemStatus);+ if (isSystemIdle && weAreNotAtMax && weAreReachingDesiredConcurrency) this.#scaleUp(systemStatus);const isSystemOverloaded = !isSystemIdle;const weAreNotAtMin = this.#desiredConcurrency > this.#minConcurrency;- if (isSystemOverloaded && weAreNotAtMin) this.scaleDown(systemStatus);+ if (isSystemOverloaded && weAreNotAtMin) this.#scaleDown(systemStatus);if (this.#loggingIntervalMillis > 0) {const now = Date.now();@@ -557,7 +560,7 @@ export class ConcurrencySystem implements IConcurrencySystem {this.#lastLoggingTime = now;} else if (now > this.#lastLoggingTime + this.#loggingIntervalMillis) {this.#lastLoggingTime = now;- this.log.info('state', {+ this.#log.info('state', {currentConcurrency: this.#currentConcurrency,desiredConcurrency: this.#desiredConcurrency,systemStatus,@@ -571,10 +574,10 @@ export class ConcurrencySystem implements IConcurrencySystem {/*** Scales the system up by increasing the desired concurrency by the scaleUpStepRatio.*/- private scaleUp(systemStatus: SystemInfo): void {- const step = Math.ceil(this.#desiredConcurrency * this.scaleUpStepRatio);+ #scaleUp(systemStatus: SystemInfo): void {+ const step = Math.ceil(this.#desiredConcurrency * this.#scaleUpStepRatio);this.#desiredConcurrency = Math.min(this.#maxConcurrency, this.#desiredConcurrency + step);- this.log.debug('scaling up', {+ this.#log.debug('scaling up', {oldConcurrency: this.#desiredConcurrency - step,newConcurrency: this.#desiredConcurrency,systemStatus,@@ -584,17 +587,17 @@ export class ConcurrencySystem implements IConcurrencySystem {/*** Scales the system down by decreasing the desired concurrency by the scaleDownStepRatio.*/- private scaleDown(systemStatus: SystemInfo): void {- const step = Math.ceil(this.#desiredConcurrency * this.scaleDownStepRatio);+ #scaleDown(systemStatus: SystemInfo): void {+ const step = Math.ceil(this.#desiredConcurrency * this.#scaleDownStepRatio);this.#desiredConcurrency = Math.max(this.#minConcurrency, this.#desiredConcurrency - step);- this.log.debug('scaling down', {+ this.#log.debug('scaling down', {oldConcurrency: this.#desiredConcurrency + step,newConcurrency: this.#desiredConcurrency,systemStatus,});}- private incrementTasksDonePerSecond(intervalCallback: () => void): void {+ #incrementTasksDonePerSecond(intervalCallback: () => void): void {this.#tasksPerMinute.unshift(0);this.#tasksPerMinute.pop();diff --git a/packages/basic-crawler/src/internals/autoscaling/memory_load_signal.ts b/packages/basic-crawler/src/internals/autoscaling/memory_load_signal.tsindex 52ea3a221..91417f3a3 100644--- a/packages/basic-crawler/src/internals/autoscaling/memory_load_signal.ts+++ b/packages/basic-crawler/src/internals/autoscaling/memory_load_signal.ts@@ -89,7 +89,7 @@ export class MemoryLoadSignal implements LoadSignal {);}// Fallback memory measurement in case memTotalBytes is missing from SystemInfo.- this.#maxMemoryBytes = await this.getTotalMemoryBytes();+ this.#maxMemoryBytes = await this.#getTotalMemoryBytes();}this.#events.on(EventType.SYSTEM_INFO, this.handle);@@ -121,10 +121,10 @@ export class MemoryLoadSignal implements LoadSignal {};this.#store.push(snapshot, createdAt);- this.memoryOverloadWarning(systemInfo, maxMemoryBytes);+ this.#memoryOverloadWarning(systemInfo, maxMemoryBytes);}- private memoryOverloadWarning(systemInfo: SystemInfo, maxMemoryBytes?: number): void {+ #memoryOverloadWarning(systemInfo: SystemInfo, maxMemoryBytes?: number): void {const effectiveMax = maxMemoryBytes ?? this.#maxMemoryBytes;const { memCurrentBytes } = systemInfo;const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();@@ -152,7 +152,7 @@ export class MemoryLoadSignal implements LoadSignal {}}- private async getTotalMemoryBytes(): Promise<number> {+ async #getTotalMemoryBytes(): Promise<number> {const containerized = this.#config.containerized ?? (await isContainerized());return (await getMemoryInfo({ containerized, logger: serviceLocator.getLogger() })).totalBytes;}diff --git a/packages/basic-crawler/src/internals/autoscaling/system_status.ts b/packages/basic-crawler/src/internals/autoscaling/system_status.tsindex 4d27c8512..ed6655f24 100644--- a/packages/basic-crawler/src/internals/autoscaling/system_status.ts+++ b/packages/basic-crawler/src/internals/autoscaling/system_status.ts@@ -104,7 +104,7 @@ export class SystemStatus {this.#historyMillis = historySecs * 1000;this.#signals = [...snapshotter.getLoadSignals(), ...loadSignals];- this.assertUniqueSignalNames();+ this.#assertUniqueSignalNames();}/**@@ -121,7 +121,7 @@ export class SystemStatus {* contradicts actual behavior: both signals are still evaluated (any overloaded one holds concurrency down), but* only the last is reported.*/- private assertUniqueSignalNames(): void {+ #assertUniqueSignalNames(): void {const seen = new Set<string>();for (const { name } of this.#signals) {@@ -155,7 +155,7 @@ export class SystemStatus {* and `true` otherwise.*/getCurrentStatus(): SystemInfo {- return this.isSystemIdle(this.#currentHistoryMillis);+ return this.#isSystemIdle(this.#currentHistoryMillis);}/**@@ -174,13 +174,13 @@ export class SystemStatus {* `historySecs` seconds and `true` otherwise.*/getHistoricalStatus(): SystemInfo {- return this.isSystemIdle(this.#historyMillis);+ return this.#isSystemIdle(this.#historyMillis);}/*** Returns a system status object.*/- private isSystemIdle(sampleDurationMillis?: number): SystemInfo {+ #isSystemIdle(sampleDurationMillis?: number): SystemInfo {const result: SystemInfo = {isSystemIdle: true,memInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex c47e11980..d00cc0e10 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -832,7 +832,7 @@ export class BasicCrawler<* Context built with this pipeline can be passed into multiple crawler pipelines at once.*/get #basicPipeline(): ContextPipeline<{ request: CrawlingRequest }, CrawlingContext> {- this.#basicContextPipeline ??= this.buildBasicContextPipeline();+ this.#basicContextPipeline ??= this.#buildBasicContextPipeline();return this.#basicContextPipeline;}@@ -842,7 +842,7 @@ export class BasicCrawler</** @internal */get contextPipeline(): ContextPipeline<CrawlingContext, ExtendedContext> {if (this.#contextPipeline === undefined) {- this.#contextPipeline = this.buildFinalContextPipeline();+ this.#contextPipeline = this.#buildFinalContextPipeline();}return this.#contextPipeline;@@ -870,7 +870,7 @@ export class BasicCrawler<readonly #maxCrawlDepth?: number;readonly #maxRequestsPerCrawl?: number;- private get handledRequestsCount(): number {+ get #handledRequestsCount(): number {return this.statistics.state.requestsSucceeded + this.statistics.state.requestsFailed;}@@ -1111,7 +1111,7 @@ export class BasicCrawler<throttleBy: 'registrableDomain',persistStateKey: `CRAWLEE_THROTTLED_DOMAINS_${this.#identity.id}`,// A factory, because the default queue is only opened on first use.- inner: suppliedManager ?? (() => this.openOwnedRequestQueue()),+ inner: suppliedManager ?? (() => this.#openOwnedRequestQueue()),}): suppliedManager;@@ -1119,7 +1119,7 @@ export class BasicCrawler<// The list is read first, while new requests still have somewhere writable to go.this.#requestManager = new RequestManagerTandem(requestList,- writableManager ?? (() => this.openOwnedRequestQueue()),+ writableManager ?? (() => this.#openOwnedRequestQueue()),);} else if (writableManager !== undefined) {// A RequestQueue is itself a request manager.@@ -1218,7 +1218,7 @@ export class BasicCrawler<this.#maxRequestsPerCrawl = maxRequestsPerCrawl;const isMaxPagesExceeded = () =>- this.#maxRequestsPerCrawl && this.#maxRequestsPerCrawl <= this.handledRequestsCount;+ this.#maxRequestsPerCrawl && this.#maxRequestsPerCrawl <= this.#handledRequestsCount;// eslint-disable-next-line prefer-constlet { isFinishedFunction, isTaskReadyFunction } = taskLoopOptions;@@ -1235,7 +1235,7 @@ export class BasicCrawler<const source = this.requestManager;if (!source) throw new Error('Request provider is not initialized!');- const request = await this.resolveRequest();+ const request = await this.#resolveRequest();if (!request) {return;}@@ -1256,7 +1256,7 @@ export class BasicCrawler<// Navigation, the navigation hooks and the request handler are timed individually, but the// phases between them are not, so a request could still get stuck indefinitely. This is the// catch-all for that - see `raceWithTimeout` for why it is a bare timer, not a timeout frame.- await this.withRequestTimeout(+ await this.#withRequestTimeout(crawlingContext,this.#basicPipeline.chain(this.contextPipeline).call(crawlingContext,@@ -1269,7 +1269,7 @@ export class BasicCrawler<// The internal timeout above fired, or `handleRequestFailure` threw a `SessionError` - both// take the normal error flow. Anything else is fatal.if (error instanceof SessionError || crawlingContext[timeoutExpiredKey]?.()) {- const unwrappedError = this.unwrapError(error);+ const unwrappedError = this.#unwrapError(error);await this.requestFunctionErrorHandler(unwrappedError,@@ -1277,12 +1277,12 @@ export class BasicCrawler<request,this.requestManager!,);- if (!this.errorAbsolvesSession(unwrappedError)) {+ if (!this.#errorAbsolvesSession(unwrappedError)) {crawlingContext.session?.markBad();}return;}- throw this.unwrapError(error);+ throw this.#unwrapError(error);}},isTaskReadyFunction: async () => {@@ -1304,14 +1304,14 @@ export class BasicCrawler<return false;}- return isTaskReadyFunction ? await isTaskReadyFunction() : await this.isTaskReadyFunction();+ return isTaskReadyFunction ? await isTaskReadyFunction() : await this.#isTaskReadyFunction();},isFinishedFunction: async () => {if (isMaxPagesExceeded()) {this.log.info(`Earlier, the crawler reached the maxRequestsPerCrawl limit of ${this.#maxRequestsPerCrawl} requests ` +'and all requests that were in progress at that time have now finished. ' +- `In total, the crawler processed ${this.handledRequestsCount} requests and will shut down.`,+ `In total, the crawler processed ${this.#handledRequestsCount} requests and will shut down.`,);return true;}@@ -1397,16 +1397,16 @@ export class BasicCrawler<* Builds the basic context pipeline that transforms `{ request }` into a full `CrawlingContext`.* This handles base context creation, session resolution, and context helpers.*/- private buildBasicContextPipeline(): ContextPipeline<{ request: CrawlingRequest }, CrawlingContext> {+ #buildBasicContextPipeline(): ContextPipeline<{ request: CrawlingRequest }, CrawlingContext> {return ContextPipeline.create<{ request: CrawlingRequest }>()- .compose(this.checkRobotsTxt.bind(this))- .compose(this.createBaseContext.bind(this))- .compose(this.resolveSession.bind(this))- .compose(this.createContextHelpers.bind(this));+ .compose(this.#checkRobotsTxt.bind(this))+ .compose(this.#createBaseContext.bind(this))+ .compose(this.#resolveSession.bind(this))+ .compose(this.#createContextHelpers.bind(this));}- private async checkRobotsTxt({ request }: { request: CrawlingRequest }) {- if (!(await this.isAllowedBasedOnRobotsTxtFile(request.url))) {+ async #checkRobotsTxt({ request }: { request: CrawlingRequest }) {+ if (!(await this.#isAllowedBasedOnRobotsTxtFile(request.url))) {this.log.warning(`Skipping request ${request.url} (${request.id}) because it is disallowed based on robots.txt`,);@@ -1423,7 +1423,7 @@ export class BasicCrawler<return {};}- private createBaseContext(context: PendingCrawlingContext, onCleanup: CleanupRegistrar) {+ #createBaseContext(context: PendingCrawlingContext, onCleanup: CleanupRegistrar) {return {id: cryptoRandomObjectId(10),log: this.log,@@ -1478,9 +1478,9 @@ export class BasicCrawler<* instance: managers key on `uniqueKey`/`id`, and it is this copy that flows back into `reclaimRequest` /* `markRequestAsHandled`, so nothing observes the difference.*/- private async resolveRequest(): Promise<CrawlingRequest | null> {- const fetched = await this.timeoutAndRetry(- this.fetchNextRequest.bind(this),+ async #resolveRequest(): Promise<CrawlingRequest | null> {+ const fetched = await this.#timeoutAndRetry(+ this.#fetchNextRequest.bind(this),this.internalTimeoutMillis,`Fetching next request timed out after ${this.internalTimeoutMillis / 1e3} seconds.`,);@@ -1496,8 +1496,8 @@ export class BasicCrawler<return request;}- private async resolveSession({ request }: { request: CrawlingRequest }) {- const session = await this.timeoutAndRetry(+ async #resolveSession({ request }: { request: CrawlingRequest }) {+ const session = await this.#timeoutAndRetry(async () => {const existingSession = await this.sessionPool.getSession(request.sessionId);@@ -1514,7 +1514,7 @@ export class BasicCrawler<return { session, proxyInfo: session?.proxyInfo };}- private async createContextHelpers({ request, session }: { request: CrawlingRequest; session: ISession }) {+ async #createContextHelpers({ request, session }: { request: CrawlingRequest; session: ISession }) {const addRequests: CrawlingContext['addRequests'] = async (requests, options = {}) => {const newCrawlDepth = request!.crawlDepth + 1;const requestsGenerator = this.addCrawlDepthRequestGenerator(requests, newCrawlDepth);@@ -1527,7 +1527,7 @@ export class BasicCrawler<return { addRequests, sendRequest };}- private buildFinalContextPipeline(): ContextPipeline<CrawlingContext, ExtendedContext> {+ #buildFinalContextPipeline(): ContextPipeline<CrawlingContext, ExtendedContext> {const subclassPipeline = (this.#contextPipelineOptions.contextPipelineBuilder?.() ??ContextPipeline.create<CrawlingContext>()) as ContextPipeline<CrawlingContext, Context>;@@ -1558,7 +1558,7 @@ export class BasicCrawler<contextPipeline = contextPipeline.compose(async (context) => {const { request } = context;- if (request && !this.requestMatchesEnqueueStrategy(request)) {+ if (request && !this.#requestMatchesEnqueueStrategy(request)) {// eslint-disable-next-line dot-notationconst message = `Skipping request ${request.id} (starting url: ${request.url} -> loaded url: ${request.loadedUrl}) because it does not match the enqueue strategy (${request['enqueueStrategy']}).`;this.log.debug(message);@@ -1616,7 +1616,7 @@ export class BasicCrawler<} satisfies EventStatusMessageData);}- private getPeriodicLogger() {+ #getPeriodicLogger() {let previousState = { ...this.statistics.state };const getOperationMode = (): { mode: 'ERROR' | 'REGULAR'; failedDelta: number } => {@@ -1746,19 +1746,19 @@ export class BasicCrawler<throw error;}- const periodicLogger = this.getPeriodicLogger();+ const periodicLogger = this.#getPeriodicLogger();this.setStatusMessage('Starting the crawler.', { level: 'INFO' });const sigintHandler = async () => {this.log.warning('Pausing... Press CTRL+C again to force exit. To resume, do: CRAWLEE_PURGE_ON_START=0 npm start',);- await this.pauseOnMigration();+ await this.#pauseOnMigration();await run.abort();};// Attach a listener to handle migration and aborting events gracefully.- const boundPauseOnMigration = this.pauseOnMigration.bind(this);+ const boundPauseOnMigration = this.#pauseOnMigration.bind(this);process.once('SIGINT', sigintHandler);const eventManager = serviceLocator.getEventManager();eventManager.on(EventType.MIGRATING, boundPauseOnMigration);@@ -1905,7 +1905,7 @@ export class BasicCrawler<*/async getRequestManager(): Promise<IRequestManager> {if (!this.#requestManager) {- this.#requestManager = await this.openOwnedRequestQueue();+ this.#requestManager = await this.#openOwnedRequestQueue();}// Apply the processing-time hint here (an async lifecycle point) rather than in the constructor,@@ -1913,7 +1913,7 @@ export class BasicCrawler<// but guard so we do not re-issue it on every call.if (!this.#requestManagerTimeoutsApplied) {this.#requestManagerTimeoutsApplied = true;- await this.applyRequestManagerTimeouts(this.#requestManager);+ await this.#applyRequestManagerTimeouts(this.#requestManager);}return this.#requestManager;@@ -1930,7 +1930,7 @@ export class BasicCrawler</*** Opens the default {@apilink RequestQueue} — the crawler's own, read from when the caller supplied nothing.*/- private async openOwnedRequestQueue(): Promise<RequestQueue> {+ async #openOwnedRequestQueue(): Promise<RequestQueue> {// The first crawler instance uses the default queue (null identifier);// subsequent instances get their own queue via a unique alias so they don't collide.const identifier = this.#identity.instanceIndex === 0 ? null : { alias: `__default_${this.#identity.id}__` };@@ -1945,7 +1945,7 @@ export class BasicCrawler<* request from being handed out a second time while it is still being processed — and it works* regardless of whether the manager is a plain {@apilink RequestQueue} or a `RequestManagerTandem`.*/- private async applyRequestManagerTimeouts(requestManager: IRequestManager): Promise<void> {+ async #applyRequestManagerTimeouts(requestManager: IRequestManager): Promise<void> {// A router route may hold a request for longer than the crawler's own timeout, and we cannot know// which routes a run will hit, so reserve for the longest one any route asked for. The hint is// raise-only, so erring high here is safe.@@ -1962,7 +1962,7 @@ export class BasicCrawler<* the request's label. Applied by the crawler on the add paths it owns — `crawler.addRequests`,* `crawler.run`, `context.addRequests` and `context.enqueueLinks`.*/- private async validateRequestUserData(source: Source | string): Promise<void> {+ async #validateRequestUserData(source: Source | string): Promise<void> {if (typeof source === 'string') {return;}@@ -2022,7 +2022,9 @@ export class BasicCrawler<const limit = Math.max(0,- this.#maxRequestsPerCrawl - this.handledRequestsCount - (await this.#getPendingRequestCountApproximation()),+ this.#maxRequestsPerCrawl -+ this.#handledRequestsCount -+ (await this.#getPendingRequestCountApproximation()),);return Math.min(limit, explicitLimit ?? Infinity);@@ -2089,7 +2091,7 @@ export class BasicCrawler<// its own per-item validation below, and validating an absent label/userData here would spuriously// check them against a registered default-route schema.if (options.label !== undefined || options.userData !== undefined) {- await this.validateRequestUserData({ label: options.label, userData: options.userData });+ await this.#validateRequestUserData({ label: options.label, userData: options.userData });}const requestLimit = await this.#calculateEnqueuedRequestLimit(options.limit);@@ -2101,9 +2103,9 @@ export class BasicCrawler<// (a URL must match an `include` pattern *and* satisfy the strategy). This mirrors crawlee-python.const enqueueStrategyPatterns = options.baseUrl ? buildEnqueueStrategyPatterns(options.baseUrl, strategy) : [];- const isAllowedBasedOnRobotsTxtFile = this.isAllowedBasedOnRobotsTxtFile.bind(this);+ const isAllowedBasedOnRobotsTxtFile = this.#isAllowedBasedOnRobotsTxtFile.bind(this);const maxCrawlDepth = this.#maxCrawlDepth;- const validateRequestUserData = this.validateRequestUserData.bind(this);+ const validateRequestUserData = this.#validateRequestUserData.bind(this);const allSkipped: { source: string | Source; reason: SkippedRequestReason }[] = [];// A skipped source (which can carry arbitrary userData) is only retained if something reads it -@@ -2346,7 +2348,7 @@ export class BasicCrawler<// floored per request so it will not actually cut them short, but the configured value is then effectively// ignored, which is worth flagging. Checked here (not in the constructor) because a subclass sets its// navigation timeout only after `super()`.- const phasesMillis = this.getNavigationTimeoutMillis() + this.resolveRequestHandlerTimeoutMillis(undefined);+ const phasesMillis = this.getNavigationTimeoutMillis() + this.#resolveRequestHandlerTimeoutMillis(undefined);if (this.internalTimeoutMillis < phasesMillis) {this.log.warning(`CRAWLEE_INTERNAL_TIMEOUT (${this.internalTimeoutMillis / 1000}s) is shorter than the navigation ` +@@ -2373,9 +2375,10 @@ export class BasicCrawler<* request, a per-route override, or a low `CRAWLEE_INTERNAL_TIMEOUT` is not cut short mid-phase. It takes* whichever is larger: the configured internal timeout, or this request's combined phase budget.*/- private async withRequestTimeout(crawlingContext: PendingCrawlingContext, work: Promise<void>): Promise<void> {+ async #withRequestTimeout(crawlingContext: PendingCrawlingContext, work: Promise<void>): Promise<void> {const { request } = crawlingContext;- const phasesMillis = this.getNavigationTimeoutMillis() + this.resolveRequestHandlerTimeoutMillis(request.label);+ const phasesMillis =+ this.getNavigationTimeoutMillis() + this.#resolveRequestHandlerTimeoutMillis(request.label);const timeoutMillis = Math.max(this.internalTimeoutMillis, phasesMillis);await raceWithTimeout(crawlingContext, work, { timeoutMillis, requestId: request.id });@@ -2388,18 +2391,18 @@ export class BasicCrawler<* @param label The request's route label, or `undefined` for the default route / no specific request.* @param fallbackMillis Timeout to use when no route overrides it.*/- private resolveRequestHandlerTimeoutMillis(+ #resolveRequestHandlerTimeoutMillis(label: string | undefined,fallbackMillis = this.#requestHandlerTimeoutMillis,): number {- return this.getRouteTimeoutMillis(label) ?? fallbackMillis;+ return this.#getRouteTimeoutMillis(label) ?? fallbackMillis;}/*** The timeout the router route with the given label asked for, or `undefined` when it did not override one* (or the request handler is not a router at all).*/- private getRouteTimeoutMillis(label: string | undefined): number | undefined {+ #getRouteTimeoutMillis(label: string | undefined): number | undefined {const getTimeoutSecs = (this.requestHandler as Partial<RouterHandler>).getTimeoutSecs;if (typeof getTimeoutSecs !== 'function') {@@ -2412,7 +2415,7 @@ export class BasicCrawler<}protected async runRequestHandler(crawlingContext: ExtendedContext): Promise<void> {- const timeoutMillis = this.resolveRequestHandlerTimeoutMillis(crawlingContext.request.label);+ const timeoutMillis = this.#resolveRequestHandlerTimeoutMillis(crawlingContext.request.label);await addTimeoutToPromise(async () => this.requestHandler(crawlingContext),@@ -2426,6 +2429,7 @@ export class BasicCrawler<* Deliberately does **not** commit on return - the pipeline hands failures to `handleRequestFailure`, so a* normal return says nothing about success. `handleRequest` commits on success.*/+ // oxlint-disable-next-line crawlee/prefer-private-fields -- accessed by testsprivate async runInStorageTransaction<T>(callback: () => Promise<T>): Promise<T> {if (!this.#transactionalStorageEnabled) {return callback();@@ -2472,7 +2476,7 @@ export class BasicCrawler<}}- private async isAllowedBasedOnRobotsTxtFile(url: string): Promise<boolean> {+ async #isAllowedBasedOnRobotsTxtFile(url: string): Promise<boolean> {if (!this.#respectRobotsTxtFile) {return true;}@@ -2483,7 +2487,7 @@ export class BasicCrawler<if (robotsTxtFile) {const crawlDelay = robotsTxtFile.getCrawlDelay(userAgent);if (crawlDelay !== undefined) {- this.applyCrawlDelay(url, crawlDelay);+ this.#applyCrawlDelay(url, crawlDelay);}}@@ -2520,7 +2524,7 @@ export class BasicCrawler<}/** Hands a robots.txt `Crawl-delay` to the request manager, warning if it will not be honoured. */- private applyCrawlDelay(url: string, delaySeconds: number): void {+ #applyCrawlDelay(url: string, delaySeconds: number): void {// robots.txt is per-origin; `hostname` is the closest pacing scope and errs wide (http and https to one// host share a clock).if (@@ -2567,7 +2571,7 @@ export class BasicCrawler<}}- private async pauseOnMigration() {+ async #pauseOnMigration() {const run = this.#dispatchingRun;if (run) {@@ -2591,7 +2595,7 @@ export class BasicCrawler</*** Fetches the next request to process from the underlying request provider.*/- private async fetchNextRequest() {+ async #fetchNextRequest() {if (this.requestManager === undefined) {throw new Error(`fetchNextRequest called on an uninitialized crawler`);}@@ -2621,7 +2625,7 @@ export class BasicCrawler<// Opened by `runInStorageTransaction`; absent when disabled or when the subclass opens its own.await currentStorageTransaction()?.commit();- await this.timeoutAndRetry(+ await this.#timeoutAndRetry(async () => requestSource.markRequestAsHandled(request!),this.internalTimeoutMillis,`Marking request ${request.url} (${request.id}) as handled timed out after ${@@ -2654,7 +2658,7 @@ export class BasicCrawler<const skipped =rawError instanceof ContextPipelineInitializationError &&rawError.cause instanceof ContextPipelineInterruptedError;- const err = this.unwrapError(rawError);+ const err = this.#unwrapError(rawError);const context = crawlingContext as CrawlingContext;const retryCountBefore = request.retryCount;const resumeRequestTimeout = crawlingContext[suspendTimeoutKey]?.();@@ -2664,7 +2668,7 @@ export class BasicCrawler<// (e.g., doesn't match enqueue strategy after redirect). Just return gracefully.if (skipped) {this.statistics.discardRequestRecord(request.id || request.uniqueKey);- await this.timeoutAndRetry(+ await this.#timeoutAndRetry(async () => requestSource.markRequestAsHandled(request),this.internalTimeoutMillis,`Marking request ${request.url} (${request.id}) as handled timed out after ${@@ -2684,7 +2688,7 @@ export class BasicCrawler<);request.state = RequestState.DONE;} catch (secondaryError) {- const unwrappedSecondaryError = this.unwrapError(secondaryError);+ const unwrappedSecondaryError = this.#unwrapError(secondaryError);const timedOut = unwrappedSecondaryError instanceof TimeoutError;request.state = RequestState.ERROR;@@ -2728,7 +2732,7 @@ export class BasicCrawler<// decrease the session score if the request fails (but the error handler did not throw);// skip when the error is a SessionError, which already retired the session- if (!skipped && !this.errorAbsolvesSession(err)) {+ if (!skipped && !this.#errorAbsolvesSession(err)) {context.session?.markBad();}}@@ -2759,7 +2763,7 @@ export class BasicCrawler<* Run async callback with given timeout and retry. Returns the result of the callback.* @ignore*/- private async timeoutAndRetry<T>(+ async #timeoutAndRetry<T>(handler: () => Promise<T>,timeout: number,error: Error | string,@@ -2772,7 +2776,7 @@ export class BasicCrawler<if (retried <= maxRetries) {// we retry on any error, not just timeoutthis.log.warning(`${(e as Error).message} (retrying ${retried}/${maxRetries})`);- return this.timeoutAndRetry(handler, timeout, error, maxRetries, retried + 1);+ return this.#timeoutAndRetry(handler, timeout, error, maxRetries, retried + 1);}throw e;@@ -2783,7 +2787,7 @@ export class BasicCrawler<* Whether the request manager has a request ready for processing. A manager that is only `waiting` also gets a* wake-up scheduled, so a paced crawl resumes on its clock rather than on the task loop's polling interval.*/- private async isTaskReadyFunction() {+ async #isTaskReadyFunction() {if (this.requestManager === undefined) {return false;}@@ -2801,13 +2805,13 @@ export class BasicCrawler<* Unwraps errors thrown by the context pipeline to get the actual user error.* RequestHandlerError, ContextPipelineInitializationError and ContextPipelineCleanupError wrap the actual error.*/- private unwrapError(error: unknown): Error {+ #unwrapError(error: unknown): Error {if (error instanceof RequestHandlerError ||error instanceof ContextPipelineInitializationError ||error instanceof ContextPipelineCleanupError) {- return this.unwrapError(error.cause);+ return this.#unwrapError(error.cause);}return error as Error;}@@ -2841,7 +2845,7 @@ export class BasicCrawler<throw error;}- const shouldRetryRequest = this.canRequestBeRetried(request, error);+ const shouldRetryRequest = this.#canRequestBeRetried(request, error);if (shouldRetryRequest) {await this.statistics.errorTrackerRetry.addAsync(error, crawlingContext);@@ -2941,11 +2945,11 @@ export class BasicCrawler<* Whether the session should be spared for this error - either because it was already retired, or because the* failure says nothing about the session (a rate limit is a property of the domain).*/- private errorAbsolvesSession(error: Error): boolean {+ #errorAbsolvesSession(error: Error): boolean {return error instanceof SessionError || error instanceof RequestThrottledError;}- private canRequestBeRetried(request: CrawlingRequest, error: Error) {+ #canRequestBeRetried(request: CrawlingRequest, error: Error) {// Request should never be retried, or the error encountered makes it not able to be retried.if (request.noRetry || error instanceof NonRetryableError) {return false;@@ -3017,7 +3021,7 @@ export class BasicCrawler<return request.headers?.Cookie || request.headers?.cookie || '';}- private requestMatchesEnqueueStrategy(request: CrawlingRequest) {+ #requestMatchesEnqueueStrategy(request: CrawlingRequest) {// If `skipNavigation` was used, just return `true`try {// eslint-disable-next-line @typescript-eslint/no-unused-expressionsdiff --git a/packages/basic-crawler/src/internals/crawlers/context_pipeline.ts b/packages/basic-crawler/src/internals/crawlers/context_pipeline.tsindex e6e24c604..a407c7cfc 100644--- a/packages/basic-crawler/src/internals/crawlers/context_pipeline.ts+++ b/packages/basic-crawler/src/internals/crawlers/context_pipeline.ts@@ -132,7 +132,7 @@ class ContextPipelineImpl<TContextBase, TCrawlingContext extends TContextBase> eother: ContextPipeline<TCrawlingContext, TFinalContext>,): ContextPipeline<TContextBase, TFinalContext> {const otherMiddlewares = Array.from(- (other as any).middlewareChain() as Iterable<ContextMiddleware<any, any>>,+ (other as unknown as ContextPipelineImpl<TCrawlingContext, TFinalContext>).#middlewareChain(),).reverse();let result: ContextPipeline<TContextBase, any> = this as any;@@ -143,7 +143,7 @@ class ContextPipelineImpl<TContextBase, TCrawlingContext extends TContextBase> ereturn result as ContextPipeline<TContextBase, TFinalContext>;}- private *middlewareChain() {+ *#middlewareChain() {let step: ContextPipelineImpl<TContextBase, TContextBase> | undefined = this as any;while (step !== undefined) {@@ -160,7 +160,7 @@ class ContextPipelineImpl<TContextBase, TCrawlingContext extends TContextBase> efinalContextConsumer: (finalContext: TCrawlingContext) => Promise<unknown>,onError: (error: unknown) => Awaitable<void>,): Promise<void> {- const middlewares = Array.from(this.middlewareChain()).reverse();+ const middlewares = Array.from(this.#middlewareChain()).reverse();const cleanupStack: Parameters<CleanupRegistrar>[0][] = [];const onCleanup: CleanupRegistrar = (cleanup) => {cleanupStack.push(cleanup);diff --git a/packages/basic-crawler/src/internals/crawlers/error_tracker.ts b/packages/basic-crawler/src/internals/crawlers/error_tracker.tsindex 48faa6d4c..76281d37d 100644--- a/packages/basic-crawler/src/internals/crawlers/error_tracker.ts+++ b/packages/basic-crawler/src/internals/crawlers/error_tracker.ts@@ -323,7 +323,7 @@ export class ErrorTracker {this.#total = 0;}- private updateGroup(error: ErrnoException) {+ #updateGroup(error: ErrnoException) {let group = this.#result;if (this.#options.showStackTrace) {@@ -350,7 +350,7 @@ export class ErrorTracker {add(error: ErrnoException) {this.#total++;- this.updateGroup(error);+ this.#updateGroup(error);if (typeof error.cause === 'object' && error.cause !== null) {this.add(error.cause);@@ -364,7 +364,7 @@ export class ErrorTracker {async addAsync(error: ErrnoException, context?: CrawlingContext) {this.#total++;- const group = this.updateGroup(error);+ const group = this.#updateGroup(error);// Capture a snapshot (screenshot and HTML) on the first occurrence of an errorif (group.count === 1 && context) {diff --git a/packages/basic-crawler/src/internals/crawlers/statistics.ts b/packages/basic-crawler/src/internals/crawlers/statistics.tsindex 7128138e5..69131be86 100644--- a/packages/basic-crawler/src/internals/crawlers/statistics.ts+++ b/packages/basic-crawler/src/internals/crawlers/statistics.ts@@ -305,7 +305,7 @@ export class Statistics<#logIntervalMillis: number;#logMessage: string;#requestsInProgress = new Map<number | string, RequestProcessingRecord>();- private readonly log: CrawleeLogger;+ readonly #log: CrawleeLogger;#logInterval: unknown;/**@@ -341,7 +341,7 @@ export class Statistics<this.id = id ?? String(Statistics.#id++);this.#persistStateKey = `CRAWLEE_CRAWLER_STATISTICS_${this.id}`;- this.log = (log ?? serviceLocator.getLogger()).child({ prefix: 'Statistics' });+ this.#log = (log ?? serviceLocator.getLogger()).child({ prefix: 'Statistics' });this.errorTracker = new ErrorTracker({ ...errorTrackerConfig, saveErrorSnapshots });this.errorTrackerRetry = new ErrorTracker({ ...errorTrackerConfig, saveErrorSnapshots });this.#logIntervalMillis = logIntervalSecs * 1000;@@ -373,7 +373,7 @@ export class Statistics<persistStateKey: this.#persistStateKey,persistenceEnabled: persistenceOptions.enable,keyValueStore,- logger: this.log,+ logger: this.#log,defaultState: () => this.#defaultState(),serialize: (state) => this.#serializeState(state),deserialize: (persistedState) => this.#deserializeState(persistedState),@@ -503,7 +503,7 @@ export class Statistics<const durationMillis = record.finish();this.state.requestsSucceeded++;this.state.requestTotalSucceededDurationMillis += durationMillis;- this.saveRetryCountForRequest(retryCount);+ this.#saveRetryCountForRequest(retryCount);if (durationMillis < this.state.requestMinDurationMillis) this.state.requestMinDurationMillis = durationMillis;if (durationMillis > this.state.requestMaxDurationMillis) this.state.requestMaxDurationMillis = durationMillis;this.#requestsInProgress.delete(id);@@ -515,7 +515,7 @@ export class Statistics<if (!record) return;this.state.requestTotalFailedDurationMillis += record.finish();this.state.requestsFailed++;- this.saveRetryCountForRequest(retryCount);+ this.#saveRetryCountForRequest(retryCount);this.#requestsInProgress.delete(id);}@@ -573,7 +573,7 @@ export class Statistics<}this.#logInterval = setInterval(() => {- this.log.info(this.#logMessage, {+ this.#log.info(this.#logMessage, {...this.calculate(),retryHistogram: this.requestRetryHistogram,});@@ -591,7 +591,7 @@ export class Statistics<await this.#recoverableState.teardown();}- private saveRetryCountForRequest(retryCount: number) {+ #saveRetryCountForRequest(retryCount: number) {if (retryCount > 0) this.state.requestsRetries++;this.requestRetryHistogram[retryCount] ??= 0;this.requestRetryHistogram[retryCount]++;@@ -607,7 +607,7 @@ export class Statistics<await this.#recoverableState.persistState().catch((error) =>- this.log.warning(`Failed to persist the statistics to ${this.#persistStateKey}`, { error }),+ this.#log.warning(`Failed to persist the statistics to ${this.#persistStateKey}`, { error }),);}@@ -626,7 +626,7 @@ export class Statistics<if (!restored.success) {// Statistics are bookkeeping - a record that cannot be made sense of is worth a warning and a fresh// start, not a failed crawl.- this.log.warning('Received invalid state from Key-value store, starting the statistics from scratch.', {+ this.#log.warning('Received invalid state from Key-value store, starting the statistics from scratch.', {persistStateKey: this.#persistStateKey,issues: restored.error.issues,});@@ -634,7 +634,7 @@ export class Statistics<return this.#defaultState();}- this.log.debug('Recreating state from KeyValueStore', { persistStateKey: this.#persistStateKey });+ this.#log.debug('Recreating state from KeyValueStore', { persistStateKey: this.#persistStateKey });return { ...this.#defaultState(), ...restored.data, ...this.#restoreStateExtension(persistedState) };}@@ -672,7 +672,7 @@ export class Statistics<} catch (error) {// Same policy as a built-in field that cannot be made sense of, but scoped to the custom ones - a// corrupt counter of your own is no reason to throw away the crawler's.- this.log.warning(+ this.#log.warning('Received invalid custom statistics fields from Key-value store, starting those from scratch.',{ persistStateKey: this.#persistStateKey, error },);diff --git a/packages/basic-crawler/src/internals/crawling_request.ts b/packages/basic-crawler/src/internals/crawling_request.tsindex 2b2ab87a7..1a7232587 100644--- a/packages/basic-crawler/src/internals/crawling_request.ts+++ b/packages/basic-crawler/src/internals/crawling_request.ts@@ -61,7 +61,7 @@ export class CrawlingRequest<UserData extends Dictionary = Dictionary> extends Rreturn super.fromSchema(schema) as CrawlingRequest<UserData>;}- private get crawleeData(): CrawlingRequestData {+ get #crawleeData(): CrawlingRequestData {return ((this.userData as Dictionary).__crawlee ??= {});}@@ -73,11 +73,11 @@ export class CrawlingRequest<UserData extends Dictionary = Dictionary> extends R* Accessing these properties will throw a {@apilink NavigationSkippedError} at runtime.*/get skipNavigation(): boolean {- return this.crawleeData.skipNavigation ?? false;+ return this.#crawleeData.skipNavigation ?? false;}set skipNavigation(value: boolean) {- this.crawleeData.skipNavigation = value;+ this.#crawleeData.skipNavigation = value;}/**@@ -85,43 +85,43 @@ export class CrawlingRequest<UserData extends Dictionary = Dictionary> extends R* Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.*/get crawlDepth(): number {- return this.crawleeData.crawlDepth ?? 0;+ return this.#crawleeData.crawlDepth ?? 0;}set crawlDepth(value: number) {- this.crawleeData.crawlDepth = value;+ this.#crawleeData.crawlDepth = value;}/** ID of a session to use for this request. When set, the crawler will fetch this session from the session pool instead of creating a new one. */get sessionId(): string | undefined {- return this.crawleeData.sessionId;+ return this.#crawleeData.sessionId;}set sessionId(value: string | undefined) {- this.crawleeData.sessionId = value;+ this.#crawleeData.sessionId = value;}/** Maximum number of retries for this request. Allows to override the global `maxRequestRetries` option of `BasicCrawler`. */get maxRetries(): number | undefined {- return this.crawleeData.maxRetries;+ return this.#crawleeData.maxRetries;}set maxRetries(value: number | undefined) {- this.crawleeData.maxRetries = value;+ this.#crawleeData.maxRetries = value;}/** Describes the request's current lifecycle state. */get state(): RequestState {- return this.crawleeData.state ?? RequestState.UNPROCESSED;+ return this.#crawleeData.state ?? RequestState.UNPROCESSED;}set state(value: RequestState) {- this.crawleeData.state = value;+ this.#crawleeData.state = value;}/** The strategy the request was enqueued under, if it came from `enqueueLinks`. @internal */get enqueueStrategy(): EnqueueStrategyOption | undefined {- return this.crawleeData.enqueueStrategy;+ return this.#crawleeData.enqueueStrategy;}/**diff --git a/packages/basic-crawler/src/internals/router.ts b/packages/basic-crawler/src/internals/router.tsindex 669e280a0..d413af6e6 100644--- a/packages/basic-crawler/src/internals/router.ts+++ b/packages/basic-crawler/src/internals/router.ts@@ -317,7 +317,7 @@ export class Router<): void;addHandler(label: string | symbol, handler: (ctx: any) => Awaitable<void>, options: RouteOptions = {}): void {- this.validate(label);+ this.#validate(label);this.#routes.set(label, handler);if (options.requestHandlerTimeoutSecs !== undefined) {@@ -336,7 +336,7 @@ export class Router<addDefaultHandler<UserData extends Dictionary = DefaultRouteUserData<Routes, GetUserDataFromRequest<Context['request']>>,>(handler: (ctx: RouterHandlerContext<Context, UserData, Routes>) => Awaitable<void>, options: RouteOptions = {}) {- this.validate(defaultRoute);+ this.#validate(defaultRoute);this.#routes.set(defaultRoute, handler);if (options.requestHandlerTimeoutSecs !== undefined) {@@ -423,7 +423,7 @@ export class Router<* Validates `request.userData` against the schema registered for its label (if any), replacing it with* the parsed value. Throws a {@apilink RequestValidationError} when validation fails.*/- private async validateRequest(context: Context) {+ async #validateRequest(context: Context) {const label = context.request.label;const schema = this.getSchema(label);@@ -439,7 +439,7 @@ export class Router</*** Throws when the label already exists in our registry.*/- private validate(label: string | symbol) {+ #validate(label: string | symbol) {if (this.#routes.has(label)) {const message =label === defaultRoute@@ -518,7 +518,7 @@ export class Router<const { url, loadedUrl, label } = context.request;context.log.debug('Page opened.', { label, url: loadedUrl ?? url });- await router.validateRequest(context);+ await router.#validateRequest(context);for (const middleware of router.#middlewares) {await middleware(context);diff --git a/packages/basic-crawler/src/internals/session_pool/session.ts b/packages/basic-crawler/src/internals/session_pool/session.tsindex e87278fd8..9182126af 100644--- a/packages/basic-crawler/src/internals/session_pool/session.ts+++ b/packages/basic-crawler/src/internals/session_pool/session.ts@@ -261,7 +261,7 @@ export class Session implements ISession {this.#errorScore -= this.#errorScoreDecrement;}- this.maybeSelfRetire();+ this.#maybeSelfRetire();}/**@@ -309,7 +309,7 @@ export class Session implements ISession {this.#errorScore += 1;this.#usageCount += 1;- this.maybeSelfRetire();+ this.#maybeSelfRetire();}/**@@ -337,7 +337,7 @@ export class Session implements ISession {* Makes an unusable session's state terminal. Unlike `retire()`, this leaves the error score alone so that* `isBlocked()` keeps telling a burned identity from one that merely ran out of uses.*/- private maybeSelfRetire(): void {+ #maybeSelfRetire(): void {if (!this.isUsable()) {this.#retired = true;}diff --git a/packages/basic-crawler/src/internals/session_pool/session_pool.ts b/packages/basic-crawler/src/internals/session_pool/session_pool.tsindex fcc869903..0c85cbff5 100644--- a/packages/basic-crawler/src/internals/session_pool/session_pool.ts+++ b/packages/basic-crawler/src/internals/session_pool/session_pool.ts@@ -265,7 +265,7 @@ export class SessionPool implements ISessionPool {// Pool Configurationthis.#maxPoolSize = maxPoolSize;- this.#createSessionFunction = createSessionFunction || this.defaultCreateSessionFunction;+ this.#createSessionFunction = createSessionFunction || this.#defaultCreateSessionFunction;// Session configuration. The pool-scoped logger is merged into per-call sessionOptions inside// `invokeCreateSessionFunction`, so every Session inherits it without custom createSessionFunctions@@ -280,7 +280,7 @@ export class SessionPool implements ISessionPool {this.#persistStateKey = persistStateKey ?? `${PERSIST_STATE_KEY}_${this.id}`;this.#stateCodec = buildSessionPoolStateCodec({- recreateSession: (sessionState) => this.invokeCreateSessionFunction(sessionState),+ recreateSession: (sessionState) => this.#invokeCreateSessionFunction(sessionState),toRecord: (sessions) => this.#buildPersistedState(sessions),log: this.#log,});@@ -308,7 +308,7 @@ export class SessionPool implements ISessionPool {* Gets count of usable sessions in the pool.*/async usableSessionsCount(): Promise<number> {- await this.ensureInitialized();+ await this.#ensureInitialized();return this.#state.currentValue.sessions.filter((session) => session.isUsable()).length;}@@ -316,7 +316,7 @@ export class SessionPool implements ISessionPool {* Gets count of retired sessions in the pool.*/async retiredSessionsCount(): Promise<number> {- await this.ensureInitialized();+ await this.#ensureInitialized();return this.#state.currentValue.sessions.filter((session) => !session.isUsable()).length;}@@ -324,14 +324,14 @@ export class SessionPool implements ISessionPool {* Starts periodic state persistence and potentially loads SessionPool state from {@apilink KeyValueStore}.* Called automatically on first use of any public method.*/- private async ensureInitialized(): Promise<void> {+ async #ensureInitialized(): Promise<void> {if (!this.#initPromise) {- this.#initPromise = this.setupPool();+ this.#initPromise = this.#setupPool();}return this.#initPromise;}- private async setupPool(): Promise<void> {+ async #setupPool(): Promise<void> {await this.#state.initialize();for (const session of this.#state.currentValue.sessions) {@@ -349,7 +349,7 @@ export class SessionPool implements ISessionPool {* @param [options] The configuration options for the session being added to the session pool.*/async addSession(options: Session | SessionOptions = {}): Promise<void> {- await this.ensureInitialized();+ await this.#ensureInitialized();const { id } = options;if (id) {const existingSession = this.#sessionMap.get(id);@@ -361,14 +361,14 @@ export class SessionPool implements ISessionPool {}}- if (!this.hasSpaceForSession()) {- this.removeRetiredSessions();+ if (!this.#hasSpaceForSession()) {+ this.#removeRetiredSessions();}- const newSession = options instanceof Session ? options : await this.invokeCreateSessionFunction(options);+ const newSession = options instanceof Session ? options : await this.#invokeCreateSessionFunction(options);this.#log.debug(`Adding new Session - ${newSession.id}`);- this.registerSession(newSession);+ this.#registerSession(newSession);}/**@@ -378,10 +378,10 @@ export class SessionPool implements ISessionPool {* @param [options] The configuration options for the session being added to the session pool.*/async newSession(sessionOptions?: SessionOptions): Promise<Session> {- await this.ensureInitialized();+ await this.#ensureInitialized();- const newSession = await this.invokeCreateSessionFunction(sessionOptions);- this.registerSession(newSession);+ const newSession = await this.#invokeCreateSessionFunction(sessionOptions);+ this.#registerSession(newSession);return newSession;}@@ -394,7 +394,7 @@ export class SessionPool implements ISessionPool {* @param [sessionId] If provided, it returns the usable session with this id, `undefined` otherwise.*/async getSession(sessionId?: string): Promise<Session | undefined> {- await this.ensureInitialized();+ await this.#ensureInitialized();await this.#queue.wait();try {@@ -404,15 +404,15 @@ export class SessionPool implements ISessionPool {return undefined;}- const pickedSession = this.pickSession();+ const pickedSession = this.#pickSession();if (pickedSession) return pickedSession;- if (this.hasSpaceForSession()) {- return await this.createSession();+ if (this.#hasSpaceForSession()) {+ return await this.#createSession();}- this.removeRetiredSessions();- return await this.createSession();+ this.#removeRetiredSessions();+ return await this.#createSession();} finally {this.#queue.shift();}@@ -445,7 +445,7 @@ export class SessionPool implements ISessionPool {* @internal*/async getState(): Promise<SessionPoolPersistedState> {- await this.ensureInitialized();+ await this.#ensureInitialized();return this.#buildPersistedState(this.#state.currentValue.sessions);}@@ -458,7 +458,7 @@ export class SessionPool implements ISessionPool {return;}- await this.ensureInitialized();+ await this.#ensureInitialized();this.#log.debug('Persisting state', {persistStateKeyValueStoreId: this.#persistStateKeyValueStoreId,@@ -483,7 +483,7 @@ export class SessionPool implements ISessionPool {*/async teardown(): Promise<void> {if (!this.#initPromise) return;- await this.ensureInitialized();+ await this.#ensureInitialized();await this.#state.teardown();// The next use initializes the pool again, restoring the record and resuming the periodic persistence -// a crawler that is run twice gets a persisting pool both times.@@ -493,7 +493,7 @@ export class SessionPool implements ISessionPool {/*** Removes retired `Session` instances from `SessionPool`.*/- private removeRetiredSessions() {+ #removeRetiredSessions() {this.#state.currentValue.sessions = this.#state.currentValue.sessions.filter((storedSession) => {if (storedSession.isUsable()) return true;@@ -519,7 +519,7 @@ export class SessionPool implements ISessionPool {* Adds `Session` instance to `SessionPool`.* @param newSession `Session` instance to be added.*/- private registerSession(newSession: Session) {+ #registerSession(newSession: Session) {this.#state.currentValue.sessions.push(newSession);this.#sessionMap.set(newSession.id, newSession);}@@ -527,7 +527,7 @@ export class SessionPool implements ISessionPool {/*** Gets random index.*/- private getRandomIndex(): number {+ #getRandomIndex(): number {return Math.floor(Math.random() * this.#state.currentValue.sessions.length);}@@ -537,7 +537,7 @@ export class SessionPool implements ISessionPool {* @param [options.sessionOptions] The configuration options for the session being created.* @returns New session.*/- private async defaultCreateSessionFunction(options: { sessionOptions?: SessionOptions } = {}): Promise<Session> {+ async #defaultCreateSessionFunction(options: { sessionOptions?: SessionOptions } = {}): Promise<Session> {const { sessionOptions } = parseArgument(options, createSessionOptionsSchema);return new Session(sessionOptions);@@ -553,7 +553,7 @@ export class SessionPool implements ISessionPool {* through `maybeLoadSessionPool` naturally wins because it arrives in* `perCallOptions`.*/- private async invokeCreateSessionFunction(perCallOptions?: SessionOptions): Promise<Session> {+ async #invokeCreateSessionFunction(perCallOptions?: SessionOptions): Promise<Session> {const sessionOptions: SessionOptions = {fingerprint: createDefaultSessionFingerprint(),...this.#sessionOptions,@@ -566,9 +566,9 @@ export class SessionPool implements ISessionPool {* Creates new session and adds it to the pool.* @returns Newly created `Session` instance.*/- private async createSession(): Promise<Session> {- const newSession = await this.invokeCreateSessionFunction();- this.registerSession(newSession);+ async #createSession(): Promise<Session> {+ const newSession = await this.#invokeCreateSessionFunction();+ this.#registerSession(newSession);this.#log.debug(`Created new Session - ${newSession.id}`);return newSession;@@ -577,7 +577,7 @@ export class SessionPool implements ISessionPool {/*** Decides whether there is enough space for creating new session.*/- private hasSpaceForSession(): boolean {+ #hasSpaceForSession(): boolean {return this.#state.currentValue.sessions.length < this.#maxPoolSize;}@@ -585,8 +585,8 @@ export class SessionPool implements ISessionPool {* Picks a session from the `SessionPool` according to the configured `sessionReuseStrategy`.* Returns `undefined` when no session should be reused and a new one should be created instead.*/- private pickSession(): Session | undefined {- if (this.#sessionReuseStrategy !== 'use-until-failure' && this.hasSpaceForSession()) return undefined;+ #pickSession(): Session | undefined {+ if (this.#sessionReuseStrategy !== 'use-until-failure' && this.#hasSpaceForSession()) return undefined;if (this.#sessionReuseStrategy === 'use-until-failure') {return this.#state.currentValue.sessions.find((session) => session.isUsable());@@ -598,7 +598,7 @@ export class SessionPool implements ISessionPool {this.#roundRobinIndex = index + 1;picked = this.#state.currentValue.sessions[index];} else {- picked = this.#state.currentValue.sessions[this.getRandomIndex()];+ picked = this.#state.currentValue.sessions[this.#getRandomIndex()];}return picked.isUsable() ? picked : undefined;diff --git a/packages/basic-crawler/src/internals/sitemap_request_loader.ts b/packages/basic-crawler/src/internals/sitemap_request_loader.tsindex f46ef7ae6..08b3e9e86 100644--- a/packages/basic-crawler/src/internals/sitemap_request_loader.ts+++ b/packages/basic-crawler/src/internals/sitemap_request_loader.ts@@ -251,7 +251,7 @@ export class SitemapRequestLoader implements IRequestLoader {this.#proxyUrl = proxyUrl;this.#enqueueStrategy = enqueueStrategy;- this.#urlQueueStream = this.createNewStream(maxBufferSize);+ this.#urlQueueStream = this.#createNewStream(maxBufferSize);this.#sitemapParsingProgress.pendingSitemapUrls = new Set(sitemapUrls);this.#events = serviceLocator.getEventManager();@@ -264,7 +264,7 @@ export class SitemapRequestLoader implements IRequestLoader {* @param highWaterMark High water mark for the stream (the maximum number of objects the stream will buffer).* @returns A new object stream.*/- private createNewStream(highWaterMark: number): Transform {+ #createNewStream(highWaterMark: number): Transform {return new Transform({objectMode: true,highWaterMark,@@ -276,7 +276,7 @@ export class SitemapRequestLoader implements IRequestLoader {* @param url URL to be checked.* @returns A matcher function that checks whether the pattern matches the closure URL.*/- private matchesUrl(url: string): (patternObject: UrlPatternObject) => boolean {+ #matchesUrl(url: string): (patternObject: UrlPatternObject) => boolean {return (patternObject) => {const { regexp, glob } = patternObject;@@ -292,10 +292,10 @@ export class SitemapRequestLoader implements IRequestLoader {* @param url URL to be checked.* @returns `true` if the URL matches the patterns, `false` otherwise.*/- private isUrlMatchingPatterns(url: string): boolean {+ #isUrlMatchingPatterns(url: string): boolean {return (- !this.#urlExcludePatternObjects.some(this.matchesUrl(url)) &&- (this.#urlPatternObjects.length === 0 || this.#urlPatternObjects.some(this.matchesUrl(url)))+ !this.#urlExcludePatternObjects.some(this.#matchesUrl(url)) &&+ (this.#urlPatternObjects.length === 0 || this.#urlPatternObjects.some(this.#matchesUrl(url))));}@@ -304,9 +304,9 @@ export class SitemapRequestLoader implements IRequestLoader {** Blocks if the stream is full until it is drained.*/- private async pushNextUrl(url: string | null) {+ async #pushNextUrl(url: string | null) {return new Promise<void>((resolve) => {- if (this.#closed || (url && !this.isUrlMatchingPatterns(url))) {+ if (this.#closed || (url && !this.#isUrlMatchingPatterns(url))) {resolve();return;}@@ -328,7 +328,7 @@ export class SitemapRequestLoader implements IRequestLoader {* If the stream is empty, blocks until a new URL is pushed.* @returns The next URL from the queue or `null` if we have read all URLs.*/- private async readNextUrl(): Promise<string | null> {+ async #readNextUrl(): Promise<string | null> {return new Promise((resolve) => {if (this.#closed) {resolve(null);@@ -366,7 +366,7 @@ export class SitemapRequestLoader implements IRequestLoader {** Resolves once all the sitemaps URLs have been fully loaded (sets `isSitemapFullyLoaded` to `true`).*/- private async load({+ async #load({parseSitemapOptions,}: {parseSitemapOptions?: SitemapRequestLoaderOptions['parseSitemapOptions'];@@ -390,7 +390,7 @@ export class SitemapRequestLoader implements IRequestLoader {}if (!this.#sitemapParsingProgress.inProgressEntries.has(item.loc)) {- await this.pushNextUrl(item.loc);+ await this.#pushNextUrl(item.loc);this.#sitemapParsingProgress.inProgressEntries.add(item.loc);}}@@ -420,8 +420,8 @@ export class SitemapRequestLoader implements IRequestLoader {...restOptions,persistStateKey: options.persistStateKey ?? STATE_PERSISTENCE_KEY,});- await requestList.restoreState();- void requestList.load({+ await requestList.#restoreState();+ void requestList.#load({parseSitemapOptions: { logger: serviceLocator.getLogger(), ...options.parseSitemapOptions, httpClient },});@@ -514,7 +514,7 @@ export class SitemapRequestLoader implements IRequestLoader {// Create a new stream, as we have read all the URLs from the current one.// Pushing the urls back to the original stream might not be possible if it has been ended.const previousStream = this.#urlQueueStream;- const newStream = this.createNewStream(previousStream.readableHighWaterMark);+ const newStream = this.#createNewStream(previousStream.readableHighWaterMark);for (const url of urlQueue) {newStream.push(url);@@ -547,7 +547,7 @@ export class SitemapRequestLoader implements IRequestLoader {} satisfies SitemapRequestLoaderState);}- private async restoreState(): Promise<void> {+ async #restoreState(): Promise<void> {await purgeDefaultStorages({ onlyPurgeOnce: true });if (this.#persistStateKey === undefined) {@@ -581,7 +581,7 @@ export class SitemapRequestLoader implements IRequestLoader {* @inheritDoc*/async fetchNextRequest(): Promise<Request | null> {- const nextUrl = await this.readNextUrl();+ const nextUrl = await this.#readNextUrl();if (!nextUrl) {return null;}@@ -626,12 +626,12 @@ export class SitemapRequestLoader implements IRequestLoader {*/async markRequestAsHandled(request: Request): Promise<void> {this.#handledUrlCount += 1;- this.ensureInProgress(request.url);+ this.#ensureInProgress(request.url);this.inProgress.delete(request.url);this.#requestData.delete(request.url);}- private ensureInProgress(url: string): void {+ #ensureInProgress(url: string): void {if (!this.inProgress.has(url)) {throw new Error(`The request is not being processed (url: ${url})`);}diff --git a/packages/basic-crawler/src/internals/throttling_request_manager.ts b/packages/basic-crawler/src/internals/throttling_request_manager.tsindex c76502e31..7eae6a649 100644--- a/packages/basic-crawler/src/internals/throttling_request_manager.ts+++ b/packages/basic-crawler/src/internals/throttling_request_manager.ts@@ -279,6 +279,7 @@ export class ThrottlingRequestManager<T extends IRequestManager = IRequestManagereadonly #throttleBy: 'hostname' | 'registrableDomain';readonly #maxThrottledDomains: number;readonly #persistStateKey: string;+ readonly #config: Configuration;readonly #subManagers = new Map<string, Promise<T>>();@@ -330,14 +331,15 @@ export class ThrottlingRequestManager<T extends IRequestManager = IRequestManageconstructor(options: ThrottlingRequestManagerOptions<T>,- private readonly config: Configuration = serviceLocator.getConfiguration(),+ config: Configuration = serviceLocator.getConfiguration(),) {+ this.#config = config;parseArgument(options, throttlingRequestManagerOptionsSchema, 'ThrottlingRequestManagerOptions');if (options.inner === undefined) {// Deferred like any other factory, so a manager nobody fetches from opens no storage. The opener is// assigned below and read only when this runs.- this.#innerFactory = () => this.#requestManagerOpener(null, { configuration: this.config });+ this.#innerFactory = () => this.#requestManagerOpener(null, { configuration: this.#config });} else if (typeof options.inner === 'function') {this.#innerFactory = options.inner;} else {@@ -523,7 +525,7 @@ export class ThrottlingRequestManager<T extends IRequestManager = IRequestManage// Backends use the alias as a directory name, and an IPv6 literal is full of characters// Windows will not accept. Ordinary hostnames survive this untouched.{ alias: `throttled-${encodeURIComponent(domain)}` },- { configuration: this.config },+ { configuration: this.#config },);this.#subManagers.set(domain, subManager);this.#ensureDomainState(domain);@@ -563,7 +565,7 @@ export class ThrottlingRequestManager<T extends IRequestManager = IRequestManageasync #ensureSubManagers(): Promise<void> {this.#subManagersReady ??= (async () => {if (this.#throttlesEveryDomain) {- this.#domainListStore = await KeyValueStore.open(null, { configuration: this.config });+ this.#domainListStore = await KeyValueStore.open(null, { configuration: this.#config });for (const domain of (await this.#domainListStore.getValue<string[]>(this.#persistStateKey)) ?? []) {this.#discoveredDomains.add(domain);diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.tsindex 4dc748e23..f1fb4814e 100644--- a/packages/browser-crawler/src/internals/browser-crawler.ts+++ b/packages/browser-crawler/src/internals/browser-crawler.ts@@ -466,7 +466,7 @@ export abstract class BrowserCrawler<);});- let pipeline = contextPipelineBuilder().compose(this.prepareNavigation.bind(this));+ let pipeline = contextPipelineBuilder().compose(this.#prepareNavigation.bind(this));for (const hook of this.#preNavigationHooks) {pipeline = pipeline.compose(windowGuard(hook));@@ -479,9 +479,9 @@ export abstract class BrowserCrawler<}return pipeline- .compose(skipGuard(this.finalizeNavigation.bind(this)))- .compose(this.handleBlockedRequestByContent.bind(this))- .compose(this.restoreRequestState.bind(this));+ .compose(skipGuard(this.#finalizeNavigation.bind(this)))+ .compose(this.#handleBlockedRequestByContent.bind(this))+ .compose(this.#restoreRequestState.bind(this));},extendContext,});@@ -509,10 +509,10 @@ export abstract class BrowserCrawler<CrawlingContext,BrowserCrawlingContext<Page, Response, Dictionary>> {- return ContextPipeline.create<CrawlingContext>().compose(this.preparePage.bind(this));+ return ContextPipeline.create<CrawlingContext>().compose(this.#preparePage.bind(this));}- private async containsSelectors(page: CommonPage, selectors: string[]): Promise<string[] | null> {+ async #containsSelectors(page: CommonPage, selectors: string[]): Promise<string[] | null> {const foundSelectors = (await Promise.all(selectors.map((selector) => (page as any).$(selector)))).map((x, i) => [x, selectors[i]] as [any, string]).filter(([x]) => x !== null)@@ -521,21 +521,21 @@ export abstract class BrowserCrawler<return foundSelectors.length > 0 ? foundSelectors : null;}- private async isRequestBlocked(crawlingContext: BrowserCrawlingContext<Page, Response>): Promise<string | false> {+ async #isRequestBlocked(crawlingContext: BrowserCrawlingContext<Page, Response>): Promise<string | false> {const { page, response } = crawlingContext;// Cloudflare specific heuristic - wait 5 seconds if we get a 403 for the JS challenge to load / resolve.- if ((await this.containsSelectors(page, CLOUDFLARE_RETRY_CSS_SELECTORS)) && response?.status() === 403) {+ if ((await this.#containsSelectors(page, CLOUDFLARE_RETRY_CSS_SELECTORS)) && response?.status() === 403) {await sleep(5000);// here we cannot test for response code, because we only have the original response, not the possible Cloudflare redirect on passed challenge.- const foundSelectors = await this.containsSelectors(page, RETRY_CSS_SELECTORS);+ const foundSelectors = await this.#containsSelectors(page, RETRY_CSS_SELECTORS);if (!foundSelectors) return false;return `Cloudflare challenge failed, found selectors: ${foundSelectors.join(', ')}`;}- const foundSelectors = await this.containsSelectors(page, RETRY_CSS_SELECTORS);+ const foundSelectors = await this.#containsSelectors(page, RETRY_CSS_SELECTORS);const statusCode = response?.status() ?? 0;if (foundSelectors) return `Found selectors: ${foundSelectors.join(', ')}`;@@ -544,7 +544,7 @@ export abstract class BrowserCrawler<return false;}- private async preparePage(+ async #preparePage(crawlingContext: CrawlingContext,onCleanup: CleanupRegistrar,): Promise<ContextDifference<CrawlingContext, BrowserCrawlingContext<Page, Response, Dictionary>>> {@@ -621,18 +621,21 @@ export abstract class BrowserCrawler<};}- private async prepareNavigation(crawlingContext: Context): Promise<Partial<Context>> {+ async #prepareNavigation(crawlingContext: Context): Promise<Partial<Context>> {if (crawlingContext.request.skipNavigation) {return {request: new Proxy(crawlingContext.request, {- get(target, propertyName, receiver) {+ get(target, propertyName) {if (propertyName === 'loadedUrl') {throw new NavigationSkippedError('The `request.loadedUrl` property is not available - `skipNavigation` was used',);}- return Reflect.get(target, propertyName, receiver);+ // `target` as receiver and bound methods, or `#` members throw on the proxy+ const value = Reflect.get(target, propertyName, target);+ return typeof value === 'function' ? value.bind(target) : value;},+ set: (target, propertyName, value) => Reflect.set(target, propertyName, value, target),}) as LoadedRequest<CrawlingRequest>,get response(): Response {throw new NavigationSkippedError(@@ -672,15 +675,15 @@ export abstract class BrowserCrawler<const cookiesBeforeHooks = readContextField<string>(crawlingContext, COOKIES_BEFORE_HOOKS);const cookiesAfterHooks = this.getCookieHeaderFromRequest(crawlingContext.request);- await this.applyCookies(crawlingContext, cookiesBeforeHooks, cookiesAfterHooks);+ await this.#applyCookies(crawlingContext, cookiesBeforeHooks, cookiesAfterHooks);let response: Response | undefined;try {response = (await this.navigationHandler(crawlingContext, gotoOptions)) ?? undefined;} catch (error) {- await this.handleNavigationTimeout(crawlingContext, error as Error);+ await this.#handleNavigationTimeout(crawlingContext, error as Error);crawlingContext.request.state = RequestState.ERROR;- this.throwIfProxyError(error as Error);+ this.#throwIfProxyError(error as Error);throw error;}tryCancel();@@ -690,7 +693,7 @@ export abstract class BrowserCrawler<return { response } as Partial<Context>;}- private async finalizeNavigation(crawlingContext: Context): Promise<Partial<Context>> {+ async #finalizeNavigation(crawlingContext: Context): Promise<Partial<Context>> {tryCancel();let response: Response | undefined;@@ -701,12 +704,12 @@ export abstract class BrowserCrawler<// navigation produced no response and no hook overrode it. Treat as undefined.}- await this.processResponse(response, crawlingContext);+ await this.#processResponse(response, crawlingContext);tryCancel();// Persist cookies from the navigation response before the user handler runs.// Cookies set during `requestHandler` are saved again afterwards.- await this.persistCookiesFromPage(crawlingContext);+ await this.#persistCookiesFromPage(crawlingContext);return { request: crawlingContext.request as LoadedRequest<CrawlingRequest> } as Partial<Context>;}@@ -714,7 +717,7 @@ export abstract class BrowserCrawler</*** Copies cookies from the live browser page into the session cookie jar.*/- private async persistCookiesFromPage(+ async #persistCookiesFromPage(crawlingContext: Pick<BrowserCrawlingContext<Page, Response>, 'session' | 'page' | 'request'>,): Promise<void> {if (!this.#saveResponseCookies || !crawlingContext.session) {@@ -748,7 +751,7 @@ export abstract class BrowserCrawler<} finally {if (!crawlingContext.request.skipNavigation) {try {- await this.persistCookiesFromPage(crawlingContext);+ await this.#persistCookiesFromPage(crawlingContext);} catch {// Page may already be closed on some failure paths; ignore.}@@ -756,21 +759,21 @@ export abstract class BrowserCrawler<}}- private async handleBlockedRequestByContent(crawlingContext: BrowserCrawlingContext<Page, Response>) {+ async #handleBlockedRequestByContent(crawlingContext: BrowserCrawlingContext<Page, Response>) {if (this.retryOnBlocked) {- const error = await this.isRequestBlocked(crawlingContext);+ const error = await this.#isRequestBlocked(crawlingContext);if (error) throw new SessionError(error);}return {};}- private async restoreRequestState(crawlingContext: CrawlingContext) {+ async #restoreRequestState(crawlingContext: CrawlingContext) {crawlingContext.request.state = RequestState.REQUEST_HANDLER;return {};}- private async applyCookies(+ async #applyCookies({ session, request, page }: BrowserCrawlingContext<Page, Response>,preHooksCookies: string,postHooksCookies: string,@@ -791,7 +794,7 @@ export abstract class BrowserCrawler</*** Marks session bad on navigation timeout, and stops in-flight page loading on any navigation error.*/- private async handleNavigationTimeout(crawlingContext: BrowserCrawlingContext, error: Error): Promise<void> {+ async #handleNavigationTimeout(crawlingContext: BrowserCrawlingContext, error: Error): Promise<void> {const { session, page } = crawlingContext;// Fire-and-forget: no user code will run on this page after a failed navigation.@@ -809,7 +812,7 @@ export abstract class BrowserCrawler</*** Transforms proxy-related errors to `SessionError`.*/- private throwIfProxyError(error: Error) {+ #throwIfProxyError(error: Error) {if (this.isProxyError(error)) {throw new SessionError(this.getMessageFromError(error));}@@ -820,10 +823,7 @@ export abstract class BrowserCrawler<gotoOptions: GoToOptions,): Promise<Context['response'] | null | undefined>;- private async processResponse(- response: Response | undefined,- crawlingContext: BrowserCrawlingContext,- ): Promise<void> {+ async #processResponse(response: Response | undefined, crawlingContext: BrowserCrawlingContext): Promise<void> {const { session, request, page } = crawlingContext;if (typeof response === 'object' && typeof response.status === 'function') {diff --git a/packages/browser-crawler/src/internals/browser-launcher.ts b/packages/browser-crawler/src/internals/browser-launcher.tsindex fb75c3f40..f3a5fe28e 100644--- a/packages/browser-crawler/src/internals/browser-launcher.ts+++ b/packages/browser-crawler/src/internals/browser-launcher.ts@@ -178,7 +178,7 @@ export abstract class BrowserLauncher<...otherLaunchContextProps} = launchContext;- this.validateProxyUrlProtocol(proxyUrl);+ this.#validateProxyUrlProtocol(proxyUrl);// those need to be reassigned otherwise they are {} in typesthis.launcher = launcher!;@@ -211,7 +211,7 @@ export abstract class BrowserLauncher<// parameter, so the argument cannot be checked here. The concrete `*BrowserPool()` factories are where the// caller-facing hook types get pinned down.return new BrowserPool<{ browserPlugins: [Plugin] }, [Plugin]>({- ...this.resolveFingerprinting(options),+ ...this.#resolveFingerprinting(options),browserPlugins: [this.createBrowserPlugin()],} as any);}@@ -225,7 +225,7 @@ export abstract class BrowserLauncher<return new RemoteBrowserPool<Page>({...options,browserPlugins: [this.createBrowserPlugin()],- browserPoolOptions: this.resolveFingerprinting(options.browserPoolOptions ?? {}),+ browserPoolOptions: this.#resolveFingerprinting(options.browserPoolOptions ?? {}),});}@@ -233,7 +233,7 @@ export abstract class BrowserLauncher<* A custom `userAgent` and Crawlee's fingerprint injection would both write the same headers, so an* explicitly requested user agent wins.*/- private resolveFingerprinting<T extends { useFingerprints?: boolean }>(options: T): T {+ #resolveFingerprinting<T extends { useFingerprints?: boolean }>(options: T): T {if (!this.userAgent) {return options;}@@ -279,7 +279,7 @@ export abstract class BrowserLauncher<}if (this.useChrome && !launchOptions.executablePath) {- launchOptions.executablePath = this.getChromeExecutablePath();+ launchOptions.executablePath = this.#getChromeExecutablePath();}return launchOptions;@@ -289,14 +289,14 @@ export abstract class BrowserLauncher<return this.configuration.headless && !this.configuration.xvfb;}- private getChromeExecutablePath(): string {- return this.configuration.chromeExecutablePath ?? this.getTypicalChromeExecutablePath();+ #getChromeExecutablePath(): string {+ return this.configuration.chromeExecutablePath ?? this.#getTypicalChromeExecutablePath();}/*** Gets a typical path to Chrome executable, depending on the current operating system.*/- private getTypicalChromeExecutablePath(): string {+ #getTypicalChromeExecutablePath(): string {/*** Returns path of Chrome executable by its OS environment variable to deal with non-english language OS.* Taking also into account the old [chrome 380177 issue](https://bugs.chromium.org/p/chromium/issues/detail?id=380177).@@ -325,7 +325,7 @@ export abstract class BrowserLauncher<}}- private validateProxyUrlProtocol(proxyUrl?: string): void {+ #validateProxyUrlProtocol(proxyUrl?: string): void {if (!proxyUrl) return;if (!/^(http|https|socks4|socks5)/i.test(proxyUrl)) {diff --git a/packages/browser-pool/src/abstract-classes/browser-plugin.ts b/packages/browser-pool/src/abstract-classes/browser-plugin.tsindex 4d94aeb22..f1cfbc9ec 100644--- a/packages/browser-pool/src/abstract-classes/browser-plugin.ts+++ b/packages/browser-pool/src/abstract-classes/browser-plugin.ts@@ -267,7 +267,7 @@ export abstract class BrowserPlugin<if (!launchContext.isRemote && this.isChromiumBasedBrowser(launchContext)) {// This will set the args for chromium based browsers to hide the webdriver.- (launchOptions as Dictionary).args = this.mergeArgsToHideWebdriver(launchOptions!.args);+ (launchOptions as Dictionary).args = this.#mergeArgsToHideWebdriver(launchOptions!.args);// When User-Agent is not set, and we're using Chromium in headless mode,// it is better to use DEFAULT_USER_AGENT to reduce chance of detection,// as otherwise 'HeadlessChrome' is present in User-Agent string.@@ -284,7 +284,7 @@ export abstract class BrowserPlugin<return this._launch(launchContext);}- private mergeArgsToHideWebdriver(originalArgs?: string[]): string[] {+ #mergeArgsToHideWebdriver(originalArgs?: string[]): string[] {if (!originalArgs?.length) {return ['--disable-blink-features=AutomationControlled'];}diff --git a/packages/browser-pool/src/browser-pool.ts b/packages/browser-pool/src/browser-pool.tsindex d6d90d66b..9ba7f42a2 100644--- a/packages/browser-pool/src/browser-pool.ts+++ b/packages/browser-pool/src/browser-pool.ts@@ -355,15 +355,17 @@ export class BrowserPool<readonly #postPageCloseHooks: PostPageCloseHook<BrowserControllerReturn>[];#pageCounter = 0;- // TS-private rather than `#`: tests observe page tracking and controller retirement directly+ // oxlint-disable-next-line crawlee/prefer-private-fields -- tests observe page tracking and controller retirement directlyprivate pages = new Map<string, PageReturn>();#pageIds = new WeakMap<PageReturn, string>();#startingBrowserControllers = new Set<BrowserControllerReturn>();+ // oxlint-disable-next-line crawlee/prefer-private-fields -- accessed by testsprivate activeBrowserControllers = new Set<BrowserControllerReturn>();+ // oxlint-disable-next-line crawlee/prefer-private-fields -- accessed by testsprivate retiredBrowserControllers = new Set<BrowserControllerReturn>();#pageToBrowserController = new WeakMap<PageReturn, BrowserControllerReturn>();- // kept as TS-private: tests replace this interval through bracket access+ // oxlint-disable-next-line crawlee/prefer-private-fields -- tests replace this interval through bracket accessprivate browserKillerInterval?: NodeJS.Timeout;#browserRetireInterval?: NodeJS.Timeout;@@ -448,7 +450,7 @@ export class BrowserPool<// fingerprintingif (useFingerprints) {- this.initializeFingerprinting();+ this.#initializeFingerprinting();// The fingerprint pre-launch hook goes last because of the fingerprint cache.// It is usual to generate proxy per browser and we want to know the proxyUrl for the caching.@@ -482,7 +484,7 @@ export class BrowserPool<const {id = nanoid(),pageOptions,- browserPlugin = this.pickBrowserPlugin(),+ browserPlugin = this.#pickBrowserPlugin(),session,proxyUrl = session?.proxyInfo?.url,ignoreTlsErrors = session?.proxyInfo?.ignoreTlsErrors,@@ -509,17 +511,17 @@ export class BrowserPool<// Limiter is necessary - https://github.com/apify/crawlee/issues/1126return this.#limiter(AsyncResource.bind(async () => {- let browserController = this.pickBrowserWithFreeCapacity(browserPlugin, { proxyUrl });+ let browserController = this.#pickBrowserWithFreeCapacity(browserPlugin, { proxyUrl });if (!browserController)- browserController = await this.launchBrowser(id, {+ browserController = await this.#launchBrowser(id, {browserPlugin,proxyUrl,ignoreTlsErrors,});tryCancel();- return await this.createPageForBrowser(id, browserController, pageOptions, proxyUrl, ignoreTlsErrors);+ return await this.#createPageForBrowser(id, browserController, pageOptions, proxyUrl, ignoreTlsErrors);}),);}@@ -532,15 +534,15 @@ export class BrowserPool<async newPageInNewBrowser(options: BrowserPoolNewPageInNewBrowserOptions<PageOptions, BrowserPlugins[number]> = {},): Promise<PageReturn> {- const { id = nanoid(), pageOptions, launchOptions, browserPlugin = this.pickBrowserPlugin() } = options;+ const { id = nanoid(), pageOptions, launchOptions, browserPlugin = this.#pickBrowserPlugin() } = options;if (this.pages.has(id)) {throw new Error(`Page with ID: ${id} already exists.`);}- const browserController = await this.launchBrowser(id, { launchOptions, browserPlugin });+ const browserController = await this.#launchBrowser(id, { launchOptions, browserPlugin });tryCancel();- return await this.createPageForBrowser(id, browserController, pageOptions);+ return await this.#createPageForBrowser(id, browserController, pageOptions);}/**@@ -613,7 +615,7 @@ export class BrowserPool<return this.#pageIds.get(page);}- private async createPageForBrowser(+ async #createPageForBrowser(pageId: string,browserController: BrowserControllerReturn,pageOptions: PageOptions = {} as PageOptions,@@ -638,7 +640,7 @@ export class BrowserPool<}}- await this.executeHooks(this.#prePageCreateHooks, pageId, browserController, finalPageOptions);+ await this.#executeHooks(this.#prePageCreateHooks, pageId, browserController, finalPageOptions);tryCancel();let page: PageReturn;@@ -660,7 +662,7 @@ export class BrowserPool<this.retireBrowserController(browserController);}- this.overridePageClose(page);+ this.#overridePageClose(page);} catch (err) {this.retireBrowserController(browserController);throw new Error(@@ -668,7 +670,7 @@ export class BrowserPool<);}- await this.executeHooks(this.#postPageCreateHooks, page, browserController);+ await this.#executeHooks(this.#postPageCreateHooks, page, browserController);tryCancel();this.emit(BROWSER_POOL_EVENTS.PAGE_CREATED, page);@@ -782,7 +784,7 @@ export class BrowserPool<* @return {Promise<void>}*/async closeAllBrowsers(): Promise<void> {- const controllers = this.getAllBrowserControllers();+ const controllers = this.#getAllBrowserControllers();const promises = [...controllers].filter((controller) => controller.isActive).map(async (controller) => controller.close());@@ -821,7 +823,7 @@ export class BrowserPool<this.removeAllListeners();}- private getAllBrowserControllers() {+ #getAllBrowserControllers() {return new Set([...this.#startingBrowserControllers,...this.activeBrowserControllers,@@ -829,7 +831,7 @@ export class BrowserPool<]);}- private async launchBrowser(pageId: string, options: InternalLaunchBrowserOptions<BrowserPlugins[number]>) {+ async #launchBrowser(pageId: string, options: InternalLaunchBrowserOptions<BrowserPlugins[number]>) {const { browserPlugin, launchOptions, proxyUrl, ignoreTlsErrors } = options;const browserController = browserPlugin.createController() as BrowserControllerReturn;@@ -854,7 +856,7 @@ export class BrowserPool<try {// If the hooks or the launch fails, we need to delete the controller,// because otherwise it would be stuck in limbo without a browser.- await this.executeHooks(this.#preLaunchHooks, pageId, launchContext);+ await this.#executeHooks(this.#preLaunchHooks, pageId, launchContext);tryCancel();const browser = await browserPlugin.launch(launchContext);tryCancel();@@ -870,7 +872,7 @@ export class BrowserPool<try {// If the launch fails on the post-launch hooks, we need to clean up// both the controller and the browser before throwing.- await this.executeHooks(this.#postLaunchHooks, pageId, browserController);+ await this.#executeHooks(this.#postLaunchHooks, pageId, browserController);} catch (err) {this.#startingBrowserControllers.delete(browserController);browserController.close().catch((closeErr) => {@@ -893,14 +895,14 @@ export class BrowserPool</*** Picks plugins round robin.*/- private pickBrowserPlugin() {+ #pickBrowserPlugin() {const pluginIndex = this.#pageCounter % this.browserPlugins.length;this.#pageCounter++;return this.browserPlugins[pluginIndex];}- private pickBrowserWithFreeCapacity(browserPlugin: BrowserPlugin, options?: { proxyUrl?: string }) {+ #pickBrowserWithFreeCapacity(browserPlugin: BrowserPlugin, options?: { proxyUrl?: string }) {return [...this.activeBrowserControllers].find((controller) => {const hasCapacity = controller.activePages < this.#maxOpenPagesPerBrowser;const isCorrectPlugin = controller.browserPlugin === browserPlugin;@@ -916,6 +918,7 @@ export class BrowserPool<});}+ // oxlint-disable-next-line crawlee/prefer-private-fields -- accessed by testsprivate async closeInactiveRetiredBrowsers() {const closedBrowserIds: string[] = [];@@ -941,7 +944,7 @@ export class BrowserPool<}}- private overridePageClose(page: PageReturn) {+ #overridePageClose(page: PageReturn) {const originalPageClose = page.close;const browserController = this.#pageToBrowserController.get(page)!;const pageId = this.getPageId(page)!;@@ -957,7 +960,7 @@ export class BrowserPool<let pageClosed = false;const closing = (async () => {- await this.executeHooks(this.#prePageCloseHooks, page, browserController);+ await this.#executeHooks(this.#prePageCloseHooks, page, browserController);await originalPageClose.apply(page, args).catch((err: Error) => {this.#log.debug(`Could not close page.\nCause:${err.message}`, { id: browserController.id });@@ -967,7 +970,7 @@ export class BrowserPool<// so that a slow hook does not get the browser retired.pageClosed = true;- await this.executeHooks(this.#postPageCloseHooks, pageId, browserController);+ await this.#executeHooks(this.#postPageCloseHooks, pageId, browserController);})();let timeout: NodeJS.Timeout | undefined;@@ -1005,19 +1008,19 @@ export class BrowserPool<browserController.registerPageClosed(page);this.pages.delete(pageId);- this.closeRetiredBrowserWithNoPages(browserController);+ this.#closeRetiredBrowserWithNoPages(browserController);this.emit(BROWSER_POOL_EVENTS.PAGE_CLOSED, page);};}- private async executeHooks(hooks: ((...args: any[]) => unknown)[], ...args: unknown[]) {+ async #executeHooks(hooks: ((...args: any[]) => unknown)[], ...args: unknown[]) {for (const hook of hooks) {await hook(...args);}}- private closeRetiredBrowserWithNoPages(browserController: BrowserControllerReturn) {+ #closeRetiredBrowserWithNoPages(browserController: BrowserControllerReturn) {if (browserController.activePages === 0 && this.retiredBrowserControllers.has(browserController)) {// Run this with a delay, otherwise page.close()// might fail with "Protocol error (Target.closeTarget): Target closed."@@ -1061,7 +1064,7 @@ export class BrowserPool<return false;}- private initializeFingerprinting(): void {+ #initializeFingerprinting(): void {const { useFingerprintCache = true, fingerprintCacheSize = 10_000 } = this.fingerprintOptions;this.fingerprintGenerator = new FingerprintGenerator(this.fingerprintOptions.fingerprintGeneratorOptions);this.fingerprintInjector = new FingerprintInjector();diff --git a/packages/browser-pool/src/playwright/playwright-plugin.ts b/packages/browser-pool/src/playwright/playwright-plugin.tsindex 59c04ec60..0dc3add59 100644--- a/packages/browser-pool/src/playwright/playwright-plugin.ts+++ b/packages/browser-pool/src/playwright/playwright-plugin.ts@@ -75,7 +75,7 @@ export class PlaywrightPlugin extends BrowserPlugin<try {if (useIncognitoPages) {browser = await this.library.launch(launchOptions).catch((error) => {- return this.throwOnFailedLaunch(launchContext, error);+ return this.#throwOnFailedLaunch(launchContext, error);});if (anonymizedProxyUrl) {@@ -87,7 +87,7 @@ export class PlaywrightPlugin extends BrowserPlugin<const browserContext = await this.library.launchPersistentContext(userDataDir, launchOptions).catch((error) => {- return this.throwOnFailedLaunch(launchContext, error);+ return this.#throwOnFailedLaunch(launchContext, error);});browserContext.once('close', () => {@@ -131,7 +131,7 @@ export class PlaywrightPlugin extends BrowserPlugin<return browser;}- private throwOnFailedLaunch(launchContext: LaunchContext<BrowserType>, cause: unknown): never {+ #throwOnFailedLaunch(launchContext: LaunchContext<BrowserType>, cause: unknown): never {this.throwAugmentedLaunchError(cause,launchContext.launchOptions?.executablePath,diff --git a/packages/browser-pool/src/remote-browser-pool.ts b/packages/browser-pool/src/remote-browser-pool.tsindex c7e30c648..9c3865a2e 100644--- a/packages/browser-pool/src/remote-browser-pool.ts+++ b/packages/browser-pool/src/remote-browser-pool.ts@@ -281,7 +281,7 @@ export class RemoteBrowserPool<Page = unknown> implements IBrowserPool<Page> {* allows it (either a new browser slot is free, or an active browser still has page capacity).*/async newPage(options?: NewPageOptions): Promise<Page> {- await this.waitForFreeSlot();+ await this.#waitForFreeSlot();return this.#pool.newPage(options);}@@ -317,9 +317,9 @@ export class RemoteBrowserPool<Page = unknown> implements IBrowserPool<Page> {}/** Resolves once the wrapped pool can serve another page without exceeding `maxOpenBrowsers`. */- private async waitForFreeSlot(): Promise<void> {+ async #waitForFreeSlot(): Promise<void> {while (!this.browserPool.hasFreeBrowserSlot() && !this.browserPool.hasActiveBrowserWithFreeCapacity()) {- await this.nextCapacityChange();+ await this.#nextCapacityChange();}}@@ -328,7 +328,7 @@ export class RemoteBrowserPool<Page = unknown> implements IBrowserPool<Page> {* concurrently-waiting `newPage` calls share a single promise (and a single pair of event listeners)* per tick, so a fleet of saturated callers doesn't fan out into N listener pairs on the pool.*/- private nextCapacityChange(): Promise<void> {+ #nextCapacityChange(): Promise<void> {this.#capacityChange ??= new Promise<void>((resolve) => {const done = () => {clearTimeout(timer);diff --git a/packages/core/src/configuration.ts b/packages/core/src/configuration.tsindex f225b6beb..613caabe0 100644--- a/packages/core/src/configuration.ts+++ b/packages/core/src/configuration.ts@@ -182,9 +182,9 @@ export class Configuration {*/constructor(options: ConfigurationInput = {}) {const fields = (this.constructor as typeof Configuration).fields;- const fileOptions = Configuration.loadFileOptions();- this.#resolvedValues = Configuration.resolveAll(fields, options as Record<string, unknown>, fileOptions);- this.registerAccessors();+ const fileOptions = Configuration.#loadFileOptions();+ this.#resolvedValues = Configuration.#resolveAll(fields, options as Record<string, unknown>, fileOptions);+ this.#registerAccessors();// Set the log levelconst logLevel = this.logLevel;@@ -206,7 +206,7 @@ export class Configuration {* Resolves all field values once using the priority chain:* constructor options > env vars > crawlee.json > schema defaults.*/- private static resolveAll(+ static #resolveAll(fields: Record<string, ConfigField>,userOptions: Record<string, unknown>,fileOptions: Record<string, unknown>,@@ -221,7 +221,7 @@ export class Configuration {}// 2. Environment variables- const envValue = Configuration.readEnvVar(fieldDef);+ const envValue = Configuration.#readEnvVar(fieldDef);if (envValue != null) {values[key] = fieldDef.schema.parse(envValue);continue;@@ -244,7 +244,7 @@ export class Configuration {/*** Registers getters (and throwing setters) on the instance for each field.*/- private registerAccessors(): void {+ #registerAccessors(): void {const fields = (this.constructor as typeof Configuration).fields;const descriptors: PropertyDescriptorMap = {};@@ -267,7 +267,7 @@ export class Configuration {* Empty strings are treated as unset, falling through to crawlee.json or schema defaults.* (Crawlee v3 coerced `''` to `false`/`0`/`''` per type — v4 drops that for consistency.)*/- private static readEnvVar(fieldDef: ConfigField): string | undefined {+ static #readEnvVar(fieldDef: ConfigField): string | undefined {if (!fieldDef.envVar) return undefined;const envVars = Array.isArray(fieldDef.envVar) ? fieldDef.envVar : [fieldDef.envVar];for (const envVar of envVars) {@@ -280,7 +280,7 @@ export class Configuration {/*** Loads config options from crawlee.json in the current working directory.*/- private static loadFileOptions(): Record<string, unknown> {+ static #loadFileOptions(): Record<string, unknown> {try {const file = readFileSync(join(process.cwd(), 'crawlee.json'));return JSON.parse(file.toString());diff --git a/packages/core/src/events/local_event_manager.ts b/packages/core/src/events/local_event_manager.tsindex 7bb93bd67..73cf30c97 100644--- a/packages/core/src/events/local_event_manager.ts+++ b/packages/core/src/events/local_event_manager.ts@@ -65,7 +65,7 @@ export class LocalEventManager extends EventManager {* @internal*/async emitSystemInfoEvent(intervalCallback: () => unknown) {- const info = await this.createSystemInfo({+ const info = await this.#createSystemInfo({maxUsedCpuRatio: serviceLocator.getConfiguration().maxUsedCpuRatio,});this.events.emit(EventType.SYSTEM_INFO, info);@@ -83,15 +83,15 @@ export class LocalEventManager extends EventManager {/*** Creates a SystemInfo object based on local metrics.*/- private async createSystemInfo(options: { maxUsedCpuRatio: number }) {+ async #createSystemInfo(options: { maxUsedCpuRatio: number }) {return {createdAt: new Date(),- ...(await this.createCpuInfo(options)),- ...(await this.createMemoryInfo()),+ ...(await this.#createCpuInfo(options)),+ ...(await this.#createMemoryInfo()),} as SystemInfo;}- private async createCpuInfo(options: { maxUsedCpuRatio: number }) {+ async #createCpuInfo(options: { maxUsedCpuRatio: number }) {const { getCurrentCpuTicksV2 } = await import('../system-info/cpu-info.js');const usedCpuRatio = await getCurrentCpuTicksV2({containerized: await this.isContainerizedWrapper(),@@ -103,9 +103,9 @@ export class LocalEventManager extends EventManager {};}- private async createMemoryInfo() {+ async #createMemoryInfo() {try {- const memInfo = await this.getMemoryInfo();+ const memInfo = await this.#getMemoryInfo();return {memTotalBytes: memInfo.totalBytes,memCurrentBytes: memInfo.mainProcessBytes + memInfo.childProcessesBytes,@@ -116,7 +116,7 @@ export class LocalEventManager extends EventManager {}}- private async getMemoryInfo() {+ async #getMemoryInfo() {const { getMemoryInfo } = await import('../system-info/memory-info.js');return getMemoryInfo({containerized: await this.isContainerizedWrapper(),diff --git a/packages/core/src/memory-storage/resource-clients/dataset.ts b/packages/core/src/memory-storage/resource-clients/dataset.tsindex a7ed375de..71fa9452c 100644--- a/packages/core/src/memory-storage/resource-clients/dataset.ts+++ b/packages/core/src/memory-storage/resource-clients/dataset.ts@@ -47,23 +47,22 @@ export class DatasetBackend<Data extends Dictionary = Dictionary>itemCount = 0;readonly #datasetEntries = new Map<string, Data>();- // kept as TS-private: storage-backend tests read this field at runtime- private readonly storageBackend: MemoryStorageBackend;+ readonly #storageBackend: MemoryStorageBackend;constructor(options: DatasetBackendOptions) {super(options.id ?? randomUUID());this.name = options.name;this.cacheKey = options.cacheKey ?? this.name ?? this.id;- this.storageBackend = options.storageBackend;+ this.#storageBackend = options.storageBackend;}async getMetadata(): Promise<storage.DatasetInfo> {- this.updateTimestamps(false);+ this.#updateTimestamps(false);return this.toDatasetInfo();}async drop(): Promise<void> {- if (this.storageBackend.evictBackend('Dataset', this.id)) {+ if (this.#storageBackend.evictBackend('Dataset', this.id)) {this.itemCount = 0;this.#datasetEntries.clear();}@@ -73,23 +72,23 @@ export class DatasetBackend<Data extends Dictionary = Dictionary>this.itemCount = 0;this.#datasetEntries.clear();- this.updateTimestamps(true);+ this.#updateTimestamps(true);}getData(options: storage.DatasetBackendListOptions = {}): Promise<storage.PaginatedList<Data>> {const { desc, limit, offset } = parseArgument(options, schemas.datasetListItemsOptions);- return this.getDataPage({+ return this.#getDataPage({desc,offset: offset ?? 0,limit: Math.min(limit ?? LIST_ITEMS_LIMIT, LIST_ITEMS_LIMIT),});}- private async getDataPage(options: storage.DatasetBackendListOptions = {}): Promise<storage.PaginatedList<Data>> {+ async #getDataPage(options: storage.DatasetBackendListOptions = {}): Promise<storage.PaginatedList<Data>> {const { limit = LIST_ITEMS_LIMIT, offset = 0, desc } = options;- const [start, end] = this.getStartAndEndIndexes(+ const [start, end] = this.#getStartAndEndIndexes(desc ? Math.max(this.itemCount - offset - limit, 0) : offset,limit,);@@ -97,11 +96,11 @@ export class DatasetBackend<Data extends Dictionary = Dictionary>const items: Data[] = [];for (let idx = start; idx < end; idx++) {- const entryNumber = this.generateLocalEntryName(idx);+ const entryNumber = this.#generateLocalEntryName(idx);items.push(this.#datasetEntries.get(entryNumber)!);}- this.updateTimestamps(false);+ this.#updateTimestamps(false);return {count: items.length,@@ -115,11 +114,11 @@ export class DatasetBackend<Data extends Dictionary = Dictionary>async pushData(items: Data[]): Promise<void> {for (const entry of items) {- const idx = this.generateLocalEntryName(++this.itemCount);+ const idx = this.#generateLocalEntryName(++this.itemCount);this.#datasetEntries.set(idx, JSON.parse(JSON.stringify(entry)) as Data);}- this.updateTimestamps(true);+ this.#updateTimestamps(true);}toDatasetInfo(): storage.DatasetInfo {@@ -133,17 +132,17 @@ export class DatasetBackend<Data extends Dictionary = Dictionary>};}- private generateLocalEntryName(idx: number): string {+ #generateLocalEntryName(idx: number): string {return idx.toString().padStart(LOCAL_ENTRY_NAME_DIGITS, '0');}- private getStartAndEndIndexes(offset: number, limit = this.itemCount) {+ #getStartAndEndIndexes(offset: number, limit = this.itemCount) {const start = offset + 1;const end = Math.min(offset + limit, this.itemCount) + 1;return [start, end] as const;}- private updateTimestamps(hasBeenModified: boolean) {+ #updateTimestamps(hasBeenModified: boolean) {this.accessedAt = new Date();if (hasBeenModified) {diff --git a/packages/core/src/memory-storage/resource-clients/key-value-store.ts b/packages/core/src/memory-storage/resource-clients/key-value-store.tsindex b56c12b13..36a84fa0f 100644--- a/packages/core/src/memory-storage/resource-clients/key-value-store.ts+++ b/packages/core/src/memory-storage/resource-clients/key-value-store.ts@@ -58,30 +58,29 @@ export class KeyValueStoreBackend extends BaseClient implements storage.KeyValuemodifiedAt = new Date();readonly #keyValueEntries = new Map<string, InternalKeyRecord>();- // kept as TS-private: storage-backend tests read this field at runtime- private readonly storageBackend: MemoryStorageBackend;+ readonly #storageBackend: MemoryStorageBackend;constructor(options: KeyValueStoreBackendOptions) {super(options.id ?? randomUUID());this.name = options.name;this.cacheKey = options.cacheKey ?? this.name ?? this.id;- this.storageBackend = options.storageBackend;+ this.#storageBackend = options.storageBackend;}async getMetadata(): Promise<storage.KeyValueStoreInfo> {- this.updateTimestamps(false);+ this.#updateTimestamps(false);return this.toKeyValueStoreInfo();}async drop(): Promise<void> {- if (this.storageBackend.evictBackend('KeyValueStore', this.id)) {+ if (this.#storageBackend.evictBackend('KeyValueStore', this.id)) {this.#keyValueEntries.clear();}}async purge(): Promise<void> {this.#keyValueEntries.clear();- this.updateTimestamps(true);+ this.#updateTimestamps(true);}async listKeys(options: storage.KeyValueStoreListKeysOptions = {}): Promise<storage.KeyValueStoreListKeysResult> {@@ -118,7 +117,7 @@ export class KeyValueStoreBackend extends BaseClient implements storage.KeyValueconst pageItems = isTruncated ? filteredItems.slice(0, limit) : filteredItems;const nextExclusiveStartKey = isTruncated ? pageItems[pageItems.length - 1].key : undefined;- this.updateTimestamps(false);+ this.#updateTimestamps(false);return {items: pageItems,@@ -172,7 +171,7 @@ export class KeyValueStoreBackend extends BaseClient implements storage.KeyValuecontentType: entry.contentType ?? (mime.contentType(entry.extension) || undefined),};- this.updateTimestamps(false);+ this.#updateTimestamps(false);return record;}@@ -214,7 +213,7 @@ export class KeyValueStoreBackend extends BaseClient implements storage.KeyValuethis.#keyValueEntries.set(key, normalizedRecord);- this.updateTimestamps(true);+ this.#updateTimestamps(true);}async deleteValue(key: string): Promise<void> {@@ -222,7 +221,7 @@ export class KeyValueStoreBackend extends BaseClient implements storage.KeyValueif (this.#keyValueEntries.has(key)) {this.#keyValueEntries.delete(key);- this.updateTimestamps(true);+ this.#updateTimestamps(true);}}@@ -236,7 +235,7 @@ export class KeyValueStoreBackend extends BaseClient implements storage.KeyValue};}- private updateTimestamps(hasBeenModified: boolean) {+ #updateTimestamps(hasBeenModified: boolean) {this.accessedAt = new Date();if (hasBeenModified) {diff --git a/packages/core/src/memory-storage/resource-clients/request-queue.ts b/packages/core/src/memory-storage/resource-clients/request-queue.tsindex 6dbd04218..067127c80 100644--- a/packages/core/src/memory-storage/resource-clients/request-queue.ts+++ b/packages/core/src/memory-storage/resource-clients/request-queue.ts@@ -73,18 +73,17 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQu* ever walk the unhandled tail instead of every request the queue has ever seen.*/readonly #pendingRequestIds = new Set<string>();- // kept as TS-private: storage-backend tests read this field at runtime- private readonly storageBackend: MemoryStorageBackend;+ readonly #storageBackend: MemoryStorageBackend;constructor(options: RequestQueueBackendOptions) {super(options.id ?? randomUUID());this.name = options.name;this.cacheKey = options.cacheKey ?? this.name ?? this.id;- this.storageBackend = options.storageBackend;+ this.#storageBackend = options.storageBackend;}async getMetadata(): Promise<storage.RequestQueueInfo> {- this.updateTimestamps(false);+ this.#updateTimestamps(false);return this.toRequestQueueInfo();}@@ -95,7 +94,7 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQuawait this.#queueStateMutex.wait();try {- if (this.storageBackend.evictBackend('RequestQueue', this.id)) {+ if (this.#storageBackend.evictBackend('RequestQueue', this.id)) {this.pendingRequestCount = 0;// Clear all in-memory state, consistent with `purge`. Clearing `requests` alone would// leave dangling ids in `forefrontRequestIds`/`inProgressRequestIds`, which a later head@@ -124,13 +123,13 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQuthis.handledRequestCount = 0;this.pendingRequestCount = 0;- this.updateTimestamps(true);+ this.#updateTimestamps(true);} finally {this.#queueStateMutex.shift();}}- private *requestKeyIterator(): IterableIterator<string> {+ *#requestKeyIterator(): IterableIterator<string> {for (let i = this.#forefrontRequestIds.length - 1; i >= 0; i--) {yield this.#forefrontRequestIds[i];}@@ -153,7 +152,7 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQu* need the head (e.g. {@link fetchNextRequest}, {@link isEmpty}) leave it off so the scan can stop as* soon as the page is filled, keeping those calls O(head) instead of O(pending).*/- private async listPendingHead(+ async #listPendingHead(limit: number,detectInProgressRequests = false,): Promise<{ items: InternalRequest[]; hasInProgressRequests?: boolean }> {@@ -165,7 +164,7 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQu// Tracks handled request IDs from `forefrontRequestIds` to be removed.const handledForefrontIds = new Set<string>();- for (const requestId of this.requestKeyIterator()) {+ for (const requestId of this.#requestKeyIterator()) {// Once the requested page is filled we can stop — unless the caller asked us to detect// in-progress requests and we have not yet seen one, in which case we must keep scanning.if (items.length >= limit && (!detectInProgressRequests || hasInProgressRequests)) {@@ -208,14 +207,14 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQu}async fetchNextRequest(): Promise<storage.UpdateRequestSchema | undefined> {- this.updateTimestamps(false);+ this.#updateTimestamps(false);await this.#queueStateMutex.wait();try {const {items: [head],- } = await this.listPendingHead(1);+ } = await this.#listPendingHead(1);if (!head) {return undefined;@@ -225,7 +224,7 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQu// reclaimed. The request keeps its `orderNo` (and thus its forefront / normal ordering).this.#inProgressRequestIds.add(head.id);- return this.jsonToRequest(head.json) ?? undefined;+ return this.#jsonToRequest(head.json) ?? undefined;} finally {this.#queueStateMutex.shift();}@@ -250,7 +249,7 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQu};for (const model of requests) {- const requestModel = this.createInternalRequest(model, options.forefront);+ const requestModel = this.#createInternalRequest(model, options.forefront);const existingRequestWithId = this.#requests.get(requestModel.id);@@ -288,7 +287,7 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQu});}- this.updateTimestamps(true);+ this.#updateTimestamps(true);return result;} finally {@@ -298,15 +297,15 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQuasync getRequest(uniqueKey: string): Promise<storage.UpdateRequestSchema | undefined> {parseArgument(uniqueKey, uniqueKeySchema);- this.updateTimestamps(false);+ this.#updateTimestamps(false);const id = uniqueKeyToRequestId(uniqueKey);const json = this.#requests.get(id)?.json;- return this.jsonToRequest(json);+ return this.#jsonToRequest(json);}async markRequestAsHandled(request: storage.UpdateRequestSchema): Promise<storage.QueueOperationInfo | undefined> {parseArgument(request, schemas.storageRequest);- this.updateTimestamps(false);+ this.#updateTimestamps(false);// Serialize against other mutators (and the head scans in `isEmpty`/`isFinished`) so the shared// `requests` map, `inProgressRequestIds` set and request counts stay consistent across the@@ -329,7 +328,7 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQuconst wasAlreadyHandled = existingRequest.orderNo === null;const handledAt = request.handledAt ?? new Date().toISOString();- const requestModel = this.createInternalRequest({ ...request, handledAt }, false);+ const requestModel = this.#createInternalRequest({ ...request, handledAt }, false);this.#requests.set(id, requestModel);this.#pendingRequestIds.delete(id);@@ -342,7 +341,7 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQuthis.handledRequestCount += 1;}- this.updateTimestamps(true);+ this.#updateTimestamps(true);return {requestId: id,@@ -360,7 +359,7 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQu): Promise<storage.QueueOperationInfo | undefined> {parseArgument(request, schemas.storageRequest);parseArgument(options, schemas.requestQueueOperationOptions);- this.updateTimestamps(false);+ this.#updateTimestamps(false);// Serialize against other mutators (and the head scans in `isEmpty`/`isFinished`) so the shared// `requests` map, `forefrontRequestIds` array and `inProgressRequestIds` set stay consistent@@ -382,7 +381,7 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQu// Reclaiming resets the `orderNo` to a fresh timestamp, restoring the request to the queue// (at the front if `forefront`).- const requestModel = this.createInternalRequest(request, options.forefront);+ const requestModel = this.#createInternalRequest(request, options.forefront);this.#requests.set(id, requestModel);@@ -393,7 +392,7 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQuthis.#forefrontRequestIds.push(id);}- this.updateTimestamps(true);+ this.#updateTimestamps(true);return {requestId: id,@@ -406,7 +405,7 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQu}async isEmpty(): Promise<boolean> {- this.updateTimestamps(false);+ this.#updateTimestamps(false);// "Empty" means there is nothing left to fetch right now — i.e. the next `fetchNextRequest`// would return `null`. Requests that are currently in progress are intentionally NOT counted@@ -419,7 +418,7 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQuawait this.#queueStateMutex.wait();try {- const { items } = await this.listPendingHead(1);+ const { items } = await this.#listPendingHead(1);return items.length === 0;} finally {this.#queueStateMutex.shift();@@ -427,7 +426,7 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQu}async isFinished(): Promise<boolean> {- this.updateTimestamps(false);+ this.#updateTimestamps(false);// The queue is finished only when there is nothing left to fetch AND nothing currently in// progress. Counting in-progress requests is what allows a crawler with concurrency to keep@@ -441,7 +440,7 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQuawait this.#queueStateMutex.wait();try {- const { items, hasInProgressRequests } = await this.listPendingHead(1, true);+ const { items, hasInProgressRequests } = await this.#listPendingHead(1, true);return items.length === 0 && !hasInProgressRequests;} finally {this.#queueStateMutex.shift();@@ -454,15 +453,15 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQu* nothing is marked in progress.*/async listItems(): Promise<storage.UpdateRequestSchema[]> {- this.updateTimestamps(false);+ this.#updateTimestamps(false);// `listPendingHead` prunes `forefrontRequestIds` as it scans, so we must hold the queue-state// mutex to avoid racing a concurrent mutator at its `await` points.await this.#queueStateMutex.wait();try {- const { items } = await this.listPendingHead(Number.POSITIVE_INFINITY);- return items.map((request) => this.jsonToRequest<storage.UpdateRequestSchema>(request.json)!);+ const { items } = await this.#listPendingHead(Number.POSITIVE_INFINITY);+ return items.map((request) => this.#jsonToRequest<storage.UpdateRequestSchema>(request.json)!);} finally {this.#queueStateMutex.shift();}@@ -481,7 +480,7 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQu};}- private updateTimestamps(hasBeenModified: boolean) {+ #updateTimestamps(hasBeenModified: boolean) {this.accessedAt = new Date();if (hasBeenModified) {@@ -489,14 +488,14 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQu}}- private jsonToRequest<T>(requestJson?: string): T | undefined {+ #jsonToRequest<T>(requestJson?: string): T | undefined {if (!requestJson) return undefined;const request = JSON.parse(requestJson);return purgeNullsFromObject(request);}- private createInternalRequest(request: storage.RequestSchema, forefront?: boolean): InternalRequest {- const orderNo = this.calculateOrderNo(request, forefront);+ #createInternalRequest(request: storage.RequestSchema, forefront?: boolean): InternalRequest {+ const orderNo = this.#calculateOrderNo(request, forefront);const id = uniqueKeyToRequestId(request.uniqueKey);if (request.id && request.id !== id) {@@ -515,7 +514,7 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQu};}- private calculateOrderNo(request: storage.RequestSchema, forefront?: boolean) {+ #calculateOrderNo(request: storage.RequestSchema, forefront?: boolean) {if (request.handledAt) return null;const timestamp = Date.now();diff --git a/packages/core/src/proxy_configuration.ts b/packages/core/src/proxy_configuration.tsindex fe17dd15f..ae62afee9 100644--- a/packages/core/src/proxy_configuration.ts+++ b/packages/core/src/proxy_configuration.ts@@ -130,8 +130,8 @@ export class ProxyConfiguration implements IProxyConfiguration {proxyConfigurationOptionsSchema,);- if (proxyUrls && newUrlFunction) this.throwCannotCombineCustomMethods();- if (!proxyUrls && !newUrlFunction && validateRequired) this.throwNoOptionsProvided();+ if (proxyUrls && newUrlFunction) this.#throwCannotCombineCustomMethods();+ if (!proxyUrls && !newUrlFunction && validateRequired) this.#throwNoOptionsProvided();this.#proxyUrls = proxyUrls;this.#newUrlFunction = newUrlFunction;@@ -169,20 +169,20 @@ export class ProxyConfiguration implements IProxyConfiguration {*/async newUrl(): Promise<string | undefined> {if (this.#newUrlFunction) {- return (await this.callNewUrlFunction()) ?? undefined;+ return (await this.#callNewUrlFunction()) ?? undefined;}- return this.handleProxyUrlsList() ?? undefined;+ return this.#handleProxyUrlsList() ?? undefined;}- private handleProxyUrlsList(): string | null {+ #handleProxyUrlsList(): string | null {return this.#proxyUrls![this.#nextCustomUrlIndex++ % this.#proxyUrls!.length];}/*** Calls the custom newUrlFunction and checks format of its return value*/- private async callNewUrlFunction() {+ async #callNewUrlFunction() {const proxyUrl = await this.#newUrlFunction!();try {if (proxyUrl) {@@ -196,13 +196,13 @@ export class ProxyConfiguration implements IProxyConfiguration {}}- private throwCannotCombineCustomMethods(): never {+ #throwCannotCombineCustomMethods(): never {throw new Error('Cannot combine custom proxies "options.proxyUrls" with custom generating function "options.newUrlFunction".',);}- private throwNoOptionsProvided(): never {+ #throwNoOptionsProvided(): never {throw new Error('One of "options.proxyUrls" or "options.newUrlFunction" needs to be provided.');}}diff --git a/packages/core/src/storages/dataset.ts b/packages/core/src/storages/dataset.tsindex 82641e34b..89e3adce1 100644--- a/packages/core/src/storages/dataset.ts+++ b/packages/core/src/storages/dataset.ts@@ -193,7 +193,7 @@ export interface DatasetExportToOptions extends DatasetExportOptions {export class Dataset<Data extends Dictionary = Dictionary> {readonly id: string;readonly name?: string;- // kept as TS-private: dataset tests spy on the backend directly+ // oxlint-disable-next-line crawlee/prefer-private-fields -- tests spy on the backend directlyprivate readonly backend: DatasetBackend<Data>;readonly #statsTracker = new StorageStatsTracker<DatasetStats>({@@ -266,7 +266,7 @@ export class Dataset<Data extends Dictionary = Dictionary> {*/async getData(options: DatasetDataOptions = {}): Promise<DatasetContent<Data>> {try {- return await this.readPage(options);+ return await this.#readPage(options);} catch (e) {const error = e as Error;if (error.message.includes('Cannot create a string longer than')) {@@ -283,8 +283,8 @@ export class Dataset<Data extends Dictionary = Dictionary> {* the private `fetchPages()`. Returns the real page concatenated with the current transaction's* buffered items, with `offset` / `limit` / `desc` windowing applied across the concatenation.*/- private async readPage(options: DatasetDataOptions): Promise<DatasetContent<Data>> {- const buffered = this.bufferedJournalEntries()?.flatMap((entry) => entry.items as Data[]);+ async #readPage(options: DatasetDataOptions): Promise<DatasetContent<Data>> {+ const buffered = this.#bufferedJournalEntries()?.flatMap((entry) => entry.items as Data[]);// Every branch below hits the backend exactly once.this.#statsTracker.add('readCount');@@ -352,7 +352,7 @@ export class Dataset<Data extends Dictionary = Dictionary> {}/** The active transaction's buffered writes to this dataset, derived from its journal. */- private bufferedJournalEntries(): DatasetJournalEntry[] | undefined {+ #bufferedJournalEntries(): DatasetJournalEntry[] | undefined {const transaction = activeStorageTransaction();return transaction?.journal.filter((entry): entry is DatasetJournalEntry => entry.type === 'dataset' && entry.participant === this,@@ -386,7 +386,7 @@ export class Dataset<Data extends Dictionary = Dictionary> {const items: Data[] = [];- for await (const page of this.fetchPages(options)) {+ for await (const page of this.#fetchPages(options)) {items.push(...page.items);}@@ -499,7 +499,7 @@ export class Dataset<Data extends Dictionary = Dictionary> {* @throws If the underlying storage no longer exists (e.g. it was deleted externally).*/async getInfo(): Promise<DatasetInfo> {- const buffered = this.bufferedJournalEntries();+ const buffered = this.#bufferedJournalEntries();const metadata = await this.backend.getMetadata();if (buffered?.length) {@@ -659,9 +659,9 @@ export class Dataset<Data extends Dictionary = Dictionary> {return currentMemo;}- private async *fetchEntryPages(options: DatasetIteratorOptions): AsyncGenerator<PaginatedList<[number, Data]>> {+ async *#fetchEntryPages(options: DatasetIteratorOptions): AsyncGenerator<PaginatedList<[number, Data]>> {let index = options.offset ?? 0;- for await (const page of this.fetchPages(options)) {+ for await (const page of this.#fetchPages(options)) {yield {...page,items: page.items.map((item) => [index++, item] as [number, Data]),@@ -669,7 +669,7 @@ export class Dataset<Data extends Dictionary = Dictionary> {}}- private async *fetchPages(+ async *#fetchPages(options: DatasetIteratorOptions,pageSize = DATASET_ITERATORS_DEFAULT_LIMIT,): AsyncGenerator<PaginatedList<Data>> {@@ -681,7 +681,7 @@ export class Dataset<Data extends Dictionary = Dictionary> {const fetchLimit = totalLimit !== undefined ? Math.min(pageSize, totalLimit - yielded) : pageSize;if (fetchLimit <= 0) break;- const page = await this.readPage({ ...options, offset, limit: fetchLimit });+ const page = await this.#readPage({ ...options, offset, limit: fetchLimit });yield page;yielded += page.items.length;@@ -717,7 +717,7 @@ export class Dataset<Data extends Dictionary = Dictionary> {tryCancel();return createDualIterable({- createPages: () => this.fetchPages(options),+ createPages: () => this.#fetchPages(options),extractItems: (page) => page.items,});}@@ -749,7 +749,7 @@ export class Dataset<Data extends Dictionary = Dictionary> {tryCancel();return createDualIterable({- createPages: () => this.fetchEntryPages(options),+ createPages: () => this.#fetchEntryPages(options),extractItems: (page) => page.items,});}diff --git a/packages/core/src/storages/key_value_store.ts b/packages/core/src/storages/key_value_store.tsindex be30eb0a9..5c5aaba68 100644--- a/packages/core/src/storages/key_value_store.ts+++ b/packages/core/src/storages/key_value_store.ts@@ -237,7 +237,7 @@ export class KeyValueStore {tryCancel();parseArgument(key, keySchema);- const record = await this.readRecord(key);+ const record = await this.#readRecord(key);// A missing record falls back to the default; a record that parses to a falsy value (including// a stored literal `null`) is returned verbatim, so callers can tell "stored null" from "absent".@@ -253,7 +253,7 @@ export class KeyValueStore {* The active transaction's last buffered write per key for this store, derived from its journal.* An entry with a `null` value is a tombstone (an in-transaction deletion).*/- private bufferedJournalEntries(): Map<string, KeyValueStoreJournalEntry> | undefined {+ #bufferedJournalEntries(): Map<string, KeyValueStoreJournalEntry> | undefined {const transaction = activeStorageTransaction();if (!transaction) return undefined;@@ -277,9 +277,9 @@ export class KeyValueStore {* which is O(journal). Single-record callers let it default (rebuilt per call); the listing paths,* which read many keys, pass a map built once so the read stays O(1) per key instead of O(journal).*/- private async readRecord(+ async #readRecord(key: string,- buffered = this.bufferedJournalEntries(),+ buffered = this.#bufferedJournalEntries(),): Promise<{ value: Buffer | ArrayBuffer; contentType: string | null } | null> {const entry = buffered?.get(key);@@ -335,7 +335,7 @@ export class KeyValueStore {tryCancel();parseArgument(key, keySchema);- return this.readRecord(key);+ return this.#readRecord(key);}/**@@ -349,7 +349,7 @@ export class KeyValueStore {parseArgument(key, keySchema);- const entry = this.bufferedJournalEntries()?.get(key);+ const entry = this.#bufferedJournalEntries()?.get(key);if (entry) {return entry.value !== null;}@@ -377,12 +377,12 @@ export class KeyValueStore {}this.#cache.set(key, value!);- this.ensurePersistStateEvent();+ this.#ensurePersistStateEvent();return value!;}- private ensurePersistStateEvent(): void {+ #ensurePersistStateEvent(): void {if (this.#persistStateEventStarted) {return;}@@ -404,19 +404,19 @@ export class KeyValueStore {this.#persistStateEventStarted = true;}- private async *fetchKeyValuePages<T>(+ async *#fetchKeyValuePages<T>(options: KeyValueStoreIteratorOptions,mapRecord: (key: string, value: unknown) => T,): AsyncGenerator<T[]> {// Reduce the journal once for the whole iteration, not once per key inside `readRecord`.- const buffered = this.bufferedJournalEntries();+ const buffered = this.#bufferedJournalEntries();- for await (const page of this.fetchKeyPages(options, buffered)) {+ for await (const page of this.#fetchKeyPages(options, buffered)) {const results: T[] = [];for (const item of page) {// The shared transaction-aware read, so a key that exists only in the transaction resolves// here instead of being dropped (`values()` would disagree with `keys()` on length).- const record = await this.readRecord(item.key, buffered);+ const record = await this.#readRecord(item.key, buffered);if (record) {const parsed = parseValue(record.value, record.contentType ?? null);results.push(mapRecord(item.key, parsed));@@ -426,9 +426,9 @@ export class KeyValueStore {}}- private async *fetchKeyPages(+ async *#fetchKeyPages(options: KeyValueStoreIteratorOptions,- buffered = this.bufferedJournalEntries(),+ buffered = this.#bufferedJournalEntries(),limit = KVS_KEYS_DEFAULT_LIMIT,): AsyncGenerator<KeyValueStoreItemData[]> {// Buffered keys are emitted first, then the real pages with any buffered (or tombstoned) key@@ -664,7 +664,7 @@ export class KeyValueStore {let index = 0;- for await (const page of this.fetchKeyPages(parsedOptions)) {+ for await (const page of this.#fetchKeyPages(parsedOptions)) {for (const item of page) {await iteratee(item.key, index++, { size: item.size });}@@ -698,7 +698,7 @@ export class KeyValueStore {tryCancel();return createDualIterable({- createPages: () => this.fetchKeyPages(options),+ createPages: () => this.#fetchKeyPages(options),extractItems: (page) => page.map((item) => item.key),});}@@ -730,7 +730,7 @@ export class KeyValueStore {tryCancel();return createDualIterable({- createPages: () => this.fetchKeyValuePages<T>(options, (_key, value) => value as T),+ createPages: () => this.#fetchKeyValuePages<T>(options, (_key, value) => value as T),extractItems: (page) => page,});}@@ -764,7 +764,7 @@ export class KeyValueStore {tryCancel();return createDualIterable({- createPages: () => this.fetchKeyValuePages<[string, T]>(options, (key, value) => [key, value as T]),+ createPages: () => this.#fetchKeyValuePages<[string, T]>(options, (key, value) => [key, value as T]),extractItems: (page) => page,});}diff --git a/packages/core/src/storages/request_dedup_cache.ts b/packages/core/src/storages/request_dedup_cache.tsindex 8d9539e68..0e79a3ef9 100644--- a/packages/core/src/storages/request_dedup_cache.ts+++ b/packages/core/src/storages/request_dedup_cache.ts@@ -14,38 +14,40 @@export class RequestDeduplicationCache {#keys: (string | undefined)[];#ids: (string | undefined)[];+ readonly #size: number;// The slot count is the same for every queue, so it's a fixed default rather than a per-consumer option.- constructor(private readonly size = 1_000_000) {+ constructor(size = 1_000_000) {+ this.#size = size;this.#keys = new Array<string | undefined>(size);this.#ids = new Array<string | undefined>(size);}get(cacheKey: string): string | null {- const index = this.indexOf(cacheKey);+ const index = this.#indexOf(cacheKey);return this.#keys[index] === cacheKey ? this.#ids[index]! : null;}add(cacheKey: string, requestId: string): void {- const index = this.indexOf(cacheKey);+ const index = this.#indexOf(cacheKey);this.#keys[index] = cacheKey;this.#ids[index] = requestId;}clear(): void {- this.#keys = new Array<string | undefined>(this.size);- this.#ids = new Array<string | undefined>(this.size);+ this.#keys = new Array<string | undefined>(this.#size);+ this.#ids = new Array<string | undefined>(this.#size);}// A cheap FNV-1a hash of the cache key — avoids pulling in a dedicated hashing dependency.- private indexOf(cacheKey: string): number {+ #indexOf(cacheKey: string): number {/* eslint-disable no-bitwise */let hash = 0x811c9dc5;for (let i = 0; i < cacheKey.length; i++) {hash ^= cacheKey.charCodeAt(i);hash = Math.imul(hash, 0x01000193);}- return (hash >>> 0) % this.size;+ return (hash >>> 0) % this.#size;/* eslint-enable no-bitwise */}}diff --git a/packages/core/src/storages/request_list.ts b/packages/core/src/storages/request_list.tsindex c7e39930f..7d194fc1b 100644--- a/packages/core/src/storages/request_list.ts+++ b/packages/core/src/storages/request_list.ts@@ -461,7 +461,7 @@ export class RequestList implements IRequestLoader {* Loads all remote sources of URLs and potentially starts periodic state persistence.* This function must be called before you can start using the instance in a meaningful way.*/- private async initialize(): Promise<this> {+ async #initialize(): Promise<this> {if (this.#isLoading) {throw new Error('RequestList sources are already loading or were loaded.');}@@ -471,7 +471,7 @@ export class RequestList implements IRequestLoader {let persistedRequests: Buffer | undefined;if (this.#persistRequestsKey) {- persistedRequests = await this.getPersistedState(this.#persistRequestsKey);+ persistedRequests = await this.#getPersistedState(this.#persistRequestsKey);if (persistedRequests)this.#log.debug('Loaded requests from key value store using the persistRequestsKey.');}@@ -479,9 +479,9 @@ export class RequestList implements IRequestLoader {// Add persisted requests / new sources in a memory efficient way because with very// large lists, we were running out of memory.if (persistedRequests) {- await this.addPersistedRequests(persistedRequests);+ await this.#addPersistedRequests(persistedRequests);} else {- await this.addRequestsFromSources();+ await this.#addRequestsFromSources();}// Restores the crawling position - the requests have to be loaded first, as the record is validated@@ -490,7 +490,7 @@ export class RequestList implements IRequestLoader {// All in-progress requests were interrupted and need to be re-crawled.this.#requestsToRetry = [...this.inProgress];this.#isInitialized = true;- if (this.#persistRequestsKey && !this.areRequestsPersisted) await this.persistRequests();+ if (this.#persistRequestsKey && !this.areRequestsPersisted) await this.#persistRequests();return this;}@@ -500,7 +500,7 @@ export class RequestList implements IRequestLoader {* This needs to be done in a memory efficient way. We should update the input* to a Stream once apify-client supports streams.*/- private async addPersistedRequests(persistedRequests: Buffer): Promise<void> {+ async #addPersistedRequests(persistedRequests: Buffer): Promise<void> {// We don't need the sources so we purge them to// prevent them from hanging in memory.for (let i = 0; i < this.#sources.length; i++) {@@ -512,7 +512,7 @@ export class RequestList implements IRequestLoader {this.areRequestsPersisted = true;const requestStream = createDeserialize(persistedRequests);for await (const request of requestStream) {- this.addRequest(request);+ this.#addRequest(request);}}@@ -522,7 +522,7 @@ export class RequestList implements IRequestLoader {* We need to avoid keeping both sources and requests in memory* to reduce memory footprint with very large sources.*/- private async addRequestsFromSources(): Promise<void> {+ async #addRequestsFromSources(): Promise<void> {// We'll load all sources in sequence to ensure that they get loaded in the right order.const sourcesCount = this.#sources.length;for (let i = 0; i < sourcesCount; i++) {@@ -533,10 +533,10 @@ export class RequestList implements IRequestLoader {delete this.#sources[i];if (typeof source === 'object' && (source as Dictionary).requestsFromUrl) {- const fetchedRequests = await this.fetchRequestsFromUrl(source as InternalSource);- await this.addFetchedRequests(source as InternalSource, fetchedRequests);+ const fetchedRequests = await this.#fetchRequestsFromUrl(source as InternalSource);+ await this.#addFetchedRequests(source as InternalSource, fetchedRequests);} else {- this.addRequest(source);+ this.#addRequest(source);}}@@ -551,7 +551,7 @@ export class RequestList implements IRequestLoader {const source = sourcesFromFunction[i];// oxlint-disable-next-line typescript/no-array-delete -- intentional, drop the slot so V8 can collect the objectdelete sourcesFromFunction[i];- this.addRequest(source);+ this.#addRequest(source);}sourcesFromFunction.length = 0;@@ -591,7 +591,7 @@ export class RequestList implements IRequestLoader {* are automatically persisted at RequestList initialization (if the persistRequestsKey is set),* but there's no reason to persist it again afterwards, because RequestList is immutable.*/- private async persistRequests(): Promise<void> {+ async #persistRequests(): Promise<void> {const serializedRequests = await serializeArray(this.requests);this.#store ??= await KeyValueStore.open();await this.#store.setValue(this.#persistRequestsKey!, serializedRequests, { contentType: CONTENT_TYPE_BINARY });@@ -603,7 +603,7 @@ export class RequestList implements IRequestLoader {* Note that the object's fields can change in future releases.*/getState(): RequestListState {- this.ensureIsInitialized();+ this.#ensureIsInitialized();return this.#stateCodec.encode(this.#state.currentValue);}@@ -612,7 +612,7 @@ export class RequestList implements IRequestLoader {* @inheritDoc*/async checkReadiness(): Promise<RequestLoaderStatus> {- this.ensureIsInitialized();+ this.#ensureIsInitialized();if (this.#requestsToRetry.length > 0 || this.#state.currentValue.nextIndex < this.requests.length) {return { status: 'ready' };@@ -626,13 +626,13 @@ export class RequestList implements IRequestLoader {* @inheritDoc*/async fetchNextRequest(): Promise<Request | null> {- this.ensureIsInitialized();+ this.#ensureIsInitialized();// First re-serve any requests that were interrupted before the last state persist.const uniqueKey = this.#requestsToRetry.shift();if (uniqueKey) {const index = this.#uniqueKeyToIndex[uniqueKey];- return this.ensureRequest(this.requests[index], index);+ return this.#ensureRequest(this.requests[index], index);}// Otherwise return next request.@@ -642,7 +642,7 @@ export class RequestList implements IRequestLoader {const request = this.requests[index];state.inProgress.add(request.uniqueKey!);state.nextIndex++;- return this.ensureRequest(request, index);+ return this.#ensureRequest(request, index);}return null;@@ -659,7 +659,7 @@ export class RequestList implements IRequestLoader {}}- private ensureRequest(requestLike: Request | RequestOptions, index: number): Request {+ #ensureRequest(requestLike: Request | RequestOptions, index: number): Request {if (requestLike instanceof Request) {return requestLike;}@@ -674,9 +674,9 @@ export class RequestList implements IRequestLoader {async markRequestAsHandled(request: Request): Promise<void> {const { uniqueKey } = request;- this.ensureUniqueKeyValid(uniqueKey);- this.ensureInProgress(uniqueKey);- this.ensureIsInitialized();+ this.#ensureUniqueKeyValid(uniqueKey);+ this.#ensureInProgress(uniqueKey);+ this.#ensureIsInitialized();this.inProgress.delete(uniqueKey);}@@ -684,11 +684,11 @@ export class RequestList implements IRequestLoader {/*** Adds all fetched requests from a URL from a remote resource.*/- private async addFetchedRequests(source: InternalSource, fetchedRequests: RequestOptions[]) {+ async #addFetchedRequests(source: InternalSource, fetchedRequests: RequestOptions[]) {const { requestsFromUrl, regex } = source;const originalLength = this.requests.length;- fetchedRequests.forEach((request) => this.addRequest(request));+ fetchedRequests.forEach((request) => this.#addRequest(request));const fetchedCount = fetchedRequests.length;const importedCount = this.requests.length - originalLength;@@ -703,7 +703,7 @@ export class RequestList implements IRequestLoader {});}- private async getPersistedState<T>(key: string): Promise<T> {+ async #getPersistedState<T>(key: string): Promise<T> {this.#store ??= await KeyValueStore.open();const state = await this.#store.getValue<T>(key);@@ -713,13 +713,13 @@ export class RequestList implements IRequestLoader {/*** Fetches URLs from requestsFromUrl and returns them in format of list of requests*/- private async fetchRequestsFromUrl(source: InternalSource): Promise<RequestOptions[]> {+ async #fetchRequestsFromUrl(source: InternalSource): Promise<RequestOptions[]> {const { requestsFromUrl, regex, ...sharedOpts } = source;// Download remote resource and parse URLs.let urlsArr;try {- urlsArr = await this.downloadListOfUrls({+ urlsArr = await this.#downloadListOfUrls({url: requestsFromUrl,urlRegExp: regex,proxyUrl: (await this.#proxyConfiguration?.newProxyInfo())?.url,@@ -742,7 +742,7 @@ export class RequestList implements IRequestLoader {* If the `source` parameter is a string or plain object and not an instance* of a `Request`, then the function creates a `Request` instance.*/- private addRequest(source: RequestListSource) {+ #addRequest(source: RequestListSource) {let request: Request | RequestOptions;const type = typeof source;@@ -765,7 +765,7 @@ export class RequestList implements IRequestLoader {}const { uniqueKey } = request;- this.ensureUniqueKeyValid(uniqueKey);+ this.#ensureUniqueKeyValid(uniqueKey);// Skip requests with duplicate uniqueKeyif (!Object.hasOwn(this.#uniqueKeyToIndex, uniqueKey)) {@@ -782,7 +782,7 @@ export class RequestList implements IRequestLoader {* Helper function that validates unique key.* Throws an error if uniqueKey is not a non-empty string.*/- private ensureUniqueKeyValid(uniqueKey: string): void {+ #ensureUniqueKeyValid(uniqueKey: string): void {if (typeof uniqueKey !== 'string' || !uniqueKey) {throw new Error("Request object's uniqueKey must be a non-empty string");}@@ -791,7 +791,7 @@ export class RequestList implements IRequestLoader {/*** Checks that a request is currently being processed and throws an error if not.*/- private ensureInProgress(uniqueKey: string): void {+ #ensureInProgress(uniqueKey: string): void {if (!this.inProgress.has(uniqueKey)) {throw new Error(`The request is not being processed (uniqueKey: ${uniqueKey})`);}@@ -800,7 +800,7 @@ export class RequestList implements IRequestLoader {/*** Throws an error if request list wasn't initialized.*/- private ensureIsInitialized(): void {+ #ensureIsInitialized(): void {if (!this.#isInitialized) {throw new Error('RequestList is not initialized; you must call "await requestList.initialize()" before using it!',@@ -812,7 +812,7 @@ export class RequestList implements IRequestLoader {* Returns the total number of unique requests present in the `RequestList`.*/async getTotalCount(): Promise<number> {- this.ensureIsInitialized();+ this.#ensureIsInitialized();return this.requests.length;}@@ -821,7 +821,7 @@ export class RequestList implements IRequestLoader {* Returns the number of pending requests in the `RequestList`.*/async getPendingCount(): Promise<number> {- this.ensureIsInitialized();+ this.#ensureIsInitialized();return this.requests.length - (this.#state.currentValue.nextIndex - this.inProgress.size);}@@ -843,7 +843,7 @@ export class RequestList implements IRequestLoader {* @inheritDoc*/async getHandledCount(): Promise<number> {- this.ensureIsInitialized();+ this.#ensureIsInitialized();return this.#state.currentValue.nextIndex - this.inProgress.size;}@@ -909,7 +909,7 @@ export class RequestList implements IRequestLoader {if (listNameOrOptions != null && typeof listNameOrOptions === 'object') {options = { ...listNameOrOptions, ...options };const rl = new RequestList(options);- await rl.initialize();+ await rl.#initialize();return rl;}@@ -926,7 +926,7 @@ export class RequestList implements IRequestLoader {persistRequestsKey: listName ? `${listName}-${REQUESTS_PERSISTENCE_KEY}` : options.persistRequestsKey,sources: sources ?? options.sources,});- await rl.initialize();+ await rl.#initialize();return rl;}@@ -934,11 +934,7 @@ export class RequestList implements IRequestLoader {/*** @internal wraps public utility for mocking purposes*/- private async downloadListOfUrls(options: {- url: string;- urlRegExp?: RegExp;- proxyUrl?: string;- }): Promise<string[]> {+ async #downloadListOfUrls(options: { url: string; urlRegExp?: RegExp; proxyUrl?: string }): Promise<string[]> {return downloadListOfUrls({...options,httpClient: this.#httpClient,diff --git a/packages/core/src/storages/request_manager_tandem.ts b/packages/core/src/storages/request_manager_tandem.tsindex 688b68b07..d707a0ff5 100644--- a/packages/core/src/storages/request_manager_tandem.ts+++ b/packages/core/src/storages/request_manager_tandem.ts@@ -57,7 +57,7 @@ export class RequestManagerTandem implements IRequestManager {/*** Resolves the writable request manager, opening it lazily (via the factory) on first use and memoizing the result.*/- private async getRequestManager(): Promise<IRequestManager> {+ async #getRequestManager(): Promise<IRequestManager> {if (this.#resolvedRequestManager === undefined) {this.#requestManagerPromise ??= Promise.resolve(this.#requestManagerFactory());this.#resolvedRequestManager = await this.#requestManagerPromise;@@ -79,14 +79,14 @@ export class RequestManagerTandem implements IRequestManager {* @returns `true` if a request was successfully transferred (or there was nothing to transfer), and `false` if a* transfer was attempted but failed - in which case the caller should not fetch from the manager this round.*/- private async transferNextRequestToQueue(): Promise<boolean> {+ async #transferNextRequestToQueue(): Promise<boolean> {const request = await this.#requestLoader.fetchNextRequest();if (request === null) {return true;}- const requestManager = await this.getRequestManager();+ const requestManager = await this.#getRequestManager();try {await requestManager.addRequest(request, { forefront: true });@@ -114,13 +114,13 @@ export class RequestManagerTandem implements IRequestManager {if ((await this.#requestLoader.checkReadiness()).status === 'ready') {// If the transfer failed, the request was dropped; don't fetch from the manager this round (matching// crawlee-python behaviour). The next `fetchNextRequest()` call will pick up where we left off.- if (!(await this.transferNextRequestToQueue())) {+ if (!(await this.#transferNextRequestToQueue())) {return null;}}// Try to fetch from manager after the transfer- return (await this.getRequestManager()).fetchNextRequest<T>();+ return (await this.#getRequestManager()).fetchNextRequest<T>();}/**@@ -128,7 +128,7 @@ export class RequestManagerTandem implements IRequestManager {* @inheritdoc*/async checkReadiness(): Promise<RequestSourceStatus> {- const requestManager = await this.getRequestManager();+ const requestManager = await this.#getRequestManager();const [loaderStatus, managerStatus] = await Promise.all([this.#requestLoader.checkReadiness(),requestManager.checkReadiness(),@@ -142,14 +142,14 @@ export class RequestManagerTandem implements IRequestManager {*/async getHandledCount(): Promise<number> {// Since one of the stores needs to have priority when both are present, we query the request manager - the request loader will first be dumped into the manager and then left empty.- return (await this.getRequestManager()).getHandledCount();+ return (await this.#getRequestManager()).getHandledCount();}/*** @inheritdoc*/async getTotalCount(): Promise<number> {- const requestManager = await this.getRequestManager();+ const requestManager = await this.#getRequestManager();const [managerTotal, loaderTotal] = await Promise.all([requestManager.getTotalCount(),// count only pending to avoid double counting, requests marked as "handled" have been moved to requestManager@@ -162,7 +162,7 @@ export class RequestManagerTandem implements IRequestManager {* @inheritdoc*/async getPendingCount(): Promise<number> {- const requestManager = await this.getRequestManager();+ const requestManager = await this.#getRequestManager();const [managerPending, loaderPending] = await Promise.all([requestManager.getPendingCount(),this.#requestLoader.getPendingCount(),@@ -185,7 +185,7 @@ export class RequestManagerTandem implements IRequestManager {* @inheritdoc*/async markRequestAsHandled(request: Request): Promise<RequestQueueOperationInfo | void | null> {- return (await this.getRequestManager()).markRequestAsHandled(request);+ return (await this.#getRequestManager()).markRequestAsHandled(request);}/**@@ -195,14 +195,14 @@ export class RequestManagerTandem implements IRequestManager {request: Request,options?: RequestQueueOperationOptions,): Promise<RequestQueueOperationInfo | null> {- return (await this.getRequestManager()).reclaimRequest(request, options);+ return (await this.#getRequestManager()).reclaimRequest(request, options);}/*** @inheritdoc*/async addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo> {- return (await this.getRequestManager()).addRequest(requestLike, options);+ return (await this.#getRequestManager()).addRequest(requestLike, options);}/**@@ -212,7 +212,7 @@ export class RequestManagerTandem implements IRequestManager {requests: RequestsLike,options?: AddRequestsBatchedOptions,): Promise<AddRequestsBatchedResult> {- return (await this.getRequestManager()).addRequestsBatched(requests, options);+ return (await this.#getRequestManager()).addRequestsBatched(requests, options);}/**@@ -221,7 +221,7 @@ export class RequestManagerTandem implements IRequestManager {* @inheritdoc*/async purge(): Promise<void> {- await (await this.getRequestManager()).purge?.();+ await (await this.#getRequestManager()).purge?.();}/**@@ -245,6 +245,6 @@ export class RequestManagerTandem implements IRequestManager {}async extendRequestProcessingTimeSecs(request: Request, secs: number): Promise<boolean> {- return (await this.getRequestManager()).extendRequestProcessingTimeSecs?.(request, secs) ?? false;+ return (await this.#getRequestManager()).extendRequestProcessingTimeSecs?.(request, secs) ?? false;}}diff --git a/packages/core/src/storages/request_queue.ts b/packages/core/src/storages/request_queue.tsindex fe80bd0a0..14e67cba3 100644--- a/packages/core/src/storages/request_queue.ts+++ b/packages/core/src/storages/request_queue.ts@@ -236,8 +236,8 @@ export class RequestQueue implements IStorage, IRequestManager {const { forefront } = parseArgument(options, operationOptionsSchema);if ('requestsFromUrl' in requestLike) {- const requests = await this.fetchRequestsFromUrl(requestLike as InternalSource);- const processedRequests = await this.addFetchedRequests(requestLike as InternalSource, requests, options);+ const requests = await this.#fetchRequestsFromUrl(requestLike as InternalSource);+ const processedRequests = await this.#addFetchedRequests(requestLike as InternalSource, requests, options);return { ...processedRequests[0], forefront };}@@ -247,7 +247,7 @@ export class RequestQueue implements IStorage, IRequestManager {const request = requestLike instanceof Request ? requestLike : new Request(requestLike as RequestOptions);if (transaction?.policy.requestQueue === 'deferred') {- return this.addRequestDeferred(transaction, request, forefront);+ return this.#addRequestDeferred(transaction, request, forefront);}const cacheKey = getRequestId(request.uniqueKey);@@ -255,7 +255,7 @@ export class RequestQueue implements IStorage, IRequestManager {if (cachedInfo) {request.id = cachedInfo.id;- this.recordRequestJournalEntry(transaction, [request], forefront, true);+ this.#recordRequestJournalEntry(transaction, [request], forefront, true);return {wasAlreadyPresent: true,// We may assume that if request is in local cache then also the information if the@@ -269,14 +269,14 @@ export class RequestQueue implements IStorage, IRequestManager {this.#statsTracker.add('writeCount');const { processedRequests } = await this.backend.addBatchOfRequests([request], { forefront });- this.recordRequestJournalEntry(transaction, [request], forefront, true);+ this.#recordRequestJournalEntry(transaction, [request], forefront, true);const queueOperationInfo = {...processedRequests[0],uniqueKey: request.uniqueKey,forefront,} satisfies RequestQueueOperationInfo;- this.cacheRequest(cacheKey, queueOperationInfo);+ this.#cacheRequest(cacheKey, queueOperationInfo);this.#requestSeenCache.add(cacheKey, request.id!);return queueOperationInfo;@@ -286,7 +286,7 @@ export class RequestQueue implements IStorage, IRequestManager {* Journals an addition for introspection only; these entries are never replayed. A no-op unless the* transaction is open, so detached and outliving writers stay out of the journal.*/- private recordRequestJournalEntry(+ #recordRequestJournalEntry(transaction: StorageTransaction | undefined,requests: Request[],forefront: boolean,@@ -311,7 +311,7 @@ export class RequestQueue implements IStorage, IRequestManager {* The requests buffered by the given transaction for this queue, keyed by `uniqueKey` — a dedup* index derived from the transaction journal.*/- private bufferedRequests(transaction: StorageTransaction): Map<string, RequestSchema> {+ #bufferedRequests(transaction: StorageTransaction): Map<string, RequestSchema> {const buffered = new Map<string, RequestSchema>();// Only `deferred` records snapshots, so scanning the journal under `writeThrough` never finds any.@@ -332,15 +332,15 @@ export class RequestQueue implements IStorage, IRequestManager {* A new request's `requestId` is the local `uniqueKey` hash and is **provisional** — never write it* to `request.id` or the dedup caches. Dedup is cheapest-first: buffer, caches, then a backend probe.*/- private async addRequestDeferred(+ async #addRequestDeferred(transaction: StorageTransaction,request: Request,forefront: boolean,- buffered = this.bufferedRequests(transaction),+ buffered = this.#bufferedRequests(transaction),): Promise<RequestQueueOperationInfo> {// This transaction's own buffered adds; the shared caches never see them (provisional ids).if (buffered.has(request.uniqueKey)) {- this.recordRequestJournalEntry(transaction, [request], forefront, false);+ this.#recordRequestJournalEntry(transaction, [request], forefront, false);return {wasAlreadyPresent: true,wasAlreadyHandled: false,@@ -357,7 +357,7 @@ export class RequestQueue implements IStorage, IRequestManager {const knownRequestId = cachedInfo?.id ?? this.#requestSeenCache.get(cacheKey);if (knownRequestId) {- this.recordRequestJournalEntry(transaction, [request], forefront, false);+ this.#recordRequestJournalEntry(transaction, [request], forefront, false);return {wasAlreadyPresent: true,// The dedup cache doesn't track the handled state; only the full record does.@@ -372,7 +372,7 @@ export class RequestQueue implements IStorage, IRequestManager {const existing = await this.backend.getRequest(request.uniqueKey);if (existing) {- this.recordRequestJournalEntry(transaction, [request], forefront, false);+ this.#recordRequestJournalEntry(transaction, [request], forefront, false);return {wasAlreadyPresent: true,wasAlreadyHandled: existing.handledAt != null,@@ -440,7 +440,7 @@ export class RequestQueue implements IStorage, IRequestManager {// Only now, with the real backend-assigned ids, may the shared dedup caches be populated.for (const processed of processedRequests) {const cacheKey = getRequestId(processed.uniqueKey);- this.cacheRequest(cacheKey, { ...processed, forefront });+ this.#cacheRequest(cacheKey, { ...processed, forefront });this.#requestSeenCache.add(cacheKey, processed.requestId);}@@ -504,8 +504,8 @@ export class RequestQueue implements IStorage, IRequestManager {if (typeof requestLike === 'string') {requests.push(new Request({ url: requestLike }));} else if ('requestsFromUrl' in requestLike) {- const fetchedRequests = await this.fetchRequestsFromUrl(requestLike as InternalSource);- await this.addFetchedRequests(requestLike as InternalSource, fetchedRequests, options);+ const fetchedRequests = await this.#fetchRequestsFromUrl(requestLike as InternalSource);+ await this.#addFetchedRequests(requestLike as InternalSource, fetchedRequests, options);} else {requests.push(requestLike instanceof Request ? requestLike : new Request(requestLike as RequestOptions),@@ -514,18 +514,18 @@ export class RequestQueue implements IStorage, IRequestManager {}if (transaction?.policy.requestQueue === 'deferred') {- const buffered = this.bufferedRequests(transaction);+ const buffered = this.#bufferedRequests(transaction);for (const request of requests) {results.processedRequests.push(- await this.addRequestDeferred(transaction, request, forefront, buffered),+ await this.#addRequestDeferred(transaction, request, forefront, buffered),);}return results;}- this.recordRequestJournalEntry(transaction, requests, forefront, true);+ this.#recordRequestJournalEntry(transaction, requests, forefront, true);const requestsToAdd = new Map<string, Request>();@@ -568,7 +568,7 @@ export class RequestQueue implements IStorage, IRequestManager {const cacheKey = getCachedRequestId(newRequest.uniqueKey);if (cache) {- this.cacheRequest(cacheKey, { ...newRequest, forefront });+ this.#cacheRequest(cacheKey, { ...newRequest, forefront });}// Unlike `requestCache`, populate this on every batch (including background ones).@@ -685,7 +685,7 @@ export class RequestQueue implements IStorage, IRequestManager {parseArgument(uniqueKey, uniqueKeySchema);// Requests buffered by the active transaction (under the `deferred` write policy) are visible to it.- const buffered = transaction && this.bufferedRequests(transaction).get(uniqueKey);+ const buffered = transaction && this.#bufferedRequests(transaction).get(uniqueKey);if (buffered) {return Request.fromSchema<T>(buffered);}@@ -766,7 +766,7 @@ export class RequestQueue implements IStorage, IRequestManager {forefront,} satisfies RequestQueueOperationInfo;- this.cacheRequest(getRequestId(request.uniqueKey), queueOperationInfo);+ this.#cacheRequest(getRequestId(request.uniqueKey), queueOperationInfo);return queueOperationInfo;}@@ -804,7 +804,7 @@ export class RequestQueue implements IStorage, IRequestManager {uniqueKey: request.uniqueKey,forefront,} satisfies RequestQueueOperationInfo;- this.cacheRequest(getRequestId(request.uniqueKey), queueOperationInfo);+ this.#cacheRequest(getRequestId(request.uniqueKey), queueOperationInfo);return queueOperationInfo;}@@ -831,7 +831,7 @@ export class RequestQueue implements IStorage, IRequestManager {const transaction = activeStorageTransaction();// Requests buffered by the active transaction count as pending from its point of view.- if (transaction && this.bufferedRequests(transaction).size > 0) {+ if (transaction && this.#bufferedRequests(transaction).size > 0) {return { status: 'ready' };}@@ -883,7 +883,7 @@ export class RequestQueue implements IStorage, IRequestManager {/*** Caches information about request to beware of unneeded addRequest() calls.*/- private cacheRequest(cacheKey: string, queueOperationInfo: RequestQueueOperationInfo): void {+ #cacheRequest(cacheKey: string, queueOperationInfo: RequestQueueOperationInfo): void {// Remove the previous entry, as otherwise our cache will never update 👀this.#requestCache.remove(cacheKey);@@ -977,7 +977,7 @@ export class RequestQueue implements IStorage, IRequestManager {const transaction = activeStorageTransaction();const metadata = await this.backend.getMetadata();- const bufferedCount = transaction ? this.bufferedRequests(transaction).size : 0;+ const bufferedCount = transaction ? this.#bufferedRequests(transaction).size : 0;if (bufferedCount > 0) {return {@@ -993,13 +993,13 @@ export class RequestQueue implements IStorage, IRequestManager {/*** Fetches URLs from requestsFromUrl and returns them in format of list of requests*/- private async fetchRequestsFromUrl(source: InternalSource): Promise<RequestOptions[]> {+ async #fetchRequestsFromUrl(source: InternalSource): Promise<RequestOptions[]> {const { requestsFromUrl, regex, ...sharedOpts } = source;// Download remote resource and parse URLs.let urlsArr;try {- urlsArr = await this.downloadListOfUrls({+ urlsArr = await this.#downloadListOfUrls({url: requestsFromUrl,urlRegExp: regex,proxyUrl: (await this.#proxyConfiguration?.newProxyInfo())?.url,@@ -1020,7 +1020,7 @@ export class RequestQueue implements IStorage, IRequestManager {/*** Adds all fetched requests from a URL from a remote resource.*/- private async addFetchedRequests(+ async #addFetchedRequests(source: InternalSource,fetchedRequests: RequestOptions[],options: RequestQueueOperationOptions,@@ -1043,11 +1043,7 @@ export class RequestQueue implements IStorage, IRequestManager {/*** @internal wraps public utility for mocking purposes*/- private async downloadListOfUrls(options: {- url: string;- urlRegExp?: RegExp;- proxyUrl?: string;- }): Promise<string[]> {+ async #downloadListOfUrls(options: { url: string; urlRegExp?: RegExp; proxyUrl?: string }): Promise<string[]> {return downloadListOfUrls({...options,httpClient: this.#httpClient,diff --git a/packages/core/src/storages/storage_instance_manager.ts b/packages/core/src/storages/storage_instance_manager.tsindex 2105b62e0..6f5ecbeac 100644--- a/packages/core/src/storages/storage_instance_manager.ts+++ b/packages/core/src/storages/storage_instance_manager.ts@@ -68,7 +68,7 @@ class StorageCache {}/** Write a single entry into a given tier. */- private setInMap<T extends IStorage>(+ #setInMap<T extends IStorage>(tier: CacheTier,cls: Constructor<T>,key: string,@@ -86,16 +86,16 @@ class StorageCache {*/set<T extends IStorage>(cls: Constructor<T>, instance: T, backendCacheKey: string, alias?: string): void {// Always cache by id.- this.setInMap(this.byId, cls, instance.id, instance, backendCacheKey);+ this.#setInMap(this.byId, cls, instance.id, instance, backendCacheKey);// Cache by name — only for named storages.if (instance.name) {- this.setInMap(this.byName, cls, instance.name, instance, backendCacheKey);+ this.#setInMap(this.byName, cls, instance.name, instance, backendCacheKey);}// Cache by alias — only for unnamed storages opened via alias.if (alias !== undefined) {- this.setInMap(this.byAlias, cls, alias, instance, backendCacheKey);+ this.#setInMap(this.byAlias, cls, alias, instance, backendCacheKey);}}diff --git a/packages/core/src/storages/transaction.ts b/packages/core/src/storages/transaction.tsindex a90be884d..8eb5965d6 100644--- a/packages/core/src/storages/transaction.ts+++ b/packages/core/src/storages/transaction.ts@@ -265,7 +265,7 @@ export class StorageTransaction implements StorageTransactionView {// handler that succeeded. Hence a fresh timeout context, which also provides the time bound.await timeoutStorage.exit(async () =>addTimeoutToPromise(- async () => this.flush(),+ async () => this.#flush(),this.#commitTimeoutMillis,`Committing the storage transaction timed out after ${this.#commitTimeoutMillis / 1000} seconds.`,),@@ -299,7 +299,7 @@ export class StorageTransaction implements StorageTransactionView {});}- private async flush(): Promise<void> {+ async #flush(): Promise<void> {// Each participating frontend replays all of its buffered entries in one call. Frontends are// ordered by storage type: key-value stores and request queues first (idempotent under retry),// datasets last (not idempotent), minimizing the blast radius of a partial commit failure.diff --git a/packages/http-crawler/src/internals/file-download.ts b/packages/http-crawler/src/internals/file-download.tsindex 1d18f0216..4a2568ee9 100644--- a/packages/http-crawler/src/internals/file-download.ts+++ b/packages/http-crawler/src/internals/file-download.ts@@ -81,10 +81,10 @@ export class FileDownload extends BasicCrawler<FileDownloadCrawlingContext> {}#buildContextPipeline(): ContextPipeline<CrawlingContext, FileDownloadCrawlingContext> {- return ContextPipeline.create<CrawlingContext>().compose(this.initiateDownload.bind(this));+ return ContextPipeline.create<CrawlingContext>().compose(this.#initiateDownload.bind(this));}- private async initiateDownload(context: CrawlingContext, onCleanup: CleanupRegistrar) {+ async #initiateDownload(context: CrawlingContext, onCleanup: CleanupRegistrar) {const response = await this.httpClient.sendRequest(context.request.intoFetchAPIRequest(), {session: context.session,});diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.tsindex c609a4707..7cdd46b7f 100644--- a/packages/http-crawler/src/internals/http-crawler.ts+++ b/packages/http-crawler/src/internals/http-crawler.ts@@ -412,7 +412,7 @@ export class HttpCrawler<});this.#supportedMimeTypes = new Set([...HTML_AND_XML_MIME_TYPES, APPLICATION_JSON_MIME_TYPE]);- if (additionalMimeTypes.length) this.extendSupportedMimeTypes(additionalMimeTypes);+ if (additionalMimeTypes.length) this.#extendSupportedMimeTypes(additionalMimeTypes);if (suggestResponseEncoding && forceResponseEncoding) {this.log.warning(@@ -429,7 +429,7 @@ export class HttpCrawler<// members `extendContext` added at runtime.this.#preNavigationHooks = preNavigationHooks as InternalHttpHook<CrawlingContext>[];this.#postNavigationHooks = [- ({ request, response }) => this.abortDownloadOfBody(request, response!),+ ({ request, response }) => this.#abortDownloadOfBody(request, response!),...(postNavigationHooks as InternalHttpHook<CrawlingContextWithResponse>[]),];@@ -474,7 +474,7 @@ export class HttpCrawler<return addTimeoutToPromise(async () => step(ctx), remaining, navigationTimedOut);});- let pipeline = ContextPipeline.create<CrawlingContext>().compose(this.prepareHttpRequest.bind(this));+ let pipeline = ContextPipeline.create<CrawlingContext>().compose(this.#prepareHttpRequest.bind(this));for (const hook of this.#preNavigationHooks) {pipeline = pipeline.compose(windowGuard(hook));@@ -487,24 +487,27 @@ export class HttpCrawler<}return pipelineWithNavigation- .compose(this.processHttpResponse.bind(this))- .compose(this.handleBlockedRequestByContent.bind(this));+ .compose(this.#processHttpResponse.bind(this))+ .compose(this.#handleBlockedRequestByContent.bind(this));}- private async prepareHttpRequest(crawlingContext: CrawlingContext): Promise<Partial<CrawlingContextWithResponse>> {+ async #prepareHttpRequest(crawlingContext: CrawlingContext): Promise<Partial<CrawlingContextWithResponse>> {const { request } = crawlingContext;if (request.skipNavigation) {return {request: new Proxy(request, {- get(target, propertyName, receiver) {+ get(target, propertyName) {if (propertyName === 'loadedUrl') {throw new NavigationSkippedError('The `request.loadedUrl` property is not available - `skipNavigation` was used',);}- return Reflect.get(target, propertyName, receiver);+ // `target` as receiver and bound methods, or `#` members throw on the proxy+ const value = Reflect.get(target, propertyName, target);+ return typeof value === 'function' ? value.bind(target) : value;},+ set: (target, propertyName, value) => Reflect.set(target, propertyName, value, target),}) as LoadedRequest<CrawlingRequest>,get response(): InternalHttpCrawlingContext['response'] {throw new NavigationSkippedError(@@ -531,7 +534,7 @@ export class HttpCrawler<// have already spent part of it), so it produces a clean navigation-timeout error rather than the raw// client abort.const httpResponse = await addTimeoutToPromise(- async () => this.requestFunction({ request, session, proxyUrl }),+ async () => this.#requestFunction({ request, session, proxyUrl }),Math.max(1, remainingNavigationWindowMillis(crawlingContext, this.#navigationTimeoutMillis)),`Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`,);@@ -543,7 +546,7 @@ export class HttpCrawler<return { request: request as LoadedRequest<CrawlingRequest>, response: httpResponse };}- private async processHttpResponse(+ async #processHttpResponse(crawlingContext: CrawlingContextWithResponse,): Promise<Omit<InternalHttpCrawlingContext, keyof CrawlingContextWithResponse> & Partial<InternalHttpCrawlingContext>@@ -600,7 +603,7 @@ export class HttpCrawler<throw new TimeoutError(`Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);}const parsed = await addTimeoutToPromise(- async () => this.parseResponse(crawlingContext.request, crawlingContext.response),+ async () => this.#parseResponse(crawlingContext.request, crawlingContext.response),remaining,`Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`,);@@ -662,7 +665,7 @@ export class HttpCrawler<};}- private async handleBlockedRequestByContent(crawlingContext: InternalHttpCrawlingContext): Promise<{}> {+ async #handleBlockedRequestByContent(crawlingContext: InternalHttpCrawlingContext): Promise<{}> {if (this.retryOnBlocked) {const error = await this.#isRequestBlocked(crawlingContext);if (error) throw new SessionError(error);@@ -693,14 +696,14 @@ export class HttpCrawler<* on the request such as only downloading the request body if the* received content type matches text/html, application/xml, application/xhtml+xml.*/- private async requestFunction({ request, session, proxyUrl }: RequestFunctionOptions): Promise<Response> {- const opts = this.getRequestOptions(request, proxyUrl);+ async #requestFunction({ request, session, proxyUrl }: RequestFunctionOptions): Promise<Response> {+ const opts = this.#getRequestOptions(request, proxyUrl);try {- return await this.requestAsBrowser(opts, session);+ return await this.#requestAsBrowser(opts, session);} catch (e) {if (e instanceof Error && e.constructor.name === 'TimeoutError') {- this.handleRequestTimeout(session);+ this.#handleRequestTimeout(session);return new Response(); // this will never happen, as handleRequestTimeout always throws}@@ -715,7 +718,7 @@ export class HttpCrawler</*** Encodes and parses response according to the provided content type*/- private async parseResponse(request: CrawlingRequest, response: Response) {+ async #parseResponse(request: CrawlingRequest, response: Response) {const { status } = response;const { type, charset } = parseContentTypeFromResponse(response);@@ -724,7 +727,7 @@ export class HttpCrawler<}if (this.isErrorStatusCode(status)) {- const { response: decoded } = this.encodeResponse(request, response, charset);+ const { response: decoded } = this.#encodeResponse(request, response, charset);const body = await decoded.text(); // TODO - this always uses UTF-8 (see https://developer.mozilla.org/en-US/docs/Web/API/Request/text)// Errors are often sent as JSON, so attempt to parse them,@@ -756,7 +759,7 @@ export class HttpCrawler<return { response, contentType: { type, encoding: 'utf-8' as BufferEncoding }, body };}- const { response: reencodedResponse, encoding } = this.encodeResponse(request, response, charset);+ const { response: reencodedResponse, encoding } = this.#encodeResponse(request, response, charset);const contentType = { type, encoding };if (HTML_AND_XML_MIME_TYPES.includes(type)) {@@ -773,7 +776,7 @@ export class HttpCrawler</*** Combines the provided `requestOptions` with mandatory (non-overridable) values.*/- private getRequestOptions(request: CrawlingRequest, proxyUrl?: string) {+ #getRequestOptions(request: CrawlingRequest, proxyUrl?: string) {const requestOptions = {url: request.url,method: request.method,@@ -793,7 +796,7 @@ export class HttpCrawler<return requestOptions;}- private encodeResponse(+ #encodeResponse(request: CrawlingRequest,response: Response,encoding: BufferEncoding,@@ -843,7 +846,7 @@ export class HttpCrawler</*** Checks and extends supported mime types*/- private extendSupportedMimeTypes(additionalMimeTypes: (string | RequestLike | ResponseLike)[]) {+ #extendSupportedMimeTypes(additionalMimeTypes: (string | RequestLike | ResponseLike)[]) {for (const mimeType of additionalMimeTypes) {if (mimeType === '*/*') {this.#supportedMimeTypes.add(mimeType);@@ -862,12 +865,12 @@ export class HttpCrawler</*** Handles timeout request*/- private handleRequestTimeout(session: ISession) {+ #handleRequestTimeout(session: ISession) {session.markBad();throw new Error(`Request timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);}- private abortDownloadOfBody(request: CrawlingRequest, response: Response) {+ #abortDownloadOfBody(request: CrawlingRequest, response: Response) {const { status } = response;const { type } = parseContentTypeFromResponse(response);@@ -885,7 +888,7 @@ export class HttpCrawler</*** @internal wraps public utility for mocking purposes*/- private requestAsBrowser = async (options: Dictionary<any>, session: ISession) => {+ #requestAsBrowser = async (options: Dictionary<any>, session: ISession) => {const opts = processHttpRequestOptions({...(options as any),responseType: 'text',diff --git a/packages/otel/src/instrumentation.ts b/packages/otel/src/instrumentation.tsindex 12d42548b..e928b1697 100644--- a/packages/otel/src/instrumentation.ts+++ b/packages/otel/src/instrumentation.ts@@ -86,21 +86,21 @@ export class CrawleeInstrumentation extends InstrumentationBase<CrawleeInstrumen'@crawlee/core',(moduleExports) => {for (const method of loggerMethods) {- const prototype = this.getPrototype(+ const prototype = this.#getPrototype(moduleExports,'@crawlee/core','BaseCrawleeLogger',method.methodName,);if (prototype) {- this._wrap(prototype, method.methodName, this.getLogPatch(method));+ this._wrap(prototype, method.methodName, this.#getLogPatch(method));}}return moduleExports;},(moduleExports) => {for (const method of loggerMethods) {- this.unwrapIfWrapped(moduleExports?.BaseCrawleeLogger?.prototype, method.methodName);+ this.#unwrapIfWrapped(moduleExports?.BaseCrawleeLogger?.prototype, method.methodName);}return moduleExports;},@@ -110,27 +110,28 @@ export class CrawleeInstrumentation extends InstrumentationBase<CrawleeInstrumenreturn definitions;}+ // oxlint-disable-next-line crawlee/prefer-private-fields -- `InstrumentationBase`'s constructor calls `init()` before `#` members existprivate instantiateModuleDefinitions(moduleDefinitions: ModuleDefinition[]): InstrumentationModuleDefinition[] {return moduleDefinitions.map((definition) => {return crawleeModuleDefinition(definition.moduleName,(moduleExports) => {for (const patch of definition.classMethodPatches) {- const prototype = this.getPrototype(+ const prototype = this.#getPrototype(moduleExports,definition.moduleName,patch.className,patch.methodName,);if (prototype) {- this._wrap(prototype, patch.methodName, this.applyClassMethodPatch(patch));+ this._wrap(prototype, patch.methodName, this.#applyClassMethodPatch(patch));}}return moduleExports;},(moduleExports) => {for (const patch of definition.classMethodPatches) {- this.unwrapIfWrapped(moduleExports?.[patch.className]?.prototype, patch.methodName);+ this.#unwrapIfWrapped(moduleExports?.[patch.className]?.prototype, patch.methodName);}return moduleExports;},@@ -142,7 +143,7 @@ export class CrawleeInstrumentation extends InstrumentationBase<CrawleeInstrumen* Resolves the prototype holding the method to patch, warning instead of throwing when the class or the method* is not there - a missing internal method must not break loading of the instrumented module.*/- private getPrototype(moduleExports: any, moduleName: string, className: string, methodName: string) {+ #getPrototype(moduleExports: any, moduleName: string, className: string, methodName: string) {const prototype = moduleExports?.[className]?.prototype;if (typeof prototype?.[methodName] !== 'function') {@@ -156,14 +157,14 @@ export class CrawleeInstrumentation extends InstrumentationBase<CrawleeInstrumenreturn prototype;}- /** Mirrors {@link getPrototype}: only methods that were actually patched are restored. */- private unwrapIfWrapped(prototype: any, methodName: string) {+ /** Mirrors `#getPrototype`: only methods that were actually patched are restored. */+ #unwrapIfWrapped(prototype: any, methodName: string) {if (prototype && isWrapped(prototype[methodName])) {this._unwrap(prototype, methodName);}}- private applyClassMethodPatch(patch: ClassMethodPatchDefinition): (original: any) => any {+ #applyClassMethodPatch(patch: ClassMethodPatchDefinition): (original: any) => any {const { spanName, spanOptions } = patch;const qualifiedName = `${patch.className}.${patch.methodName}`;const codeAttributes = { [ATTR_CODE_FUNCTION_NAME]: qualifiedName };@@ -183,7 +184,7 @@ export class CrawleeInstrumentation extends InstrumentationBase<CrawleeInstrumen};}- private getLogPatch(method: LoggerMethodDefinition) {+ #getLogPatch(method: LoggerMethodDefinition) {// oxlint-disable-next-line no-this-aliasconst instrumentation = this;@@ -195,7 +196,7 @@ export class CrawleeInstrumentation extends InstrumentationBase<CrawleeInstrumentry {original.apply(this, args);} finally {- instrumentation.forwardLogRecord(method, args);+ instrumentation.#forwardLogRecord(method, args);}};};@@ -208,7 +209,7 @@ export class CrawleeInstrumentation extends InstrumentationBase<CrawleeInstrumen* the application's own call to `log.info()` and friends, so a failure anywhere in it is reported and dropped* rather than raised.*/- private forwardLogRecord(method: LoggerMethodDefinition, args: unknown[]): void {+ #forwardLogRecord(method: LoggerMethodDefinition, args: unknown[]): void {try {const { message, data } = method.read(args);diff --git a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.tsindex 04f205436..ea15be6a4 100644--- a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts+++ b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts@@ -495,11 +495,11 @@ export class AdaptivePlaywrightCrawler<this.#browserCrawler = browserCrawler;this.#staticContextPipeline = staticCrawler.contextPipeline.compose(- this.adaptCheerioContext.bind(this),+ this.#adaptCheerioContext.bind(this),) as unknown as ContextPipeline<CrawlingContext, ExtendedContext>;this.#browserContextPipeline = browserCrawler.contextPipeline.compose(- this.adaptPlaywrightContext.bind(this),+ this.#adaptPlaywrightContext.bind(this),) as unknown as ContextPipeline<CrawlingContext, ExtendedContext>;}@@ -544,7 +544,7 @@ export class AdaptivePlaywrightCrawler<}));}- private async adaptCheerioContext(cheerioContext: CheerioCrawlingContext) {+ async #adaptCheerioContext(cheerioContext: CheerioCrawlingContext) {return {get page(): Page {throw new Error('Page object was used in HTTP-only request handler');@@ -561,13 +561,13 @@ export class AdaptivePlaywrightCrawler<options.selector,options.baseUrl ?? cheerioContext.request.loadedUrl,);- return (await this.enqueueLinks(urls, options, cheerioContext.request)) as unknown as void;+ return (await this.#enqueueLinks(urls, options, cheerioContext.request)) as unknown as void;},response: cheerioContext.response,};}- private async adaptPlaywrightContext(playwrightContext: PlaywrightCrawlingContext) {+ async #adaptPlaywrightContext(playwrightContext: PlaywrightCrawlingContext) {// Capture the original response to avoid infinite recursion when the getter is copied to the contextconst originalResponse = playwrightContext.response;@@ -602,7 +602,7 @@ export class AdaptivePlaywrightCrawler<options.baseUrl ?? playwrightContext.request.loadedUrl,);- return (await this.enqueueLinks(urls, options, playwrightContext.request)) as unknown as void;+ return (await this.#enqueueLinks(urls, options, playwrightContext.request)) as unknown as void;},};}@@ -613,7 +613,7 @@ export class AdaptivePlaywrightCrawler<* time, before the `try`* - the `ok: false` branch of the returned {@apilink Result} carries no* result, and failed attempts are routine here. The caller owns the outcome and disposal.*/- private async crawlOne(+ async #crawlOne(renderingType: RenderingType,context: CrawlingContext,useStateFunction: (defaultValue?: Dictionary) => Promise<Dictionary>,@@ -631,7 +631,7 @@ export class AdaptivePlaywrightCrawler<const attemptBoundContextHelpers = {useState: useStateFunction,- log: this.createLogProxy(context.log, logs),+ log: this.#createLogProxy(context.log, logs),registerDeferredCleanup: (cleanup: () => Promise<unknown>) => deferredCleanup.push(cleanup),};@@ -702,7 +702,7 @@ export class AdaptivePlaywrightCrawler<crawlingContext.log.debug(`Running HTTP-only request handler for ${crawlingContext.request.url}`);this.statistics.state.httpOnlyRequestHandlerRuns++;- const plainHTTPRun = await this.crawlOne(+ const plainHTTPRun = await this.#crawlOne('static',crawlingContext,crawlingContext.useState,@@ -766,7 +766,7 @@ export class AdaptivePlaywrightCrawler<},};- const browserRun = await this.crawlOne(+ const browserRun = await this.#crawlOne('clientOnly',crawlingContext,stateTracker.getLiveState.bind(stateTracker),@@ -785,7 +785,7 @@ export class AdaptivePlaywrightCrawler<crawlingContext.log.debug(`Detecting rendering type for ${crawlingContext.request.url}`);// The detection attempt's transaction is never committed - its writes exist only for the// result comparison.- const plainHTTPRun = await this.crawlOne(+ const plainHTTPRun = await this.#crawlOne('static',crawlingContext,stateTracker.getStateCopy.bind(stateTracker),@@ -846,7 +846,7 @@ export class AdaptivePlaywrightCrawler<}}- private async enqueueLinks(+ async #enqueueLinks(urls: readonly string[],options: EnqueueLinksOptions,request: RestrictedCrawlingContext['request'],@@ -869,7 +869,7 @@ export class AdaptivePlaywrightCrawler<});}- private createLogProxy(log: CrawleeLogger, logs: LogProxyCall[]) {+ #createLogProxy(log: CrawleeLogger, logs: LogProxyCall[]) {return new Proxy(log, {get(target: CrawleeLogger, propertyName: (typeof proxyLogMethods)[number]) {if (proxyLogMethods.includes(propertyName)) {diff --git a/packages/playwright-crawler/src/internals/playwright-crawler.ts b/packages/playwright-crawler/src/internals/playwright-crawler.tsindex 408568ffd..28e5c23eb 100644--- a/packages/playwright-crawler/src/internals/playwright-crawler.ts+++ b/packages/playwright-crawler/src/internals/playwright-crawler.ts@@ -258,7 +258,7 @@ export class PlaywrightCrawler<}#buildContextPipeline(): ContextPipeline<CrawlingContext, PlaywrightCrawlingContext> {- return this.buildContextPipeline().compose(this.enhanceContext.bind(this));+ return this.buildContextPipeline().compose(this.#enhanceContext.bind(this));}protected override async navigationHandler(@@ -268,7 +268,7 @@ export class PlaywrightCrawler<return gotoExtended(crawlingContext.page, crawlingContext.request, gotoOptions);}- private async enhanceContext(context: BrowserCrawlingContext<Page, Response, Dictionary>) {+ async #enhanceContext(context: BrowserCrawlingContext<Page, Response, Dictionary>) {const waitForSelector = async (selector: string, timeoutMs = 5_000) => {const locator = context.page.locator(selector).first();await locator.waitFor({ timeout: timeoutMs, state: 'attached' });diff --git a/packages/playwright-crawler/src/internals/utils/rendering-type-prediction.ts b/packages/playwright-crawler/src/internals/utils/rendering-type-prediction.tsindex 437b471ca..29030539c 100644--- a/packages/playwright-crawler/src/internals/utils/rendering-type-prediction.ts+++ b/packages/playwright-crawler/src/internals/utils/rendering-type-prediction.ts@@ -175,7 +175,7 @@ export class RenderingTypePredictor implements IRenderingTypePredictor {const predictionUrl = new URL(loadedUrl ?? url);- const urlFeature = new Matrix([this.calculateFeatureVector(urlComponents(predictionUrl), label)]);+ const urlFeature = new Matrix([this.#calculateFeatureVector(urlComponents(predictionUrl), label)]);const [prediction] = logreg.predict(urlFeature);const scores = [logreg.classifiers[0].testScores(urlFeature), logreg.classifiers[1].testScores(urlFeature)];@@ -184,7 +184,7 @@ export class RenderingTypePredictor implements IRenderingTypePredictor {detectionProbabilityRecommendation:Math.abs(scores[0] - scores[1]) < 0.1? 1- : this.#detectionRatio * Math.max(1, 5 - this.resultCount(label)),+ : this.#detectionRatio * Math.max(1, 5 - this.#resultCount(label)),};}@@ -208,16 +208,16 @@ export class RenderingTypePredictor implements IRenderingTypePredictor {state.detectionResults.get(renderingType)!.get(label)!.push(urlComponents(resultUrl));}- this.retrain();+ this.#retrain();}- private resultCount(label: string | undefined): number {+ #resultCount(label: string | undefined): number {return Array.from(this.#state.currentValue.detectionResults.values()).map((results) => results.get(label)?.length ?? 0).reduce((acc, value) => acc + value, 0);}- private calculateFeatureVector(url: URLComponents, label: string | undefined): FeatureVector {+ #calculateFeatureVector(url: URLComponents, label: string | undefined): FeatureVector {return [mean((this.#state.currentValue.detectionResults.get('static')?.get(label) ?? []).map(@@ -232,7 +232,7 @@ export class RenderingTypePredictor implements IRenderingTypePredictor {];}- private retrain(): void {+ #retrain(): void {const X: FeatureVector[] = [[0, 1],[1, 0],@@ -242,7 +242,7 @@ export class RenderingTypePredictor implements IRenderingTypePredictor {for (const [renderingType, urlsByLabel] of this.#state.currentValue.detectionResults.entries()) {for (const [label, urls] of urlsByLabel) {for (const url of urls) {- X.push(this.calculateFeatureVector(url, label));+ X.push(this.#calculateFeatureVector(url, label));Y.push(renderingType === 'static' ? 1 : 0);}}diff --git a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.tsindex e533c7214..41e5c648c 100644--- a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts+++ b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts@@ -255,10 +255,10 @@ export class PuppeteerCrawler<}#buildContextPipeline(): ContextPipeline<CrawlingContext, PuppeteerCrawlingContext> {- return this.buildContextPipeline().compose(this.enhanceContext.bind(this));+ return this.buildContextPipeline().compose(this.#enhanceContext.bind(this));}- private async enhanceContext(context: BrowserCrawlingContext<Page, HTTPResponse>) {+ async #enhanceContext(context: BrowserCrawlingContext<Page, HTTPResponse>) {const waitForSelector = async (selector: string, timeoutMs = 5_000) => {await context.page.waitForSelector(selector, { timeout: timeoutMs });};diff --git a/packages/stagehand-crawler/src/internals/stagehand-controller.ts b/packages/stagehand-crawler/src/internals/stagehand-controller.tsindex 21c7e7eb7..102ee8a18 100644--- a/packages/stagehand-crawler/src/internals/stagehand-controller.ts+++ b/packages/stagehand-crawler/src/internals/stagehand-controller.ts@@ -63,7 +63,7 @@ export class StagehandController extends BrowserController<BrowserType, LaunchOp});try {- await this.waitForStagehandToRegisterPage(page);+ await this.#waitForStagehandToRegisterPage(page);} catch (error) {await page.close().catch(() => {});throw error;@@ -86,9 +86,9 @@ export class StagehandController extends BrowserController<BrowserType, LaunchOp* a short window, and the AI methods fail with 'Failed to resolve V3 Page from Playwright page'.* Stagehand's own `newPage()` polls for the same reason.*/- private async waitForStagehandToRegisterPage(page: Page, timeoutMs = 10_000): Promise<void> {+ async #waitForStagehandToRegisterPage(page: Page, timeoutMs = 10_000): Promise<void> {const stagehand = this.getStagehand();- const mainFrameId = await this.getMainFrameId(page);+ const mainFrameId = await this.#getMainFrameId(page);const deadline = Date.now() + timeoutMs;while (Date.now() < deadline) {@@ -107,7 +107,7 @@ export class StagehandController extends BrowserController<BrowserType, LaunchOp/*** Reads the page's main frame id, which is the key Stagehand resolves pages by.*/- private async getMainFrameId(page: Page): Promise<string> {+ async #getMainFrameId(page: Page): Promise<string> {const cdpSession = await page.context().newCDPSession(page);try {diff --git a/packages/stagehand-crawler/src/internals/stagehand-crawler.ts b/packages/stagehand-crawler/src/internals/stagehand-crawler.tsindex 026b434b5..6c29fae76 100644--- a/packages/stagehand-crawler/src/internals/stagehand-crawler.ts+++ b/packages/stagehand-crawler/src/internals/stagehand-crawler.ts@@ -469,7 +469,7 @@ export class StagehandCrawler<}#buildContextPipeline(): ContextPipeline<CrawlingContext, StagehandCrawlingContext> {- return this.buildContextPipeline().compose(this.setUpStagehand.bind(this));+ return this.buildContextPipeline().compose(this.#setUpStagehand.bind(this));}/**@@ -481,7 +481,7 @@ export class StagehandCrawler<* instance bound to the page's browser, which is why it reaches past the* {@apilink IBrowserPool} abstraction here.*/- private getBrowserControllerByPage(page: StagehandPage): StagehandController | undefined {+ #getBrowserControllerByPage(page: StagehandPage): StagehandController | undefined {if ('getBrowserControllerByPage' in this.browserPool) {return (this.browserPool as unknown as {@@ -496,10 +496,8 @@ export class StagehandCrawler</*** Enhance the page with Stagehand AI methods.*/- private async setUpStagehand(crawlingContext: {- page: Page;- }): Promise<{ stagehand: Stagehand; page: StagehandPage }> {- const controller = this.getBrowserControllerByPage(crawlingContext.page as StagehandPage);+ async #setUpStagehand(crawlingContext: { page: Page }): Promise<{ stagehand: Stagehand; page: StagehandPage }> {+ const controller = this.#getBrowserControllerByPage(crawlingContext.page as StagehandPage);if (!controller) {throw new Error(diff --git a/packages/stagehand-crawler/src/internals/stagehand-plugin.ts b/packages/stagehand-crawler/src/internals/stagehand-plugin.tsindex 23e0b0e24..bbe2778d6 100644--- a/packages/stagehand-crawler/src/internals/stagehand-plugin.ts+++ b/packages/stagehand-crawler/src/internals/stagehand-plugin.ts@@ -108,7 +108,7 @@ export class StagehandPlugin extends BrowserPlugin<BrowserType, LaunchOptions, P// Handle browser disconnection - cleanup both Stagehand and anonymized proxybrowser.on('disconnected', async () => {- await this.cleanupStagehand(browser);+ await this.#cleanupStagehand(browser);await closeAnonymizedProxy();});@@ -118,7 +118,7 @@ export class StagehandPlugin extends BrowserPlugin<BrowserType, LaunchOptions, Pawait stagehand.close().catch(() => {});await closeAnonymizedProxy();- const augmentedError = this.augmentLaunchError(error, launchContext);+ const augmentedError = this.#augmentLaunchError(error, launchContext);serviceLocator.getLogger().error('Stagehand browser launch failed', { message: augmentedError.message });throw augmentedError;}@@ -127,7 +127,7 @@ export class StagehandPlugin extends BrowserPlugin<BrowserType, LaunchOptions, P/*** Cleans up Stagehand instance when browser disconnects.*/- private async cleanupStagehand(browser: PlaywrightBrowser): Promise<void> {+ async #cleanupStagehand(browser: PlaywrightBrowser): Promise<void> {const stagehand = this.#stagehandInstances.get(browser);if (stagehand) {try {@@ -176,7 +176,7 @@ export class StagehandPlugin extends BrowserPlugin<BrowserType, LaunchOptions, P/*** Augments launch errors with helpful context.*/- private augmentLaunchError(error: unknown, launchContext: LaunchContext<BrowserType>): Error {+ #augmentLaunchError(error: unknown, launchContext: LaunchContext<BrowserType>): Error {const message = error instanceof Error ? error.message : String(error);const model = this.#stagehandOptions.model;diff --git a/packages/utils/src/internals/robots.ts b/packages/utils/src/internals/robots.tsindex 81f41ee30..2550a9b97 100644--- a/packages/utils/src/internals/robots.ts+++ b/packages/utils/src/internals/robots.ts@@ -74,7 +74,7 @@ export class RobotsTxtFile {robotsTxtFileUrl.pathname = '/robots.txt';robotsTxtFileUrl.search = '';- return RobotsTxtFile.load(robotsTxtFileUrl.toString(), options);+ return RobotsTxtFile.#load(robotsTxtFileUrl.toString(), options);}/**@@ -88,7 +88,7 @@ export class RobotsTxtFile {return new RobotsTxtFile(url, robotsParser(url, content), proxyUrl);}- private static async load(+ static async #load(url: string,options?: {signal?: AbortSignal;diff --git a/packages/utils/src/internals/sitemap.ts b/packages/utils/src/internals/sitemap.tsindex 6117c657d..29aa5bce2 100644--- a/packages/utils/src/internals/sitemap.ts+++ b/packages/utils/src/internals/sitemap.ts@@ -42,17 +42,17 @@ class SitemapTxtParser extends Transform {super({readableObjectMode: true,transform: (chunk, _encoding, callback) => {- this.processBuffer(this.#decoder.write(chunk), false);+ this.#processBuffer(this.#decoder.write(chunk), false);callback();},flush: (callback) => {- this.processBuffer(this.#decoder.end(), true);+ this.#processBuffer(this.#decoder.end(), true);callback();},});}- private processBuffer(input: string, finalize: boolean): void {+ #processBuffer(input: string, finalize: boolean): void {this.#buffer += input;if (finalize || this.#buffer.includes('\n')) {@@ -110,16 +110,16 @@ class SitemapXmlParser extends Transform {});this.#parser = parser;- this.#parser.onopentag = this.onOpenTag.bind(this);- this.#parser.onclosetag = this.onCloseTag.bind(this);+ this.#parser.onopentag = this.#onOpenTag.bind(this);+ this.#parser.onclosetag = this.#onCloseTag.bind(this);- this.#parser.ontext = this.onText.bind(this);- this.#parser.oncdata = this.onText.bind(this);+ this.#parser.ontext = this.#onText.bind(this);+ this.#parser.oncdata = this.#onText.bind(this);this.#parser.onerror = this.destroy.bind(this);}- private onOpenTag(node: sax.Tag | sax.QualifiedTag) {+ #onOpenTag(node: sax.Tag | sax.QualifiedTag) {if (this.#rootTagName !== undefined) {if (node.name === 'loc' ||@@ -138,7 +138,7 @@ class SitemapXmlParser extends Transform {}}- private onCloseTag(name: string) {+ #onCloseTag(name: string) {if (name === 'loc' || name === 'lastmod' || name === 'priority' || name === 'changefreq') {this.#currentTag = undefined;}@@ -151,7 +151,7 @@ class SitemapXmlParser extends Transform {}}- private onText(text: string) {+ #onText(text: string) {if (this.#currentTag === 'loc') {if (this.#rootTagName === 'sitemapindex') {this.push({ type: 'sitemapUrl', url: text.trim() } satisfies SitemapItem);@@ -487,7 +487,7 @@ export class Sitemap {proxyUrl?: string,parseSitemapOptions?: ParseSitemapOptions,): Promise<Sitemap> {- return await this.parse(+ return await this.#parse((Array.isArray(urls) ? urls : [urls]).map((url) => ({ type: 'url', url })),proxyUrl,parseSitemapOptions,@@ -504,10 +504,10 @@ export class Sitemap {proxyUrl?: string,parseSitemapOptions?: ParseSitemapOptions,): Promise<Sitemap> {- return await this.parse([{ type: 'raw', content }], proxyUrl, parseSitemapOptions);+ return await this.#parse([{ type: 'raw', content }], proxyUrl, parseSitemapOptions);}- private static async parse(+ static async #parse(sources: SitemapSource[],proxyUrl?: string,parseSitemapOptions?: ParseSitemapOptions,diff --git a/test/core/crawlers/file_download.test.ts b/test/core/crawlers/file_download.test.tsindex 06751a996..e3806065f 100644--- a/test/core/crawlers/file_download.test.ts+++ b/test/core/crawlers/file_download.test.ts@@ -12,7 +12,7 @@ import { startExpressAppPromise } from '../../shared/_helper.js';import { afterAll, beforeAll, expect, test } from 'vitest';class ReadableStreamGenerator {- private static async generateRandomData(size: number, seed: number): Promise<Uint8Array> {+ static async #generateRandomData(size: number, seed: number): Promise<Uint8Array> {const chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789';const array = new Uint8Array(size);for (let i = 0; i < size; i++) {@@ -29,7 +29,7 @@ class ReadableStreamGenerator {start: async (controller) => {while (bytesRead < size) {const chunkSize = Math.min(size - bytesRead, 1024);- const chunk = await this.generateRandomData(chunkSize, seed);+ const chunk = await this.#generateRandomData(chunkSize, seed);bytesRead += chunk.length;controller.enqueue(chunk);