Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
refactor!: use native private class fields and remove underscore prefixes (#3980)
Martin Adámek4d1407208bf3
diff --git a/docs/public-api/crawlee-basic.api.md b/docs/public-api/crawlee-basic.api.mdindex c83e150c1..80fbacb56 100644--- a/docs/public-api/crawlee-basic.api.md+++ b/docs/public-api/crawlee-basic.api.md@@ -82,10 +82,10 @@ export class BasicCrawler<Context extends CrawlingContext = CrawlingContext, Con// (undocumented)protected readonly failedRequestHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;// (undocumented)- protected _getCookieHeaderFromRequest(request: Request_2): string;+ protected getCookieHeaderFromRequest(request: Request_2): string;getData(...args: Parameters<Dataset['getData']>): ReturnType<Dataset['getData']>;getDataset(identifier?: string | StorageIdentifier): Promise<Dataset>;- protected _getMessageFromError(error: Error, forceStack?: boolean): string | TimeoutError | undefined;+ protected getMessageFromError(error: Error, forceStack?: boolean): string | TimeoutError | undefined;protected getNavigationTimeoutMillis(): number;// (undocumented)protected getPendingRequestCountApproximation(): Promise<number>;@@ -102,7 +102,7 @@ export class BasicCrawler<Context extends CrawlingContext = CrawlingContext, Conprotected readonly httpClient: BaseHttpClient;// (undocumented)protected readonly identity: CrawlerIdentity;- protected _init(): Promise<void>;+ protected init(): Promise<void>;// (undocumented)protected readonly internalTimeoutMillis: number;protected isErrorStatusCode(status: number): boolean;@@ -176,7 +176,7 @@ export class BasicCrawler<Context extends CrawlingContext = CrawlingContext, Conget stats(): IStatistics;stop(reason?: string): void;teardown(): Promise<void>;- protected _throwOnBlockedRequest(statusCode: number): void;+ protected throwOnBlockedRequest(statusCode: number): void;// (undocumented)useState<State extends Dictionary = Dictionary>(defaultValue?: State): Promise<State>;}diff --git a/docs/public-api/crawlee-browser-pool.api.md b/docs/public-api/crawlee-browser-pool.api.mdindex 9907685ac..bfd59d56e 100644--- a/docs/public-api/crawlee-browser-pool.api.md+++ b/docs/public-api/crawlee-browser-pool.api.md@@ -126,17 +126,17 @@ interface BrowserOptions {export abstract class BrowserPlugin<Library extends CommonLibrary = CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> {constructor(library: Library, options?: BrowserPluginOptions<LibraryOptions>);// (undocumented)- protected abstract _addProxyToLaunchOptions(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): Promise<void>;+ protected abstract addProxyToLaunchOptions(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): Promise<void>;// (undocumented)readonly browserPerProxy?: boolean;- protected _connectToRemoteBrowser(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>, connect: (url: string) => Promise<LaunchResult>): Promise<LaunchResult>;+ protected connectToRemoteBrowser(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>, connect: (url: string) => Promise<LaunchResult>): Promise<LaunchResult>;// (undocumented)abstract createController(): BrowserController<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;createLaunchContext(options?: CreateLaunchContextOptions<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;// (undocumented)readonly ignoreProxyCertificate?: boolean;// (undocumented)- protected abstract _isChromiumBasedBrowser(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): boolean;+ protected abstract isChromiumBasedBrowser(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): boolean;launch(launchContext?: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): Promise<LaunchResult>;// (undocumented)protected abstract _launch(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): Promise<LaunchResult>;@@ -151,7 +151,7 @@ export abstract class BrowserPlugin<Library extends CommonLibrary = CommonLibrar// (undocumented)readonly proxyUrl?: string;// (undocumented)- protected _throwAugmentedLaunchError(cause: unknown, executablePath: string | undefined, dockerImage: string, moduleInstallCommand: string): never;+ protected throwAugmentedLaunchError(cause: unknown, executablePath: string | undefined, dockerImage: string, moduleInstallCommand: string): never;// (undocumented)useIncognitoPages: boolean;// (undocumented)@@ -495,11 +495,11 @@ export class PlaywrightController extends BrowserController<BrowserType, SafePar// @public (undocumented)export class PlaywrightPlugin extends BrowserPlugin<BrowserType, SafeParameters<BrowserType['launch']>[0], Browser> {// (undocumented)- protected _addProxyToLaunchOptions(launchContext: LaunchContext<BrowserType>): Promise<void>;+ protected addProxyToLaunchOptions(launchContext: LaunchContext<BrowserType>): Promise<void>;// (undocumented)createController(): PlaywrightController;// (undocumented)- protected _isChromiumBasedBrowser(): boolean;+ protected isChromiumBasedBrowser(): boolean;// (undocumented)protected _launch(launchContext: LaunchContext<BrowserType>): Promise<Browser>;useRemoteConnection(connection: RemoteConnection, parameters?: RemoteConnectionParameters): void;@@ -551,11 +551,11 @@ interface PuppeteerNewPageOptions extends PuppeteerTypes.BrowserContextOptions {// @public (undocumented)export class PuppeteerPlugin extends BrowserPlugin<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions> {// (undocumented)- protected _addProxyToLaunchOptions(_launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): Promise<void>;+ protected addProxyToLaunchOptions(_launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): Promise<void>;// (undocumented)createController(): PuppeteerController;// (undocumented)- protected _isChromiumBasedBrowser(_launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): boolean;+ protected isChromiumBasedBrowser(_launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): boolean;// (undocumented)protected _launch(launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): Promise<PuppeteerTypes.Browser>;useRemoteConnection(connection: RemoteConnection, parameters?: RemoteConnectionParameters): void;diff --git a/docs/public-api/crawlee-browser.api.md b/docs/public-api/crawlee-browser.api.mdindex 5fd79f7f0..a20e2fafe 100644--- a/docs/public-api/crawlee-browser.api.md+++ b/docs/public-api/crawlee-browser.api.md@@ -62,7 +62,7 @@ export abstract class BrowserCrawler<Page extends CommonPage = CommonPage, Respo// (undocumented)launchContext: BrowserLaunchContext<LaunchOptions, unknown>;// (undocumented)- protected abstract _navigationHandler(crawlingContext: BrowserCrawlingContext<Page, Response>, gotoOptions: GoToOptions): Promise<Context['response'] | null | undefined>;+ protected abstract navigationHandler(crawlingContext: BrowserCrawlingContext<Page, Response>, gotoOptions: GoToOptions): Promise<Context['response'] | null | undefined>;// (undocumented)protected static optionsShape: {navigationTimeoutSecs: NumberPredicate & BasePredicate<number | undefined>;diff --git a/docs/public-api/crawlee-playwright.api.md b/docs/public-api/crawlee-playwright.api.mdindex f267e0481..177b5fcb1 100644--- a/docs/public-api/crawlee-playwright.api.md+++ b/docs/public-api/crawlee-playwright.api.md@@ -90,7 +90,7 @@ export class AdaptivePlaywrightCrawler<ContextExtension = Dictionary<never>, Extreadonly parseWithCheerio: AdaptivePlaywrightCrawlerContext["parseWithCheerio"];}>;// (undocumented)- protected _init(): Promise<void>;+ protected init(): Promise<void>;// (undocumented)protected runRequestHandler(crawlingContext: CrawlingContext_2): Promise<void>;// (undocumented)@@ -341,7 +341,7 @@ export class PlaywrightCrawler<ContextExtension = Dictionary<never>, ExtendedConhandleCloudflareChallenge: (options?: HandleCloudflareChallengeOptions) => Promise<Response_2 | undefined>;}>;// (undocumented)- protected _navigationHandler(crawlingContext: PlaywrightCrawlingContext, gotoOptions: PlaywrightDirectNavigationOptions): Promise<Response_2 | null>;+ protected navigationHandler(crawlingContext: PlaywrightCrawlingContext, gotoOptions: PlaywrightDirectNavigationOptions): Promise<Response_2 | null>;// (undocumented)protected static optionsShape: {browserPoolOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;diff --git a/docs/public-api/crawlee-puppeteer.api.md b/docs/public-api/crawlee-puppeteer.api.mdindex 526617c50..ec45e4f86 100644--- a/docs/public-api/crawlee-puppeteer.api.md+++ b/docs/public-api/crawlee-puppeteer.api.md@@ -198,7 +198,7 @@ export class PuppeteerCrawler<ContextExtension = Dictionary<never>, ExtendedContcloseCookieModals: () => Promise<void>;}>;// (undocumented)- protected _navigationHandler(crawlingContext: PuppeteerCrawlingContext, gotoOptions: PuppeteerDirectNavigationOptions): Promise<HTTPResponse | null>;+ protected navigationHandler(crawlingContext: PuppeteerCrawlingContext, gotoOptions: PuppeteerDirectNavigationOptions): Promise<HTTPResponse | null>;// (undocumented)protected static optionsShape: {browserPoolOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;diff --git a/docs/public-api/crawlee-stagehand.api.md b/docs/public-api/crawlee-stagehand.api.mdindex 887d34d18..84b28934d 100644--- a/docs/public-api/crawlee-stagehand.api.md+++ b/docs/public-api/crawlee-stagehand.api.md@@ -85,7 +85,7 @@ export class StagehandCrawler<ContextExtension = Dictionary<never>, ExtendedContconstructor(options?: StagehandCrawlerOptions<ContextExtension, ExtendedContext, Routes>);// (undocumented)protected buildContextPipeline(): ContextPipeline<CrawlingContext, StagehandCrawlingContext>;- protected _navigationHandler(crawlingContext: StagehandCrawlingContext, gotoOptions: StagehandGotoOptions): Promise<Response_2 | null>;+ protected navigationHandler(crawlingContext: StagehandCrawlingContext, gotoOptions: StagehandGotoOptions): Promise<Response_2 | null>;// (undocumented)protected static optionsShape: {stagehandOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;@@ -204,10 +204,10 @@ export interface StagehandPage extends Page {// @publicclass StagehandPlugin extends BrowserPlugin<BrowserType, LaunchOptions, Browser> {constructor(library: BrowserType, options?: StagehandPluginOptions);- protected _addProxyToLaunchOptions(launchContext: LaunchContext<BrowserType>): Promise<void>;+ protected addProxyToLaunchOptions(launchContext: LaunchContext<BrowserType>): Promise<void>;createController(): BrowserController<BrowserType, LaunchOptions, Browser>;getStagehandForBrowser(browser: Browser): Stagehand | undefined;- protected _isChromiumBasedBrowser(): boolean;+ protected isChromiumBasedBrowser(): boolean;protected _launch(launchContext: LaunchContext<BrowserType>): Promise<Browser>;// (undocumented)readonly stagehandOptions: StagehandOptions;diff --git a/docs/upgrading/upgrading_v4.md b/docs/upgrading/upgrading_v4.mdindex 98bf8d962..a537dd5d4 100644--- a/docs/upgrading/upgrading_v4.md+++ b/docs/upgrading/upgrading_v4.md@@ -650,8 +650,32 @@ The leading underscore was dropped from protected and private class members acroThe file-system storage backends' shared `CachedIdClient._cachedId` protected field was also renamed to `cachedId` (this only affects custom `@crawlee/fs-storage` backends that subclass it).+If you subclass a crawler or implement a custom browser plugin, these `protected` extension points lost their underscore too:++- `BasicCrawler._init` -> `init`+- `BasicCrawler._throwOnBlockedRequest` -> `throwOnBlockedRequest`+- `BasicCrawler._getMessageFromError` -> `getMessageFromError`+- `BasicCrawler._getCookieHeaderFromRequest` -> `getCookieHeaderFromRequest`+- `BrowserCrawler._navigationHandler` -> `navigationHandler` (including the `PlaywrightCrawler`, `PuppeteerCrawler` and `StagehandCrawler` overrides)+- `BrowserPlugin._addProxyToLaunchOptions` -> `addProxyToLaunchOptions`+- `BrowserPlugin._isChromiumBasedBrowser` -> `isChromiumBasedBrowser`+- `BrowserPlugin._connectToRemoteBrowser` -> `connectToRemoteBrowser`+- `BrowserPlugin._throwAugmentedLaunchError` -> `throwAugmentedLaunchError`++Two `@internal` members were renamed the same way: `LaunchContext._remoteToken` -> `remoteToken` and `PlaywrightBrowser._setBrowserType` -> `setBrowserType`.++A handful of hooks intentionally keep the underscore, because the un-prefixed name is taken by their public wrapper method: `BrowserPlugin._launch` (wrapped by `launch()`) and `BrowserController._close`, `_kill`, `_newPage`, `_getCookies`, `_setCookies` (wrapped by the same names without the underscore). Custom plugin or controller implementations override these under their existing names, unchanged.+Members that were also made `private` in the same pass are listed under [Unintentionally exposed internals are now private](#unintentionally-exposed-internals-are-now-private) below.+### Private properties are native `#` fields now++Private class properties across the codebase were converted from TypeScript's compile-time `private` to native ECMAScript private fields (`#name`). Where a `private _foo` field backed a `get foo()` accessor, the field is now `#foo` — the public accessor is unchanged.++TypeScript's `private` was purely a compile-time construct: the properties still existed on the instances at runtime, so code could reach them via `(crawler as any).something` or `crawler['something']`, and they showed up in `Object.keys()`, object spread and `JSON.stringify()`. Native `#` fields close that hole — they are inaccessible outside the declaring class and invisible to enumeration and serialization. If you were reaching into any of them, that now fails at runtime, not just in the type checker. As with the visibility tightening above, the supported extension points (handlers, hooks, `ContextPipeline` composition and the `ISessionPool` / `IBrowserPool` / `IRequestManager` interfaces) are the way to go; open an issue if something you need is missing.++One related behavior change: `LaunchContext.extend()` now consistently rejects all declared fields as reserved keys — including `fingerprint`, `proxyUrl` and `remoteToken`, which previously slipped through the reserved-name check. Set those directly instead (e.g. `launchContext.fingerprint = ...`); `extend()` is only for attaching your own extra fields.+### Unintentionally exposed internals are now privateA number of class members were `public` or `protected` only by accident — they were never meant to be part of the extension surface, are not used by any subclass, and in most cases also carried a leading underscore to signal that. In v4 they are `private` (and where it applies, `readonly`). If you were reaching into any of these — either to read internal state or to override a helper in a subclass — that no longer compiles.diff --git a/oxlint.config.ts b/oxlint.config.tsindex d36301988..75ec13ccd 100644--- a/oxlint.config.ts+++ b/oxlint.config.ts@@ -32,6 +32,30 @@ export default defineConfig({'no-throw-literal': 'error',eqeqeq: ['error', 'smart'],yoda: 'error',+ // Enforces the naming convention from #3108: no `_`-prefixed members; private+ // properties use native `#` fields instead. The allow list covers the template-method+ // hooks that collide with their public wrappers, platform contracts (`Readable._read`),+ // and documented internals (`__crawlee`, `_currentConcurrency`).+ 'no-underscore-dangle': [+ 'error',+ {+ enforceInClassFields: true,+ enforceInMethodNames: true,+ allow: [+ '_launch',+ '_close',+ '_kill',+ '_newPage',+ '_getCookies',+ '_setCookies',+ '_read',+ '_currentConcurrency',+ '__crawlee',+ '__purged',+ '__originalHistory__',+ ],+ },+ ],'typescript/consistent-type-imports': ['error', { disallowTypeAnnotations: false }],'typescript/consistent-type-definitions': ['error', 'interface'],@@ -97,6 +121,7 @@ export default defineConfig({{files: ['test/**/*', 'packages/*/test/**/*'],rules: {+ 'no-underscore-dangle': 'off','typescript/consistent-type-imports': 'off','typescript/no-floating-promises': 'off','typescript/only-throw-error': 'off',diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex 266769b17..6e7dece43 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -107,10 +107,10 @@ import {import { createSendRequest } from './send-request.js';class LazyDefaultHttpClient implements BaseHttpClient {- private readonly _delegatePromise: Promise<BaseHttpClient>;+ readonly #delegatePromise: Promise<BaseHttpClient>;constructor(options?: { logger?: CrawleeLogger }) {- this._delegatePromise = import('@crawlee/impit-client')+ this.#delegatePromise = import('@crawlee/impit-client').then(({ ImpitHttpClient }) => new ImpitHttpClient(options)).catch(() => {(options?.logger ?? log).warning(@@ -122,7 +122,7 @@ class LazyDefaultHttpClient implements BaseHttpClient {}async sendRequest(...args: Parameters<BaseHttpClient['sendRequest']>): Promise<Response> {- return (await this._delegatePromise).sendRequest(...args);+ return (await this.#delegatePromise).sendRequest(...args);}}@@ -638,23 +638,24 @@ export class BasicCrawler<* request queue; subsequent ones get their own queue via a unique alias so they don't* collide.*/+ // kept as TS-private: tests reset the counter at runtimeprivate static instanceCount = 0;/*** Tracks crawler instances that accessed shared state without having an explicit id.* Used to detect and warn about multiple crawlers sharing the same state.*/- private static useStateAnonymousIndices = new Set<number>();+ static #useStateAnonymousIndices = new Set<number>();/** Backs the {@apilink BasicCrawler.stats|`stats`} getter. */- private statsDep: OwnedOrInjected<IStatistics, Statistics>;+ #statsDep: OwnedOrInjected<IStatistics, Statistics>;/*** The statistics instance collecting the crawler's run statistics - either the injected `statistics` option or a* crawler-built default. Typed as {@apilink IStatistics} so custom implementations can be plugged in.*/get stats(): IStatistics {- return this.statsDep.value;+ return this.#statsDep.value;}/**@@ -665,14 +666,14 @@ export class BasicCrawler<protected requestManager?: IRequestManager;/** Backs the {@apilink BasicCrawler.sessionPool|`sessionPool`} getter. */- private sessionPoolDep: OwnedOrInjected<ISessionPool, SessionPool>;+ #sessionPoolDep: OwnedOrInjected<ISessionPool, SessionPool>;/*** A reference to the underlying session pool that manages the crawler's {@apilink Session|sessions}. Typed as* {@apilink ISessionPool} so custom implementations can be plugged in via the `sessionPool` constructor option.*/get sessionPool(): ISessionPool {- return this.sessionPoolDep.value;+ return this.#sessionPoolDep.value;}/**@@ -680,24 +681,24 @@ export class BasicCrawler<* around a user-supplied `requestList` — so the owned-only purge between repeated `run()` calls never reaches* through to a borrowed loader. Filled lazily in {@apilink BasicCrawler.openOwnedRequestQueue|`openOwnedRequestQueue()`}.*/- private ownedRequestQueue = OwnedOrInjected.resolve<RequestQueue>();+ #ownedRequestQueue = OwnedOrInjected.resolve<RequestQueue>();/*** Whether the request-processing-time hint has already been forwarded to the request manager. The hint* derives only from `requestHandlerTimeoutMillis` (constant for the crawler's lifetime) and is raise-only,* so it only needs to be applied once, at the first async access of the manager.*/- private requestManagerTimeoutsApplied = false;+ #requestManagerTimeoutsApplied = false;/*** Resolves the governor for one run: either the injected* {@apilink BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} (borrowed) or a freshly built default with* the concurrency shortcuts folded in (owned, so the crawler starts and stops it).*/- private readonly resolveConcurrencySystem: () => OwnedOrInjected<IConcurrencySystem, ConcurrencySystem>;+ readonly #resolveConcurrencySystem: () => OwnedOrInjected<IConcurrencySystem, ConcurrencySystem>;- /** As resolved by `_init()`. Absent until the first run, so a `teardown()` before it is a no-op. */- private concurrencySystemDep?: OwnedOrInjected<IConcurrencySystem, ConcurrencySystem>;+ /** As resolved by `init()`. Absent until the first run, so a `teardown()` before it is a no-op. */+ #concurrencySystemDep?: OwnedOrInjected<IConcurrencySystem, ConcurrencySystem>;/*** The concurrency governor this run is booking its requests against — either the@@ -712,7 +713,7 @@ export class BasicCrawler<* `minConcurrency`/`maxConcurrency`/`desiredConcurrency` on your own reference.*/get concurrencySystem(): IConcurrencySystem | undefined {- return this.concurrencySystemDep?.maybeValue;+ return this.#concurrencySystemDep?.maybeValue;}/**@@ -722,7 +723,7 @@ export class BasicCrawler<* {@apilink BasicCrawler.resume|`resume()`}, {@apilink BasicCrawler.teardown|`teardown()`} and* {@apilink BasicCrawler.concurrencySystem|`concurrencySystem`}.*/- private autoscaledPool?: AutoscaledPool;+ #autoscaledPool?: AutoscaledPool;/*** A reference to the underlying {@apilink IProxyConfiguration} instance that manages the crawler's proxies.@@ -739,7 +740,7 @@ export class BasicCrawler<Routes>;- private _basicContextPipeline?: ContextPipeline<{ request: Request }, CrawlingContext>;+ #basicContextPipeline?: ContextPipeline<{ request: Request }, CrawlingContext>;/*** The basic part of the context pipeline. Unlike the subclass pipeline, this@@ -750,26 +751,26 @@ export class BasicCrawler<* This is used e.g. in the {@apilink AdaptivePlaywrightCrawler|`AdaptivePlaywrightCrawler`}.*/get basicContextPipeline(): ContextPipeline<{ request: Request }, CrawlingContext> {- if (this._basicContextPipeline === undefined) {- this._basicContextPipeline = this.buildBasicContextPipeline();+ if (this.#basicContextPipeline === undefined) {+ this.#basicContextPipeline = this.buildBasicContextPipeline();}- return this._basicContextPipeline;+ return this.#basicContextPipeline;}- private _contextPipeline?: ContextPipeline<CrawlingContext, ExtendedContext>;+ #contextPipeline?: ContextPipeline<CrawlingContext, ExtendedContext>;get contextPipeline(): ContextPipeline<CrawlingContext, ExtendedContext> {- if (this._contextPipeline === undefined) {- this._contextPipeline = this.buildFinalContextPipeline();+ if (this.#contextPipeline === undefined) {+ this.#contextPipeline = this.buildFinalContextPipeline();}- return this._contextPipeline;+ return this.#contextPipeline;}running = false;hasFinishedBefore = false;- private unexpectedStop = false;+ #unexpectedStop = false;#log!: CrawleeLogger;@@ -780,43 +781,45 @@ export class BasicCrawler<protected readonly requestHandler!: RequestHandler<ExtendedContext>;protected readonly errorHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;protected readonly failedRequestHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;+ // kept as TS-private: tests read it at runtimeprivate requestHandlerTimeoutMillis!: number;protected readonly internalTimeoutMillis: number;protected readonly maxRequestRetries: number;protected readonly maxCrawlDepth?: number;- private sameDomainDelayMillis: number;- private domainAccessedTime: Map<string, number>;+ #sameDomainDelayMillis: number;+ #domainAccessedTime: Map<string, number>;protected readonly maxRequestsPerCrawl?: number;private get handledRequestsCount(): number {return this.stats.state.requestsFinished + this.stats.state.requestsFailed;}- private statusMessageLoggingInterval: number;- private statusMessageCallback?: StatusMessageCallback;+ #statusMessageLoggingInterval: number;+ #statusMessageCallback?: StatusMessageCallback;protected blockedStatusCodes = new Set<number>();protected readonly additionalHttpErrorStatusCodes: Set<number>;- private ignoreHttpErrorStatusCodes: Set<number>;+ #ignoreHttpErrorStatusCodes: Set<number>;/*** The resolved options for the crawler's own task loop — the crawler-owned `runTaskFunction`, the (possibly* user-overridden) ready/finished predicates and cadence/logging. Concurrency configuration lives on the* {@apilink ConcurrencySystem} instead, and the loop's `consumer` identity is the crawler's own, so neither is* settable here.*/+ // kept as TS-private: tests mutate it at runtimeprivate taskLoopOptions: Omit<AutoscaledPoolOptions, 'concurrencySystem' | 'consumer'>;protected readonly httpClient: BaseHttpClient;protected readonly retryOnBlocked: boolean;- private respectRobotsTxtFile: boolean | { userAgent?: string };+ #respectRobotsTxtFile: boolean | { userAgent?: string };/** Whether `runInStorageTransaction()` opens a transaction at all. */- private readonly transactionalStorageEnabled: boolean;+ readonly #transactionalStorageEnabled: boolean;/** The resolved per-storage-type write policy overrides forwarded to each request's transaction. */- private readonly storageWritePolicy: Partial<StorageWritePolicy>;+ readonly #storageWritePolicy: Partial<StorageWritePolicy>;protected readonly onSkippedRequest?: SkippedRequestCallback;- private _ownsEventManager = false;- private loggedPerRun = new Set<string>();- private readonly robotsTxtFileCache: LruCache<RobotsTxtFile>;+ #ownsEventManager = false;+ #loggedPerRun = new Set<string>();+ readonly #robotsTxtFileCache: LruCache<RobotsTxtFile>;protected readonly identity: CrawlerIdentity;- private readonly contextPipelineOptions: {+ readonly #contextPipelineOptions: {contextPipelineBuilder?: () => ContextPipeline<CrawlingContext, Context>;extendContext?: (context: CrawlingContext) => Awaitable<ContextExtension>;};@@ -967,7 +970,7 @@ export class BasicCrawler<try {serviceLocatorScope.enterScope();- this.contextPipelineOptions = {+ this.#contextPipelineOptions = {contextPipelineBuilder: options.contextPipelineBuilder,extendContext: options.extendContext,};@@ -1002,14 +1005,14 @@ export class BasicCrawler<this.httpClient = httpClient ?? new LazyDefaultHttpClient({ logger: this.log });this.proxyConfiguration = proxyConfiguration;- this.statusMessageLoggingInterval = statusMessageLoggingInterval;- this.statusMessageCallback = statusMessageCallback as StatusMessageCallback;- this.domainAccessedTime = new Map();- this.robotsTxtFileCache = new LruCache({ maxLength: 1000 });+ this.#statusMessageLoggingInterval = statusMessageLoggingInterval;+ this.#statusMessageCallback = statusMessageCallback as StatusMessageCallback;+ this.#domainAccessedTime = new Map();+ this.#robotsTxtFileCache = new LruCache({ maxLength: 1000 });this.handleSkippedRequest = this.handleSkippedRequest.bind(this);this.additionalHttpErrorStatusCodes = new Set([...additionalHttpErrorStatusCodes]);- this.ignoreHttpErrorStatusCodes = new Set([...ignoreHttpErrorStatusCodes]);+ this.#ignoreHttpErrorStatusCodes = new Set([...ignoreHttpErrorStatusCodes]);this.requestHandler = requestHandler ?? this.router;this.failedRequestHandler = failedRequestHandler;@@ -1022,14 +1025,14 @@ export class BasicCrawler<}this.retryOnBlocked = retryOnBlocked;- this.respectRobotsTxtFile = respectRobotsTxtFile;+ this.#respectRobotsTxtFile = respectRobotsTxtFile;// The cast undoes ow's assertion signature, which mangles `boolean | object` unions.const transactionalStorageOption = transactionalStorage as| boolean| Partial<StorageWritePolicy>| undefined;- this.transactionalStorageEnabled = transactionalStorageOption !== false;- this.storageWritePolicy = typeof transactionalStorageOption === 'object' ? transactionalStorageOption : {};+ this.#transactionalStorageEnabled = transactionalStorageOption !== false;+ this.#storageWritePolicy = typeof transactionalStorageOption === 'object' ? transactionalStorageOption : {};this.onSkippedRequest = onSkippedRequest;// allow at least 5min for internal timeouts@@ -1039,8 +1042,8 @@ export class BasicCrawler<this.maxRequestRetries = maxRequestRetries;this.maxCrawlDepth = maxCrawlDepth;- this.sameDomainDelayMillis = sameDomainDelaySecs * 1000;- this.statsDep = OwnedOrInjected.resolve<IStatistics, Statistics>(+ this.#sameDomainDelayMillis = sameDomainDelaySecs * 1000;+ this.#statsDep = OwnedOrInjected.resolve<IStatistics, Statistics>(statistics,() =>new Statistics({@@ -1059,7 +1062,7 @@ export class BasicCrawler<);}- this.sessionPoolDep = OwnedOrInjected.resolve(+ this.#sessionPoolDep = OwnedOrInjected.resolve(sessionPool,() =>new SessionPool({@@ -1194,7 +1197,7 @@ export class BasicCrawler<return false;}- if (this.unexpectedStop) {+ if (this.#unexpectedStop) {this.logOncePerRun('shuttingDown','No new requests are allowed because the `stop()` method has been called. ' +@@ -1215,7 +1218,7 @@ export class BasicCrawler<return true;}- if (this.unexpectedStop) {+ if (this.#unexpectedStop) {this.log.info('The crawler has finished all the remaining ongoing requests and will shut down now.',);@@ -1240,7 +1243,7 @@ export class BasicCrawler<this.taskLoopOptions = { ...taskLoopOptions, ...crawlerOwnedTaskLoopConfiguration };- this.resolveConcurrencySystem = () =>+ this.#resolveConcurrencySystem = () =>OwnedOrInjected.resolve<IConcurrencySystem, ConcurrencySystem>(concurrencySystem, () =>this.createDefaultConcurrencySystem({minConcurrency,@@ -1273,7 +1276,7 @@ export class BasicCrawler<* @returns `true` if the status code is considered an error, `false` otherwise*/protected isErrorStatusCode(status: number): boolean {- const excludeError = this.ignoreHttpErrorStatusCodes.has(status);+ const excludeError = this.#ignoreHttpErrorStatusCodes.has(status);const includeError = this.additionalHttpErrorStatusCodes.has(status);return (status >= 500 && !excludeError) || includeError;@@ -1398,7 +1401,7 @@ export class BasicCrawler<}private buildFinalContextPipeline(): ContextPipeline<CrawlingContext, ExtendedContext> {- const subclassPipeline = (this.contextPipelineOptions.contextPipelineBuilder?.() ??+ const subclassPipeline = (this.#contextPipelineOptions.contextPipelineBuilder?.() ??this.buildContextPipeline()) as ContextPipeline<CrawlingContext, Context>;// `extendContext` runs *before* the subclass navigation pipeline (which includes the@@ -1411,7 +1414,7 @@ export class BasicCrawler<// TypeScript cannot express that `Context` transitively includes `ContextExtension` here. The// casts below are sound because `buildFinalContextPipeline` is declared to return the fully// resolved `ExtendedContext` (= `Context & ContextExtension`).- const { extendContext } = this.contextPipelineOptions;+ const { extendContext } = this.#contextPipelineOptions;let contextPipeline: ContextPipeline<CrawlingContext, Context>;if (extendContext !== undefined) {contextPipeline = ContextPipeline.create<CrawlingContext>()@@ -1455,7 +1458,7 @@ export class BasicCrawler<* @param error The error to check.*/protected isProxyError(error: Error): boolean {- return ROTATE_PROXY_ERRORS.some((x: string) => (this._getMessageFromError(error) as any)?.includes(x));+ return ROTATE_PROXY_ERRORS.some((x: string) => (this.getMessageFromError(error) as any)?.includes(x));}/**@@ -1507,7 +1510,7 @@ export class BasicCrawler<let message: string;if (operationMode === 'ERROR') {- message = `Experiencing problems, ${failedDelta} failed requests in the past ${this.statusMessageLoggingInterval} seconds.`;+ message = `Experiencing problems, ${failedDelta} failed requests in the past ${this.#statusMessageLoggingInterval} seconds.`;} else {const total = await this.requestManager?.getTotalCount();message = `Crawled ${this.stats.state.requestsFinished}${total ? `/${total}` : ''} pages, ${@@ -1515,8 +1518,8 @@ export class BasicCrawler<} failed requests, desired concurrency ${this.concurrencySystem?.desiredConcurrency ?? 0}.`;}- if (this.statusMessageCallback) {- await this.statusMessageCallback({+ if (this.#statusMessageCallback) {+ await this.#statusMessageCallback({crawler: this as any,state: this.stats.state,previousState,@@ -1528,7 +1531,7 @@ export class BasicCrawler<this.setStatusMessage(message);};- const interval = setInterval(log, this.statusMessageLoggingInterval * 1e3);+ const interval = setInterval(log, this.#statusMessageLoggingInterval * 1e3);return { log, stop: () => clearInterval(interval) };}@@ -1562,23 +1565,23 @@ export class BasicCrawler<// When `purgeRequestQueue` is explicitly `false`, nothing is purged.const shouldPurge = purgeRequestQueue !== false;const managerToPurge =- this.ownedRequestQueue.maybeValue ?? (purgeRequestQueue === true ? this.requestManager : undefined);+ this.#ownedRequestQueue.maybeValue ?? (purgeRequestQueue === true ? this.requestManager : undefined);if (managerToPurge?.purge && shouldPurge) {await managerToPurge.purge();}// A supplied statistics instance keeps whatever state it was handed - only wipe a default we built.- await this.statsDep.ifOwned(async (stats) => {+ await this.#statsDep.ifOwned(async (stats) => {stats.reset();await stats.resetStore();});- await this.sessionPoolDep.ifOwned((pool) => pool.resetStore());+ await this.#sessionPoolDep.ifOwned((pool) => pool.resetStore());}- this.unexpectedStop = false;+ this.#unexpectedStop = false;this.running = true;- this.loggedPerRun.clear();+ this.#loggedPerRun.clear();await purgeDefaultStorages({onlyPurgeOnce: true,@@ -1591,7 +1594,7 @@ export class BasicCrawler<}try {- await this._init();+ await this.init();await this.stats.startCapturing();} catch (error) {// Clean up here before propagating, otherwise a failed startup would leave the process hanging.@@ -1612,7 +1615,7 @@ export class BasicCrawler<'Pausing... Press CTRL+C again to force exit. To resume, do: CRAWLEE_PURGE_ON_START=0 npm start',);await this.pauseOnMigration();- await this.autoscaledPool!.abort();+ await this.#autoscaledPool!.abort();};// Attach a listener to handle migration and aborting events gracefully.@@ -1625,7 +1628,7 @@ export class BasicCrawler<let stats = {} as FinalStatistics;try {- await this.autoscaledPool!.run();+ await this.#autoscaledPool!.run();} finally {await this.stats.stopCapturing();await this.teardown();@@ -1690,11 +1693,11 @@ export class BasicCrawler<* To stop the crawler immediately, use {@apilink BasicCrawler.teardown|`crawler.teardown()`} instead.*/stop(reason = 'The crawler has been gracefully stopped.'): void {- if (this.unexpectedStop) {+ if (this.#unexpectedStop) {return;}this.log.info(reason);- this.unexpectedStop = true;+ this.#unexpectedStop = true;}/**@@ -1706,12 +1709,12 @@ export class BasicCrawler<* throughout, since a shared one may still be serving other crawlers.*/async pause(timeoutSecs?: number): Promise<void> {- if (!this.autoscaledPool) {+ if (!this.#autoscaledPool) {this.log.warning('Cannot pause a crawler that is not running.');return;}- await this.autoscaledPool.pause(timeoutSecs);+ await this.#autoscaledPool.pause(timeoutSecs);}/**@@ -1719,12 +1722,12 @@ export class BasicCrawler<* again. A no-op on a crawler that is not paused.*/resume(): void {- if (!this.autoscaledPool) {+ if (!this.#autoscaledPool) {this.log.warning('Cannot resume a crawler that is not running.');return;}- this.autoscaledPool.resume();+ this.#autoscaledPool.resume();}/**@@ -1739,8 +1742,8 @@ export class BasicCrawler<// Apply the processing-time hint here (an async lifecycle point) rather than in the constructor,// now that `setExpectedRequestProcessingTimeSecs` is async. The hint is raise-only and idempotent,// but guard so we do not re-issue it on every call.- if (!this.requestManagerTimeoutsApplied) {- this.requestManagerTimeoutsApplied = true;+ if (!this.#requestManagerTimeoutsApplied) {+ this.#requestManagerTimeoutsApplied = true;await this.applyRequestManagerTimeouts(this.requestManager);}@@ -1766,7 +1769,7 @@ export class BasicCrawler<const identifier = this.identity.instanceIndex === 0 ? null : { alias: `__default_${this.identity.id}__` };const requestQueue = await RequestQueue.open(identifier, { configuration: serviceLocator.getConfiguration() });- return this.ownedRequestQueue.set(requestQueue);+ return this.#ownedRequestQueue.set(requestQueue);}/**@@ -1826,9 +1829,9 @@ export class BasicCrawler<return kvs.getAutoSavedValue<State>(stateKey, defaultValue);}- BasicCrawler.useStateAnonymousIndices.add(this.identity.instanceIndex);+ BasicCrawler.#useStateAnonymousIndices.add(this.identity.instanceIndex);- if (BasicCrawler.useStateAnonymousIndices.size > 1) {+ if (BasicCrawler.#useStateAnonymousIndices.size > 1) {serviceLocator.getLogger().warningOnce(@@ -1883,9 +1886,9 @@ export class BasicCrawler<}private logOncePerRun(key: string, message: string): void {- if (!this.loggedPerRun.has(key)) {+ if (!this.#loggedPerRun.has(key)) {this.log.info(message);- this.loggedPerRun.add(key);+ this.#loggedPerRun.add(key);}}@@ -2066,12 +2069,12 @@ export class BasicCrawler</*** Initializes the crawler.*/- protected async _init(): Promise<void> {+ protected async init(): Promise<void> {const eventManager = serviceLocator.getEventManager();if (!eventManager.isInitialized()) {await eventManager.init();- this._ownsEventManager = true;+ this.#ownsEventManager = true;}// Warn once at startup if the internal timeout is shorter than the phases it is meant to outlast. It is@@ -2090,12 +2093,12 @@ export class BasicCrawler<// An owned governor is rebuilt (and started) for every run, so it always starts from a clean slate — stale// resource snapshots or a previous run's scaled desired concurrency would otherwise distort this run's// scaling. An injected one is long-lived and its lifecycle belongs to the caller.- this.concurrencySystemDep = this.resolveConcurrencySystem();- await this.concurrencySystemDep.ifOwned((system) => system.start());+ this.#concurrencySystemDep = this.#resolveConcurrencySystem();+ await this.#concurrencySystemDep.ifOwned((system) => system.start());- this.autoscaledPool = new AutoscaledPool({+ this.#autoscaledPool = new AutoscaledPool({...this.taskLoopOptions,- concurrencySystem: this.concurrencySystemDep.value,+ concurrencySystem: this.#concurrencySystemDep.value,consumer: this.identity,});@@ -2171,12 +2174,12 @@ export class BasicCrawler<* a normal return says nothing about success. `handleRequest` owns the outcome.*/private async runInStorageTransaction<T>(callback: () => Promise<T>): Promise<T> {- if (!this.transactionalStorageEnabled) {+ if (!this.#transactionalStorageEnabled) {return callback();}const transaction = createStorageTransaction({- policy: this.storageWritePolicy,+ policy: this.#storageWritePolicy,commitTimeoutMillis: this.internalTimeoutMillis,});@@ -2208,7 +2211,7 @@ export class BasicCrawler</*** Handles blocked request*/- protected _throwOnBlockedRequest(statusCode: number) {+ protected throwOnBlockedRequest(statusCode: number) {if (this.retryOnBlocked) return;if (this.blockedStatusCodes.has(statusCode)) {@@ -2217,31 +2220,31 @@ export class BasicCrawler<}private async isAllowedBasedOnRobotsTxtFile(url: string): Promise<boolean> {- if (!this.respectRobotsTxtFile) {+ if (!this.#respectRobotsTxtFile) {return true;}const robotsTxtFile = await this.getRobotsTxtFileForUrl(url);- const userAgent = typeof this.respectRobotsTxtFile === 'object' ? this.respectRobotsTxtFile?.userAgent : '*';+ const userAgent = typeof this.#respectRobotsTxtFile === 'object' ? this.#respectRobotsTxtFile?.userAgent : '*';return !robotsTxtFile || robotsTxtFile.isAllowed(url, userAgent);}protected async getRobotsTxtFileForUrl(url: string): Promise<RobotsTxtFile | undefined> {- if (!this.respectRobotsTxtFile) {+ if (!this.#respectRobotsTxtFile) {return undefined;}try {const origin = new URL(url).origin;- const cachedRobotsTxtFile = this.robotsTxtFileCache.get(origin);+ const cachedRobotsTxtFile = this.#robotsTxtFileCache.get(origin);if (cachedRobotsTxtFile) {return cachedRobotsTxtFile;}const robotsTxtFile = await RobotsTxtFile.find(url, { logger: this.log });- this.robotsTxtFileCache.add(origin, robotsTxtFile);+ this.#robotsTxtFileCache.add(origin, robotsTxtFile);return robotsTxtFile;} catch (e: any) {@@ -2251,9 +2254,9 @@ export class BasicCrawler<}private async pauseOnMigration() {- if (this.autoscaledPool) {+ if (this.#autoscaledPool) {// if run wasn't called, this is going to crash- await this.autoscaledPool.pause(SAFE_MIGRATION_WAIT_MILLIS).catch((err) => {+ await this.#autoscaledPool.pause(SAFE_MIGRATION_WAIT_MILLIS).catch((err) => {if (err.message.includes('running tasks did not finish')) {this.log.error('The crawler was paused due to migration to another host, ' +@@ -2315,14 +2318,14 @@ export class BasicCrawler<}const now = Date.now();- const lastAccessTime = this.domainAccessedTime.get(domain);+ const lastAccessTime = this.#domainAccessedTime.get(domain);- if (!lastAccessTime || now - lastAccessTime >= this.sameDomainDelayMillis) {- this.domainAccessedTime.set(domain, now);+ if (!lastAccessTime || now - lastAccessTime >= this.#sameDomainDelayMillis) {+ this.#domainAccessedTime.set(domain, now);return false;}- const delay = lastAccessTime + this.sameDomainDelayMillis - now;+ const delay = lastAccessTime + this.#sameDomainDelayMillis - now;this.log.debug(`Request ${request.url} (${request.id}) will be reclaimed after ${delay} milliseconds due to same domain delay`,);@@ -2495,7 +2498,7 @@ export class BasicCrawler<// (e.g. `enqueueLinks({ urls, limit: config.limit })`) cannot discard the crawler defaults ⤵requestManager: options.requestManager ?? requestManager,robotsTxtFile: options.robotsTxtFile ?? (await this.getRobotsTxtFileForUrl(request.url)),- respectRobotsTxtFile: options.respectRobotsTxtFile ?? this.respectRobotsTxtFile,+ respectRobotsTxtFile: options.respectRobotsTxtFile ?? this.#respectRobotsTxtFile,onSkippedRequest,limit,transformRequestFunction: transformRequestFunctionWrapper,@@ -2610,7 +2613,7 @@ export class BasicCrawler<// We don't want to see the stack trace in the logs by default, when we are going to retry the request.// Thus, we print the full stack trace only when CRAWLEE_VERBOSE_LOG environment variable is set to true.- const message = this._getMessageFromError(error);+ const message = this.getMessageFromError(error);this.log.warning(`Reclaiming failed request back to the list or queue. ${message}`, {id,url,@@ -2648,7 +2651,7 @@ export class BasicCrawler<private async handleFailedRequestHandler(crawlingContext: CrawlingContext, error: Error): Promise<void> {// Always log the last error regardless if the user provided a failedRequestHandlerconst { id, url, method, uniqueKey } = crawlingContext.request;- const message = this._getMessageFromError(error, true);+ const message = this.getMessageFromError(error, true);this.log.error(`Request failed and reached maximum retries. ${message}`, { id, url, method, uniqueKey });@@ -2665,7 +2668,7 @@ export class BasicCrawler<* @param error The error received* @returns The message to be logged*/- protected _getMessageFromError(error: Error, forceStack = false) {+ protected getMessageFromError(error: Error, forceStack = false) {if ([TypeError, SyntaxError, ReferenceError].some((type) => error instanceof type)) {forceStack = true;}@@ -2711,21 +2714,21 @@ export class BasicCrawler<// When this crawler initialized the event manager, its close() call emits// the final persistence event after the crawler-specific state has been// saved. External event managers still need an explicit event here.- if (!this._ownsEventManager) {+ if (!this.#ownsEventManager) {serviceLocator.getEventManager().emit(EventType.PERSIST_STATE, { isMigrating: false });}- await this.sessionPoolDep.ifOwned(async (pool) => pool.teardown({ persistState: this._ownsEventManager }));+ await this.#sessionPoolDep.ifOwned(async (pool) => pool.teardown({ persistState: this.#ownsEventManager }));- if (this._ownsEventManager) {+ if (this.#ownsEventManager) {await serviceLocator.getEventManager().close();}- await this.autoscaledPool?.abort();- await this.concurrencySystemDep?.ifOwned((system) => system.stop());+ await this.#autoscaledPool?.abort();+ await this.#concurrencySystemDep?.ifOwned((system) => system.stop());}- protected _getCookieHeaderFromRequest(request: Request) {+ protected getCookieHeaderFromRequest(request: Request) {if (request.headers?.Cookie && request.headers?.cookie) {this.log.warning(`Encountered mixed casing for the cookie headers for request ${request.url} (${request.id}). Their values will be merged.`,diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.tsindex 54f30f698..5cc17696e 100644--- a/packages/browser-crawler/src/internals/browser-crawler.ts+++ b/packages/browser-crawler/src/internals/browser-crawler.ts@@ -360,14 +360,14 @@ export abstract class BrowserCrawler<GoToOptions extends Dictionary = Dictionary,> extends BasicCrawler<Context, ContextExtension, ExtendedContext, Routes> {/** Backs the {@apilink BrowserCrawler.browserPool|`browserPool`} getter. */- private browserPoolDep: OwnedOrInjected<IBrowserPool<Page>, OwnedBrowserPool<Page>>;+ #browserPoolDep: OwnedOrInjected<IBrowserPool<Page>, OwnedBrowserPool<Page>>;/*** A reference to the underlying browser pool that manages the crawler's browsers. Typed as* {@apilink IBrowserPool} so custom implementations can be plugged in via the `browserPool` constructor option.*/get browserPool(): IBrowserPool<Page> {- return this.browserPoolDep.value;+ return this.#browserPoolDep.value;}launchContext: BrowserLaunchContext<LaunchOptions, unknown>;@@ -375,10 +375,10 @@ export abstract class BrowserCrawler<protected readonly ignoreShadowRoots: boolean;protected readonly ignoreIframes: boolean;- private readonly navigationTimeoutMillis: number;- private readonly preNavigationHooks: BrowserHook<Context>[];- private readonly postNavigationHooks: BrowserHook<Context>[];- private readonly saveResponseCookies: boolean;+ readonly #navigationTimeoutMillis: number;+ readonly #preNavigationHooks: BrowserHook<Context>[];+ readonly #postNavigationHooks: BrowserHook<Context>[];+ readonly #saveResponseCookies: boolean;protected static override optionsShape = {...BasicCrawler.optionsShape,@@ -439,28 +439,28 @@ export abstract class BrowserCrawler<step: (ctx: Ctx) => Awaitable<void | Partial<Ctx>>,): ContextMiddleware<Ctx, Partial<Ctx>> =>skipGuard(async (ctx: Ctx) => {- const remaining = remainingNavigationWindowMillis(ctx, this.navigationTimeoutMillis);+ const remaining = remainingNavigationWindowMillis(ctx, this.#navigationTimeoutMillis);if (remaining <= 0) {throw new TimeoutError(- `Navigation timed out after ${this.navigationTimeoutMillis / 1000} seconds.`,+ `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`,);}return addTimeoutToPromise(async () => step(ctx),remaining,- `Navigation timed out after ${this.navigationTimeoutMillis / 1000} seconds.`,+ `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`,);});let pipeline = contextPipelineBuilder().compose({ action: this.prepareNavigation.bind(this) });- for (const hook of this.preNavigationHooks) {+ for (const hook of this.#preNavigationHooks) {pipeline = pipeline.compose(windowGuard(hook));}pipeline = pipeline.compose(skipGuard(this.navigate.bind(this)));- for (const hook of this.postNavigationHooks) {+ for (const hook of this.#postNavigationHooks) {pipeline = pipeline.compose(windowGuard(hook));}@@ -473,12 +473,12 @@ export abstract class BrowserCrawler<});this.launchContext = launchContext;- this.navigationTimeoutMillis = navigationTimeoutSecs * 1000;+ this.#navigationTimeoutMillis = navigationTimeoutSecs * 1000;// The public option hooks are extension-aware; internal storage uses the base context type// (the pipeline composes hooks against the concrete context, which does not statically carry// `ContextExtension`). The extension members are present at runtime regardless.- this.preNavigationHooks = preNavigationHooks as BrowserHook<Context>[];- this.postNavigationHooks = postNavigationHooks as BrowserHook<Context>[];+ this.#preNavigationHooks = preNavigationHooks as BrowserHook<Context>[];+ this.#postNavigationHooks = postNavigationHooks as BrowserHook<Context>[];this.ignoreIframes = ignoreIframes;this.ignoreShadowRoots = ignoreShadowRoots;@@ -487,12 +487,12 @@ export abstract class BrowserCrawler<(this.launchContext.launchOptions as Dictionary).headless = headless;}- this.saveResponseCookies = saveResponseCookies;+ this.#saveResponseCookies = saveResponseCookies;// `browserPool` wins over `remoteBrowser` — a passed-in pool is used as-is (borrowed), the sugar is ignored.// The default is only built when no pool was injected, so all the option/launchContext fiddling below stays// inside the factory.- this.browserPoolDep = OwnedOrInjected.resolve(browserPool, () => {+ this.#browserPoolDep = OwnedOrInjected.resolve(browserPool, () => {const resolvedBrowserPoolOptions = browserPoolOptions ?? ({} as Partial<BrowserPoolOptions>);if (launchContext?.userAgent) {@@ -523,7 +523,7 @@ export abstract class BrowserCrawler<}protected override getNavigationTimeoutMillis(): number {- return this.navigationTimeoutMillis;+ return this.#navigationTimeoutMillis;}protected override buildContextPipeline(): ContextPipeline<@@ -648,8 +648,8 @@ export abstract class BrowserCrawler<return {// Default to the full navigation timeout so a pre-navigation hook can read it; `navigate` narrows it// to the remaining shared window unless a hook overrode it (see there).- gotoOptions: { timeout: this.navigationTimeoutMillis } as unknown as GoToOptions,- [COOKIES_BEFORE_HOOKS]: this._getCookieHeaderFromRequest(crawlingContext.request),+ gotoOptions: { timeout: this.#navigationTimeoutMillis } as unknown as GoToOptions,+ [COOKIES_BEFORE_HOOKS]: this.getCookieHeaderFromRequest(crawlingContext.request),} as unknown as Partial<Context>;}@@ -657,26 +657,26 @@ export abstract class BrowserCrawler<tryCancel();const gotoOptions = crawlingContext.gotoOptions as GoToOptions;- const remaining = remainingNavigationWindowMillis(crawlingContext, this.navigationTimeoutMillis);+ const remaining = remainingNavigationWindowMillis(crawlingContext, this.#navigationTimeoutMillis);if (remaining <= 0) {- throw new TimeoutError(`Navigation timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);+ throw new TimeoutError(`Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);}// If a hook left the default `navigationTimeoutMillis` in place, bound the goto to whatever is left of the// shared navigation window. If it overrode the value - including `0`, Playwright's "no timeout" - honour// that verbatim as the goto's own timeout. The driver enforces this natively (so a timed-out goto is// aborted, not left lingering) and `handleNavigationTimeout` turns its error into our own message.const gotoTimeout = gotoOptions as { timeout?: number };- if (gotoTimeout.timeout === this.navigationTimeoutMillis) {+ if (gotoTimeout.timeout === this.#navigationTimeoutMillis) {gotoTimeout.timeout = remaining;}const cookiesBeforeHooks = readContextField<string>(crawlingContext, COOKIES_BEFORE_HOOKS);- const cookiesAfterHooks = this._getCookieHeaderFromRequest(crawlingContext.request);+ const cookiesAfterHooks = this.getCookieHeaderFromRequest(crawlingContext.request);await this.applyCookies(crawlingContext, cookiesBeforeHooks, cookiesAfterHooks);let response: Response | undefined;try {- response = (await this._navigationHandler(crawlingContext, gotoOptions)) ?? undefined;+ response = (await this.navigationHandler(crawlingContext, gotoOptions)) ?? undefined;} catch (error) {await this.handleNavigationTimeout(crawlingContext, error as Error);crawlingContext.request.state = RequestState.ERROR;@@ -717,7 +717,7 @@ export abstract class BrowserCrawler<private async persistCookiesFromPage(crawlingContext: Pick<BrowserCrawlingContext<Page, Response>, 'session' | 'page' | 'request'>,): Promise<void> {- if (!this.saveResponseCookies || !crawlingContext.session) {+ if (!this.#saveResponseCookies || !crawlingContext.session) {return;}@@ -799,7 +799,7 @@ export abstract class BrowserCrawler<session?.markBad();// The driver was handed the remaining window (usually shorter than `navigationTimeoutSecs` once the// hooks have run), so it names that value in its own error; report the configured window instead.- throw new TimeoutError(`Navigation timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);+ throw new TimeoutError(`Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);}}@@ -808,11 +808,11 @@ export abstract class BrowserCrawler<*/private throwIfProxyError(error: Error) {if (this.isProxyError(error)) {- throw new SessionError(this._getMessageFromError(error) as string);+ throw new SessionError(this.getMessageFromError(error) as string);}}- protected abstract _navigationHandler(+ protected abstract navigationHandler(crawlingContext: BrowserCrawlingContext<Page, Response>,gotoOptions: GoToOptions,): Promise<Context['response'] | null | undefined>;@@ -839,7 +839,7 @@ export abstract class BrowserCrawler<if (this.sessionPool && response && session) {if (typeof response === 'object' && typeof response.status === 'function') {- this._throwOnBlockedRequest(response.status());+ this.throwOnBlockedRequest(response.status());} else {this.log.debug('Got a malformed Browser response.', { request, response });}@@ -853,7 +853,7 @@ export abstract class BrowserCrawler<* @ignore*/override async teardown(): Promise<void> {- await this.browserPoolDep.ifOwned((pool) => pool.destroy());+ await this.#browserPoolDep.ifOwned((pool) => pool.destroy());await super.teardown();}}diff --git a/packages/browser-pool/src/abstract-classes/browser-controller.ts b/packages/browser-pool/src/abstract-classes/browser-controller.tsindex 011652129..ead1d0860 100644--- a/packages/browser-pool/src/abstract-classes/browser-controller.ts+++ b/packages/browser-pool/src/abstract-classes/browser-controller.ts@@ -144,16 +144,17 @@ export abstract class BrowserController<lastPageOpenedAt = Date.now();- private _activate!: () => void;+ #activate!: () => void;+ // kept as TS-private: `BrowserPool` awaits it through cross-object bracket accessprivate isActivePromise = new Promise<void>((resolve) => {- this._activate = resolve;+ this.#activate = resolve;});- private commitBrowser!: () => void;+ #commitBrowser!: () => void;- private hasBrowserPromise = new Promise<void>((resolve) => {- this.commitBrowser = resolve;+ #hasBrowserPromise = new Promise<void>((resolve) => {+ this.#commitBrowser = resolve;});constructor(browserPlugin: BrowserPlugin<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>) {@@ -172,7 +173,7 @@ export abstract class BrowserController<if (!this.browser) {throw new Error('Cannot activate BrowserController without an assigned browser.');}- this._activate();+ this.#activate();this.isActive = true;}@@ -188,7 +189,7 @@ export abstract class BrowserController<}this.browser = browser;this.launchContext = launchContext;- this.commitBrowser();+ this.#commitBrowser();}/**@@ -198,7 +199,7 @@ export abstract class BrowserController<* Emits 'browserClosed' event.*/async close(): Promise<void> {- await this.hasBrowserPromise;+ await this.#hasBrowserPromise;try {await this._close();@@ -225,7 +226,7 @@ export abstract class BrowserController<* Emits 'browserClosed' event.*/async kill(): Promise<void> {- await this.hasBrowserPromise;+ await this.#hasBrowserPromise;await this._kill();this.emit(BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED, this);}diff --git a/packages/browser-pool/src/abstract-classes/browser-plugin.ts b/packages/browser-pool/src/abstract-classes/browser-plugin.tsindex b289263a4..b565005cd 100644--- a/packages/browser-pool/src/abstract-classes/browser-plugin.ts+++ b/packages/browser-pool/src/abstract-classes/browser-plugin.ts@@ -169,7 +169,7 @@ export abstract class BrowserPlugin<* Subclasses implement only the `connect` callback — the resolve / token / release / error-wrap scaffolding* lives here so it stays identical across plugins.*/- protected async _connectToRemoteBrowser(+ protected async connectToRemoteBrowser(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>,connect: (url: string) => Promise<LaunchResult>,): Promise<LaunchResult> {@@ -183,7 +183,7 @@ export abstract class BrowserPlugin<throw new BrowserLaunchError('Failed to resolve the remote browser endpoint.', { cause });}- launchContext._remoteToken = token;+ launchContext.remoteToken = token;try {return await connect(url);@@ -256,12 +256,12 @@ export abstract class BrowserPlugin<const { proxyUrl, launchOptions } = launchContext;if (proxyUrl && !launchContext.isRemote) {- await this._addProxyToLaunchOptions(launchContext);+ await this.addProxyToLaunchOptions(launchContext);}- if (!launchContext.isRemote && this._isChromiumBasedBrowser(launchContext)) {+ if (!launchContext.isRemote && this.isChromiumBasedBrowser(launchContext)) {// This will set the args for chromium based browsers to hide the webdriver.- (launchOptions as Dictionary).args = this._mergeArgsToHideWebdriver(launchOptions!.args);+ (launchOptions as Dictionary).args = this.mergeArgsToHideWebdriver(launchOptions!.args);// When User-Agent is not set, and we're using Chromium in headless mode,// it is better to use DEFAULT_USER_AGENT to reduce chance of detection,// as otherwise 'HeadlessChrome' is present in User-Agent string.@@ -278,7 +278,7 @@ export abstract class BrowserPlugin<return this._launch(launchContext);}- private _mergeArgsToHideWebdriver(originalArgs?: string[]): string[] {+ private mergeArgsToHideWebdriver(originalArgs?: string[]): string[] {if (!originalArgs?.length) {return ['--disable-blink-features=AutomationControlled'];}@@ -294,7 +294,7 @@ export abstract class BrowserPlugin<return originalArgs;}- protected _throwAugmentedLaunchError(+ protected throwAugmentedLaunchError(cause: unknown,executablePath: string | undefined,dockerImage: string,@@ -325,11 +325,11 @@ export abstract class BrowserPlugin</*** @private*/- protected abstract _addProxyToLaunchOptions(+ protected abstract addProxyToLaunchOptions(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>,): Promise<void>;- protected abstract _isChromiumBasedBrowser(+ protected abstract isChromiumBasedBrowser(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>,): boolean;diff --git a/packages/browser-pool/src/browser-pool.ts b/packages/browser-pool/src/browser-pool.tsindex 5f283a651..5f9945af0 100644--- a/packages/browser-pool/src/browser-pool.ts+++ b/packages/browser-pool/src/browser-pool.ts@@ -333,19 +333,20 @@ export class BrowserPool<fingerprintGenerator?: FingerprintGenerator;fingerprintCache?: QuickLRU<string, BrowserFingerprintWithHeaders>;+ // kept as TS-private: tests replace this interval through bracket accessprivate browserKillerInterval? = setInterval(- async () => this._closeInactiveRetiredBrowsers(),+ async () => this.closeInactiveRetiredBrowsers(),BROWSER_KILLER_INTERVAL_MILLIS,);- private browserRetireInterval?: NodeJS.Timeout;+ #browserRetireInterval?: NodeJS.Timeout;- private limiter = pLimit(1);- private log!: CrawleeLogger;+ #limiter = pLimit(1);+ #log!: CrawleeLogger;constructor(options: Options & BrowserPoolHooks<BrowserControllerReturn, LaunchContextReturn, PageReturn>) {super();- this.log = serviceLocator.getLogger().child({ prefix: 'BrowserPool' });+ this.#log = serviceLocator.getLogger().child({ prefix: 'BrowserPool' });this.browserKillerInterval!.unref();@@ -410,7 +411,7 @@ export class BrowserPool<this.useFingerprints = useFingerprints;this.fingerprintOptions = fingerprintOptions;- this.browserRetireInterval = setInterval(+ this.#browserRetireInterval = setInterval(async () =>this.activeBrowserControllers.forEach((controller) => {if (@@ -423,7 +424,7 @@ export class BrowserPool<retireInactiveBrowserAfterSecs * 1000,);- this.browserRetireInterval!.unref();+ this.#browserRetireInterval!.unref();// hooksthis.preLaunchHooks = preLaunchHooks;@@ -435,7 +436,7 @@ export class BrowserPool<// fingerprintingif (this.useFingerprints) {- this._initializeFingerprinting();+ this.initializeFingerprinting();}}@@ -460,7 +461,7 @@ export class BrowserPool<const {id = nanoid(),pageOptions,- browserPlugin = this._pickBrowserPlugin(),+ browserPlugin = this.pickBrowserPlugin(),session,proxyUrl = session?.proxyInfo?.url,ignoreTlsErrors = session?.proxyInfo?.ignoreTlsErrors,@@ -485,19 +486,19 @@ export class BrowserPool<// context and run request B's storage writes inside request A's transaction.// TODO(crawlee@v4): bump p-limit to v5 and drop this AsyncResource.bind wrapper.// Limiter is necessary - https://github.com/apify/crawlee/issues/1126- return this.limiter(+ return this.#limiter(AsyncResource.bind(async () => {- let browserController = this._pickBrowserWithFreeCapacity(browserPlugin, { proxyUrl });+ let browserController = this.pickBrowserWithFreeCapacity(browserPlugin, { proxyUrl });if (!browserController)- browserController = await this._launchBrowser(id, {+ browserController = await this.launchBrowser(id, {browserPlugin,proxyUrl,ignoreTlsErrors,});tryCancel();- return await this._createPageForBrowser(id, browserController, pageOptions, proxyUrl, ignoreTlsErrors);+ return await this.createPageForBrowser(id, browserController, pageOptions, proxyUrl, ignoreTlsErrors);}),);}@@ -510,15 +511,15 @@ export class BrowserPool<async newPageInNewBrowser(options: BrowserPoolNewPageInNewBrowserOptions<PageOptions, BrowserPlugins[number]> = {},): Promise<PageReturn> {- const { id = nanoid(), pageOptions, launchOptions, browserPlugin = this._pickBrowserPlugin() } = options;+ const { id = nanoid(), pageOptions, launchOptions, browserPlugin = this.pickBrowserPlugin() } = options;if (this.pages.has(id)) {throw new Error(`Page with ID: ${id} already exists.`);}- const browserController = await this._launchBrowser(id, { launchOptions, browserPlugin });+ const browserController = await this.launchBrowser(id, { launchOptions, browserPlugin });tryCancel();- return await this._createPageForBrowser(id, browserController, pageOptions);+ return await this.createPageForBrowser(id, browserController, pageOptions);}/**@@ -591,7 +592,7 @@ export class BrowserPool<return this.pageIds.get(page);}- private async _createPageForBrowser(+ private async createPageForBrowser(pageId: string,browserController: BrowserControllerReturn,pageOptions: PageOptions = {} as PageOptions,@@ -617,7 +618,7 @@ export class BrowserPool<}}- await this._executeHooks(this.prePageCreateHooks, pageId, browserController, finalPageOptions);+ await this.executeHooks(this.prePageCreateHooks, pageId, browserController, finalPageOptions);tryCancel();let page: PageReturn;@@ -639,7 +640,7 @@ export class BrowserPool<this.retireBrowserController(browserController);}- this._overridePageClose(page);+ this.overridePageClose(page);} catch (err) {this.retireBrowserController(browserController);throw new Error(@@ -647,7 +648,7 @@ export class BrowserPool<);}- await this._executeHooks(this.postPageCreateHooks, page, browserController);+ await this.executeHooks(this.postPageCreateHooks, page, browserController);tryCancel();this.emit(BROWSER_POOL_EVENTS.PAGE_CREATED, page);@@ -762,7 +763,7 @@ export class BrowserPool<* @return {Promise<void>}*/async closeAllBrowsers(): Promise<void> {- const controllers = this._getAllBrowserControllers();+ const controllers = this.getAllBrowserControllers();const promises = [...controllers].filter((controller) => controller.isActive).map(async (controller) => controller.close());@@ -775,16 +776,16 @@ export class BrowserPool<*/async destroy(): Promise<void> {clearInterval(this.browserKillerInterval!);- clearInterval(this.browserRetireInterval!);+ clearInterval(this.#browserRetireInterval!);this.browserKillerInterval = undefined;- this.browserRetireInterval = undefined;+ this.#browserRetireInterval = undefined;await this.closeAllBrowsers();- this._teardown();+ this.teardown();}- private _teardown() {+ private teardown() {this.startingBrowserControllers.clear();this.activeBrowserControllers.clear();this.retiredBrowserControllers.clear();@@ -792,7 +793,7 @@ export class BrowserPool<this.removeAllListeners();}- private _getAllBrowserControllers() {+ private getAllBrowserControllers() {return new Set([...this.startingBrowserControllers,...this.activeBrowserControllers,@@ -800,7 +801,7 @@ export class BrowserPool<]);}- private async _launchBrowser(pageId: string, options: InternalLaunchBrowserOptions<BrowserPlugins[number]>) {+ private async launchBrowser(pageId: string, options: InternalLaunchBrowserOptions<BrowserPlugins[number]>) {const { browserPlugin, launchOptions, proxyUrl, ignoreTlsErrors } = options;const browserController = browserPlugin.createController() as BrowserControllerReturn;@@ -825,7 +826,7 @@ export class BrowserPool<try {// If the hooks or the launch fails, we need to delete the controller,// because otherwise it would be stuck in limbo without a browser.- await this._executeHooks(this.preLaunchHooks, pageId, launchContext);+ await this.executeHooks(this.preLaunchHooks, pageId, launchContext);tryCancel();const browser = await browserPlugin.launch(launchContext);tryCancel();@@ -835,17 +836,17 @@ export class BrowserPool<throw err;}- this.log.debug('Launched new browser.', { id: browserController.id });+ this.#log.debug('Launched new browser.', { id: browserController.id });browserController.proxyUrl = proxyUrl;try {// If the launch fails on the post-launch hooks, we need to clean up// both the controller and the browser before throwing.- await this._executeHooks(this.postLaunchHooks, pageId, browserController);+ await this.executeHooks(this.postLaunchHooks, pageId, browserController);} catch (err) {this.startingBrowserControllers.delete(browserController);browserController.close().catch((closeErr) => {- this.log.error(`Could not close browser whose post-launch hooks failed.\nCause:${closeErr.message}`, {+ this.#log.error(`Could not close browser whose post-launch hooks failed.\nCause:${closeErr.message}`, {id: browserController.id,});});@@ -865,14 +866,14 @@ export class BrowserPool<* Picks plugins round robin.* @private*/- private _pickBrowserPlugin() {+ private pickBrowserPlugin() {const pluginIndex = this.pageCounter % this.browserPlugins.length;this.pageCounter++;return this.browserPlugins[pluginIndex];}- private _pickBrowserWithFreeCapacity(browserPlugin: BrowserPlugin, options?: { proxyUrl?: string }) {+ private pickBrowserWithFreeCapacity(browserPlugin: BrowserPlugin, options?: { proxyUrl?: string }) {return [...this.activeBrowserControllers].find((controller) => {const hasCapacity = controller.activePages < this.maxOpenPagesPerBrowser;const isCorrectPlugin = controller.browserPlugin === browserPlugin;@@ -888,7 +889,7 @@ export class BrowserPool<});}- private async _closeInactiveRetiredBrowsers() {+ private async closeInactiveRetiredBrowsers() {const closedBrowserIds: string[] = [];for (const controller of this.retiredBrowserControllers) {@@ -898,7 +899,7 @@ export class BrowserPool<if (isBrowserIdle || isBrowserEmpty) {const { id } = controller;- this.log.debug('Closing retired browser.', { id });+ this.#log.debug('Closing retired browser.', { id });await controller.close();this.retiredBrowserControllers.delete(controller);closedBrowserIds.push(id);@@ -906,46 +907,46 @@ export class BrowserPool<}if (closedBrowserIds.length) {- this.log.debug('Closed retired browsers.', {+ this.#log.debug('Closed retired browsers.', {count: closedBrowserIds.length,closedBrowserIds,});}}- private _overridePageClose(page: PageReturn) {+ private overridePageClose(page: PageReturn) {const originalPageClose = page.close;const browserController = this.pageToBrowserController.get(page)!;const pageId = this.getPageId(page)!;page.close = async (...args: unknown[]) => {- await this._executeHooks(this.prePageCloseHooks, page, browserController);+ await this.executeHooks(this.prePageCloseHooks, page, browserController);await originalPageClose.apply(page, args).catch((err: Error) => {- this.log.debug(`Could not close page.\nCause:${err.message}`, { id: browserController.id });+ this.#log.debug(`Could not close page.\nCause:${err.message}`, { id: browserController.id });});- await this._executeHooks(this.postPageCloseHooks, pageId, browserController);+ await this.executeHooks(this.postPageCloseHooks, pageId, browserController);this.pages.delete(pageId);- this._closeRetiredBrowserWithNoPages(browserController);+ this.closeRetiredBrowserWithNoPages(browserController);this.emit(BROWSER_POOL_EVENTS.PAGE_CLOSED, page);};}- private async _executeHooks(hooks: ((...args: any[]) => unknown)[], ...args: unknown[]) {+ private async executeHooks(hooks: ((...args: any[]) => unknown)[], ...args: unknown[]) {for (const hook of hooks) {await hook(...args);}}- private _closeRetiredBrowserWithNoPages(browserController: BrowserControllerReturn) {+ private closeRetiredBrowserWithNoPages(browserController: BrowserControllerReturn) {if (browserController.activePages === 0 && this.retiredBrowserControllers.has(browserController)) {// Run this with a delay, otherwise page.close()// might fail with "Protocol error (Target.closeTarget): Target closed."setTimeout(() => {- this.log.debug('Closing retired browser because it has no active pages', { id: browserController.id });+ this.#log.debug('Closing retired browser because it has no active pages', { id: browserController.id });void browserController.close().finally(() => {this.retiredBrowserControllers.delete(browserController);});@@ -975,7 +976,7 @@ export class BrowserPool<return false;}- private _initializeFingerprinting(): void {+ private initializeFingerprinting(): void {const { useFingerprintCache = true, fingerprintCacheSize = 10_000 } = this.fingerprintOptions;this.fingerprintGenerator = new FingerprintGenerator(this.fingerprintOptions.fingerprintGeneratorOptions);this.fingerprintInjector = new FingerprintInjector();@@ -984,10 +985,10 @@ export class BrowserPool<this.fingerprintCache = new QuickLRU({ maxSize: fingerprintCacheSize });}- this._addFingerprintHooks();+ this.addFingerprintHooks();}- private _addFingerprintHooks() {+ private addFingerprintHooks() {this.preLaunchHooks = [...this.preLaunchHooks,// This is flipped because of the fingerprint cache.diff --git a/packages/browser-pool/src/fingerprinting/hooks.ts b/packages/browser-pool/src/fingerprinting/hooks.tsindex 9a95752ad..ae823b43d 100644--- a/packages/browser-pool/src/fingerprinting/hooks.ts+++ b/packages/browser-pool/src/fingerprinting/hooks.ts@@ -53,7 +53,8 @@ export function createFingerprintPreLaunchHook(browserPool: BrowserPool<any, anyif (cacheKey) fingerprintCache?.set(cacheKey, fingerprint);}- launchContext.extend({ fingerprint });+ // `fingerprint` is a declared field, so it cannot go through `extend()` (which rejects reserved names)+ launchContext.fingerprint = fingerprint;if (useIncognitoPages) {return;diff --git a/packages/browser-pool/src/launch-context.ts b/packages/browser-pool/src/launch-context.tsindex 92f3e6b62..14f02e180 100644--- a/packages/browser-pool/src/launch-context.ts+++ b/packages/browser-pool/src/launch-context.ts@@ -80,8 +80,8 @@ export class LaunchContext<readonly isRemote: boolean;ignoreProxyCertificate?: boolean;- private _proxyUrl?: string;- private readonly _reservedFieldNames = [...Reflect.ownKeys(this), 'extend'];+ #proxyUrl?: string;+ readonly #reservedFieldNames: (string | symbol)[];fingerprint?: BrowserFingerprintWithHeaders;@@ -90,7 +90,7 @@ export class LaunchContext<* the {@apilink RemoteBrowserPool} to release the session on close. Only present for remote connections.* @internal*/- _remoteToken?: number;+ remoteToken?: number;[K: PropertyKey]: unknown;@@ -116,7 +116,11 @@ export class LaunchContext<this.ignoreProxyCertificate = ignoreProxyCertificate ?? false;this.isRemote = isRemote ?? false;- this._proxyUrl = proxyUrl;+ this.#proxyUrl = proxyUrl;++ // Computed here (not in a field initializer) so that all fields already exist; the accessors live on+ // the prototype, so they are never own keys and have to be listed explicitly.+ this.#reservedFieldNames = [...Reflect.ownKeys(this), 'proxyUrl', 'remoteToken', 'extend'];}/**@@ -128,7 +132,7 @@ export class LaunchContext<*/extend<T extends Record<PropertyKey, unknown>>(fields: T): void {Object.entries(fields).forEach(([key, value]) => {- if (this._reservedFieldNames.includes(key)) {+ if (this.#reservedFieldNames.includes(key)) {throw new Error(`Cannot extend LaunchContext with key: ${key}, because it's reserved.`);} else {Reflect.set(this, key, value);@@ -142,7 +146,7 @@ export class LaunchContext<*/set proxyUrl(url: string | undefined) {if (!url) {- this._proxyUrl = undefined;+ this.#proxyUrl = undefined;return;}@@ -152,13 +156,13 @@ export class LaunchContext<urlInstance.hash = '';// https://www.chromium.org/developers/design-documents/network-settings/#command-line-options-for-proxy-settings- this._proxyUrl = urlInstance.href.slice(0, -1);+ this.#proxyUrl = urlInstance.href.slice(0, -1);}/*** Returns the proxy URL of the browser.*/get proxyUrl(): string | undefined {- return this._proxyUrl;+ return this.#proxyUrl;}}diff --git a/packages/browser-pool/src/playwright/playwright-browser.ts b/packages/browser-pool/src/playwright/playwright-browser.tsindex c1e2c65b8..20cede4bd 100644--- a/packages/browser-pool/src/playwright/playwright-browser.ts+++ b/packages/browser-pool/src/playwright/playwright-browser.ts@@ -11,20 +11,20 @@ export interface BrowserOptions {* Browser wrapper created to have consistent API with persistent and non-persistent contexts.*/export class PlaywrightBrowser extends EventEmitter {- private _browserContext: BrowserContext;- private _version: string;- private _isConnected = true;- private _browserType?: BrowserType;+ #browserContext: BrowserContext;+ #version: string;+ #isConnected = true;+ #browserType?: BrowserType;constructor(options: BrowserOptions) {super();const { browserContext, version } = options;- this._browserContext = browserContext;- this._version = version;+ this.#browserContext = browserContext;+ this.#version = version;- this._browserContext.once('close', () => {- this._isConnected = false;+ this.#browserContext.once('close', () => {+ this.#isConnected = false;this.emit('disconnected');});}@@ -34,32 +34,32 @@ export class PlaywrightBrowser extends EventEmitter {}async close(): Promise<void> {- await this._browserContext.close();+ await this.#browserContext.close();}contexts(): BrowserContext[] {- return [this._browserContext];+ return [this.#browserContext];}isConnected(): boolean {- return this._isConnected;+ return this.#isConnected;}version(): string {- return this._version;+ return this.#version;}/** @internal */- _setBrowserType(browserType: BrowserType): void {- this._browserType = browserType;+ setBrowserType(browserType: BrowserType): void {+ this.#browserType = browserType;}browserType(): BrowserType {- return this._browserType!;+ return this.#browserType!;}async newPage(...args: Parameters<BrowserContext['newPage']>): ReturnType<BrowserContext['newPage']> {- return this._browserContext.newPage(...args);+ return this.#browserContext.newPage(...args);}async newContext(): Promise<never> {diff --git a/packages/browser-pool/src/playwright/playwright-plugin.ts b/packages/browser-pool/src/playwright/playwright-plugin.tsindex 064d747f7..26454f6ac 100644--- a/packages/browser-pool/src/playwright/playwright-plugin.ts+++ b/packages/browser-pool/src/playwright/playwright-plugin.ts@@ -16,7 +16,7 @@ export class PlaywrightPlugin extends BrowserPlugin<SafeParameters<BrowserType['launch']>[0],PlaywrightBrowser> {- private _browserVersion?: string;+ #browserVersion?: string;/*** Playwright remote connections only support incognito pages — `connect()` / `connectOverCDP()` don't@@ -36,7 +36,7 @@ export class PlaywrightPlugin extends BrowserPlugin<protected async _launch(launchContext: LaunchContext<BrowserType>): Promise<PlaywrightBrowser> {if (this.remoteConnection) {- return this._connectToRemoteBrowser(launchContext, async (url) => {+ return this.connectToRemoteBrowser(launchContext, async (url) => {const connectOptions = (this.remoteConnectionParameters?.connectOptions ?? {}) as any;if (this.remoteConnectionParameters?.protocol === 'playwright') {this.log.info('Connecting to remote browser via connect (Playwright WebSocket).');@@ -73,7 +73,7 @@ export class PlaywrightPlugin extends BrowserPlugin<try {if (useIncognitoPages) {browser = await this.library.launch(launchOptions).catch((error) => {- return this._throwOnFailedLaunch(launchContext, error);+ return this.throwOnFailedLaunch(launchContext, error);});if (anonymizedProxyUrl) {@@ -85,7 +85,7 @@ export class PlaywrightPlugin extends BrowserPlugin<const browserContext = await this.library.launchPersistentContext(userDataDir, launchOptions).catch((error) => {- return this._throwOnFailedLaunch(launchContext, error);+ return this.throwOnFailedLaunch(launchContext, error);});browserContext.once('close', () => {@@ -103,10 +103,10 @@ export class PlaywrightPlugin extends BrowserPlugin<});}- if (!this._browserVersion) {+ if (!this.#browserVersion) {// Launches unused browser just to get the browser version.const inactiveBrowser = await this.library.launch(launchOptions);- this._browserVersion = inactiveBrowser.version();+ this.#browserVersion = inactiveBrowser.version();inactiveBrowser.close().catch((error) => {this.log.exception(error, 'Failed to close browser.');@@ -115,7 +115,7 @@ export class PlaywrightPlugin extends BrowserPlugin<browser = new PlaywrightBrowserWithPersistentContext({browserContext,- version: this._browserVersion,+ version: this.#browserVersion,}) as unknown as PlaywrightBrowser;}} catch (error) {@@ -127,8 +127,8 @@ export class PlaywrightPlugin extends BrowserPlugin<return browser;}- private _throwOnFailedLaunch(launchContext: LaunchContext<BrowserType>, cause: unknown): never {- this._throwAugmentedLaunchError(+ private throwOnFailedLaunch(launchContext: LaunchContext<BrowserType>, cause: unknown): never {+ this.throwAugmentedLaunchError(cause,launchContext.launchOptions?.executablePath,'`apify/actor-node-playwright-*` (with a correct browser name)',@@ -140,7 +140,7 @@ export class PlaywrightPlugin extends BrowserPlugin<return new PlaywrightController(this as any);}- protected async _addProxyToLaunchOptions(launchContext: LaunchContext<BrowserType>): Promise<void> {+ protected async addProxyToLaunchOptions(launchContext: LaunchContext<BrowserType>): Promise<void> {launchContext.launchOptions ??= {};const { launchOptions, proxyUrl } = launchContext;@@ -156,7 +156,7 @@ export class PlaywrightPlugin extends BrowserPlugin<}}- protected _isChromiumBasedBrowser(): boolean {+ protected isChromiumBasedBrowser(): boolean {const name = this.library.name();return name === 'chromium';}diff --git a/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts b/packages/browser-pool/src/puppeteer/puppeteer-plugin.tsindex e2edf022c..1f8c29339 100644--- a/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts+++ b/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts@@ -58,7 +58,7 @@ export class PuppeteerPlugin extends BrowserPlugin<let browser: PuppeteerTypes.Browser;if (this.remoteConnection) {- browser = await this._connectToRemoteBrowser(launchContext, async (url) => {+ browser = await this.connectToRemoteBrowser(launchContext, async (url) => {const connectOptions = this.remoteConnectionParameters?.connectOptions ?? {};this.log.info('Connecting to remote browser via connect (CDP).');return this.library.connect({ ...connectOptions, browserWSEndpoint: url });@@ -110,7 +110,7 @@ export class PuppeteerPlugin extends BrowserPlugin<} catch (error: any) {await close();- this._throwAugmentedLaunchError(+ this.throwAugmentedLaunchError(error,launchContext.launchOptions?.executablePath,'`apify/actor-node-puppeteer-chrome`',@@ -238,7 +238,7 @@ export class PuppeteerPlugin extends BrowserPlugin<return new PuppeteerController(this);}- protected async _addProxyToLaunchOptions(+ protected async addProxyToLaunchOptions(_launchContext: LaunchContext<typeof Puppeteer,PuppeteerTypes.LaunchOptions,@@ -273,7 +273,7 @@ export class PuppeteerPlugin extends BrowserPlugin<*/}- protected _isChromiumBasedBrowser(+ protected isChromiumBasedBrowser(_launchContext: LaunchContext<typeof Puppeteer,PuppeteerTypes.LaunchOptions,diff --git a/packages/browser-pool/src/remote-browser-pool.ts b/packages/browser-pool/src/remote-browser-pool.tsindex eef70fa19..d07478dac 100644--- a/packages/browser-pool/src/remote-browser-pool.ts+++ b/packages/browser-pool/src/remote-browser-pool.ts@@ -53,22 +53,25 @@ export interface RemoteConnection {* {@apilink RemoteConnection} so it can be injected into a plugin.*/class RemoteSessionRegistry implements RemoteConnection {- private readonly sessions = new Map<- number,- { url: string; context?: Record<string, unknown>; released: boolean }- >();- private nextToken = 0;+ readonly #sessions = new Map<number, { url: string; context?: Record<string, unknown>; released: boolean }>();+ #nextToken = 0;++ readonly #endpoint: RemoteBrowserEndpoint;+ readonly #onRelease: ((info: { endpoint: string; context?: Record<string, unknown> }) => unknown) | undefined;+ readonly #log: CrawleeLogger;constructor(- private readonly endpoint: RemoteBrowserEndpoint,- private readonly onRelease:- | ((info: { endpoint: string; context?: Record<string, unknown> }) => unknown)- | undefined,- private readonly log: CrawleeLogger,- ) {}+ endpoint: RemoteBrowserEndpoint,+ onRelease: ((info: { endpoint: string; context?: Record<string, unknown> }) => unknown) | undefined,+ log: CrawleeLogger,+ ) {+ this.#endpoint = endpoint;+ this.#onRelease = onRelease;+ this.#log = log;+ }async resolve(options?: { proxyUrl?: string }): Promise<{ url: string; token: number }> {- const resolved = typeof this.endpoint === 'function' ? await this.endpoint(options) : this.endpoint;+ const resolved = typeof this.#endpoint === 'function' ? await this.#endpoint(options) : this.#endpoint;let result: ResolvedRemoteEndpoint;if (typeof resolved === 'string') {@@ -80,30 +83,30 @@ class RemoteSessionRegistry implements RemoteConnection {result = resolved;}- const token = this.nextToken++;- this.sessions.set(token, { url: result.url, context: result.context, released: false });+ const token = this.#nextToken++;+ this.#sessions.set(token, { url: result.url, context: result.context, released: false });return { url: result.url, token };}async release(token: number): Promise<void> {- const session = this.sessions.get(token);+ const session = this.#sessions.get(token);// Release at most once per session — guards a close()/teardown race (the `released` flag is set// synchronously before the awaited onRelease, so releaseAll() can't double-fire an in-flight release).if (!session || session.released) return;session.released = true;try {- await this.onRelease?.({ endpoint: session.url, context: session.context });+ await this.#onRelease?.({ endpoint: session.url, context: session.context });} catch (err) {- this.log.warning('Remote browser release() failed.', { error: (err as Error)?.message });+ this.#log.warning('Remote browser release() failed.', { error: (err as Error)?.message });} finally {- this.sessions.delete(token);+ this.#sessions.delete(token);}}/** Releases every session that is still open. Called on pool teardown so no remote session leaks. */async releaseAll(): Promise<void> {- await Promise.all([...this.sessions.keys()].map(async (token) => this.release(token)));+ await Promise.all([...this.#sessions.keys()].map(async (token) => this.release(token)));}}@@ -199,14 +202,14 @@ export class RemoteBrowserPool<Page = unknown> implements IBrowserPool<Page> {readonly browserPool: BrowserPool;/** The wrapped pool viewed through the {@apilink IBrowserPool} contract (the bare type widens pages to `never`). */- private readonly pool: IBrowserPool<Page>;+ readonly #pool: IBrowserPool<Page>;- private readonly registry: RemoteSessionRegistry;- private readonly slotPollIntervalMillis: number;- private readonly log: CrawleeLogger;+ readonly #registry: RemoteSessionRegistry;+ readonly #slotPollIntervalMillis: number;+ readonly #log: CrawleeLogger;/** Shared by all `newPage` callers waiting for a free slot, so they don't each register their own listeners. */- private _capacityChange?: Promise<void>;+ #capacityChange?: Promise<void>;constructor(options: RemoteBrowserPoolOptions) {const {@@ -219,8 +222,8 @@ export class RemoteBrowserPool<Page = unknown> implements IBrowserPool<Page> {slotPollIntervalMillis = 500,} = options;- this.log = serviceLocator.getLogger().child({ prefix: 'RemoteBrowserPool' });- this.slotPollIntervalMillis = slotPollIntervalMillis;+ this.#log = serviceLocator.getLogger().child({ prefix: 'RemoteBrowserPool' });+ this.#slotPollIntervalMillis = slotPollIntervalMillis;// A RemoteBrowserProvider carries its own endpoint, release, and maxOpenBrowsers.const provider = endpoint instanceof RemoteBrowserProvider ? endpoint : undefined;@@ -232,22 +235,22 @@ export class RemoteBrowserPool<Page = unknown> implements IBrowserPool<Page> {: release;const resolvedMax = maxOpenBrowsers ?? provider?.maxOpenBrowsers;- this.registry = new RemoteSessionRegistry(resolvedEndpoint, resolvedRelease, this.log);+ this.#registry = new RemoteSessionRegistry(resolvedEndpoint, resolvedRelease, this.#log);// Wire every plugin for remote connection.for (const plugin of browserPlugins) {- plugin.useRemoteConnection(this.registry, connection);+ plugin.useRemoteConnection(this.#registry, connection);}this.browserPool = new BrowserPool({ ...browserPoolOptions, browserPlugins }) as unknown as BrowserPool;- this.pool = this.browserPool as unknown as IBrowserPool<Page>;+ this.#pool = this.browserPool as unknown as IBrowserPool<Page>;// Release a browser's remote session once it closes. The registry dedupes (close() schedules a delayed// kill(), so BROWSER_CLOSED can fire twice), and destroy()'s releaseAll() backstops any that never close.this.browserPool.on(BROWSER_POOL_EVENTS.BROWSER_LAUNCHED, (controller: BrowserController) => {controller.once(BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED, () => {- const token = controller.launchContext._remoteToken;- if (token !== undefined) void this.registry.release(token);+ const token = controller.launchContext.remoteToken;+ if (token !== undefined) void this.#registry.release(token);});});@@ -270,33 +273,33 @@ export class RemoteBrowserPool<Page = unknown> implements IBrowserPool<Page> {* allows it (either a new browser slot is free, or an active browser still has page capacity).*/async newPage(options?: NewPageOptions): Promise<Page> {- await this._waitForFreeSlot();- return this.pool.newPage(options);+ await this.waitForFreeSlot();+ return this.#pool.newPage(options);}async closePage(page: Page, options?: { error?: Error }): Promise<void> {- return this.pool.closePage(page, options);+ return this.#pool.closePage(page, options);}async extractPageState(page: Page): Promise<PageState> {- return this.pool.extractPageState(page);+ return this.#pool.extractPageState(page);}async injectPageState(page: Page, state: PageState): Promise<void> {- return this.pool.injectPageState(page, state);+ return this.#pool.injectPageState(page, state);}/** Closes all browsers, releases any still-open remote sessions, and tears down the wrapped pool. */async destroy(): Promise<void> {await this.browserPool.destroy();// Backstop: release any sessions whose browser never emitted a close (e.g. dropped on teardown).- await this.registry.releaseAll();+ await this.#registry.releaseAll();}/** Resolves once the wrapped pool can serve another page without exceeding `maxOpenBrowsers`. */- private async _waitForFreeSlot(): Promise<void> {+ private async waitForFreeSlot(): Promise<void> {while (!this.browserPool.hasFreeBrowserSlot() && !this.browserPool.hasActiveBrowserWithFreeCapacity()) {- await this._nextCapacityChange();+ await this.nextCapacityChange();}}@@ -305,22 +308,22 @@ export class RemoteBrowserPool<Page = unknown> implements IBrowserPool<Page> {* concurrently-waiting `newPage` calls share a single promise (and a single pair of event listeners)* per tick, so a fleet of saturated callers doesn't fan out into N listener pairs on the pool.*/- private _nextCapacityChange(): Promise<void> {- this._capacityChange ??= new Promise<void>((resolve) => {+ private nextCapacityChange(): Promise<void> {+ this.#capacityChange ??= new Promise<void>((resolve) => {const done = () => {clearTimeout(timer);this.browserPool.off(BROWSER_POOL_EVENTS.BROWSER_RETIRED, done);this.browserPool.off(BROWSER_POOL_EVENTS.PAGE_CLOSED, done);- this._capacityChange = undefined;+ this.#capacityChange = undefined;resolve();};- const timer = setTimeout(done, this.slotPollIntervalMillis);+ const timer = setTimeout(done, this.#slotPollIntervalMillis);timer.unref?.();this.browserPool.once(BROWSER_POOL_EVENTS.BROWSER_RETIRED, done);this.browserPool.once(BROWSER_POOL_EVENTS.PAGE_CLOSED, done);});- return this._capacityChange;+ return this.#capacityChange;}}diff --git a/packages/browser-pool/test/remote-browser-pool.test.ts b/packages/browser-pool/test/remote-browser-pool.test.tsindex 32cdbe676..d02e9391a 100644--- a/packages/browser-pool/test/remote-browser-pool.test.ts+++ b/packages/browser-pool/test/remote-browser-pool.test.ts@@ -117,7 +117,7 @@ describe('RemoteBrowserPool — release lifecycle', () => {// Mimic the inner pool launching a controller, then that controller closing.const controller: any = new EventEmitter();- controller.launchContext = { _remoteToken: token };+ controller.launchContext = { remoteToken: token };pool.browserPool.emit(BROWSER_POOL_EVENTS.BROWSER_LAUNCHED, controller);controller.emit(BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED, controller);diff --git a/packages/browser-pool/test/remote-browser.test.ts b/packages/browser-pool/test/remote-browser.test.tsindex cadacf7e9..abeaad02c 100644--- a/packages/browser-pool/test/remote-browser.test.ts+++ b/packages/browser-pool/test/remote-browser.test.ts@@ -129,7 +129,7 @@ describe('Remote connection — PlaywrightPlugin', () => {expect(lib.connectOverCDP).toHaveBeenCalledWith('http://remote:9222', { timeout: 5000 });expect(lib.connect).not.toHaveBeenCalled();expect(lib.launch).not.toHaveBeenCalled();- expect(ctx._remoteToken).toBe(42);+ expect(ctx.remoteToken).toBe(42);});it("connects via connect() when protocol is 'playwright'", async () => {@@ -195,7 +195,7 @@ describe('Remote connection — PuppeteerPlugin', () => {expect(connection.resolve).toHaveBeenCalledTimes(1);expect(lib.connect).toHaveBeenCalledWith({ protocolTimeout: 1000, browserWSEndpoint: 'ws://remote:9222' });expect(lib.launch).not.toHaveBeenCalled();- expect(ctx._remoteToken).toBe(42);+ expect(ctx.remoteToken).toBe(42);});it('releases the session and throws BrowserLaunchError when connect fails', async () => {diff --git a/packages/core/src/autoscaling/autoscaled_pool.ts b/packages/core/src/autoscaling/autoscaled_pool.tsindex d86f61bce..63915cf27 100644--- a/packages/core/src/autoscaling/autoscaled_pool.ts+++ b/packages/core/src/autoscaling/autoscaled_pool.ts@@ -124,31 +124,31 @@ export interface AutoscaledPoolOptions extends TaskLoopPredicates {* @internal*/export class AutoscaledPool {- private readonly log: CrawleeLogger;+ readonly #log: CrawleeLogger;// Configurable properties.- private readonly maybeRunIntervalMillis: number;- private readonly taskTimeoutMillis: number;- private readonly runTaskFunction: () => Promise<unknown>;- private readonly isFinishedFunction: () => Promise<boolean>;- private readonly isTaskReadyFunction: () => Promise<boolean>;+ readonly #maybeRunIntervalMillis: number;+ readonly #taskTimeoutMillis: number;+ readonly #runTaskFunction: () => Promise<unknown>;+ readonly #isFinishedFunction: () => Promise<boolean>;+ readonly #isTaskReadyFunction: () => Promise<boolean>;- private readonly concurrencySystem: IConcurrencySystem;- private readonly consumer: ConcurrencyConsumer;+ readonly #concurrencySystem: IConcurrencySystem;+ readonly #consumer: ConcurrencyConsumer;// Internal properties.- private isStopped = false;- private resolve: ((val?: unknown) => void) | null = null;- private reject: ((reason?: unknown) => void) | null = null;- private maybeRunInterval!: BetterIntervalID;- private queryingIsTaskReady!: boolean;- private queryingIsFinished!: boolean;+ #isStopped = false;+ #resolve: ((val?: unknown) => void) | null = null;+ #reject: ((reason?: unknown) => void) | null = null;+ #maybeRunInterval!: BetterIntervalID;+ #queryingIsTaskReady!: boolean;+ #queryingIsFinished!: boolean;/*** This pool's *own* in-flight task count, as opposed to {@apilink AutoscaledPool.currentConcurrency}, which is the* (possibly shared) governor's total. `pause()` and `maybeFinish()` care only about this pool draining.*/- private ownConcurrency = 0;+ #ownConcurrency = 0;constructor(options: AutoscaledPoolOptions) {ow(@@ -176,22 +176,22 @@ export class AutoscaledPool {consumer,} = options;- this.log = log.child({ prefix: 'AutoscaledPool' });+ this.#log = log.child({ prefix: 'AutoscaledPool' });// Configurable properties.- this.maybeRunIntervalMillis = maybeRunIntervalSecs * 1000;- this.taskTimeoutMillis = taskTimeoutSecs * 1000;- this.runTaskFunction = runTaskFunction;- this.isFinishedFunction = isFinishedFunction;- this.isTaskReadyFunction = isTaskReadyFunction;+ this.#maybeRunIntervalMillis = maybeRunIntervalSecs * 1000;+ this.#taskTimeoutMillis = taskTimeoutSecs * 1000;+ this.#runTaskFunction = runTaskFunction;+ this.#isFinishedFunction = isFinishedFunction;+ this.#isTaskReadyFunction = isTaskReadyFunction;- this.concurrencySystem = concurrencySystem;- this.consumer = consumer;+ this.#concurrencySystem = concurrencySystem;+ this.#consumer = consumer;// Internal properties.- this.isStopped = false;- this.resolve = null;- this.reject = null;+ this.#isStopped = false;+ this.#resolve = null;+ this.#reject = null;this.maybeRunTask = this.maybeRunTask.bind(this);}@@ -203,12 +203,12 @@ export class AutoscaledPool {* {@apilink ConcurrencySystem} its owner holds, never through the pool.*/get system(): IConcurrencySystem {- return this.concurrencySystem;+ return this.#concurrencySystem;}/** The estimated number of parallel tasks the governor can currently support. */get desiredConcurrency(): number {- return this.concurrencySystem.desiredConcurrency;+ return this.#concurrencySystem.desiredConcurrency;}/**@@ -216,7 +216,7 @@ export class AutoscaledPool {* borrowing pool's tasks, not just this one's.*/get currentConcurrency(): number {- return this.concurrencySystem.currentConcurrency;+ return this.#concurrencySystem.currentConcurrency;}/**@@ -229,7 +229,7 @@ export class AutoscaledPool {async run(): Promise<void> {// Checked here, on an awaited path — the capacity queries inside the task loop run from intervals and// `setImmediate`, where a throw would become an unhandled rejection and hang `run()` forever.- if (!this.concurrencySystem.isRunning) {+ if (!this.#concurrencySystem.isRunning) {throw new CriticalError('The ConcurrencySystem this AutoscaledPool borrows has not been started, so system load would not be ' +'monitored and the concurrency would never be adjusted. Whoever creates a ConcurrencySystem owns ' +@@ -239,20 +239,20 @@ export class AutoscaledPool {}const poolPromise = new Promise((resolve, reject) => {- this.resolve = resolve;- this.reject = reject;+ this.#resolve = resolve;+ this.#reject = reject;});// This is here because if we scale down to let's say 1, then after each promise is finished// this.maybeRunTask() doesn't trigger another one. So if that 1 instance gets stuck it results// in the crawler getting stuck and even after scaling up it never triggers another promise.- this.maybeRunInterval = betterSetInterval(this.maybeRunTask, this.maybeRunIntervalMillis);+ this.#maybeRunInterval = betterSetInterval(this.maybeRunTask, this.#maybeRunIntervalMillis);try {await poolPromise;} finally {// If resolve is null, the pool is already destroyed.- if (this.resolve) await this.destroy();+ if (this.#resolve) await this.destroy();}}@@ -268,9 +268,9 @@ export class AutoscaledPool {* parts of their asynchronous chains of commands will not execute.*/async abort(): Promise<void> {- this.isStopped = true;- if (this.resolve) {- this.resolve();+ this.#isStopped = true;+ if (this.#resolve) {+ this.#resolve();await this.destroy();}}@@ -291,8 +291,8 @@ export class AutoscaledPool {* it during a long pause, its owner can `stop()` and `start()` it again.*/async pause(timeoutSecs?: number): Promise<void> {- if (this.isStopped) return;- this.isStopped = true;+ if (this.#isStopped) return;+ this.#isStopped = true;await new Promise<void>((resolve, reject) => {let timeout: NodeJS.Timeout;let interval: NodeJS.Timeout;@@ -309,13 +309,13 @@ export class AutoscaledPool {}interval = setInterval(() => {- if (this.ownConcurrency <= 0) {+ if (this.#ownConcurrency <= 0) {// Clean up timeout and interval to prevent process hanging.if (timeout) clearTimeout(timeout);clearInterval(interval);resolve();}- }, this.maybeRunIntervalMillis);+ }, this.#maybeRunIntervalMillis);});}@@ -326,7 +326,7 @@ export class AutoscaledPool {* Tasks will automatically start running again in `options.maybeRunIntervalSecs`.*/resume(): void {- this.isStopped = false;+ this.#isStopped = false;}/**@@ -341,28 +341,28 @@ export class AutoscaledPool {* Starts a new task* if the number of running tasks (current concurrency) is lower than desired concurrency* and the system is not currently overloaded- * and this.isTaskReadyFunction() returns true.+ * and `isTaskReadyFunction()` returns true.** It doesn't allow multiple concurrent runs of this method.*/private async maybeRunTask(intervalCallback?: () => void): Promise<void> {- this.log.perf('Attempting to run a task.');+ this.#log.perf('Attempting to run a task.');// Check if the function was invoked by the maybeRunInterval and use an empty function if not.const done = intervalCallback || (() => {});// Prevent starting a new task if:// - the pool is paused or aborted- if (this.isStopped) {- this.log.perf('Task will not run. AutoscaledPool is stopped.');+ if (this.#isStopped) {+ this.#log.perf('Task will not run. AutoscaledPool is stopped.');return done();}// - we are already querying for a task.- if (this.queryingIsTaskReady) {- this.log.perf('Task will not run. Waiting for a ready task.');+ if (this.#queryingIsTaskReady) {+ this.#log.perf('Task will not run. Waiting for a ready task.');return done();}// - the budget has room for us.- if (!this.concurrencySystem.hasCapacityForTask(this.consumer)) {+ if (!this.#concurrencySystem.hasCapacityForTask(this.#consumer)) {done();// A shared governor's budget can stay saturated by another pool indefinitely, so we still have to be able// to notice that *this* pool has run out of work — `maybeFinish()` is the only thing that ever resolves@@ -371,25 +371,25 @@ export class AutoscaledPool {return this.maybeFinish();}// - a task is ready.- this.queryingIsTaskReady = true;+ this.#queryingIsTaskReady = true;let isTaskReady;try {- this.log.perf('Checking for ready tasks.');- isTaskReady = await this.isTaskReadyFunction();+ this.#log.perf('Checking for ready tasks.');+ isTaskReady = await this.#isTaskReadyFunction();} catch (e) {const err = e as Error;- this.log.perf('Checking for ready tasks failed.');+ this.#log.perf('Checking for ready tasks failed.');// We might have already rejected this promise.- if (this.reject) {+ if (this.#reject) {// No need to log all concurrent errors.- this.log.exception(err, 'isTaskReadyFunction failed');- this.reject(err);+ this.#log.exception(err, 'isTaskReadyFunction failed');+ this.#reject(err);}} finally {- this.queryingIsTaskReady = false;+ this.#queryingIsTaskReady = false;}if (!isTaskReady) {- this.log.perf('Task will not run. No tasks are ready.');+ this.#log.perf('Task will not run. No tasks are ready.');done();// No tasks could mean that we're finished with all tasks.return this.maybeFinish();@@ -397,11 +397,11 @@ export class AutoscaledPool {// - the budget still has room. Re-checked atomically, because another pool sharing the governor may have taken// the last free slot while we awaited `isTaskReadyFunction` above.- if (!this.concurrencySystem.tryRegisterTaskStart(this.consumer)) {+ if (!this.#concurrencySystem.tryRegisterTaskStart(this.#consumer)) {return done();}- this.ownConcurrency++;+ this.#ownConcurrency++;try {// Everything's fine. Run task.@@ -413,67 +413,67 @@ export class AutoscaledPool {done();// Execute the current task.- this.log.perf('Running a task.');+ this.#log.perf('Running a task.');- if (this.taskTimeoutMillis > 0) {+ if (this.#taskTimeoutMillis > 0) {await addTimeoutToPromise(- async () => this.runTaskFunction(),- this.taskTimeoutMillis,- `runTaskFunction timed out after ${this.taskTimeoutMillis / 1000} seconds.`,+ async () => this.#runTaskFunction(),+ this.#taskTimeoutMillis,+ `runTaskFunction timed out after ${this.#taskTimeoutMillis / 1000} seconds.`,);} else {- await this.runTaskFunction();+ await this.#runTaskFunction();}- this.log.perf('Task finished.');+ this.#log.perf('Task finished.');// Run task after the previous one finished. Only on success: a failed task rejects the pool, and// nudging the loop afterwards could start work on an already destroyed pool.setImmediate(this.maybeRunTask);} catch (e) {const err = e as Error;- this.log.perf('Running a task failed.');+ this.#log.perf('Running a task failed.');// We might have already rejected this promise.- if (this.reject) {+ if (this.#reject) {// No need to log all concurrent errors.if (// avoid reprinting the same critical error multiple times, as it will be printed by Nodejs at the end anyway!(e instanceof CriticalError)) {- this.log.exception(err, 'runTaskFunction failed.');+ this.#log.exception(err, 'runTaskFunction failed.');}- this.reject(err);+ this.#reject(err);}} finally {- this.concurrencySystem.registerTaskEnd(this.consumer);- this.ownConcurrency--;+ this.#concurrencySystem.registerTaskEnd(this.#consumer);+ this.#ownConcurrency--;}return undefined;}/**- * If there are no running tasks and this.isFinishedFunction() returns true then closes+ * If there are no running tasks and `isFinishedFunction()` returns true then closes* the pool and resolves the pool's promise returned by the run() method.** It doesn't allow multiple concurrent runs of this method.*/private async maybeFinish(): Promise<void> {- if (this.queryingIsFinished) return;- if (this.ownConcurrency > 0) return;+ if (this.#queryingIsFinished) return;+ if (this.#ownConcurrency > 0) return;- this.queryingIsFinished = true;+ this.#queryingIsFinished = true;try {- const isFinished = await this.isFinishedFunction();- if (isFinished && this.resolve) this.resolve();+ const isFinished = await this.#isFinishedFunction();+ if (isFinished && this.#resolve) this.#resolve();} catch (e) {const err = e as Error;- if (this.reject) {+ if (this.#reject) {// No need to log all concurrent errors.- this.log.exception(err, 'isFinishedFunction failed.');- this.reject(err);+ this.#log.exception(err, 'isFinishedFunction failed.');+ this.#reject(err);}} finally {- this.queryingIsFinished = false;+ this.#queryingIsFinished = false;}}@@ -481,9 +481,9 @@ export class AutoscaledPool {* Cleans up resources.*/private async destroy(): Promise<void> {- this.resolve = null;- this.reject = null;+ this.#resolve = null;+ this.#reject = null;- betterClearInterval(this.maybeRunInterval);+ betterClearInterval(this.#maybeRunInterval);}}diff --git a/packages/core/src/autoscaling/client_load_signal.ts b/packages/core/src/autoscaling/client_load_signal.tsindex 1dd21c5d9..c30d028bd 100644--- a/packages/core/src/autoscaling/client_load_signal.ts+++ b/packages/core/src/autoscaling/client_load_signal.ts@@ -56,39 +56,39 @@ export class ClientLoadSignal implements LoadSignal {readonly name = 'clientInfo';readonly overloadedRatio: number;- private readonly store = new SnapshotStore<ClientSnapshot>();- private readonly intervalMillis: number;- private readonly maxErrors: number;- private interval?: BetterIntervalID;- private client?: StorageBackend;+ readonly #store = new SnapshotStore<ClientSnapshot>();+ readonly #intervalMillis: number;+ readonly #maxErrors: number;+ #interval?: BetterIntervalID;+ #client?: StorageBackend;constructor(options: ClientLoadSignalOptions = {}) {this.overloadedRatio = options.overloadedRatio ?? 0.3;- this.intervalMillis = (options.snapshotIntervalSecs ?? 1) * 1000;- this.maxErrors = options.maxErrors ?? 3;+ this.#intervalMillis = (options.snapshotIntervalSecs ?? 1) * 1000;+ this.#maxErrors = options.maxErrors ?? 3;this.handle = this.handle.bind(this);}async start(context: LoadSignalStartContext): Promise<void> {- this.store.useSampleWindow(context.maxSampleWindowMillis);+ this.#store.useSampleWindow(context.maxSampleWindowMillis);// A new session starts from a clean slate, or its first measurement diffs the error count against the previous// session's — possibly against a different backend, since the client is resolved afresh just below.- this.store.clear();+ this.#store.clear();// Resolved here rather than in the constructor, where asking for the backend would instantiate a default one// as a side effect - long before the crawler that owns the run has had a chance to register its own.- this.client = serviceLocator.getStorageBackend();- this.interval = betterSetInterval(this.handle, this.intervalMillis);+ this.#client = serviceLocator.getStorageBackend();+ this.#interval = betterSetInterval(this.handle, this.#intervalMillis);}async stop(): Promise<void> {- if (this.interval) betterClearInterval(this.interval);- this.interval = undefined;- this.client = undefined;+ if (this.#interval) betterClearInterval(this.#interval);+ this.#interval = undefined;+ this.#client = undefined;}getSample(sampleDurationMillis?: number): LoadSnapshot[] {- return this.store.getSample(sampleDurationMillis);+ return this.#store.getSample(sampleDurationMillis);}/**@@ -99,7 +99,7 @@ export class ClientLoadSignal implements LoadSignal {handle(intervalCallback: () => unknown): void {const now = new Date();- const allErrorCounts = this.client?.stats?.rateLimitErrors ?? [];+ const allErrorCounts = this.#client?.stats?.rateLimitErrors ?? [];const currentErrCount = allErrorCounts[CLIENT_RATE_LIMIT_ERROR_RETRY_COUNT] || 0;const snapshot: ClientSnapshot = {@@ -108,15 +108,15 @@ export class ClientLoadSignal implements LoadSignal {rateLimitErrorCount: currentErrCount,};- const all = this.store.getAll();+ const all = this.#store.getAll();const previousSnapshot = all[all.length - 1];if (previousSnapshot) {const delta = currentErrCount - previousSnapshot.rateLimitErrorCount;- if (delta > this.maxErrors) snapshot.isOverloaded = true;+ if (delta > this.#maxErrors) snapshot.isOverloaded = true;}- this.store.push(snapshot, now);+ this.#store.push(snapshot, now);intervalCallback();}}diff --git a/packages/core/src/autoscaling/concurrency_system.ts b/packages/core/src/autoscaling/concurrency_system.tsindex 0dd676a5c..a89031fd8 100644--- a/packages/core/src/autoscaling/concurrency_system.ts+++ b/packages/core/src/autoscaling/concurrency_system.ts@@ -187,32 +187,33 @@ export class ConcurrencySystem implements IConcurrencySystem {private readonly desiredConcurrencyRatio: number;private readonly scaleUpStepRatio: number;private readonly scaleDownStepRatio: number;- private readonly loggingIntervalMillis: number;- private readonly autoscaleIntervalMillis: number;+ readonly #loggingIntervalMillis: number;+ readonly #autoscaleIntervalMillis: number;private readonly maxTasksPerMinute: number;- private _minConcurrency: number;- private _maxConcurrency: number;- private _desiredConcurrency: number;+ #minConcurrency: number;+ #maxConcurrency: number;+ #desiredConcurrency: number;+ // kept as TS-private _-prefixed: autoscaled_pool tests write this backing field directlyprivate _currentConcurrency = 0;- private lastLoggingTime?: number;- private _tasksPerMinute: number[] = Array.from({ length: 60 }, () => 0);+ #lastLoggingTime?: number;+ #tasksPerMinute: number[] = Array.from({ length: 60 }, () => 0);private readonly snapshotter: Snapshotter;- private readonly loadSignals: LoadSignal[];+ readonly #loadSignals: LoadSignal[];private readonly systemStatus: SystemStatus;- private autoscaleInterval?: BetterIntervalID;- private tasksDonePerSecondInterval?: BetterIntervalID;+ #autoscaleInterval?: BetterIntervalID;+ #tasksDonePerSecondInterval?: BetterIntervalID;/** Whether the snapshotter and autoscaling intervals are currently running. */- private running = false;+ #running = false;/** The in-flight (or completed) startup, memoized so concurrent `start()` calls await one boot. */- private startPromise?: Promise<void>;+ #startPromise?: Promise<void>;/** Set once per session, so a pool outliving `stop()` is reported once rather than every half second. */- private warnedAboutQueryWhileStopped = false;+ #warnedAboutQueryWhileStopped = false;constructor(options: ConcurrencySystemOptions = {}) {ow(@@ -255,23 +256,23 @@ export class ConcurrencySystem implements IConcurrencySystem {this.desiredConcurrencyRatio = desiredConcurrencyRatio;this.scaleUpStepRatio = scaleUpStepRatio;this.scaleDownStepRatio = scaleDownStepRatio;- this.loggingIntervalMillis = (loggingIntervalSecs ?? 0) * 1000;- this.autoscaleIntervalMillis = autoscaleIntervalSecs * 1000;+ this.#loggingIntervalMillis = (loggingIntervalSecs ?? 0) * 1000;+ this.#autoscaleIntervalMillis = autoscaleIntervalSecs * 1000;this.maxTasksPerMinute = maxTasksPerMinute;- this._minConcurrency = minConcurrency;- this._maxConcurrency = maxConcurrency;- this._desiredConcurrency = desiredConcurrency ?? minConcurrency;+ this.#minConcurrency = minConcurrency;+ this.#maxConcurrency = maxConcurrency;+ this.#desiredConcurrency = desiredConcurrency ?? minConcurrency;this.clampDesiredConcurrency();- this._autoscale = this._autoscale.bind(this);- this._incrementTasksDonePerSecond = this._incrementTasksDonePerSecond.bind(this);+ this.autoscale = this.autoscale.bind(this);+ this.incrementTasksDonePerSecond = this.incrementTasksDonePerSecond.bind(this);// The built-in signals are collected by the snapshotter; custom ones are simply evaluated alongside them.const { custom: customLoadSignals = [], ...builtinSignalOptions } = loadSignals;this.snapshotter = new Snapshotter(builtinSignalOptions);- this.loadSignals = customLoadSignals;+ this.#loadSignals = customLoadSignals;this.systemStatus = new SystemStatus({snapshotter: this.snapshotter,loadSignals: customLoadSignals,@@ -286,7 +287,7 @@ export class ConcurrencySystem implements IConcurrencySystem {* Gets the minimum number of tasks running in parallel.*/get minConcurrency(): number {- return this._minConcurrency;+ return this.#minConcurrency;}/**@@ -297,7 +298,7 @@ export class ConcurrencySystem implements IConcurrencySystem {*/set minConcurrency(value: number) {ow(value, ow.optional.number.integer.greaterThanOrEqual(1));- this._minConcurrency = value;+ this.#minConcurrency = value;this.clampDesiredConcurrency();}@@ -305,7 +306,7 @@ export class ConcurrencySystem implements IConcurrencySystem {* Gets the maximum number of tasks running in parallel.*/get maxConcurrency(): number {- return this._maxConcurrency;+ return this.#maxConcurrency;}/**@@ -316,7 +317,7 @@ export class ConcurrencySystem implements IConcurrencySystem {*/set maxConcurrency(value: number) {ow(value, ow.optional.number.integer.greaterThanOrEqual(1));- this._maxConcurrency = value;+ this.#maxConcurrency = value;this.clampDesiredConcurrency();}@@ -325,7 +326,7 @@ export class ConcurrencySystem implements IConcurrencySystem {* which is an estimated number of parallel tasks that the system can currently support.*/get desiredConcurrency(): number {- return this._desiredConcurrency;+ return this.#desiredConcurrency;}/**@@ -334,7 +335,7 @@ export class ConcurrencySystem implements IConcurrencySystem {*/set desiredConcurrency(value: number) {ow(value, ow.optional.number.integer.greaterThanOrEqual(1));- this._desiredConcurrency = value;+ this.#desiredConcurrency = value;this.clampDesiredConcurrency();}@@ -345,8 +346,8 @@ export class ConcurrencySystem implements IConcurrencySystem {* that is the limit callers set in order to protect something.*/private clampDesiredConcurrency(): void {- const atLeastMin = Math.max(this._desiredConcurrency, this._minConcurrency);- this._desiredConcurrency = Math.min(atLeastMin, this._maxConcurrency);+ const atLeastMin = Math.max(this.#desiredConcurrency, this.#minConcurrency);+ this.#desiredConcurrency = Math.min(atLeastMin, this.#maxConcurrency);}get currentConcurrency(): number {@@ -355,7 +356,7 @@ export class ConcurrencySystem implements IConcurrencySystem {/** Whether the system is currently monitoring load and autoscaling the budget. */get isRunning(): boolean {- return this.running;+ return this.#running;}/**@@ -366,60 +367,60 @@ export class ConcurrencySystem implements IConcurrencySystem {async start(): Promise<void> {// Unwound and dropped again on failure, so a later `start()` retries instead of resolving instantly against// a system that is down.- this.startPromise ??= this.boot().catch(async (error) => {- this.startPromise = undefined;+ this.#startPromise ??= this.boot().catch(async (error) => {+ this.#startPromise = undefined;await this.shutDown();throw error;});- await this.startPromise;+ await this.#startPromise;}private async boot(): Promise<void> {// Per-session measurement state, reset so a restarted system isn't judged on the previous session. The// per-minute window matters most: its ageing interval is cleared while we are down, so starts from before an// arbitrarily long stop would otherwise still count against "this minute" and trip the cap immediately.- this._tasksPerMinute = Array.from({ length: 60 }, () => 0);- this.lastLoggingTime = undefined;- this.warnedAboutQueryWhileStopped = false;+ this.#tasksPerMinute = Array.from({ length: 60 }, () => 0);+ this.#lastLoggingTime = undefined;+ this.#warnedAboutQueryWhileStopped = false;// Signals are told how much history to keep when they start: exactly the longest window they will be sampled// over, so nobody has to guess a retention value that matches this system's configuration.const startContext = { maxSampleWindowMillis: this.systemStatus.maxSampleWindowMillis };await this.snapshotter.start(startContext);- await Promise.all(this.loadSignals.map(async (s) => s.start(startContext)));+ await Promise.all(this.#loadSignals.map(async (s) => s.start(startContext)));- this.autoscaleInterval = betterSetInterval(this._autoscale, this.autoscaleIntervalMillis);+ this.#autoscaleInterval = betterSetInterval(this.autoscale, this.#autoscaleIntervalMillis);if (this.maxTasksPerMinute !== Infinity) {- this.tasksDonePerSecondInterval = betterSetInterval(this._incrementTasksDonePerSecond, 1000);+ this.#tasksDonePerSecondInterval = betterSetInterval(this.incrementTasksDonePerSecond, 1000);}// Last, so `isRunning` never claims a system whose signals aren't collecting yet.- this.running = true;+ this.#running = true;}/*** Stops the snapshotter and intervals. Idempotent and safe to call even if the system was never started.*/async stop(): Promise<void> {- if (this.startPromise === undefined) {+ if (this.#startPromise === undefined) {return;}// Waited out rather than interrupted, or the intervals a starting signal is about to register outlive us.- await this.startPromise.catch(() => {});- this.startPromise = undefined;- this.running = false;+ await this.#startPromise.catch(() => {});+ this.#startPromise = undefined;+ this.#running = false;await this.shutDown();}private async shutDown(): Promise<void> {- if (this.autoscaleInterval) betterClearInterval(this.autoscaleInterval);- if (this.tasksDonePerSecondInterval) betterClearInterval(this.tasksDonePerSecondInterval);+ if (this.#autoscaleInterval) betterClearInterval(this.#autoscaleInterval);+ if (this.#tasksDonePerSecondInterval) betterClearInterval(this.#tasksDonePerSecondInterval);await this.snapshotter.stop();- await Promise.all(this.loadSignals.map(async (s) => s.stop()));+ await Promise.all(this.#loadSignals.map(async (s) => s.stop()));}/**@@ -429,11 +430,11 @@ export class ConcurrencySystem implements IConcurrencySystem {* `desiredConcurrency` are frozen at that point, so the borrowing pool would otherwise just quietly mis-scale.*/private warnIfNotRunning(): void {- if (this.running || this.warnedAboutQueryWhileStopped) {+ if (this.#running || this.#warnedAboutQueryWhileStopped) {return;}- this.warnedAboutQueryWhileStopped = true;+ this.#warnedAboutQueryWhileStopped = true;this.log.warning('Capacity is being queried on a ConcurrencySystem that is not running, so system load is no longer being ' +'monitored and the concurrency will no longer be adjusted. Whoever creates a ConcurrencySystem owns ' +@@ -452,14 +453,14 @@ export class ConcurrencySystem implements IConcurrencySystem {hasCapacityForTask(_consumer?: ConcurrencyConsumer): boolean {this.warnIfNotRunning();- if (this._currentConcurrency >= this._desiredConcurrency) {+ if (this._currentConcurrency >= this.#desiredConcurrency) {this.log.perf('Task will not run. Desired concurrency achieved.');return false;}const currentStatus = this.systemStatus.getCurrentStatus();const { isSystemIdle } = currentStatus;- if (!isSystemIdle && this._currentConcurrency >= this._minConcurrency) {+ if (!isSystemIdle && this._currentConcurrency >= this.#minConcurrency) {this.log.perf('Task will not be run. System is overloaded.',currentStatus as unknown as Record<string, unknown>,@@ -476,7 +477,7 @@ export class ConcurrencySystem implements IConcurrencySystem {return false;}- return this._tasksPerMinute.reduce((acc, curr) => acc + curr, 0) >= this.maxTasksPerMinute;+ return this.#tasksPerMinute.reduce((acc, curr) => acc + curr, 0) >= this.maxTasksPerMinute;}/**@@ -499,7 +500,7 @@ export class ConcurrencySystem implements IConcurrencySystem {}this._currentConcurrency++;- this._tasksPerMinute[0]++;+ this.#tasksPerMinute[0]++;return true;}@@ -521,32 +522,32 @@ export class ConcurrencySystem implements IConcurrencySystem {* Evaluates the historical system status and scales the shared desired concurrency up or down accordingly. Driven* by the autoscaling interval started in {@apilink ConcurrencySystem.start|`start()`}.*/- private _autoscale(intervalCallback: () => void): void {+ private autoscale(intervalCallback: () => void): void {if (this.isOverMaxRequestLimit) return intervalCallback();const systemStatus = this.systemStatus.getHistoricalStatus();const { isSystemIdle } = systemStatus;- const weAreNotAtMax = this._desiredConcurrency < this._maxConcurrency;- const minCurrentConcurrency = Math.floor(this._desiredConcurrency * this.desiredConcurrencyRatio);+ const weAreNotAtMax = this.#desiredConcurrency < this.#maxConcurrency;+ const minCurrentConcurrency = Math.floor(this.#desiredConcurrency * this.desiredConcurrencyRatio);const weAreReachingDesiredConcurrency = this._currentConcurrency >= minCurrentConcurrency;- if (isSystemIdle && weAreNotAtMax && weAreReachingDesiredConcurrency) this._scaleUp(systemStatus);+ if (isSystemIdle && weAreNotAtMax && weAreReachingDesiredConcurrency) this.scaleUp(systemStatus);const isSystemOverloaded = !isSystemIdle;- const weAreNotAtMin = this._desiredConcurrency > this._minConcurrency;+ const weAreNotAtMin = this.#desiredConcurrency > this.#minConcurrency;- if (isSystemOverloaded && weAreNotAtMin) this._scaleDown(systemStatus);+ if (isSystemOverloaded && weAreNotAtMin) this.scaleDown(systemStatus);- if (this.loggingIntervalMillis > 0) {+ if (this.#loggingIntervalMillis > 0) {const now = Date.now();- if (this.lastLoggingTime == null) {- this.lastLoggingTime = now;- } else if (now > this.lastLoggingTime + this.loggingIntervalMillis) {- this.lastLoggingTime = now;+ if (this.#lastLoggingTime == null) {+ this.#lastLoggingTime = now;+ } else if (now > this.#lastLoggingTime + this.#loggingIntervalMillis) {+ this.#lastLoggingTime = now;this.log.info('state', {currentConcurrency: this._currentConcurrency,- desiredConcurrency: this._desiredConcurrency,+ desiredConcurrency: this.#desiredConcurrency,systemStatus,});}@@ -558,12 +559,12 @@ export class ConcurrencySystem implements IConcurrencySystem {/*** Scales the system up by increasing the desired concurrency by the scaleUpStepRatio.*/- private _scaleUp(systemStatus: SystemInfo): void {- const step = Math.ceil(this._desiredConcurrency * this.scaleUpStepRatio);- this._desiredConcurrency = Math.min(this._maxConcurrency, this._desiredConcurrency + step);+ private scaleUp(systemStatus: SystemInfo): void {+ const step = Math.ceil(this.#desiredConcurrency * this.scaleUpStepRatio);+ this.#desiredConcurrency = Math.min(this.#maxConcurrency, this.#desiredConcurrency + step);this.log.debug('scaling up', {- oldConcurrency: this._desiredConcurrency - step,- newConcurrency: this._desiredConcurrency,+ oldConcurrency: this.#desiredConcurrency - step,+ newConcurrency: this.#desiredConcurrency,systemStatus,});}@@ -571,19 +572,19 @@ export class ConcurrencySystem implements IConcurrencySystem {/*** Scales the system down by decreasing the desired concurrency by the scaleDownStepRatio.*/- private _scaleDown(systemStatus: SystemInfo): void {- const step = Math.ceil(this._desiredConcurrency * this.scaleDownStepRatio);- this._desiredConcurrency = Math.max(this._minConcurrency, this._desiredConcurrency - step);+ private scaleDown(systemStatus: SystemInfo): void {+ const step = Math.ceil(this.#desiredConcurrency * this.scaleDownStepRatio);+ this.#desiredConcurrency = Math.max(this.#minConcurrency, this.#desiredConcurrency - step);this.log.debug('scaling down', {- oldConcurrency: this._desiredConcurrency + step,- newConcurrency: this._desiredConcurrency,+ oldConcurrency: this.#desiredConcurrency + step,+ newConcurrency: this.#desiredConcurrency,systemStatus,});}- private _incrementTasksDonePerSecond(intervalCallback: () => void): void {- this._tasksPerMinute.unshift(0);- this._tasksPerMinute.pop();+ private incrementTasksDonePerSecond(intervalCallback: () => void): void {+ this.#tasksPerMinute.unshift(0);+ this.#tasksPerMinute.pop();return intervalCallback();}diff --git a/packages/core/src/autoscaling/cpu_load_signal.ts b/packages/core/src/autoscaling/cpu_load_signal.tsindex 123a0e1c7..b2ea269a7 100644--- a/packages/core/src/autoscaling/cpu_load_signal.ts+++ b/packages/core/src/autoscaling/cpu_load_signal.ts@@ -38,8 +38,8 @@ export class CpuLoadSignal implements LoadSignal {readonly name = 'cpuInfo';readonly overloadedRatio: number;- private readonly store = new SnapshotStore<CpuSnapshot>();- private events?: EventManager;+ readonly #store = new SnapshotStore<CpuSnapshot>();+ #events?: EventManager;constructor(options: CpuLoadSignalOptions = {}) {this.overloadedRatio = options.overloadedRatio ?? 0.4;@@ -47,23 +47,23 @@ export class CpuLoadSignal implements LoadSignal {}async start(context: LoadSignalStartContext): Promise<void> {- this.store.useSampleWindow(context.maxSampleWindowMillis);+ this.#store.useSampleWindow(context.maxSampleWindowMillis);// A new session starts from a clean slate, so it is not judged on measurements from before the downtime.- this.store.clear();+ this.#store.clear();// Resolved here rather than in the constructor, so an instance built ahead of time (to be wrapped, or shared// between systems) cannot capture whichever event manager happened to be registered at that moment.- this.events = serviceLocator.getEventManager();- this.events.on(EventType.SYSTEM_INFO, this.handle);+ this.#events = serviceLocator.getEventManager();+ this.#events.on(EventType.SYSTEM_INFO, this.handle);}async stop(): Promise<void> {- this.events?.off(EventType.SYSTEM_INFO, this.handle);- this.events = undefined;+ this.#events?.off(EventType.SYSTEM_INFO, this.handle);+ this.#events = undefined;}getSample(sampleDurationMillis?: number): LoadSnapshot[] {- return this.store.getSample(sampleDurationMillis);+ return this.#store.getSample(sampleDurationMillis);}/** @internal Records a snapshot from a `SYSTEM_INFO` payload. Exposed for tests. */@@ -71,7 +71,7 @@ export class CpuLoadSignal implements LoadSignal {const { cpuCurrentUsage, isCpuOverloaded } = systemInfo;const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();- this.store.push(+ this.#store.push({createdAt,isOverloaded: isCpuOverloaded!,diff --git a/packages/core/src/autoscaling/event_loop_load_signal.ts b/packages/core/src/autoscaling/event_loop_load_signal.tsindex 37f71f4cb..ac90277d7 100644--- a/packages/core/src/autoscaling/event_loop_load_signal.ts+++ b/packages/core/src/autoscaling/event_loop_load_signal.ts@@ -48,33 +48,33 @@ export class EventLoopLoadSignal implements LoadSignal {readonly name = 'eventLoopInfo';readonly overloadedRatio: number;- private readonly store = new SnapshotStore<EventLoopSnapshot>();- private readonly intervalMillis: number;- private readonly maxBlockedMillis: number;- private interval?: BetterIntervalID;+ readonly #store = new SnapshotStore<EventLoopSnapshot>();+ readonly #intervalMillis: number;+ readonly #maxBlockedMillis: number;+ #interval?: BetterIntervalID;constructor(options: EventLoopLoadSignalOptions = {}) {this.overloadedRatio = options.overloadedRatio ?? 0.6;- this.intervalMillis = (options.snapshotIntervalSecs ?? 0.5) * 1000;- this.maxBlockedMillis = options.maxBlockedMillis ?? 50;+ this.#intervalMillis = (options.snapshotIntervalSecs ?? 0.5) * 1000;+ this.#maxBlockedMillis = options.maxBlockedMillis ?? 50;this.handle = this.handle.bind(this);}async start(context: LoadSignalStartContext): Promise<void> {- this.store.useSampleWindow(context.maxSampleWindowMillis);+ this.#store.useSampleWindow(context.maxSampleWindowMillis);// A new session starts from a clean slate, or the downtime gets charged to the event loop: `handle()` measures// the gap since the previous snapshot, which across a restart is however long the system was stopped.- this.store.clear();- this.interval = betterSetInterval(this.handle, this.intervalMillis);+ this.#store.clear();+ this.#interval = betterSetInterval(this.handle, this.#intervalMillis);}async stop(): Promise<void> {- if (this.interval) betterClearInterval(this.interval);- this.interval = undefined;+ if (this.#interval) betterClearInterval(this.#interval);+ this.#interval = undefined;}getSample(sampleDurationMillis?: number): LoadSnapshot[] {- return this.store.getSample(sampleDurationMillis);+ return this.#store.getSample(sampleDurationMillis);}/**@@ -90,17 +90,17 @@ export class EventLoopLoadSignal implements LoadSignal {exceededMillis: 0,};- const all = this.store.getAll();+ const all = this.#store.getAll();const previousSnapshot = all[all.length - 1];if (previousSnapshot) {- const delta = now.getTime() - +previousSnapshot.createdAt - this.intervalMillis;+ const delta = now.getTime() - +previousSnapshot.createdAt - this.#intervalMillis;- if (delta > this.maxBlockedMillis) snapshot.isOverloaded = true;- snapshot.exceededMillis = Math.max(delta - this.maxBlockedMillis, 0);+ if (delta > this.#maxBlockedMillis) snapshot.isOverloaded = true;+ snapshot.exceededMillis = Math.max(delta - this.#maxBlockedMillis, 0);}- this.store.push(snapshot, now);+ this.#store.push(snapshot, now);intervalCallback();}}diff --git a/packages/core/src/autoscaling/load_signal.ts b/packages/core/src/autoscaling/load_signal.tsindex 11d8228fc..d32a69354 100644--- a/packages/core/src/autoscaling/load_signal.ts+++ b/packages/core/src/autoscaling/load_signal.ts@@ -69,9 +69,10 @@ export interface LoadSignal {* and so can yours — it is the only part of their machinery worth reusing.*/export class SnapshotStore<T extends LoadSnapshot = LoadSnapshot> {- private snapshots: T[] = [];+ #snapshots: T[] = [];/** Retention window in milliseconds. Unbounded until {@apilink SnapshotStore.useSampleWindow|`useSampleWindow()`}. */+ // kept as TS-private: concurrency_system tests read this retention window directlyprivate historyMillis = Infinity;/**@@ -89,29 +90,29 @@ export class SnapshotStore<T extends LoadSnapshot = LoadSnapshot> {push(snapshot: T, now: Date = snapshot.createdAt): void {// Inline pruning to avoid private-method transpilation issueslet oldCount = 0;- for (let i = 0; i < this.snapshots.length; i++) {- const { createdAt } = this.snapshots[i];+ for (let i = 0; i < this.#snapshots.length; i++) {+ const { createdAt } = this.#snapshots[i];if (now.getTime() - new Date(createdAt).getTime() > this.historyMillis) oldCount++;else break;}- if (oldCount) this.snapshots.splice(0, oldCount);+ if (oldCount) this.#snapshots.splice(0, oldCount);- this.snapshots.push(snapshot);+ this.#snapshots.push(snapshot);}/*** Return all snapshots, or only those within the given time window.*/getSample(sampleDurationMillis?: number): T[] {- if (!sampleDurationMillis) return this.snapshots;+ if (!sampleDurationMillis) return this.#snapshots;const sample: T[] = [];- let idx = this.snapshots.length;+ let idx = this.#snapshots.length;if (!idx) return sample;- const latestTime = this.snapshots[idx - 1].createdAt;+ const latestTime = this.#snapshots[idx - 1].createdAt;while (idx--) {- const snapshot = this.snapshots[idx];+ const snapshot = this.#snapshots[idx];if (+latestTime - +snapshot.createdAt <= sampleDurationMillis) {sample.unshift(snapshot);} else {@@ -127,7 +128,7 @@ export class SnapshotStore<T extends LoadSnapshot = LoadSnapshot> {* to compute a delta (e.g. the event loop and client signals read the last entry to measure change since it).*/getAll(): T[] {- return this.snapshots;+ return this.#snapshots;}/**@@ -137,7 +138,7 @@ export class SnapshotStore<T extends LoadSnapshot = LoadSnapshot> {* rather than on stop leaves a finished session readable.*/clear(): void {- this.snapshots = [];+ this.#snapshots = [];}}diff --git a/packages/core/src/autoscaling/memory_load_signal.ts b/packages/core/src/autoscaling/memory_load_signal.tsindex 893283c9a..027cca393 100644--- a/packages/core/src/autoscaling/memory_load_signal.ts+++ b/packages/core/src/autoscaling/memory_load_signal.ts@@ -51,61 +51,61 @@ export class MemoryLoadSignal implements LoadSignal {readonly name = 'memInfo';readonly overloadedRatio: number;- private readonly store = new SnapshotStore<MemorySnapshot>();- private readonly maxUsedRatio: number;+ readonly #store = new SnapshotStore<MemorySnapshot>();+ readonly #maxUsedRatio: number;/** All resolved in `start()`, before anything that reads them can fire. */- private config!: Configuration;- private log!: CrawleeLogger;- private maxMemoryBytes!: number;- private events?: EventManager;- private maxMemoryRatio: number | undefined;- private lastLoggedCriticalMemoryOverloadAt: Date | null = null;+ #config!: Configuration;+ #log!: CrawleeLogger;+ #maxMemoryBytes!: number;+ #events?: EventManager;+ #maxMemoryRatio: number | undefined;+ #lastLoggedCriticalMemoryOverloadAt: Date | null = null;constructor(options: MemoryLoadSignalOptions = {}) {- this.maxUsedRatio = options.maxUsedRatio ?? 0.9;+ this.#maxUsedRatio = options.maxUsedRatio ?? 0.9;this.overloadedRatio = options.overloadedRatio ?? 0.2;this.handle = this.handle.bind(this);}async start(context: LoadSignalStartContext): Promise<void> {- this.store.useSampleWindow(context.maxSampleWindowMillis);+ this.#store.useSampleWindow(context.maxSampleWindowMillis);// A new session starts from a clean slate, so it is not judged on measurements from before the downtime.- this.store.clear();+ this.#store.clear();// Resolved here rather than in the constructor: an instance built ahead of time (to be wrapped, or shared// between systems) must not capture whichever services happened to be registered at that moment.- this.config = serviceLocator.getConfiguration();- this.events = serviceLocator.getEventManager();- this.log = serviceLocator.getLogger().child({ prefix: 'MemoryLoadSignal' });+ this.#config = serviceLocator.getConfiguration();+ this.#events = serviceLocator.getEventManager();+ this.#log = serviceLocator.getLogger().child({ prefix: 'MemoryLoadSignal' });- const memoryMbytes = this.config.memoryMbytes ?? 0;+ const memoryMbytes = this.#config.memoryMbytes ?? 0;if (memoryMbytes > 0) {- this.maxMemoryBytes = memoryMbytes * 1024 * 1024;+ this.#maxMemoryBytes = memoryMbytes * 1024 * 1024;} else {- this.maxMemoryRatio = this.config.availableMemoryRatio;- if (!this.maxMemoryRatio) {+ this.#maxMemoryRatio = this.#config.availableMemoryRatio;+ if (!this.#maxMemoryRatio) {throw new Error('availableMemoryRatio is not set in configuration.');} else {- this.log.debug(- `Setting max memory of this run to ${this.maxMemoryRatio * 100} % of available memory. ` ++ this.#log.debug(+ `Setting max memory of this run to ${this.#maxMemoryRatio * 100} % of available memory. ` +'Use the CRAWLEE_MEMORY_MBYTES or CRAWLEE_AVAILABLE_MEMORY_RATIO environment variable to override it.',);}// Fallback memory measurement in case memTotalBytes is missing from SystemInfo.- this.maxMemoryBytes = await this._getTotalMemoryBytes();+ this.#maxMemoryBytes = await this.getTotalMemoryBytes();}- this.events.on(EventType.SYSTEM_INFO, this.handle);+ this.#events.on(EventType.SYSTEM_INFO, this.handle);}async stop(): Promise<void> {- this.events?.off(EventType.SYSTEM_INFO, this.handle);- this.events = undefined;+ this.#events?.off(EventType.SYSTEM_INFO, this.handle);+ this.#events = undefined;}getSample(sampleDurationMillis?: number): LoadSnapshot[] {- return this.store.getSample(sampleDurationMillis);+ return this.#store.getSample(sampleDurationMillis);}/** @internal Records a snapshot from a `SYSTEM_INFO` payload. Exposed for tests. */@@ -113,52 +113,51 @@ export class MemoryLoadSignal implements LoadSignal {const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();const { memCurrentBytes, memTotalBytes } = systemInfo;- let maxMemoryBytes = this.maxMemoryBytes;- if (this.maxMemoryRatio !== undefined && this.maxMemoryRatio > 0) {- maxMemoryBytes = this.maxMemoryRatio * (memTotalBytes ?? this.maxMemoryBytes);+ let maxMemoryBytes = this.#maxMemoryBytes;+ if (this.#maxMemoryRatio !== undefined && this.#maxMemoryRatio > 0) {+ maxMemoryBytes = this.#maxMemoryRatio * (memTotalBytes ?? this.#maxMemoryBytes);}const snapshot: MemorySnapshot = {createdAt,- isOverloaded: memCurrentBytes! / maxMemoryBytes > this.maxUsedRatio,+ isOverloaded: memCurrentBytes! / maxMemoryBytes > this.#maxUsedRatio,usedBytes: memCurrentBytes,};- this.store.push(snapshot, createdAt);- this._memoryOverloadWarning(systemInfo, maxMemoryBytes);+ this.#store.push(snapshot, createdAt);+ this.memoryOverloadWarning(systemInfo, maxMemoryBytes);}- /** @internal */- _memoryOverloadWarning(systemInfo: SystemInfo, maxMemoryBytes?: number): void {- const effectiveMax = maxMemoryBytes ?? this.maxMemoryBytes;+ private memoryOverloadWarning(systemInfo: SystemInfo, maxMemoryBytes?: number): void {+ const effectiveMax = maxMemoryBytes ?? this.#maxMemoryBytes;const { memCurrentBytes } = systemInfo;const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();if (- this.lastLoggedCriticalMemoryOverloadAt &&- +createdAt < +this.lastLoggedCriticalMemoryOverloadAt + CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS+ this.#lastLoggedCriticalMemoryOverloadAt &&+ +createdAt < +this.#lastLoggedCriticalMemoryOverloadAt + CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS)return;- const maxDesiredMemoryBytes = this.maxUsedRatio * effectiveMax;- const reserveMemory = effectiveMax * (1 - this.maxUsedRatio) * RESERVE_MEMORY_RATIO;+ const maxDesiredMemoryBytes = this.#maxUsedRatio * effectiveMax;+ const reserveMemory = effectiveMax * (1 - this.#maxUsedRatio) * RESERVE_MEMORY_RATIO;const criticalOverloadBytes = maxDesiredMemoryBytes + reserveMemory;const isCriticalOverload = memCurrentBytes! > criticalOverloadBytes;if (isCriticalOverload) {const usedPercentage = Math.round((memCurrentBytes! / effectiveMax) * 100);const toMb = (bytes: number) => Math.round(bytes / 1024 ** 2);- this.log.warning(+ this.#log.warning('Memory is critically overloaded. ' +`Using ${toMb(memCurrentBytes!)} MB of ${toMb(effectiveMax,)} MB (${usedPercentage}%). Consider increasing available memory.`,);- this.lastLoggedCriticalMemoryOverloadAt = createdAt;+ this.#lastLoggedCriticalMemoryOverloadAt = createdAt;}}- private async _getTotalMemoryBytes(): Promise<number> {- const containerized = this.config.containerized ?? (await isContainerized());+ private async getTotalMemoryBytes(): Promise<number> {+ const containerized = this.#config.containerized ?? (await isContainerized());return (await getMemoryInfo({ containerized, logger: serviceLocator.getLogger() })).totalBytes;}}diff --git a/packages/core/src/autoscaling/snapshotter.ts b/packages/core/src/autoscaling/snapshotter.tsindex cf18f7d12..167fcb27d 100644--- a/packages/core/src/autoscaling/snapshotter.ts+++ b/packages/core/src/autoscaling/snapshotter.ts@@ -73,10 +73,10 @@ export type SnapshotterOptions = Omit<LoadSignalsOptions, 'custom'>;*/export class Snapshotter {// Absent when switched off through the corresponding option (e.g. `client: false`).- private readonly memorySignal?: MemoryLoadSignal;- private readonly eventLoopSignal?: EventLoopLoadSignal;- private readonly cpuSignal?: CpuLoadSignal;- private readonly clientSignal?: ClientLoadSignal;+ readonly #memorySignal?: MemoryLoadSignal;+ readonly #eventLoopSignal?: EventLoopLoadSignal;+ readonly #cpuSignal?: CpuLoadSignal;+ readonly #clientSignal?: ClientLoadSignal;/*** Returns the enabled built-in signals, so `SystemStatus` can iterate them alongside any custom `LoadSignal`@@ -85,10 +85,10 @@ export class Snapshotter {*/getLoadSignals(): LoadSignal[] {const builtin: (LoadSignal | undefined)[] = [- this.memorySignal,- this.eventLoopSignal,- this.cpuSignal,- this.clientSignal,+ this.#memorySignal,+ this.#eventLoopSignal,+ this.#cpuSignal,+ this.#clientSignal,];return builtin.filter((signal): signal is LoadSignal => signal !== undefined);@@ -102,10 +102,10 @@ export class Snapshotter {// Each signal resolves its own ambient dependencies when started, and is told the window it will be sampled// over then too - so there is nothing to thread in here beyond the caller's tuning.- if (memory !== false) this.memorySignal = new MemoryLoadSignal(memory);- if (eventLoop !== false) this.eventLoopSignal = new EventLoopLoadSignal(eventLoop);- if (cpu !== false) this.cpuSignal = new CpuLoadSignal(cpu);- if (client !== false) this.clientSignal = new ClientLoadSignal(client);+ if (memory !== false) this.#memorySignal = new MemoryLoadSignal(memory);+ if (eventLoop !== false) this.#eventLoopSignal = new EventLoopLoadSignal(eventLoop);+ if (cpu !== false) this.#cpuSignal = new CpuLoadSignal(cpu);+ if (client !== false) this.#clientSignal = new ClientLoadSignal(client);}/**diff --git a/packages/core/src/autoscaling/system_status.ts b/packages/core/src/autoscaling/system_status.tsindex e5f1f3ede..37157b2cf 100644--- a/packages/core/src/autoscaling/system_status.ts+++ b/packages/core/src/autoscaling/system_status.ts@@ -127,9 +127,9 @@ const BUILTIN_SIGNAL_NAMES = new Set(Object.keys(BUILTIN_SIGNAL_OPTION_KEYS));* @internal*/export class SystemStatus {- private readonly currentHistoryMillis: number;- private readonly historyMillis: number;- private readonly signals: LoadSignal[];+ readonly #currentHistoryMillis: number;+ readonly #historyMillis: number;+ readonly #signals: LoadSignal[];constructor(options: SystemStatusOptions) {const {@@ -139,10 +139,10 @@ export class SystemStatus {loadSignals = [],} = options;- this.currentHistoryMillis = currentHistorySecs * 1000;- this.historyMillis = historySecs * 1000;+ this.#currentHistoryMillis = currentHistorySecs * 1000;+ this.#historyMillis = historySecs * 1000;- this.signals = [...snapshotter.getLoadSignals(), ...loadSignals];+ this.#signals = [...snapshotter.getLoadSignals(), ...loadSignals];this.assertUniqueSignalNames();}@@ -152,7 +152,7 @@ export class SystemStatus {* defaults.*/get maxSampleWindowMillis(): number {- return Math.max(this.currentHistoryMillis, this.historyMillis);+ return Math.max(this.#currentHistoryMillis, this.#historyMillis);}/**@@ -163,7 +163,7 @@ export class SystemStatus {private assertUniqueSignalNames(): void {const seen = new Set<string>();- for (const { name } of this.signals) {+ for (const { name } of this.#signals) {if (!seen.has(name)) {seen.add(name);continue;@@ -194,7 +194,7 @@ export class SystemStatus {* and `true` otherwise.*/getCurrentStatus(): SystemInfo {- return this.isSystemIdle(this.currentHistoryMillis);+ return this.isSystemIdle(this.#currentHistoryMillis);}/**@@ -213,7 +213,7 @@ export class SystemStatus {* `historySecs` seconds and `true` otherwise.*/getHistoricalStatus(): SystemInfo {- return this.isSystemIdle(this.historyMillis);+ return this.isSystemIdle(this.#historyMillis);}/**@@ -230,7 +230,7 @@ export class SystemStatus {let loadSignalInfo: Record<string, ClientInfo> | undefined;- for (const signal of this.signals) {+ for (const signal of this.#signals) {const sample = signal.getSample(sampleDurationMillis);const info = evaluateLoadSignalSample(sample, signal.overloadedRatio);diff --git a/packages/core/src/configuration.ts b/packages/core/src/configuration.tsindex 4a8402945..059857507 100644--- a/packages/core/src/configuration.ts+++ b/packages/core/src/configuration.ts@@ -185,7 +185,7 @@ export class Configuration {*/protected static fields: Record<string, ConfigField> = crawleeConfigFields;- private resolvedValues: Record<string, unknown>;+ #resolvedValues: Record<string, unknown>;/*** Creates new `Configuration` instance with provided options.@@ -195,7 +195,7 @@ export class Configuration {constructor(options: ConfigurationInput = {}) {const fields = (this.constructor as typeof Configuration).fields;const fileOptions = Configuration.loadFileOptions();- this.resolvedValues = Configuration.resolveAll(fields, options as Record<string, unknown>, fileOptions);+ this.#resolvedValues = Configuration.resolveAll(fields, options as Record<string, unknown>, fileOptions);this.registerAccessors();// Set the log level@@ -262,7 +262,7 @@ export class Configuration {for (const key of Object.keys(fields)) {descriptors[key] = {- get: () => this.resolvedValues[key],+ get: () => this.#resolvedValues[key],set() {throw new TypeError('Configuration is immutable. Pass options via the constructor instead.');},diff --git a/packages/core/src/crawlers/context_pipeline.ts b/packages/core/src/crawlers/context_pipeline.tsindex f22f0bb0e..bfe82d681 100644--- a/packages/core/src/crawlers/context_pipeline.ts+++ b/packages/core/src/crawlers/context_pipeline.ts@@ -100,11 +100,16 @@ class ContextPipelineImpl<TContextBase, TCrawlingContext extends TContextBase> eTContextBase,TCrawlingContext> {+ readonly #middleware: ContextMiddleware<TContextBase, TCrawlingContext>;+ readonly #parent?: ContextPipelineImpl<TContextBase, TContextBase>;+constructor(- private middleware: ContextMiddleware<TContextBase, TCrawlingContext>,- private parent?: ContextPipelineImpl<TContextBase, TContextBase>,+ middleware: ContextMiddleware<TContextBase, TCrawlingContext>,+ parent?: ContextPipelineImpl<TContextBase, TContextBase>,) {super();+ this.#middleware = middleware;+ this.#parent = parent;}/**@@ -138,8 +143,8 @@ class ContextPipelineImpl<TContextBase, TCrawlingContext extends TContextBase> elet step: ContextPipelineImpl<TContextBase, TContextBase> | undefined = this as any;while (step !== undefined) {- yield step.middleware;- step = step.parent;+ yield step.#middleware;+ step = step.#parent;}}diff --git a/packages/core/src/crawlers/statistics.ts b/packages/core/src/crawlers/statistics.tsindex f404ceb16..24c9f2a20 100644--- a/packages/core/src/crawlers/statistics.ts+++ b/packages/core/src/crawlers/statistics.ts@@ -11,16 +11,16 @@ import { ErrorTracker } from './error_tracker.js';* @ignore*/class Job {- private lastRunAt: number | null = null;- private durationMillis?: number;+ #lastRunAt: number | null = null;+ #durationMillis?: number;run() {- this.lastRunAt = Date.now();+ this.#lastRunAt = Date.now();}finish() {- this.durationMillis = Date.now() - this.lastRunAt!;- return this.durationMillis;+ this.#durationMillis = Date.now() - this.#lastRunAt!;+ return this.#durationMillis;}}@@ -134,6 +134,7 @@ export interface CalculatedStatistics {* @category Crawlers*/export class Statistics implements IStatistics {+ // kept as TS-private: statistics tests read the static counter directlyprivate static id = 0;/**@@ -163,21 +164,21 @@ export class Statistics implements IStatistics {protected keyValueStore?: KeyValueStore = undefined;protected readonly persistStateKey: string;- private logIntervalMillis: number;- private logMessage: string;- private listener: () => Promise<void>;- private requestsInProgress = new Map<number | string, Job>();+ #logIntervalMillis: number;+ #logMessage: string;+ #listener: () => Promise<void>;+ #requestsInProgress = new Map<number | string, Job>();private readonly log: CrawleeLogger;- private instanceStart!: number;- private logInterval: unknown;- private _events?: EventManager;- private persistenceOptions: PersistenceOptions;+ #instanceStart!: number;+ #logInterval: unknown;+ #events?: EventManager;+ #persistenceOptions: PersistenceOptions;private get events(): EventManager {- if (!this._events) {- this._events = serviceLocator.getEventManager();+ if (!this.#events) {+ this.#events = serviceLocator.getEventManager();}- return this._events;+ return this.#events;}/**@@ -215,11 +216,11 @@ export class Statistics implements IStatistics {this.log = (options.log ?? serviceLocator.getLogger()).child({ prefix: 'Statistics' });this.errorTracker = new ErrorTracker({ ...errorTrackerConfig, saveErrorSnapshots });this.errorTrackerRetry = new ErrorTracker({ ...errorTrackerConfig, saveErrorSnapshots });- this.logIntervalMillis = logIntervalSecs * 1000;- this.logMessage = logMessage;+ this.#logIntervalMillis = logIntervalSecs * 1000;+ this.#logMessage = logMessage;this.keyValueStore = keyValueStore;- this.listener = this.persistState.bind(this);- this.persistenceOptions = persistenceOptions;+ this.#listener = this.persistState.bind(this);+ this.#persistenceOptions = persistenceOptions;// initialize by "resetting"this.reset();@@ -252,8 +253,8 @@ export class Statistics implements IStatistics {};this.requestRetryHistogram.length = 0;- this.requestsInProgress.clear();- this.instanceStart = Date.now();+ this.#requestsInProgress.clear();+ this.#instanceStart = Date.now();this.teardown();}@@ -262,7 +263,7 @@ export class Statistics implements IStatistics {* @param options - Override the persistence options provided in the constructor*/async resetStore(options?: PersistenceOptions) {- if (!this.persistenceOptions.enable && !options?.enable) {+ if (!this.#persistenceOptions.enable && !options?.enable) {return;}@@ -291,10 +292,10 @@ export class Statistics implements IStatistics {* @ignore*/startJob(id: number | string) {- let job = this.requestsInProgress.get(id);+ let job = this.#requestsInProgress.get(id);if (!job) job = new Job();job.run();- this.requestsInProgress.set(id, job);+ this.#requestsInProgress.set(id, job);}/**@@ -302,7 +303,7 @@ export class Statistics implements IStatistics {* @ignore*/finishJob(id: number | string, retryCount: number) {- const job = this.requestsInProgress.get(id);+ const job = this.#requestsInProgress.get(id);if (!job) return;const jobDurationMillis = job.finish();this.state.requestsFinished++;@@ -312,7 +313,7 @@ export class Statistics implements IStatistics {this.state.requestMinDurationMillis = jobDurationMillis;if (jobDurationMillis > this.state.requestMaxDurationMillis)this.state.requestMaxDurationMillis = jobDurationMillis;- this.requestsInProgress.delete(id);+ this.#requestsInProgress.delete(id);}/**@@ -320,12 +321,12 @@ export class Statistics implements IStatistics {* @ignore*/failJob(id: number | string, retryCount: number) {- const job = this.requestsInProgress.get(id);+ const job = this.#requestsInProgress.get(id);if (!job) return;this.state.requestTotalFailedDurationMillis += job.finish();this.state.requestsFailed++;this.saveRetryCountForJob(retryCount);- this.requestsInProgress.delete(id);+ this.#requestsInProgress.delete(id);}/**@@ -334,7 +335,7 @@ export class Statistics implements IStatistics {* @ignore*/discardJob(id: number | string) {- this.requestsInProgress.delete(id);+ this.#requestsInProgress.delete(id);}/**@@ -347,7 +348,7 @@ export class Statistics implements IStatistics {requestTotalFailedDurationMillis,requestTotalFinishedDurationMillis,} = this.state;- const totalMillis = Date.now() - this.instanceStart;+ const totalMillis = Date.now() - this.#instanceStart;const totalMinutes = totalMillis / 1000 / 60;return {@@ -369,7 +370,7 @@ export class Statistics implements IStatistics {async startCapturing() {// A single instance drives one logging interval and one PERSIST_STATE listener, so a second concurrent// capture (e.g. sharing one instance across crawlers running at once) would orphan the first. Fail loudly.- if (this.logInterval) {+ if (this.#logInterval) {throw new Error('Statistics.startCapturing() was already called - this instance is already capturing.');}@@ -379,17 +380,17 @@ export class Statistics implements IStatistics {this.state.crawlerStartedAt = new Date();}- if (this.persistenceOptions.enable) {+ if (this.#persistenceOptions.enable) {await this.maybeLoadStatistics();- this.events.on(EventType.PERSIST_STATE, this.listener);+ this.events.on(EventType.PERSIST_STATE, this.#listener);}- this.logInterval = setInterval(() => {- this.log.info(this.logMessage, {+ this.#logInterval = setInterval(() => {+ this.log.info(this.#logMessage, {...this.calculate(),retryHistogram: this.requestRetryHistogram,});- }, this.logIntervalMillis);+ }, this.#logIntervalMillis);}/**@@ -414,7 +415,7 @@ export class Statistics implements IStatistics {* @param options - Override the persistence options provided in the constructor*/async persistState(options?: PersistenceOptions) {- if (!this.persistenceOptions.enable && !options?.enable) {+ if (!this.#persistenceOptions.enable && !options?.enable) {return;}@@ -472,7 +473,7 @@ export class Statistics implements IStatistics {this.state.crawlerStartedAt = savedState.crawlerStartedAt ? new Date(savedState.crawlerStartedAt) : null;this.state.statsPersistedAt = savedState.statsPersistedAt ? new Date(savedState.statsPersistedAt) : null;this.state.crawlerRuntimeMillis = savedState.crawlerRuntimeMillis;- this.instanceStart = Date.now() - (+this.state.statsPersistedAt! - savedState.crawlerLastStartTimestamp);+ this.#instanceStart = Date.now() - (+this.state.statsPersistedAt! - savedState.crawlerLastStartTimestamp);this.log.debug('Loaded from KeyValueStore');}@@ -481,11 +482,11 @@ export class Statistics implements IStatistics {// this can be called before a call to startCapturing happens (or in a 'finally' block)// Only unsubscribe if event manager was already resolved — avoid eagerly resolving it// (e.g. during the constructor's reset() call, which would capture the wrong context)- this._events?.off(EventType.PERSIST_STATE, this.listener);+ this.#events?.off(EventType.PERSIST_STATE, this.#listener);- if (this.logInterval) {- clearInterval(this.logInterval as number);- this.logInterval = null;+ if (this.#logInterval) {+ clearInterval(this.#logInterval as number);+ this.#logInterval = null;}}@@ -499,7 +500,7 @@ export class Statistics implements IStatistics {// omit duplicated informationconst result = {...this.state,- crawlerLastStartTimestamp: this.instanceStart,+ crawlerLastStartTimestamp: this.#instanceStart,crawlerFinishedAt: this.state.crawlerFinishedAt? new Date(this.state.crawlerFinishedAt).toISOString(): null,diff --git a/packages/core/src/events/event_manager.ts b/packages/core/src/events/event_manager.tsindex b08814985..95b7f5862 100644--- a/packages/core/src/events/event_manager.ts+++ b/packages/core/src/events/event_manager.ts@@ -62,10 +62,10 @@ export abstract class EventManager {protected initialized = false;protected intervals: Intervals = {};protected log = serviceLocator.getLogger().child({ prefix: 'Events' });- private persistStateIntervalMillis: number;+ #persistStateIntervalMillis: number;constructor(options: EventManagerOptions) {- this.persistStateIntervalMillis = options.persistStateIntervalMillis;+ this.#persistStateIntervalMillis = options.persistStateIntervalMillis;this.events.setMaxListeners(50);}@@ -81,7 +81,7 @@ export abstract class EventManager {this.intervals.persistState = betterSetInterval((intervalCallback: () => unknown) => {this.emit(EventType.PERSIST_STATE, { isMigrating: false });intervalCallback();- }, this.persistStateIntervalMillis);+ }, this.#persistStateIntervalMillis);this.initialized = true;}diff --git a/packages/core/src/events/local_event_manager.ts b/packages/core/src/events/local_event_manager.tsindex 494a887e3..493745ca2 100644--- a/packages/core/src/events/local_event_manager.ts+++ b/packages/core/src/events/local_event_manager.ts@@ -11,11 +11,11 @@ export interface LocalEventManagerOptions extends EventManagerOptions {}export class LocalEventManager extends EventManager {- private systemInfoIntervalMillis: number;+ #systemInfoIntervalMillis: number;constructor(options: LocalEventManagerOptions) {super(options);- this.systemInfoIntervalMillis = options.systemInfoIntervalMillis;+ this.#systemInfoIntervalMillis = options.systemInfoIntervalMillis;}/**@@ -45,7 +45,7 @@ export class LocalEventManager extends EventManager {this.emitSystemInfoEvent = this.emitSystemInfoEvent.bind(this);this.intervals.systemInfo = betterSetInterval(this.emitSystemInfoEvent.bind(this),- this.systemInfoIntervalMillis,+ this.#systemInfoIntervalMillis,);}diff --git a/packages/core/src/log.ts b/packages/core/src/log.tsindex 01231bc33..a37168019 100644--- a/packages/core/src/log.ts+++ b/packages/core/src/log.ts@@ -36,6 +36,8 @@ export type { CrawleeLogger, CrawleeLoggerOptions };* ```*/export abstract class BaseCrawleeLogger implements CrawleeLogger {+ // kept as TS-private: the adaptive crawler's log proxy calls non-intercepted methods with `this === proxy`,+ // where `#`-field access would throw at runtimeprivate options: CrawleeLoggerOptions;private readonly warningsLogged = new Set<string>();@@ -126,6 +128,7 @@ export abstract class BaseCrawleeLogger implements CrawleeLogger {*/export class ApifyLogAdapter extends BaseCrawleeLogger {constructor(+ // kept as a TS-private parameter property: reached through the adaptive crawler's log proxy, see aboveprivate readonly apifyLog: Log,options?: Partial<CrawleeLoggerOptions>,) {diff --git a/packages/core/src/memory-storage/memory-storage.ts b/packages/core/src/memory-storage/memory-storage.tsindex c5ec9cca1..77f218ed6 100644--- a/packages/core/src/memory-storage/memory-storage.ts+++ b/packages/core/src/memory-storage/memory-storage.ts@@ -21,7 +21,7 @@ export class MemoryStorageBackend implements storage.StorageBackend {* Unique per-instance cache partition key. Mirrors the way `FileSystemStorageBackend` partitions its* cache by storage directory: two distinct `MemoryStorageBackend` instances must not share cached backends.*/- private readonly instanceCacheKey = `MemoryStorageBackend:${randomUUID()}`;+ readonly #instanceCacheKey = `MemoryStorageBackend:${randomUUID()}`;readonly keyValueStoreBackendCache: KeyValueStoreBackend[] = [];readonly datasetBackendCache: DatasetBackend[] = [];@@ -36,7 +36,7 @@ export class MemoryStorageBackend implements storage.StorageBackend {* cache partitions in the storage backend cache.*/getStorageBackendCacheKey(): string {- return this.instanceCacheKey;+ return this.#instanceCacheKey;}private static resolveStorageKey(options: { id?: string; name?: string; alias?: string }): {diff --git a/packages/core/src/memory-storage/resource-clients/dataset.ts b/packages/core/src/memory-storage/resource-clients/dataset.tsindex 14cd2684b..1023f035d 100644--- a/packages/core/src/memory-storage/resource-clients/dataset.ts+++ b/packages/core/src/memory-storage/resource-clients/dataset.ts@@ -46,7 +46,8 @@ export class DatasetBackend<Data extends Dictionary = Dictionary>modifiedAt = new Date();itemCount = 0;- private readonly datasetEntries = new Map<string, Data>();+ readonly #datasetEntries = new Map<string, Data>();+ // kept as TS-private: storage-backend tests read this field at runtimeprivate readonly storageBackend: MemoryStorageBackend;constructor(options: DatasetBackendOptions) {@@ -67,13 +68,13 @@ export class DatasetBackend<Data extends Dictionary = Dictionary>if (storeIndex !== -1) {const [oldBackend] = this.storageBackend.datasetBackendCache.splice(storeIndex, 1);oldBackend.itemCount = 0;- oldBackend.datasetEntries.clear();+ oldBackend.#datasetEntries.clear();}}async purge(): Promise<void> {this.itemCount = 0;- this.datasetEntries.clear();+ this.#datasetEntries.clear();this.updateTimestamps(true);}@@ -106,7 +107,7 @@ export class DatasetBackend<Data extends Dictionary = Dictionary>for (let idx = start; idx < end; idx++) {const entryNumber = this.generateLocalEntryName(idx);- items.push(this.datasetEntries.get(entryNumber)!);+ items.push(this.#datasetEntries.get(entryNumber)!);}this.updateTimestamps(false);@@ -124,7 +125,7 @@ export class DatasetBackend<Data extends Dictionary = Dictionary>async pushData(items: Data[]): Promise<void> {for (const entry of items) {const idx = this.generateLocalEntryName(++this.itemCount);- this.datasetEntries.set(idx, JSON.parse(JSON.stringify(entry)) as Data);+ this.#datasetEntries.set(idx, JSON.parse(JSON.stringify(entry)) as Data);}this.updateTimestamps(true);diff --git a/packages/core/src/memory-storage/resource-clients/key-value-store.ts b/packages/core/src/memory-storage/resource-clients/key-value-store.tsindex 87e608483..c07da456a 100644--- a/packages/core/src/memory-storage/resource-clients/key-value-store.ts+++ b/packages/core/src/memory-storage/resource-clients/key-value-store.ts@@ -46,7 +46,8 @@ export class KeyValueStoreBackend extends BaseClient implements storage.KeyValueaccessedAt = new Date();modifiedAt = new Date();- private readonly keyValueEntries = new Map<string, InternalKeyRecord>();+ readonly #keyValueEntries = new Map<string, InternalKeyRecord>();+ // kept as TS-private: storage-backend tests read this field at runtimeprivate readonly storageBackend: MemoryStorageBackend;constructor(options: KeyValueStoreBackendOptions) {@@ -66,12 +67,12 @@ export class KeyValueStoreBackend extends BaseClient implements storage.KeyValueif (storeIndex !== -1) {const [oldBackend] = this.storageBackend.keyValueStoreBackendCache.splice(storeIndex, 1);- oldBackend.keyValueEntries.clear();+ oldBackend.#keyValueEntries.clear();}}async purge(): Promise<void> {- this.keyValueEntries.clear();+ this.#keyValueEntries.clear();this.updateTimestamps(true);}@@ -82,9 +83,9 @@ export class KeyValueStoreBackend extends BaseClient implements storage.KeyValue* preserve the bare `INPUT` key only.*/async purgeExceptInput(): Promise<void> {- for (const key of this.keyValueEntries.keys()) {+ for (const key of this.#keyValueEntries.keys()) {if (key !== KEY_VALUE_STORE_INPUT_KEY) {- this.keyValueEntries.delete(key);+ this.#keyValueEntries.delete(key);}}@@ -102,7 +103,7 @@ export class KeyValueStoreBackend extends BaseClient implements storage.KeyValueconst items: storage.KeyValueStoreItemData[] = [];- for (const record of this.keyValueEntries.values()) {+ for (const record of this.#keyValueEntries.values()) {const size = Buffer.byteLength(record.value);items.push({key: record.key,@@ -163,13 +164,13 @@ export class KeyValueStoreBackend extends BaseClient implements storage.KeyValueasync recordExists(key: string): Promise<boolean> {s.string().parse(key);- return this.keyValueEntries.has(key);+ return this.#keyValueEntries.has(key);}async getValue(key: string): Promise<storage.KeyValueStoreRecord | undefined> {s.string().parse(key);- const entry = this.keyValueEntries.get(key);+ const entry = this.#keyValueEntries.get(key);if (!entry) {return undefined;@@ -231,14 +232,14 @@ export class KeyValueStoreBackend extends BaseClient implements storage.KeyValue? Buffer.from(value, 'utf-8'): toBuffer(value as Buffer | ArrayBuffer | ArrayBufferView);- const _record = {+ const normalizedRecord = {extension,key,value: normalizedValue,contentType,} satisfies InternalKeyRecord;- this.keyValueEntries.set(key, _record);+ this.#keyValueEntries.set(key, normalizedRecord);this.updateTimestamps(true);}@@ -246,8 +247,8 @@ export class KeyValueStoreBackend extends BaseClient implements storage.KeyValueasync deleteValue(key: string): Promise<void> {s.string().parse(key);- if (this.keyValueEntries.has(key)) {- this.keyValueEntries.delete(key);+ if (this.#keyValueEntries.has(key)) {+ this.#keyValueEntries.delete(key);this.updateTimestamps(true);}}diff --git a/packages/core/src/memory-storage/resource-clients/request-queue.ts b/packages/core/src/memory-storage/resource-clients/request-queue.tsindex 9974bb779..b8b47dcfe 100644--- a/packages/core/src/memory-storage/resource-clients/request-queue.ts+++ b/packages/core/src/memory-storage/resource-clients/request-queue.ts@@ -68,8 +68,8 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQu* `addBatchOfRequests` pushes to it). Held by every mutating method as well as by `isEmpty`/* `isFinished`, whose head scan also prunes `forefrontRequestIds`.*/- private readonly queueStateMutex = new AsyncQueue();- private forefrontRequestIds: string[] = [];+ readonly #queueStateMutex = new AsyncQueue();+ #forefrontRequestIds: string[] = [];/*** IDs of requests currently fetched but not yet handled or reclaimed. A request in this set is@@ -79,9 +79,10 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQu* process and is never shared with another consumer, so there is no need for an expiring,* cross-process-visible lock — tracking in-progress requests in this set is enough.*/- private readonly inProgressRequestIds = new Set<string>();+ readonly #inProgressRequestIds = new Set<string>();- private readonly requests = new Map<string, InternalRequest>();+ readonly #requests = new Map<string, InternalRequest>();+ // kept as TS-private: storage-backend tests read this field at runtimeprivate readonly storageBackend: MemoryStorageBackend;constructor(options: RequestQueueBackendOptions) {@@ -100,7 +101,7 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQu// Serialize against other mutators (and the head scans in `isEmpty`/`isFinished`) so a concurrent// operation cannot observe half-cleared state — e.g. a forefront id whose request has already been// removed, which `listPendingHead` would then dereference as `undefined`.- await this.queueStateMutex.wait();+ await this.#queueStateMutex.wait();try {const storeIndex = this.storageBackend.requestQueueBackendCache.findIndex((queue) => queue.id === this.id);@@ -111,40 +112,40 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQu// Clear all in-memory state, consistent with `purge`. Clearing `requests` alone would// leave dangling ids in `forefrontRequestIds`/`inProgressRequestIds`, which a later head// scan would resolve to a missing request and dereference.- oldBackend.requests.clear();- oldBackend.forefrontRequestIds = [];- oldBackend.inProgressRequestIds.clear();+ oldBackend.#requests.clear();+ oldBackend.#forefrontRequestIds = [];+ oldBackend.#inProgressRequestIds.clear();}} finally {- this.queueStateMutex.shift();+ this.#queueStateMutex.shift();}}async purge(): Promise<void> {// Serialize against other mutators (and the head scans in `isEmpty`/`isFinished`) so a concurrent// operation cannot observe or repopulate half-cleared state across the `await` below.- await this.queueStateMutex.wait();+ await this.#queueStateMutex.wait();try {// Clear all in-memory state- this.requests.clear();- this.forefrontRequestIds = [];- this.inProgressRequestIds.clear();+ this.#requests.clear();+ this.#forefrontRequestIds = [];+ this.#inProgressRequestIds.clear();this.handledRequestCount = 0;this.pendingRequestCount = 0;this.updateTimestamps(true);} finally {- this.queueStateMutex.shift();+ this.#queueStateMutex.shift();}}private *requestKeyIterator(): IterableIterator<string> {- for (let i = this.forefrontRequestIds.length - 1; i >= 0; i--) {- yield this.forefrontRequestIds[i];+ for (let i = this.#forefrontRequestIds.length - 1; i >= 0; i--) {+ yield this.#forefrontRequestIds[i];}- for (const key of this.requests.keys()) {+ for (const key of this.#requests.keys()) {yield key;}}@@ -189,11 +190,11 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQuseenRequestIds.add(requestId);- const request = this.requests.get(requestId)!;+ const request = this.#requests.get(requestId)!;// Permanently-handled requests (`orderNo === null`) are in a terminal state and can be skipped.if (request.orderNo === null) {- if (this.forefrontRequestIds.includes(requestId)) {+ if (this.#forefrontRequestIds.includes(requestId)) {handledForefrontIds.add(requestId);}continue;@@ -201,7 +202,7 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQu// In progress (fetched but not yet handled or reclaimed) — skip it, but remember that the// queue is not truly empty.- if (this.inProgressRequestIds.has(requestId)) {+ if (this.#inProgressRequestIds.has(requestId)) {hasInProgressRequests = true;continue;}@@ -211,7 +212,7 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQu}}- this.forefrontRequestIds = this.forefrontRequestIds.filter((id) => !handledForefrontIds.has(id));+ this.#forefrontRequestIds = this.#forefrontRequestIds.filter((id) => !handledForefrontIds.has(id));return {items: items.sort((a, b) => a.orderNo! - b.orderNo!),@@ -222,7 +223,7 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQuasync fetchNextRequest(): Promise<storage.UpdateRequestSchema | undefined> {this.updateTimestamps(false);- await this.queueStateMutex.wait();+ await this.#queueStateMutex.wait();try {const {@@ -235,11 +236,11 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQu// Mark the request as in progress so it is not handed out again until it is handled or// reclaimed. The request keeps its `orderNo` (and thus its forefront / normal ordering).- this.inProgressRequestIds.add(head.id);+ this.#inProgressRequestIds.add(head.id);- return this._jsonToRequest(head.json) ?? undefined;+ return this.jsonToRequest(head.json) ?? undefined;} finally {- this.queueStateMutex.shift();+ this.#queueStateMutex.shift();}}@@ -253,7 +254,7 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQu// Serialize against other mutators (and the head scans in `isEmpty`/`isFinished`) so that the// shared `requests` map, `forefrontRequestIds` array and request counts are not corrupted by a// concurrent operation interleaving at one of the `await` points below.- await this.queueStateMutex.wait();+ await this.#queueStateMutex.wait();try {const result: storage.BatchAddRequestsResult = {@@ -262,9 +263,9 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQu};for (const model of requests) {- const requestModel = this._createInternalRequest(model, options.forefront);+ const requestModel = this.createInternalRequest(model, options.forefront);- const existingRequestWithId = this.requests.get(requestModel.id);+ const existingRequestWithId = this.#requests.get(requestModel.id);if (existingRequestWithId) {result.processedRequests.push({@@ -277,7 +278,7 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQucontinue;}- this.requests.set(requestModel.id, requestModel);+ this.#requests.set(requestModel.id, requestModel);if (requestModel.orderNo) {this.pendingRequestCount += 1;@@ -286,7 +287,7 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQu}if (options.forefront) {- this.forefrontRequestIds.push(requestModel.id);+ this.#forefrontRequestIds.push(requestModel.id);}result.processedRequests.push({@@ -303,7 +304,7 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQureturn result;} finally {- this.queueStateMutex.shift();+ this.#queueStateMutex.shift();}}@@ -311,8 +312,8 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQus.string().parse(uniqueKey);this.updateTimestamps(false);const id = uniqueKeyToRequestId(uniqueKey);- const json = this.requests.get(id)?.json;- return this._jsonToRequest(json);+ const json = this.#requests.get(id)?.json;+ return this.jsonToRequest(json);}async markRequestAsHandled(request: storage.UpdateRequestSchema): Promise<storage.QueueOperationInfo | undefined> {@@ -322,12 +323,12 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQu// Serialize against other mutators (and the head scans in `isEmpty`/`isFinished`) so the shared// `requests` map, `inProgressRequestIds` set and request counts stay consistent across the// `await` points below.- await this.queueStateMutex.wait();+ await this.#queueStateMutex.wait();try {const id = uniqueKeyToRequestId(request.uniqueKey);- const existingRequest = this.requests.get(id);+ const existingRequest = this.#requests.get(id);// The request must exist to be marked as handled. We intentionally do NOT require it to still// be in progress: marking an already-released request handled must still succeed, otherwise@@ -340,12 +341,12 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQuconst wasAlreadyHandled = existingRequest.orderNo === null;const handledAt = request.handledAt ?? new Date().toISOString();- const requestModel = this._createInternalRequest({ ...request, handledAt }, false);+ const requestModel = this.createInternalRequest({ ...request, handledAt }, false);- this.requests.set(id, requestModel);+ this.#requests.set(id, requestModel);// The request is no longer in progress for this client.- this.inProgressRequestIds.delete(id);+ this.#inProgressRequestIds.delete(id);if (!wasAlreadyHandled) {this.pendingRequestCount -= 1;@@ -360,7 +361,7 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQuwasAlreadyPresent: true,};} finally {- this.queueStateMutex.shift();+ this.#queueStateMutex.shift();}}@@ -375,12 +376,12 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQu// Serialize against other mutators (and the head scans in `isEmpty`/`isFinished`) so the shared// `requests` map, `forefrontRequestIds` array and `inProgressRequestIds` set stay consistent// across the `await` points below.- await this.queueStateMutex.wait();+ await this.#queueStateMutex.wait();try {const id = uniqueKeyToRequestId(request.uniqueKey);- const existingRequest = this.requests.get(id);+ const existingRequest = this.#requests.get(id);// The request must exist and not already be handled to be reclaimed. As with// `markRequestAsHandled`, we do NOT require it to still be in progress — returning an@@ -392,15 +393,15 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQu// Reclaiming resets the `orderNo` to a fresh timestamp, restoring the request to the queue// (at the front if `forefront`).- const requestModel = this._createInternalRequest(request, options.forefront);+ const requestModel = this.createInternalRequest(request, options.forefront);- this.requests.set(id, requestModel);+ this.#requests.set(id, requestModel);// The request is no longer in progress for this client.- this.inProgressRequestIds.delete(id);+ this.#inProgressRequestIds.delete(id);if (options.forefront) {- this.forefrontRequestIds.push(id);+ this.#forefrontRequestIds.push(id);}this.updateTimestamps(true);@@ -411,7 +412,7 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQuwasAlreadyPresent: true,};} finally {- this.queueStateMutex.shift();+ this.#queueStateMutex.shift();}}@@ -426,13 +427,13 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQu//// `listPendingHead` prunes `forefrontRequestIds` as it scans, so we must hold the queue-state mutex to avoid// racing a concurrent mutator (e.g. `addBatchOfRequests`) at its `await` points.- await this.queueStateMutex.wait();+ await this.#queueStateMutex.wait();try {const { items } = await this.listPendingHead(1);return items.length === 0;} finally {- this.queueStateMutex.shift();+ this.#queueStateMutex.shift();}}@@ -448,13 +449,13 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQu//// `listPendingHead` prunes `forefrontRequestIds` as it scans, so we must hold the queue-state mutex to avoid// racing a concurrent mutator (e.g. `addBatchOfRequests`) at its `await` points.- await this.queueStateMutex.wait();+ await this.#queueStateMutex.wait();try {const { items, hasInProgressRequests } = await this.listPendingHead(1, true);return items.length === 0 && !hasInProgressRequests;} finally {- this.queueStateMutex.shift();+ this.#queueStateMutex.shift();}}@@ -468,13 +469,13 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQu// `listPendingHead` prunes `forefrontRequestIds` as it scans, so we must hold the queue-state// mutex to avoid racing a concurrent mutator at its `await` points.- await this.queueStateMutex.wait();+ await this.#queueStateMutex.wait();try {const { items } = await this.listPendingHead(Number.POSITIVE_INFINITY);- return items.map((request) => this._jsonToRequest<storage.UpdateRequestSchema>(request.json)!);+ return items.map((request) => this.jsonToRequest<storage.UpdateRequestSchema>(request.json)!);} finally {- this.queueStateMutex.shift();+ this.#queueStateMutex.shift();}}@@ -487,7 +488,7 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQumodifiedAt: this.modifiedAt,name: this.name,pendingRequestCount: this.pendingRequestCount,- totalRequestCount: this.requests.size,+ totalRequestCount: this.#requests.size,};}@@ -499,14 +500,14 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQu}}- private _jsonToRequest<T>(requestJson?: string): T | undefined {+ private jsonToRequest<T>(requestJson?: string): T | undefined {if (!requestJson) return undefined;const request = JSON.parse(requestJson);return purgeNullsFromObject(request);}- private _createInternalRequest(request: storage.RequestSchema, forefront?: boolean): InternalRequest {- const orderNo = this._calculateOrderNo(request, forefront);+ private createInternalRequest(request: storage.RequestSchema, forefront?: boolean): InternalRequest {+ const orderNo = this.calculateOrderNo(request, forefront);const id = uniqueKeyToRequestId(request.uniqueKey);if (request.id && request.id !== id) {@@ -525,7 +526,7 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQu};}- private _calculateOrderNo(request: storage.RequestSchema, forefront?: boolean) {+ private calculateOrderNo(request: storage.RequestSchema, forefront?: boolean) {if (request.handledAt) return null;const timestamp = Date.now();diff --git a/packages/core/src/owned_or_injected.ts b/packages/core/src/owned_or_injected.tsindex 15ce7a33d..3363d8045 100644--- a/packages/core/src/owned_or_injected.ts+++ b/packages/core/src/owned_or_injected.ts@@ -10,14 +10,14 @@* @internal*/export class OwnedOrInjected<Injected, Owned extends Injected = Injected> {- private _value: Injected | undefined;- private readonly _owned: boolean;- private _present: boolean;+ #value: Injected | undefined;+ readonly #owned: boolean;+ #present: boolean;private constructor(value: Injected | undefined, owned: boolean, present: boolean) {- this._value = value;- this._owned = owned;- this._present = present;+ this.#value = value;+ this.#owned = owned;+ this.#present = present;}/**@@ -50,7 +50,7 @@ export class OwnedOrInjected<Injected, Owned extends Injected = Injected> {* crawler-built defaults, `false` for user-injected instances.*/get isOwned(): boolean {- return this._owned;+ return this.#owned;}/**@@ -58,7 +58,7 @@ export class OwnedOrInjected<Injected, Owned extends Injected = Injected> {* (e.g. a lazily-opened request queue before its first use).*/get isPresent(): boolean {- return this._present;+ return this.#present;}/**@@ -66,11 +66,11 @@ export class OwnedOrInjected<Injected, Owned extends Injected = Injected> {* expect a lazily-filled owned slot should read {@apilink OwnedOrInjected.maybeValue|`maybeValue`} instead.*/get value(): Injected {- if (!this._present) {+ if (!this.#present) {throw new Error('OwnedOrInjected value is not initialized yet');}- return this._value as Injected;+ return this.#value as Injected;}/**@@ -79,7 +79,7 @@ export class OwnedOrInjected<Injected, Owned extends Injected = Injected> {* a possibly-empty slot without the `isPresent ? value : …` dance.*/get maybeValue(): Injected | undefined {- return this._present ? (this._value as Injected) : undefined;+ return this.#present ? (this.#value as Injected) : undefined;}/**@@ -88,16 +88,16 @@ export class OwnedOrInjected<Injected, Owned extends Injected = Injected> {* would silently orphan the previous instance's lifecycle).*/set(value: Owned): Owned {- if (!this._owned) {+ if (!this.#owned) {throw new Error('Cannot set() a borrowed OwnedOrInjected value');}- if (this._present) {+ if (this.#present) {throw new Error('OwnedOrInjected value is already initialized');}- this._value = value;- this._present = true;+ this.#value = value;+ this.#present = true;return value;}@@ -107,10 +107,10 @@ export class OwnedOrInjected<Injected, Owned extends Injected = Injected> {* owns a present instance — a no-op for a borrowed instance or an owned-but-not-yet-built slot.*/async ifOwned<R>(fn: (value: Owned) => R | Promise<R>): Promise<R | undefined> {- if (!this._owned || !this._present) {+ if (!this.#owned || !this.#present) {return undefined;}- return fn(this._value as Owned);+ return fn(this.#value as Owned);}}diff --git a/packages/core/src/proxy_configuration.ts b/packages/core/src/proxy_configuration.tsindex a336ad5dc..6a55f6d81 100644--- a/packages/core/src/proxy_configuration.ts+++ b/packages/core/src/proxy_configuration.ts@@ -78,9 +78,9 @@ export interface IProxyConfiguration {*/export class ProxyConfiguration implements IProxyConfiguration {readonly isManInTheMiddle = false;- private nextCustomUrlIndex = 0;- private proxyUrls?: UrlList;- private newUrlFunction?: ProxyConfigurationFunction;+ #nextCustomUrlIndex = 0;+ #proxyUrls?: UrlList;+ #newUrlFunction?: ProxyConfigurationFunction;/*** Creates a {@apilink ProxyConfiguration} instance based on the provided options. Proxy servers are used to prevent target websites from@@ -125,8 +125,8 @@ export class ProxyConfiguration implements IProxyConfiguration {if (proxyUrls && newUrlFunction) this.throwCannotCombineCustomMethods();if (!proxyUrls && !newUrlFunction && validateRequired) this.throwNoOptionsProvided();- this.proxyUrls = proxyUrls;- this.newUrlFunction = newUrlFunction;+ this.#proxyUrls = proxyUrls;+ this.#newUrlFunction = newUrlFunction;}/**@@ -160,7 +160,7 @@ export class ProxyConfiguration implements IProxyConfiguration {* For example, `http://bob:[email protected]:8000`*/async newUrl(options?: NewUrlOptions): Promise<string | undefined> {- if (this.newUrlFunction) {+ if (this.#newUrlFunction) {return (await this.callNewUrlFunction({ request: options?.request })) ?? undefined;}@@ -168,14 +168,14 @@ export class ProxyConfiguration implements IProxyConfiguration {}private handleProxyUrlsList(): string | null {- return this.proxyUrls![this.nextCustomUrlIndex++ % this.proxyUrls!.length];+ return this.#proxyUrls![this.#nextCustomUrlIndex++ % this.#proxyUrls!.length];}/*** Calls the custom newUrlFunction and checks format of its return value*/private async callNewUrlFunction(options?: { request?: Request }) {- const proxyUrl = await this.newUrlFunction!(options);+ const proxyUrl = await this.#newUrlFunction!(options);try {if (proxyUrl) {new URL(proxyUrl); // eslint-disable-line no-newdiff --git a/packages/core/src/recoverable_state.ts b/packages/core/src/recoverable_state.tsindex 22b120daf..dbb0ff37a 100644--- a/packages/core/src/recoverable_state.ts+++ b/packages/core/src/recoverable_state.ts@@ -72,16 +72,16 @@ export interface RecoverableStateOptions<* The class automatically hooks into the event system to persist state when needed.*/export class RecoverableState<TStateModel = Record<string, unknown>> {- private readonly defaultState: TStateModel;- private state: TStateModel | null = null;- private readonly persistenceEnabled: boolean;- private readonly persistStateKey: string;- private readonly persistStateKvsName?: string;- private readonly persistStateKvsId?: string;- private keyValueStore: KeyValueStore | null = null;- private readonly log: CrawleeLogger;- private readonly serialize: (state: TStateModel) => string;- private readonly deserialize: (serializedState: string) => TStateModel;+ readonly #defaultState: TStateModel;+ #state: TStateModel | null = null;+ readonly #persistenceEnabled: boolean;+ readonly #persistStateKey: string;+ readonly #persistStateKvsName?: string;+ readonly #persistStateKvsId?: string;+ #keyValueStore: KeyValueStore | null = null;+ readonly #log: CrawleeLogger;+ readonly #serialize: (state: TStateModel) => string;+ readonly #deserialize: (serializedState: string) => TStateModel;/*** Initialize a new recoverable state object.@@ -89,14 +89,14 @@ export class RecoverableState<TStateModel = Record<string, unknown>> {* @param options Configuration options for the recoverable state*/constructor(options: RecoverableStateOptions<TStateModel>) {- this.defaultState = options.defaultState;- this.persistStateKey = options.persistStateKey;- this.persistenceEnabled = options.persistenceEnabled ?? false;- this.persistStateKvsName = options.persistStateKvsName;- this.persistStateKvsId = options.persistStateKvsId;- this.log = options.logger ?? serviceLocator.getLogger().child({ prefix: 'RecoverableState' });- this.serialize = options.serialize ?? JSON.stringify;- this.deserialize = options.deserialize ?? JSON.parse;+ this.#defaultState = options.defaultState;+ this.#persistStateKey = options.persistStateKey;+ this.#persistenceEnabled = options.persistenceEnabled ?? false;+ this.#persistStateKvsName = options.persistStateKvsName;+ this.#persistStateKvsId = options.persistStateKvsId;+ this.#log = options.logger ?? serviceLocator.getLogger().child({ prefix: 'RecoverableState' });+ this.#serialize = options.serialize ?? JSON.stringify;+ this.#deserialize = options.deserialize ?? JSON.parse;this.persistState = this.persistState.bind(this);}@@ -110,24 +110,24 @@ export class RecoverableState<TStateModel = Record<string, unknown>> {* @returns The loaded state object*/async initialize(): Promise<TStateModel> {- if (this.state !== null && this.state !== undefined) {+ if (this.#state !== null && this.#state !== undefined) {return this.currentValue;}- if (!this.persistenceEnabled) {- this.state = this.deserialize(this.serialize(this.defaultState));+ if (!this.#persistenceEnabled) {+ this.#state = this.#deserialize(this.#serialize(this.#defaultState));return this.currentValue;}let kvsIdentifier: { name: string } | { id: string } | null = null;- if (this.persistStateKvsName) {- kvsIdentifier = { name: this.persistStateKvsName };- } else if (this.persistStateKvsId) {- kvsIdentifier = { id: this.persistStateKvsId };+ if (this.#persistStateKvsName) {+ kvsIdentifier = { name: this.#persistStateKvsName };+ } else if (this.#persistStateKvsId) {+ kvsIdentifier = { id: this.#persistStateKvsId };}- this.keyValueStore = await KeyValueStore.open(kvsIdentifier, {+ this.#keyValueStore = await KeyValueStore.open(kvsIdentifier, {configuration: serviceLocator.getConfiguration(),});@@ -147,7 +147,7 @@ export class RecoverableState<TStateModel = Record<string, unknown>> {* and persists the current state one last time.*/async teardown(): Promise<void> {- if (!this.persistenceEnabled || !this.persistState) {+ if (!this.#persistenceEnabled || !this.persistState) {return;}@@ -160,11 +160,11 @@ export class RecoverableState<TStateModel = Record<string, unknown>> {* Get the current state.*/get currentValue(): TStateModel {- if (this.state === null) {+ if (this.#state === null) {throw new Error('Recoverable state has not yet been loaded');}- return this.state;+ return this.#state;}/**@@ -174,14 +174,14 @@ export class RecoverableState<TStateModel = Record<string, unknown>> {* clears the persisted state from the KeyValueStore.*/async reset(): Promise<void> {- this.state = this.deserialize(this.serialize(this.defaultState));+ this.#state = this.#deserialize(this.#serialize(this.#defaultState));- if (this.persistenceEnabled) {- if (this.keyValueStore === null) {+ if (this.#persistenceEnabled) {+ if (this.#keyValueStore === null) {throw new Error('Recoverable state has not yet been initialized');}- await this.keyValueStore.setValue(this.persistStateKey, null);+ await this.#keyValueStore.setValue(this.#persistStateKey, null);}}@@ -194,14 +194,14 @@ export class RecoverableState<TStateModel = Record<string, unknown>> {* @param eventData Optional data associated with a PERSIST_STATE event*/async persistState(eventData?: { isMigrating: boolean }): Promise<void> {- this.log.debug(`Persisting state of the RecoverableState (eventData=${JSON.stringify(eventData)}).`);+ this.#log.debug(`Persisting state of the RecoverableState (eventData=${JSON.stringify(eventData)}).`);- if (this.keyValueStore === null || this.state === null) {+ if (this.#keyValueStore === null || this.#state === null) {throw new Error('Recoverable state has not yet been initialized');}- if (this.persistenceEnabled) {- await this.keyValueStore.setValue(this.persistStateKey, this.serialize(this.state), {+ if (this.#persistenceEnabled) {+ await this.#keyValueStore.setValue(this.#persistStateKey, this.#serialize(this.#state), {contentType: 'text/plain', // HACK - the result is expected to be JSON, but we do this to avoid the implicit JSON.parse in `KeyValueStore.getValue`});}@@ -211,15 +211,15 @@ export class RecoverableState<TStateModel = Record<string, unknown>> {* Load the saved state from the KeyValueStore*/private async loadSavedState(): Promise<void> {- if (this.keyValueStore === null) {+ if (this.#keyValueStore === null) {throw new Error('Recoverable state has not yet been initialized');}- const storedState = await this.keyValueStore.getValue(this.persistStateKey);+ const storedState = await this.#keyValueStore.getValue(this.#persistStateKey);if (storedState === null || storedState === undefined) {- this.state = this.deserialize(this.serialize(this.defaultState));+ this.#state = this.#deserialize(this.#serialize(this.#defaultState));} else {- this.state = this.deserialize(storedState as string);+ this.#state = this.#deserialize(storedState as string);}}}diff --git a/packages/core/src/request.ts b/packages/core/src/request.tsindex 53620fe67..ac1a2ac51 100644--- a/packages/core/src/request.ts+++ b/packages/core/src/request.ts@@ -121,7 +121,7 @@ class CrawleeRequest<UserData extends Dictionary = Dictionary> {headers?: Record<string, string>;/** Private store for the custom user data assigned to the request. */- private _userData: Record<string, any> = {};+ #userData: Record<string, any> = {};/*** Custom user data assigned to the request.@@ -225,37 +225,35 @@ class CrawleeRequest<UserData extends Dictionary = Dictionary> {userData.label = label;}+ this.#userData = { __crawlee: {}, ...userData };++ // `userData` must stay an enumerable own accessor — serialization in the storages relies on itObject.defineProperties(this, {- _userData: {- value: { __crawlee: {}, ...userData },- enumerable: false,- writable: true,- },userData: {- get: () => this._userData,+ get: () => this.#userData,set: (value: Record<string, any>) => {Object.defineProperties(value, {__crawlee: {- value: this._userData.__crawlee,+ value: this.#userData.__crawlee,enumerable: false,writable: true,},toJSON: {value: () => {- if (Object.keys(this._userData.__crawlee).length > 0) {+ if (Object.keys(this.#userData.__crawlee).length > 0) {return {- ...this._userData,- __crawlee: this._userData.__crawlee,+ ...this.#userData,+ __crawlee: this.#userData.__crawlee,};}- return this._userData;+ return this.#userData;},enumerable: false,writable: true,},});- this._userData = value;+ this.#userData = value;},enumerable: true,},diff --git a/packages/core/src/router.ts b/packages/core/src/router.tsindex 520baec54..38a54bbdb 100644--- a/packages/core/src/router.ts+++ b/packages/core/src/router.ts@@ -279,10 +279,10 @@ export class Router<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'>,Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>,> {- private readonly routes: Map<string | symbol, (ctx: any) => Awaitable<void>> = new Map();- private readonly schemas: Map<string | symbol, StandardSchemaV1> = new Map();- private readonly timeouts: Map<string | symbol, number> = new Map();- private readonly middlewares: ((ctx: Context) => Awaitable<void>)[] = [];+ readonly #routes: Map<string | symbol, (ctx: any) => Awaitable<void>> = new Map();+ readonly #schemas: Map<string | symbol, StandardSchemaV1> = new Map();+ readonly #timeouts: Map<string | symbol, number> = new Map();+ readonly #middlewares: ((ctx: Context) => Awaitable<void>)[] = [];/*** use Router.create() instead!@@ -315,10 +315,10 @@ export class Router<addHandler(label: string | symbol, handler: (ctx: any) => Awaitable<void>, options: RouteOptions = {}): void {this.validate(label);- this.routes.set(label, handler);+ this.#routes.set(label, handler);if (options.requestHandlerTimeoutSecs !== undefined) {- this.timeouts.set(label, options.requestHandlerTimeoutSecs);+ this.#timeouts.set(label, options.requestHandlerTimeoutSecs);}}@@ -334,10 +334,10 @@ export class Router<UserData extends Dictionary = DefaultRouteUserData<Routes, GetUserDataFromRequest<Context['request']>>,>(handler: (ctx: RouterHandlerContext<Context, UserData, Routes>) => Awaitable<void>, options: RouteOptions = {}) {this.validate(defaultRoute);- this.routes.set(defaultRoute, handler);+ this.#routes.set(defaultRoute, handler);if (options.requestHandlerTimeoutSecs !== undefined) {- this.timeouts.set(defaultRoute, options.requestHandlerTimeoutSecs);+ this.#timeouts.set(defaultRoute, options.requestHandlerTimeoutSecs);}}@@ -348,21 +348,21 @@ export class Router<*/getSchema(label?: string | symbol): StandardSchemaV1 | undefined {if (label != null) {- const schema = this.schemas.get(label);+ const schema = this.#schemas.get(label);if (schema) {return schema;}// A label with its own route is fully specified; don't fall back to the default-route schema.- if (this.routes.has(label)) {+ if (this.#routes.has(label)) {return undefined;}}// Requests with no route of their own fall through to the default handler, so validate their// `userData` against the default-route schema, if one was registered.- return this.schemas.get(defaultRoute);+ return this.#schemas.get(defaultRoute);}/**@@ -370,7 +370,7 @@ export class Router<* Multiple middlewares can be registered, they will be fired in the same order.*/use(middleware: (ctx: Context) => Awaitable<void>) {- this.middlewares.push(middleware);+ this.#middlewares.push(middleware);}/**@@ -380,11 +380,11 @@ export class Router<* the default route asked for. Used by the crawler; not meant to be called directly.*/getTimeoutSecs(label?: string | symbol): number | undefined {- if (label && this.routes.has(label)) {- return this.timeouts.get(label);+ if (label && this.#routes.has(label)) {+ return this.#timeouts.get(label);}- return this.timeouts.get(defaultRoute);+ return this.#timeouts.get(defaultRoute);}/**@@ -392,19 +392,19 @@ export class Router<* The crawler needs an upper bound up front, before it knows which routes a run will actually hit.*/getMaxTimeoutSecs(): number | undefined {- return this.timeouts.size > 0 ? Math.max(...this.timeouts.values()) : undefined;+ return this.#timeouts.size > 0 ? Math.max(...this.#timeouts.values()) : undefined;}/*** Returns route handler for given label. If no label is provided, the default request handler will be returned.*/getHandler(label?: string | symbol): (ctx: Context) => Awaitable<void> {- if (label && this.routes.has(label)) {- return this.routes.get(label)!;+ if (label && this.#routes.has(label)) {+ return this.#routes.get(label)!;}- if (this.routes.has(defaultRoute)) {- return this.routes.get(defaultRoute)!;+ if (this.#routes.has(defaultRoute)) {+ return this.#routes.get(defaultRoute)!;}throw new MissingRouteError(@@ -435,7 +435,7 @@ export class Router<* Throws when the label already exists in our registry.*/private validate(label: string | symbol) {- if (this.routes.has(label)) {+ if (this.#routes.has(label)) {const message =label === defaultRoute? `Default route is already defined!`@@ -505,7 +505,7 @@ export class Router<if (typeof value === 'function') {router.addHandler(label as string, value as (ctx: any) => Awaitable<void>);} else {- router.schemas.set(label, value);+ router.#schemas.set(label, value);}}@@ -515,7 +515,7 @@ export class Router<await router.validateRequest(context);- for (const middleware of router.middlewares) {+ for (const middleware of router.#middlewares) {await middleware(context);}diff --git a/packages/core/src/serialization.ts b/packages/core/src/serialization.tsindex bb640726d..eabf34a28 100644--- a/packages/core/src/serialization.ts+++ b/packages/core/src/serialization.ts@@ -16,34 +16,32 @@ type Chain = ReturnType<typeof StreamArray.withParser>;* @internal*/class ArrayToJson<T> extends Readable {- private offset = 0;- private readonly batchSize: number;+ #offset = 0;+ readonly #batchSize: number;+ readonly #data: T[];- constructor(- private data: T[],- options: { batchSize?: number } = {},- ) {+ constructor(data: T[], options: { batchSize?: number } = {}) {super({...options,autoDestroy: true,emitClose: true,});const { batchSize = 10000 } = options;- this.batchSize = batchSize;- this.data = data;+ this.#batchSize = batchSize;+ this.#data = data;this.push('[');}override _read(): void {try {- const items = this.data.slice(this.offset, this.offset + this.batchSize);+ const items = this.#data.slice(this.#offset, this.#offset + this.#batchSize);if (items.length) {const json = JSON.stringify(items);// Strip brackets to flatten the batch.const itemString = json.substring(1, json.length - 1);- if (this.offset > 0) this.push(',', 'utf8');+ if (this.#offset > 0) this.push(',', 'utf8');this.push(itemString, 'utf8');- this.offset += this.batchSize;+ this.#offset += this.#batchSize;} else {this.push(']');this.push(null);diff --git a/packages/core/src/service_locator.ts b/packages/core/src/service_locator.tsindex bd5937e7d..cad4b8a3f 100644--- a/packages/core/src/service_locator.ts+++ b/packages/core/src/service_locator.ts@@ -123,17 +123,17 @@ interface ServiceLocatorInterface {* ```*/export class ServiceLocator implements ServiceLocatorInterface {- private configuration?: Configuration;- private eventManager?: EventManager;- private storageBackend?: StorageBackend;- private logger?: CrawleeLogger;+ #configuration?: Configuration;+ #eventManager?: EventManager;+ #storageBackend?: StorageBackend;+ #logger?: CrawleeLogger;/*** Unified storage instance manager for Dataset, KeyValueStore, and RequestQueue.* Shared across all ServiceLocator instances (global singleton), matching crawlee-python.* Per-crawler isolation is achieved via `clientCacheKey`, not separate manager instances.*/- private static storageInstanceManager?: StorageInstanceManager;+ static #storageInstanceManager?: StorageInstanceManager;/*** Creates a new ServiceLocator instance.@@ -149,76 +149,76 @@ export class ServiceLocator implements ServiceLocatorInterface {storageBackend?: StorageBackend,logger?: CrawleeLogger,) {- this.configuration = configuration;- this.eventManager = eventManager;- this.storageBackend = storageBackend;- this.logger = logger;+ this.#configuration = configuration;+ this.#eventManager = eventManager;+ this.#storageBackend = storageBackend;+ this.#logger = logger;}getConfiguration(): Configuration {- if (!this.configuration) {+ if (!this.#configuration) {this.getLogger().debug('No configuration set, implicitly creating and using default Configuration.');- this.configuration = new Configuration();+ this.#configuration = new Configuration();}- return this.configuration;+ return this.#configuration;}setConfiguration(configuration: Configuration): void {// Same instance, no need to do anything- if (this.configuration === configuration) {+ if (this.#configuration === configuration) {return;}// Already have a different configuration that was retrieved- if (this.configuration) {- throw new ServiceConflictError('Configuration', configuration, this.configuration);+ if (this.#configuration) {+ throw new ServiceConflictError('Configuration', configuration, this.#configuration);}- this.configuration = configuration;+ this.#configuration = configuration;}getEventManager(): EventManager {- if (!this.eventManager) {+ if (!this.#eventManager) {this.getLogger().debug('No event manager set, implicitly creating and using default LocalEventManager.');- if (!this.configuration) {+ if (!this.#configuration) {this.getLogger().warning('Implicit creation of event manager will implicitly set configuration as side effect. ' +'It is advised to explicitly first set the configuration instead.',);}- this.eventManager = LocalEventManager.fromConfiguration(this.getConfiguration());+ this.#eventManager = LocalEventManager.fromConfiguration(this.getConfiguration());}- return this.eventManager;+ return this.#eventManager;}setEventManager(eventManager: EventManager): void {// Same instance, no need to do anything- if (this.eventManager === eventManager) {+ if (this.#eventManager === eventManager) {return;}// Already have a different event manager that was retrieved- if (this.eventManager) {- throw new ServiceConflictError('EventManager', eventManager, this.eventManager);+ if (this.#eventManager) {+ throw new ServiceConflictError('EventManager', eventManager, this.#eventManager);}- this.eventManager = eventManager;+ this.#eventManager = eventManager;}getStorageBackend(): StorageBackend {- if (!this.storageBackend) {+ if (!this.#storageBackend) {this.getLogger().debug('No storage backend set, implicitly creating and using the default storage backend ' +'(FileSystemStorageBackend when persistStorage is enabled, MemoryStorageBackend otherwise).',);- if (!this.configuration) {+ if (!this.#configuration) {this.getLogger().warning('Implicit creation of storage backend will implicitly set configuration as side effect. ' +'It is advised to explicitly first set the configuration instead.',);}const configuration = this.getConfiguration();- this.storageBackend = configuration.persistStorage+ this.#storageBackend = configuration.persistStorage? new FileSystemStorageBackend({localDataDirectory: configuration.storageDir,logger: this.getLogger().child({ prefix: 'FileSystemStorageBackend' }),@@ -227,40 +227,40 @@ export class ServiceLocator implements ServiceLocatorInterface {logger: this.getLogger().child({ prefix: 'MemoryStorageBackend' }),});}- return this.storageBackend;+ return this.#storageBackend;}setStorageBackend(storageBackend: StorageBackend): void {// Same instance, no need to do anything- if (this.storageBackend === storageBackend) {+ if (this.#storageBackend === storageBackend) {return;}// Already have a different storage backend that was retrieved- if (this.storageBackend) {- throw new ServiceConflictError('StorageBackend', storageBackend, this.storageBackend);+ if (this.#storageBackend) {+ throw new ServiceConflictError('StorageBackend', storageBackend, this.#storageBackend);}- this.storageBackend = storageBackend;+ this.#storageBackend = storageBackend;}getLogger(): CrawleeLogger {- if (!this.logger) {- this.logger = new ApifyLogAdapter(log);+ if (!this.#logger) {+ this.#logger = new ApifyLogAdapter(log);}- return this.logger;+ return this.#logger;}setLogger(logger: CrawleeLogger): void {- if (this.logger === logger) {+ if (this.#logger === logger) {return;}- if (this.logger) {- throw new ServiceConflictError('Logger', logger, this.logger);+ if (this.#logger) {+ throw new ServiceConflictError('Logger', logger, this.#logger);}- this.logger = logger;+ this.#logger = logger;}getChildLog(prefix: string): CrawleeLogger {@@ -268,19 +268,19 @@ export class ServiceLocator implements ServiceLocatorInterface {}getStorageInstanceManager(): StorageInstanceManager {- if (!ServiceLocator.storageInstanceManager) {- ServiceLocator.storageInstanceManager = new StorageInstanceManager();+ if (!ServiceLocator.#storageInstanceManager) {+ ServiceLocator.#storageInstanceManager = new StorageInstanceManager();}- return ServiceLocator.storageInstanceManager;+ return ServiceLocator.#storageInstanceManager;}reset(): void {- this.configuration = undefined;- this.eventManager = undefined;- this.storageBackend = undefined;- this.logger = undefined;- ServiceLocator.storageInstanceManager?.clearCache();- ServiceLocator.storageInstanceManager = undefined;+ this.#configuration = undefined;+ this.#eventManager = undefined;+ this.#storageBackend = undefined;+ this.#logger = undefined;+ ServiceLocator.#storageInstanceManager?.clearCache();+ ServiceLocator.#storageInstanceManager = undefined;}}diff --git a/packages/core/src/session_pool/session.ts b/packages/core/src/session_pool/session.tsindex e5529ba8d..27c627c72 100644--- a/packages/core/src/session_pool/session.ts+++ b/packages/core/src/session_pool/session.ts@@ -86,61 +86,61 @@ export interface SessionOptions {export class Session implements ISession {readonly id: string;readonly userData: Dictionary;- private _maxErrorScore: number;- private _errorScoreDecrement: number;- private _createdAt: Date;- private _expiresAt: Date;- private _usageCount: number;- private _maxUsageCount: number;- private _errorScore: number;- private _retired = false;- private _proxyInfo?: ProxyInfo;- private _cookieJar: CookieJar;- private _fingerprint?: SessionFingerprint;- private log: CrawleeLogger;+ #maxErrorScore: number;+ #errorScoreDecrement: number;+ #createdAt: Date;+ #expiresAt: Date;+ #usageCount: number;+ #maxUsageCount: number;+ #errorScore: number;+ #retired = false;+ #proxyInfo?: ProxyInfo;+ #cookieJar: CookieJar;+ #fingerprint?: SessionFingerprint;+ #log: CrawleeLogger;get errorScore() {- return this._errorScore;+ return this.#errorScore;}get usageCount() {- return this._usageCount;+ return this.#usageCount;}get maxErrorScore() {- return this._maxErrorScore;+ return this.#maxErrorScore;}get errorScoreDecrement() {- return this._errorScoreDecrement;+ return this.#errorScoreDecrement;}get expiresAt() {- return this._expiresAt;+ return this.#expiresAt;}get createdAt() {- return this._createdAt;+ return this.#createdAt;}get maxUsageCount() {- return this._maxUsageCount;+ return this.#maxUsageCount;}get cookieJar() {- return this._cookieJar;+ return this.#cookieJar;}get proxyInfo() {- return this._proxyInfo;+ return this.#proxyInfo;}get fingerprint(): SessionFingerprint | undefined {- return this._fingerprint;+ return this.#fingerprint;}set fingerprint(fingerprint: SessionFingerprint | undefined) {- this._fingerprint = fingerprint;+ this.#fingerprint = fingerprint;}/**@@ -148,7 +148,7 @@ export class Session implements ISession {* a retired session is never picked by the pool and cannot be revived via `markGood()`.*/get retired() {- return this._retired;+ return this.#retired;}/**@@ -195,23 +195,23 @@ export class Session implements ISession {const { expiresAt = getDefaultCookieExpirationDate(maxAgeSecs) } = options;- this.log = log.child({ prefix: 'Session' });+ this.#log = log.child({ prefix: 'Session' });- this._cookieJar = (cookieJar.setCookie as unknown) ? cookieJar : CookieJar.fromJSON(JSON.stringify(cookieJar));- this._proxyInfo = proxyInfo;- this._fingerprint = fingerprint;+ this.#cookieJar = (cookieJar.setCookie as unknown) ? cookieJar : CookieJar.fromJSON(JSON.stringify(cookieJar));+ this.#proxyInfo = proxyInfo;+ this.#fingerprint = fingerprint;this.id = id;this.userData = userData;- this._maxErrorScore = maxErrorScore;- this._errorScoreDecrement = errorScoreDecrement;+ this.#maxErrorScore = maxErrorScore;+ this.#errorScoreDecrement = errorScoreDecrement;// Internal- this._expiresAt = expiresAt;- this._createdAt = createdAt;- this._usageCount = usageCount; // indicates how many times the session has been used- this._errorScore = errorScore; // indicates number of markBaded request with the session- this._maxUsageCount = maxUsageCount;- this._retired = retired;+ this.#expiresAt = expiresAt;+ this.#createdAt = createdAt;+ this.#usageCount = usageCount; // indicates how many times the session has been used+ this.#errorScore = errorScore; // indicates number of markBaded request with the session+ this.#maxUsageCount = maxUsageCount;+ this.#retired = retired;}/**@@ -244,7 +244,7 @@ export class Session implements ISession {* Session is usable when it is not retired, not expired, not blocked and the maximum usage count has not be reached.*/isUsable(): boolean {- return !this._retired && !this.isBlocked() && !this.isExpired() && !this.isMaxUsageCountReached();+ return !this.#retired && !this.isBlocked() && !this.isExpired() && !this.isMaxUsageCountReached();}/**@@ -252,10 +252,10 @@ export class Session implements ISession {* It increases `usageCount` and potentially lowers the `errorScore` by the `errorScoreDecrement`.*/markGood() {- this._usageCount += 1;+ this.#usageCount += 1;- if (this._errorScore > 0) {- this._errorScore -= this._errorScoreDecrement;+ if (this.#errorScore > 0) {+ this.#errorScore -= this.#errorScoreDecrement;}this.maybeSelfRetire();@@ -269,9 +269,9 @@ export class Session implements ISession {return {id: this.id,cookieJar: this.cookieJar.toJSON()!,- proxyInfo: this._proxyInfo,+ proxyInfo: this.#proxyInfo,userData: this.userData,- fingerprint: this._fingerprint,+ fingerprint: this.#fingerprint,maxErrorScore: this.maxErrorScore,errorScoreDecrement: this.errorScoreDecrement,expiresAt: this.expiresAt.toISOString(),@@ -279,7 +279,7 @@ export class Session implements ISession {usageCount: this.usageCount,maxUsageCount: this.maxUsageCount,errorScore: this.errorScore,- retired: this._retired,+ retired: this.#retired,};}@@ -291,10 +291,10 @@ export class Session implements ISession {* For transient external failures (such as `5XX` responses), use `markBad()` instead.*/retire() {- if (this._retired) return;- this._errorScore += this._maxErrorScore;- this._usageCount += 1;- this._retired = true;+ if (this.#retired) return;+ this.#errorScore += this.#maxErrorScore;+ this.#usageCount += 1;+ this.#retired = true;}/**@@ -302,8 +302,8 @@ export class Session implements ISession {* Should be used when the session has been used unsuccessfully. For example because of timeouts.*/markBad() {- this._errorScore += 1;- this._usageCount += 1;+ this.#errorScore += 1;+ this.#usageCount += 1;this.maybeSelfRetire();}@@ -325,7 +325,7 @@ export class Session implements ISession {try {this.cookieJar.setCookieSync(rawCookie, url);} catch (e) {- this.log.warning('Could not set cookie.', { url, error: (e as Error).message });+ this.#log.warning('Could not set cookie.', { url, error: (e as Error).message });}}diff --git a/packages/core/src/session_pool/session_pool.ts b/packages/core/src/session_pool/session_pool.tsindex e0f8432f7..16754c7c7 100644--- a/packages/core/src/session_pool/session_pool.ts+++ b/packages/core/src/session_pool/session_pool.ts@@ -131,26 +131,27 @@ export interface SessionPoolOptions {* @category Scaling*/export class SessionPool implements ISessionPool {- private static nextId = 0;+ static #nextId = 0;readonly id: string;- private log: CrawleeLogger;+ #log: CrawleeLogger;+ #sessions: Session[] = [];+ // kept as TS-private: session_pool tests read/override the members below directlyprivate maxPoolSize: number;private createSessionFunction: CreateSession;private keyValueStore?: KeyValueStore;- private sessions: Session[] = [];private sessionMap = new Map<string, Session>();private sessionOptions: SessionOptions;private persistStateKeyValueStoreId?: string;private persistStateKey: string;- private listener?: () => Promise<void>;- private events: EventManager;- private persistenceOptions: PersistenceOptions;- private sessionReuseStrategy: SessionReuseStrategy;+ #listener?: () => Promise<void>;+ #events: EventManager;+ #persistenceOptions: PersistenceOptions;+ #sessionReuseStrategy: SessionReuseStrategy;- private initPromise?: Promise<void>;- private queue = new AsyncQueue();- private roundRobinIndex = 0;+ #initPromise?: Promise<void>;+ #queue = new AsyncQueue();+ #roundRobinIndex = 0;constructor(options: SessionPoolOptions = {}) {ow(@@ -182,22 +183,22 @@ export class SessionPool implements ISessionPool {sessionReuseStrategy = 'random',} = options;- this.id = id != null ? String(id) : String(SessionPool.nextId++);- this.sessionReuseStrategy = sessionReuseStrategy;- this.events = serviceLocator.getEventManager();- this.log = log.child({ prefix: 'SessionPool' });- this.persistenceOptions = persistenceOptions;+ this.id = id != null ? String(id) : String(SessionPool.#nextId++);+ this.#sessionReuseStrategy = sessionReuseStrategy;+ this.#events = serviceLocator.getEventManager();+ this.#log = log.child({ prefix: 'SessionPool' });+ this.#persistenceOptions = persistenceOptions;// Pool Configurationthis.maxPoolSize = maxPoolSize;this.createSessionFunction = createSessionFunction || this.defaultCreateSessionFunction;// Session configuration. The pool-scoped logger is merged into per-call sessionOptions inside- // `_invokeCreateSessionFunction`, so every Session inherits it without custom createSessionFunctions+ // `invokeCreateSessionFunction`, so every Session inherits it without custom createSessionFunctions// having to know about it.this.sessionOptions = {...sessionOptions,- log: this.log,+ log: this.#log,};// Session keyValueStore@@ -210,7 +211,7 @@ export class SessionPool implements ISessionPool {*/async usableSessionsCount(): Promise<number> {await this.ensureInitialized();- return this.sessions.filter((session) => session.isUsable()).length;+ return this.#sessions.filter((session) => session.isUsable()).length;}/**@@ -218,7 +219,7 @@ export class SessionPool implements ISessionPool {*/async retiredSessionsCount(): Promise<number> {await this.ensureInitialized();- return this.sessions.filter((session) => !session.isUsable()).length;+ return this.#sessions.filter((session) => !session.isUsable()).length;}/**@@ -226,14 +227,14 @@ export class SessionPool implements ISessionPool {* Called automatically on first use of any public method.*/private async ensureInitialized(): Promise<void> {- if (!this.initPromise) {- this.initPromise = this.setupPool();+ if (!this.#initPromise) {+ this.#initPromise = this.setupPool();}- return this.initPromise;+ return this.#initPromise;}private async setupPool(): Promise<void> {- if (!this.persistenceOptions.enable) {+ if (!this.#persistenceOptions.enable) {return;}@@ -245,7 +246,7 @@ export class SessionPool implements ISessionPool {);if (!this.persistStateKeyValueStoreId) {- this.log.debug(+ this.#log.debug(`No 'persistStateKeyValueStoreId' options specified, this session pool's data has been saved in the KeyValueStore with the id: ${this.keyValueStore.id}`,);}@@ -253,8 +254,8 @@ export class SessionPool implements ISessionPool {// in case of migration happened and SessionPool state should be restored from the keyValueStore.await this.maybeLoadSessionPool();- this.listener = this.persistState.bind(this);- this.events.on(EventType.PERSIST_STATE, this.listener);+ this.#listener = this.persistState.bind(this);+ this.#events.on(EventType.PERSIST_STATE, this.#listener);}/**@@ -277,8 +278,8 @@ export class SessionPool implements ISessionPool {this.removeRetiredSessions();}- const newSession = options instanceof Session ? options : await this._invokeCreateSessionFunction(options);- this.log.debug(`Adding new Session - ${newSession.id}`);+ const newSession = options instanceof Session ? options : await this.invokeCreateSessionFunction(options);+ this.#log.debug(`Adding new Session - ${newSession.id}`);this.registerSession(newSession);}@@ -292,7 +293,7 @@ export class SessionPool implements ISessionPool {async newSession(sessionOptions?: SessionOptions): Promise<Session> {await this.ensureInitialized();- const newSession = await this._invokeCreateSessionFunction(sessionOptions);+ const newSession = await this.invokeCreateSessionFunction(sessionOptions);this.registerSession(newSession);return newSession;@@ -308,7 +309,7 @@ export class SessionPool implements ISessionPool {async getSession(sessionId?: string): Promise<Session | undefined> {await this.ensureInitialized();- await this.queue.wait();+ await this.#queue.wait();try {if (sessionId) {const session = this.sessionMap.get(sessionId);@@ -326,7 +327,7 @@ export class SessionPool implements ISessionPool {this.removeRetiredSessions();return await this.createSession();} finally {- this.queue.shift();+ this.#queue.shift();}}@@ -334,7 +335,7 @@ export class SessionPool implements ISessionPool {* @param options - Override the persistence options provided in the constructor*/async resetStore(options?: PersistenceOptions) {- if (!this.persistenceOptions.enable && !options?.enable) {+ if (!this.#persistenceOptions.enable && !options?.enable) {return;}@@ -351,7 +352,7 @@ export class SessionPool implements ISessionPool {return {usableSessionsCount: await this.usableSessionsCount(),retiredSessionsCount: await this.retiredSessionsCount(),- sessions: this.sessions.map((session) => session.getState()),+ sessions: this.#sessions.map((session) => session.getState()),};}@@ -361,13 +362,13 @@ export class SessionPool implements ISessionPool {* @param options - Override the persistence options provided in the constructor*/async persistState(options?: PersistenceOptions): Promise<void> {- if (!this.persistenceOptions.enable && !options?.enable) {+ if (!this.#persistenceOptions.enable && !options?.enable) {return;}await this.ensureInitialized();- this.log.debug('Persisting state', {+ this.#log.debug('Persisting state', {persistStateKeyValueStoreId: this.persistStateKeyValueStoreId,persistStateKey: this.persistStateKey,});@@ -375,7 +376,7 @@ export class SessionPool implements ISessionPool {await this.keyValueStore?.setValue(this.persistStateKey, await this.getState()).catch((error) =>- this.log.warning(`Failed to persist the session pool stats to ${this.persistStateKey}`, { error }),+ this.#log.warning(`Failed to persist the session pool stats to ${this.persistStateKey}`, { error }),);}@@ -385,10 +386,10 @@ export class SessionPool implements ISessionPool {* @param options - Set `persistState` to false when the final state was already persisted by the event manager.*/async teardown({ persistState = true }: { persistState?: boolean } = {}): Promise<void> {- if (!this.initPromise) return;+ if (!this.#initPromise) return;await this.ensureInitialized();- if (this.listener) {- this.events.off(EventType.PERSIST_STATE, this.listener);+ if (this.#listener) {+ this.#events.off(EventType.PERSIST_STATE, this.#listener);}if (persistState) {await this.persistState();@@ -399,11 +400,11 @@ export class SessionPool implements ISessionPool {* Removes retired `Session` instances from `SessionPool`.*/private removeRetiredSessions() {- this.sessions = this.sessions.filter((storedSession) => {+ this.#sessions = this.#sessions.filter((storedSession) => {if (storedSession.isUsable()) return true;this.sessionMap.delete(storedSession.id);- this.log.debug(`Removed Session - ${storedSession.id}`);+ this.#log.debug(`Removed Session - ${storedSession.id}`);return false;});@@ -414,7 +415,7 @@ export class SessionPool implements ISessionPool {* @param newSession `Session` instance to be added.*/private registerSession(newSession: Session) {- this.sessions.push(newSession);+ this.#sessions.push(newSession);this.sessionMap.set(newSession.id, newSession);}@@ -422,7 +423,7 @@ export class SessionPool implements ISessionPool {* Gets random index.*/private getRandomIndex(): number {- return Math.floor(Math.random() * this.sessions.length);+ return Math.floor(Math.random() * this.#sessions.length);}/**@@ -448,7 +449,7 @@ export class SessionPool implements ISessionPool {* through `maybeLoadSessionPool` naturally wins because it arrives in* `perCallOptions`.*/- private async _invokeCreateSessionFunction(perCallOptions?: SessionOptions): Promise<Session> {+ private async invokeCreateSessionFunction(perCallOptions?: SessionOptions): Promise<Session> {const sessionOptions: SessionOptions = {fingerprint: createDefaultSessionFingerprint(),...this.sessionOptions,@@ -462,9 +463,9 @@ export class SessionPool implements ISessionPool {* @returns Newly created `Session` instance.*/private async createSession(): Promise<Session> {- const newSession = await this._invokeCreateSessionFunction();+ const newSession = await this.invokeCreateSessionFunction();this.registerSession(newSession);- this.log.debug(`Created new Session - ${newSession.id}`);+ this.#log.debug(`Created new Session - ${newSession.id}`);return newSession;}@@ -473,7 +474,7 @@ export class SessionPool implements ISessionPool {* Decides whether there is enough space for creating new session.*/private hasSpaceForSession(): boolean {- return this.sessions.length < this.maxPoolSize;+ return this.#sessions.length < this.maxPoolSize;}/**@@ -481,19 +482,19 @@ export class SessionPool implements ISessionPool {* Returns `undefined` when no session should be reused and a new one should be created instead.*/private pickSession(): Session | undefined {- if (this.sessionReuseStrategy !== 'use-until-failure' && this.hasSpaceForSession()) return undefined;+ if (this.#sessionReuseStrategy !== 'use-until-failure' && this.hasSpaceForSession()) return undefined;- if (this.sessionReuseStrategy === 'use-until-failure') {- return this.sessions.find((session) => session.isUsable());+ if (this.#sessionReuseStrategy === 'use-until-failure') {+ return this.#sessions.find((session) => session.isUsable());}let picked: Session;- if (this.sessionReuseStrategy === 'round-robin') {- const index = this.roundRobinIndex % this.sessions.length;- this.roundRobinIndex = index + 1;- picked = this.sessions[index];+ if (this.#sessionReuseStrategy === 'round-robin') {+ const index = this.#roundRobinIndex % this.#sessions.length;+ this.#roundRobinIndex = index + 1;+ picked = this.#sessions[index];} else {- picked = this.sessions[this.getRandomIndex()];+ picked = this.#sessions[this.getRandomIndex()];}return picked.isUsable() ? picked : undefined;@@ -509,7 +510,7 @@ export class SessionPool implements ISessionPool {if (!loadedSessionPool) return;// Invalidate old sessions and load active sessions only- this.log.debug('Recreating state from KeyValueStore', {+ this.#log.debug('Recreating state from KeyValueStore', {persistStateKeyValueStoreId: this.persistStateKeyValueStoreId,persistStateKey: this.persistStateKey,});@@ -517,13 +518,13 @@ export class SessionPool implements ISessionPool {for (const sessionObject of loadedSessionPool.sessions) {sessionObject.createdAt = new Date(sessionObject.createdAt as string);sessionObject.expiresAt = new Date(sessionObject.expiresAt as string);- const recreatedSession = await this._invokeCreateSessionFunction(sessionObject);+ const recreatedSession = await this.invokeCreateSessionFunction(sessionObject);if (recreatedSession.isUsable()) {this.registerSession(recreatedSession);}}- this.log.debug(`${this.sessions.length} active sessions loaded from KeyValueStore`);+ this.#log.debug(`${this.#sessions.length} active sessions loaded from KeyValueStore`);}}diff --git a/packages/core/src/storages/dataset.ts b/packages/core/src/storages/dataset.tsindex 4f71cb166..4d46b321b 100644--- a/packages/core/src/storages/dataset.ts+++ b/packages/core/src/storages/dataset.ts@@ -192,7 +192,7 @@ export class Dataset<Data extends Dictionary = Dictionary> {backend: DatasetBackend<Data>;log: CrawleeLogger;- private readonly statsTracker = new StorageStatsTracker<DatasetStats>({+ readonly #statsTracker = new StorageStatsTracker<DatasetStats>({readCount: 0,writeCount: 0,});@@ -215,7 +215,7 @@ export class Dataset<Data extends Dictionary = Dictionary> {* the underlying storage backend). Counted per backend call.*/get stats(): DatasetStats {- return this.statsTracker.current;+ return this.#statsTracker.current;}/**@@ -252,7 +252,7 @@ export class Dataset<Data extends Dictionary = Dictionary> {return;}- this.statsTracker.add('writeCount');+ this.#statsTracker.add('writeCount');await this.backend.pushData(items);}@@ -282,7 +282,7 @@ export class Dataset<Data extends Dictionary = Dictionary> {const buffered = this.bufferedJournalEntries()?.flatMap((entry) => entry.items as Data[]);// Every branch below hits the backend exactly once.- this.statsTracker.add('readCount');+ this.#statsTracker.add('readCount');if (!buffered?.length) {return this.backend.getData(options);@@ -367,7 +367,7 @@ export class Dataset<Data extends Dictionary = Dictionary> {// One backend call with all journaled items, in order - as close to atomic as the backend allows.// Straight to the backend: the items were validated and snapshotted at write time.if (items.length > 0) {- this.statsTracker.add('writeCount');+ this.#statsTracker.add('writeCount');await this.backend.pushData(items);}}diff --git a/packages/core/src/storages/key_value_store.ts b/packages/core/src/storages/key_value_store.tsindex 286188fe1..f5fec0a5e 100644--- a/packages/core/src/storages/key_value_store.ts+++ b/packages/core/src/storages/key_value_store.ts@@ -91,13 +91,14 @@ const KVS_KEYS_DEFAULT_LIMIT = 1000;export class KeyValueStore {readonly id: string;readonly name?: string;+ // kept as TS-private: key_value_store tests spy on the backend directlyprivate readonly backend: KeyValueStoreBackend;- private persistStateEventStarted = false;+ #persistStateEventStarted = false;/** Cache for persistent (auto-saved) values. When we try to set such value, the cache will be updated automatically. */- private readonly cache = new Map<string, Dictionary>();+ readonly #cache = new Map<string, Dictionary>();- private readonly statsTracker = new StorageStatsTracker<KeyValueStoreStats>({+ readonly #statsTracker = new StorageStatsTracker<KeyValueStoreStats>({readCount: 0,writeCount: 0,deleteCount: 0,@@ -121,7 +122,7 @@ export class KeyValueStore {* list operations issued to the underlying storage backend). Counted per backend call.*/get stats(): KeyValueStoreStats {- return this.statsTracker.current;+ return this.#statsTracker.current;}/**@@ -282,7 +283,7 @@ export class KeyValueStore {};}- this.statsTracker.add('readCount');+ this.#statsTracker.add('readCount');const record = await this.backend.getValue(key);if (!record) return null;@@ -347,8 +348,8 @@ export class KeyValueStore {async getAutoSavedValue<T extends Dictionary = Dictionary>(key: string, defaultValue = {} as T): Promise<T> {tryCancel();- if (this.cache.has(key)) {- return this.cache.get(key) as T;+ if (this.#cache.has(key)) {+ return this.#cache.get(key) as T;}// Auto-saved state is deliberately *not* transactional. The direct read bypasses any active@@ -359,25 +360,25 @@ export class KeyValueStore {// the value will in cache at this point, and returning the new fetched value would introduce two different instances of// the auto-saved object, and only the latter one would be persisted.// Therefore we re-check the cache here, and if such race condition happened, we drop the fetched value and return the cached one.- if (this.cache.has(key)) {- return this.cache.get(key) as T;+ if (this.#cache.has(key)) {+ return this.#cache.get(key) as T;}- this.cache.set(key, value!);+ this.#cache.set(key, value!);this.ensurePersistStateEvent();return value!;}private ensurePersistStateEvent(): void {- if (this.persistStateEventStarted) {+ if (this.#persistStateEventStarted) {return;}serviceLocator.getEventManager().on('persistState', async () => {const promises: Promise<void>[] = [];- for (const [key, value] of this.cache) {+ for (const [key, value] of this.#cache) {promises.push(this.setValue(key, value).catch((error) =>serviceLocator.getLogger().warning(`Failed to persist the state value to ${key}`, { error }),@@ -388,7 +389,7 @@ export class KeyValueStore {await Promise.all(promises);});- this.persistStateEventStarted = true;+ this.#persistStateEventStarted = true;}private async *fetchKeyValuePages<T>(@@ -443,7 +444,7 @@ export class KeyValueStore {let exclusiveStartKey: string | undefined;while (true) {- this.statsTracker.add('listCount');+ this.#statsTracker.add('listCount');const { items, isTruncated, nextExclusiveStartKey } = await this.backend.listKeys({...options,exclusiveStartKey,@@ -559,17 +560,17 @@ export class KeyValueStore {}// If we try to set the value of a cached state to a different reference, we need to update the cache accordingly.- const cachedValue = this.cache.get(key);+ const cachedValue = this.#cache.get(key);if (cachedValue && cachedValue !== value) {if (value === null) {// Cached state can be only object, so a propagation of `null` means removing all its properties.- Object.keys(cachedValue).forEach((k) => this.cache.delete(k));+ Object.keys(cachedValue).forEach((k) => this.#cache.delete(k));} else if (typeof value === 'object') {// We need to remove the keys that are no longer present in the new value.Object.keys(cachedValue).filter((k) => !(k in (value as Dictionary)))- .forEach((k) => this.cache.delete(k));+ .forEach((k) => this.#cache.delete(k));// And update the existing ones + add new ones.Object.assign(cachedValue, value);}@@ -577,13 +578,13 @@ export class KeyValueStore {// In this case delete the record.if (value === null) {- this.statsTracker.add('deleteCount');+ this.#statsTracker.add('deleteCount');return this.backend.deleteValue(key);}const serialized = serializeValue(value, optionsCopy.contentType);- this.statsTracker.add('writeCount');+ this.#statsTracker.add('writeCount');return this.backend.setValue({key,value: serialized.value,@@ -622,7 +623,7 @@ export class KeyValueStore {clearCache(): void {rejectOperationInTransaction('KeyValueStore.clearCache()');- this.cache.clear();+ this.#cache.clear();}/**diff --git a/packages/core/src/storages/request_dedup_cache.ts b/packages/core/src/storages/request_dedup_cache.tsindex 632e56a0c..8d9539e68 100644--- a/packages/core/src/storages/request_dedup_cache.ts+++ b/packages/core/src/storages/request_dedup_cache.ts@@ -12,29 +12,29 @@* @internal*/export class RequestDeduplicationCache {- private keys: (string | undefined)[];- private ids: (string | undefined)[];+ #keys: (string | undefined)[];+ #ids: (string | undefined)[];// The slot count is the same for every queue, so it's a fixed default rather than a per-consumer option.constructor(private readonly size = 1_000_000) {- this.keys = new Array<string | undefined>(size);- this.ids = new Array<string | undefined>(size);+ this.#keys = new Array<string | undefined>(size);+ this.#ids = new Array<string | undefined>(size);}get(cacheKey: string): string | null {const index = this.indexOf(cacheKey);- return this.keys[index] === cacheKey ? this.ids[index]! : null;+ return this.#keys[index] === cacheKey ? this.#ids[index]! : null;}add(cacheKey: string, requestId: string): void {const index = this.indexOf(cacheKey);- this.keys[index] = cacheKey;- this.ids[index] = requestId;+ this.#keys[index] = cacheKey;+ this.#ids[index] = requestId;}clear(): void {- this.keys = new Array<string | undefined>(this.size);- this.ids = new Array<string | undefined>(this.size);+ this.#keys = new Array<string | undefined>(this.size);+ this.#ids = new Array<string | undefined>(this.size);}// A cheap FNV-1a hash of the cache key — avoids pulling in a dedicated hashing dependency.diff --git a/packages/core/src/storages/request_list.ts b/packages/core/src/storages/request_list.tsindex 5cbf933a6..491339a66 100644--- a/packages/core/src/storages/request_list.ts+++ b/packages/core/src/storages/request_list.ts@@ -239,7 +239,7 @@ export interface RequestListOptions {* @category Sources*/export class RequestList implements IRequestLoader {- private log: CrawleeLogger = serviceLocator.getLogger().child({ prefix: 'RequestList' });+ #log: CrawleeLogger = serviceLocator.getLogger().child({ prefix: 'RequestList' });/*** Array of all requests from all sources, in the order as they appeared in sources.@@ -249,10 +249,10 @@ export class RequestList implements IRequestLoader {readonly requests: (Request | RequestOptions)[] = [];/** Index to the next item in requests array to fetch. All previous requests are either handled or in progress. */- private nextIndex = 0;+ #nextIndex = 0;/** Dictionary, key is Request.uniqueKey, value is corresponding index in the requests array. */- private uniqueKeyToIndex: Record<string, number> = {};+ #uniqueKeyToIndex: Record<string, number> = {};/*** Set of `uniqueKey`s of requests that were returned by fetchNextRequest().@@ -265,7 +265,7 @@ export class RequestList implements IRequestLoader {* re-crawled after a restart. They are served before advancing through the rest of the sources.* @internal*/- private requestsToRetry: string[] = [];+ #requestsToRetry: string[] = [];/*** Starts as true because until we handle the first request, the list is effectively persisted by doing nothing.@@ -279,17 +279,18 @@ export class RequestList implements IRequestLoader {*/areRequestsPersisted = false;- private isLoading = false;- private isInitialized = false;- private persistStateKey?: string;- private persistRequestsKey?: string;- private initialState?: RequestListState;- private store?: KeyValueStore;- private keepDuplicateUrls: boolean;+ #isLoading = false;+ #isInitialized = false;+ #persistStateKey?: string;+ #persistRequestsKey?: string;+ #initialState?: RequestListState;+ #store?: KeyValueStore;+ #keepDuplicateUrls: boolean;+ // kept as TS-private: request_list tests read this field directlyprivate sources: RequestListSource[];- private sourcesFunction?: RequestListSourcesFunction;- private proxyConfiguration?: IProxyConfiguration;- private httpClient?: BaseHttpClient;+ #sourcesFunction?: RequestListSourcesFunction;+ #proxyConfiguration?: IProxyConfiguration;+ #httpClient?: BaseHttpClient;/*** To create new instance of `RequestList` we need to use `RequestList.open()` factory method.@@ -332,20 +333,20 @@ export class RequestList implements IRequestLoader {}),);- this.persistStateKey = persistStateKey ? `CRAWLEE_${persistStateKey}` : persistStateKey;- this.persistRequestsKey = persistRequestsKey ? `CRAWLEE_${persistRequestsKey}` : persistRequestsKey;- this.initialState = state;- this.httpClient = httpClient;+ this.#persistStateKey = persistStateKey ? `CRAWLEE_${persistStateKey}` : persistStateKey;+ this.#persistRequestsKey = persistRequestsKey ? `CRAWLEE_${persistRequestsKey}` : persistRequestsKey;+ this.#initialState = state;+ this.#httpClient = httpClient;// If this option is set then all requests will get a pre-generated unique ID and duplicate URLs will be kept in the list.- this.keepDuplicateUrls = keepDuplicateUrls;+ this.#keepDuplicateUrls = keepDuplicateUrls;// Will be empty after initialization to save memory.this.sources = sources ? [...sources] : [];- this.sourcesFunction = sourcesFunction;+ this.#sourcesFunction = sourcesFunction;// The proxy configuration used for `requestsFromUrl` requests.- this.proxyConfiguration = proxyConfiguration;+ this.#proxyConfiguration = proxyConfiguration;this.persistState = this.persistState.bind(this);}@@ -355,11 +356,11 @@ export class RequestList implements IRequestLoader {* This function must be called before you can start using the instance in a meaningful way.*/private async initialize(): Promise<this> {- if (this.isLoading) {+ if (this.#isLoading) {throw new Error('RequestList sources are already loading or were loaded.');}- this.isLoading = true;+ this.#isLoading = true;await purgeDefaultStorages({ onlyPurgeOnce: true });const [state, persistedRequests] = await this.loadStateAndPersistedRequests();@@ -373,9 +374,9 @@ export class RequestList implements IRequestLoader {}this.restoreState(state as RequestListState);- this.isInitialized = true;- if (this.persistRequestsKey && !this.areRequestsPersisted) await this.persistRequests();- if (this.persistStateKey) {+ this.#isInitialized = true;+ if (this.#persistRequestsKey && !this.areRequestsPersisted) await this.persistRequests();+ if (this.#persistStateKey) {serviceLocator.getEventManager().on(EventType.PERSIST_STATE, this.persistState);}@@ -430,9 +431,9 @@ export class RequestList implements IRequestLoader {// Drop the original array full of empty indexes.this.sources = [];- if (this.sourcesFunction) {+ if (this.#sourcesFunction) {try {- const sourcesFromFunction = await this.sourcesFunction();+ const sourcesFromFunction = await this.#sourcesFunction();const sourcesFromFunctionCount = sourcesFromFunction.length;for (let i = 0; i < sourcesFromFunctionCount; i++) {const source = sourcesFromFunction[i];@@ -453,17 +454,17 @@ export class RequestList implements IRequestLoader {* @inheritDoc*/async persistState(): Promise<void> {- if (!this.persistStateKey) {+ if (!this.#persistStateKey) {throw new Error('Cannot persist state. options.persistStateKey is not set.');}if (this.isStatePersisted) return;try {- this.store ??= await KeyValueStore.open();- await this.store.setValue(this.persistStateKey, this.getState());+ this.#store ??= await KeyValueStore.open();+ await this.#store.setValue(this.#persistStateKey, this.getState());this.isStatePersisted = true;} catch (e) {const err = e as Error;- this.log.exception(err, 'Attempted to persist state, but failed.');+ this.#log.exception(err, 'Attempted to persist state, but failed.');}}@@ -475,7 +476,7 @@ export class RequestList implements IRequestLoader {async teardown(): Promise<void> {serviceLocator.getEventManager().off(EventType.PERSIST_STATE, this.persistState);- if (this.persistStateKey) {+ if (this.#persistStateKey) {await this.persistState();}}@@ -487,8 +488,8 @@ export class RequestList implements IRequestLoader {*/private async persistRequests(): Promise<void> {const serializedRequests = await serializeArray(this.requests);- this.store ??= await KeyValueStore.open();- await this.store.setValue(this.persistRequestsKey!, serializedRequests, { contentType: CONTENT_TYPE_BINARY });+ this.#store ??= await KeyValueStore.open();+ await this.#store.setValue(this.#persistRequestsKey!, serializedRequests, { contentType: CONTENT_TYPE_BINARY });this.areRequestsPersisted = true;}@@ -516,7 +517,7 @@ export class RequestList implements IRequestLoader {const deleteFromInProgress: string[] = [];state.inProgress.forEach((uniqueKey) => {- const index = this.uniqueKeyToIndex[uniqueKey];+ const index = this.#uniqueKeyToIndex[uniqueKey];if (typeof index !== 'number') {throw new Error('The state object is not consistent with RequestList. Unknown uniqueKey is present in the state.',@@ -527,7 +528,7 @@ export class RequestList implements IRequestLoader {}});- this.nextIndex = state.nextIndex;+ this.#nextIndex = state.nextIndex;this.inProgress = new Set(state.inProgress);// WORKAROUND:@@ -544,7 +545,7 @@ export class RequestList implements IRequestLoader {// As a workaround, we just remove all inProgress requests whose index >= nextIndex,// since they will be crawled again.if (deleteFromInProgress.length) {- this.log.warning(+ this.#log.warning("RequestList's in-progress field is not consistent, skipping invalid in-progress entries",{deleteFromInProgress,@@ -556,7 +557,7 @@ export class RequestList implements IRequestLoader {}// All in-progress requests were interrupted and need to be re-crawled.- this.requestsToRetry = [...this.inProgress];+ this.#requestsToRetry = [...this.inProgress];}/**@@ -567,17 +568,18 @@ export class RequestList implements IRequestLoader {let state!: RequestListState;let persistedRequests!: Buffer;- if (this.initialState) {- state = this.initialState;- this.log.debug('Loaded state from options.state argument.');- } else if (this.persistStateKey) {- state = await this.getPersistedState(this.persistStateKey);- if (state) this.log.debug('Loaded state from key value store using the persistStateKey.');+ if (this.#initialState) {+ state = this.#initialState;+ this.#log.debug('Loaded state from options.state argument.');+ } else if (this.#persistStateKey) {+ state = await this.getPersistedState(this.#persistStateKey);+ if (state) this.#log.debug('Loaded state from key value store using the persistStateKey.');}- if (this.persistRequestsKey) {- persistedRequests = await this.getPersistedState(this.persistRequestsKey);- if (persistedRequests) this.log.debug('Loaded requests from key value store using the persistRequestsKey.');+ if (this.#persistRequestsKey) {+ persistedRequests = await this.getPersistedState(this.#persistRequestsKey);+ if (persistedRequests)+ this.#log.debug('Loaded requests from key value store using the persistRequestsKey.');}return [state, persistedRequests];@@ -591,8 +593,8 @@ export class RequestList implements IRequestLoader {this.ensureIsInitialized();return {- nextIndex: this.nextIndex,- nextUniqueKey: this.nextIndex < this.requests.length ? this.requests[this.nextIndex].uniqueKey! : null,+ nextIndex: this.#nextIndex,+ nextUniqueKey: this.#nextIndex < this.requests.length ? this.requests[this.#nextIndex].uniqueKey! : null,inProgress: [...this.inProgress],};}@@ -603,7 +605,7 @@ export class RequestList implements IRequestLoader {async isEmpty(): Promise<boolean> {this.ensureIsInitialized();- return this.requestsToRetry.length === 0 && this.nextIndex >= this.requests.length;+ return this.#requestsToRetry.length === 0 && this.#nextIndex >= this.requests.length;}/**@@ -612,7 +614,7 @@ export class RequestList implements IRequestLoader {async isFinished(): Promise<boolean> {this.ensureIsInitialized();- return this.inProgress.size === 0 && this.nextIndex >= this.requests.length;+ return this.inProgress.size === 0 && this.#nextIndex >= this.requests.length;}/**@@ -622,18 +624,18 @@ export class RequestList implements IRequestLoader {this.ensureIsInitialized();// First re-serve any requests that were interrupted before the last state persist.- const uniqueKey = this.requestsToRetry.shift();+ const uniqueKey = this.#requestsToRetry.shift();if (uniqueKey) {- const index = this.uniqueKeyToIndex[uniqueKey];+ const index = this.#uniqueKeyToIndex[uniqueKey];return this.ensureRequest(this.requests[index], index);}// Otherwise return next request.- if (this.nextIndex < this.requests.length) {- const index = this.nextIndex;+ if (this.#nextIndex < this.requests.length) {+ const index = this.#nextIndex;const request = this.requests[index];this.inProgress.add(request.uniqueKey!);- this.nextIndex++;+ this.#nextIndex++;this.isStatePersisted = false;return this.ensureRequest(request, index);}@@ -687,7 +689,7 @@ export class RequestList implements IRequestLoader {const fetchedCount = fetchedRequests.length;const importedCount = this.requests.length - originalLength;- this.log.info('Fetched and loaded Requests from a remote resource.', {+ this.#log.info('Fetched and loaded Requests from a remote resource.', {requestsFromUrl,regex,fetchedCount,@@ -698,8 +700,8 @@ export class RequestList implements IRequestLoader {}private async getPersistedState<T>(key: string): Promise<T> {- this.store ??= await KeyValueStore.open();- const state = await this.store.getValue<T>(key);+ this.#store ??= await KeyValueStore.open();+ const state = await this.#store.getValue<T>(key);return state!;}@@ -713,10 +715,10 @@ export class RequestList implements IRequestLoader {// Download remote resource and parse URLs.let urlsArr;try {- urlsArr = await this._downloadListOfUrls({+ urlsArr = await this.downloadListOfUrls({url: requestsFromUrl,urlRegExp: regex,- proxyUrl: (await this.proxyConfiguration?.newProxyInfo())?.url,+ proxyUrl: (await this.#proxyConfiguration?.newProxyInfo())?.url,});} catch (err) {throw new Error(`Cannot fetch a request list from ${requestsFromUrl}: ${err}`);@@ -724,7 +726,7 @@ export class RequestList implements IRequestLoader {// Skip if resource contained no URLs.if (!urlsArr.length) {- this.log.warning('The fetched list contains no valid URLs.', { requestsFromUrl, regex });+ this.#log.warning('The fetched list contains no valid URLs.', { requestsFromUrl, regex });return [];}@@ -754,7 +756,7 @@ export class RequestList implements IRequestLoader {request.uniqueKey ??= Request.computeUniqueKey(request as any);// Add index to uniqueKey if duplicates are to be kept- if (this.keepDuplicateUrls && !hasUniqueKey) {+ if (this.#keepDuplicateUrls && !hasUniqueKey) {request.uniqueKey += `-${this.requests.length}`;}@@ -762,11 +764,11 @@ export class RequestList implements IRequestLoader {this.ensureUniqueKeyValid(uniqueKey);// Skip requests with duplicate uniqueKey- if (!Object.hasOwn(this.uniqueKeyToIndex, uniqueKey)) {- this.uniqueKeyToIndex[uniqueKey] = this.requests.length;+ if (!Object.hasOwn(this.#uniqueKeyToIndex, uniqueKey)) {+ this.#uniqueKeyToIndex[uniqueKey] = this.requests.length;this.requests.push(request);- } else if (this.keepDuplicateUrls) {- this.log.warning(+ } else if (this.#keepDuplicateUrls) {+ this.#log.warning(`Duplicate uniqueKey: ${uniqueKey} found while the keepDuplicateUrls option was set. Check your sources' unique keys.`,);}@@ -795,7 +797,7 @@ export class RequestList implements IRequestLoader {* Throws an error if request list wasn't initialized.*/private ensureIsInitialized(): void {- if (!this.isInitialized) {+ if (!this.#isInitialized) {throw new Error('RequestList is not initialized; you must call "await requestList.initialize()" before using it!',);@@ -817,7 +819,7 @@ export class RequestList implements IRequestLoader {async getPendingCount(): Promise<number> {this.ensureIsInitialized();- return this.requests.length - (this.nextIndex - this.inProgress.size);+ return this.requests.length - (this.#nextIndex - this.inProgress.size);}/**@@ -839,7 +841,7 @@ export class RequestList implements IRequestLoader {async getHandledCount(): Promise<number> {this.ensureIsInitialized();- return this.nextIndex - this.inProgress.size;+ return this.#nextIndex - this.inProgress.size;}/**@@ -931,14 +933,14 @@ export class RequestList implements IRequestLoader {/*** @internal wraps public utility for mocking purposes*/- private async _downloadListOfUrls(options: {+ private async downloadListOfUrls(options: {url: string;urlRegExp?: RegExp;proxyUrl?: string;}): Promise<string[]> {return downloadListOfUrls({...options,- httpClient: this.httpClient,+ httpClient: this.#httpClient,});}}diff --git a/packages/core/src/storages/request_manager_tandem.ts b/packages/core/src/storages/request_manager_tandem.tsindex 9ba885368..21aa2ab0f 100644--- a/packages/core/src/storages/request_manager_tandem.ts+++ b/packages/core/src/storages/request_manager_tandem.ts@@ -18,18 +18,18 @@ import type {* It first reads requests from the loader and then, when needed, transfers them in batches to the manager.*/export class RequestManagerTandem implements IRequestManager {- private log: CrawleeLogger;- private requestLoader: IRequestLoader;- private requestManagerPromise?: Promise<IRequestManager>;- private resolvedRequestManager?: IRequestManager;+ #log: CrawleeLogger;+ #requestLoader: IRequestLoader;+ #requestManagerPromise?: Promise<IRequestManager>;+ #resolvedRequestManager?: IRequestManager;- private requestManagerFactory: () => IRequestManager | Promise<IRequestManager>;+ #requestManagerFactory: () => IRequestManager | Promise<IRequestManager>;/*** The latest expected request-processing time hinted via {@link setExpectedRequestProcessingTimeSecs}.* Remembered so it can be applied to the writable manager once it is lazily resolved.*/- private expectedRequestProcessingSecs?: number;+ #expectedRequestProcessingSecs?: number;/*** @param requestLoader The read-only loader to read requests from first.@@ -41,9 +41,9 @@ export class RequestManagerTandem implements IRequestManager {requestLoader: IRequestLoader,requestManager: IRequestManager | (() => IRequestManager | Promise<IRequestManager>),) {- this.log = serviceLocator.getLogger().child({ prefix: 'RequestManagerTandem' });- this.requestLoader = requestLoader;- this.requestManagerFactory = typeof requestManager === 'function' ? requestManager : () => requestManager;+ this.#log = serviceLocator.getLogger().child({ prefix: 'RequestManagerTandem' });+ this.#requestLoader = requestLoader;+ this.#requestManagerFactory = typeof requestManager === 'function' ? requestManager : () => requestManager;}/**@@ -51,18 +51,18 @@ export class RequestManagerTandem implements IRequestManager {* @private*/private async getRequestManager(): Promise<IRequestManager> {- if (this.resolvedRequestManager === undefined) {- this.requestManagerPromise ??= Promise.resolve(this.requestManagerFactory());- this.resolvedRequestManager = await this.requestManagerPromise;+ if (this.#resolvedRequestManager === undefined) {+ this.#requestManagerPromise ??= Promise.resolve(this.#requestManagerFactory());+ this.#resolvedRequestManager = await this.#requestManagerPromise;// Apply any hint received before the manager was resolved.- if (this.expectedRequestProcessingSecs !== undefined) {- await this.resolvedRequestManager.setExpectedRequestProcessingTimeSecs?.(- this.expectedRequestProcessingSecs,+ if (this.#expectedRequestProcessingSecs !== undefined) {+ await this.#resolvedRequestManager.setExpectedRequestProcessingTimeSecs?.(+ this.#expectedRequestProcessingSecs,);}}- return this.resolvedRequestManager;+ return this.#resolvedRequestManager;}/**@@ -74,7 +74,7 @@ export class RequestManagerTandem implements IRequestManager {* @private*/private async transferNextRequestToQueue(): Promise<boolean> {- const request = await this.requestLoader.fetchNextRequest();+ const request = await this.#requestLoader.fetchNextRequest();if (request === null) {return true;@@ -86,7 +86,7 @@ export class RequestManagerTandem implements IRequestManager {await requestManager.addRequest(request, { forefront: true });return true;} catch (error) {- this.log.exception(+ this.#log.exception(error as Error,'Adding request from the RequestLoader to the RequestManager failed, the request has been dropped.',{ url: request.url, uniqueKey: request.uniqueKey },@@ -94,7 +94,7 @@ export class RequestManagerTandem implements IRequestManager {return false;} finally {// Mark it as handled so that the request doesn't get stuck in the `inProgress` state in the loader.- await this.requestLoader.markRequestAsHandled(request);+ await this.#requestLoader.markRequestAsHandled(request);}}@@ -106,8 +106,8 @@ export class RequestManagerTandem implements IRequestManager {async fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request<T> | null> {// First, try to transfer a request from the requestListconst [listEmpty, listFinished] = await Promise.all([- this.requestLoader.isEmpty(),- this.requestLoader.isFinished(),+ this.#requestLoader.isEmpty(),+ this.#requestLoader.isFinished(),]);if (!listEmpty && !listFinished) {@@ -127,7 +127,7 @@ export class RequestManagerTandem implements IRequestManager {*/async isFinished(): Promise<boolean> {const requestManager = await this.getRequestManager();- const storagesFinished = await Promise.all([this.requestLoader.isFinished(), requestManager.isFinished()]);+ const storagesFinished = await Promise.all([this.#requestLoader.isFinished(), requestManager.isFinished()]);return storagesFinished.every(Boolean);}@@ -136,7 +136,7 @@ export class RequestManagerTandem implements IRequestManager {*/async isEmpty(): Promise<boolean> {const requestManager = await this.getRequestManager();- const storagesEmpty = await Promise.all([this.requestLoader.isEmpty(), requestManager.isEmpty()]);+ const storagesEmpty = await Promise.all([this.#requestLoader.isEmpty(), requestManager.isEmpty()]);return storagesEmpty.every(Boolean);}@@ -156,7 +156,7 @@ export class RequestManagerTandem implements IRequestManager {const [managerTotal, loaderTotal] = await Promise.all([requestManager.getTotalCount(),// count only pending to avoid double counting, requests marked as "handled" have been moved to requestManager- this.requestLoader.getPendingCount(),+ this.#requestLoader.getPendingCount(),]);return managerTotal + loaderTotal;}@@ -168,7 +168,7 @@ export class RequestManagerTandem implements IRequestManager {const requestManager = await this.getRequestManager();const [managerPending, loaderPending] = await Promise.all([requestManager.getPendingCount(),- this.requestLoader.getPendingCount(),+ this.#requestLoader.getPendingCount(),]);return managerPending + loaderPending;}@@ -223,7 +223,7 @@ export class RequestManagerTandem implements IRequestManager {* @inheritdoc*/async persistState(): Promise<void> {- await this.requestLoader.persistState?.();+ await this.#requestLoader.persistState?.();}/**@@ -241,7 +241,7 @@ export class RequestManagerTandem implements IRequestManager {* @inheritdoc*/async setExpectedRequestProcessingTimeSecs(secs: number): Promise<void> {- this.expectedRequestProcessingSecs = secs;- await this.resolvedRequestManager?.setExpectedRequestProcessingTimeSecs?.(secs);+ this.#expectedRequestProcessingSecs = secs;+ await this.#resolvedRequestManager?.setExpectedRequestProcessingTimeSecs?.(secs);}}diff --git a/packages/core/src/storages/request_queue.ts b/packages/core/src/storages/request_queue.tsindex e01b5ab9b..4cad6a9d5 100644--- a/packages/core/src/storages/request_queue.ts+++ b/packages/core/src/storages/request_queue.ts@@ -82,10 +82,11 @@ export class RequestQueue implements IStorage, IRequestManager {readonly id: string;readonly name?: string;readonly backend: RequestQueueBackend;- private proxyConfiguration?: IProxyConfiguration;+ #proxyConfiguration?: IProxyConfiguration;readonly log: CrawleeLogger;+ // kept as TS-private: request_queue tests read this cache directlyprivate requestCache: LruCache<RequestLruItem>;/**@@ -93,10 +94,11 @@ export class RequestQueue implements IStorage, IRequestManager {* batches that `requestCache` skips — so overlapping URL sets aren't re-submitted.* See {@link RequestDeduplicationCache} for why this is a separate, cheaper cache.*/- private requestSeenCache: RequestDeduplicationCache;+ #requestSeenCache: RequestDeduplicationCache;- private queuePausedForMigration = false;+ #queuePausedForMigration = false;+ // kept as TS-private: packages/core/test request-queue tests write this counter directlyprivate inProgressRequestBatchCount = 0;/**@@ -104,13 +106,13 @@ export class RequestQueue implements IStorage, IRequestManager {* {@link setExpectedRequestProcessingTimeSecs}. Used to ensure that value is only ever raised, never* lowered, before being forwarded to the storage backend.*/- private expectedRequestProcessingSecs = 0;+ #expectedRequestProcessingSecs = 0;- private httpClient?: BaseHttpClient;+ #httpClient?: BaseHttpClient;- private readonly events: EventManager;+ readonly #events: EventManager;- private readonly statsTracker = new StorageStatsTracker<RequestQueueStats>({+ readonly #statsTracker = new StorageStatsTracker<RequestQueueStats>({writeCount: 0,headItemReadCount: 0,});@@ -120,7 +122,7 @@ export class RequestQueue implements IStorage, IRequestManager {* queue-head reads issued to the underlying storage backend). Counted per backend call.*/get stats(): RequestQueueStats {- return this.statsTracker.current;+ return this.#statsTracker.current;}/**@@ -129,17 +131,17 @@ export class RequestQueue implements IStorage, IRequestManager {constructor(options: RequestQueueOptions) {this.id = options.metadata.id;this.name = options.metadata.name;- this.events = serviceLocator.getEventManager();+ this.#events = serviceLocator.getEventManager();this.backend = options.backend;- this.proxyConfiguration = options.proxyConfiguration;+ this.#proxyConfiguration = options.proxyConfiguration;this.requestCache = new LruCache({ maxLength: MAX_CACHED_REQUESTS });- this.requestSeenCache = new RequestDeduplicationCache();+ this.#requestSeenCache = new RequestDeduplicationCache();this.log = serviceLocator.getLogger().child({ prefix: `RequestQueue(${this.id}, ${this.name ?? 'no-name'})` });- this.events.on(EventType.MIGRATING, async () => {- this.queuePausedForMigration = true;+ this.#events.on(EventType.MIGRATING, async () => {+ this.#queuePausedForMigration = true;});}@@ -231,7 +233,7 @@ export class RequestQueue implements IStorage, IRequestManager {};}- this.statsTracker.add('writeCount');+ this.#statsTracker.add('writeCount');const { processedRequests } = await this.backend.addBatchOfRequests([request], { forefront });this.recordRequestJournalEntry(transaction, [request], forefront, true);const queueOperationInfo = {@@ -241,7 +243,7 @@ export class RequestQueue implements IStorage, IRequestManager {} satisfies RequestQueueOperationInfo;this.cacheRequest(cacheKey, queueOperationInfo);- this.requestSeenCache.add(cacheKey, request.id!);+ this.#requestSeenCache.add(cacheKey, request.id!);return queueOperationInfo;}@@ -318,7 +320,7 @@ export class RequestQueue implements IStorage, IRequestManager {// reading saves a probe. Same lookup as the write-through path.const cacheKey = getRequestId(request.uniqueKey);const cachedInfo = this.requestCache.get(cacheKey);- const knownRequestId = cachedInfo?.id ?? this.requestSeenCache.get(cacheKey);+ const knownRequestId = cachedInfo?.id ?? this.#requestSeenCache.get(cacheKey);if (knownRequestId) {this.recordRequestJournalEntry(transaction, [request], forefront, false);@@ -396,7 +398,7 @@ export class RequestQueue implements IStorage, IRequestManager {if (requests.length === 0) continue;- this.statsTracker.add('writeCount');+ this.#statsTracker.add('writeCount');const { processedRequests, unprocessedRequests } = await this.backend.addBatchOfRequests(requests, {forefront,});@@ -405,7 +407,7 @@ export class RequestQueue implements IStorage, IRequestManager {for (const processed of processedRequests) {const cacheKey = getRequestId(processed.uniqueKey);this.cacheRequest(cacheKey, { ...processed, forefront });- this.requestSeenCache.add(cacheKey, processed.requestId);+ this.#requestSeenCache.add(cacheKey, processed.requestId);}if (unprocessedRequests.length > 0) {@@ -510,7 +512,7 @@ export class RequestQueue implements IStorage, IRequestManager {const cacheKey = getCachedRequestId(request.uniqueKey);// Prefer the full `requestCache` record; fall back to the dedup cache for background batches it skips.const cachedInfo = this.requestCache.get(cacheKey);- const knownRequestId = cachedInfo?.id ?? this.requestSeenCache.get(cacheKey);+ const knownRequestId = cachedInfo?.id ?? this.#requestSeenCache.get(cacheKey);if (knownRequestId) {request.id = knownRequestId;@@ -531,7 +533,7 @@ export class RequestQueue implements IStorage, IRequestManager {return results;}- this.statsTracker.add('writeCount');+ this.#statsTracker.add('writeCount');const apiResults = await this.backend.addBatchOfRequests([...requestsToAdd.values()], { forefront });// Report unprocessed requests@@ -549,7 +551,7 @@ export class RequestQueue implements IStorage, IRequestManager {}// Unlike `requestCache`, populate this on every batch (including background ones).- this.requestSeenCache.add(cacheKey, newRequest.requestId!);+ this.#requestSeenCache.add(cacheKey, newRequest.requestId!);}return results;@@ -802,11 +804,11 @@ export class RequestQueue implements IStorage, IRequestManager {'it is part of the crawler request-processing bookkeeping, which a transaction must not affect.',);- if (this.queuePausedForMigration) {+ if (this.#queuePausedForMigration) {return null;}- this.statsTracker.add('headItemReadCount');+ this.#statsTracker.add('headItemReadCount');const requestOptions = await this.backend.fetchNextRequest();if (!requestOptions) return null;@@ -837,7 +839,7 @@ export class RequestQueue implements IStorage, IRequestManager {const forefront = this.requestCache.get(getRequestId(request.uniqueKey))?.forefront ?? false;const handledAt = request.handledAt ?? new Date().toISOString();- this.statsTracker.add('writeCount');+ this.#statsTracker.add('writeCount');const processedRequest = await this.backend.markRequestAsHandled({...request,handledAt,@@ -892,7 +894,7 @@ export class RequestQueue implements IStorage, IRequestManager {const { forefront = false } = options;- this.statsTracker.add('writeCount');+ this.#statsTracker.add('writeCount');const processedRequest = await this.backend.reclaimRequest(request, { forefront });// The request was not in progress — nothing to reclaim.@@ -966,11 +968,11 @@ export class RequestQueue implements IStorage, IRequestManager {* short the reservation of a long-lived one and have its in-flight request stolen.*/async setExpectedRequestProcessingTimeSecs(secs: number): Promise<void> {- if (secs <= this.expectedRequestProcessingSecs) {+ if (secs <= this.#expectedRequestProcessingSecs) {return;}- this.expectedRequestProcessingSecs = secs;+ this.#expectedRequestProcessingSecs = secs;await this.backend.setExpectedRequestProcessingTimeSecs?.(secs);}@@ -1013,13 +1015,13 @@ export class RequestQueue implements IStorage, IRequestManager {// Reset in-memory bookkeeping so the queue behaves as if freshly opened.this.requestCache.clear();- this.requestSeenCache.clear();+ this.#requestSeenCache.clear();this.inProgressRequestBatchCount = 0;// Reset the expected-processing-time high-water mark too, otherwise the monotonic-raise guard// in `setExpectedRequestProcessingTimeSecs` would let a value raised in an earlier run leak into a// later one and silently swallow a lower hint (the queue is meant to be reusable across runs).- this.expectedRequestProcessingSecs = 0;+ this.#expectedRequestProcessingSecs = 0;}/**@@ -1094,10 +1096,10 @@ export class RequestQueue implements IStorage, IRequestManager {// Download remote resource and parse URLs.let urlsArr;try {- urlsArr = await this._downloadListOfUrls({+ urlsArr = await this.downloadListOfUrls({url: requestsFromUrl,urlRegExp: regex,- proxyUrl: (await this.proxyConfiguration?.newProxyInfo())?.url,+ proxyUrl: (await this.#proxyConfiguration?.newProxyInfo())?.url,});} catch (err) {throw new Error(`Cannot fetch a request list from ${requestsFromUrl}: ${err}`);@@ -1138,14 +1140,14 @@ export class RequestQueue implements IStorage, IRequestManager {/*** @internal wraps public utility for mocking purposes*/- private async _downloadListOfUrls(options: {+ private async downloadListOfUrls(options: {url: string;urlRegExp?: RegExp;proxyUrl?: string;}): Promise<string[]> {return downloadListOfUrls({...options,- httpClient: this.httpClient,+ httpClient: this.#httpClient,});}@@ -1196,8 +1198,8 @@ export class RequestQueue implements IStorage, IRequestManager {backendOpener: () => storageBackend.createRequestQueueBackend(resolved),backendCacheKey: storageBackend.getStorageBackendCacheKey?.() ?? storageBackend.constructor.name,});- queue.proxyConfiguration = options.proxyConfiguration;- queue.httpClient = options.httpClient;+ queue.#proxyConfiguration = options.proxyConfiguration;+ queue.#httpClient = options.httpClient;return queue;}diff --git a/packages/core/src/storages/sitemap_request_loader.ts b/packages/core/src/storages/sitemap_request_loader.tsindex 41f8664f2..90bb02e27 100644--- a/packages/core/src/storages/sitemap_request_loader.ts+++ b/packages/core/src/storages/sitemap_request_loader.ts@@ -131,12 +131,12 @@ export class SitemapRequestLoader implements IRequestLoader {** We use this to persist custom user fields on the in-progress requests.*/- private requestData = new Map<string, Request>();+ #requestData = new Map<string, Request>();/*** Object for keeping track of the sitemap parsing progress.*/- private sitemapParsingProgress: SitemapParsingProgress = {+ #sitemapParsingProgress: SitemapParsingProgress = {/*** URL of the sitemap that is currently being parsed. `null` if no sitemap is being parsed.*/@@ -157,7 +157,7 @@ export class SitemapRequestLoader implements IRequestLoader {** Fetch the next URL to be processed using `fetchNextRequest()`.*/- private urlQueueStream: Transform;+ #urlQueueStream: Transform;/*** Indicates whether the request list sitemap loading was aborted.@@ -167,39 +167,39 @@ export class SitemapRequestLoader implements IRequestLoader {** If the loading is aborted and all the requests are handled, `isFinished()` will return `true`.*/- private abortLoading = false;+ #abortLoading = false;/** Number of URLs that were marked as handled */- private handledUrlCount = 0;+ #handledUrlCount = 0;- private persistStateKey?: string;+ #persistStateKey?: string;- private store?: KeyValueStore;+ #store?: KeyValueStore;- private closed = false;+ #closed = false;/*** Proxy URL to be used for sitemap loading.*/- private proxyUrl?: string;+ #proxyUrl?: string;/*** Enqueue strategy applied to sitemap-derived URLs and stamped onto the emitted `Request` objects.*/- private enqueueStrategy: EnqueueStrategy | `${EnqueueStrategy}`;+ #enqueueStrategy: EnqueueStrategy | `${EnqueueStrategy}`;/*** Logger instance.*/- private log: CrawleeLogger;+ #log: CrawleeLogger;- private urlExcludePatternObjects: UrlPatternObject[] = [];- private urlPatternObjects: UrlPatternObject[] = [];+ #urlExcludePatternObjects: UrlPatternObject[] = [];+ #urlPatternObjects: UrlPatternObject[] = [];/** EventManager used to handle persistence */- private events: EventManager;+ #events: EventManager;- private persistenceOptions: RequiredDeep<SitemapRequestLoaderOptions['persistenceOptions']>;+ #persistenceOptions: RequiredDeep<SitemapRequestLoaderOptions['persistenceOptions']>;/** @internal */private constructor(options: SitemapRequestLoaderOptions) {@@ -230,26 +230,26 @@ export class SitemapRequestLoader implements IRequestLoader {const { include, exclude } = options;- this.log = serviceLocator.getLogger().child({ prefix: 'SitemapRequestLoader' });+ this.#log = serviceLocator.getLogger().child({ prefix: 'SitemapRequestLoader' });if (exclude?.length) {- this.urlExcludePatternObjects.push(...constructUrlPatternObjects(exclude));+ this.#urlExcludePatternObjects.push(...constructUrlPatternObjects(exclude));}if (include?.length) {- this.urlPatternObjects.push(...constructUrlPatternObjects(include));+ this.#urlPatternObjects.push(...constructUrlPatternObjects(include));}- this.persistStateKey = options.persistStateKey;- this.persistenceOptions = { enable: true, ...options.persistenceOptions };+ this.#persistStateKey = options.persistStateKey;+ this.#persistenceOptions = { enable: true, ...options.persistenceOptions };- this.proxyUrl = options.proxyUrl;- this.enqueueStrategy = options.enqueueStrategy ?? EnqueueStrategy.SameHostname;+ this.#proxyUrl = options.proxyUrl;+ this.#enqueueStrategy = options.enqueueStrategy ?? EnqueueStrategy.SameHostname;- this.urlQueueStream = this.createNewStream(options.maxBufferSize ?? 200);+ this.#urlQueueStream = this.createNewStream(options.maxBufferSize ?? 200);- this.sitemapParsingProgress.pendingSitemapUrls = new Set(options.sitemapUrls);- this.events = serviceLocator.getEventManager();+ this.#sitemapParsingProgress.pendingSitemapUrls = new Set(options.sitemapUrls);+ this.#events = serviceLocator.getEventManager();this.persistState = this.persistState.bind(this);}@@ -289,8 +289,8 @@ export class SitemapRequestLoader implements IRequestLoader {*/private isUrlMatchingPatterns(url: string): boolean {return (- !this.urlExcludePatternObjects.some(this.matchesUrl(url)) &&- (this.urlPatternObjects.length === 0 || this.urlPatternObjects.some(this.matchesUrl(url)))+ !this.#urlExcludePatternObjects.some(this.matchesUrl(url)) &&+ (this.#urlPatternObjects.length === 0 || this.#urlPatternObjects.some(this.matchesUrl(url))));}@@ -301,14 +301,14 @@ export class SitemapRequestLoader implements IRequestLoader {*/private async pushNextUrl(url: string | null) {return new Promise<void>((resolve) => {- if (this.closed || (url && !this.isUrlMatchingPatterns(url))) {+ if (this.#closed || (url && !this.isUrlMatchingPatterns(url))) {resolve();return;}- if (!this.urlQueueStream.push(url)) {+ if (!this.#urlQueueStream.push(url)) {// This doesn't work with the 'drain' event (it's not emitted for some reason).- this.urlQueueStream.once('readdata', () => {+ this.#urlQueueStream.once('readdata', () => {resolve();});} else {@@ -325,22 +325,22 @@ export class SitemapRequestLoader implements IRequestLoader {*/private async readNextUrl(): Promise<string | null> {return new Promise((resolve) => {- if (this.closed) {+ if (this.#closed) {resolve(null);return;}- const result = this.urlQueueStream.read();+ const result = this.#urlQueueStream.read();if (!result && !this.isSitemapFullyLoaded()) {- this.urlQueueStream.once('readable', () => {- const nextUrl = this.urlQueueStream.read();+ this.#urlQueueStream.once('readable', () => {+ const nextUrl = this.#urlQueueStream.read();resolve(nextUrl);});} else {resolve(result);}- this.urlQueueStream.emit('readdata');+ this.#urlQueueStream.emit('readdata');});}@@ -351,8 +351,8 @@ export class SitemapRequestLoader implements IRequestLoader {*/isSitemapFullyLoaded(): boolean {return (- this.sitemapParsingProgress.inProgressSitemapUrl === null &&- this.sitemapParsingProgress.pendingSitemapUrls.size === 0+ this.#sitemapParsingProgress.inProgressSitemapUrl === null &&+ this.#sitemapParsingProgress.pendingSitemapUrls.size === 0);}@@ -366,39 +366,39 @@ export class SitemapRequestLoader implements IRequestLoader {}: {parseSitemapOptions?: SitemapRequestLoaderOptions['parseSitemapOptions'];}): Promise<void> {- while (!this.isSitemapFullyLoaded() && !this.abortLoading) {+ while (!this.isSitemapFullyLoaded() && !this.#abortLoading) {const sitemapUrl =- this.sitemapParsingProgress.inProgressSitemapUrl ??- this.sitemapParsingProgress.pendingSitemapUrls.values().next().value!;+ this.#sitemapParsingProgress.inProgressSitemapUrl ??+ this.#sitemapParsingProgress.pendingSitemapUrls.values().next().value!;try {- for await (const item of parseSitemap([{ type: 'url', url: sitemapUrl }], this.proxyUrl, {+ for await (const item of parseSitemap([{ type: 'url', url: sitemapUrl }], this.#proxyUrl, {...parseSitemapOptions,maxDepth: 0,emitNestedSitemaps: true,- enqueueStrategy: this.enqueueStrategy,+ enqueueStrategy: this.#enqueueStrategy,})) {if (!item.originSitemapUrl) {// This is a nested sitemap- this.sitemapParsingProgress.pendingSitemapUrls.add(item.loc);+ this.#sitemapParsingProgress.pendingSitemapUrls.add(item.loc);continue;}- if (!this.sitemapParsingProgress.inProgressEntries.has(item.loc)) {+ if (!this.#sitemapParsingProgress.inProgressEntries.has(item.loc)) {await this.pushNextUrl(item.loc);- this.sitemapParsingProgress.inProgressEntries.add(item.loc);+ this.#sitemapParsingProgress.inProgressEntries.add(item.loc);}}} catch (e: any) {- this.log.error('Error loading sitemap contents:', e);+ this.#log.error('Error loading sitemap contents:', e);}- this.sitemapParsingProgress.pendingSitemapUrls.delete(sitemapUrl);- this.sitemapParsingProgress.inProgressEntries.clear();- this.sitemapParsingProgress.inProgressSitemapUrl = null;+ this.#sitemapParsingProgress.pendingSitemapUrls.delete(sitemapUrl);+ this.#sitemapParsingProgress.inProgressEntries.clear();+ this.#sitemapParsingProgress.inProgressSitemapUrl = null;}- this.urlQueueStream.end();+ this.#urlQueueStream.end();}/**@@ -420,17 +420,17 @@ export class SitemapRequestLoader implements IRequestLoader {parseSitemapOptions: { logger: serviceLocator.getLogger(), ...options.parseSitemapOptions, httpClient },});- if (requestList.persistenceOptions.enable) {- requestList.events.on(EventType.PERSIST_STATE, requestList.persistState);+ if (requestList.#persistenceOptions.enable) {+ requestList.#events.on(EventType.PERSIST_STATE, requestList.persistState);}options?.signal?.addEventListener('abort', () => {- requestList.abortLoading = true;+ requestList.#abortLoading = true;});if (options.timeoutMillis) {setTimeout(() => {- requestList.abortLoading = true;+ requestList.#abortLoading = true;}, options.timeoutMillis);}@@ -443,7 +443,7 @@ export class SitemapRequestLoader implements IRequestLoader {async getTotalCount(): Promise<number> {// Total known so far = not-yet-fetched (still buffered in the stream) + in-progress (fetched but not// yet handled) + already handled.- return this.urlQueueStream.readableLength + this.inProgress.size + this.handledUrlCount;+ return this.#urlQueueStream.readableLength + this.inProgress.size + this.#handledUrlCount;}/**@@ -451,7 +451,7 @@ export class SitemapRequestLoader implements IRequestLoader {*/async getPendingCount(): Promise<number> {// Pending = everything not yet handled = not-yet-fetched + in-progress.- return this.urlQueueStream.readableLength + this.inProgress.size;+ return this.#urlQueueStream.readableLength + this.inProgress.size;}/**@@ -472,7 +472,7 @@ export class SitemapRequestLoader implements IRequestLoader {*/async isFinished(): Promise<boolean> {return (- (await this.isEmpty()) && this.inProgress.size === 0 && (this.isSitemapFullyLoaded() || this.abortLoading)+ (await this.isEmpty()) && this.inProgress.size === 0 && (this.isSitemapFullyLoaded() || this.#abortLoading));}@@ -480,30 +480,30 @@ export class SitemapRequestLoader implements IRequestLoader {* @inheritDoc*/async isEmpty(): Promise<boolean> {- return this.urlQueueStream.readableLength === 0;+ return this.#urlQueueStream.readableLength === 0;}/*** @inheritDoc*/async getHandledCount(): Promise<number> {- return this.handledUrlCount;+ return this.#handledUrlCount;}/*** @inheritDoc*/async persistState(): Promise<void> {- if (this.persistStateKey === undefined) {+ if (this.#persistStateKey === undefined) {return;}- this.store ??= await KeyValueStore.open();+ this.#store ??= await KeyValueStore.open();const urlQueue = [];- while (this.urlQueueStream.readableLength > 0) {- const url = this.urlQueueStream.read();+ while (this.#urlQueueStream.readableLength > 0) {+ const url = this.#urlQueueStream.read();if (url === null) {break;}@@ -512,7 +512,7 @@ export class SitemapRequestLoader implements IRequestLoader {// Create a new stream, as we have read all the URLs from the current one.// Pushing the urls back to the original stream might not be possible if it has been ended.- const previousStream = this.urlQueueStream;+ const previousStream = this.#urlQueueStream;const newStream = this.createNewStream(previousStream.readableHighWaterMark);for (const url of urlQueue) {@@ -523,7 +523,7 @@ export class SitemapRequestLoader implements IRequestLoader {newStream.end();}- this.urlQueueStream = newStream;+ this.#urlQueueStream = newStream;// A `pushNextUrl()` call may be blocked on backpressure, waiting for a `readdata` event on the// previous stream. That event is only ever emitted by `readNextUrl()` on the current stream, so@@ -532,48 +532,48 @@ export class SitemapRequestLoader implements IRequestLoader {// been transferred to the new stream above).previousStream.emit('readdata');- await this.store.setValue(this.persistStateKey, {+ await this.#store.setValue(this.#persistStateKey, {sitemapParsingProgress: {- pendingSitemapUrls: Array.from(this.sitemapParsingProgress.pendingSitemapUrls),- inProgressSitemapUrl: this.sitemapParsingProgress.inProgressSitemapUrl,- inProgressEntries: Array.from(this.sitemapParsingProgress.inProgressEntries),+ pendingSitemapUrls: Array.from(this.#sitemapParsingProgress.pendingSitemapUrls),+ inProgressSitemapUrl: this.#sitemapParsingProgress.inProgressSitemapUrl,+ inProgressEntries: Array.from(this.#sitemapParsingProgress.inProgressEntries),},// Re-queue in-progress requests to the front so they are retried if the state is restored.urlQueue: [...this.inProgress, ...urlQueue],- requestData: Array.from(this.requestData.entries()),- abortLoading: this.abortLoading,- closed: this.closed,+ requestData: Array.from(this.#requestData.entries()),+ abortLoading: this.#abortLoading,+ closed: this.#closed,} satisfies SitemapRequestLoaderState);}private async restoreState(): Promise<void> {await purgeDefaultStorages({ onlyPurgeOnce: true });- if (this.persistStateKey === undefined) {+ if (this.#persistStateKey === undefined) {return;}- this.store ??= await KeyValueStore.open();- const state = await this.store.getValue<SitemapRequestLoaderState>(this.persistStateKey);+ this.#store ??= await KeyValueStore.open();+ const state = await this.#store.getValue<SitemapRequestLoaderState>(this.#persistStateKey);if (state === null) {return;}- this.sitemapParsingProgress = {+ this.#sitemapParsingProgress = {pendingSitemapUrls: new Set(state.sitemapParsingProgress.pendingSitemapUrls),inProgressSitemapUrl: state.sitemapParsingProgress.inProgressSitemapUrl,inProgressEntries: new Set(state.sitemapParsingProgress.inProgressEntries),};- this.requestData = new Map(state.requestData ?? []);+ this.#requestData = new Map(state.requestData ?? []);for (const url of state.urlQueue) {- this.urlQueueStream.push(url);+ this.#urlQueueStream.push(url);}- this.abortLoading = state.abortLoading;- this.closed = state.closed;+ this.#abortLoading = state.abortLoading;+ this.#closed = state.closed;}/**@@ -586,12 +586,12 @@ export class SitemapRequestLoader implements IRequestLoader {}// A restored in-progress request already has its Request data; don't overwrite it.- if (!this.requestData.has(nextUrl)) {- this.requestData.set(nextUrl, new Request({ url: nextUrl, enqueueStrategy: this.enqueueStrategy }));+ if (!this.#requestData.has(nextUrl)) {+ this.#requestData.set(nextUrl, new Request({ url: nextUrl, enqueueStrategy: this.#enqueueStrategy }));}this.inProgress.add(nextUrl);- return this.requestData.get(nextUrl)!;+ return this.#requestData.get(nextUrl)!;}/**@@ -612,22 +612,22 @@ export class SitemapRequestLoader implements IRequestLoader {* Calling `fetchNextRequest()` after this method will always return `null`.*/async teardown(): Promise<void> {- this.closed = true;- this.abortLoading = true;- this.events.off(EventType.PERSIST_STATE, this.persistState);+ this.#closed = true;+ this.#abortLoading = true;+ this.#events.off(EventType.PERSIST_STATE, this.persistState);await this.persistState();- this.urlQueueStream.emit('readdata'); // unblocks the potentially waiting `pushNextUrl` call+ this.#urlQueueStream.emit('readdata'); // unblocks the potentially waiting `pushNextUrl` call}/*** @inheritDoc*/async markRequestAsHandled(request: Request): Promise<void> {- this.handledUrlCount += 1;+ this.#handledUrlCount += 1;this.ensureInProgress(request.url);this.inProgress.delete(request.url);- this.requestData.delete(request.url);+ this.#requestData.delete(request.url);}private ensureInProgress(url: string): void {diff --git a/packages/core/src/storages/storage_instance_manager.ts b/packages/core/src/storages/storage_instance_manager.tsindex db6fedb75..f1ec38d9e 100644--- a/packages/core/src/storages/storage_instance_manager.ts+++ b/packages/core/src/storages/storage_instance_manager.ts@@ -178,8 +178,8 @@ class StorageCache {* assigns a reserved default alias.*/export class StorageInstanceManager {- private readonly cache = new StorageCache();- private readonly openerLocks = new Map<string, AsyncQueue>();+ readonly #cache = new StorageCache();+ readonly #openerLocks = new Map<string, AsyncQueue>();/*** Open (or retrieve from cache) a storage instance.@@ -214,40 +214,40 @@ export class StorageInstanceManager {// Fast-path cache check (no lock).if (alias !== undefined) {- const cached = this.cache.get(cls, { alias, backendCacheKey });+ const cached = this.#cache.get(cls, { alias, backendCacheKey });if (cached) return cached;} else if (id) {- const cached = this.cache.get(cls, { id, backendCacheKey });+ const cached = this.#cache.get(cls, { id, backendCacheKey });if (cached) return cached;} else if (name) {- const cached = this.cache.get(cls, { name, backendCacheKey });+ const cached = this.#cache.get(cls, { name, backendCacheKey });if (cached) return cached;}const identifierKey = id ?? name ?? alias ?? DEFAULT_STORAGE_ALIAS;const lockKey = `${cls.name}:${identifierKey}:${backendCacheKey}`;- if (!this.openerLocks.has(lockKey)) {- this.openerLocks.set(lockKey, new AsyncQueue());+ if (!this.#openerLocks.has(lockKey)) {+ this.#openerLocks.set(lockKey, new AsyncQueue());}- const queue = this.openerLocks.get(lockKey)!;+ const queue = this.#openerLocks.get(lockKey)!;await queue.wait();try {// Double-check cache under lock (another caller may have filled it while we waited).if (alias !== undefined) {- const cached = this.cache.get(cls, { alias, backendCacheKey });+ const cached = this.#cache.get(cls, { alias, backendCacheKey });if (cached) return cached;} else if (id) {- const cached = this.cache.get(cls, { id, backendCacheKey });+ const cached = this.#cache.get(cls, { id, backendCacheKey });if (cached) return cached;} else if (name) {- const cached = this.cache.get(cls, { name, backendCacheKey });+ const cached = this.#cache.get(cls, { name, backendCacheKey });if (cached) return cached;}// Prevent the same string from being used as both a name and an alias.- this.cache.checkNameAliasConflict(cls, { name, alias, backendCacheKey });+ this.#cache.checkNameAliasConflict(cls, { name, alias, backendCacheKey });// Cache miss — create the sub-backend and storage instance.const subBackend = await backendOpener();@@ -260,7 +260,7 @@ export class StorageInstanceManager {const instance = new cls({ metadata: storageInfo, backend: subBackend });// Atomic cache writes (no awaits between these).- this.cache.set(cls, instance, backendCacheKey, alias);+ this.#cache.set(cls, instance, backendCacheKey, alias);return instance;} finally {@@ -269,7 +269,7 @@ export class StorageInstanceManager {// Clean up idle locks so the map doesn't grow unboundedly// (mirrors crawlee-python's WeakValueDictionary behaviour).if (queue.remaining === 0) {- this.openerLocks.delete(lockKey);+ this.#openerLocks.delete(lockKey);}}}@@ -278,7 +278,7 @@ export class StorageInstanceManager {* Remove a storage instance from the cache (called from `storage.drop()`).*/removeFromCache(instance: IStorage): void {- this.cache.removeFromCache(instance);+ this.#cache.removeFromCache(instance);}/**@@ -287,13 +287,13 @@ export class StorageInstanceManager {* Called during service locator reset.*/clearCache(): void {- for (const instance of this.cache.allValues()) {+ for (const instance of this.#cache.allValues()) {if ('clearCache' in instance && typeof (instance as any).clearCache === 'function') {(instance as any).clearCache();}}- this.cache.clear();+ this.#cache.clear();}}diff --git a/packages/core/src/storages/storage_stats.ts b/packages/core/src/storages/storage_stats.tsindex cbf1ff686..5a78344ff 100644--- a/packages/core/src/storages/storage_stats.ts+++ b/packages/core/src/storages/storage_stats.ts@@ -43,19 +43,19 @@ export interface RequestQueueStats {* the buckets that make sense for it.*/export class StorageStatsTracker<T extends Record<keyof T, number>> {- private readonly counters: T;+ readonly #counters: T;constructor(initial: T) {- this.counters = { ...initial };+ this.#counters = { ...initial };}/** Increment a counter bucket by `by` (default `1`). */add(key: keyof T, by = 1): void {- (this.counters[key] as number) += by;+ (this.#counters[key] as number) += by;}/** Return a snapshot of the current counters. The returned object is a copy and safe to keep. */get current(): T {- return { ...this.counters };+ return { ...this.#counters };}}diff --git a/packages/core/src/storages/transaction.ts b/packages/core/src/storages/transaction.tsindex b3c8e7800..b16abd1b9 100644--- a/packages/core/src/storages/transaction.ts+++ b/packages/core/src/storages/transaction.ts@@ -169,20 +169,20 @@ export class StorageTransaction implements StorageTransactionView {/** Per-storage-type write policy. */readonly policy: StorageWritePolicy;- private readonly commitTimeoutMillis: number;+ readonly #commitTimeoutMillis: number;- private _state: StorageTransactionState = 'open';+ #state: StorageTransactionState = 'open';- private disposed = false;+ #disposed = false;/** @internal */constructor(options: StorageTransactionOptions = {}) {this.policy = { ...DEFAULT_STORAGE_WRITE_POLICY, ...options.policy };- this.commitTimeoutMillis = options.commitTimeoutMillis ?? DEFAULT_COMMIT_TIMEOUT_MILLIS;+ this.#commitTimeoutMillis = options.commitTimeoutMillis ?? DEFAULT_COMMIT_TIMEOUT_MILLIS;}get state(): StorageTransactionState {- return this._state;+ return this.#state;}/**@@ -190,7 +190,7 @@ export class StorageTransaction implements StorageTransactionView {* consults — operations performed after the transaction is closed pass through to the real backend.*/get isActive(): boolean {- return this._state === 'open';+ return this.#state === 'open';}/** Runs `callback` with this transaction installed in the async context. */@@ -204,7 +204,7 @@ export class StorageTransaction implements StorageTransactionView {*/recordJournalEntry(entry: JournalEntry): void {if (!this.isActive) {- throw new Error(`Cannot record a journal entry on a transaction in the '${this._state}' state`);+ throw new Error(`Cannot record a journal entry on a transaction in the '${this.#state}' state`);}this.journal.push(entry);@@ -219,11 +219,11 @@ export class StorageTransaction implements StorageTransactionView {* partway may have applied some of the writes already.*/async commit(): Promise<void> {- if (this._state !== 'open') {+ if (this.#state !== 'open') {return;}- this._state = 'committing';+ this.#state = 'committing';try {// The replay re-drives the frontend write path, which checks for cancellation (`tryCancel`)@@ -233,13 +233,13 @@ export class StorageTransaction implements StorageTransactionView {await timeoutStorage.exit(async () =>addTimeoutToPromise(async () => this.flush(),- this.commitTimeoutMillis,- `Committing the storage transaction timed out after ${this.commitTimeoutMillis / 1000} seconds.`,+ this.#commitTimeoutMillis,+ `Committing the storage transaction timed out after ${this.#commitTimeoutMillis / 1000} seconds.`,),);- this._state = 'committed';+ this.#state = 'committed';} catch (error) {- this._state = 'failed';+ this.#state = 'failed';throw error;}}@@ -274,11 +274,11 @@ export class StorageTransaction implements StorageTransactionView {* and never throws.*/rollback(): void {- if (this._state !== 'open') {+ if (this.#state !== 'open') {return;}- this._state = 'rolledBack';+ this.#state = 'rolledBack';}/**@@ -287,11 +287,11 @@ export class StorageTransaction implements StorageTransactionView {* {@apilink StorageTransactionView} of this transaction is only valid until this is called.*/dispose(): void {- if (this.disposed) {+ if (this.#disposed) {return;}- if (this._state === 'open') {+ if (this.#state === 'open') {// Disposing an open transaction is an internal invariant violation - roll back first.try {serviceLocator@@ -303,7 +303,7 @@ export class StorageTransaction implements StorageTransactionView {this.rollback();}- this.disposed = true;+ this.#disposed = true;this.journal.length = 0;}diff --git a/packages/core/src/system-info/runtime.ts b/packages/core/src/system-info/runtime.tsindex 63dfdefe4..32e86cc36 100644--- a/packages/core/src/system-info/runtime.ts+++ b/packages/core/src/system-info/runtime.ts@@ -65,29 +65,29 @@ export function isLambda() {return !!process.env.AWS_LAMBDA_FUNCTION_MEMORY_SIZE;}-let _cgroupsVersion: null | 'V1' | 'V2';+let cgroupsVersion: null | 'V1' | 'V2';/*** gets the cgroup version by checking for a file at /sys/fs/cgroup/memory* @returns "V1" or "V2" for the version of cgroup or null if cgroup is not found.*/export async function getCgroupsVersion(forceReset?: boolean) {// Parameter forceReset is just internal for unit tests.- if (_cgroupsVersion !== undefined && !forceReset) {- return _cgroupsVersion;+ if (cgroupsVersion !== undefined && !forceReset) {+ return cgroupsVersion;}try {// If this directory does not exists, cgroups are not availableawait fs.access('/sys/fs/cgroup/');} catch (e) {- _cgroupsVersion = null;+ cgroupsVersion = null;return null;}- _cgroupsVersion = 'V1';+ cgroupsVersion = 'V1';try {// If this directory does not exists, assume the container is using cgroups V2await fs.access('/sys/fs/cgroup/memory/');} catch (e) {- _cgroupsVersion = 'V2';+ cgroupsVersion = 'V2';}- return _cgroupsVersion;+ return cgroupsVersion;}diff --git a/packages/fs-storage/src/resource-clients/dataset.ts b/packages/fs-storage/src/resource-clients/dataset.tsindex f8f415b70..b6cb62a75 100644--- a/packages/fs-storage/src/resource-clients/dataset.ts+++ b/packages/fs-storage/src/resource-clients/dataset.ts@@ -42,19 +42,19 @@ export class DatasetBackend<Data extends Dictionary = Dictionary>readonly name?: string;readonly cacheKey: string;- private readonly nativeBackend: NativeFileSystemDatasetBackend;- private readonly logger?: CrawleeLogger;+ readonly #nativeBackend: NativeFileSystemDatasetBackend;+ readonly #logger?: CrawleeLogger;constructor(options: DatasetBackendOptions) {super();this.name = options.name;this.cacheKey = options.cacheKey;- this.nativeBackend = options.nativeBackend;- this.logger = options.logger;+ this.#nativeBackend = options.nativeBackend;+ this.#logger = options.logger;}get datasetDirectory(): string {- return this.nativeBackend.pathToDataset;+ return this.#nativeBackend.pathToDataset;}static async create<Data extends Dictionary = Dictionary>(@@ -66,26 +66,26 @@ export class DatasetBackend<Data extends Dictionary = Dictionary>}async getMetadata(): Promise<storage.DatasetInfo> {- return this.nativeBackend.getMetadata();+ return this.#nativeBackend.getMetadata();}async drop(): Promise<void> {- await this.nativeBackend.dropStorage();+ await this.#nativeBackend.dropStorage();}async purge(): Promise<void> {- await this.nativeBackend.purge();+ await this.#nativeBackend.purge();}async pushData(items: Data[]): Promise<void> {- await this.nativeBackend.pushData(items);+ await this.#nativeBackend.pushData(items);}async getData(options: storage.DatasetBackendListOptions = {}): Promise<storage.PaginatedList<Data>> {const passedOptions = options as Record<string, unknown>;const ignored = UNSUPPORTED_GET_DATA_OPTIONS.filter((key) => passedOptions[key] !== undefined);if (ignored.length > 0) {- this.logger?.warning?.(+ this.#logger?.warning?.(`getData() options [${ignored.join(', ')}] are not supported by the file-system dataset ` +`and were ignored. Only "offset", "limit" and "desc" are honored.`,);@@ -99,7 +99,7 @@ export class DatasetBackend<Data extends Dictionary = Dictionary>}).parse(options);- const page = await this.nativeBackend.getData(offset ?? 0, limit, desc ?? false, false);+ const page = await this.#nativeBackend.getData(offset ?? 0, limit, desc ?? false, false);return {count: page.count,diff --git a/packages/fs-storage/src/resource-clients/key-value-store.ts b/packages/fs-storage/src/resource-clients/key-value-store.tsindex 90f9a956d..ca5f293dd 100644--- a/packages/fs-storage/src/resource-clients/key-value-store.ts+++ b/packages/fs-storage/src/resource-clients/key-value-store.ts@@ -85,17 +85,17 @@ export class KeyValueStoreBackend extends CachedIdClient implements storage.KeyVreadonly name?: string;readonly cacheKey: string;- private readonly nativeBackend: NativeFileSystemKeyValueStoreBackend;+ readonly #nativeBackend: NativeFileSystemKeyValueStoreBackend;constructor(options: KeyValueStoreBackendOptions) {super();this.name = options.name;this.cacheKey = options.cacheKey;- this.nativeBackend = options.nativeBackend;+ this.#nativeBackend = options.nativeBackend;}get keyValueStoreDirectory(): string {- return this.nativeBackend.pathToKvs;+ return this.#nativeBackend.pathToKvs;}static async create(options: KeyValueStoreBackendOptions): Promise<KeyValueStoreBackend> {@@ -105,15 +105,15 @@ export class KeyValueStoreBackend extends CachedIdClient implements storage.KeyV}async getMetadata(): Promise<storage.KeyValueStoreInfo> {- return this.nativeBackend.getMetadata();+ return this.#nativeBackend.getMetadata();}async drop(): Promise<void> {- await this.nativeBackend.dropStorage();+ await this.#nativeBackend.dropStorage();}async purge(): Promise<void> {- await this.nativeBackend.purge();+ await this.#nativeBackend.purge();}/**@@ -125,7 +125,7 @@ export class KeyValueStoreBackend extends CachedIdClient implements storage.KeyV* filename the input might live under (`INPUT`, `INPUT.json`, `INPUT.txt`, `INPUT.bin`).*/async purgeExceptInput(): Promise<void> {- await this.nativeBackend.purge(BARE_FILE_FALLBACKS.flatMap(({ extension }) => `INPUT${extension}`));+ await this.#nativeBackend.purge(BARE_FILE_FALLBACKS.flatMap(({ extension }) => `INPUT${extension}`));}async listKeys(options: storage.KeyValueStoreListKeysOptions = {}): Promise<storage.KeyValueStoreListKeysResult> {@@ -142,7 +142,7 @@ export class KeyValueStoreBackend extends CachedIdClient implements storage.KeyV// everything it needs off the filesystem index — no per-file reads — so this stays cheap.// The native `listKeys` already returns a self-describing page (items + pagination cursors)// matching the `KeyValueStoreListKeysResult` contract, so we only post-process the items.- const page = await this.nativeBackend.listKeys(exclusiveStartKey, limit, prefix, LIST_BARE_FALLBACKS);+ const page = await this.#nativeBackend.listKeys(exclusiveStartKey, limit, prefix, LIST_BARE_FALLBACKS);const presentKeys = new Set(page.items.map((record) => record.key));@@ -183,7 +183,7 @@ export class KeyValueStoreBackend extends CachedIdClient implements storage.KeyVif (resolvedKey === undefined) {return undefined;}- return (await this.nativeBackend.getPublicUrl(resolvedKey)) ?? undefined;+ return (await this.#nativeBackend.getPublicUrl(resolvedKey)) ?? undefined;}/**@@ -202,8 +202,8 @@ export class KeyValueStoreBackend extends CachedIdClient implements storage.KeyVconst fallbacks = this.bareFallbacksFor(key);const record = fallbacks- ? await this.nativeBackend.resolveValue(key, fallbacks)- : await this.nativeBackend.getValue(key);+ ? await this.#nativeBackend.resolveValue(key, fallbacks)+ : await this.#nativeBackend.getValue(key);if (record) {return {@@ -244,7 +244,7 @@ export class KeyValueStoreBackend extends CachedIdClient implements storage.KeyV// consumes a Web `ReadableStream`, so convert the Node `Readable` we get from the frontend.if (isStream(value)) {const webStream = Readable.toWeb(value as Readable) as ReadableStream<Uint8Array>;- await this.nativeBackend.setValueStream(key, webStream, contentType);+ await this.#nativeBackend.setValueStream(key, webStream, contentType);return;}@@ -257,12 +257,12 @@ export class KeyValueStoreBackend extends CachedIdClient implements storage.KeyV? Buffer.from(value.buffer, value.byteOffset, value.byteLength): Buffer.from(value as string);- await this.nativeBackend.setValue(key, buffer, contentType);+ await this.#nativeBackend.setValue(key, buffer, contentType);}async deleteValue(key: string): Promise<void> {s.string().parse(key);- await this.nativeBackend.deleteValue(key);+ await this.#nativeBackend.deleteValue(key);}/**@@ -277,13 +277,13 @@ export class KeyValueStoreBackend extends CachedIdClient implements storage.KeyVconst fallbacks = this.bareFallbacksFor(key);if (fallbacks) {return (- (await this.nativeBackend.resolveExistingKey(+ (await this.#nativeBackend.resolveExistingKey(key,fallbacks.map(({ extension }) => extension),)) ?? undefined);}- return (await this.nativeBackend.recordExists(key)) ? key : undefined;+ return (await this.#nativeBackend.recordExists(key)) ? key : undefined;}/**diff --git a/packages/fs-storage/src/resource-clients/request-queue.ts b/packages/fs-storage/src/resource-clients/request-queue.tsindex 95b9c0fff..5e41b51ee 100644--- a/packages/fs-storage/src/resource-clients/request-queue.ts+++ b/packages/fs-storage/src/resource-clients/request-queue.ts@@ -63,17 +63,17 @@ export class RequestQueueBackend extends CachedIdClient implements storage.Requereadonly name?: string;readonly cacheKey: string;- private readonly nativeBackend: NativeFileSystemRequestQueueBackend;+ readonly #nativeBackend: NativeFileSystemRequestQueueBackend;constructor(options: RequestQueueBackendOptions) {super();this.name = options.name;this.cacheKey = options.cacheKey;- this.nativeBackend = options.nativeBackend;+ this.#nativeBackend = options.nativeBackend;}get requestQueueDirectory(): string {- return this.nativeBackend.pathToRq;+ return this.#nativeBackend.pathToRq;}static async create(options: RequestQueueBackendOptions): Promise<RequestQueueBackend> {@@ -87,19 +87,19 @@ export class RequestQueueBackend extends CachedIdClient implements storage.Reque* available again.*/async setExpectedRequestProcessingTimeSecs(secs: number): Promise<void> {- await this.nativeBackend.setExpectedRequestProcessingTime(secs);+ await this.#nativeBackend.setExpectedRequestProcessingTime(secs);}async getMetadata(): Promise<storage.RequestQueueInfo> {- return this.nativeBackend.getMetadata();+ return this.#nativeBackend.getMetadata();}async drop(): Promise<void> {- await this.nativeBackend.dropStorage();+ await this.#nativeBackend.dropStorage();}async purge(): Promise<void> {- await this.nativeBackend.purge();+ await this.#nativeBackend.purge();}async addBatchOfRequests(@@ -109,7 +109,7 @@ export class RequestQueueBackend extends CachedIdClient implements storage.RequebatchRequestShapeWithoutId.parse(requests);requestOptionsShape.parse(options);- const response = await this.nativeBackend.addBatchOfRequests(+ const response = await this.#nativeBackend.addBatchOfRequests(requests.map((request) => plainifyRequest(request)),options.forefront ?? false,);@@ -128,16 +128,16 @@ export class RequestQueueBackend extends CachedIdClient implements storage.Reque// The native client tags requests with an internal `orderNo`; it's harmless to leak, so we// hand the request back as-is rather than copying it just to drop one undeclared property.// The native client already returns `undefined` for a missing request, matching this contract.- return (await this.nativeBackend.getRequest(uniqueKey)) as storage.UpdateRequestSchema | undefined;+ return (await this.#nativeBackend.getRequest(uniqueKey)) as storage.UpdateRequestSchema | undefined;}async fetchNextRequest(): Promise<storage.UpdateRequestSchema | undefined> {- return (await this.nativeBackend.fetchNextRequest()) as storage.UpdateRequestSchema | undefined;+ return (await this.#nativeBackend.fetchNextRequest()) as storage.UpdateRequestSchema | undefined;}async markRequestAsHandled(request: storage.UpdateRequestSchema): Promise<storage.QueueOperationInfo | undefined> {requestShape.parse(request);- return (await this.nativeBackend.markRequestAsHandled(plainifyRequest(request))) ?? undefined;+ return (await this.#nativeBackend.markRequestAsHandled(plainifyRequest(request))) ?? undefined;}async reclaimRequest(@@ -147,16 +147,17 @@ export class RequestQueueBackend extends CachedIdClient implements storage.RequerequestShape.parse(request);requestOptionsShape.parse(options);return (- (await this.nativeBackend.reclaimRequest(plainifyRequest(request), options.forefront ?? false)) ?? undefined+ (await this.#nativeBackend.reclaimRequest(plainifyRequest(request), options.forefront ?? false)) ??+ undefined);}async isEmpty(): Promise<boolean> {- return this.nativeBackend.isEmpty();+ return this.#nativeBackend.isEmpty();}async isFinished(): Promise<boolean> {- return this.nativeBackend.isFinished();+ return this.#nativeBackend.isFinished();}/**@@ -165,6 +166,6 @@ export class RequestQueueBackend extends CachedIdClient implements storage.Reque* for the next consumer of the same on-disk queue.*/async persistState(): Promise<void> {- await this.nativeBackend.persistState();+ await this.#nativeBackend.persistState();}}diff --git a/packages/http-client/src/base-http-client.ts b/packages/http-client/src/base-http-client.tsindex b9ea44d62..daec82dba 100644--- a/packages/http-client/src/base-http-client.ts+++ b/packages/http-client/src/base-http-client.ts@@ -41,10 +41,10 @@ export interface CustomFetchOptions {* implement only the low-level network call in `fetch`.*/export abstract class BaseHttpClient implements BaseHttpClientInterface {- private log?: CrawleeLogger;+ #log?: CrawleeLogger;constructor(options?: { logger?: CrawleeLogger }) {- this.log = options?.logger;+ this.#log = options?.logger;}/**@@ -82,7 +82,7 @@ export abstract class BaseHttpClient implements BaseHttpClientInterface {request.headers.set('cookie', cookieString);}} catch (e) {- this.log?.warning(`Failed to get cookies for URL "${request.url}": ${(e as Error).message}`);+ this.#log?.warning(`Failed to get cookies for URL "${request.url}": ${(e as Error).message}`);}return request;@@ -95,7 +95,7 @@ export abstract class BaseHttpClient implements BaseHttpClientInterface {try {await cookieJar.setCookie(header, response.url);} catch (e) {- this.log?.warning(`Failed to set cookie for URL "${response.url}": ${(e as Error).message}`);+ this.#log?.warning(`Failed to set cookie for URL "${response.url}": ${(e as Error).message}`);}}}diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.tsindex 9b606e384..e0ad1f3f7 100644--- a/packages/http-crawler/src/internals/http-crawler.ts+++ b/packages/http-crawler/src/internals/http-crawler.ts@@ -200,6 +200,10 @@ interface CrawlingContextWithResponse<response: Response;}+type InternalHttpPostNavigationHook = (+ crawlingContext: CrawlingContextWithResponse,+) => Awaitable<void | Partial<CrawlingContextWithResponse>>;+export interface InternalHttpCrawlingContext<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawlerJSONData extends JsonValue = any, // with default to Dictionary we cant use a typed router in untyped crawler@@ -340,16 +344,15 @@ export class HttpCrawler<// extension-aware for consumer DX, but internally the pipeline composes hooks against the// concrete crawling context, which does not statically carry `ContextExtension`. The members// added by `extendContext` are present at runtime regardless.- private preNavigationHooks: InternalHttpHook<CrawlingContext>[];- private postNavigationHooks: ((- crawlingContext: CrawlingContextWithResponse,- ) => Awaitable<void | Partial<CrawlingContextWithResponse>>)[];- private saveResponseCookies: boolean;- private navigationTimeoutMillis: number;+ #preNavigationHooks: InternalHttpHook<CrawlingContext>[];+ #postNavigationHooks: InternalHttpPostNavigationHook[];+ #saveResponseCookies: boolean;+ #navigationTimeoutMillis: number;+ // kept as TS-private: tests read it at runtimeprivate ignoreSslErrors: boolean;- private suggestResponseEncoding?: string;- private forceResponseEncoding?: string;- private readonly supportedMimeTypes: Set<string>;+ #suggestResponseEncoding?: string;+ #forceResponseEncoding?: string;+ readonly #supportedMimeTypes: Set<string>;protected static override optionsShape = {...BasicCrawler.optionsShape,@@ -396,7 +399,7 @@ export class HttpCrawler<(() => this.buildContextPipeline() as ContextPipeline<CrawlingContext, Context>),});- this.supportedMimeTypes = new Set([...HTML_AND_XML_MIME_TYPES, APPLICATION_JSON_MIME_TYPE]);+ this.#supportedMimeTypes = new Set([...HTML_AND_XML_MIME_TYPES, APPLICATION_JSON_MIME_TYPE]);if (additionalMimeTypes.length) this.extendSupportedMimeTypes(additionalMimeTypes);if (suggestResponseEncoding && forceResponseEncoding) {@@ -405,24 +408,24 @@ export class HttpCrawler<);}- this.navigationTimeoutMillis = navigationTimeoutSecs * 1000;+ this.#navigationTimeoutMillis = navigationTimeoutSecs * 1000;this.ignoreSslErrors = ignoreSslErrors;- this.suggestResponseEncoding = suggestResponseEncoding;- this.forceResponseEncoding = forceResponseEncoding;+ this.#suggestResponseEncoding = suggestResponseEncoding;+ this.#forceResponseEncoding = forceResponseEncoding;// Cast away the extension-aware option types to the base internal storage types (see the field// declarations above). This is sound - the hooks only ever receive the base context plus the// members `extendContext` added at runtime.- this.preNavigationHooks = preNavigationHooks as InternalHttpHook<CrawlingContext>[];- this.postNavigationHooks = [- ({ request, response }) => this._abortDownloadOfBody(request, response!),- ...(postNavigationHooks as typeof this.postNavigationHooks),+ this.#preNavigationHooks = preNavigationHooks as InternalHttpHook<CrawlingContext>[];+ this.#postNavigationHooks = [+ ({ request, response }) => this.abortDownloadOfBody(request, response!),+ ...(postNavigationHooks as InternalHttpPostNavigationHook[]),];- this.saveResponseCookies = saveResponseCookies;+ this.#saveResponseCookies = saveResponseCookies;}protected override getNavigationTimeoutMillis(): number {- return this.navigationTimeoutMillis;+ return this.#navigationTimeoutMillis;}/**@@ -450,12 +453,12 @@ export class HttpCrawler<// A single navigation window covers the pre-navigation hooks, the navigation, and the post-navigation// hooks: the whole phase shares one `navigationTimeoutSecs` budget, so a slow hook eats into the same// window the navigation uses instead of each step being timed on its own.- const navigationTimedOut = `Navigation timed out after ${this.navigationTimeoutMillis / 1000} seconds.`;+ const navigationTimedOut = `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`;const windowGuard = <Ctx extends CrawlingContext, Ext>(step: (ctx: Ctx) => Awaitable<void | Ext>,): ContextMiddleware<Ctx, Ext> =>skipGuard(async (ctx: Ctx) => {- const remaining = remainingNavigationWindowMillis(ctx, this.navigationTimeoutMillis);+ const remaining = remainingNavigationWindowMillis(ctx, this.#navigationTimeoutMillis);if (remaining <= 0) {throw new TimeoutError(navigationTimedOut);}@@ -466,13 +469,13 @@ export class HttpCrawler<action: this.prepareHttpRequest.bind(this),});- for (const hook of this.preNavigationHooks) {+ for (const hook of this.#preNavigationHooks) {pipeline = pipeline.compose(windowGuard(hook));}let pipelineWithNavigation = pipeline.compose(skipGuard(this.makeHttpRequest.bind(this)));- for (const hook of this.postNavigationHooks) {+ for (const hook of this.#postNavigationHooks) {pipelineWithNavigation = pipelineWithNavigation.compose(windowGuard(hook));}@@ -521,8 +524,8 @@ export class HttpCrawler<// client abort.const httpResponse = await addTimeoutToPromise(async () => this.requestFunction({ request, session, proxyUrl }),- Math.max(1, remainingNavigationWindowMillis(crawlingContext, this.navigationTimeoutMillis)),- `Navigation timed out after ${this.navigationTimeoutMillis / 1000} seconds.`,+ Math.max(1, remainingNavigationWindowMillis(crawlingContext, this.#navigationTimeoutMillis)),+ `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`,);tryCancel();@@ -572,14 +575,14 @@ export class HttpCrawler<// Reading the body is still part of the navigation, so it draws from the same shared window: on a server// that streams the body slowly the request completes (headers arrive) but the body read would otherwise// run unbounded. `extendTimeout` from a post-navigation hook has already pushed this deadline out if asked.- const remaining = remainingNavigationWindowMillis(crawlingContext, this.navigationTimeoutMillis);+ const remaining = remainingNavigationWindowMillis(crawlingContext, this.#navigationTimeoutMillis);if (remaining <= 0) {- throw new TimeoutError(`Navigation timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);+ throw new TimeoutError(`Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);}const parsed = await addTimeoutToPromise(async () => this.parseResponse(crawlingContext.request, crawlingContext.response),remaining,- `Navigation timed out after ${this.navigationTimeoutMillis / 1000} seconds.`,+ `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`,);tryCancel();const response = parsed.response!;@@ -604,9 +607,9 @@ export class HttpCrawler<return $;};- this._throwOnBlockedRequest(response.status);+ this.throwOnBlockedRequest(response.status);- if (this.saveResponseCookies) {+ if (this.#saveResponseCookies) {try {for (const cookie of getCookiesFromResponse(response)) {if (!cookie) continue;@@ -669,7 +672,7 @@ export class HttpCrawler<const opts = this.getRequestOptions(request, session, proxyUrl);try {- return await this._requestAsBrowser(opts, session);+ return await this.requestAsBrowser(opts, session);} catch (e) {if (e instanceof Error && e.constructor.name === 'TimeoutError') {this.handleRequestTimeout(session);@@ -677,7 +680,7 @@ export class HttpCrawler<}if (this.isProxyError(e as Error)) {- throw new SessionError(this._getMessageFromError(e as Error) as string);+ throw new SessionError(this.getMessageFromError(e as Error) as string);} else {throw e;}@@ -716,10 +719,10 @@ export class HttpCrawler<// It's not a JSON, so it's probably some text. Get the first 100 chars of it.throw new Error(`${status} - Internal Server Error: ${body.slice(0, 100)}`);} else if (HTML_AND_XML_MIME_TYPES.includes(type)) {- if (!charset && !this.forceResponseEncoding) {+ if (!charset && !this.#forceResponseEncoding) {const rawBytes = Buffer.from(await response.arrayBuffer());const metaCharset = extractCharsetFromHtmlBytes(rawBytes);- const charsetToUse = metaCharset ?? this.suggestResponseEncoding ?? 'utf-8';+ const charsetToUse = metaCharset ?? this.#suggestResponseEncoding ?? 'utf-8';const body = iconv.encodingExists(charsetToUse)? iconv.decode(rawBytes, charsetToUse): rawBytes.toString('utf8');@@ -744,7 +747,7 @@ export class HttpCrawler<url: request.url,method: request.method,proxyUrl,- timeout: this.navigationTimeoutMillis,+ timeout: this.#navigationTimeoutMillis,sessionToken: session,headers: request.headers,https: {@@ -754,7 +757,7 @@ export class HttpCrawler<};if (requestOptions.headers?.cookie || requestOptions.headers?.Cookie) {- requestOptions.headers!.Cookie = this._getCookieHeaderFromRequest(request);+ requestOptions.headers!.Cookie = this.getCookieHeaderFromRequest(request);delete requestOptions.headers!.cookie;}@@ -779,10 +782,10 @@ export class HttpCrawler<encoding: BufferEncoding;response: Response;} {- if (this.forceResponseEncoding) {- encoding = this.forceResponseEncoding as BufferEncoding;- } else if (!encoding && this.suggestResponseEncoding) {- encoding = this.suggestResponseEncoding as BufferEncoding;+ if (this.#forceResponseEncoding) {+ encoding = this.#forceResponseEncoding as BufferEncoding;+ } else if (!encoding && this.#suggestResponseEncoding) {+ encoding = this.#suggestResponseEncoding as BufferEncoding;}// Fall back to utf-8 if we still don't have encoding.@@ -824,13 +827,13 @@ export class HttpCrawler<private extendSupportedMimeTypes(additionalMimeTypes: (string | RequestLike | ResponseLike)[]) {for (const mimeType of additionalMimeTypes) {if (mimeType === '*/*') {- this.supportedMimeTypes.add(mimeType);+ this.#supportedMimeTypes.add(mimeType);continue;}try {const parsedType = contentTypeParser.parse(mimeType);- this.supportedMimeTypes.add(parsedType.type);+ this.#supportedMimeTypes.add(parsedType.type);} catch (err) {throw new Error(`Can not parse mime type ${mimeType} from "options.additionalMimeTypes".`);}@@ -842,20 +845,20 @@ export class HttpCrawler<*/private handleRequestTimeout(session: ISession) {session.markBad();- throw new Error(`Request timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);+ throw new Error(`Request timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);}- private _abortDownloadOfBody(request: CrawleeRequest, response: Response) {+ private abortDownloadOfBody(request: CrawleeRequest, response: Response) {const { status } = response;const { type } = parseContentTypeFromResponse(response);const isTransientContentType = status >= 500 || this.blockedStatusCodes.has(status);- if (!this.supportedMimeTypes.has(type) && !this.supportedMimeTypes.has('*/*') && !isTransientContentType) {+ if (!this.#supportedMimeTypes.has(type) && !this.#supportedMimeTypes.has('*/*') && !isTransientContentType) {request.noRetry = true;throw new Error(`Resource ${request.url} served Content-Type ${type}, ` +- `but only ${Array.from(this.supportedMimeTypes).join(', ')} are allowed. Skipping resource.`,+ `but only ${Array.from(this.#supportedMimeTypes).join(', ')} are allowed. Skipping resource.`,);}}@@ -863,7 +866,7 @@ export class HttpCrawler</*** @internal wraps public utility for mocking purposes*/- private _requestAsBrowser = async (options: Dictionary<any>, session: ISession) => {+ private requestAsBrowser = async (options: Dictionary<any>, session: ISession) => {const opts = processHttpRequestOptions({...(options as any),responseType: 'text',@@ -872,7 +875,7 @@ export class HttpCrawler<// When saveResponseCookies is false, the response cookies must not mutate the// session jar. Reads still go through the session (so session.setCookie() in pre-nav// hooks keeps working) but a per-request clone is passed in so writes are discarded.- const cookieJar = this.saveResponseCookies ? session.cookieJar : await session.cookieJar.clone();+ const cookieJar = this.#saveResponseCookies ? session.cookieJar : await session.cookieJar.clone();// Bind the request to the shared navigation window instead of a fixed per-request timeout, so// `extendTimeout()` can push the deadline and a fixed `AbortSignal.timeout` won't fire on its own anddiff --git a/packages/impit-client/src/index.ts b/packages/impit-client/src/index.tsindex 48f8eed70..9647a0fb2 100644--- a/packages/impit-client/src/index.ts+++ b/packages/impit-client/src/index.ts@@ -36,8 +36,8 @@ export const Browser = {* A HTTP client implementation based on the `impit` library.*/export class ImpitHttpClient extends BaseHttpClient {- private impitOptions: ImpitOptions;- private cacheClients: boolean;+ #impitOptions: ImpitOptions;+ #cacheClients: boolean;/*** Enables reuse of `impit` clients for the same set of options.@@ -45,31 +45,31 @@ export class ImpitHttpClient extends BaseHttpClient {* a new client for each request breaks TCP connection* (and other resources) reuse.*/- private clientCache: LruCache<{ client: Impit; cookieJar: ToughCookieJar }> = new LruCache({ maxLength: 10 });+ #clientCache: LruCache<{ client: Impit; cookieJar: ToughCookieJar }> = new LruCache({ maxLength: 10 });/*** Stable impit impersonation version per fingerprint object, so the same* session keeps impersonating the same browser version across requests* instead of rerolling on every call.*/- private impitBrowserByFingerprint = new WeakMap<SessionFingerprint, ImpitBrowser>();+ #impitBrowserByFingerprint = new WeakMap<SessionFingerprint, ImpitBrowser>();private getClient(options: ImpitOptions): Impit {- if (!this.cacheClients) {+ if (!this.#cacheClients) {return new Impit(options);}const { cookieJar, ...rest } = options;const cacheKey = JSON.stringify(rest);- const existingClient = this.clientCache.get(cacheKey);+ const existingClient = this.#clientCache.get(cacheKey);if (existingClient && (!cookieJar || existingClient.cookieJar === cookieJar)) {return existingClient.client;}const client = new Impit(options);- this.clientCache.add(cacheKey, { client, cookieJar: cookieJar as ToughCookieJar });+ this.#clientCache.add(cacheKey, { client, cookieJar: cookieJar as ToughCookieJar });return client;}@@ -83,8 +83,8 @@ export class ImpitHttpClient extends BaseHttpClient {super({ logger: options?.logger });const { cacheClients = true, logger: _logger, ...impitOptions } = options ?? {};- this.impitOptions = impitOptions;- this.cacheClients = cacheClients;+ this.#impitOptions = impitOptions;+ this.#cacheClients = cacheClients;}/**@@ -96,7 +96,7 @@ export class ImpitHttpClient extends BaseHttpClient {const impitBrowser = this.resolveImpitBrowser(fingerprint);const impit = this.getClient({- ...this.impitOptions,+ ...this.#impitOptions,...(impitBrowser ? { browser: impitBrowser } : {}),proxyUrl,followRedirects: redirect === 'follow',@@ -111,7 +111,7 @@ export class ImpitHttpClient extends BaseHttpClient {private resolveImpitBrowser(fingerprint?: SessionFingerprint): ImpitBrowser | undefined {if (!fingerprint?.browser) return undefined;- const cached = this.impitBrowserByFingerprint.get(fingerprint);+ const cached = this.#impitBrowserByFingerprint.get(fingerprint);if (cached) return cached;// impit can only impersonate Chrome and Firefox. Map other (Chromium-based or@@ -120,7 +120,7 @@ export class ImpitHttpClient extends BaseHttpClient {const versions = IMPIT_VERSIONS_BY_BROWSER[fingerprint.browser] ?? IMPIT_VERSIONS_BY_BROWSER.chrome!;const picked = versions[Math.floor(Math.random() * versions.length)];- this.impitBrowserByFingerprint.set(fingerprint, picked);+ this.#impitBrowserByFingerprint.set(fingerprint, picked);return picked;}}diff --git a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts b/packages/jsdom-crawler/src/internals/jsdom-crawler.tsindex 39e5cdb06..001e806d4 100644--- a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts+++ b/packages/jsdom-crawler/src/internals/jsdom-crawler.ts@@ -199,9 +199,9 @@ export class JSDOMCrawler<hideInternalConsole: ow.optional.boolean,};- private runScripts: boolean;- private hideInternalConsole: boolean;- private virtualConsole: VirtualConsole | null = null;+ #runScripts: boolean;+ #hideInternalConsole: boolean;+ #virtualConsole: VirtualConsole | null = null;constructor(options: JSDOMCrawlerOptions<ContextExtension, ExtendedContext, any, any, Routes> = {}) {const { runScripts = false, hideInternalConsole = false, contextPipelineBuilder, ...httpOptions } = options;@@ -211,8 +211,8 @@ export class JSDOMCrawler<contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),});- this.runScripts = runScripts;- this.hideInternalConsole = hideInternalConsole;+ this.#runScripts = runScripts;+ this.#hideInternalConsole = hideInternalConsole;}protected override buildContextPipeline() {@@ -243,19 +243,19 @@ export class JSDOMCrawler<* ```*/getVirtualConsole() {- if (this.virtualConsole) {- return this.virtualConsole;+ if (this.#virtualConsole) {+ return this.#virtualConsole;}- this.virtualConsole = new VirtualConsole();+ this.#virtualConsole = new VirtualConsole();- if (!this.hideInternalConsole) {- this.virtualConsole.sendTo(console, { omitJSDOMErrors: true });+ if (!this.#hideInternalConsole) {+ this.#virtualConsole.sendTo(console, { omitJSDOMErrors: true });}- this.virtualConsole.on('jsdomError', this.jsdomErrorHandler);+ this.#virtualConsole.on('jsdomError', this.jsdomErrorHandler);- return this.virtualConsole;+ return this.#virtualConsole;}private readonly jsdomErrorHandler = (error: Error) => this.log.debug('JSDOM error from console', { error });@@ -268,7 +268,7 @@ export class JSDOMCrawler<const { window } = new JSDOM(crawlingContext.body.toString(), {url: crawlingContext.response.url,contentType: isXml ? 'text/xml' : 'text/html',- runScripts: this.runScripts ? 'dangerously' : undefined,+ runScripts: this.#runScripts ? 'dangerously' : undefined,resources,virtualConsole: this.getVirtualConsole(),pretendToBeVisual: true,@@ -295,7 +295,7 @@ export class JSDOMCrawler<return range;};- if (this.runScripts) {+ if (this.#runScripts) {try {await addTimeoutToPromise(async () => {diff --git a/packages/linkedom-crawler/src/internals/linkedom-crawler.ts b/packages/linkedom-crawler/src/internals/linkedom-crawler.tsindex dae37dfc0..3593b4f4b 100644--- a/packages/linkedom-crawler/src/internals/linkedom-crawler.ts+++ b/packages/linkedom-crawler/src/internals/linkedom-crawler.ts@@ -177,7 +177,7 @@ export class LinkeDOMCrawler<GetUserDataFromRequest<LinkeDOMCrawlingContext['request']>>,> extends HttpCrawler<LinkeDOMCrawlingContext, ContextExtension, ExtendedContext, Routes> {- private static parser = new DOMParser();+ static #parser = new DOMParser();constructor(options: LinkeDOMCrawlerOptions<ContextExtension, ExtendedContext, any, any, Routes>) {const { contextPipelineBuilder, ...rest } = options;@@ -200,7 +200,7 @@ export class LinkeDOMCrawler<private async parseContent(crawlingContext: InternalHttpCrawlingContext) {try {const isXml = crawlingContext.contentType.type.includes('xml');- const document = LinkeDOMCrawler.parser.parseFromString(+ const document = LinkeDOMCrawler.#parser.parseFromString(crawlingContext.body.toString(),isXml ? 'text/xml' : 'text/html',);diff --git a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.tsindex 87a57a820..4b62c7c1c 100644--- a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts+++ b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts@@ -318,28 +318,26 @@ export class AdaptivePlaywrightCrawler<GetUserDataFromRequest<AdaptivePlaywrightCrawlerContext['request']>>,> extends BasicCrawler<AdaptivePlaywrightCrawlerContext, ContextExtension, ExtendedContext, Routes> {- private renderingTypePredictor: OwnedOrInjected<IRenderingTypePredictor, RenderingTypePredictor>;- private resultChecker: NonNullable<AdaptivePlaywrightCrawlerOptions['resultChecker']>;- private shouldPropagateError: NonNullable<AdaptivePlaywrightCrawlerOptions['shouldPropagateError']>;- private resultComparator: NonNullable<AdaptivePlaywrightCrawlerOptions['resultComparator']>;- private staticContextPipeline: ContextPipeline<CrawlingContext, ExtendedContext>;- private browserContextPipeline: ContextPipeline<CrawlingContext, ExtendedContext>;- private individualRequestHandlerTimeoutMillis: number;+ #renderingTypePredictor: OwnedOrInjected<IRenderingTypePredictor, RenderingTypePredictor>;+ #resultChecker: NonNullable<AdaptivePlaywrightCrawlerOptions['resultChecker']>;+ #shouldPropagateError: NonNullable<AdaptivePlaywrightCrawlerOptions['shouldPropagateError']>;+ #resultComparator: NonNullable<AdaptivePlaywrightCrawlerOptions['resultComparator']>;+ #staticContextPipeline: ContextPipeline<CrawlingContext, ExtendedContext>;+ #browserContextPipeline: ContextPipeline<CrawlingContext, ExtendedContext>;+ #individualRequestHandlerTimeoutMillis: number;// The constructor always injects an `AdaptivePlaywrightCrawlerStatistics`, so narrowing the cast is sound.override get stats(): AdaptivePlaywrightCrawlerStatistics {return super.stats as AdaptivePlaywrightCrawlerStatistics;}- private inFlightRenderingTypeDetections = 0;-/*** The write policy of the per-attempt transactions. Defaults the request queue to `deferred`:* a discarded attempt's enqueues must never reach the queue.*/- private readonly attemptWritePolicy: Partial<StorageWritePolicy>;+ readonly #attemptWritePolicy: Partial<StorageWritePolicy>;- private teardownHooks: (() => Promise<unknown>)[] = [];+ #teardownHooks: (() => Promise<unknown>)[] = [];constructor(options: AdaptivePlaywrightCrawlerOptions<ContextExtension, ExtendedContext, Routes> = {}) {const {@@ -400,28 +398,28 @@ export class AdaptivePlaywrightCrawler<// user-facing option (validated above) to those.transactionalStorage: false,});- this.individualRequestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;+ this.#individualRequestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;// `renderingTypeDetectionRatio` only configures the default predictor - an injected one brings its own// detection ratio (and its own state), so the option is ignored in that case.- this.renderingTypePredictor = OwnedOrInjected.resolve<IRenderingTypePredictor, RenderingTypePredictor>(+ this.#renderingTypePredictor = OwnedOrInjected.resolve<IRenderingTypePredictor, RenderingTypePredictor>(renderingTypePredictor,() => new RenderingTypePredictor({ detectionRatio: renderingTypeDetectionRatio }),);- this.attemptWritePolicy = {+ this.#attemptWritePolicy = {requestQueue: 'deferred',...(typeof transactionalStorage === 'object' ? transactionalStorage : {}),};- this.resultChecker = resultChecker ?? (() => true);- this.shouldPropagateError = shouldPropagateError ?? (() => false);+ this.#resultChecker = resultChecker ?? (() => true);+ this.#shouldPropagateError = shouldPropagateError ?? (() => false);if (resultComparator !== undefined) {- this.resultComparator = resultComparator;+ this.#resultComparator = resultComparator;} else if (resultChecker !== undefined) {- this.resultComparator = (resultA, resultB) => this.resultChecker(resultA) && this.resultChecker(resultB);+ this.#resultComparator = (resultA, resultB) => this.#resultChecker(resultA) && this.#resultChecker(resultB);} else {- this.resultComparator = (resultA, resultB) => {+ this.#resultComparator = (resultA, resultB) => {return (resultA.datasetItems.length === resultB.datasetItems.length &&resultA.datasetItems.every((itemA, i) => {@@ -455,22 +453,22 @@ export class AdaptivePlaywrightCrawler<extendContext,});- this.teardownHooks.push(browserCrawler.teardown.bind(browserCrawler));+ this.#teardownHooks.push(browserCrawler.teardown.bind(browserCrawler));- this.staticContextPipeline = staticCrawler.contextPipeline.compose({+ this.#staticContextPipeline = staticCrawler.contextPipeline.compose({action: this.adaptCheerioContext.bind(this),}) as unknown as ContextPipeline<CrawlingContext, ExtendedContext>;- this.browserContextPipeline = browserCrawler.contextPipeline.compose({+ this.#browserContextPipeline = browserCrawler.contextPipeline.compose({action: this.adaptPlaywrightContext.bind(this),}) as unknown as ContextPipeline<CrawlingContext, ExtendedContext>;}- protected override async _init(): Promise<void> {+ protected override async init(): Promise<void> {// Only the predictor we built ourselves is ours to initialize - an injected one is borrowed, so its// lifecycle (including restoring persisted state) stays with whoever created it.- await this.renderingTypePredictor.ifOwned((predictor) => predictor.initialize());- return await super._init();+ await this.#renderingTypePredictor.ifOwned((predictor) => predictor.initialize());+ return await super.init();}protected override buildContextPipeline() {@@ -590,7 +588,7 @@ export class AdaptivePlaywrightCrawler<transactions: StorageTransaction[],): Promise<Result<StorageTransaction>> {const transaction = createStorageTransaction({- policy: this.attemptWritePolicy,+ policy: this.#attemptWritePolicy,commitTimeoutMillis: this.internalTimeoutMillis,});transactions.push(transaction);@@ -619,9 +617,9 @@ export class AdaptivePlaywrightCrawler<try {const callAdaptiveRequestHandler = async () => {if (renderingType === 'static') {- await this.staticContextPipeline.call(subCrawlerContext, this.requestHandler.bind(this));+ await this.#staticContextPipeline.call(subCrawlerContext, this.requestHandler.bind(this));} else if (renderingType === 'clientOnly') {- await this.browserContextPipeline.call(subCrawlerContext, this.requestHandler.bind(this));+ await this.#browserContextPipeline.call(subCrawlerContext, this.requestHandler.bind(this));}};@@ -631,7 +629,7 @@ export class AdaptivePlaywrightCrawler<context.request.label,);const timeoutMillis =- routeTimeoutSecs === undefined ? this.individualRequestHandlerTimeoutMillis : routeTimeoutSecs * 1000;+ routeTimeoutSecs === undefined ? this.#individualRequestHandlerTimeoutMillis : routeTimeoutSecs * 1000;await addTimeoutToPromise(async () => transaction.run(callAdaptiveRequestHandler),@@ -648,7 +646,7 @@ export class AdaptivePlaywrightCrawler<}protected override async runRequestHandler(crawlingContext: CrawlingContext): Promise<void> {- const renderingTypePrediction = this.renderingTypePredictor.value.predict(crawlingContext.request);+ const renderingTypePrediction = this.#renderingTypePredictor.value.predict(crawlingContext.request);const shouldDetectRenderingType = Math.random() < renderingTypePrediction.detectionProbabilityRecommendation;if (!shouldDetectRenderingType) {@@ -657,10 +655,6 @@ export class AdaptivePlaywrightCrawler<);}- if (shouldDetectRenderingType) {- this.inFlightRenderingTypeDetections += 1;- }-// Every transaction created for this request - up to two, since the static-then-browser// fall-through and the browser-then-detection pair are mutually exclusive. Disposed in the// `finally` below, not earlier: the comparators read the journals after `crawlOne` returns.@@ -678,7 +672,7 @@ export class AdaptivePlaywrightCrawler<transactions,);- if (plainHTTPRun.ok && this.resultChecker(plainHTTPRun.result)) {+ if (plainHTTPRun.ok && this.#resultChecker(plainHTTPRun.result)) {crawlingContext.log.debug(`HTTP-only request handler succeeded for ${crawlingContext.request.url}`);plainHTTPRun.logs?.forEach(([log, method, ...args]) => log[method](...(args as [any, any])));await plainHTTPRun.result.commit();@@ -692,7 +686,7 @@ export class AdaptivePlaywrightCrawler<? (plainHTTPRun.error.cause as Error): (plainHTTPRun.error as Error);- if (await this.shouldPropagateError(actualError, crawlingContext as any)) {+ if (await this.#shouldPropagateError(actualError, crawlingContext as any)) {throw actualError;}@@ -764,7 +758,7 @@ export class AdaptivePlaywrightCrawler<return 'clientOnly';}- const comparisonResult = this.resultComparator(plainHTTPRun.result, browserRun.result);+ const comparisonResult = this.#resultComparator(plainHTTPRun.result, browserRun.result);if (comparisonResult === true || comparisonResult === 'equal') {return 'static';}@@ -781,14 +775,10 @@ export class AdaptivePlaywrightCrawler<);if (detectionResult !== undefined) {- this.renderingTypePredictor.value.storeResult(crawlingContext.request, detectionResult);+ this.#renderingTypePredictor.value.storeResult(crawlingContext.request, detectionResult);}}} finally {- if (shouldDetectRenderingType) {- this.inFlightRenderingTypeDetections -= 1;- }-// A still-open transaction here belongs to a discarded attempt - roll it back, then release.for (const transaction of transactions) {transaction.rollback();@@ -828,7 +818,7 @@ export class AdaptivePlaywrightCrawler<override async teardown() {await super.teardown();- for (const hook of this.teardownHooks) {+ for (const hook of this.#teardownHooks) {await hook();}}diff --git a/packages/playwright-crawler/src/internals/playwright-crawler.ts b/packages/playwright-crawler/src/internals/playwright-crawler.tsindex 24aafe029..cacc5bad2 100644--- a/packages/playwright-crawler/src/internals/playwright-crawler.ts+++ b/packages/playwright-crawler/src/internals/playwright-crawler.ts@@ -263,7 +263,7 @@ export class PlaywrightCrawler<return super.buildContextPipeline().compose({ action: this.enhanceContext.bind(this) });}- protected override async _navigationHandler(+ protected override async navigationHandler(crawlingContext: PlaywrightCrawlingContext,gotoOptions: DirectNavigationOptions,) {diff --git a/packages/playwright-crawler/src/internals/utils/rendering-type-prediction.ts b/packages/playwright-crawler/src/internals/utils/rendering-type-prediction.tsindex a7b4c4e5e..e385ef1ce 100644--- a/packages/playwright-crawler/src/internals/utils/rendering-type-prediction.ts+++ b/packages/playwright-crawler/src/internals/utils/rendering-type-prediction.ts@@ -69,14 +69,15 @@ export interface IRenderingTypePredictor {* @experimental*/export class RenderingTypePredictor implements IRenderingTypePredictor {- private detectionRatio: number;+ #detectionRatio: number;+ // kept as TS-private: tests reach for it at runtimeprivate state: RecoverableState<{logreg: LogisticRegression;detectionResults: Map<RenderingType, Map<string | undefined, URLComponents[]>>;}>;constructor({ detectionRatio, persistenceOptions }: RenderingTypePredictorOptions) {- this.detectionRatio = detectionRatio;+ this.#detectionRatio = detectionRatio;this.state = new RecoverableState({defaultState: {logreg: new LogisticRegression({ numSteps: 1000, learningRate: 0.05 }),@@ -144,7 +145,7 @@ export class RenderingTypePredictor implements IRenderingTypePredictor {detectionProbabilityRecommendation:Math.abs(scores[0] - scores[1]) < 0.1? 1- : this.detectionRatio * Math.max(1, 5 - this.resultCount(label)),+ : this.#detectionRatio * Math.max(1, 5 - this.resultCount(label)),};}diff --git a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.tsindex c7165ba11..45a898c58 100644--- a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts+++ b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts@@ -302,7 +302,7 @@ export class PuppeteerCrawler<};}- protected override async _navigationHandler(+ protected override async navigationHandler(crawlingContext: PuppeteerCrawlingContext,gotoOptions: DirectNavigationOptions,) {diff --git a/packages/stagehand-crawler/src/internals/stagehand-controller.ts b/packages/stagehand-crawler/src/internals/stagehand-controller.tsindex 8e75206ae..a934addf6 100644--- a/packages/stagehand-crawler/src/internals/stagehand-controller.ts+++ b/packages/stagehand-crawler/src/internals/stagehand-controller.ts@@ -21,25 +21,25 @@ import type { StagehandPlugin } from './stagehand-plugin';* @ignore*/export class StagehandController extends BrowserController<BrowserType, LaunchOptions, PlaywrightBrowser> {- private stagehand: Stagehand | null = null;- private readonly stagehandInstances: WeakMap<PlaywrightBrowser, Stagehand>;+ #stagehand: Stagehand | null = null;+ readonly #stagehandInstances: WeakMap<PlaywrightBrowser, Stagehand>;constructor(browserPlugin: StagehandPlugin, stagehandInstances: WeakMap<PlaywrightBrowser, Stagehand>) {super(browserPlugin);- this.stagehandInstances = stagehandInstances;+ this.#stagehandInstances = stagehandInstances;}/*** Gets the Stagehand instance associated with this controller's browser.*/getStagehand(): Stagehand {- if (!this.stagehand) {- this.stagehand = this.stagehandInstances.get(this.browser)!;- if (!this.stagehand) {+ if (!this.#stagehand) {+ this.#stagehand = this.#stagehandInstances.get(this.browser)!;+ if (!this.#stagehand) {throw new Error('Stagehand instance not found for browser');}}- return this.stagehand;+ return this.#stagehand;}/**diff --git a/packages/stagehand-crawler/src/internals/stagehand-crawler.ts b/packages/stagehand-crawler/src/internals/stagehand-crawler.tsindex 74120644e..7b9117f7f 100644--- a/packages/stagehand-crawler/src/internals/stagehand-crawler.ts+++ b/packages/stagehand-crawler/src/internals/stagehand-crawler.ts@@ -494,7 +494,7 @@ export class StagehandCrawler<* Navigation handler for Stagehand crawler.* Uses standard Playwright navigation.*/- protected override async _navigationHandler(+ protected override async navigationHandler(crawlingContext: StagehandCrawlingContext,gotoOptions: StagehandGotoOptions,): Promise<Response | null> {diff --git a/packages/stagehand-crawler/src/internals/stagehand-launcher.ts b/packages/stagehand-crawler/src/internals/stagehand-launcher.tsindex fbec0d931..efae47f76 100644--- a/packages/stagehand-crawler/src/internals/stagehand-launcher.ts+++ b/packages/stagehand-crawler/src/internals/stagehand-launcher.ts@@ -70,7 +70,7 @@ export class StagehandLauncher extends BrowserLauncher<StagehandPlugin> {stagehandOptions: ow.optional.object,};- private readonly stagehandOptions: StagehandOptions;+ readonly #stagehandOptions: StagehandOptions;/*** All StagehandLauncher parameters are passed via the launchContext object.@@ -104,7 +104,7 @@ export class StagehandLauncher extends BrowserLauncher<StagehandPlugin> {configuration,);- this.stagehandOptions = {+ this.#stagehandOptions = {env: 'LOCAL',model: 'openai/gpt-4.1-mini',...stagehandOptions,@@ -121,7 +121,7 @@ export class StagehandLauncher extends BrowserLauncher<StagehandPlugin> {...this.otherLaunchContextProps,proxyUrl: this.proxyUrl,launchOptions: this.createLaunchOptions(),- stagehandOptions: this.stagehandOptions, // Set AFTER to override any unresolved options+ stagehandOptions: this.#stagehandOptions, // Set AFTER to override any unresolved options});}}diff --git a/packages/stagehand-crawler/src/internals/stagehand-plugin.ts b/packages/stagehand-crawler/src/internals/stagehand-plugin.tsindex a86029c21..f74e40322 100644--- a/packages/stagehand-crawler/src/internals/stagehand-plugin.ts+++ b/packages/stagehand-crawler/src/internals/stagehand-plugin.ts@@ -35,7 +35,7 @@ export interface StagehandPluginOptions extends BrowserPluginOptions<LaunchOptio*/export class StagehandPlugin extends BrowserPlugin<BrowserType, LaunchOptions, PlaywrightBrowser> {readonly stagehandOptions: StagehandOptions;- private readonly stagehandInstances: WeakMap<PlaywrightBrowser, Stagehand> = new WeakMap();+ readonly #stagehandInstances: WeakMap<PlaywrightBrowser, Stagehand> = new WeakMap();constructor(library: BrowserType, options: StagehandPluginOptions = {}) {super(library, options);@@ -104,11 +104,11 @@ export class StagehandPlugin extends BrowserPlugin<BrowserType, LaunchOptions, Pconst browser = await chromium.connectOverCDP(cdpUrl);// Store the Stagehand instance for AI operations- this.stagehandInstances.set(browser, stagehand);+ this.#stagehandInstances.set(browser, stagehand);// Handle browser disconnection - cleanup both Stagehand and anonymized proxybrowser.on('disconnected', async () => {- await this._cleanupStagehand(browser);+ await this.cleanupStagehand(browser);await closeAnonymizedProxy();});@@ -118,7 +118,7 @@ export class StagehandPlugin extends BrowserPlugin<BrowserType, LaunchOptions, Pawait stagehand.close().catch(() => {});await closeAnonymizedProxy();- const augmentedError = this._augmentLaunchError(error, launchContext);+ const augmentedError = this.augmentLaunchError(error, launchContext);serviceLocator.getLogger().error('Stagehand browser launch failed', { message: augmentedError.message });throw augmentedError;}@@ -127,15 +127,15 @@ export class StagehandPlugin extends BrowserPlugin<BrowserType, LaunchOptions, P/*** Cleans up Stagehand instance when browser disconnects.*/- private async _cleanupStagehand(browser: PlaywrightBrowser): Promise<void> {- const stagehand = this.stagehandInstances.get(browser);+ private async cleanupStagehand(browser: PlaywrightBrowser): Promise<void> {+ const stagehand = this.#stagehandInstances.get(browser);if (stagehand) {try {await stagehand.close();} catch {// Ignore cleanup errors}- this.stagehandInstances.delete(browser);+ this.#stagehandInstances.delete(browser);}}@@ -143,13 +143,13 @@ export class StagehandPlugin extends BrowserPlugin<BrowserType, LaunchOptions, P* Creates a controller for the Stagehand browser.*/override createController(): BrowserController<BrowserType, LaunchOptions, PlaywrightBrowser> {- return new StagehandController(this, this.stagehandInstances) as any;+ return new StagehandController(this, this.#stagehandInstances) as any;}/*** Adds proxy configuration to launch options.*/- protected async _addProxyToLaunchOptions(launchContext: LaunchContext<BrowserType>): Promise<void> {+ protected async addProxyToLaunchOptions(launchContext: LaunchContext<BrowserType>): Promise<void> {launchContext.launchOptions ??= {};const { launchOptions, proxyUrl } = launchContext;@@ -168,7 +168,7 @@ export class StagehandPlugin extends BrowserPlugin<BrowserType, LaunchOptions, P/*** Determines if this is a Chromium-based browser.*/- protected _isChromiumBasedBrowser(): boolean {+ protected isChromiumBasedBrowser(): boolean {const name = this.library?.name?.();return name === 'chromium';}@@ -176,7 +176,7 @@ export class StagehandPlugin extends BrowserPlugin<BrowserType, LaunchOptions, P/*** Augments launch errors with helpful context.*/- private _augmentLaunchError(error: unknown, launchContext: LaunchContext<BrowserType>): Error {+ private augmentLaunchError(error: unknown, launchContext: LaunchContext<BrowserType>): Error {const message = error instanceof Error ? error.message : String(error);const model = this.stagehandOptions.model;@@ -205,6 +205,6 @@ export class StagehandPlugin extends BrowserPlugin<BrowserType, LaunchOptions, P* Gets the Stagehand instance for a given browser.*/getStagehandForBrowser(browser: PlaywrightBrowser): Stagehand | undefined {- return this.stagehandInstances.get(browser);+ return this.#stagehandInstances.get(browser);}}diff --git a/packages/utils/src/internals/robots.ts b/packages/utils/src/internals/robots.tsindex e20ff678f..6c5f90470 100644--- a/packages/utils/src/internals/robots.ts+++ b/packages/utils/src/internals/robots.ts@@ -34,12 +34,22 @@ export interface RobotsTxtFileSitemapsOptions {* ```*/export class RobotsTxtFile {+ #url: string;+ #robots: Pick<Robot, 'isAllowed' | 'getSitemaps'>;+ #proxyUrl?: string;+ #logger?: CrawleeLogger;+private constructor(- private url: string,- private robots: Pick<Robot, 'isAllowed' | 'getSitemaps'>,- private proxyUrl?: string,- private logger?: CrawleeLogger,- ) {}+ url: string,+ robots: Pick<Robot, 'isAllowed' | 'getSitemaps'>,+ proxyUrl?: string,+ logger?: CrawleeLogger,+ ) {+ this.#url = url;+ this.#robots = robots;+ this.#proxyUrl = proxyUrl;+ this.#logger = logger;+ }/*** Determine the location of a robots.txt file for a URL and fetch it.@@ -123,7 +133,7 @@ export class RobotsTxtFile {* @param [userAgent] relevant user agent, default to `*`*/isAllowed(url: string, userAgent = '*'): boolean {- return this.robots.isAllowed(url, userAgent) ?? true; // `undefined` means that there is no explicit rule for the requested URL - assume it's allowed+ return this.#robots.isAllowed(url, userAgent) ?? true; // `undefined` means that there is no explicit rule for the requested URL - assume it's allowed}/**@@ -135,11 +145,11 @@ export class RobotsTxtFile {const { enqueueStrategy = 'same-hostname' } = options;const sitemaps: string[] = [];- for (const sitemapUrl of this.robots.getSitemaps()) {+ for (const sitemapUrl of this.#robots.getSitemaps()) {// `filterUrl` tolerates an unparseable origin (returns not-allowed) rather than throwing.- const { allowed, reason } = filterUrl(sitemapUrl, this.url, enqueueStrategy);+ const { allowed, reason } = filterUrl(sitemapUrl, this.#url, enqueueStrategy);if (!allowed) {- this.logger?.warning(`Skipping sitemap ${sitemapUrl} listed in robots.txt at ${this.url}: ${reason}.`);+ this.#logger?.warning(`Skipping sitemap ${sitemapUrl} listed in robots.txt at ${this.#url}: ${reason}.`);continue;}sitemaps.push(sitemapUrl);@@ -153,7 +163,7 @@ export class RobotsTxtFile {* and the sitemap parser.*/async parseSitemaps(options: RobotsTxtFileSitemapsOptions = {}): Promise<Sitemap> {- return Sitemap.load(this.getSitemaps(options), this.proxyUrl, { ...options, logger: this.logger });+ return Sitemap.load(this.getSitemaps(options), this.#proxyUrl, { ...options, logger: this.#logger });}/**diff --git a/packages/utils/src/internals/sitemap.ts b/packages/utils/src/internals/sitemap.tsindex 514f1b180..ffaaf276a 100644--- a/packages/utils/src/internals/sitemap.ts+++ b/packages/utils/src/internals/sitemap.ts@@ -34,28 +34,28 @@ type SitemapSource = ({ type: 'url'; url: string } | { type: 'raw'; content: strtype SitemapItem = ({ type: 'url' } & SitemapUrlData) | { type: 'sitemapUrl'; url: string };class SitemapTxtParser extends Transform {- private decoder: StringDecoder = new StringDecoder('utf8');- private buffer = '';+ #decoder: StringDecoder = new StringDecoder('utf8');+ #buffer = '';constructor() {super({readableObjectMode: true,transform: (chunk, _encoding, callback) => {- this.processBuffer(this.decoder.write(chunk), false);+ this.processBuffer(this.#decoder.write(chunk), false);callback();},flush: (callback) => {- this.processBuffer(this.decoder.end(), true);+ this.processBuffer(this.#decoder.end(), true);callback();},});}private processBuffer(input: string, finalize: boolean): void {- this.buffer += input;+ this.#buffer += input;- if (finalize || this.buffer.includes('\n')) {- const parts = this.buffer+ if (finalize || this.#buffer.includes('\n')) {+ const parts = this.#buffer.split('\n').map((part) => part.trim()).filter((part) => part.length > 0);@@ -65,113 +65,113 @@ class SitemapTxtParser extends Transform {this.push({ type: 'url', loc: url } satisfies SitemapItem);}- this.buffer = '';+ this.#buffer = '';} else if (parts.length > 0) {for (const url of parts.slice(0, -1)) {this.push({ type: 'url', loc: url } satisfies SitemapItem);}- this.buffer = parts.at(-1)!;+ this.#buffer = parts.at(-1)!;}}}}class SitemapXmlParser extends Transform {- private decoder: StringDecoder = new StringDecoder('utf8');- private parser = new sax.SAXParser(true);+ #decoder: StringDecoder = new StringDecoder('utf8');+ #parser = new sax.SAXParser(true);- private rootTagName?: 'sitemapindex' | 'urlset';- private currentTag?: 'loc' | 'lastmod' | 'changefreq' | 'priority' = undefined;- private url: Partial<SitemapUrl> = {};+ #rootTagName?: 'sitemapindex' | 'urlset';+ #currentTag?: 'loc' | 'lastmod' | 'changefreq' | 'priority' = undefined;+ #url: Partial<SitemapUrl> = {};constructor() {super({readableObjectMode: true,transform: (chunk, _encoding, callback) => {- this.parser.write(this.decoder.write(chunk));+ this.#parser.write(this.#decoder.write(chunk));callback();},flush: (callback) => {- const rest = this.decoder.end();+ const rest = this.#decoder.end();if (rest.length > 0) {- this.parser.write(rest);+ this.#parser.write(rest);}- this.parser.end();+ this.#parser.end();callback();},});- this.parser.onopentag = this.onOpenTag.bind(this);- this.parser.onclosetag = this.onCloseTag.bind(this);+ this.#parser.onopentag = this.onOpenTag.bind(this);+ this.#parser.onclosetag = this.onCloseTag.bind(this);- this.parser.ontext = this.onText.bind(this);- this.parser.oncdata = this.onText.bind(this);+ this.#parser.ontext = this.onText.bind(this);+ this.#parser.oncdata = this.onText.bind(this);- this.parser.onerror = this.destroy.bind(this);+ this.#parser.onerror = this.destroy.bind(this);}private onOpenTag(node: sax.Tag | sax.QualifiedTag) {- if (this.rootTagName !== undefined) {+ if (this.#rootTagName !== undefined) {if (node.name === 'loc' ||node.name === 'lastmod' ||node.name === 'priority' ||node.name === 'changefreq') {- this.currentTag = node.name;+ this.#currentTag = node.name;}}if (node.name === 'urlset') {- this.rootTagName = 'urlset';+ this.#rootTagName = 'urlset';}if (node.name === 'sitemapindex') {- this.rootTagName = 'sitemapindex';+ this.#rootTagName = 'sitemapindex';}}private onCloseTag(name: string) {if (name === 'loc' || name === 'lastmod' || name === 'priority' || name === 'changefreq') {- this.currentTag = undefined;+ this.#currentTag = undefined;}if (name === 'url') {- if (this.url.loc !== undefined) {- this.push({ type: 'url', ...this.url, loc: this.url.loc } satisfies SitemapItem);+ if (this.#url.loc !== undefined) {+ this.push({ type: 'url', ...this.#url, loc: this.#url.loc } satisfies SitemapItem);}- this.url = {};+ this.#url = {};}}private onText(text: string) {- if (this.currentTag === 'loc') {- if (this.rootTagName === 'sitemapindex') {+ if (this.#currentTag === 'loc') {+ if (this.#rootTagName === 'sitemapindex') {this.push({ type: 'sitemapUrl', url: text.trim() } satisfies SitemapItem);}- if (this.rootTagName === 'urlset') {- this.url ??= {};- this.url.loc = text.trim();+ if (this.#rootTagName === 'urlset') {+ this.#url ??= {};+ this.#url.loc = text.trim();}}text = text.trim();- if (this.currentTag === 'lastmod') {+ if (this.#currentTag === 'lastmod') {const lastmod = new Date(text);if (!Number.isNaN(lastmod.getTime())) {- this.url.lastmod = lastmod;+ this.#url.lastmod = lastmod;}}- if (this.currentTag === 'priority') {- this.url.priority = Number(text);+ if (this.#currentTag === 'priority') {+ this.#url.priority = Number(text);}- if (this.currentTag === 'changefreq') {+ if (this.#currentTag === 'changefreq') {if (['always', 'hourly', 'daily', 'weekly', 'monthly', 'yearly', 'never'].includes(text)) {- this.url.changefreq = text as SitemapUrl['changefreq'];+ this.#url.changefreq = text as SitemapUrl['changefreq'];}}}diff --git a/test/browser-pool/browser-plugins/plugins.test.ts b/test/browser-pool/browser-plugins/plugins.test.tsindex c6ff407eb..855d59db9 100644--- a/test/browser-pool/browser-plugins/plugins.test.ts+++ b/test/browser-pool/browser-plugins/plugins.test.ts@@ -82,7 +82,7 @@ const runPluginTest = <id,launchOptions,browserPlugin: plugin,- _proxyUrl: proxyUrl.slice(0, -1),+ proxyUrl: proxyUrl.slice(0, -1),one: 1,useIncognitoPages: false,};@@ -93,7 +93,7 @@ const runPluginTest = <expect(context.id).toEqual(desiredObject.id);expect(context.launchOptions).toEqual(desiredObject.launchOptions);expect(context.browserPlugin).toEqual(desiredObject.browserPlugin);- expect(context['_proxyUrl']).toEqual(desiredObject._proxyUrl); // eslint-disable-line+ expect(context.proxyUrl).toEqual(desiredObject.proxyUrl);expect(context.one).toEqual(desiredObject.one);expect(context.useIncognitoPages).toEqual(desiredObject.useIncognitoPages);});@@ -634,8 +634,9 @@ describe('Plugins', () => {const contexts = browser.contexts();expect(contexts).toHaveLength(1);- // Cast to any to access private property- expect(contexts[0]).toEqual((browser as any)._browserContext);+ // the returned context is the one new pages are created in+ const page = await browser.newPage();+ expect(page.context()).toBe(contexts[0]);});test('should return correct connected status', async () => {diff --git a/test/browser-pool/browser-pool.test.ts b/test/browser-pool/browser-pool.test.tsindex 4fcd1e644..ec4485e44 100644--- a/test/browser-pool/browser-pool.test.ts+++ b/test/browser-pool/browser-pool.test.ts@@ -162,7 +162,7 @@ describe.each([const timeout = browserPool.operationTimeoutMillis;browserPool.operationTimeoutMillis = 500;// @ts-expect-error mocking private method- const spy = vitest.spyOn(BrowserPool.prototype, '_executeHooks');+ const spy = vitest.spyOn(BrowserPool.prototype, 'executeHooks');await browserPool.newPage();expect(spy).toBeCalledTimes(4);@@ -209,11 +209,11 @@ describe.each([test('should correctly override page close', async () => {// @ts-expect-error Private function- vitest.spyOn(browserPool!, '_overridePageClose');+ vitest.spyOn(browserPool!, 'overridePageClose');const page = await browserPool.newPage();- expect(browserPool['_overridePageClose']).toBeCalled();+ expect(browserPool['overridePageClose']).toBeCalled();const controller = browserPool.getBrowserControllerByPage(page)!;@@ -245,7 +245,7 @@ describe.each([test('should allow max pages per browser', async () => {browserPool.maxOpenPagesPerBrowser = 1;// @ts-expect-error Private function- vitest.spyOn(browserPool!, '_launchBrowser');+ vitest.spyOn(browserPool!, 'launchBrowser');await browserPool.newPage();expect(browserPool.activeBrowserControllers.size).toBe(1);@@ -254,13 +254,13 @@ describe.each([await browserPool.newPage();expect(browserPool.activeBrowserControllers.size).toBe(3);- expect(browserPool['_launchBrowser']).toBeCalledTimes(3);+ expect(browserPool['launchBrowser']).toBeCalledTimes(3);});test('should allow max pages per browser - no race condition', async () => {browserPool.maxOpenPagesPerBrowser = 1;// @ts-expect-error Private function- vitest.spyOn(browserPool, '_launchBrowser');+ vitest.spyOn(browserPool, 'launchBrowser');const usePlugin = {browserPlugin: plugin,@@ -270,7 +270,7 @@ describe.each([expect(browserPool.activeBrowserControllers.size).toBe(2);- expect(browserPool['_launchBrowser']).toBeCalledTimes(2);+ expect(browserPool['launchBrowser']).toBeCalledTimes(2);});test('should close retired browsers', async () => {@@ -279,12 +279,12 @@ describe.each([clearInterval(browserPool['browserKillerInterval']!);browserPool['browserKillerInterval'] = setInterval(- async () => browserPool['_closeInactiveRetiredBrowsers'](),+ async () => browserPool['closeInactiveRetiredBrowsers'](),100,);// @ts-expect-error Private function- vitest.spyOn(browserPool!, '_closeRetiredBrowserWithNoPages');+ vitest.spyOn(browserPool!, 'closeRetiredBrowserWithNoPages');expect(browserPool.retiredBrowserControllers.size).toBe(0);const page = await browserPool.newPage();@@ -300,7 +300,7 @@ describe.each([}, 1000),);- expect(browserPool['_closeRetiredBrowserWithNoPages']).toHaveBeenCalled();+ expect(browserPool['closeRetiredBrowserWithNoPages']).toHaveBeenCalled();expect(controller.close).toHaveBeenCalled();expect(browserPool.retiredBrowserControllers.size).toBe(0);});@@ -318,7 +318,7 @@ describe.each([hooks[i] = createAsyncHookReturningIndex(i);}- await browserPool['_executeHooks'](hooks);+ await browserPool['executeHooks'](hooks);expect(indexArray).toHaveLength(10);indexArray.forEach((v, index) => expect(v).toEqual(index));});@@ -367,12 +367,12 @@ describe.each([browserPool.preLaunchHooks.push(myAsyncHook);// @ts-expect-error Private function- vitest.spyOn(browserPool!, '_executeHooks');+ vitest.spyOn(browserPool!, 'executeHooks');const page = await browserPool.newPage();const pageId = browserPool.getPageId(page)!;const { launchContext } = browserPool.getBrowserControllerByPage(page)!;- expect(browserPool['_executeHooks']).toHaveBeenNthCalledWith(+ expect(browserPool['executeHooks']).toHaveBeenNthCalledWith(1,browserPool.preLaunchHooks,pageId,@@ -411,13 +411,13 @@ describe.each([browserPool.postLaunchHooks = [myAsyncHook];// @ts-expect-error Private function- vitest.spyOn(browserPool, '_executeHooks');+ vitest.spyOn(browserPool, 'executeHooks');const page = await browserPool.newPage();const pageId = browserPool.getPageId(page)!;const browserController = browserPool.getBrowserControllerByPage(page)!;- expect(browserPool['_executeHooks']).toHaveBeenNthCalledWith(+ expect(browserPool['executeHooks']).toHaveBeenNthCalledWith(2,browserPool.postLaunchHooks,pageId,@@ -470,13 +470,13 @@ describe.each([browserPool.prePageCreateHooks = [myAsyncHook];// @ts-expect-error Private function- vitest.spyOn(browserPool, '_executeHooks');+ vitest.spyOn(browserPool, 'executeHooks');const page = await browserPool.newPage();const pageId = browserPool.getPageId(page)!;const browserController = browserPool.getBrowserControllerByPage(page)!;- expect(browserPool['_executeHooks']).toHaveBeenNthCalledWith(+ expect(browserPool['executeHooks']).toHaveBeenNthCalledWith(3,browserPool.prePageCreateHooks,pageId,@@ -492,12 +492,12 @@ describe.each([browserPool.postPageCreateHooks = [myAsyncHook];// @ts-expect-error Private function- vitest.spyOn(browserPool, '_executeHooks');+ vitest.spyOn(browserPool, 'executeHooks');const page = await browserPool.newPage();const browserController = browserPool.getBrowserControllerByPage(page);- expect(browserPool['_executeHooks']).toHaveBeenNthCalledWith(+ expect(browserPool['executeHooks']).toHaveBeenNthCalledWith(4,browserPool.postPageCreateHooks,page,@@ -512,13 +512,13 @@ describe.each([browserPool.prePageCloseHooks = [myAsyncHook];// @ts-expect-error Private function- vitest.spyOn(browserPool, '_executeHooks');+ vitest.spyOn(browserPool, 'executeHooks');const page = await browserPool.newPage();await page.close();const browserController = browserPool.getBrowserControllerByPage(page);- expect(browserPool['_executeHooks']).toHaveBeenNthCalledWith(+ expect(browserPool['executeHooks']).toHaveBeenNthCalledWith(5,browserPool.prePageCloseHooks,page,@@ -533,14 +533,14 @@ describe.each([browserPool.postPageCloseHooks = [myAsyncHook];// @ts-expect-error Private function- vitest.spyOn(browserPool, '_executeHooks');+ vitest.spyOn(browserPool, 'executeHooks');const page = await browserPool.newPage();const pageId = browserPool.getPageId(page);await page.close();const browserController = browserPool.getBrowserControllerByPage(page);- expect(browserPool['_executeHooks']).toHaveBeenNthCalledWith(+ expect(browserPool['executeHooks']).toHaveBeenNthCalledWith(6,browserPool.postPageCloseHooks,pageId,diff --git a/test/browser-pool/launch-context.test.ts b/test/browser-pool/launch-context.test.tsnew file mode 100644index 000000000..49965037d--- /dev/null+++ b/test/browser-pool/launch-context.test.ts@@ -0,0 +1,47 @@+import { LaunchContext } from '@crawlee/browser-pool';++const createLaunchContext = () =>+ new LaunchContext({ id: 'id', browserPlugin: {} as never, launchOptions: {} as never });++describe('LaunchContext', () => {+ test('extend() attaches custom fields', () => {+ const context = createLaunchContext();+ context.extend({ one: 1, two: 'two' });++ expect(context.one).toBe(1);+ expect(context.two).toBe('two');+ });++ test('extend() rejects every declared field as reserved', () => {+ const context = createLaunchContext();++ // Includes `fingerprint`, `proxyUrl` and `remoteToken`, which slipped through the check in v3.+ const reservedKeys = [+ 'id',+ 'browserPlugin',+ 'launchOptions',+ 'fingerprint',+ 'proxyUrl',+ 'remoteToken',+ 'extend',+ ];++ for (const key of reservedKeys) {+ expect(() => context.extend({ [key]: 'anything' })).toThrow(+ `Cannot extend LaunchContext with key: ${key}, because it's reserved.`,+ );+ }+ });++ test('declared fields are set directly instead', () => {+ const context = createLaunchContext();++ const fingerprint = { fingerprint: {}, headers: {} } as never;+ context.fingerprint = fingerprint;+ expect(context.fingerprint).toBe(fingerprint);++ context.proxyUrl = 'http://proxy.com:1111/';+ // The setter normalizes the URL (no trailing slash).+ expect(context.proxyUrl).toBe('http://proxy.com:1111');+ });+});diff --git a/test/core/autoscaling/autoscaled_pool.test.ts b/test/core/autoscaling/autoscaled_pool.test.tsindex ab7812403..da5bfaac4 100644--- a/test/core/autoscaling/autoscaled_pool.test.ts+++ b/test/core/autoscaling/autoscaled_pool.test.ts@@ -198,29 +198,29 @@ describe('AutoscaledPool', () => {test('works with low values', () => {// @ts-expect-error Calling private method on the governor- pool.system._autoscale(cb);+ pool.system.autoscale(cb);expect(pool.desiredConcurrency).toBe(2);// @ts-expect-error Calling private method on the governor- pool.system._autoscale(cb);+ pool.system.autoscale(cb);expect(pool.desiredConcurrency).toBe(2); // because currentConcurrency is not high enough;// @ts-expect-error Overwriting readonly private proppool.system._currentConcurrency = 2;// @ts-expect-error Calling private method on the governor- pool.system._autoscale(cb);+ pool.system.autoscale(cb);expect(pool.desiredConcurrency).toBe(3);systemStatus.okNow = false; // this should have no effect// @ts-expect-error Overwriting readonly private proppool.system._currentConcurrency = 3;// @ts-expect-error Calling private method on the governor- pool.system._autoscale(cb);+ pool.system.autoscale(cb);expect(pool.desiredConcurrency).toBe(4);systemStatus.okLately = false;// @ts-expect-error Calling private method on the governor- pool.system._autoscale(cb);+ pool.system.autoscale(cb);expect(pool.desiredConcurrency).toBe(3);});@@ -233,7 +233,7 @@ describe('AutoscaledPool', () => {Math.floor(pool.desiredConcurrency * pool.system.desiredConcurrencyRatio) - 1;systemStatus.okLately = true;// @ts-expect-error Calling private method on the governor- pool.system._autoscale(cb);+ pool.system.autoscale(cb);expect(pool.desiredConcurrency).toBe(50);// Should scale because we bumped up current concurrency.@@ -245,7 +245,7 @@ describe('AutoscaledPool', () => {// @ts-expect-error Accessing private prop on the governorpool.desiredConcurrency + Math.ceil(pool.desiredConcurrency * pool.system.scaleUpStepRatio);// @ts-expect-error Calling private method on the governor- pool.system._autoscale(cb);+ pool.system.autoscale(cb);expect(pool.desiredConcurrency).toEqual(newConcurrency);// Should scale down.@@ -254,7 +254,7 @@ describe('AutoscaledPool', () => {// @ts-expect-error Accessing private prop on the governorpool.desiredConcurrency - Math.ceil(pool.desiredConcurrency * pool.system.scaleDownStepRatio);// @ts-expect-error Calling private method on the governor- pool.system._autoscale(cb);+ pool.system.autoscale(cb);expect(pool.desiredConcurrency).toEqual(newConcurrency);});@@ -262,17 +262,22 @@ describe('AutoscaledPool', () => {let limit = 5;let concurrencyLog: number[] = [];let count = 0;- // @ts-expect-error Overwriting readonly private prop on the governor- pool.system.systemStatus.okNow = false;- // @ts-expect-error Overwriting readonly private prop- pool.runTaskFunction = async () => {- await sleep(10);- count++;- };- // @ts-expect-error Overwriting readonly private prop- pool.isFinishedFunction = async () => count >= limit;- // @ts-expect-error Overwriting readonly private prop- pool.isTaskReadyFunction = async () => count < limit;++ // The task loop is configuration, so this test builds its own pool instead of reusing the shared one.+ pool = await makePool(+ {+ runTaskFunction: async () => {+ await sleep(10);+ count++;+ },+ isFinishedFunction: async () => count >= limit,+ isTaskReadyFunction: async () => count < limit,+ },+ { minConcurrency: 1, maxConcurrency: 100 },+ );+ // @ts-expect-error Mock+ pool.system.systemStatus = systemStatus;+ systemStatus.okNow = false;systemOf(pool).desiredConcurrency = 10;// Spy on the governor's concurrency accounting - that is where per-task current concurrency now lives.@@ -505,13 +510,12 @@ describe('AutoscaledPool', () => {}),isFinishedFunction: isFinished,isTaskReadyFunction: async () => !(await isFinished()),+ // Speed up the test.+ maybeRunIntervalSecs: 0.005,},{ minConcurrency: 3 },);- // @ts-expect-error Overwriting readonly private prop- pool.maybeRunIntervalMillis = 5;-await pool.run();await sleep(10);expect(count).toBeGreaterThanOrEqual(11);@@ -544,11 +548,11 @@ describe('AutoscaledPool', () => {},isFinishedFunction: async () => isFinished,isTaskReadyFunction: async () => !isFinished && isTaskReady,+ // Speed up the test.+ maybeRunIntervalSecs: 0.001,},{ maxConcurrency: 1 },);- // @ts-expect-error Overwriting readonly private prop- pool.maybeRunIntervalMillis = 1;await pool.run();// Check finished tasks.@@ -566,7 +570,7 @@ describe('AutoscaledPool', () => {{ minConcurrency: 1, maxConcurrency: 100, loggingIntervalSecs: null },);// @ts-expect-error Calling private method on the governor- pool.system._autoscale(() => {});+ pool.system.autoscale(() => {});expect(pool.desiredConcurrency).toBe(2);});diff --git a/test/core/autoscaling/runtime.test.ts b/test/core/autoscaling/runtime.test.tsindex bc5269dff..2e5667e47 100644--- a/test/core/autoscaling/runtime.test.ts+++ b/test/core/autoscaling/runtime.test.ts@@ -115,7 +115,7 @@ describe('isLambda()', () => {});describe('getCgroupsVersion()', () => {- // Reset the module cache so that _cgroupsVersion is not retained across tests.+ // Reset the module cache so that cgroupsVersion is not retained across tests.beforeEach(() => {vitest.resetModules();});diff --git a/test/core/autoscaling/snapshotter.test.ts b/test/core/autoscaling/snapshotter.test.tsindex 35d1ac2f7..d74be4dfb 100644--- a/test/core/autoscaling/snapshotter.test.ts+++ b/test/core/autoscaling/snapshotter.test.ts@@ -3,6 +3,7 @@ import os from 'node:os';import {ClientLoadSignal,Configuration,+ type EventLoopLoadSignal,EventType,LocalEventManager,MemoryLoadSignal,@@ -25,11 +26,11 @@ const noop = () => {};const START_CONTEXT = { maxSampleWindowMillis: 30_000 };/** Reads a signal's sample by name, the same way production (`SystemStatus`) does. */+const signalOf = (snapshotter: Snapshotter, name: string) =>+ snapshotter.getLoadSignals().find((signal) => signal.name === name)!;+const sampleOf = (snapshotter: Snapshotter, name: string, sampleDurationMillis?: number): any[] =>- snapshotter- .getLoadSignals()- .find((signal) => signal.name === name)!- .getSample(sampleDurationMillis);+ signalOf(snapshotter, name).getSample(sampleDurationMillis);describe('Snapshotter', () => {let logLevel: number;@@ -205,8 +206,7 @@ describe('Snapshotter', () => {const clock = vitest.useFakeTimers();try {const snapshotter = new Snapshotter({ eventLoop: { maxBlockedMillis: 5, snapshotIntervalSecs: 0 } });- // @ts-expect-error Accessing private property- const eventLoopSignal = snapshotter.eventLoopSignal!;+ const eventLoopSignal = signalOf(snapshotter, 'eventLoopInfo') as EventLoopLoadSignal;eventLoopSignal.handle(noop);clock.advanceTimersByTime(1);eventLoopSignal.handle(noop);diff --git a/test/core/autoscaling/system_status.test.ts b/test/core/autoscaling/system_status.test.tsindex c6e67cecb..626fddf81 100644--- a/test/core/autoscaling/system_status.test.ts+++ b/test/core/autoscaling/system_status.test.ts@@ -184,45 +184,34 @@ describe('SystemStatus', () => {test('should show different values for now and lately', () => {const ratios = { memInfo: 0.5, eventLoopInfo: 0.5, cpuInfo: 0.5, clientInfo: 0.5 };+ // The "now" window is configuration, so each width gets its own instance rather than a mutated one.+ const statusWithCurrentWindow = (snaps: any[], currentHistoryMillis: number) =>+ new SystemStatus({+ snapshotter: new MockSnapshotter(snaps, snaps, snaps, snaps, ratios) as any,+ currentHistorySecs: currentHistoryMillis / 1000,+ });+let snaps = generateSnapsSync(95, false);- let systemStatus = new SystemStatus({- snapshotter: new MockSnapshotter(snaps, snaps, snaps, snaps, ratios) as any,- });- // @ts-expect-error Overwriting readonly private prop- systemStatus.currentHistoryMillis = 5;- expect(systemStatus.getCurrentStatus().isSystemIdle).toBe(false);- expect(systemStatus.getHistoricalStatus().isSystemIdle).toBe(true);+ expect(statusWithCurrentWindow(snaps, 5).getCurrentStatus().isSystemIdle).toBe(false);+ expect(statusWithCurrentWindow(snaps, 5).getHistoricalStatus().isSystemIdle).toBe(true);- // @ts-expect-error Overwriting readonly private prop- systemStatus.currentHistoryMillis = 10;- expect(systemStatus.getCurrentStatus().isSystemIdle).toBe(false);- expect(systemStatus.getHistoricalStatus().isSystemIdle).toBe(true);+ expect(statusWithCurrentWindow(snaps, 10).getCurrentStatus().isSystemIdle).toBe(false);+ expect(statusWithCurrentWindow(snaps, 10).getHistoricalStatus().isSystemIdle).toBe(true);- // @ts-expect-error Overwriting readonly private prop- systemStatus.currentHistoryMillis = 12;- expect(systemStatus.getCurrentStatus().isSystemIdle).toBe(true);- expect(systemStatus.getHistoricalStatus().isSystemIdle).toBe(true);+ expect(statusWithCurrentWindow(snaps, 12).getCurrentStatus().isSystemIdle).toBe(true);+ expect(statusWithCurrentWindow(snaps, 12).getHistoricalStatus().isSystemIdle).toBe(true);snaps = generateSnapsSync(95, true);- systemStatus = new SystemStatus({- snapshotter: new MockSnapshotter(snaps, snaps, snaps, snaps, ratios) as any,- });- // @ts-expect-error Overwriting readonly private prop- systemStatus.currentHistoryMillis = 5;- expect(systemStatus.getCurrentStatus().isSystemIdle).toBe(true);- expect(systemStatus.getHistoricalStatus().isSystemIdle).toBe(false);+ expect(statusWithCurrentWindow(snaps, 5).getCurrentStatus().isSystemIdle).toBe(true);+ expect(statusWithCurrentWindow(snaps, 5).getHistoricalStatus().isSystemIdle).toBe(false);- // @ts-expect-error Overwriting readonly private prop- systemStatus.currentHistoryMillis = 10;- expect(systemStatus.getCurrentStatus().isSystemIdle).toBe(true);- expect(systemStatus.getHistoricalStatus().isSystemIdle).toBe(false);+ expect(statusWithCurrentWindow(snaps, 10).getCurrentStatus().isSystemIdle).toBe(true);+ expect(statusWithCurrentWindow(snaps, 10).getHistoricalStatus().isSystemIdle).toBe(false);- // @ts-expect-error Overwriting readonly private prop- systemStatus.currentHistoryMillis = 12;- expect(systemStatus.getCurrentStatus().isSystemIdle).toBe(false);- expect(systemStatus.getHistoricalStatus().isSystemIdle).toBe(false);+ expect(statusWithCurrentWindow(snaps, 12).getCurrentStatus().isSystemIdle).toBe(false);+ expect(statusWithCurrentWindow(snaps, 12).getHistoricalStatus().isSystemIdle).toBe(false);});test('the historical window is requested explicitly, so a long-memory custom signal cannot widen it', () => {diff --git a/test/core/crawlers/basic_browser_crawler.ts b/test/core/crawlers/basic_browser_crawler.tsindex 6b0ff9778..0dc756f41 100644--- a/test/core/crawlers/basic_browser_crawler.ts+++ b/test/core/crawlers/basic_browser_crawler.ts@@ -26,7 +26,7 @@ export class BrowserCrawlerTest extends BrowserCrawler<});}- protected async _navigationHandler(+ protected async navigationHandler(ctx: PuppeteerCrawlingContext,gotoOptions: PuppeteerGoToOptions,): Promise<HTTPResponse | null | undefined> {diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.tsindex 3f98cc951..73227eb88 100644--- a/test/core/crawlers/basic_crawler.test.ts+++ b/test/core/crawlers/basic_crawler.test.ts@@ -330,13 +330,13 @@ describe('BasicCrawler', () => {});const failure = new Error('Could not open the request queue');- // `_init()` starts the concurrency system before it resolves the request manager, so this fails after the+ // `init()` starts the concurrency system before it resolves the request manager, so this fails after the// system's intervals are already ticking.const getRequestManager = vitest.spyOn(crawler, 'getRequestManager').mockImplementation(async () => Promise.reject(failure));- // No initial requests — `addRequests()` would resolve the request manager before `_init()` even runs.+ // No initial requests — `addRequests()` would resolve the request manager before `init()` even runs.await expect(crawler.run()).rejects.toThrow(failure);// The intervals would otherwise keep the event loop alive for the rest of the process's life.@@ -788,8 +788,7 @@ describe('BasicCrawler', () => {expect(setValueSpy).toBeCalled();// clean up- // @ts-expect-error Accessing a private property and its private method- await basicCrawler.autoscaledPool!.destroy();+ await basicCrawler.teardown();},);diff --git a/test/core/crawlers/browser_crawler.test.ts b/test/core/crawlers/browser_crawler.test.tsindex 02e8cce0f..9aa3460ca 100644--- a/test/core/crawlers/browser_crawler.test.ts+++ b/test/core/crawlers/browser_crawler.test.ts@@ -217,7 +217,7 @@ describe('BrowserCrawler', () => {let markBadCalled = false;let sessionGoto!: ISession;const browserCrawler = new (class extends BrowserCrawlerTest {- protected override async _navigationHandler(+ protected override async navigationHandler(ctx: TestCrawlingContext,): Promise<HTTPResponse | null | undefined> {sessionGoto = ctx.session!;@@ -408,7 +408,7 @@ describe('BrowserCrawler', () => {const pageClosedStates: boolean[] = [];const browserCrawler = new (class extends BrowserCrawlerTest {- protected override async _navigationHandler(): Promise<HTTPResponse | null | undefined> {+ protected override async navigationHandler(): Promise<HTTPResponse | null | undefined> {throw new Error('net::ERR_NAME_NOT_RESOLVED');}})({@@ -482,7 +482,7 @@ describe('BrowserCrawler', () => {});let optionsGoto: PuppeteerGoToOptions;const browserCrawler = new (class extends BrowserCrawlerTest {- protected override async _navigationHandler(+ protected override async navigationHandler(ctx: TestCrawlingContext,gotoOptions: PuppeteerGoToOptions,): Promise<HTTPResponse | null | undefined> {@@ -751,7 +751,7 @@ describe('BrowserCrawler', () => {});// @ts-expect-error Overriding protected method- crawler._navigationHandler = async ({ request }) => {+ crawler.navigationHandler = async ({ request }) => {return { status: () => request.userData.statusCode };};@@ -826,7 +826,7 @@ describe('BrowserCrawler', () => {});// @ts-expect-error Overriding protected method- crawler._navigationHandler = async ({ request }) => {+ crawler.navigationHandler = async ({ request }) => {return { status: () => request.userData.statusCode };};@@ -868,7 +868,7 @@ describe('BrowserCrawler', () => {});// @ts-expect-error Overriding protected method- crawler._navigationHandler = async ({ request }) => {+ crawler.navigationHandler = async ({ request }) => {return { status: () => request.userData.statusCode };};@@ -987,9 +987,8 @@ describe('BrowserCrawler', () => {],});- const proxyConfiguration = new ProxyConfiguration({- proxyUrls: ['http://proxy.com:1111', 'http://proxy.com:2222', 'http://proxy.com:3333'],- });+ const proxyUrls = ['http://proxy.com:1111', 'http://proxy.com:2222', 'http://proxy.com:3333'];+ const proxyConfiguration = new ProxyConfiguration({ proxyUrls });const browserProxies: string[] = [];@@ -1012,10 +1011,8 @@ describe('BrowserCrawler', () => {await browserCrawler.run();- // @ts-expect-error Accessing private property- const proxiesToUse = proxyConfiguration.proxyUrls!;- for (const proxyUrl of proxiesToUse) {- expect(browserProxies.includes(new URL(proxyUrl!).href.slice(0, -1))).toBeTruthy();+ for (const proxyUrl of proxyUrls) {+ expect(browserProxies.includes(new URL(proxyUrl).href.slice(0, -1))).toBeTruthy();}delete process.env[ENV_VARS.PROXY_PASSWORD];@@ -1039,7 +1036,7 @@ describe('BrowserCrawler', () => {const requestHandler = vitest.fn();const browserCrawler = new (class extends BrowserCrawlerTest {- protected override async _navigationHandler(+ protected override async navigationHandler(ctx: TestCrawlingContext,): Promise<HTTPResponse | null | undefined> {const proxyInfo = ctx.session?.proxyInfo;@@ -1090,7 +1087,7 @@ describe('BrowserCrawler', () => {*/let numberOfRotations = -(await requestList!.getTotalCount());const browserCrawler = new (class extends BrowserCrawlerTest {- protected override async _navigationHandler(+ protected override async navigationHandler(ctx: TestCrawlingContext,): Promise<HTTPResponse | null | undefined> {const proxyInfo = ctx.session?.proxyInfo;@@ -1138,7 +1135,7 @@ describe('BrowserCrawler', () => {'Proxy responded with 400 - Bad request. Also, this error message contains some useful payload.';const crawler = new (class extends BrowserCrawlerTest {- protected override async _navigationHandler(+ protected override async navigationHandler(ctx: TestCrawlingContext,): Promise<HTTPResponse | null | undefined> {const proxyInfo = ctx.session?.proxyInfo;@@ -1224,7 +1221,7 @@ describe('BrowserCrawler', () => {failedRequestHandler,});// @ts-expect-error Overriding protected method- browserCrawler._navigationHandler = gotoFunction;+ browserCrawler.navigationHandler = gotoFunction;await browserCrawler.run();} finally {diff --git a/test/core/crawlers/cheerio_crawler.test.ts b/test/core/crawlers/cheerio_crawler.test.tsindex 13c21976e..aaa733282 100644--- a/test/core/crawlers/cheerio_crawler.test.ts+++ b/test/core/crawlers/cheerio_crawler.test.ts@@ -1147,8 +1147,7 @@ describe('CheerioCrawler', () => {await cheerioCrawler.run();- // @ts-expect-error private symbol- const sessions: Session[] = cheerioCrawler.sessionPool!.sessions;+ const { sessions } = await (cheerioCrawler.sessionPool as SessionPool).getState();expect(sessions.length).toBe(4);sessions.forEach((session) => {// TODO this test is flaky in CI and we need some more info to debug why.@@ -1188,8 +1187,7 @@ describe('CheerioCrawler', () => {});await crawler.run();- // @ts-expect-error private symbol- const poolSessions: Session[] = crawler.sessionPool.sessions;+ const { sessions: poolSessions } = await (crawler.sessionPool as SessionPool).getState();// each request retires its session on every retry, so we get// (maxRequestRetries + 1) sessions per request (retired ones + the final one)expect(poolSessions.length).toBe(4 * (maxRequestRetries + 1));diff --git a/test/core/proxy_configuration.test.ts b/test/core/proxy_configuration.test.tsindex bd42904f6..bc37ef1f7 100644--- a/test/core/proxy_configuration.test.ts+++ b/test/core/proxy_configuration.test.ts@@ -107,12 +107,9 @@ describe('ProxyConfiguration', () => {describe('With proxyUrls options', () => {test('should rotate custom URLs correctly', async () => {- const proxyConfiguration = new ProxyConfiguration({- proxyUrls: ['http://proxy.com:1111', 'http://proxy.com:2222', 'http://proxy.com:3333'],- });+ const proxyUrls = ['http://proxy.com:1111', 'http://proxy.com:2222', 'http://proxy.com:3333'];+ const proxyConfiguration = new ProxyConfiguration({ proxyUrls });- // @ts-expect-error private property- const proxyUrls = proxyConfiguration.proxyUrls!;expect(await proxyConfiguration.newUrl()).toEqual(proxyUrls[0]);expect(await proxyConfiguration.newUrl()).toEqual(proxyUrls[1]);expect(await proxyConfiguration.newUrl()).toEqual(proxyUrls[2]);@@ -122,12 +119,9 @@ describe('ProxyConfiguration', () => {});test('newProxyInfo() should return correctly rotated URL', async () => {- const proxyConfiguration = new ProxyConfiguration({- proxyUrls: ['http://proxy.com:1111', 'http://proxy.com:2222', 'http://proxy.com:3333'],- });+ const proxyUrls = ['http://proxy.com:1111', 'http://proxy.com:2222', 'http://proxy.com:3333'];+ const proxyConfiguration = new ProxyConfiguration({ proxyUrls });- // @ts-expect-error TODO private property?- const proxyUrls = proxyConfiguration.proxyUrls!;expect((await proxyConfiguration.newProxyInfo())!.url).toEqual(proxyUrls[0]);expect((await proxyConfiguration.newProxyInfo())!.url).toEqual(proxyUrls[1]);expect((await proxyConfiguration.newProxyInfo())!.url).toEqual(proxyUrls[2]);diff --git a/test/core/request_list.test.ts b/test/core/request_list.test.tsindex 4148528d9..f3a93cf59 100644--- a/test/core/request_list.test.ts+++ b/test/core/request_list.test.ts@@ -7,7 +7,9 @@ import {ProxyConfiguration,Request,RequestList,+ REQUESTS_PERSISTENCE_KEY,serviceLocator,+ STATE_PERSISTENCE_KEY,} from '@crawlee/core';import { sleep } from '@crawlee/utils';import { beforeAll, type MockedFunction } from 'vitest';@@ -164,7 +166,7 @@ describe('RequestList', () => {});test('should correctly load list from hosted files in correct order', async () => {- const spy = vitest.spyOn(RequestList.prototype as any, '_downloadListOfUrls');+ const spy = vitest.spyOn(RequestList.prototype as any, 'downloadListOfUrls');const list1 = ['https://example.com', 'https://google.com', 'https://wired.com'];const list2 = ['https://another.com', 'https://page.com'];spy.mockImplementationOnce(() => new Promise((resolve) => setTimeout(() => resolve(list1) as any, 100)) as any);@@ -242,7 +244,7 @@ describe('RequestList', () => {});test('should handle requestsFromUrl with no URLs', async () => {- const spy = vitest.spyOn(RequestList.prototype as any, '_downloadListOfUrls');+ const spy = vitest.spyOn(RequestList.prototype as any, 'downloadListOfUrls');spy.mockResolvedValueOnce([]);const requestList = await RequestList.open({@@ -263,7 +265,7 @@ describe('RequestList', () => {test('should use the defined proxy server when using `requestsFromUrl`', async () => {const proxyUrls = ['http://proxyurl.usedforthe.download', 'http://another.proxy.url'];- const spy = vitest.spyOn(RequestList.prototype as any, '_downloadListOfUrls');+ const spy = vitest.spyOn(RequestList.prototype as any, 'downloadListOfUrls');spy.mockResolvedValue([]);const proxyConfiguration = new ProxyConfiguration({@@ -430,7 +432,7 @@ describe('RequestList', () => {const PERSIST_REQUESTS_KEY = 'some-key';const getValueSpy = vitest.spyOn(KeyValueStore.prototype, 'getValue');const setValueSpy = vitest.spyOn(KeyValueStore.prototype, 'setValue');- const spy = vitest.spyOn(RequestList.prototype as any, '_downloadListOfUrls');+ const spy = vitest.spyOn(RequestList.prototype as any, 'downloadListOfUrls');let persistedRequests: any;const opts = {@@ -575,6 +577,9 @@ describe('RequestList', () => {});describe('Apify.RequestList.open()', () => {+ /** The keys a spied {@apilink KeyValueStore} method was called with, in call order. */+ const keysPassedTo = (spy: MockedFunction<any>) => spy.mock.calls.map(([key]: [string]) => key);+test('should work', async () => {const getValueSpy = vitest.spyOn(KeyValueStore.prototype, 'getValue');const setValueSpy = vitest.spyOn(KeyValueStore.prototype, 'setValue');@@ -586,16 +591,16 @@ describe('RequestList', () => {const rl = await RequestList.open(name, sources);expect(rl).toBeInstanceOf(RequestList);// @ts-expect-error accessing private var- expect(rl.persistStateKey.startsWith(CRAWLEE_KEY)).toBe(true);- // @ts-expect-error accessing private var- expect(rl.persistRequestsKey.startsWith(CRAWLEE_KEY)).toBe(true);- // @ts-expect-error accessing private varexpect(rl.sources).toEqual([]);- // @ts-expect-error accessing private var- expect(rl.isInitialized).toBe(true);-- expect(getValueSpy).toBeCalledTimes(2);- expect(setValueSpy).toBeCalledTimes(1);+ // An uninitialized list throws here, so this is the observable form of "open() initialized it".+ await expect(rl.isEmpty()).resolves.toBe(false);++ // The persistence keys are derived from the list name, which shows in the keys it reads and writes.+ expect(keysPassedTo(getValueSpy)).toEqual([+ `${CRAWLEE_KEY}-${STATE_PERSISTENCE_KEY}`,+ `${CRAWLEE_KEY}-${REQUESTS_PERSISTENCE_KEY}`,+ ]);+ expect(keysPassedTo(setValueSpy)).toEqual([`${CRAWLEE_KEY}-${REQUESTS_PERSISTENCE_KEY}`]);});test('should work with string sources', async () => {@@ -609,16 +614,14 @@ describe('RequestList', () => {const rl = await RequestList.open(name, sources);expect(rl).toBeInstanceOf(RequestList);- // @ts-expect-error accessing private var- expect(rl.persistStateKey.startsWith(CRAWLEE_KEY)).toBe(true);- // @ts-expect-error accessing private var- expect(rl.persistRequestsKey.startsWith(CRAWLEE_KEY)).toBe(true);expect(rl.requests).toEqual(requests);- // @ts-expect-error accessing private var- expect(rl.isInitialized).toBe(true);+ await expect(rl.isEmpty()).resolves.toBe(false);- expect(getValueSpy).toBeCalledTimes(2);- expect(setValueSpy).toBeCalledTimes(1);+ expect(keysPassedTo(getValueSpy)).toEqual([+ `${CRAWLEE_KEY}-${STATE_PERSISTENCE_KEY}`,+ `${CRAWLEE_KEY}-${REQUESTS_PERSISTENCE_KEY}`,+ ]);+ expect(keysPassedTo(setValueSpy)).toEqual([`${CRAWLEE_KEY}-${REQUESTS_PERSISTENCE_KEY}`]);});test('should correctly pass options', async () => {@@ -637,18 +640,16 @@ describe('RequestList', () => {const rl = await RequestList.open(name, sources, options);expect(rl).toBeInstanceOf(RequestList);- // @ts-expect-error accessing private var- expect(rl.persistStateKey.startsWith(CRAWLEE_KEY)).toBe(true);- // @ts-expect-error accessing private var- expect(rl.persistRequestsKey.startsWith(CRAWLEE_KEY)).toBe(true);+ // The counter suffix on the unique key is what `keepDuplicateUrls: true` does.expect(rl.requests).toEqual(requests);- // @ts-expect-error accessing private var- expect(rl.isInitialized).toBe(true);- // @ts-expect-error accessing private var- expect(rl.keepDuplicateUrls).toBe(true);-- expect(getValueSpy).toBeCalledTimes(2);- expect(setValueSpy).toBeCalledTimes(1);+ await expect(rl.isEmpty()).resolves.toBe(false);++ // The list name wins over the `persistStateKey` option.+ expect(keysPassedTo(getValueSpy)).toEqual([+ `${CRAWLEE_KEY}-${STATE_PERSISTENCE_KEY}`,+ `${CRAWLEE_KEY}-${REQUESTS_PERSISTENCE_KEY}`,+ ]);+ expect(keysPassedTo(setValueSpy)).toEqual([`${CRAWLEE_KEY}-${REQUESTS_PERSISTENCE_KEY}`]);});test('should work with null name', async () => {@@ -661,14 +662,10 @@ describe('RequestList', () => {const rl = await RequestList.open(name, sources);expect(rl).toBeInstanceOf(RequestList);- // @ts-expect-error accessing private var- expect(rl.persistStateKey == null).toBe(true);- // @ts-expect-error accessing private var- expect(rl.persistRequestsKey == null).toBe(true);expect(rl.requests).toEqual(requests);- // @ts-expect-error accessing private var- expect(rl.isInitialized).toBe(true);+ await expect(rl.isEmpty()).resolves.toBe(false);+ // A nameless list has no persistence keys, so it never touches the store.expect(getValueSpy).not.toBeCalled();expect(setValueSpy).not.toBeCalled();});diff --git a/test/core/session_pool/session.test.ts b/test/core/session_pool/session.test.tsindex bd93eb294..98ddac123 100644--- a/test/core/session_pool/session.test.ts+++ b/test/core/session_pool/session.test.ts@@ -16,8 +16,8 @@ describe('Session - testing session behaviour', () => {session.markGood();expect(session.usageCount).toBe(1);expect(session.errorScore).toBe(0);- // @ts-expect-error Private property- session._errorScore = 1;+ session.markBad();+ expect(session.errorScore).toBe(1);session.markGood();expect(session.errorScore).toBe(0.5);});@@ -36,16 +36,14 @@ describe('Session - testing session behaviour', () => {});test('should max out session usage', () => {- // @ts-expect-error Private property- session._maxUsageCount = 1;+ session = new Session({ maxUsageCount: 1 });session.markGood();expect(session.isMaxUsageCountReached()).toBe(true);expect(session.isUsable()).toBe(false);});test('should block session', () => {- // @ts-expect-error Private property- session._errorScore += session.maxErrorScore;+ session = new Session({ maxErrorScore: 3, errorScore: 3 });expect(session.isBlocked()).toBe(true);expect(session.isUsable()).toBe(false);});diff --git a/test/core/session_pool/session_pool.test.ts b/test/core/session_pool/session_pool.test.tsindex 2c65b9181..0319255f4 100644--- a/test/core/session_pool/session_pool.test.ts+++ b/test/core/session_pool/session_pool.test.ts@@ -23,8 +23,7 @@ describe('SessionPool - testing session pool', () => {});test('should initialize with default values for first time', async () => {- // @ts-expect-error private symbol- expect(sessionPool.sessions.length).toBeDefined();+ expect((await sessionPool.getState()).sessions).toEqual([]);// @ts-expect-error private symbolexpect(sessionPool.maxPoolSize).toBeDefined();// @ts-expect-error private symbol@@ -65,8 +64,7 @@ describe('SessionPool - testing session pool', () => {test('should retrieve session with correct shape', async () => {sessionPool = new SessionPool({ sessionOptions: { maxAgeSecs: 100, maxUsageCount: 10 } });const session = await sessionPool.getSession();- // @ts-expect-error private symbol- expect(sessionPool.sessions.length).toBe(1);+ expect((await sessionPool.getState()).sessions).toHaveLength(1);expect(session?.id).toBeDefined();expect(session!.expiresAt.getTime() - session!.createdAt.getTime()).toEqual(// @ts-expect-error Accessing protected property@@ -102,17 +100,14 @@ describe('SessionPool - testing session pool', () => {await sessionPool.addSession();const session = await sessionPool.getSession();- // @ts-expect-error private symbol- expect(sessionPool.sessions[0].id === session.id).toBe(true);+ expect((await sessionPool.getState()).sessions[0].id).toBe(session!.id);- // @ts-expect-error Overriding private property- session._errorScore += session.maxErrorScore;+ session!.retire();await sessionPool.getSession();- // @ts-expect-error private symbol- expect(sessionPool.sessions[0].id === session.id).toBe(false);- // @ts-expect-error private symbol- expect(sessionPool.sessions).toHaveLength(1);+ const { sessions } = await sessionPool.getState();+ expect(sessions[0].id).not.toBe(session!.id);+ expect(sessions).toHaveLength(1);});});@@ -146,30 +141,13 @@ describe('SessionPool - testing session pool', () => {expect(sessionPoolSaved!.usableSessionsCount).toEqual(currentState.usableSessionsCount);expect(sessionPoolSaved!.retiredSessionsCount).toEqual(currentState.retiredSessionsCount);- // @ts-expect-error private symbol- expect(sessionPoolSaved.sessions.length).toEqual(sessionPool.sessions.length);-- sessionPoolSaved!.sessions.forEach((session, index) => {- entries(session).forEach(([key, value]) => {- // @ts-expect-error private symbol- if (sessionPool.sessions[index][key] instanceof Date) {- // @ts-expect-error private symbol- expect(value).toEqual((sessionPool.sessions[index][key] as Date).toISOString());- } else if (key === 'cookieJar') {- // @ts-expect-error private symbol- expect(value).toEqual(sessionPool.sessions[index][key].toJSON());- } else {- // @ts-expect-error private symbol- expect(sessionPool.sessions[index][key]).toEqual(value);- }- });- });+ // Every persisted session round-trips field for field, dates and cookie jar included.+ expect(sessionPoolSaved!.sessions).toEqual(currentState.sessions);const loadedSessionPool = new SessionPool({ persistStateKey });- // @ts-expect-error Accessing protected method- await loadedSessionPool.ensureInitialized();- // @ts-expect-error private symbol- expect(sessionPool.sessions).toHaveLength(loadedSessionPool.sessions.length);+ expect((await sessionPool.getState()).sessions).toHaveLength(+ (await loadedSessionPool.getState()).sessions.length,+ );// @ts-expect-error private symbolexpect(sessionPool.maxPoolSize).toEqual(loadedSessionPool.maxPoolSize);// @ts-expect-error private symbol@@ -178,13 +156,12 @@ describe('SessionPool - testing session pool', () => {});test('should create only maxPoolSize number of sessions', async () => {- // @ts-expect-error private symbol- const max = sessionPool.maxPoolSize;- for (let i = 0; i < max + 100; i++) {+ const maxPoolSize = 10;+ sessionPool = new SessionPool({ maxPoolSize });+ for (let i = 0; i < maxPoolSize + 100; i++) {await sessionPool.getSession();}- // @ts-expect-error private symbol- expect(sessionPool.sessions.length).toEqual(sessionPool.maxPoolSize);+ expect((await sessionPool.getState()).sessions).toHaveLength(maxPoolSize);});test('should create session', async () => {@@ -192,10 +169,9 @@ describe('SessionPool - testing session pool', () => {await sessionPool.ensureInitialized();// @ts-expect-error private symbolawait sessionPool.createSession();- // @ts-expect-error private symbol- expect(sessionPool.sessions.length).toBe(1);- // @ts-expect-error private symbol- expect(sessionPool.sessions[0].id).toBeDefined();+ const { sessions } = await sessionPool.getState();+ expect(sessions).toHaveLength(1);+ expect(sessions[0].id).toBeDefined();});describe('should persist state', () => {@@ -212,8 +188,7 @@ describe('SessionPool - testing session pool', () => {test('on persist event', async () => {await sessionPool.getSession();- // @ts-expect-error private symbol- expect(sessionPool.sessions.length).toBe(1);+ expect((await sessionPool.getState()).sessions).toHaveLength(1);serviceLocator.getEventManager().emit(EventType.PERSIST_STATE);@@ -238,18 +213,15 @@ describe('SessionPool - testing session pool', () => {test('should remove retired sessions', async () => {// @ts-expect-error private symbolsessionPool.maxPoolSize = 1;- await sessionPool.getSession();+ const session = (await sessionPool.getSession())!;- // @ts-expect-error private symbol- const session = sessionPool.sessions[0];- // @ts-expect-error Overriding private property- session._errorScore += session.maxErrorScore;+ session.retire();const { id: retiredSessionId } = session;await sessionPool.getSession();- // @ts-expect-error private symbol- expect(sessionPool.sessions.find((s) => s.id === retiredSessionId)).toEqual(undefined);+ const { sessions } = await sessionPool.getState();+ expect(sessions.find((s) => s.id === retiredSessionId)).toBeUndefined();});test('should recreate only usable sessions', async () => {@@ -261,8 +233,7 @@ describe('SessionPool - testing session pool', () => {const session = await sessionPool.getSession();if (i % 2 === 0) {- // @ts-expect-error Overriding private property- session._errorScore += session.maxErrorScore;+ session!.retire();invalidSessionsCount += 1;}}@@ -271,10 +242,7 @@ describe('SessionPool - testing session pool', () => {await sessionPool.persistState();const newSessionPool = new SessionPool({ persistStateKey });- // @ts-expect-error Accessing protected method- await newSessionPool.ensureInitialized();- // @ts-expect-error Accessing private property- expect(newSessionPool.sessions).toHaveLength(10 - invalidSessionsCount);+ expect((await newSessionPool.getState()).sessions).toHaveLength(10 - invalidSessionsCount);await newSessionPool.teardown();});@@ -355,9 +323,8 @@ describe('SessionPool - testing session pool', () => {test('should be able to create session with provided id', async () => {await sessionPool.addSession({ id: 'test-session' });- // @ts-expect-error private symbol- const session = sessionPool.sessions[0];- expect(session.id).toBe('test-session');+ const session = await sessionPool.getSession('test-session');+ expect(session?.id).toBe('test-session');});test('should be able to add session instance and create new session with provided sessionOptions with addSession()', async () => {@@ -397,12 +364,12 @@ describe('SessionPool - testing session pool', () => {for (let i = 0; i < 20; i++) await sessionPool.getSession();- // @ts-expect-error private symbol- expect(sessionPool.sessions.length).toEqual(10);+ const { sessions } = await sessionPool.getState();+ expect(sessions).toHaveLength(10);// @ts-expect-error private symbolexpect(sessionPool.sessionMap.size).toEqual(10);// @ts-expect-error private symbol- expect(sessionPool.sessions.length).toEqual(sessionPool.sessionMap.size);+ expect(sessionPool.sessionMap.size).toEqual(sessions.length);});test('should correctly remove retired sessions both from array and session map', async () => {@@ -412,18 +379,17 @@ describe('SessionPool - testing session pool', () => {for (let i = 0; i < 10; i++) {await sessionPool.addSession({ id: `session_${i}` });const session = await sessionPool.getSession(`session_${i}`);- // @ts-expect-error Overriding private property- session._errorScore += session.maxErrorScore;+ session!.retire();}await sessionPool.getSession();- // @ts-expect-error private symbol- expect(sessionPool.sessions.length).toEqual(1);+ const { sessions } = await sessionPool.getState();+ expect(sessions).toHaveLength(1);// @ts-expect-error private symbolexpect(sessionPool.sessionMap.size).toEqual(1);// @ts-expect-error private symbol- expect(sessionPool.sessions.length).toEqual(sessionPool.sessionMap.size);+ expect(sessionPool.sessionMap.size).toEqual(sessions.length);});describe('sessionReuseStrategy', () => {@@ -478,15 +444,13 @@ describe('SessionPool - testing session pool', () => {s1?.retire();- // @ts-expect-error private symbol- expect(sessionPool.sessions).toHaveLength(3);+ expect((await sessionPool.getState()).sessions).toHaveLength(3);for (let i = 0; i < 50; i++) await sessionPool.getSession();- // @ts-expect-error private symbol- expect(sessionPool.sessions).toHaveLength(3);- // @ts-expect-error private symbol- expect(sessionPool.sessions.find((s) => s.id === s1.id)).toBeUndefined();+ const { sessions } = await sessionPool.getState();+ expect(sessions).toHaveLength(3);+ expect(sessions.find((s) => s.id === s1!.id)).toBeUndefined();},);@@ -543,15 +507,8 @@ describe('SessionPool - testing session pool', () => {persistStateKey: pool2.persistStateKey,});- // @ts-expect-error -- we're reading the private sessions field, public methods initialize the instance automatically- await pool1Reloaded.ensureInitialized();- // @ts-expect-error- await pool2Reloaded.ensureInitialized();-- // @ts-expect-error private symbol- expect(pool1Reloaded.sessions).toHaveLength(3);- // @ts-expect-error private symbol- expect(pool2Reloaded.sessions).toHaveLength(5);+ expect((await pool1Reloaded.getState()).sessions).toHaveLength(3);+ expect((await pool2Reloaded.getState()).sessions).toHaveLength(5);await pool1.teardown();await pool2.teardown();diff --git a/test/core/storages/request_queue.test.ts b/test/core/storages/request_queue.test.tsindex d2b697aed..afdfa4ac9 100644--- a/test/core/storages/request_queue.test.ts+++ b/test/core/storages/request_queue.test.ts@@ -415,7 +415,7 @@ describe('RequestQueue with requestsFromUrl', () => {});test('should correctly load list from hosted files in correct order', async () => {- const spy = vitest.spyOn(RequestQueue.prototype as any, '_downloadListOfUrls');+ const spy = vitest.spyOn(RequestQueue.prototype as any, 'downloadListOfUrls');const list1 = ['https://example.com', 'https://google.com', 'https://wired.com'];const list2 = ['https://another.com', 'https://page.com'];spy.mockImplementationOnce(() => new Promise((resolve) => setTimeout(() => resolve(list1) as any, 100)) as any);@@ -491,7 +491,7 @@ describe('RequestQueue with requestsFromUrl', () => {});test('should handle requestsFromUrl with no URLs', async () => {- const spy = vitest.spyOn(RequestQueue.prototype as any, '_downloadListOfUrls');+ const spy = vitest.spyOn(RequestQueue.prototype as any, 'downloadListOfUrls');spy.mockResolvedValueOnce([]);const queue = await RequestQueue.open();@@ -509,7 +509,7 @@ describe('RequestQueue with requestsFromUrl', () => {test('should use the defined proxy server when using `requestsFromUrl`', async () => {const proxyUrls = ['http://proxyurl.usedforthe.download', 'http://another.proxy.url'];- const spy = vitest.spyOn(RequestQueue.prototype as any, '_downloadListOfUrls');+ const spy = vitest.spyOn(RequestQueue.prototype as any, 'downloadListOfUrls');spy.mockResolvedValue([]);const proxyConfiguration = new ProxyConfiguration({