Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
refactor!: Privatize unintentionally exposed internals (#3893)
Jan Buchara78527dce393
diff --git a/docs/public-api/crawlee-basic.api.md b/docs/public-api/crawlee-basic.api.mdindex 82ac79d97..4273d6438 100644--- a/docs/public-api/crawlee-basic.api.md+++ b/docs/public-api/crawlee-basic.api.md@@ -62,38 +62,26 @@ import { TimeoutError } from '@apify/timeout';export class BasicCrawler<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension> {constructor(options?: BasicCrawlerOptions<Context, ContextExtension, ExtendedContext> & RequireContextPipeline<CrawlingContext, Context>);// (undocumented)- protected additionalHttpErrorStatusCodes: Set<number>;+ protected readonly additionalHttpErrorStatusCodes: Set<number>;addRequests(requests: ReadonlyDeep<RequestsLike>, options?: CrawlerAddRequestsOptions): Promise<CrawlerAddRequestsResult>;autoscaledPool?: AutoscaledPool;- // (undocumented)- protected autoscaledPoolOptions: AutoscaledPoolOptions;get basicContextPipeline(): ContextPipeline<{request: Request_2;}, CrawlingContext>;// (undocumented)protected blockedStatusCodes: Set<number>;- protected buildBasicContextPipeline(): ContextPipeline<{- request: Request_2;- }, CrawlingContext>;protected buildContextPipeline(): ContextPipeline<CrawlingContext, CrawlingContext>;// (undocumented)protected calculateEnqueuedRequestLimit(explicitLimit?: number): Promise<number | undefined>;// (undocumented)- protected _canRequestBeRetried(request: Request_2, error: Error): boolean;- // (undocumented)get contextPipeline(): ContextPipeline<CrawlingContext, ExtendedContext>;// (undocumented)protected static readonly CRAWLEE_STATE_KEY = "CRAWLEE_STATE";- protected _defaultIsFinishedFunction(): Promise<boolean>;- protected delayRequest(request: Request_2, source: IRequestManager): boolean;- // (undocumented)- protected domainAccessedTime: Map<string, number>;// (undocumented)- protected errorHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;+ protected readonly errorHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;exportData<Data>(path: string, format?: 'json' | 'csv', options?: DatasetExportOptions): Promise<Data[]>;// (undocumented)- protected failedRequestHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;- protected _fetchNextRequest(): Promise<Request_2<Dictionary> | null>;+ protected readonly failedRequestHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;// (undocumented)protected _getCookieHeaderFromRequest(request: Request_2): string;getData(...args: Parameters<Dataset['getData']>): ReturnType<Dataset['getData']>;@@ -107,37 +95,28 @@ export class BasicCrawler<Context extends CrawlingContext = CrawlingContext, Con// (undocumented)protected getRobotsTxtFileForUrl(url: string): Promise<RobotsTxtFile | undefined>;// (undocumented)- protected get handledRequestsCount(): number;- protected set handledRequestsCount(_value: number);- // (undocumented)- protected _handleFailedRequestHandler(crawlingContext: CrawlingContext, error: Error): Promise<void>;- protected handleRequest(crawlingContext: ExtendedContext, requestSource: IRequestManager, request: Request_2): Promise<void>;- // (undocumented)protected handleSkippedRequest(options: Parameters<SkippedRequestCallback>[0]): Promise<void>;// (undocumented)hasFinishedBefore: boolean;// (undocumented)- protected httpClient: BaseHttpClient;+ protected readonly httpClient: BaseHttpClient;// (undocumented)protected readonly identity: CrawlerIdentity;- // (undocumented)- protected ignoreHttpErrorStatusCodes: Set<number>;protected _init(): Promise<void>;// (undocumented)- protected internalTimeoutMillis: number;+ protected readonly internalTimeoutMillis: number;protected isErrorStatusCode(status: number): boolean;protected isProxyError(error: Error): boolean;- protected _isTaskReadyFunction(): Promise<boolean>;// (undocumented)get log(): CrawleeLogger;// (undocumented)- protected maxCrawlDepth?: number;+ protected readonly maxCrawlDepth?: number;// (undocumented)- protected maxRequestRetries: number;+ protected readonly maxRequestRetries: number;// (undocumented)- protected maxRequestsPerCrawl?: number;+ protected readonly maxRequestsPerCrawl?: number;// (undocumented)- protected onSkippedRequest?: SkippedRequestCallback;+ protected readonly onSkippedRequest?: SkippedRequestCallback;// (undocumented)protected static optionsShape: {contextPipelineBuilder: ObjectPredicate<object> & BasePredicate<object | undefined>;@@ -175,48 +154,27 @@ export class BasicCrawler<Context extends CrawlingContext = CrawlingContext, ConstatisticsOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;id: StringPredicate & BasePredicate<string | undefined>;};- // (undocumented)- protected _pauseOnMigration(): Promise<void>;- proxyConfiguration?: IProxyConfiguration;+ readonly proxyConfiguration?: IProxyConfiguration;pushData(data: Parameters<Dataset['pushData']>[0], datasetIdentifier?: string | StorageIdentifier): Promise<void>;- protected _requestFunctionErrorHandler(error: Error, crawlingContext: CrawlingContext, request: Request_2, source: IRequestManager): Promise<void>;// (undocumented)- protected requestHandler: RequestHandler<ExtendedContext>;- // (undocumented)- protected requestHandlerTimeoutMillis: number;+ protected readonly requestHandler: RequestHandler<ExtendedContext>;protected requestManager?: IRequestManager;// (undocumented)- protected respectRobotsTxtFile: boolean | {- userAgent?: string;- };- // (undocumented)- protected retryOnBlocked: boolean;+ protected readonly retryOnBlocked: boolean;readonly router: RouterHandler<Context>;run(requests?: RequestsLike, options?: CrawlerRunOptions): Promise<FinalStatistics>;// (undocumented)running: boolean;// (undocumented)protected runRequestHandler(crawlingContext: ExtendedContext): Promise<void>;- // (undocumented)- protected sameDomainDelayMillis: number;- sessionPool: ISessionPool;+ readonly sessionPool: ISessionPool;setStatusMessage(message: string, options?: SetStatusMessageOptions): void;readonly stats: Statistics;- // (undocumented)- protected statusMessageCallback?: StatusMessageCallback;- // (undocumented)- protected statusMessageLoggingInterval: number;stop(reason?: string): void;- // (undocumented)- protected _tagUserHandlerError<T>(cb: () => unknown): Promise<T>;teardown(): Promise<void>;protected _throwOnBlockedRequest(statusCode: number): void;- protected _timeoutAndRetry<T>(handler: () => Promise<T>, timeout: number, error: Error | string, maxRetries?: number, retried?: number): Promise<T>;- // (undocumented)- protected unexpectedStop: boolean;// (undocumented)useState<State extends Dictionary = Dictionary>(defaultValue?: State): Promise<State>;- protected validateRequestUserData(source: Source | string): Promise<void>;}// @public (undocumented)diff --git a/docs/public-api/crawlee-browser-pool.api.md b/docs/public-api/crawlee-browser-pool.api.mdindex 9ba695c44..88599d0c3 100644--- a/docs/public-api/crawlee-browser-pool.api.md+++ b/docs/public-api/crawlee-browser-pool.api.md@@ -60,7 +60,7 @@ export abstract class BrowserController<Library extends CommonLibrary = CommonLi// (undocumented)assignBrowser(browser: LaunchResult, launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): void;browser: LaunchResult;- browserPlugin: BrowserPlugin<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;+ readonly browserPlugin: BrowserPlugin<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;close(): Promise<void>;// (undocumented)protected abstract _close(): Promise<void>;@@ -69,7 +69,7 @@ export abstract class BrowserController<Library extends CommonLibrary = CommonLi// (undocumented)protected abstract _getCookies(page: NewPageResult): Promise<Cookie[]>;// (undocumented)- id: string;+ readonly id: string;// (undocumented)isActive: boolean;kill(): Promise<void>;@@ -79,7 +79,7 @@ export abstract class BrowserController<Library extends CommonLibrary = CommonLilastPageOpenedAt: number;launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;// (undocumented)- protected log: CrawleeLogger;+ protected readonly log: CrawleeLogger;newPage(pageOptions?: NewPageOptions): Promise<NewPageResult>;// (undocumented)protected abstract _newPage(pageOptions?: NewPageOptions): Promise<NewPageResult>;@@ -123,34 +123,34 @@ export abstract class BrowserPlugin<Library extends CommonLibrary = CommonLibrar// (undocumented)protected abstract _addProxyToLaunchOptions(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): Promise<void>;// (undocumented)- browserPerProxy?: boolean;+ readonly browserPerProxy?: boolean;protected _connectToRemoteBrowser(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>, connect: (url: string) => Promise<LaunchResult>): Promise<LaunchResult>;// (undocumented)abstract createController(): BrowserController<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;createLaunchContext(options?: CreateLaunchContextOptions<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;// (undocumented)- ignoreProxyCertificate?: boolean;+ readonly ignoreProxyCertificate?: boolean;// (undocumented)protected abstract _isChromiumBasedBrowser(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): boolean;launch(launchContext?: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): Promise<LaunchResult>;// (undocumented)protected abstract _launch(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): Promise<LaunchResult>;// (undocumented)- launchOptions: LibraryOptions;+ readonly launchOptions: LibraryOptions;// (undocumented)- library: Library;+ readonly library: Library;// (undocumented)- protected log: CrawleeLogger;+ protected readonly log: CrawleeLogger;// (undocumented)- name: string;+ readonly name: string;// (undocumented)- proxyUrl?: string;+ readonly proxyUrl?: string;// (undocumented)protected _throwAugmentedLaunchError(cause: unknown, executablePath: string | undefined, dockerImage: string, moduleInstallCommand: string): never;// (undocumented)useIncognitoPages: boolean;// (undocumented)- userDataDir?: string;+ readonly userDataDir?: string;}// @public (undocumented)@@ -471,8 +471,6 @@ export class PlaywrightPlugin extends BrowserPlugin<BrowserType, SafeParameters<// (undocumented)protected _addProxyToLaunchOptions(launchContext: LaunchContext<BrowserType>): Promise<void>;// (undocumented)- _containerProxyServer?: Awaited<ReturnType<typeof createProxyServerForContainers>>;- // (undocumented)createController(): PlaywrightController;// (undocumented)protected _isChromiumBasedBrowser(): boolean;diff --git a/docs/public-api/crawlee-core.api.md b/docs/public-api/crawlee-core.api.mdindex c42eb4e02..4ef2103b9 100644--- a/docs/public-api/crawlee-core.api.md+++ b/docs/public-api/crawlee-core.api.md@@ -32,7 +32,6 @@ import { LoggerJson } from '@apify/log';import type { LoggerOptions } from '@apify/log';import { LoggerText } from '@apify/log';import { LogLevel } from '@apify/log';-import { LruCache } from '@apify/datastructures';import { ParseSitemapOptions } from '@crawlee/utils';import type { ProcessedRequest } from '@crawlee/types';import type { ProxyInfo } from '@crawlee/types';@@ -86,27 +85,17 @@ export function assertJsonSerializable<T>(item: T, index?: number): void;export class AutoscaledPool {constructor(options: AutoscaledPoolOptions);abort(): Promise<void>;- protected _autoscale(intervalCallback: () => void): void;get currentConcurrency(): number;get desiredConcurrency(): number;set desiredConcurrency(value: number);- protected _destroy(): Promise<void>;- // (undocumented)- protected _incrementTasksDonePerSecond(intervalCallback: () => void): void;- // (undocumented)- protected get _isOverMaxRequestLimit(): boolean;get maxConcurrency(): number;set maxConcurrency(value: number);- protected _maybeFinish(): Promise<void>;- protected _maybeRunTask(intervalCallback?: () => void): Promise<void>;get minConcurrency(): number;set minConcurrency(value: number);notify(): Promise<void>;pause(timeoutSecs?: number): Promise<void>;resume(): void;run(): Promise<void>;- protected _scaleDown(systemStatus: SystemInfo): void;- protected _scaleUp(systemStatus: SystemInfo): void;}// @public (undocumented)@@ -1001,29 +990,10 @@ export interface PersistenceOptions {// @publicexport class ProxyConfiguration implements IProxyConfiguration {constructor(options?: ProxyConfigurationOptions);- protected _callNewUrlFunction(options?: {- request?: Request_2;- }): Promise<string | null>;// (undocumented)- protected _handleProxyUrlsList(): string | null;- // (undocumented)- isManInTheMiddle: boolean;- // (undocumented)- protected log: CrawleeLogger;+ readonly isManInTheMiddle = false;newProxyInfo(options?: NewUrlOptions): Promise<ProxyInfo | undefined>;newUrl(options?: NewUrlOptions): Promise<string | undefined>;- // (undocumented)- protected newUrlFunction?: ProxyConfigurationFunction;- // (undocumented)- protected nextCustomUrlIndex: number;- // (undocumented)- protected proxyUrls?: UrlList;- // (undocumented)- protected _throwCannotCombineCustomMethods(): never;- // (undocumented)- protected _throwNoOptionsProvided(): never;- // (undocumented)- protected usedProxyUrls: Map<string, string | null>;}// @public (undocumented)@@ -1181,35 +1151,22 @@ export class RequestHandlerResult {export class RequestList implements IRequestLoader {// (undocumented)[Symbol.asyncIterator](): AsyncGenerator<Request_2<Dictionary>, void, unknown>;- protected _addFetchedRequests(source: InternalSource, fetchedRequests: RequestOptions[]): Promise<void>;- protected _addPersistedRequests(persistedRequests: Buffer): Promise<void>;- protected _addRequest(source: RequestListSource): void;- protected _addRequestsFromSources(): Promise<void>;- protected _ensureInProgress(uniqueKey: string): void;- protected _ensureIsInitialized(): void;- protected _ensureUniqueKeyValid(uniqueKey: string): void;// (undocumented)fetchNextRequest(): Promise<Request_2 | null>;- protected _fetchRequestsFromUrl(source: InternalSource): Promise<RequestOptions[]>;// (undocumented)getHandledCount(): Promise<number>;getPendingCount(): Promise<number>;- // (undocumented)- protected _getPersistedState<T>(key: string): Promise<T>;getState(): RequestListState;getTotalCount(): Promise<number>;// (undocumented)isEmpty(): Promise<boolean>;// (undocumented)isFinished(): Promise<boolean>;- protected _loadStateAndPersistedRequests(): Promise<[RequestListState, Buffer]>;// (undocumented)markRequestAsHandled(request: Request_2): Promise<void>;static open(listNameOrOptions: string | null | RequestListOptions, sources?: RequestListSource[], options?: RequestListOptions): Promise<RequestList>;- protected _persistRequests(): Promise<void>;// (undocumented)persistState(): Promise<void>;- protected _restoreState(state?: RequestListState): void;teardown(): Promise<void>;toTandem(requestManager?: IRequestManager): Promise<IRequestManager>;}@@ -1288,49 +1245,30 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {export class RequestQueue implements IStorage, IRequestManager {// (undocumented)[Symbol.asyncIterator](): AsyncGenerator<Request_2<Dictionary>, void, unknown>;- protected _addFetchedRequests(source: InternalSource, fetchedRequests: RequestOptions[], options: RequestQueueOperationOptions): Promise<ProcessedRequest[]>;addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;addRequests(requestsLike: RequestsLike, options?: RequestQueueOperationOptions): Promise<BatchAddRequestsResult>;addRequestsBatched(requests: ReadonlyDeep<RequestsLike>, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;// (undocumented)- backend: RequestQueueBackend;- protected _cacheRequest(cacheKey: string, queueOperationInfo: RequestQueueOperationInfo): void;- // (undocumented)- protected readonly config: Configuration;+ readonly backend: RequestQueueBackend;drop(): Promise<void>;- // (undocumented)- protected readonly events: EventManager;- protected expectedRequestProcessingSecs: number;fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request_2<T> | null>;- protected _fetchRequestsFromUrl(source: InternalSource): Promise<RequestOptions[]>;getHandledCount(): Promise<number>;getInfo(): Promise<RequestQueueInfo>;getPendingCount(): Promise<number>;getRequest<T extends Dictionary = Dictionary>(uniqueKey: string): Promise<Request_2<T> | null>;getTotalCount(): Promise<number>;// (undocumented)- protected httpClient?: BaseHttpClient;- // (undocumented)- id: string;- // (undocumented)- protected inProgressRequestBatchCount: number;+ readonly id: string;isEmpty(): Promise<boolean>;isFinished(): Promise<boolean>;// (undocumented)- log: CrawleeLogger;+ readonly log: CrawleeLogger;markRequestAsHandled(request: Request_2): Promise<RequestQueueOperationInfo | null>;// (undocumented)- name?: string;+ readonly name?: string;static open(identifier?: string | StorageIdentifier | null, options?: StorageOpenOptions): Promise<RequestQueue>;- // (undocumented)- protected proxyConfiguration?: IProxyConfiguration;purge(): Promise<void>;- // (undocumented)- protected queuePausedForMigration: boolean;reclaimRequest(request: Request_2, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;- // (undocumented)- protected requestCache: LruCache<RequestLruItem>;- protected requestSeenCache: RequestDeduplicationCache;setExpectedRequestProcessingTimeSecs(secs: number): Promise<void>;get stats(): RequestQueueStats;}@@ -1439,7 +1377,6 @@ export class RetryRequestError extends Error {// @publicexport class Router<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'>, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> {- protected constructor();addDefaultHandler<UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(handler: (ctx: RouterHandlerContext<Context, UserData>) => Awaitable_2<void>): void;addHandler<Label extends keyof Routes & string>(label: Label, handler: (ctx: RouterHandlerContext<Context, Routes[Label]>) => Awaitable_2<void>): void;addHandler<UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(label: RouterLabel<Routes>, handler: (ctx: RouterHandlerContext<Context, UserData>) => Awaitable_2<void>): void;@@ -1554,7 +1491,6 @@ export class Session implements ISession {get maxErrorScore(): number;// (undocumented)get maxUsageCount(): number;- protected _maybeSelfRetire(): void;// (undocumented)get proxyInfo(): ProxyInfo | undefined;retire(): void;@@ -1563,7 +1499,7 @@ export class Session implements ISession {// (undocumented)get usageCount(): number;// (undocumented)- userData: Dictionary;+ readonly userData: Dictionary;}// @public@@ -1598,56 +1534,19 @@ export interface SessionOptions {export class SessionPool implements ISessionPool {constructor(options?: SessionPoolOptions);addSession(options?: Session | SessionOptions): Promise<void>;- protected _addSession(newSession: Session): void;- protected _createSession(): Promise<Session>;- // (undocumented)- protected createSessionFunction: CreateSession;- protected _defaultCreateSessionFunction(options?: {- sessionOptions?: SessionOptions;- }): Promise<Session>;- protected ensureInitialized(): Promise<void>;- // (undocumented)- protected events: EventManager;- protected _getRandomIndex(): number;getSession(sessionId?: string): Promise<Session | undefined>;getState(): Promise<{usableSessionsCount: number;retiredSessionsCount: number;sessions: SessionState[];}>;- protected _hasSpaceForSession(): boolean;// (undocumented)readonly id: string;- // (undocumented)- protected keyValueStore?: KeyValueStore;- // (undocumented)- protected _listener?: () => Promise<void>;- // (undocumented)- protected log: CrawleeLogger;- // (undocumented)- protected maxPoolSize: number;- protected _maybeLoadSessionPool(): Promise<void>;newSession(sessionOptions?: SessionOptions): Promise<Session>;- // (undocumented)- protected persistenceOptions: PersistenceOptions;persistState(options?: PersistenceOptions): Promise<void>;// (undocumented)- protected persistStateKey: string;- // (undocumented)- protected persistStateKeyValueStoreId?: string;- protected _pickSession(): Session | undefined;- protected _removeRetiredSessions(): void;- // (undocumented)resetStore(options?: PersistenceOptions): Promise<void>;retiredSessionsCount(): Promise<number>;- // (undocumented)- protected sessionMap: Map<string, Session>;- // (undocumented)- protected sessionOptions: SessionOptions;- // (undocumented)- protected sessionReuseStrategy: SessionReuseStrategy;- // (undocumented)- protected sessions: Session[];teardown(): Promise<void>;usableSessionsCount(): Promise<number>;}@@ -1760,11 +1659,11 @@ export class SnapshotStore<T extends LoadSnapshot = LoadSnapshot> {export class Snapshotter {constructor(options?: SnapshotterOptions);// (undocumented)- client: StorageBackend;+ readonly client: StorageBackend;// (undocumented)get clientSnapshots(): ClientSnapshot[];// (undocumented)- config: Configuration;+ readonly config: Configuration;// (undocumented)get cpuSnapshots(): CpuSnapshot[];// (undocumented)@@ -1775,21 +1674,9 @@ export class Snapshotter {getLoadSignals(): LoadSignal[];getMemorySample(sampleDurationMillis?: number): MemorySnapshot[];// (undocumented)- log: CrawleeLogger;- // @deprecated (undocumented)- protected _memoryOverloadWarning(systemInfo: SystemInfo): void;+ readonly log: CrawleeLogger;// (undocumented)get memorySnapshots(): MemorySnapshot[];- // @deprecated (undocumented)- protected _pruneSnapshots(_snapshots: any[], _now: Date): void;- // @deprecated (undocumented)- protected _snapshotClient(intervalCallback: () => unknown): void;- // @deprecated (undocumented)- protected _snapshotCpu(systemInfo: SystemInfo): void;- // @deprecated (undocumented)- protected _snapshotEventLoop(intervalCallback: () => unknown): void;- // @deprecated (undocumented)- protected _snapshotMemory(systemInfo: SystemInfo): void;start(): Promise<void>;stop(): Promise<void>;}@@ -1842,30 +1729,26 @@ export class Statistics {crawlerRuntimeMillis: number;};discardJob(id: number | string): void;- errorTracker: ErrorTracker;- errorTrackerRetry: ErrorTracker;+ readonly errorTracker: ErrorTracker;+ readonly errorTrackerRetry: ErrorTracker;failJob(id: number | string, retryCount: number): void;finishJob(id: number | string, retryCount: number): void;readonly id: string;// (undocumented)protected keyValueStore?: KeyValueStore;- protected _maybeLoadStatistics(): Promise<void>;+ protected maybeLoadStatistics(): Promise<void>;persistState(options?: PersistenceOptions): Promise<void>;// (undocumented)- protected persistStateKey: string;+ protected readonly persistStateKey: string;registerStatusCode(code: number): void;readonly requestRetryHistogram: number[];reset(): void;// (undocumented)resetStore(options?: PersistenceOptions): Promise<void>;- // (undocumented)- protected _saveRetryCountForJob(retryCount: number): void;startCapturing(): Promise<void>;startJob(id: number | string): void;state: StatisticState;stopCapturing(): Promise<void>;- // (undocumented)- protected _teardown(): void;toJSON(): StatisticPersistedState;}@@ -1968,7 +1851,6 @@ export class SystemStatus {constructor(options?: SystemStatusOptions);getCurrentStatus(): SystemInfo;getHistoricalStatus(): SystemInfo;- protected _isSystemIdle(sampleDurationMillis?: number): SystemInfo;}// @public (undocumented)diff --git a/docs/public-api/crawlee-http-client.api.md b/docs/public-api/crawlee-http-client.api.mdindex 98c0a2e64..761c08f20 100644--- a/docs/public-api/crawlee-http-client.api.md+++ b/docs/public-api/crawlee-http-client.api.md@@ -16,8 +16,6 @@ export abstract class BaseHttpClient implements BaseHttpClient_2 {logger?: CrawleeLogger;});protected abstract fetch(input: Request, init?: RequestInit & CustomFetchOptions): Promise<Response>;- // (undocumented)- protected log?: CrawleeLogger;sendRequest(initialRequest: Request, options?: SendRequestOptions): Promise<Response>;}diff --git a/docs/public-api/crawlee-http.api.md b/docs/public-api/crawlee-http.api.mdindex 8c8775db4..90843c2cd 100644--- a/docs/public-api/crawlee-http.api.md+++ b/docs/public-api/crawlee-http.api.md@@ -4,7 +4,6 @@```ts-import { AllowedHttpMethods } from '@crawlee/types';import { AnyPredicate } from 'ow';import { ArrayPredicate } from 'ow';import type { Awaitable } from '@crawlee/types';@@ -19,7 +18,6 @@ import type { CrawlingContext as CrawlingContext_2 } from '@crawlee/core';import type { Dictionary } from '@crawlee/types';import { ErrorHandler } from '@crawlee/basic';import { GetUserDataFromRequest } from '@crawlee/basic';-import type { ISession } from '@crawlee/types';import type { JsonValue } from 'type-fest';import { LoadedRequest } from '@crawlee/core';import { NumberPredicate } from 'ow';@@ -28,9 +26,7 @@ import { Predicate } from 'ow';import { Request as Request_2 } from '@crawlee/basic';import type { Request as Request_3 } from '@crawlee/core';import { RequestHandler } from '@crawlee/basic';-import type { RequestLike } from 'content-type';import type { RequireContextPipeline } from '@crawlee/basic';-import type { ResponseLike } from 'content-type';import { ResponseWithUrl } from '@crawlee/http-client';import { RouterHandler } from '@crawlee/basic';import { RouterRoutes } from '@crawlee/basic';@@ -101,33 +97,8 @@ export class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any> =// (undocumented)protected buildContextPipeline(): ContextPipeline<CrawlingContext, InternalHttpCrawlingContext>;// (undocumented)- protected _encodeResponse(request: Request_2, response: Response, encoding: BufferEncoding): {- encoding: BufferEncoding;- response: Response;- };- protected _extendSupportedMimeTypes(additionalMimeTypes: (string | RequestLike | ResponseLike)[]): void;- // (undocumented)- protected forceResponseEncoding?: string;- protected _getRequestOptions(request: Request_2, session: ISession, proxyUrl?: string): {- url: string;- method: AllowedHttpMethods;- proxyUrl: string | undefined;- timeout: number;- sessionToken: ISession;- headers: Record<string, string> | undefined;- https: {- rejectUnauthorized: boolean;- };- body: string | undefined;- };- protected _handleRequestTimeout(session: ISession): void;- // (undocumented)- protected ignoreSslErrors: boolean;- // (undocumented)protected isRequestBlocked(crawlingContext: InternalHttpCrawlingContext): Promise<string | false>;// (undocumented)- protected navigationTimeoutMillis: number;- // (undocumented)protected static optionsShape: {navigationTimeoutSecs: NumberPredicate & BasePredicate<number | undefined>;ignoreSslErrors: BooleanPredicate & BasePredicate<boolean | undefined>;@@ -172,32 +143,6 @@ export class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any> =statisticsOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;id: StringPredicate & BasePredicate<string | undefined>;};- protected _parseResponse(request: Request_2, response: Response): Promise<{- response: Response;- contentType: {- type: string;- encoding: BufferEncoding;- };- body: string;- } | {- body: Buffer<ArrayBuffer>;- response: Response;- contentType: {- type: string;- encoding: BufferEncoding;- };- }>;- // (undocumented)- protected postNavigationHooks: ((crawlingContext: CrawlingContextWithResponse) => Awaitable<void | Partial<CrawlingContextWithResponse>>)[];- // (undocumented)- protected preNavigationHooks: InternalHttpHook<CrawlingContext>[];- protected _requestFunction(input: RequestFunctionOptions): Promise<Response>;- // (undocumented)- protected saveResponseCookies: boolean;- // (undocumented)- protected suggestResponseEncoding?: string;- // (undocumented)- protected readonly supportedMimeTypes: Set<string>;}// @public (undocumented)diff --git a/docs/public-api/crawlee-jsdom.api.md b/docs/public-api/crawlee-jsdom.api.mdindex c3584fdb8..27dcf81e8 100644--- a/docs/public-api/crawlee-jsdom.api.md+++ b/docs/public-api/crawlee-jsdom.api.md@@ -60,8 +60,6 @@ export class JSDOMCrawler<ContextExtension = Dictionary<never>, ExtendedContext}>;getVirtualConsole(): VirtualConsole;// (undocumented)- protected hideInternalConsole: boolean;- // (undocumented)protected static optionsShape: {runScripts: BooleanPredicate & BasePredicate<boolean | undefined>;hideInternalConsole: BooleanPredicate & BasePredicate<boolean | undefined>;@@ -108,10 +106,6 @@ export class JSDOMCrawler<ContextExtension = Dictionary<never>, ExtendedContextstatisticsOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;id: StringPredicate & BasePredicate<string | undefined>;};- // (undocumented)- protected runScripts: boolean;- // (undocumented)- protected virtualConsole: VirtualConsole | null;}// @public (undocumented)diff --git a/docs/public-api/crawlee-playwright.api.md b/docs/public-api/crawlee-playwright.api.mdindex 1cd0c8163..f0cdc3770 100644--- a/docs/public-api/crawlee-playwright.api.md+++ b/docs/public-api/crawlee-playwright.api.md@@ -53,13 +53,11 @@ import type { RequestHandler } from '@crawlee/browser';import { RequestHandlerResult } from '@crawlee/core';import type { RequestTransform } from '@crawlee/browser';import type { Response as Response_2 } from 'playwright';-import type { RestrictedCrawlingContext } from '@crawlee/core';import type { RouterHandler } from '@crawlee/browser';import type { RouterRoutes } from '@crawlee/browser';import type { RouterRoutes as RouterRoutes_2 } from '@crawlee/core';import type { RouteSchemas } from '@crawlee/browser';import type { RoutesFromSchemas } from '@crawlee/browser';-import type { SetRequired } from 'type-fest';import type { SkippedRequestCallback } from '@crawlee/browser';import { Statistics } from '@crawlee/core';import type { StatisticsOptions } from '@crawlee/core';@@ -70,8 +68,6 @@ import { StringPredicate } from 'ow';export class AdaptivePlaywrightCrawler<ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext> extends BasicCrawler<AdaptivePlaywrightCrawlerContext, ExtendedContext> {constructor(options?: AdaptivePlaywrightCrawlerOptions<ExtendedContext>);// (undocumented)- protected allowStorageAccess<R, TArgs extends any[]>(func: (...args: TArgs) => Promise<R>): (...args: TArgs) => Promise<R>;- // (undocumented)protected buildContextPipeline(): ContextPipeline_2<CrawlingContext_2<Dictionary_2>, CrawlingContext_2<Dictionary_2> & {readonly request: LoadedRequest<Request_3<Dictionary_2>>;readonly response: Response;@@ -80,10 +76,6 @@ export class AdaptivePlaywrightCrawler<ExtendedContext extends AdaptivePlaywrighreadonly waitForSelector: AdaptivePlaywrightCrawlerContext["waitForSelector"];readonly parseWithCheerio: AdaptivePlaywrightCrawlerContext["parseWithCheerio"];}>;- // (undocumented)- protected commitResult(crawlingContext: CrawlingContext_2, input: RequestHandlerResult): Promise<void>;- // (undocumented)- protected enqueueLinks(options: SetRequired<EnqueueLinksOptions, 'urls'>, request: RestrictedCrawlingContext['request'], result: RequestHandlerResult): Promise<BatchAddRequestsResult>;protected getPendingRequestCountApproximation(): Promise<number>;// (undocumented)protected _init(): Promise<void>;@@ -420,15 +412,11 @@ export type RenderingType = 'clientOnly' | 'static';// @publicexport class RenderingTypePredictor {constructor(input: RenderingTypePredictorOptions);- // (undocumented)- protected calculateFeatureVector(url: URLComponents, label: string | undefined): FeatureVector;initialize(): Promise<void>;predict(input: Request_2): {renderingType: RenderingType;detectionProbabilityRecommendation: number;};- // (undocumented)- protected retrain(): void;storeResult(requests: Request_2 | Request_2[], renderingType: RenderingType): void;}diff --git a/docs/public-api/crawlee-puppeteer.api.md b/docs/public-api/crawlee-puppeteer.api.mdindex bc6e01783..8b5463033 100644--- a/docs/public-api/crawlee-puppeteer.api.md+++ b/docs/public-api/crawlee-puppeteer.api.md@@ -287,7 +287,7 @@ export class PuppeteerLauncher extends BrowserLauncher<PuppeteerPlugin, unknown>// (undocumented)readonly config: Configuration;// (undocumented)- protected _getDefaultHeadlessOption(): boolean;+ protected getDefaultHeadlessOption(): boolean;// (undocumented)protected static optionsShape: {launcher: ObjectPredicate<object> & BasePredicate<object | undefined>;diff --git a/docs/public-api/crawlee-utils.api.md b/docs/public-api/crawlee-utils.api.mdindex 9e7e450f9..3a2b79685 100644--- a/docs/public-api/crawlee-utils.api.md+++ b/docs/public-api/crawlee-utils.api.md@@ -195,14 +195,6 @@ class RobotsTxtFile {static from(url: string, content: string, proxyUrl?: string): RobotsTxtFile;getSitemaps(): string[];isAllowed(url: string, userAgent?: string): boolean;- // (undocumented)- protected static load(url: string, options?: {- signal?: AbortSignal;- timeoutMillis?: number;- proxyUrl?: string;- httpClient?: BaseHttpClient;- logger?: CrawleeLogger;- }): Promise<RobotsTxtFile>;parseSitemaps(): Promise<Sitemap>;parseUrlsFromSitemaps(): Promise<string[]>;}@@ -217,8 +209,6 @@ export class Sitemap {constructor(urls: string[]);static fromXmlString(content: string, proxyUrl?: string, parseSitemapOptions?: ParseSitemapOptions): Promise<Sitemap>;static load(urls: string | string[], proxyUrl?: string, parseSitemapOptions?: ParseSitemapOptions): Promise<Sitemap>;- // (undocumented)- protected static parse(sources: SitemapSource[], proxyUrl?: string, parseSitemapOptions?: ParseSitemapOptions): Promise<Sitemap>;static tryCommonNames(url: string, proxyUrl?: string, parseSitemapOptions?: ParseSitemapOptions): Promise<Sitemap>;// (undocumented)readonly urls: string[];diff --git a/docs/upgrading/upgrading_v4.md b/docs/upgrading/upgrading_v4.mdindex 685b57196..6b9d24969 100644--- a/docs/upgrading/upgrading_v4.md+++ b/docs/upgrading/upgrading_v4.md@@ -34,8 +34,47 @@ The crawler following options are removed:## Underscore prefix is removed from many protected and private methods+The leading underscore was dropped from protected and private class members across the codebase. The most visible rename is:+- `BasicCrawler._runRequestHandler` -> `BasicCrawler.runRequestHandler`+The file-system storage backends' shared `CachedIdClient._cachedId` protected field was also renamed to `cachedId` (this only affects custom `@crawlee/fs-storage` backends that subclass it).++Members that were also made `private` in the same pass are listed under [Unintentionally exposed internals are now private](#unintentionally-exposed-internals-are-now-private) below.++## Unintentionally exposed internals are now private++A number of class members were `public` or `protected` only by accident — they were never meant to be part of the extension surface, are not used by any subclass, and in most cases also carried a leading underscore to signal that. In v4 they are `private` (and where it applies, `readonly`). If you were reaching into any of these — either to read internal state or to override a helper in a subclass — that no longer compiles.++This is intentional: these were never a supported API. If you relied on overriding one of the now-private helpers, the supported extension points (the `requestHandler`, `errorHandler`, `failedRequestHandler`, `preNavigationHooks`/`postNavigationHooks`, `ContextPipeline` composition, and the `ISessionPool` / `IBrowserPool` / `IRequestManager` interfaces) should cover the same use cases. If something you genuinely need is missing, open an issue.++The change spans, among others:++- **`BasicCrawler`** — `unexpectedStop`, `requestHandlerTimeoutMillis`, `sameDomainDelayMillis`, `domainAccessedTime`, `handledRequestsCount`, `statusMessageLoggingInterval`, `statusMessageCallback`, `ignoreHttpErrorStatusCodes`, `autoscaledPoolOptions`, `respectRobotsTxtFile`, and the helpers `buildBasicContextPipeline`, `validateRequestUserData`, `pauseOnMigration`, `fetchNextRequest`, `delayRequest`, `handleRequest`, `timeoutAndRetry`, `isTaskReadyFunction`, `defaultIsFinishedFunction`, `requestFunctionErrorHandler`, `handleFailedRequestHandler`, `canRequestBeRetried`+- **`HttpCrawler`** — `preNavigationHooks`, `postNavigationHooks`, `saveResponseCookies`, `navigationTimeoutMillis`, `ignoreSslErrors`, `suggestResponseEncoding`, `forceResponseEncoding`, `supportedMimeTypes`, and the helpers `requestFunction`, `parseResponse`, `getRequestOptions`, `encodeResponse`, `extendSupportedMimeTypes`, `handleRequestTimeout`+- **`AutoscaledPool`** — `autoscale`, `maybeRunTask`, `incrementTasksDonePerSecond`, `maybeFinish`, `destroy`, `scaleUp`, `scaleDown`, `isOverMaxRequestLimit`+- **`SessionPool`** — all pool internals (`log`, `maxPoolSize`, `createSessionFunction`, `keyValueStore`, `sessions`, `sessionMap`, `sessionOptions`, `persistStateKey`, `persistStateKeyValueStoreId`, `events`, `persistenceOptions`, `sessionReuseStrategy`, and the helpers `ensureInitialized`, `maybeLoadSessionPool`, `registerSession`, `createSession`, `hasSpaceForSession`, `pickSession`, `removeRetiredSessions`, `getRandomIndex`, `defaultCreateSessionFunction`)+- **`Session`** — `maybeSelfRetire` (`userData` is now `readonly`)+- **`RequestList`** — all `_`-prefixed helpers (`addFetchedRequests`, `addPersistedRequests`, `addRequest`, `addRequestsFromSources`, `ensureInProgress`, `ensureIsInitialized`, `ensureUniqueKeyValid`, `fetchRequestsFromUrl`, `getPersistedState`, `loadStateAndPersistedRequests`, `persistRequests`, `restoreState`)+- **`RequestQueue`** — `proxyConfiguration`, `requestCache`, `requestSeenCache`, `queuePausedForMigration`, `inProgressRequestBatchCount`, `expectedRequestProcessingSecs`, `httpClient`, `events`, and the helpers `cacheRequest`, `fetchRequestsFromUrl`, `addFetchedRequests` (`id`, `name`, `backend`, `log` are now `readonly`)+- **`ProxyConfiguration`** — `nextCustomUrlIndex`, `proxyUrls`, `newUrlFunction`, and the helpers `handleProxyUrlsList`, `callNewUrlFunction`, `throwCannotCombineCustomMethods`, `throwNoOptionsProvided` (the internal `log` field and `usedProxyUrls` map are removed; `isManInTheMiddle` is now `readonly`)+- **`Statistics`** — `saveRetryCountForJob`, `teardown` (`errorTracker`, `errorTrackerRetry` are now `readonly`)+- **`SystemStatus`** — `isSystemIdle`+- **`Router`** — the constructor is now `private`; use the static `Router.create()` factory+- **`BaseHttpClient`** — `log` (subclasses receive it via the constructor `logger` option instead of reading `this.log`)+- **`JSDOMCrawler`** — `runScripts`, `hideInternalConsole`, `virtualConsole`+- **`AdaptivePlaywrightCrawler`** — `commitResult`, `allowStorageAccess`, `enqueueLinks`+- **`RenderingTypePredictor`** — `calculateFeatureVector`, `retrain`+- **`BrowserCrawler`** — `navigationTimeoutMillis`, `preNavigationHooks`, `postNavigationHooks`, `saveResponseCookies` (now `private readonly`; configure them through the constructor options as before), and the helpers `isRequestBlocked`, `applyCookies` (was `_applyCookies`), `handleNavigationTimeout` (was `_handleNavigationTimeout`), `throwIfProxyError` (was `_throwIfProxyError`)+- **`BrowserLauncher`** — the helpers `getChromeExecutablePath`, `getTypicalChromeExecutablePath`, `validateProxyUrlProtocol` (were `_`-prefixed). `getDefaultHeadlessOption` (was `_getDefaultHeadlessOption`) stays `protected` — it is an override point (`PuppeteerLauncher` overrides it) — but lost its underscore prefix+- **`Statistics.maybeLoadStatistics`** (was `_maybeLoadStatistics`) — stays `protected` (it is overridden by `AdaptivePlaywrightCrawler`'s statistics), but lost its underscore prefix; rename any `super._maybeLoadStatistics()` overrides accordingly+- **`RobotsTxtFile.load` and `Sitemap.parse`** — internal static factory helpers, now `private` (use the public `RobotsTxtFile.from` / `Sitemap.load` / `Sitemap.fromXmlString` entry points)+- Various internal fields on `BrowserController` (`id`, `browserPlugin`, `log`) and `BrowserPlugin` (`name`, `library`, `launchOptions`, `proxyUrl`, `userDataDir`, `browserPerProxy`, `ignoreProxyCertificate`, `log`) are now `readonly`++## The `RequestQueue` constructor no longer takes a `Configuration`++The internal `RequestQueue` constructor dropped its second `config: Configuration` parameter (it also stopped exposing a `protected config` field). You should not be constructing `RequestQueue` directly anyway — use `RequestQueue.open()`, which resolves configuration for you.+## Removed symbols- `BasicCrawler._cleanupContext` (protected) - this is now handled by the `ContextPipeline`@@ -50,8 +89,11 @@ The crawler following options are removed:- `HttpCrawler._handleNavigation` (protected)- `HttpCrawler._applyCookies` (protected) - cookie merging is now handled by `BaseHttpClient`- `HttpCrawler._parseHTML` (protected)-- `HttpCrawler._parseResponse` (protected) - made private- `HttpCrawler.use` and the `CrawlerExtension` class (experimental) - the `ContextPipeline` should be used for extending the crawler+- `BasicCrawler._tagUserHandlerError` (protected) - internal error-tagging helper, no longer part of the crawler surface+- `BasicCrawler.handledRequestsCount` setter (`@deprecated`) - the throw-on-assign guard is gone; the getter is now internal-only and the count is derived from `this.stats`+- `PlaywrightPlugin._containerProxyServer` (public) - was an unused, never-populated field+- `Snapshotter._snapshotMemory`, `Snapshotter._memoryOverloadWarning`, `Snapshotter._snapshotEventLoop`, `Snapshotter._snapshotCpu`, `Snapshotter._snapshotClient`, `Snapshotter._pruneSnapshots` (all `@deprecated` protected stubs) - snapshotting is handled by the individual load signals, use `Snapshotter.getMemorySample()` / `getEventLoopSample()` / `getCpuSample()` / `getClientSample()` instead- `FileDownloadOptions.streamHandler` - streaming should now be handled directly in the `requestHandler` instead- `playwrightUtils.registerUtilsToContext` and `puppeteerUtils.registerUtilsToContext` - this is now added to the context via `ContextPipeline` composition- `puppeteerUtils.blockResources` and `puppeteerUtils.cacheResponses` (deprecated)diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex 1ce8f4453..726c743a7 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -594,7 +594,7 @@ export class BasicCrawler<* A reference to the underlying session pool that manages the crawler's {@apilink Session|sessions}. Typed as* {@apilink ISessionPool} so custom implementations can be plugged in via the `sessionPool` constructor option.*/- sessionPool: ISessionPool;+ readonly sessionPool: ISessionPool;/*** Set when the crawler constructed its own {@apilink SessionPool} (no `sessionPool` option was provided).@@ -631,7 +631,7 @@ export class BasicCrawler<* A reference to the underlying {@apilink IProxyConfiguration} instance that manages the crawler's proxies.* Only available if used by the crawler.*/- proxyConfiguration?: IProxyConfiguration;+ readonly proxyConfiguration?: IProxyConfiguration;/*** Default {@apilink Router} instance that will be used if we don't specify any {@apilink BasicCrawlerOptions.requestHandler|`requestHandler`}.@@ -669,7 +669,7 @@ export class BasicCrawler<running = false;hasFinishedBefore = false;- protected unexpectedStop = false;+ private unexpectedStop = false;#log!: CrawleeLogger;@@ -677,39 +677,31 @@ export class BasicCrawler<return this.#log;}- protected requestHandler!: RequestHandler<ExtendedContext>;- protected errorHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;- protected failedRequestHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;- protected requestHandlerTimeoutMillis!: number;- protected internalTimeoutMillis: number;- protected maxRequestRetries: number;- protected maxCrawlDepth?: number;- protected sameDomainDelayMillis: number;- protected domainAccessedTime: Map<string, number>;- protected maxRequestsPerCrawl?: number;-- protected get handledRequestsCount(): number {+ protected readonly requestHandler!: RequestHandler<ExtendedContext>;+ protected readonly errorHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;+ protected readonly failedRequestHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;+ private requestHandlerTimeoutMillis!: number;+ protected readonly internalTimeoutMillis: number;+ protected readonly maxRequestRetries: number;+ protected readonly maxCrawlDepth?: number;+ private sameDomainDelayMillis: number;+ private domainAccessedTime: Map<string, number>;+ protected readonly maxRequestsPerCrawl?: number;++ private get handledRequestsCount(): number {return this.stats.state.requestsFinished + this.stats.state.requestsFailed;}- /** @deprecated Setting `handledRequestsCount` directly is no longer supported. The count is now derived from `this.stats`. */- protected set handledRequestsCount(_value: number) {- throw new Error(- 'Setting `handledRequestsCount` directly is no longer supported. ' +- 'The count is now derived from `this.stats.state.requestsFinished` and `this.stats.state.requestsFailed`.',- );- }-- protected statusMessageLoggingInterval: number;- protected statusMessageCallback?: StatusMessageCallback;+ private statusMessageLoggingInterval: number;+ private statusMessageCallback?: StatusMessageCallback;protected blockedStatusCodes = new Set<number>();- protected additionalHttpErrorStatusCodes: Set<number>;- protected ignoreHttpErrorStatusCodes: Set<number>;- protected autoscaledPoolOptions: AutoscaledPoolOptions;- protected httpClient: BaseHttpClient;- protected retryOnBlocked: boolean;- protected respectRobotsTxtFile: boolean | { userAgent?: string };- protected onSkippedRequest?: SkippedRequestCallback;+ protected readonly additionalHttpErrorStatusCodes: Set<number>;+ private ignoreHttpErrorStatusCodes: Set<number>;+ private autoscaledPoolOptions: AutoscaledPoolOptions;+ protected readonly httpClient: BaseHttpClient;+ protected readonly retryOnBlocked: boolean;+ private respectRobotsTxtFile: boolean | { userAgent?: string };+ protected readonly onSkippedRequest?: SkippedRequestCallback;private _ownsEventManager = false;private loggedPerRun = new Set<string>();private readonly robotsTxtFileCache: LruCache<RobotsTxtFile>;@@ -992,7 +984,7 @@ export class BasicCrawler<// (e.g., doesn't match enqueue strategy after redirect). Just return gracefully.if (error instanceof ContextPipelineInterruptedError) {this.stats.discardJob(request.id || request.uniqueKey);- await this._timeoutAndRetry(+ await this.timeoutAndRetry(async () => this.requestManager?.markRequestAsHandled(request),this.internalTimeoutMillis,`Marking request ${crawlingContext.request.url} (${crawlingContext.request.id}) as handled timed out after ${@@ -1009,13 +1001,13 @@ export class BasicCrawler<if (isPipelineError) {const unwrappedError = this.unwrapError(error);- await this._requestFunctionErrorHandler(+ await this.requestFunctionErrorHandler(unwrappedError,crawlingContext as CrawlingContext,request,this.requestManager!,);- // SessionError already retired the session in `_requestFunctionErrorHandler`;+ // SessionError already retired the session in `requestFunctionErrorHandler`;// skip `markBad` to avoid double-counting usage/error score.if (!(unwrappedError instanceof SessionError)) {crawlingContext.session?.markBad();@@ -1057,7 +1049,7 @@ export class BasicCrawler<return false;}- return isTaskReadyFunction ? await isTaskReadyFunction() : await this._isTaskReadyFunction();+ return isTaskReadyFunction ? await isTaskReadyFunction() : await this.isTaskReadyFunction();},isFinishedFunction: async () => {if (isMaxPagesExceeded()) {@@ -1078,7 +1070,7 @@ export class BasicCrawler<const isFinished = isFinishedFunction? await isFinishedFunction()- : await this._defaultIsFinishedFunction();+ : await this.defaultIsFinishedFunction();if (isFinished) {const reason = isFinishedFunction@@ -1115,7 +1107,7 @@ export class BasicCrawler<* Builds the basic context pipeline that transforms `{ request }` into a full `CrawlingContext`.* This handles base context creation, session resolution, and context helpers.*/- protected buildBasicContextPipeline(): ContextPipeline<{ request: Request }, CrawlingContext> {+ private buildBasicContextPipeline(): ContextPipeline<{ request: Request }, CrawlingContext> {return ContextPipeline.create<{ request: Request }>().compose({ action: this.checkRobotsTxt.bind(this) }).compose({ action: () => this.createBaseContext() })@@ -1166,8 +1158,8 @@ export class BasicCrawler<}private async resolveRequest(): Promise<Request | null> {- const request = await this._timeoutAndRetry(- this._fetchNextRequest.bind(this),+ const request = await this.timeoutAndRetry(+ this.fetchNextRequest.bind(this),this.internalTimeoutMillis,`Fetching next request timed out after ${this.internalTimeoutMillis / 1e3} seconds.`,);@@ -1181,7 +1173,7 @@ export class BasicCrawler<}private async resolveSession({ request }: { request: Request }) {- const session = await this._timeoutAndRetry(+ const session = await this.timeoutAndRetry(async () => {const existingSession = await this.sessionPool.getSession(request.sessionId);@@ -1396,12 +1388,12 @@ export class BasicCrawler<this.log.warning('Pausing... Press CTRL+C again to force exit. To resume, do: CRAWLEE_PURGE_ON_START=0 npm start',);- await this._pauseOnMigration();+ await this.pauseOnMigration();await this.autoscaledPool!.abort();};// Attach a listener to handle migration and aborting events gracefully.- const boundPauseOnMigration = this._pauseOnMigration.bind(this);+ const boundPauseOnMigration = this.pauseOnMigration.bind(this);process.once('SIGINT', sigintHandler);const eventManager = serviceLocator.getEventManager();eventManager.on(EventType.MIGRATING, boundPauseOnMigration);@@ -1545,7 +1537,7 @@ export class BasicCrawler<* the request's label. Applied by the crawler on the add paths it owns — `crawler.addRequests`,* `crawler.run`, `context.addRequests` and `context.enqueueLinks`.*/- protected async validateRequestUserData(source: Source | string): Promise<void> {+ private async validateRequestUserData(source: Source | string): Promise<void> {if (typeof source === 'string') {return;}@@ -1880,7 +1872,7 @@ export class BasicCrawler<}}- protected async _pauseOnMigration() {+ private async pauseOnMigration() {if (this.autoscaledPool) {// if run wasn't called, this is going to crashawait this.autoscaledPool.pause(SAFE_MIGRATION_WAIT_MILLIS).catch((err) => {@@ -1924,9 +1916,9 @@ export class BasicCrawler</*** Fetches the next request to process from the underlying request provider.*/- protected async _fetchNextRequest() {+ private async fetchNextRequest() {if (this.requestManager === undefined) {- throw new Error(`_fetchNextRequest called on an uninitialized crawler`);+ throw new Error(`fetchNextRequest called on an uninitialized crawler`);}return this.requestManager.fetchNextRequest();@@ -1937,7 +1929,7 @@ export class BasicCrawler<* adding it back to the queue after the timeout passes. Returns `true` if the request* should be ignored and will be reclaimed to the queue once ready.*/- protected delayRequest(request: Request, source: IRequestManager) {+ private delayRequest(request: Request, source: IRequestManager) {const domain = getDomain(request.url);if (!domain || !request) {@@ -1966,7 +1958,7 @@ export class BasicCrawler<}/** Handles a single request - runs the request handler with retries, error handling, and lifecycle management. */- protected async handleRequest(crawlingContext: ExtendedContext, requestSource: IRequestManager, request: Request) {+ private async handleRequest(crawlingContext: ExtendedContext, requestSource: IRequestManager, request: Request) {const statisticsId = request.id || request.uniqueKey;let isRequestLocked = true;@@ -1975,7 +1967,7 @@ export class BasicCrawler<request.state = RequestState.REQUEST_HANDLER;await this.runRequestHandler(crawlingContext);- await this._timeoutAndRetry(+ await this.timeoutAndRetry(async () => requestSource.markRequestAsHandled(request!),this.internalTimeoutMillis,`Marking request ${request.url} (${request.id}) as handled timed out after ${@@ -1995,14 +1987,14 @@ export class BasicCrawler<try {request.state = RequestState.ERROR_HANDLER;await addTimeoutToPromise(- async () => this._requestFunctionErrorHandler(err, crawlingContext, request, requestSource),+ async () => this.requestFunctionErrorHandler(err, crawlingContext, request, requestSource),this.internalTimeoutMillis,`Handling request failure of ${request.url} (${request.id}) timed out after ${this.internalTimeoutMillis / 1e3} seconds.`,);if (!(err instanceof CriticalError)) {- isRequestLocked = false; // _requestFunctionErrorHandler calls either markRequestAsHandled or reclaimRequest+ isRequestLocked = false; // requestFunctionErrorHandler calls either markRequestAsHandled or reclaimRequest}request.state = RequestState.DONE;} catch (secondaryError: any) {@@ -2136,7 +2128,7 @@ export class BasicCrawler<* Run async callback with given timeout and retry. Returns the result of the callback.* @ignore*/- protected async _timeoutAndRetry<T>(+ private async timeoutAndRetry<T>(handler: () => Promise<T>,timeout: number,error: Error | string,@@ -2149,7 +2141,7 @@ export class BasicCrawler<if (retried <= maxRetries) {// we retry on any error, not just timeoutthis.log.warning(`${(e as Error).message} (retrying ${retried}/${maxRetries})`);- return this._timeoutAndRetry(handler, timeout, error, maxRetries, retried + 1);+ return this.timeoutAndRetry(handler, timeout, error, maxRetries, retried + 1);}throw e;@@ -2159,14 +2151,14 @@ export class BasicCrawler</*** Returns true if either RequestList or RequestQueue have a request ready for processing.*/- protected async _isTaskReadyFunction() {+ private async isTaskReadyFunction() {return this.requestManager !== undefined && !(await this.requestManager.isEmpty());}/*** Returns true if both RequestList and RequestQueue have all requests finished.*/- protected async _defaultIsFinishedFunction() {+ private async defaultIsFinishedFunction() {return !this.requestManager || (await this.requestManager.isFinished());}@@ -2190,7 +2182,7 @@ export class BasicCrawler<** @param request The request object, passed separately to circumvent potential dynamic logic in crawlingContext.request*/- protected async _requestFunctionErrorHandler(+ private async requestFunctionErrorHandler(error: Error,crawlingContext: CrawlingContext,request: Request,@@ -2202,7 +2194,7 @@ export class BasicCrawler<throw error;}- const shouldRetryRequest = this._canRequestBeRetried(request, error);+ const shouldRetryRequest = this.canRequestBeRetried(request, error);if (shouldRetryRequest) {await this.stats.errorTrackerRetry.addAsync(error, crawlingContext);@@ -2254,19 +2246,10 @@ export class BasicCrawler<await source.markRequestAsHandled(request);this.stats.failJob(request.id || request.uniqueKey, request.retryCount);- await this._handleFailedRequestHandler(crawlingContext, error); // This function prints an error message.- }-- protected async _tagUserHandlerError<T>(cb: () => unknown): Promise<T> {- try {- return (await cb()) as T;- } catch (e: any) {- Object.defineProperty(e, 'triggeredFromUserHandler', { value: true });- throw e;- }+ await this.handleFailedRequestHandler(crawlingContext, error); // This function prints an error message.}- protected async _handleFailedRequestHandler(crawlingContext: CrawlingContext, error: Error): Promise<void> {+ private async handleFailedRequestHandler(crawlingContext: CrawlingContext, error: Error): Promise<void> {// Always log the last error regardless if the user provided a failedRequestHandlerconst { id, url, method, uniqueKey } = crawlingContext.request;const message = this._getMessageFromError(error, true);@@ -2305,7 +2288,7 @@ export class BasicCrawler<: [error.message || error, userLine].join('\n');}- protected _canRequestBeRetried(request: Request, error: Error) {+ private canRequestBeRetried(request: Request, error: Error) {// Request should never be retried, or the error encountered makes it not able to be retried.if (request.noRetry || error instanceof NonRetryableError) {return false;diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.tsindex 3539c9847..82a1450a7 100644--- a/packages/browser-crawler/src/internals/browser-crawler.ts+++ b/packages/browser-crawler/src/internals/browser-crawler.ts@@ -347,10 +347,10 @@ export abstract class BrowserCrawler<protected readonly ignoreShadowRoots: boolean;protected readonly ignoreIframes: boolean;- protected navigationTimeoutMillis: number;- protected preNavigationHooks: BrowserHook<Context>[];- protected postNavigationHooks: BrowserHook<Context>[];- protected saveResponseCookies: boolean;+ private readonly navigationTimeoutMillis: number;+ private readonly preNavigationHooks: BrowserHook<Context>[];+ private readonly postNavigationHooks: BrowserHook<Context>[];+ private readonly saveResponseCookies: boolean;protected static override optionsShape = {...BasicCrawler.optionsShape,@@ -507,7 +507,7 @@ export abstract class BrowserCrawler<return foundSelectors.length > 0 ? foundSelectors : null;}- protected async isRequestBlocked(crawlingContext: BrowserCrawlingContext<Page, Response>): Promise<string | false> {+ private async isRequestBlocked(crawlingContext: BrowserCrawlingContext<Page, Response>): Promise<string | false> {const { page, response } = crawlingContext;// Cloudflare specific heuristic - wait 5 seconds if we get a 403 for the JS challenge to load / resolve.@@ -604,15 +604,15 @@ export abstract class BrowserCrawler<const cookiesBeforeHooks = readContextField<string>(crawlingContext, COOKIES_BEFORE_HOOKS);const cookiesAfterHooks = this._getCookieHeaderFromRequest(crawlingContext.request);- await this._applyCookies(crawlingContext, cookiesBeforeHooks, cookiesAfterHooks);+ await this.applyCookies(crawlingContext, cookiesBeforeHooks, cookiesAfterHooks);let response: Response | undefined;try {response = (await this._navigationHandler(crawlingContext, gotoOptions)) ?? undefined;} catch (error) {- await this._handleNavigationTimeout(crawlingContext, error as Error);+ await this.handleNavigationTimeout(crawlingContext, error as Error);crawlingContext.request.state = RequestState.ERROR;- this._throwIfProxyError(error as Error);+ this.throwIfProxyError(error as Error);throw error;}tryCancel();@@ -669,7 +669,7 @@ export abstract class BrowserCrawler<return {};}- protected async _applyCookies(+ private async applyCookies({ session, request, page }: BrowserCrawlingContext<Page, Response>,preHooksCookies: string,postHooksCookies: string,@@ -688,7 +688,7 @@ export abstract class BrowserCrawler</*** Marks session bad on navigation timeout, and stops in-flight page loading on any navigation error.*/- protected async _handleNavigationTimeout(crawlingContext: BrowserCrawlingContext, error: Error): Promise<void> {+ private async handleNavigationTimeout(crawlingContext: BrowserCrawlingContext, error: Error): Promise<void> {const { session, page } = crawlingContext;if (error?.constructor.name === 'TimeoutError') {@@ -703,7 +703,7 @@ export abstract class BrowserCrawler</*** Transforms proxy-related errors to `SessionError`.*/- protected _throwIfProxyError(error: Error) {+ private throwIfProxyError(error: Error) {if (this.isProxyError(error)) {throw new SessionError(this._getMessageFromError(error) as string);}diff --git a/packages/browser-crawler/src/internals/browser-launcher.ts b/packages/browser-crawler/src/internals/browser-launcher.tsindex bfac15205..09f7c2799 100644--- a/packages/browser-crawler/src/internals/browser-launcher.ts+++ b/packages/browser-crawler/src/internals/browser-launcher.ts@@ -149,7 +149,7 @@ export abstract class BrowserLauncher<...otherLaunchContextProps} = launchContext;- this._validateProxyUrlProtocol(proxyUrl);+ this.validateProxyUrlProtocol(proxyUrl);// those need to be reassigned otherwise they are {} in typesthis.launcher = launcher!;@@ -198,28 +198,28 @@ export abstract class BrowserLauncher<}if (launchOptions.headless == null) {- launchOptions.headless = this._getDefaultHeadlessOption();+ launchOptions.headless = this.getDefaultHeadlessOption();}if (this.useChrome && !launchOptions.executablePath) {- launchOptions.executablePath = this._getChromeExecutablePath();+ launchOptions.executablePath = this.getChromeExecutablePath();}return launchOptions;}- protected _getDefaultHeadlessOption(): boolean {+ protected getDefaultHeadlessOption(): boolean {return this.config.headless && !this.config.xvfb;}- protected _getChromeExecutablePath(): string {- return this.config.chromeExecutablePath ?? this._getTypicalChromeExecutablePath();+ private getChromeExecutablePath(): string {+ return this.config.chromeExecutablePath ?? this.getTypicalChromeExecutablePath();}/*** Gets a typical path to Chrome executable, depending on the current operating system.*/- protected _getTypicalChromeExecutablePath(): string {+ private getTypicalChromeExecutablePath(): string {/*** Returns path of Chrome executable by its OS environment variable to deal with non-english language OS.* Taking also into account the old [chrome 380177 issue](https://bugs.chromium.org/p/chromium/issues/detail?id=380177).@@ -248,7 +248,7 @@ export abstract class BrowserLauncher<}}- protected _validateProxyUrlProtocol(proxyUrl?: string): void {+ private validateProxyUrlProtocol(proxyUrl?: string): void {if (!proxyUrl) return;if (!/^(http|https|socks4|socks5)/i.test(proxyUrl)) {diff --git a/packages/browser-pool/src/abstract-classes/browser-controller.ts b/packages/browser-pool/src/abstract-classes/browser-controller.tsindex f76bfdeb2..011652129 100644--- a/packages/browser-pool/src/abstract-classes/browser-controller.ts+++ b/packages/browser-pool/src/abstract-classes/browser-controller.ts@@ -112,13 +112,13 @@ export abstract class BrowserController<extends TypedEmitter<BrowserControllerEvents<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>>implements IBrowserController<NewPageResult>{- id = nanoid();- protected log!: CrawleeLogger;+ readonly id = nanoid();+ protected readonly log!: CrawleeLogger;/*** The `BrowserPlugin` instance used to launch the browser.*/- browserPlugin: BrowserPlugin<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;+ readonly browserPlugin: BrowserPlugin<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;/*** Browser representation of the underlying automation library.diff --git a/packages/browser-pool/src/abstract-classes/browser-plugin.ts b/packages/browser-pool/src/abstract-classes/browser-plugin.tsindex aa600ab7a..4c30f26c4 100644--- a/packages/browser-pool/src/abstract-classes/browser-plugin.ts+++ b/packages/browser-pool/src/abstract-classes/browser-plugin.ts@@ -108,16 +108,16 @@ export abstract class BrowserPlugin<NewPageOptions = Parameters<LaunchResult['newPage']>[0],NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>,> {- name = this.constructor.name;- protected log!: CrawleeLogger;- library: Library;- launchOptions: LibraryOptions;- proxyUrl?: string;- userDataDir?: string;+ readonly name = this.constructor.name;+ protected readonly log!: CrawleeLogger;+ readonly library: Library;+ readonly launchOptions: LibraryOptions;+ readonly proxyUrl?: string;+ readonly userDataDir?: string;useIncognitoPages: boolean;- browserPerProxy?: boolean;+ readonly browserPerProxy?: boolean;- ignoreProxyCertificate?: boolean;+ readonly ignoreProxyCertificate?: boolean;/*** Set by {@apilink RemoteBrowserPool} when this plugin connects to a remote browser service instead ofdiff --git a/packages/browser-pool/src/playwright/playwright-plugin.ts b/packages/browser-pool/src/playwright/playwright-plugin.tsindex bd71092fc..064d747f7 100644--- a/packages/browser-pool/src/playwright/playwright-plugin.ts+++ b/packages/browser-pool/src/playwright/playwright-plugin.ts@@ -4,7 +4,6 @@ import type { Browser as PlaywrightBrowser, BrowserType } from 'playwright';import { BrowserPlugin } from '../abstract-classes/browser-plugin.js';import { anonymizeProxySugar } from '../anonymize-proxy.js';-import type { createProxyServerForContainers } from '../container-proxy-server.js';import type { LaunchContext } from '../launch-context.js';import { getLocalProxyAddress } from '../proxy-server.js';import type { RemoteConnection, RemoteConnectionParameters } from '../remote-browser-pool.js';@@ -18,7 +17,6 @@ export class PlaywrightPlugin extends BrowserPlugin<PlaywrightBrowser> {private _browserVersion?: string;- _containerProxyServer?: Awaited<ReturnType<typeof createProxyServerForContainers>>;/*** Playwright remote connections only support incognito pages — `connect()` / `connectOverCDP()` don'tdiff --git a/packages/core/src/autoscaling/autoscaled_pool.ts b/packages/core/src/autoscaling/autoscaled_pool.tsindex 1aab92945..3acb0ca04 100644--- a/packages/core/src/autoscaling/autoscaled_pool.ts+++ b/packages/core/src/autoscaling/autoscaled_pool.ts@@ -282,9 +282,9 @@ export class AutoscaledPool {this.isStopped = false;this.resolve = null;this.reject = null;- this._autoscale = this._autoscale.bind(this);- this._maybeRunTask = this._maybeRunTask.bind(this);- this._incrementTasksDonePerSecond = this._incrementTasksDonePerSecond.bind(this);+ this.autoscale = this.autoscale.bind(this);+ this.maybeRunTask = this.maybeRunTask.bind(this);+ this.incrementTasksDonePerSecond = this.incrementTasksDonePerSecond.bind(this);// Create instances with correct options.const ssoCopy = { ...systemStatusOptions };@@ -368,23 +368,23 @@ export class AutoscaledPool {await Promise.all(this.loadSignals.map((s) => s.start()));// This interval checks the system status and updates the desired concurrency accordingly.- this.autoscaleInterval = betterSetInterval(this._autoscale, this.autoscaleIntervalMillis);+ this.autoscaleInterval = betterSetInterval(this.autoscale, this.autoscaleIntervalMillis);// This is here because if we scale down to let's say 1, then after each promise is finished- // this._maybeRunTask() doesn't trigger another one. So if that 1 instance gets stuck it results+ // this.maybeRunTask() doesn't trigger another one. So if that 1 instance gets stuck it results// in the crawler getting stuck and even after scaling up it never triggers another promise.- this.maybeRunInterval = betterSetInterval(this._maybeRunTask, this.maybeRunIntervalMillis);+ this.maybeRunInterval = betterSetInterval(this.maybeRunTask, this.maybeRunIntervalMillis);if (this.maxTasksPerMinute !== Infinity) {// Start the interval that resets the counter of tasks per minute.- this.tasksDonePerSecondInterval = betterSetInterval(this._incrementTasksDonePerSecond, 1000);+ this.tasksDonePerSecondInterval = betterSetInterval(this.incrementTasksDonePerSecond, 1000);}try {await poolPromise;} finally {// If resolve is null, the pool is already destroyed.- if (this.resolve) await this._destroy();+ if (this.resolve) await this.destroy();}}@@ -403,7 +403,7 @@ export class AutoscaledPool {this.isStopped = true;if (this.resolve) {this.resolve();- await this._destroy();+ await this.destroy();}}@@ -462,7 +462,7 @@ export class AutoscaledPool {* every `maybeRunIntervalSecs` seconds. If you want to trigger the processing immediately, use this method.*/async notify(): Promise<void> {- setImmediate(this._maybeRunTask);+ setImmediate(this.maybeRunTask);}/**@@ -473,7 +473,7 @@ export class AutoscaledPool {** It doesn't allow multiple concurrent runs of this method.*/- protected async _maybeRunTask(intervalCallback?: () => void): Promise<void> {+ private async maybeRunTask(intervalCallback?: () => void): Promise<void> {this.log.perf('Attempting to run a task.');// Check if the function was invoked by the maybeRunInterval and use an empty function if not.const done = intervalCallback || (() => {});@@ -526,13 +526,13 @@ export class AutoscaledPool {this.log.perf('Task will not run. No tasks are ready.');done();// No tasks could mean that we're finished with all tasks.- return this._maybeFinish();+ return this.maybeFinish();}// - we have already reached the maximum tasks per minute// we need to check this *after* checking if a task is ready to prevent hanging the pool// for an extra minute if there are no more tasks- if (this._isOverMaxRequestLimit) {+ if (this.isOverMaxRequestLimit) {this.log.perf('Task will not run. Maximum tasks per minute reached.');return done();}@@ -543,7 +543,7 @@ export class AutoscaledPool {this._tasksPerMinute[0]++;// Try to run next task to build up concurrency,// but defer it so it doesn't create a cycle.- setImmediate(this._maybeRunTask);+ setImmediate(this.maybeRunTask);// We need to restart interval here, so that it doesn't get blocked by a stalled task.done();@@ -564,7 +564,7 @@ export class AutoscaledPool {this.log.perf('Task finished.');this._currentConcurrency--;// Run task after the previous one finished.- setImmediate(this._maybeRunTask);+ setImmediate(this.maybeRunTask);} catch (e) {const err = e as Error;this.log.perf('Running a task failed.');@@ -589,12 +589,12 @@ export class AutoscaledPool {* If the system IS NOT overloaded and the settings allow it, it scales up.* If the system IS overloaded and the settings allow it, it scales down.*/- protected _autoscale(intervalCallback: () => void) {+ private autoscale(intervalCallback: () => void) {// Don't scale if paused.if (this.isStopped) return intervalCallback();// Don't scale if we've hit the maximum requests per minute- if (this._isOverMaxRequestLimit) return intervalCallback();+ if (this.isOverMaxRequestLimit) return intervalCallback();// Only scale up if:// - system has not been overloaded lately.@@ -606,7 +606,7 @@ export class AutoscaledPool {const minCurrentConcurrency = Math.floor(this._desiredConcurrency * this.desiredConcurrencyRatio);const weAreReachingDesiredConcurrency = this._currentConcurrency >= minCurrentConcurrency;- if (isSystemIdle && weAreNotAtMax && weAreReachingDesiredConcurrency) this._scaleUp(systemStatus);+ if (isSystemIdle && weAreNotAtMax && weAreReachingDesiredConcurrency) this.scaleUp(systemStatus);// Always scale down if:// - the system has been overloaded lately.@@ -614,7 +614,7 @@ export class AutoscaledPool {// - we're over min concurrency.const weAreNotAtMin = this._desiredConcurrency > this._minConcurrency;- if (isSystemOverloaded && weAreNotAtMin) this._scaleDown(systemStatus);+ if (isSystemOverloaded && weAreNotAtMin) this.scaleDown(systemStatus);// On periodic intervals, print comprehensive log informationif (this.loggingIntervalMillis > 0) {@@ -642,7 +642,7 @@ export class AutoscaledPool {** @param systemStatus for logging*/- protected _scaleUp(systemStatus: SystemInfo): void {+ private scaleUp(systemStatus: SystemInfo): void {const step = Math.ceil(this._desiredConcurrency * this.scaleUpStepRatio);this._desiredConcurrency = Math.min(this._maxConcurrency, this._desiredConcurrency + step);this.log.debug('scaling up', {@@ -658,7 +658,7 @@ export class AutoscaledPool {** @param systemStatus for logging*/- protected _scaleDown(systemStatus: SystemInfo): void {+ private scaleDown(systemStatus: SystemInfo): void {const step = Math.ceil(this._desiredConcurrency * this.scaleDownStepRatio);this._desiredConcurrency = Math.max(this._minConcurrency, this._desiredConcurrency - step);this.log.debug('scaling down', {@@ -674,7 +674,7 @@ export class AutoscaledPool {** It doesn't allow multiple concurrent runs of this method.*/- protected async _maybeFinish(): Promise<void> {+ private async maybeFinish(): Promise<void> {if (this.queryingIsFinished) return;if (this._currentConcurrency > 0) return;@@ -697,7 +697,7 @@ export class AutoscaledPool {/*** Cleans up resources.*/- protected async _destroy(): Promise<void> {+ private async destroy(): Promise<void> {this.resolve = null;this.reject = null;@@ -708,7 +708,7 @@ export class AutoscaledPool {await Promise.all(this.loadSignals.map((s) => s.stop()));}- protected _incrementTasksDonePerSecond(intervalCallback: () => void) {+ private incrementTasksDonePerSecond(intervalCallback: () => void) {this._tasksPerMinute.unshift(0);this._tasksPerMinute.pop();@@ -716,7 +716,7 @@ export class AutoscaledPool {return intervalCallback();}- protected get _isOverMaxRequestLimit() {+ private get isOverMaxRequestLimit() {if (this.maxTasksPerMinute === Infinity) {return false;}diff --git a/packages/core/src/autoscaling/snapshotter.ts b/packages/core/src/autoscaling/snapshotter.tsindex 4e34552ee..1cdc5e77b 100644--- a/packages/core/src/autoscaling/snapshotter.ts+++ b/packages/core/src/autoscaling/snapshotter.ts@@ -13,7 +13,6 @@ import { createEventLoopLoadSignal } from './event_loop_load_signal.js';import type { LoadSignal } from './load_signal.js';import type { MemorySnapshot } from './memory_load_signal.js';import { MemoryLoadSignal } from './memory_load_signal.js';-import type { SystemInfo } from './system_status.js';export interface SnapshotterOptions {/**@@ -94,9 +93,9 @@ export interface SnapshotterOptions {* @category Scaling*/export class Snapshotter {- log: CrawleeLogger;- client: StorageBackend;- config: Configuration;+ readonly log: CrawleeLogger;+ readonly client: StorageBackend;+ readonly config: Configuration;private readonly memorySignal: MemoryLoadSignal;private readonly eventLoopSignal: EventLoopLoadSignal;@@ -246,46 +245,4 @@ export class Snapshotter {getClientSample(sampleDurationMillis?: number): ClientSnapshot[] {return this.clientSignal.getSample(sampleDurationMillis);}-- /**- * @deprecated Kept for backward compatibility.- */- protected _snapshotMemory(systemInfo: SystemInfo) {- this.memorySignal._onSystemInfo(systemInfo);- }-- /**- * @deprecated Kept for backward compatibility.- */- protected _memoryOverloadWarning(systemInfo: SystemInfo) {- this.memorySignal._memoryOverloadWarning(systemInfo);- }-- /**- * @deprecated Kept for backward compatibility.- */- protected _snapshotEventLoop(intervalCallback: () => unknown) {- this.eventLoopSignal.handle(intervalCallback);- }-- /**- * @deprecated Kept for backward compatibility.- */- protected _snapshotCpu(systemInfo: SystemInfo) {- this.cpuSignal.handle(systemInfo);- }-- /**- * @deprecated Kept for backward compatibility.- */- protected _snapshotClient(intervalCallback: () => unknown) {- this.clientSignal.handle(intervalCallback);- }-- /**- * @deprecated Pruning is now handled by individual signals.- */- protected _pruneSnapshots(_snapshots: any[], _now: Date) {- // no-op — signals prune themselves- }}diff --git a/packages/core/src/autoscaling/system_status.ts b/packages/core/src/autoscaling/system_status.tsindex cfbd73259..310663619 100644--- a/packages/core/src/autoscaling/system_status.ts+++ b/packages/core/src/autoscaling/system_status.ts@@ -200,7 +200,7 @@ export class SystemStatus {* and `true` otherwise.*/getCurrentStatus(): SystemInfo {- return this._isSystemIdle(this.currentHistoryMillis);+ return this.isSystemIdle(this.currentHistoryMillis);}/**@@ -220,13 +220,13 @@ export class SystemStatus {* (which is configurable in the {@apilink Snapshotter}) and `true` otherwise.*/getHistoricalStatus(): SystemInfo {- return this._isSystemIdle();+ return this.isSystemIdle();}/*** Returns a system status object.*/- protected _isSystemIdle(sampleDurationMillis?: number): SystemInfo {+ private isSystemIdle(sampleDurationMillis?: number): SystemInfo {const result: SystemInfo = {isSystemIdle: true,memInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },diff --git a/packages/core/src/crawlers/statistics.ts b/packages/core/src/crawlers/statistics.tsindex 17d5ce491..dd115cf9e 100644--- a/packages/core/src/crawlers/statistics.ts+++ b/packages/core/src/crawlers/statistics.ts@@ -60,12 +60,12 @@ export class Statistics {/*** An error tracker for final retry errors.*/- errorTracker: ErrorTracker;+ readonly errorTracker: ErrorTracker;/*** An error tracker for retry errors prior to the final retry.*/- errorTrackerRetry: ErrorTracker;+ readonly errorTrackerRetry: ErrorTracker;/*** Statistic instance id.@@ -83,7 +83,7 @@ export class Statistics {readonly requestRetryHistogram: number[] = [];protected keyValueStore?: KeyValueStore = undefined;- protected persistStateKey: string;+ protected readonly persistStateKey: string;private logIntervalMillis: number;private logMessage: string;private listener: () => Promise<void>;@@ -175,7 +175,7 @@ export class Statistics {this.requestsInProgress.clear();this.instanceStart = Date.now();- this._teardown();+ this.teardown();}/**@@ -227,7 +227,7 @@ export class Statistics {const jobDurationMillis = job.finish();this.state.requestsFinished++;this.state.requestTotalFinishedDurationMillis += jobDurationMillis;- this._saveRetryCountForJob(retryCount);+ this.saveRetryCountForJob(retryCount);if (jobDurationMillis < this.state.requestMinDurationMillis)this.state.requestMinDurationMillis = jobDurationMillis;if (jobDurationMillis > this.state.requestMaxDurationMillis)@@ -244,7 +244,7 @@ export class Statistics {if (!job) return;this.state.requestTotalFailedDurationMillis += job.finish();this.state.requestsFailed++;- this._saveRetryCountForJob(retryCount);+ this.saveRetryCountForJob(retryCount);this.requestsInProgress.delete(id);}@@ -294,7 +294,7 @@ export class Statistics {}if (this.persistenceOptions.enable) {- await this._maybeLoadStatistics();+ await this.maybeLoadStatistics();this.events.on(EventType.PERSIST_STATE, this.listener);}@@ -310,14 +310,14 @@ export class Statistics {* Stops logging and remove event listeners, then persist*/async stopCapturing() {- this._teardown();+ this.teardown();this.state.crawlerFinishedAt = new Date();await this.persistState();}- protected _saveRetryCountForJob(retryCount: number) {+ private saveRetryCountForJob(retryCount: number) {if (retryCount > 0) this.state.requestsRetries++;this.requestRetryHistogram[retryCount] ??= 0;this.requestRetryHistogram[retryCount]++;@@ -349,7 +349,7 @@ export class Statistics {/*** Loads the current statistic from the key value store if any*/- protected async _maybeLoadStatistics() {+ protected async maybeLoadStatistics() {// this might be called before startCapturing was called without using await, should not crashif (!this.keyValueStore) {return;@@ -391,7 +391,7 @@ export class Statistics {this.log.debug('Loaded from KeyValueStore');}- protected _teardown(): void {+ private teardown(): void {// this can be called before a call to startCapturing happens (or in a 'finally' block)// Only unsubscribe if event manager was already resolved — avoid eagerly resolving it// (e.g. during the constructor's reset() call, which would capture the wrong context)diff --git a/packages/core/src/proxy_configuration.ts b/packages/core/src/proxy_configuration.tsindex 83b959388..a336ad5dc 100644--- a/packages/core/src/proxy_configuration.ts+++ b/packages/core/src/proxy_configuration.ts@@ -2,7 +2,6 @@ import type { Dictionary, ProxyInfo } from '@crawlee/types';import ow from 'ow';import type { Request } from './request.js';-import { serviceLocator } from './service_locator.js';export interface ProxyConfigurationFunction {(options?: { request?: Request }): string | null | Promise<string | null>;@@ -78,12 +77,10 @@ export interface IProxyConfiguration {* @category Scaling*/export class ProxyConfiguration implements IProxyConfiguration {- isManInTheMiddle = false;- protected nextCustomUrlIndex = 0;- protected proxyUrls?: UrlList;- protected usedProxyUrls = new Map<string, string | null>();- protected newUrlFunction?: ProxyConfigurationFunction;- protected log = serviceLocator.getLogger().child({ prefix: 'ProxyConfiguration' });+ readonly isManInTheMiddle = false;+ private nextCustomUrlIndex = 0;+ private proxyUrls?: UrlList;+ private newUrlFunction?: ProxyConfigurationFunction;/*** Creates a {@apilink ProxyConfiguration} instance based on the provided options. Proxy servers are used to prevent target websites from@@ -125,8 +122,8 @@ export class ProxyConfiguration implements IProxyConfiguration {const { proxyUrls, newUrlFunction } = options;- if (proxyUrls && newUrlFunction) this._throwCannotCombineCustomMethods();- if (!proxyUrls && !newUrlFunction && validateRequired) this._throwNoOptionsProvided();+ if (proxyUrls && newUrlFunction) this.throwCannotCombineCustomMethods();+ if (!proxyUrls && !newUrlFunction && validateRequired) this.throwNoOptionsProvided();this.proxyUrls = proxyUrls;this.newUrlFunction = newUrlFunction;@@ -164,20 +161,20 @@ export class ProxyConfiguration implements IProxyConfiguration {*/async newUrl(options?: NewUrlOptions): Promise<string | undefined> {if (this.newUrlFunction) {- return (await this._callNewUrlFunction({ request: options?.request })) ?? undefined;+ return (await this.callNewUrlFunction({ request: options?.request })) ?? undefined;}- return this._handleProxyUrlsList() ?? undefined;+ return this.handleProxyUrlsList() ?? undefined;}- protected _handleProxyUrlsList(): string | null {+ private handleProxyUrlsList(): string | null {return this.proxyUrls![this.nextCustomUrlIndex++ % this.proxyUrls!.length];}/*** Calls the custom newUrlFunction and checks format of its return value*/- protected async _callNewUrlFunction(options?: { request?: Request }) {+ private async callNewUrlFunction(options?: { request?: Request }) {const proxyUrl = await this.newUrlFunction!(options);try {if (proxyUrl) {@@ -191,13 +188,13 @@ export class ProxyConfiguration implements IProxyConfiguration {}}- protected _throwCannotCombineCustomMethods(): never {+ private throwCannotCombineCustomMethods(): never {throw new Error('Cannot combine custom proxies "options.proxyUrls" with custom generating function "options.newUrlFunction".',);}- protected _throwNoOptionsProvided(): never {+ private throwNoOptionsProvided(): never {throw new Error('One of "options.proxyUrls" or "options.newUrlFunction" needs to be provided.');}}diff --git a/packages/core/src/router.ts b/packages/core/src/router.tsindex 64626e8a9..91fe8ed3f 100644--- a/packages/core/src/router.ts+++ b/packages/core/src/router.ts@@ -238,7 +238,7 @@ export class Router<* use Router.create() instead!* @ignore*/- protected constructor() {}+ private constructor() {}/*** Registers new route handler for given label. When the router declares a route map, thediff --git a/packages/core/src/session_pool/session.ts b/packages/core/src/session_pool/session.tsindex b44c4b2dc..e5529ba8d 100644--- a/packages/core/src/session_pool/session.ts+++ b/packages/core/src/session_pool/session.ts@@ -85,7 +85,7 @@ export interface SessionOptions {*/export class Session implements ISession {readonly id: string;- userData: Dictionary;+ readonly userData: Dictionary;private _maxErrorScore: number;private _errorScoreDecrement: number;private _createdAt: Date;@@ -258,7 +258,7 @@ export class Session implements ISession {this._errorScore -= this._errorScoreDecrement;}- this._maybeSelfRetire();+ this.maybeSelfRetire();}/**@@ -305,7 +305,7 @@ export class Session implements ISession {this._errorScore += 1;this._usageCount += 1;- this._maybeSelfRetire();+ this.maybeSelfRetire();}/**@@ -332,7 +332,7 @@ export class Session implements ISession {/*** Checks if session is not usable. if it is not retires the session.*/- protected _maybeSelfRetire(): void {+ private maybeSelfRetire(): void {if (!this.isUsable()) {this.retire();}diff --git a/packages/core/src/session_pool/session_pool.ts b/packages/core/src/session_pool/session_pool.tsindex a771e493d..dbc24b52d 100644--- a/packages/core/src/session_pool/session_pool.ts+++ b/packages/core/src/session_pool/session_pool.ts@@ -134,19 +134,19 @@ export class SessionPool implements ISessionPool {private static nextId = 0;readonly id: string;- protected log: CrawleeLogger;- protected maxPoolSize: number;- protected createSessionFunction: CreateSession;- protected keyValueStore?: KeyValueStore;- protected sessions: Session[] = [];- protected sessionMap = new Map<string, Session>();- protected sessionOptions: SessionOptions;- protected persistStateKeyValueStoreId?: string;- protected persistStateKey: string;- protected _listener?: () => Promise<void>;- protected events: EventManager;- protected persistenceOptions: PersistenceOptions;- protected sessionReuseStrategy: SessionReuseStrategy;+ private log: CrawleeLogger;+ private maxPoolSize: number;+ private createSessionFunction: CreateSession;+ private keyValueStore?: KeyValueStore;+ private sessions: Session[] = [];+ private sessionMap = new Map<string, Session>();+ private sessionOptions: SessionOptions;+ private persistStateKeyValueStoreId?: string;+ private persistStateKey: string;+ private listener?: () => Promise<void>;+ private events: EventManager;+ private persistenceOptions: PersistenceOptions;+ private sessionReuseStrategy: SessionReuseStrategy;private initPromise?: Promise<void>;private queue = new AsyncQueue();@@ -190,7 +190,7 @@ export class SessionPool implements ISessionPool {// Pool Configurationthis.maxPoolSize = maxPoolSize;- this.createSessionFunction = createSessionFunction || this._defaultCreateSessionFunction;+ this.createSessionFunction = createSessionFunction || this.defaultCreateSessionFunction;// Session configuration. The pool-scoped logger is merged into per-call sessionOptions inside// `_invokeCreateSessionFunction`, so every Session inherits it without custom createSessionFunctions@@ -225,7 +225,7 @@ export class SessionPool implements ISessionPool {* Starts periodic state persistence and potentially loads SessionPool state from {@apilink KeyValueStore}.* Called automatically on first use of any public method.*/- protected async ensureInitialized(): Promise<void> {+ private async ensureInitialized(): Promise<void> {if (!this.initPromise) {this.initPromise = this.setupPool();}@@ -251,10 +251,10 @@ export class SessionPool implements ISessionPool {}// in case of migration happened and SessionPool state should be restored from the keyValueStore.- await this._maybeLoadSessionPool();+ await this.maybeLoadSessionPool();- this._listener = this.persistState.bind(this);- this.events.on(EventType.PERSIST_STATE, this._listener);+ this.listener = this.persistState.bind(this);+ this.events.on(EventType.PERSIST_STATE, this.listener);}/**@@ -273,14 +273,14 @@ export class SessionPool implements ISessionPool {}}- if (!this._hasSpaceForSession()) {- this._removeRetiredSessions();+ if (!this.hasSpaceForSession()) {+ this.removeRetiredSessions();}const newSession = options instanceof Session ? options : await this._invokeCreateSessionFunction(options);this.log.debug(`Adding new Session - ${newSession.id}`);- this._addSession(newSession);+ this.registerSession(newSession);}/**@@ -293,7 +293,7 @@ export class SessionPool implements ISessionPool {await this.ensureInitialized();const newSession = await this._invokeCreateSessionFunction(sessionOptions);- this._addSession(newSession);+ this.registerSession(newSession);return newSession;}@@ -316,15 +316,15 @@ export class SessionPool implements ISessionPool {return undefined;}- const pickedSession = this._pickSession();+ const pickedSession = this.pickSession();if (pickedSession) return pickedSession;- if (this._hasSpaceForSession()) {- return await this._createSession();+ if (this.hasSpaceForSession()) {+ return await this.createSession();}- this._removeRetiredSessions();- return await this._createSession();+ this.removeRetiredSessions();+ return await this.createSession();} finally {this.queue.shift();}@@ -387,8 +387,8 @@ export class SessionPool implements ISessionPool {async teardown({ persistState = true }: { persistState?: boolean } = {}): Promise<void> {if (!this.initPromise) return;await this.ensureInitialized();- if (this._listener) {- this.events.off(EventType.PERSIST_STATE, this._listener);+ if (this.listener) {+ this.events.off(EventType.PERSIST_STATE, this.listener);}if (persistState) {await this.persistState();@@ -398,7 +398,7 @@ export class SessionPool implements ISessionPool {/*** Removes retired `Session` instances from `SessionPool`.*/- protected _removeRetiredSessions() {+ private removeRetiredSessions() {this.sessions = this.sessions.filter((storedSession) => {if (storedSession.isUsable()) return true;@@ -413,7 +413,7 @@ export class SessionPool implements ISessionPool {* Adds `Session` instance to `SessionPool`.* @param newSession `Session` instance to be added.*/- protected _addSession(newSession: Session) {+ private registerSession(newSession: Session) {this.sessions.push(newSession);this.sessionMap.set(newSession.id, newSession);}@@ -421,7 +421,7 @@ export class SessionPool implements ISessionPool {/*** Gets random index.*/- protected _getRandomIndex(): number {+ private getRandomIndex(): number {return Math.floor(Math.random() * this.sessions.length);}@@ -431,7 +431,7 @@ export class SessionPool implements ISessionPool {* @param [options.sessionOptions] The configuration options for the session being created.* @returns New session.*/- protected async _defaultCreateSessionFunction(options: { sessionOptions?: SessionOptions } = {}): Promise<Session> {+ private async defaultCreateSessionFunction(options: { sessionOptions?: SessionOptions } = {}): Promise<Session> {ow(options, ow.object.exactShape({ sessionOptions: ow.optional.object }));const { sessionOptions = {} } = options;@@ -445,7 +445,7 @@ export class SessionPool implements ISessionPool {* A default {@apilink SessionFingerprint} is generated up front (host OS as* `platform`, a random valid `browser`/`device` for that platform). Pool-wide* and per-call options override it, and a persisted fingerprint coming- * through `_maybeLoadSessionPool` naturally wins because it arrives in+ * through `maybeLoadSessionPool` naturally wins because it arrives in* `perCallOptions`.*/private async _invokeCreateSessionFunction(perCallOptions?: SessionOptions): Promise<Session> {@@ -461,9 +461,9 @@ export class SessionPool implements ISessionPool {* Creates new session and adds it to the pool.* @returns Newly created `Session` instance.*/- protected async _createSession(): Promise<Session> {+ private async createSession(): Promise<Session> {const newSession = await this._invokeCreateSessionFunction();- this._addSession(newSession);+ this.registerSession(newSession);this.log.debug(`Created new Session - ${newSession.id}`);return newSession;@@ -472,7 +472,7 @@ export class SessionPool implements ISessionPool {/*** Decides whether there is enough space for creating new session.*/- protected _hasSpaceForSession(): boolean {+ private hasSpaceForSession(): boolean {return this.sessions.length < this.maxPoolSize;}@@ -480,8 +480,8 @@ export class SessionPool implements ISessionPool {* Picks a session from the `SessionPool` according to the configured `sessionReuseStrategy`.* Returns `undefined` when no session should be reused and a new one should be created instead.*/- protected _pickSession(): Session | undefined {- if (this.sessionReuseStrategy !== 'use-until-failure' && this._hasSpaceForSession()) return undefined;+ private pickSession(): Session | undefined {+ if (this.sessionReuseStrategy !== 'use-until-failure' && this.hasSpaceForSession()) return undefined;if (this.sessionReuseStrategy === 'use-until-failure') {return this.sessions.find((session) => session.isUsable());@@ -493,7 +493,7 @@ export class SessionPool implements ISessionPool {this.roundRobinIndex = index + 1;picked = this.sessions[index];} else {- picked = this.sessions[this._getRandomIndex()];+ picked = this.sessions[this.getRandomIndex()];}return picked.isUsable() ? picked : undefined;@@ -503,7 +503,7 @@ export class SessionPool implements ISessionPool {* Potentially loads `SessionPool`.* If the state was persisted it loads the `SessionPool` from the persisted state.*/- protected async _maybeLoadSessionPool(): Promise<void> {+ private async maybeLoadSessionPool(): Promise<void> {const loadedSessionPool = await this.keyValueStore?.getValue<{ sessions: Dictionary[] }>(this.persistStateKey);if (!loadedSessionPool) return;@@ -520,7 +520,7 @@ export class SessionPool implements ISessionPool {const recreatedSession = await this._invokeCreateSessionFunction(sessionObject);if (recreatedSession.isUsable()) {- this._addSession(recreatedSession);+ this.registerSession(recreatedSession);}}diff --git a/packages/core/src/storages/request_list.ts b/packages/core/src/storages/request_list.tsindex 68fe272fc..169eab144 100644--- a/packages/core/src/storages/request_list.ts+++ b/packages/core/src/storages/request_list.ts@@ -246,7 +246,7 @@ export class RequestList implements IRequestLoader {* All requests in the array have distinct uniqueKey!* @internal*/- requests: (Request | RequestOptions)[] = [];+ readonly requests: (Request | RequestOptions)[] = [];/** Index to the next item in requests array to fetch. All previous requests are either handled or in progress. */private nextIndex = 0;@@ -362,19 +362,19 @@ export class RequestList implements IRequestLoader {this.isLoading = true;await purgeDefaultStorages({ onlyPurgeOnce: true });- const [state, persistedRequests] = await this._loadStateAndPersistedRequests();+ const [state, persistedRequests] = await this.loadStateAndPersistedRequests();// Add persisted requests / new sources in a memory efficient way because with very// large lists, we were running out of memory.if (persistedRequests) {- await this._addPersistedRequests(persistedRequests as Buffer);+ await this.addPersistedRequests(persistedRequests as Buffer);} else {- await this._addRequestsFromSources();+ await this.addRequestsFromSources();}- this._restoreState(state as RequestListState);+ this.restoreState(state as RequestListState);this.isInitialized = true;- if (this.persistRequestsKey && !this.areRequestsPersisted) await this._persistRequests();+ if (this.persistRequestsKey && !this.areRequestsPersisted) await this.persistRequests();if (this.persistStateKey) {serviceLocator.getEventManager().on(EventType.PERSIST_STATE, this.persistState);}@@ -387,7 +387,7 @@ export class RequestList implements IRequestLoader {* This needs to be done in a memory efficient way. We should update the input* to a Stream once apify-client supports streams.*/- protected async _addPersistedRequests(persistedRequests: Buffer): Promise<void> {+ private async addPersistedRequests(persistedRequests: Buffer): Promise<void> {// We don't need the sources so we purge them to// prevent them from hanging in memory.for (let i = 0; i < this.sources.length; i++) {@@ -399,7 +399,7 @@ export class RequestList implements IRequestLoader {this.areRequestsPersisted = true;const requestStream = createDeserialize(persistedRequests);for await (const request of requestStream) {- this._addRequest(request);+ this.addRequest(request);}}@@ -409,7 +409,7 @@ export class RequestList implements IRequestLoader {* We need to avoid keeping both sources and requests in memory* to reduce memory footprint with very large sources.*/- protected async _addRequestsFromSources(): Promise<void> {+ private async addRequestsFromSources(): Promise<void> {// We'll load all sources in sequence to ensure that they get loaded in the right order.const sourcesCount = this.sources.length;for (let i = 0; i < sourcesCount; i++) {@@ -420,10 +420,10 @@ export class RequestList implements IRequestLoader {delete this.sources[i];if (typeof source === 'object' && (source as Dictionary).requestsFromUrl) {- const fetchedRequests = await this._fetchRequestsFromUrl(source as InternalSource);- await this._addFetchedRequests(source as InternalSource, fetchedRequests);+ const fetchedRequests = await this.fetchRequestsFromUrl(source as InternalSource);+ await this.addFetchedRequests(source as InternalSource, fetchedRequests);} else {- this._addRequest(source);+ this.addRequest(source);}}@@ -438,7 +438,7 @@ export class RequestList implements IRequestLoader {const source = sourcesFromFunction[i];// oxlint-disable-next-line typescript/no-array-delete -- intentional, drop the slot so V8 can collect the objectdelete sourcesFromFunction[i];- this._addRequest(source);+ this.addRequest(source);}sourcesFromFunction.length = 0;@@ -485,7 +485,7 @@ export class RequestList implements IRequestLoader {* are automatically persisted at RequestList initialization (if the persistRequestsKey is set),* but there's no reason to persist it again afterwards, because RequestList is immutable.*/- protected async _persistRequests(): Promise<void> {+ private async persistRequests(): Promise<void> {const serializedRequests = await serializeArray(this.requests);this.store ??= await KeyValueStore.open();await this.store.setValue(this.persistRequestsKey!, serializedRequests, { contentType: CONTENT_TYPE_BINARY });@@ -495,7 +495,7 @@ export class RequestList implements IRequestLoader {/*** Restores RequestList state from a state object.*/- protected _restoreState(state?: RequestListState): void {+ private restoreState(state?: RequestListState): void {// If there's no state it means we've not persisted any (yet).if (!state) return;// Restore previous state.@@ -563,7 +563,7 @@ export class RequestList implements IRequestLoader {* Attempts to load state and requests using the `RequestList` configuration* and returns a tuple of [state, requests] where each may be null if not loaded.*/- protected async _loadStateAndPersistedRequests(): Promise<[RequestListState, Buffer]> {+ private async loadStateAndPersistedRequests(): Promise<[RequestListState, Buffer]> {let state!: RequestListState;let persistedRequests!: Buffer;@@ -571,12 +571,12 @@ export class RequestList implements IRequestLoader {state = this.initialState;this.log.debug('Loaded state from options.state argument.');} else if (this.persistStateKey) {- state = await this._getPersistedState(this.persistStateKey);+ state = await this.getPersistedState(this.persistStateKey);if (state) this.log.debug('Loaded state from key value store using the persistStateKey.');}if (this.persistRequestsKey) {- persistedRequests = await this._getPersistedState(this.persistRequestsKey);+ persistedRequests = await this.getPersistedState(this.persistRequestsKey);if (persistedRequests) this.log.debug('Loaded requests from key value store using the persistRequestsKey.');}@@ -588,7 +588,7 @@ export class RequestList implements IRequestLoader {* Note that the object's fields can change in future releases.*/getState(): RequestListState {- this._ensureIsInitialized();+ this.ensureIsInitialized();return {nextIndex: this.nextIndex,@@ -601,7 +601,7 @@ export class RequestList implements IRequestLoader {* @inheritDoc*/async isEmpty(): Promise<boolean> {- this._ensureIsInitialized();+ this.ensureIsInitialized();return this.requestsToRetry.length === 0 && this.nextIndex >= this.requests.length;}@@ -610,7 +610,7 @@ export class RequestList implements IRequestLoader {* @inheritDoc*/async isFinished(): Promise<boolean> {- this._ensureIsInitialized();+ this.ensureIsInitialized();return this.inProgress.size === 0 && this.nextIndex >= this.requests.length;}@@ -619,7 +619,7 @@ export class RequestList implements IRequestLoader {* @inheritDoc*/async fetchNextRequest(): Promise<Request | null> {- this._ensureIsInitialized();+ this.ensureIsInitialized();// First re-serve any requests that were interrupted before the last state persist.const uniqueKey = this.requestsToRetry.shift();@@ -667,9 +667,9 @@ export class RequestList implements IRequestLoader {async markRequestAsHandled(request: Request): Promise<void> {const { uniqueKey } = request;- this._ensureUniqueKeyValid(uniqueKey);- this._ensureInProgress(uniqueKey);- this._ensureIsInitialized();+ this.ensureUniqueKeyValid(uniqueKey);+ this.ensureInProgress(uniqueKey);+ this.ensureIsInitialized();this.inProgress.delete(uniqueKey);this.isStatePersisted = false;@@ -678,11 +678,11 @@ export class RequestList implements IRequestLoader {/*** Adds all fetched requests from a URL from a remote resource.*/- protected async _addFetchedRequests(source: InternalSource, fetchedRequests: RequestOptions[]) {+ private async addFetchedRequests(source: InternalSource, fetchedRequests: RequestOptions[]) {const { requestsFromUrl, regex } = source;const originalLength = this.requests.length;- fetchedRequests.forEach((request) => this._addRequest(request));+ fetchedRequests.forEach((request) => this.addRequest(request));const fetchedCount = fetchedRequests.length;const importedCount = this.requests.length - originalLength;@@ -697,7 +697,7 @@ export class RequestList implements IRequestLoader {});}- protected async _getPersistedState<T>(key: string): Promise<T> {+ private async getPersistedState<T>(key: string): Promise<T> {this.store ??= await KeyValueStore.open();const state = await this.store.getValue<T>(key);@@ -707,7 +707,7 @@ export class RequestList implements IRequestLoader {/*** Fetches URLs from requestsFromUrl and returns them in format of list of requests*/- protected async _fetchRequestsFromUrl(source: InternalSource): Promise<RequestOptions[]> {+ private async fetchRequestsFromUrl(source: InternalSource): Promise<RequestOptions[]> {const { requestsFromUrl, regex, ...sharedOpts } = source;// Download remote resource and parse URLs.@@ -736,7 +736,7 @@ export class RequestList implements IRequestLoader {* If the `source` parameter is a string or plain object and not an instance* of a `Request`, then the function creates a `Request` instance.*/- protected _addRequest(source: RequestListSource) {+ private addRequest(source: RequestListSource) {let request: Request | RequestOptions;const type = typeof source;@@ -759,7 +759,7 @@ export class RequestList implements IRequestLoader {}const { uniqueKey } = request;- this._ensureUniqueKeyValid(uniqueKey);+ this.ensureUniqueKeyValid(uniqueKey);// Skip requests with duplicate uniqueKeyif (!Object.hasOwn(this.uniqueKeyToIndex, uniqueKey)) {@@ -776,7 +776,7 @@ export class RequestList implements IRequestLoader {* Helper function that validates unique key.* Throws an error if uniqueKey is not a non-empty string.*/- protected _ensureUniqueKeyValid(uniqueKey: string): void {+ private ensureUniqueKeyValid(uniqueKey: string): void {if (typeof uniqueKey !== 'string' || !uniqueKey) {throw new Error("Request object's uniqueKey must be a non-empty string");}@@ -785,7 +785,7 @@ export class RequestList implements IRequestLoader {/*** Checks that a request is currently being processed and throws an error if not.*/- protected _ensureInProgress(uniqueKey: string): void {+ private ensureInProgress(uniqueKey: string): void {if (!this.inProgress.has(uniqueKey)) {throw new Error(`The request is not being processed (uniqueKey: ${uniqueKey})`);}@@ -794,7 +794,7 @@ export class RequestList implements IRequestLoader {/*** Throws an error if request list wasn't initialized.*/- protected _ensureIsInitialized(): void {+ private ensureIsInitialized(): void {if (!this.isInitialized) {throw new Error('RequestList is not initialized; you must call "await requestList.initialize()" before using it!',@@ -806,7 +806,7 @@ export class RequestList implements IRequestLoader {* Returns the total number of unique requests present in the `RequestList`.*/async getTotalCount(): Promise<number> {- this._ensureIsInitialized();+ this.ensureIsInitialized();return this.requests.length;}@@ -815,7 +815,7 @@ export class RequestList implements IRequestLoader {* Returns the number of pending requests in the `RequestList`.*/async getPendingCount(): Promise<number> {- this._ensureIsInitialized();+ this.ensureIsInitialized();return this.requests.length - (this.nextIndex - this.inProgress.size);}@@ -837,7 +837,7 @@ export class RequestList implements IRequestLoader {* @inheritDoc*/async getHandledCount(): Promise<number> {- this._ensureIsInitialized();+ this.ensureIsInitialized();return this.nextIndex - this.inProgress.size;}diff --git a/packages/core/src/storages/request_queue.ts b/packages/core/src/storages/request_queue.tsindex b929f643a..5366e1c34 100644--- a/packages/core/src/storages/request_queue.ts+++ b/packages/core/src/storages/request_queue.ts@@ -89,36 +89,36 @@ const MAX_UNPROCESSED_REQUESTS_RETRIES = 3;* @category Sources*/export class RequestQueue implements IStorage, IRequestManager {- id: string;- name?: string;- backend: RequestQueueBackend;- protected proxyConfiguration?: IProxyConfiguration;+ readonly id: string;+ readonly name?: string;+ readonly backend: RequestQueueBackend;+ private proxyConfiguration?: IProxyConfiguration;- log: CrawleeLogger;+ readonly log: CrawleeLogger;- protected requestCache: LruCache<RequestLruItem>;+ private requestCache: LruCache<RequestLruItem>;/*** Remembers the `requestId` of every request already submitted to the client — including background* batches that `requestCache` skips — so overlapping URL sets aren't re-submitted.* See {@link RequestDeduplicationCache} for why this is a separate, cheaper cache.*/- protected requestSeenCache: RequestDeduplicationCache;+ private requestSeenCache: RequestDeduplicationCache;- protected queuePausedForMigration = false;+ private queuePausedForMigration = false;- protected inProgressRequestBatchCount = 0;+ private inProgressRequestBatchCount = 0;/*** The largest expected request-processing time (in seconds) seen so far via* {@link setExpectedRequestProcessingTimeSecs}. Used to ensure that value is only ever raised, never* lowered, before being forwarded to the storage backend.*/- protected expectedRequestProcessingSecs = 0;+ private expectedRequestProcessingSecs = 0;- protected httpClient?: BaseHttpClient;+ private httpClient?: BaseHttpClient;- protected readonly events: EventManager;+ private readonly events: EventManager;private readonly statsTracker = new StorageStatsTracker<RequestQueueStats>({writeCount: 0,@@ -136,10 +136,7 @@ export class RequestQueue implements IStorage, IRequestManager {/*** @internal*/- constructor(- options: RequestQueueOptions,- protected readonly config: Configuration = Configuration.getGlobalConfig(),- ) {+ constructor(options: RequestQueueOptions) {this.id = options.metadata.id;this.name = options.metadata.name;this.events = serviceLocator.getEventManager();@@ -207,8 +204,8 @@ export class RequestQueue implements IStorage, IRequestManager {const { forefront = false } = options;if ('requestsFromUrl' in requestLike) {- const requests = await this._fetchRequestsFromUrl(requestLike as InternalSource);- const processedRequests = await this._addFetchedRequests(requestLike as InternalSource, requests, options);+ const requests = await this.fetchRequestsFromUrl(requestLike as InternalSource);+ const processedRequests = await this.addFetchedRequests(requestLike as InternalSource, requests, options);return { ...processedRequests[0], forefront };}@@ -247,7 +244,7 @@ export class RequestQueue implements IStorage, IRequestManager {forefront,} satisfies RequestQueueOperationInfo;- this._cacheRequest(cacheKey, queueOperationInfo);+ this.cacheRequest(cacheKey, queueOperationInfo);this.requestSeenCache.add(cacheKey, request.id!);return queueOperationInfo;@@ -312,8 +309,8 @@ export class RequestQueue implements IStorage, IRequestManager {if (typeof requestLike === 'string') {requests.push(new Request({ url: requestLike }));} else if ('requestsFromUrl' in requestLike) {- const fetchedRequests = await this._fetchRequestsFromUrl(requestLike as InternalSource);- await this._addFetchedRequests(requestLike as InternalSource, fetchedRequests, options);+ const fetchedRequests = await this.fetchRequestsFromUrl(requestLike as InternalSource);+ await this.addFetchedRequests(requestLike as InternalSource, fetchedRequests, options);} else {requests.push(requestLike instanceof Request ? requestLike : new Request(requestLike as RequestOptions),@@ -362,7 +359,7 @@ export class RequestQueue implements IStorage, IRequestManager {const cacheKey = getCachedRequestId(newRequest.uniqueKey);if (cache) {- this._cacheRequest(cacheKey, { ...newRequest, forefront });+ this.cacheRequest(cacheKey, { ...newRequest, forefront });}// Unlike `requestCache`, populate this on every batch (including background ones).@@ -663,7 +660,7 @@ export class RequestQueue implements IStorage, IRequestManager {forefront,} satisfies RequestQueueOperationInfo;- this._cacheRequest(getRequestId(request.uniqueKey), queueOperationInfo);+ this.cacheRequest(getRequestId(request.uniqueKey), queueOperationInfo);return queueOperationInfo;}@@ -709,7 +706,7 @@ export class RequestQueue implements IStorage, IRequestManager {uniqueKey: request.uniqueKey,forefront,} satisfies RequestQueueOperationInfo;- this._cacheRequest(getRequestId(request.uniqueKey), queueOperationInfo);+ this.cacheRequest(getRequestId(request.uniqueKey), queueOperationInfo);return queueOperationInfo;}@@ -771,7 +768,7 @@ export class RequestQueue implements IStorage, IRequestManager {/*** Caches information about request to beware of unneeded addRequest() calls.*/- protected _cacheRequest(cacheKey: string, queueOperationInfo: RequestQueueOperationInfo): void {+ private cacheRequest(cacheKey: string, queueOperationInfo: RequestQueueOperationInfo): void {// Remove the previous entry, as otherwise our cache will never update 👀this.requestCache.remove(cacheKey);@@ -871,7 +868,7 @@ export class RequestQueue implements IStorage, IRequestManager {/*** Fetches URLs from requestsFromUrl and returns them in format of list of requests*/- protected async _fetchRequestsFromUrl(source: InternalSource): Promise<RequestOptions[]> {+ private async fetchRequestsFromUrl(source: InternalSource): Promise<RequestOptions[]> {const { requestsFromUrl, regex, ...sharedOpts } = source;// Download remote resource and parse URLs.@@ -898,7 +895,7 @@ export class RequestQueue implements IStorage, IRequestManager {/*** Adds all fetched requests from a URL from a remote resource.*/- protected async _addFetchedRequests(+ private async addFetchedRequests(source: InternalSource,fetchedRequests: RequestOptions[],options: RequestQueueOperationOptions,diff --git a/packages/core/test/request-queue/adding-the-same-request-should-not-call-the-api.test.ts b/packages/core/test/request-queue/adding-the-same-request-should-not-call-the-api.test.tsindex 8eab8ea8d..35e0fe778 100644--- a/packages/core/test/request-queue/adding-the-same-request-should-not-call-the-api.test.ts+++ b/packages/core/test/request-queue/adding-the-same-request-should-not-call-the-api.test.ts@@ -12,9 +12,8 @@ beforeEach(async () => {describe('RequestQueue#addRequest should not call the API if the request is already in the queue', () => {test('should not call the API if the request is already in the queue', async () => {- const config = serviceLocator.getConfiguration();const rqInfo = await rqClient.getMetadata();- const requestQueue = new RequestQueue({ metadata: rqInfo, backend: rqClient }, config);+ const requestQueue = new RequestQueue({ metadata: rqInfo, backend: rqClient });const clientSpy = vitest.spyOn(requestQueue.backend, 'addBatchOfRequests');@@ -35,9 +34,8 @@ describe('RequestQueue#addRequest should not call the API if the request is alredescribe('RequestQueue#addRequests should not call the API if the request is already in the queue', () => {test('should not call the API if the request is already in the queue', async () => {- const config = serviceLocator.getConfiguration();const rqInfo = await rqClient.getMetadata();- const requestQueue = new RequestQueue({ metadata: rqInfo, backend: rqClient }, config);+ const requestQueue = new RequestQueue({ metadata: rqInfo, backend: rqClient });const clientSpy = vitest.spyOn(requestQueue.backend, 'addBatchOfRequests');diff --git a/packages/fs-storage/src/resource-clients/cached-id-client.ts b/packages/fs-storage/src/resource-clients/cached-id-client.tsindex ac0a2e200..4e5ad8e99 100644--- a/packages/fs-storage/src/resource-clients/cached-id-client.ts+++ b/packages/fs-storage/src/resource-clients/cached-id-client.ts@@ -5,10 +5,10 @@*/export abstract class CachedIdClient {/** The storage id assigned by the native client. Set once by the subclass `create()`. */- protected _cachedId!: string;+ protected cachedId!: string;/** The storage id assigned by the native client. */get id(): string {- return this._cachedId;+ return this.cachedId;}}diff --git a/packages/fs-storage/src/resource-clients/dataset.ts b/packages/fs-storage/src/resource-clients/dataset.tsindex 88dd62781..f8f415b70 100644--- a/packages/fs-storage/src/resource-clients/dataset.ts+++ b/packages/fs-storage/src/resource-clients/dataset.ts@@ -61,7 +61,7 @@ export class DatasetBackend<Data extends Dictionary = Dictionary>options: DatasetBackendOptions,): Promise<DatasetBackend<Data>> {const backend = new DatasetBackend<Data>(options);- backend._cachedId = (await options.nativeBackend.getMetadata()).id;+ backend.cachedId = (await options.nativeBackend.getMetadata()).id;return backend;}diff --git a/packages/fs-storage/src/resource-clients/key-value-store.ts b/packages/fs-storage/src/resource-clients/key-value-store.tsindex 6b38e985a..90f9a956d 100644--- a/packages/fs-storage/src/resource-clients/key-value-store.ts+++ b/packages/fs-storage/src/resource-clients/key-value-store.ts@@ -100,7 +100,7 @@ export class KeyValueStoreBackend extends CachedIdClient implements storage.KeyVstatic async create(options: KeyValueStoreBackendOptions): Promise<KeyValueStoreBackend> {const backend = new KeyValueStoreBackend(options);- backend._cachedId = (await options.nativeBackend.getMetadata()).id;+ backend.cachedId = (await options.nativeBackend.getMetadata()).id;return backend;}diff --git a/packages/fs-storage/src/resource-clients/request-queue.ts b/packages/fs-storage/src/resource-clients/request-queue.tsindex ac26cde11..95b9c0fff 100644--- a/packages/fs-storage/src/resource-clients/request-queue.ts+++ b/packages/fs-storage/src/resource-clients/request-queue.ts@@ -78,7 +78,7 @@ export class RequestQueueBackend extends CachedIdClient implements storage.Requestatic async create(options: RequestQueueBackendOptions): Promise<RequestQueueBackend> {const backend = new RequestQueueBackend(options);- backend._cachedId = (await options.nativeBackend.getMetadata()).id;+ backend.cachedId = (await options.nativeBackend.getMetadata()).id;return backend;}diff --git a/packages/http-client/src/base-http-client.ts b/packages/http-client/src/base-http-client.tsindex 9e72d9a4c..b9ea44d62 100644--- a/packages/http-client/src/base-http-client.ts+++ b/packages/http-client/src/base-http-client.ts@@ -41,7 +41,7 @@ export interface CustomFetchOptions {* implement only the low-level network call in `fetch`.*/export abstract class BaseHttpClient implements BaseHttpClientInterface {- protected log?: CrawleeLogger;+ private log?: CrawleeLogger;constructor(options?: { logger?: CrawleeLogger }) {this.log = options?.logger;diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.tsindex 8d312d5a8..499c7524c 100644--- a/packages/http-crawler/src/internals/http-crawler.ts+++ b/packages/http-crawler/src/internals/http-crawler.ts@@ -316,16 +316,16 @@ export class HttpCrawler<ContextExtension = Dictionary<never>,ExtendedContext extends Context = Context & ContextExtension,> extends BasicCrawler<Context, ContextExtension, ExtendedContext> {- protected preNavigationHooks: InternalHttpHook<CrawlingContext>[];- protected postNavigationHooks: ((+ private preNavigationHooks: InternalHttpHook<CrawlingContext>[];+ private postNavigationHooks: ((crawlingContext: CrawlingContextWithResponse,) => Awaitable<void | Partial<CrawlingContextWithResponse>>)[];- protected saveResponseCookies: boolean;- protected navigationTimeoutMillis: number;- protected ignoreSslErrors: boolean;- protected suggestResponseEncoding?: string;- protected forceResponseEncoding?: string;- protected readonly supportedMimeTypes: Set<string>;+ private saveResponseCookies: boolean;+ private navigationTimeoutMillis: number;+ private ignoreSslErrors: boolean;+ private suggestResponseEncoding?: string;+ private forceResponseEncoding?: string;+ private readonly supportedMimeTypes: Set<string>;protected static override optionsShape = {...BasicCrawler.optionsShape,@@ -375,7 +375,7 @@ export class HttpCrawler<});this.supportedMimeTypes = new Set([...HTML_AND_XML_MIME_TYPES, APPLICATION_JSON_MIME_TYPE]);- if (additionalMimeTypes.length) this._extendSupportedMimeTypes(additionalMimeTypes);+ if (additionalMimeTypes.length) this.extendSupportedMimeTypes(additionalMimeTypes);if (suggestResponseEncoding && forceResponseEncoding) {this.log.warning(@@ -460,7 +460,7 @@ export class HttpCrawler<const proxyUrl = crawlingContext.proxyInfo?.url;const httpResponse = await addTimeoutToPromise(- async () => this._requestFunction({ request, session, proxyUrl }),+ async () => this.requestFunction({ request, session, proxyUrl }),this.navigationTimeoutMillis,`request timed out after ${this.navigationTimeoutMillis / 1000} seconds.`,);@@ -509,7 +509,7 @@ export class HttpCrawler<tryCancel();- const parsed = await this._parseResponse(crawlingContext.request, crawlingContext.response);+ const parsed = await this.parseResponse(crawlingContext.request, crawlingContext.response);tryCancel();const response = parsed.response!;const contentType = parsed.contentType!;@@ -594,15 +594,15 @@ export class HttpCrawler<* on the request such as only downloading the request body if the* received content type matches text/html, application/xml, application/xhtml+xml.*/- protected async _requestFunction({ request, session, proxyUrl }: RequestFunctionOptions): Promise<Response> {- const opts = this._getRequestOptions(request, session, proxyUrl);+ private async requestFunction({ request, session, proxyUrl }: RequestFunctionOptions): Promise<Response> {+ const opts = this.getRequestOptions(request, session, proxyUrl);try {return await this._requestAsBrowser(opts, session);} catch (e) {if (e instanceof Error && e.constructor.name === 'TimeoutError') {- this._handleRequestTimeout(session);- return new Response(); // this will never happen, as _handleRequestTimeout always throws+ this.handleRequestTimeout(session);+ return new Response(); // this will never happen, as handleRequestTimeout always throws}if (this.isProxyError(e as Error)) {@@ -616,10 +616,10 @@ export class HttpCrawler</*** Encodes and parses response according to the provided content type*/- protected async _parseResponse(request: CrawleeRequest, response: Response) {+ private async parseResponse(request: CrawleeRequest, response: Response) {const { status } = response;const { type, charset } = parseContentTypeFromResponse(response);- const { response: reencodedResponse, encoding } = this._encodeResponse(request, response, charset);+ const { response: reencodedResponse, encoding } = this.encodeResponse(request, response, charset);const contentType = { type, encoding };if (status >= 400 && status <= 599) {@@ -668,7 +668,7 @@ export class HttpCrawler</*** Combines the provided `requestOptions` with mandatory (non-overridable) values.*/- protected _getRequestOptions(request: CrawleeRequest, session: ISession, proxyUrl?: string) {+ private getRequestOptions(request: CrawleeRequest, session: ISession, proxyUrl?: string) {const requestOptions = {url: request.url,method: request.method,@@ -700,7 +700,7 @@ export class HttpCrawler<return requestOptions;}- protected _encodeResponse(+ private encodeResponse(request: CrawleeRequest,response: Response,encoding: BufferEncoding,@@ -750,7 +750,7 @@ export class HttpCrawler</*** Checks and extends supported mime types*/- protected _extendSupportedMimeTypes(additionalMimeTypes: (string | RequestLike | ResponseLike)[]) {+ private extendSupportedMimeTypes(additionalMimeTypes: (string | RequestLike | ResponseLike)[]) {for (const mimeType of additionalMimeTypes) {if (mimeType === '*/*') {this.supportedMimeTypes.add(mimeType);@@ -769,7 +769,7 @@ export class HttpCrawler</*** Handles timeout request*/- protected _handleRequestTimeout(session: ISession) {+ private handleRequestTimeout(session: ISession) {session.markBad();throw new Error(`request timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);}diff --git a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts b/packages/jsdom-crawler/src/internals/jsdom-crawler.tsindex 6eaff6040..e7640d186 100644--- a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts+++ b/packages/jsdom-crawler/src/internals/jsdom-crawler.ts@@ -194,9 +194,9 @@ export class JSDOMCrawler<hideInternalConsole: ow.optional.boolean,};- protected runScripts: boolean;- protected hideInternalConsole: boolean;- protected virtualConsole: VirtualConsole | null = null;+ private runScripts: boolean;+ private hideInternalConsole: boolean;+ private virtualConsole: VirtualConsole | null = null;constructor(options: JSDOMCrawlerOptions<ContextExtension, ExtendedContext> = {}) {const { runScripts = false, hideInternalConsole = false, contextPipelineBuilder, ...httpOptions } = options;diff --git a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.tsindex 8692c4c59..fcbbce1e9 100644--- a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts+++ b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts@@ -79,8 +79,8 @@ class AdaptivePlaywrightCrawlerStatistics extends Statistics {this.state.renderingTypeMispredictions = 0;}- protected override async _maybeLoadStatistics(): Promise<void> {- await super._maybeLoadStatistics();+ protected override async maybeLoadStatistics(): Promise<void> {+ await super.maybeLoadStatistics();const savedState = await this.keyValueStore?.getValue<AdaptivePlaywrightCrawlerPersistedStatisticState>(this.persistStateKey,);@@ -718,7 +718,7 @@ export class AdaptivePlaywrightCrawler<}}- protected async commitResult(+ private async commitResult(crawlingContext: CrawlingContext,{ calls, keyValueStoreChanges }: RequestHandlerResult,): Promise<void> {@@ -736,7 +736,7 @@ export class AdaptivePlaywrightCrawler<]);}- protected allowStorageAccess<R, TArgs extends any[]>(+ private allowStorageAccess<R, TArgs extends any[]>(func: (...args: TArgs) => Promise<R>,): (...args: TArgs) => Promise<R> {return async (...args: TArgs) =>@@ -755,7 +755,7 @@ export class AdaptivePlaywrightCrawler<return this.allowStorageAccess(() => super.getPendingRequestCountApproximation())();}- protected async enqueueLinks(+ private async enqueueLinks(options: SetRequired<EnqueueLinksOptions, 'urls'>,request: RestrictedCrawlingContext['request'],result: RequestHandlerResult,diff --git a/packages/playwright-crawler/src/internals/utils/rendering-type-prediction.ts b/packages/playwright-crawler/src/internals/utils/rendering-type-prediction.tsindex e325d6f74..db8639f32 100644--- a/packages/playwright-crawler/src/internals/utils/rendering-type-prediction.ts+++ b/packages/playwright-crawler/src/internals/utils/rendering-type-prediction.ts@@ -151,7 +151,7 @@ export class RenderingTypePredictor {.reduce((acc, value) => acc + value, 0);}- protected calculateFeatureVector(url: URLComponents, label: string | undefined): FeatureVector {+ private calculateFeatureVector(url: URLComponents, label: string | undefined): FeatureVector {return [mean((this.state.currentValue.detectionResults.get('static')?.get(label) ?? []).map(@@ -166,7 +166,7 @@ export class RenderingTypePredictor {];}- protected retrain(): void {+ private retrain(): void {const X: FeatureVector[] = [[0, 1],[1, 0],diff --git a/packages/puppeteer-crawler/src/internals/puppeteer-launcher.ts b/packages/puppeteer-crawler/src/internals/puppeteer-launcher.tsindex ec023a535..5be972128 100644--- a/packages/puppeteer-crawler/src/internals/puppeteer-launcher.ts+++ b/packages/puppeteer-crawler/src/internals/puppeteer-launcher.ts@@ -103,8 +103,8 @@ export class PuppeteerLauncher extends BrowserLauncher<PuppeteerPlugin, unknown>this.Plugin = PuppeteerPlugin;}- protected override _getDefaultHeadlessOption(): boolean {- const headless = super._getDefaultHeadlessOption();+ protected override getDefaultHeadlessOption(): boolean {+ const headless = super.getDefaultHeadlessOption();return headless ? ('new' as any) : headless;}}diff --git a/packages/utils/src/internals/robots.ts b/packages/utils/src/internals/robots.tsindex 4cfc55b87..ff2c7907f 100644--- a/packages/utils/src/internals/robots.ts+++ b/packages/utils/src/internals/robots.ts@@ -76,7 +76,7 @@ export class RobotsTxtFile {return new RobotsTxtFile(url, robotsParser(url, content), proxyUrl);}- protected static async load(+ private static async load(url: string,options?: {signal?: AbortSignal;diff --git a/packages/utils/src/internals/sitemap.ts b/packages/utils/src/internals/sitemap.tsindex 82cdb3c42..5969b259d 100644--- a/packages/utils/src/internals/sitemap.ts+++ b/packages/utils/src/internals/sitemap.ts@@ -499,7 +499,7 @@ export class Sitemap {return await this.parse([{ type: 'raw', content }], proxyUrl, parseSitemapOptions);}- protected static async parse(+ private static async parse(sources: SitemapSource[],proxyUrl?: string,parseSitemapOptions?: ParseSitemapOptions,diff --git a/test/core/autoscaling/autoscaled_pool.test.ts b/test/core/autoscaling/autoscaled_pool.test.tsindex 96e03ab9b..8e8e63b11 100644--- a/test/core/autoscaling/autoscaled_pool.test.ts+++ b/test/core/autoscaling/autoscaled_pool.test.ts@@ -157,29 +157,29 @@ describe('AutoscaledPool', () => {test('works with low values', () => {// @ts-expect-error Calling private method- pool._autoscale(cb);+ pool.autoscale(cb);expect(pool.desiredConcurrency).toBe(2);// @ts-expect-error Calling private method- pool._autoscale(cb);+ pool.autoscale(cb);expect(pool.desiredConcurrency).toBe(2); // because currentConcurrency is not high enough;// @ts-expect-error Overwriting readonly private proppool._currentConcurrency = 2;// @ts-expect-error Calling private method- pool._autoscale(cb);+ pool.autoscale(cb);expect(pool.desiredConcurrency).toBe(3);systemStatus.okNow = false; // this should have no effect// @ts-expect-error Overwriting readonly private proppool._currentConcurrency = 3;// @ts-expect-error Calling private method- pool._autoscale(cb);+ pool.autoscale(cb);expect(pool.desiredConcurrency).toBe(4);systemStatus.okLately = false;// @ts-expect-error Calling private method- pool._autoscale(cb);+ pool.autoscale(cb);expect(pool.desiredConcurrency).toBe(3);});@@ -190,7 +190,7 @@ describe('AutoscaledPool', () => {pool._currentConcurrency = Math.floor(pool.desiredConcurrency * pool.desiredConcurrencyRatio) - 1;systemStatus.okLately = true;// @ts-expect-error Calling private method- pool._autoscale(cb);+ pool.autoscale(cb);expect(pool.desiredConcurrency).toBe(50);// Should scale because we bumped up current concurrency.@@ -199,7 +199,7 @@ describe('AutoscaledPool', () => {// @ts-expect-error Accessing private proplet newConcurrency = pool.desiredConcurrency + Math.ceil(pool.desiredConcurrency * pool.scaleUpStepRatio);// @ts-expect-error Calling private method- pool._autoscale(cb);+ pool.autoscale(cb);expect(pool.desiredConcurrency).toEqual(newConcurrency);// Should scale down.@@ -207,7 +207,7 @@ describe('AutoscaledPool', () => {// @ts-expect-error Accessing private propnewConcurrency = pool.desiredConcurrency - Math.ceil(pool.desiredConcurrency * pool.scaleDownStepRatio);// @ts-expect-error Calling private method- pool._autoscale(cb);+ pool.autoscale(cb);expect(pool.desiredConcurrency).toEqual(newConcurrency);});@@ -408,7 +408,7 @@ describe('AutoscaledPool', () => {loggingIntervalSecs: null,});// @ts-expect-error Calling private method- pool._autoscale(() => {});+ pool.autoscale(() => {});expect(pool.desiredConcurrency).toBe(2);});diff --git a/test/core/autoscaling/snapshotter.test.ts b/test/core/autoscaling/snapshotter.test.tsindex 6f805c164..2e1cbee4e 100644--- a/test/core/autoscaling/snapshotter.test.ts+++ b/test/core/autoscaling/snapshotter.test.ts@@ -184,20 +184,17 @@ describe('Snapshotter', () => {const clock = vitest.useFakeTimers();try {const snapshotter = new Snapshotter({ maxBlockedMillis: 5, eventLoopSnapshotIntervalSecs: 0 });- // @ts-expect-error Calling protected method- snapshotter._snapshotEventLoop(noop);+ // @ts-expect-error Accessing private property+ const eventLoopSignal = snapshotter.eventLoopSignal;+ eventLoopSignal.handle(noop);clock.advanceTimersByTime(1);- // @ts-expect-error Calling protected method- snapshotter._snapshotEventLoop(noop);+ eventLoopSignal.handle(noop);clock.advanceTimersByTime(2);- // @ts-expect-error Calling protected method- snapshotter._snapshotEventLoop(noop);+ eventLoopSignal.handle(noop);clock.advanceTimersByTime(7);- // @ts-expect-error Calling protected method- snapshotter._snapshotEventLoop(noop);+ eventLoopSignal.handle(noop);clock.advanceTimersByTime(3);- // @ts-expect-error Calling protected method- snapshotter._snapshotEventLoop(noop);+ eventLoopSignal.handle(noop);const loopSnapshots = snapshotter.getEventLoopSample();expect(loopSnapshots.length).toBe(5);@@ -297,17 +294,15 @@ describe('Snapshotter', () => {apifyClient.stats!.rateLimitErrors = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0];const snapshotter = new Snapshotter({ maxClientErrors: 1 });- // @ts-expect-error Calling protected method- snapshotter._snapshotClient(noop);+ // @ts-expect-error Accessing private property+ const clientSignal = snapshotter.clientSignal;+ clientSignal.handle(noop);apifyClient.stats!.rateLimitErrors = [1, 1, 1, 0, 0, 0, 0, 0, 0, 0];- // @ts-expect-error Calling protected method- snapshotter._snapshotClient(noop);+ clientSignal.handle(noop);apifyClient.stats!.rateLimitErrors = [10, 5, 2, 0, 0, 0, 0, 0, 0, 0];- // @ts-expect-error Calling protected method- snapshotter._snapshotClient(noop);+ clientSignal.handle(noop);apifyClient.stats!.rateLimitErrors = [100, 24, 4, 2, 0, 0, 0, 0, 0, 0];- // @ts-expect-error Calling protected method- snapshotter._snapshotClient(noop);+ clientSignal.handle(noop);const clientSnapshots = snapshotter.getClientSample();diff --git a/test/core/browser_launchers/playwright_launcher.test.ts b/test/core/browser_launchers/playwright_launcher.test.tsindex 7729b6aa3..374edac79 100644--- a/test/core/browser_launchers/playwright_launcher.test.ts+++ b/test/core/browser_launchers/playwright_launcher.test.ts@@ -147,7 +147,7 @@ describe('launchPlaywright()', () => {});test('supports useChrome option', async () => {- const spy = vitest.spyOn(BrowserLauncher.prototype as any, '_getTypicalChromeExecutablePath');+ const spy = vitest.spyOn(BrowserLauncher.prototype as any, 'getTypicalChromeExecutablePath');let browser;const opts = {useChrome: true,@@ -200,7 +200,7 @@ describe('launchPlaywright()', () => {const plugin = launcher.createBrowserPlugin();// @ts-expect-error private method- expect(plugin.launchOptions.executablePath).toBe(launcher._getTypicalChromeExecutablePath());+ expect(plugin.launchOptions.executablePath).toBe(launcher.getTypicalChromeExecutablePath());}, 60e3);test('allows to be overridden', () => {diff --git a/test/core/browser_launchers/puppeteer_launcher.test.ts b/test/core/browser_launchers/puppeteer_launcher.test.tsindex 87f401b33..b4d5eb0f6 100644--- a/test/core/browser_launchers/puppeteer_launcher.test.ts+++ b/test/core/browser_launchers/puppeteer_launcher.test.ts@@ -199,7 +199,7 @@ describe('launchPuppeteer()', () => {});test('supports useChrome option', async () => {- const spy = vitest.spyOn(BrowserLauncher.prototype as any, '_getTypicalChromeExecutablePath');+ const spy = vitest.spyOn(BrowserLauncher.prototype as any, 'getTypicalChromeExecutablePath');let browser;const opts = {diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.tsindex 06fffa89f..eeeaf8189 100644--- a/test/core/crawlers/basic_crawler.test.ts+++ b/test/core/crawlers/basic_crawler.test.ts@@ -657,7 +657,7 @@ describe('BasicCrawler', () => {// clean up// @ts-expect-error Accessing private method- await basicCrawler.autoscaledPool!._destroy();+ await basicCrawler.autoscaledPool!.destroy();},);@@ -1110,7 +1110,7 @@ describe('BasicCrawler', () => {});// @ts-expect-error Accessing private prop- expect(await crawler._isTaskReadyFunction()).toBe(false);+ expect(await crawler.isTaskReadyFunction()).toBe(false);});test('should be possible to override isFinishedFunction and isTaskReadyFunction of underlying AutoscaledPool', async () => {diff --git a/test/core/crawlers/cheerio_crawler.test.ts b/test/core/crawlers/cheerio_crawler.test.tsindex 7e5756457..3753c0ae4 100644--- a/test/core/crawlers/cheerio_crawler.test.ts+++ b/test/core/crawlers/cheerio_crawler.test.ts@@ -335,7 +335,7 @@ describe('CheerioCrawler', () => {});// @ts-expect-error Overriding private method- cheerioCrawler._requestFunction = async () => {+ cheerioCrawler.requestFunction = async () => {await sleep(300);return '<html><head></head><body>Body</body></html>';};@@ -663,7 +663,7 @@ describe('CheerioCrawler', () => {});// @ts-expect-error Using private method- const { response, encoding } = crawler._encodeResponse({}, new Response(new Uint8Array(buf)));+ const { response, encoding } = crawler.encodeResponse({}, new Response(new Uint8Array(buf)));expect(encoding).toBe('utf8');expect(await response.text()).toBe(html);});@@ -684,7 +684,7 @@ describe('CheerioCrawler', () => {});// @ts-expect-error Using private method- const { response, encoding } = crawler._encodeResponse({}, new Response(new Uint8Array(buf)), 'ascii');+ const { response, encoding } = crawler.encodeResponse({}, new Response(new Uint8Array(buf)), 'ascii');expect(encoding).toBe('utf8');expect(await response.text()).toBe(html);});@@ -792,23 +792,26 @@ describe('CheerioCrawler', () => {proxyUrls: ['http://localhost', 'http://localhost:1234', goodProxyUrl],});const check = vitest.fn();+ const impit = new ImpitHttpClient();- const crawler = new (class extends CheerioCrawler {- protected override async _requestFunction(...args: any[]): Promise<any> {- check(...args);-- if (args[0].proxyUrl === goodProxyUrl) {- return null;- }-- throw new Error('Proxy responded with 400 - Bad request');- }- })({+ const crawler = new CheerioCrawler({maxRequestRetries: 2,maxConcurrency: 1,proxyConfiguration,requestHandler: () => {},+ httpClient: {+ sendRequest: async (request, opts) => {+ const proxyUrl = opts?.session?.proxyInfo?.url;+ check({ ...opts, proxyUrl });++ if (proxyUrl === goodProxyUrl) {+ return await impit.sendRequest(request);+ }++ throw new Error('Proxy responded with 400 - Bad request');+ },+ },});await expect(crawler.run([serverAddress])).resolves.not.toThrow();diff --git a/test/core/session_pool/session.test.ts b/test/core/session_pool/session.test.tsindex 1cf8a79d5..7f40a7a9f 100644--- a/test/core/session_pool/session.test.ts+++ b/test/core/session_pool/session.test.ts@@ -80,8 +80,6 @@ describe('Session - testing session behaviour', () => {});test('should retire session after marking bad', () => {- // @ts-expect-error Private property- vitest.spyOn(session, '_maybeSelfRetire');vitest.spyOn(session, 'retire');session.markBad();expect(session.retire).toBeCalledTimes(0);@@ -91,8 +89,6 @@ describe('Session - testing session behaviour', () => {});test('should retire session after marking good', () => {- // @ts-expect-error Private property- vitest.spyOn(session, '_maybeSelfRetire');vitest.spyOn(session, 'retire');session.markGood();@@ -104,14 +100,18 @@ describe('Session - testing session behaviour', () => {});test('should reevaluate usability of session after marking the session', () => {- // @ts-expect-error Private property- vitest.spyOn(session, '_maybeSelfRetire');+ vitest.spyOn(session, 'retire');++ // A usable session is not retired when marked.session.markGood();- // @ts-expect-error Private property- expect(session._maybeSelfRetire).toBeCalledTimes(1);+ expect(session.retire).toBeCalledTimes(0);++ // Once the session becomes unusable, marking it (good or bad) retires it.+ session.isUsable = () => false;+ session.markGood();+ expect(session.retire).toBeCalledTimes(1);session.markBad();- // @ts-expect-error Private property- expect(session._maybeSelfRetire).toBeCalledTimes(2);+ expect(session.retire).toBeCalledTimes(2);});test('should get state', () => {diff --git a/test/core/session_pool/session_pool.test.ts b/test/core/session_pool/session_pool.test.tsindex 6dedc84b6..c4022e6de 100644--- a/test/core/session_pool/session_pool.test.ts+++ b/test/core/session_pool/session_pool.test.ts@@ -31,7 +31,7 @@ describe('SessionPool - testing session pool', () => {// @ts-expect-error private symbolexpect(sessionPool.persistStateKey).toBeDefined();// @ts-expect-error private symbol- expect(sessionPool.createSessionFunction).toEqual(sessionPool._defaultCreateSessionFunction);+ expect(sessionPool.createSessionFunction).toEqual(sessionPool.defaultCreateSessionFunction);});test('should override default values', async () => {@@ -82,10 +82,10 @@ describe('SessionPool - testing session pool', () => {await sessionPool.getSession();let isCalled = false;// @ts-expect-error Accessing private property- const oldPick = sessionPool._pickSession;+ const oldPick = sessionPool.pickSession;// @ts-expect-error Overriding private property- sessionPool._pickSession = () => {+ sessionPool.pickSession = () => {isCalled = true;return oldPick.bind(sessionPool)();};@@ -190,7 +190,7 @@ describe('SessionPool - testing session pool', () => {// @ts-expect-error Accessing protected methodawait sessionPool.ensureInitialized();// @ts-expect-error private symbol- await sessionPool._createSession();+ await sessionPool.createSession();// @ts-expect-error private symbolexpect(sessionPool.sessions.length).toBe(1);// @ts-expect-error private symbol