Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
refactor!: Continue public API minimization (#4036)
Jan Bucharceb936af2a83
diff --git a/docs/public-api/crawlee-basic.api.md b/docs/public-api/crawlee-basic.api.mdindex 9ddff7730..77f5c7260 100644--- a/docs/public-api/crawlee-basic.api.md+++ b/docs/public-api/crawlee-basic.api.md@@ -8,9 +8,6 @@ import type { AddRequestsBatchedOptions } from '@crawlee/core';import type { AddRequestsBatchedResult } from '@crawlee/core';import type { Awaitable } from '@crawlee/types';import { BaseHttpClient } from '@crawlee/http-client';-import { Cheerio } from '@crawlee/utils/internal';-import { CheerioAPI } from '@crawlee/utils/internal';-import { CheerioRoot } from '@crawlee/utils/internal';import { ConcurrencySystem } from '@crawlee/core';import { ConcurrencySystemOptions } from '@crawlee/core';import { Configuration } from '@crawlee/core';@@ -20,7 +17,6 @@ import { CrawlingContext } from '@crawlee/core';import { Dataset } from '@crawlee/core';import type { DatasetExportOptions } from '@crawlee/core';import type { Dictionary } from '@crawlee/types';-import { Element as Element_2 } from '@crawlee/utils/internal';import type { EnqueueUrlsOptions } from '@crawlee/core';import { EventManager } from '@crawlee/core';import type { FinalStatistics } from '@crawlee/core';@@ -48,7 +44,6 @@ import { StorageWritePolicy } from '@crawlee/core';import type { TaskLoopPredicates } from '@crawlee/core';import { TimeoutError } from '@apify/timeout';import type { TypedRequestsLike } from '@crawlee/core';-import { z } from 'zod';// @public (undocumented)export class BasicCrawler<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> {@@ -62,40 +57,26 @@ export class BasicCrawler<Context extends CrawlingContext = CrawlingContext, Con// (undocumented)protected blockedStatusCodes: Set<number>;protected buildContextPipeline(): ContextPipeline<CrawlingContext, CrawlingContext>;- // (undocumented)- protected calculateEnqueuedRequestLimit(explicitLimit?: number): Promise<number | undefined>;get concurrencySystem(): IConcurrencySystem | undefined;// (undocumented)get contextPipeline(): ContextPipeline<CrawlingContext, ExtendedContext>;- // (undocumented)- protected static readonly CRAWLEE_STATE_KEY = "CRAWLEE_STATE";protected createDefaultConcurrencySystem(options: ConcurrencySystemOptions): ConcurrencySystem;- // (undocumented)- protected readonly errorHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;exportData<Data>(path: string, format?: 'json' | 'csv', options?: DatasetExportOptions): Promise<Data[]>;// (undocumented)- protected readonly failedRequestHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;- // (undocumented)protected getCookieHeaderFromRequest(request: Request_2): string;getData(...args: Parameters<Dataset['getData']>): ReturnType<Dataset['getData']>;getDataset(identifier?: string | StorageIdentifier): Promise<Dataset>;protected getMessageFromError(error: Error, forceStack?: boolean): string | TimeoutError | undefined;protected getNavigationTimeoutMillis(): number;- // (undocumented)- protected getPendingRequestCountApproximation(): Promise<number>;getRequestManager(): Promise<IRequestManager>;// @deprecated (undocumented)getRequestQueue(): Promise<IRequestManager>;// (undocumented)protected getRobotsTxtFileForUrl(url: string): Promise<RobotsTxtFile | undefined>;// (undocumented)- protected handleSkippedRequest(options: Parameters<SkippedRequestCallback>[0]): Promise<void>;- // (undocumented)hasFinishedBefore: boolean;// (undocumented)protected readonly httpClient: BaseHttpClient;- // (undocumented)- protected readonly identity: CrawlerIdentity;protected init(): Promise<void>;// (undocumented)protected readonly internalTimeoutMillis: number;@@ -103,104 +84,6 @@ export class BasicCrawler<Context extends CrawlingContext = CrawlingContext, Conprotected isProxyError(error: Error): boolean;// (undocumented)get log(): CrawleeLogger;- // (undocumented)- protected readonly maxCrawlDepth?: number;- // (undocumented)- protected readonly maxRequestRetries: number;- // (undocumented)- protected readonly maxRequestsPerCrawl?: number;- // (undocumented)- protected readonly onSkippedRequest?: SkippedRequestCallback;- // (undocumented)- protected static optionsSchema: z.ZodObject<{- contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;- errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;- sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;- maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;- maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;- taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;- statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;- ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;- blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;- retryOnBlocked: z.ZodDefault<z.ZodBoolean>;- respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;- transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{- requestQueue: z.ZodOptional<z.ZodEnum<{- deferred: "deferred";- writeThrough: "writeThrough";- }>>;- }, z.core.$strict>]>>;- onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- httpClient: z.ZodOptional<z.ZodCustom<BaseHttpClient, BaseHttpClient>>;- configuration: z.ZodOptional<z.ZodCustom<Configuration, Configuration>>;- storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- eventManager: z.ZodOptional<z.ZodCustom<EventManager, EventManager>>;- logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;- maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;- maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;- keepAlive: z.ZodOptional<z.ZodBoolean>;- statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- id: z.ZodOptional<z.ZodString>;- }, z.core.$strict>;- // (undocumented)- protected static optionsShape: {- contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;- errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;- sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;- maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;- maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;- taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;- statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;- ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;- blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;- retryOnBlocked: z.ZodDefault<z.ZodBoolean>;- respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;- transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{- requestQueue: z.ZodOptional<z.ZodEnum<{- deferred: "deferred";- writeThrough: "writeThrough";- }>>;- }, z.core.$strict>]>>;- onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- httpClient: z.ZodOptional<z.ZodCustom<BaseHttpClient, BaseHttpClient>>;- configuration: z.ZodOptional<z.ZodCustom<Configuration, Configuration>>;- storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- eventManager: z.ZodOptional<z.ZodCustom<EventManager, EventManager>>;- logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;- maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;- maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;- keepAlive: z.ZodOptional<z.ZodBoolean>;- statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- id: z.ZodOptional<z.ZodString>;- };pause(timeoutSecs?: number): Promise<void>;readonly proxyConfiguration?: IProxyConfiguration;pushData(data: Parameters<Dataset['pushData']>[0], datasetIdentifier?: string | StorageIdentifier): Promise<void>;@@ -276,12 +159,6 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingCexport interface BasicCrawlingContext<UserData extends Dictionary = Dictionary> extends CrawlingContext<UserData> {}-export { Cheerio }--export { CheerioAPI }--export { CheerioRoot }-// @public (undocumented)export interface CrawlerAddRequestsOptions extends AddRequestsBatchedOptions, EnqueueUrlsOptions {}@@ -290,14 +167,6 @@ export interface CrawlerAddRequestsOptions extends AddRequestsBatchedOptions, Enexport interface CrawlerAddRequestsResult extends AddRequestsBatchedResult {}-// Not exported by the entry point; reachable only as a referenced type.-// @public-interface CrawlerIdentity {- readonly hasExplicitId: boolean;- readonly id: string;- readonly instanceIndex: number;-}-// @public (undocumented)export interface CrawlerRunOptions extends CrawlerAddRequestsOptions {purgeRequestQueue?: boolean;@@ -319,8 +188,6 @@ export interface CreateContextOptions {session: ISession;}-export { Element_2 as Element }-// @publicexport type ErrorHandler<BaseContext extends CrawlingContext = CrawlingContext, ExtendedContext extends BaseContext = BaseContext> = (inputs: BaseContext & Partial<ExtendedContext>, error: Error) => Awaitable<void>;diff --git a/docs/public-api/crawlee-browser.api.md b/docs/public-api/crawlee-browser.api.mdindex da5aff65b..ab664946a 100644--- a/docs/public-api/crawlee-browser.api.md+++ b/docs/public-api/crawlee-browser.api.md@@ -6,21 +6,18 @@import type { AddRequestsBatchedResult } from '@crawlee/basic';import type { Awaitable } from '@crawlee/types';-import { BaseHttpClient } from '@crawlee/http-client';import { BasicCrawler } from '@crawlee/basic';import { BasicCrawlerOptions } from '@crawlee/basic';import type { BrowserPluginOptions } from '@crawlee/browser-pool';import type { BrowserPoolHooks } from '@crawlee/browser-pool';import type { BrowserPoolOptions } from '@crawlee/browser-pool';import type { CommonPage } from '@crawlee/browser-pool';-import { Configuration } from '@crawlee/basic';import { ContextPipeline } from '@crawlee/basic';import type { CrawlerRemoteBrowserOptions } from '@crawlee/browser-pool';import type { CrawlingContext } from '@crawlee/basic';import type { Dictionary } from '@crawlee/types';import type { EnqueueLinksOptions } from '@crawlee/basic';import type { ErrorHandler } from '@crawlee/basic';-import { EventManager } from '@crawlee/basic';import type { ExtractLinksOptions } from '@crawlee/basic';import type { GetUserDataFromRequest } from '@crawlee/basic';import type { IBrowserPool } from '@crawlee/types';@@ -29,7 +26,6 @@ import type { RemoteBrowserPoolOptions } from '@crawlee/browser-pool';import { Request as Request_2 } from '@crawlee/basic';import type { RequestHandler } from '@crawlee/basic';import type { RouterHandler } from '@crawlee/basic';-import { z } from 'zod';// @publicexport function assertBrowserPoolNotConfigured(crawlerName: string, ignoredOptions: Dictionary): void;@@ -61,116 +57,6 @@ export abstract class BrowserCrawler<Page extends CommonPage = CommonPage, RespolaunchContext: BrowserLaunchContext<LaunchOptions, unknown>;// (undocumented)protected abstract navigationHandler(crawlingContext: BrowserCrawlingContext<Page, Response>, gotoOptions: GoToOptions): Promise<Context['response'] | null | undefined>;- // (undocumented)- protected static optionsSchema: z.ZodObject<{- navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;- preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;- postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;- launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;- browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- saveResponseCookies: z.ZodDefault<z.ZodBoolean>;- proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- ignoreIframes: z.ZodDefault<z.ZodBoolean>;- ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;- contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;- errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;- sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;- maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;- maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;- taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;- statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;- ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;- blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;- retryOnBlocked: z.ZodDefault<z.ZodBoolean>;- respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;- transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{- requestQueue: z.ZodOptional<z.ZodEnum<{- deferred: "deferred";- writeThrough: "writeThrough";- }>>;- }, z.core.$strict>]>>;- onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- httpClient: z.ZodOptional<z.ZodCustom<BaseHttpClient, BaseHttpClient>>;- configuration: z.ZodOptional<z.ZodCustom<Configuration, Configuration>>;- storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- eventManager: z.ZodOptional<z.ZodCustom<EventManager, EventManager>>;- logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;- maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;- maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;- keepAlive: z.ZodOptional<z.ZodBoolean>;- statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- id: z.ZodOptional<z.ZodString>;- }, z.core.$strict>;- // (undocumented)- protected static optionsShape: {- navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;- preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;- postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;- launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;- browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- saveResponseCookies: z.ZodDefault<z.ZodBoolean>;- proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- ignoreIframes: z.ZodDefault<z.ZodBoolean>;- ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;- contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;- errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;- sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;- maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;- maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;- taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;- statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;- ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;- blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;- retryOnBlocked: z.ZodDefault<z.ZodBoolean>;- respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;- transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{- requestQueue: z.ZodOptional<z.ZodEnum<{- deferred: "deferred";- writeThrough: "writeThrough";- }>>;- }, z.core.$strict>]>>;- onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- httpClient: z.ZodOptional<z.ZodCustom<BaseHttpClient, BaseHttpClient>>;- configuration: z.ZodOptional<z.ZodCustom<Configuration, Configuration>>;- storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- eventManager: z.ZodOptional<z.ZodCustom<EventManager, EventManager>>;- logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;- maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;- maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;- keepAlive: z.ZodOptional<z.ZodBoolean>;- statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- id: z.ZodOptional<z.ZodString>;- };protected runRequestHandler(crawlingContext: ExtendedContext): Promise<void>;}diff --git a/docs/public-api/crawlee-cheerio.api.md b/docs/public-api/crawlee-cheerio.api.mdindex 6c1168a23..45d334529 100644--- a/docs/public-api/crawlee-cheerio.api.md+++ b/docs/public-api/crawlee-cheerio.api.md@@ -7,8 +7,7 @@import type { AddRequestsBatchedResult } from '@crawlee/http';import * as cheerio from 'cheerio';import type { CheerioAPI } from 'cheerio';-import { CheerioRoot } from '@crawlee/utils/internal';-import { ContextPipeline } from '@crawlee/http';+import type { ContextPipeline } from '@crawlee/http';import type { CrawlingContext } from '@crawlee/http';import type { Dictionary } from '@crawlee/types';import type { EnqueueLinksOptions } from '@crawlee/http';@@ -29,15 +28,7 @@ import type { RoutesFromSchemas } from '@crawlee/http';export class CheerioCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends CheerioCrawlingContext = CheerioCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<CheerioCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends HttpCrawler<CheerioCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {constructor(options?: CheerioCrawlerOptions<ContextExtension, ExtendedContext, any, any, Routes, StatisticStateExtension>);// (undocumented)- protected buildContextPipeline(): ContextPipeline<CrawlingContext<Dictionary>, InternalHttpCrawlingContext<any, any> & {- readonly body: string;- readonly $: CheerioAPI;- } & {- extractLinks: (options?: ExtractLinksOptions) => Promise<string[]>;- enqueueLinks: (options?: EnqueueLinksOptions) => Promise<AddRequestsBatchedResult>;- waitForSelector: (selector: string, _timeoutMs?: number) => Promise<void>;- parseWithCheerio: (selector?: string, timeoutMs?: number) => Promise<CheerioAPI>;- }>;+ protected buildContextPipeline(): ContextPipeline<CrawlingContext, CheerioCrawlingContext>;}// @public (undocumented)@@ -53,7 +44,7 @@ JSONData extends Dictionary = any> extends InternalHttpCrawlingContext<UserData,body: string;enqueueLinks(options?: EnqueueLinksOptions): Promise<AddRequestsBatchedResult>;extractLinks(options?: ExtractLinksOptions): Promise<string[]>;- parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;+ parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;waitForSelector(selector: string, timeoutMs?: number): Promise<void>;}diff --git a/docs/public-api/crawlee-core.api.md b/docs/public-api/crawlee-core.api.mdindex 3b24c8034..f7ded0625 100644--- a/docs/public-api/crawlee-core.api.md+++ b/docs/public-api/crawlee-core.api.md@@ -31,7 +31,6 @@ import { LoggerJson } from '@apify/log';import type { LoggerOptions } from '@apify/log';import { LoggerText } from '@apify/log';import { LogLevel } from '@apify/log';-import { parseArgument } from '@crawlee/utils';import { ParseSitemapOptions } from '@crawlee/utils';import type { ProcessedRequest } from '@crawlee/types';import type { ProxyInfo } from '@crawlee/types';@@ -39,7 +38,6 @@ import type { QueueOperationInfo } from '@crawlee/types';import type { ReadonlyDeep } from 'type-fest';import type { RequestQueueBackend } from '@crawlee/types';import type { RequestQueueInfo } from '@crawlee/types';-import { schemas } from '@crawlee/utils/internal';import type { SendRequestOptions } from '@crawlee/types';import type { SessionFingerprint } from '@crawlee/types';import { SessionState } from '@crawlee/types';@@ -47,7 +45,6 @@ import type { StandardSchemaV1 } from '@standard-schema/spec';import type * as storage from '@crawlee/types';import { StorageBackend } from '@crawlee/types';import { StorageIdentifier } from '@crawlee/types';-import { tryAbsoluteURL } from '@crawlee/utils/internal';import { z } from 'zod';import { ZodType } from 'zod';@@ -357,47 +354,6 @@ export class Dataset<Data extends Dictionary = Dictionary> {values(options?: DatasetIteratorOptions): AsyncIterable<Data> & Promise<Data[]>;}-// Not exported by the entry point; reachable only as a referenced type.-// @public (undocumented)-class DatasetBackend_2<Data extends Dictionary = Dictionary> extends BaseClient implements storage.DatasetBackend<Data> {- constructor(options: DatasetBackendOptions);- // (undocumented)- accessedAt: Date;- cacheKey: string;- // (undocumented)- createdAt: Date;- // (undocumented)- drop(): Promise<void>;- // (undocumented)- getData(options?: storage.DatasetBackendListOptions): Promise<storage.PaginatedList<Data>>;- // (undocumented)- getMetadata(): Promise<storage.DatasetInfo>;- // (undocumented)- itemCount: number;- // (undocumented)- modifiedAt: Date;- // (undocumented)- name?: string;- // (undocumented)- purge(): Promise<void>;- // (undocumented)- pushData(items: Data[]): Promise<void>;- // (undocumented)- toDatasetInfo(): storage.DatasetInfo;-}--// Not exported by the entry point; reachable only as a referenced type.-// @public (undocumented)-interface DatasetBackendOptions {- cacheKey?: string;- // (undocumented)- id?: string;- // (undocumented)- name?: string;- // (undocumented)- storageBackend: MemoryStorageBackend;-}-// @publicexport interface DatasetConsumer<Data> {// (undocumented)@@ -881,52 +837,6 @@ export class KeyValueStore {values<T = unknown>(options?: KeyValueStoreIteratorOptions): AsyncIterable<T> & Promise<T[]>;}-// Not exported by the entry point; reachable only as a referenced type.-// @public (undocumented)-class KeyValueStoreBackend_2 extends BaseClient implements storage.KeyValueStoreBackend {- constructor(options: KeyValueStoreBackendOptions);- // (undocumented)- accessedAt: Date;- cacheKey: string;- // (undocumented)- createdAt: Date;- // (undocumented)- deleteValue(key: string): Promise<void>;- // (undocumented)- drop(): Promise<void>;- // (undocumented)- getMetadata(): Promise<storage.KeyValueStoreInfo>;- getPublicUrl(key: string): Promise<string | undefined>;- // (undocumented)- getValue(key: string): Promise<storage.KeyValueStoreRecord | undefined>;- // (undocumented)- listKeys(options?: storage.KeyValueStoreListKeysOptions): Promise<storage.KeyValueStoreListKeysResult>;- // (undocumented)- modifiedAt: Date;- // (undocumented)- name?: string;- // (undocumented)- purge(): Promise<void>;- purgeExceptInput(): Promise<void>;- recordExists(key: string): Promise<boolean>;- // (undocumented)- setValue(record: storage.KeyValueStoreInputRecord): Promise<void>;- // (undocumented)- toKeyValueStoreInfo(): storage.KeyValueStoreInfo;-}--// Not exported by the entry point; reachable only as a referenced type.-// @public (undocumented)-interface KeyValueStoreBackendOptions {- cacheKey?: string;- // (undocumented)- id?: string;- // (undocumented)- name?: string;- // (undocumented)- storageBackend: MemoryStorageBackend;-}-// @public (undocumented)export interface KeyValueStoreIteratorOptions {prefix?: string;@@ -1085,17 +995,11 @@ export class MemoryStorageBackend implements storage.StorageBackend {createKeyValueStoreBackend(options?: storage.StorageIdentifier): Promise<storage.KeyValueStoreBackend>;// (undocumented)createRequestQueueBackend(options?: storage.StorageIdentifier): Promise<RequestQueueBackend_2>;- // (undocumented)- readonly datasetBackendCache: DatasetBackend_2[];getStorageBackendCacheKey(): string;// (undocumented)- readonly keyValueStoreBackendCache: KeyValueStoreBackend_2[];- // (undocumented)readonly logger?: CrawleeLogger;purge(): Promise<void>;// (undocumented)- readonly requestQueueBackendCache: RequestQueueBackend_2[];- // (undocumented)storageExists(id: string, type: 'Dataset' | 'KeyValueStore' | 'RequestQueue'): Promise<boolean>;teardown(): Promise<void>;}@@ -1125,8 +1029,6 @@ interface NewUrlOptions {export class NonRetryableError extends Error {}-export { parseArgument }-// @publicexport function parseRetryAfterHeader(value?: string | null): number | null;@@ -1643,8 +1545,6 @@ export interface SchemaIssue {})[];}-export { schemas }-// Not exported by the entry point; reachable only as a referenced type.// @publictype SchemaUserData<Schema extends StandardSchemaV1> = StandardSchemaV1.InferOutput<Schema> extends Dictionary ? StandardSchemaV1.InferOutput<Schema> : Dictionary;@@ -2218,8 +2118,6 @@ export interface ThrottlingRequestManagerOptions<T extends IRequestManager = IRethrottleBy?: 'hostname' | 'registrableDomain';}-export { tryAbsoluteURL }-// @publicexport type TypedContextAddRequests<Routes extends Record<keyof Routes, Dictionary>> = (requestsLike: ReadonlyDeep<LabeledSource<Routes>[]>, options?: ReadonlyDeep<EnqueueUrlsOptions>) => Promise<AddRequestsBatchedResult>;diff --git a/docs/public-api/crawlee-fs-storage.api.md b/docs/public-api/crawlee-fs-storage.api.mdindex 206442236..16fad365e 100644--- a/docs/public-api/crawlee-fs-storage.api.md+++ b/docs/public-api/crawlee-fs-storage.api.md@@ -5,54 +5,8 @@```tsimport type { CrawleeLogger } from '@crawlee/types';-import type { Dictionary } from '@crawlee/types';-import type { FileSystemDatasetClient } from '@crawlee/fs-storage-native';-import type { FileSystemKeyValueStoreClient } from '@crawlee/fs-storage-native';-import type { FileSystemRequestQueueClient } from '@crawlee/fs-storage-native';import type * as storage from '@crawlee/types';-// Not exported by the entry point; reachable only as a referenced type.-// @public-abstract class CachedIdClient {- protected cachedId: string;- get id(): string;-}--// Not exported by the entry point; reachable only as a referenced type.-// @public-class DatasetBackend<Data extends Dictionary = Dictionary> extends CachedIdClient implements storage.DatasetBackend<Data> {- constructor(options: DatasetBackendOptions);- // (undocumented)- readonly cacheKey: string;- // (undocumented)- static create<Data extends Dictionary = Dictionary>(options: DatasetBackendOptions): Promise<DatasetBackend<Data>>;- // (undocumented)- get datasetDirectory(): string;- // (undocumented)- drop(): Promise<void>;- // (undocumented)- getData(options?: storage.DatasetBackendListOptions): Promise<storage.PaginatedList<Data>>;- // (undocumented)- getMetadata(): Promise<storage.DatasetInfo>;- // (undocumented)- readonly name?: string;- // (undocumented)- purge(): Promise<void>;- // (undocumented)- pushData(items: Data[]): Promise<void>;-}--// Not exported by the entry point; reachable only as a referenced type.-// @public (undocumented)-interface DatasetBackendOptions {- cacheKey: string;- // (undocumented)- logger?: CrawleeLogger;- name?: string;- // (undocumented)- nativeBackend: FileSystemDatasetClient;-}-// @publicexport class FileSystemStorageBackend implements storage.StorageBackend {constructor(options: FileSystemStorageOptions);@@ -63,13 +17,9 @@ export class FileSystemStorageBackend implements storage.StorageBackend {// (undocumented)createRequestQueueBackend(options?: storage.StorageIdentifier): Promise<storage.RequestQueueBackend>;// (undocumented)- readonly datasetBackendCache: DatasetBackend[];- // (undocumented)readonly datasetsDirectory: string;getStorageBackendCacheKey(): string;// (undocumented)- readonly keyValueStoreBackendCache: KeyValueStoreBackend[];- // (undocumented)readonly keyValueStoresDirectory: string;// (undocumented)readonly localDataDirectory: string;@@ -79,8 +29,6 @@ export class FileSystemStorageBackend implements storage.StorageBackend {// (undocumented)readonly requestQueueAccess: 'single' | 'shared';// (undocumented)- readonly requestQueueBackendCache: RequestQueueBackend[];- // (undocumented)readonly requestQueuesDirectory: string;// (undocumented)storageExists(id: string, type: 'Dataset' | 'KeyValueStore' | 'RequestQueue'): Promise<boolean>;@@ -94,95 +42,6 @@ export interface FileSystemStorageOptions {requestQueueAccess?: 'single' | 'shared';}-// Not exported by the entry point; reachable only as a referenced type.-// @public-class KeyValueStoreBackend extends CachedIdClient implements storage.KeyValueStoreBackend {- constructor(options: KeyValueStoreBackendOptions);- // (undocumented)- readonly cacheKey: string;- // (undocumented)- static create(options: KeyValueStoreBackendOptions): Promise<KeyValueStoreBackend>;- // (undocumented)- deleteValue(key: string): Promise<void>;- // (undocumented)- drop(): Promise<void>;- // (undocumented)- getMetadata(): Promise<storage.KeyValueStoreInfo>;- getPublicUrl(key: string): Promise<string | undefined>;- // (undocumented)- getValue(key: string): Promise<storage.KeyValueStoreRecord | undefined>;- // (undocumented)- get keyValueStoreDirectory(): string;- // (undocumented)- listKeys(options?: storage.KeyValueStoreListKeysOptions): Promise<storage.KeyValueStoreListKeysResult>;- // (undocumented)- readonly name?: string;- // (undocumented)- purge(): Promise<void>;- purgeExceptInput(): Promise<void>;- recordExists(key: string): Promise<boolean>;- // (undocumented)- setValue(record: storage.KeyValueStoreInputRecord): Promise<void>;-}--// Not exported by the entry point; reachable only as a referenced type.-// @public (undocumented)-interface KeyValueStoreBackendOptions {- cacheKey: string;- // (undocumented)- logger?: CrawleeLogger;- name?: string;- // (undocumented)- nativeBackend: FileSystemKeyValueStoreClient;-}--// Not exported by the entry point; reachable only as a referenced type.-// @public-class RequestQueueBackend extends CachedIdClient implements storage.RequestQueueBackend {- constructor(options: RequestQueueBackendOptions);- // (undocumented)- addBatchOfRequests(requests: storage.RequestSchema[], options?: storage.RequestQueueOperationOptions): Promise<storage.BatchAddRequestsResult>;- // (undocumented)- readonly cacheKey: string;- // (undocumented)- static create(options: RequestQueueBackendOptions): Promise<RequestQueueBackend>;- // (undocumented)- drop(): Promise<void>;- // (undocumented)- fetchNextRequest(): Promise<storage.UpdateRequestSchema | undefined>;- // (undocumented)- getMetadata(): Promise<storage.RequestQueueInfo>;- // (undocumented)- getRequest(uniqueKey: string): Promise<storage.UpdateRequestSchema | undefined>;- // (undocumented)- isEmpty(): Promise<boolean>;- // (undocumented)- isFinished(): Promise<boolean>;- // (undocumented)- markRequestAsHandled(request: storage.UpdateRequestSchema): Promise<storage.QueueOperationInfo | undefined>;- // (undocumented)- readonly name?: string;- persistState(): Promise<void>;- // (undocumented)- purge(): Promise<void>;- // (undocumented)- reclaimRequest(request: storage.UpdateRequestSchema, options?: storage.RequestQueueOperationOptions): Promise<storage.QueueOperationInfo | undefined>;- // (undocumented)- get requestQueueDirectory(): string;- setExpectedRequestProcessingTimeSecs(secs: number): Promise<void>;-}--// Not exported by the entry point; reachable only as a referenced type.-// @public (undocumented)-interface RequestQueueBackendOptions {- cacheKey: string;- // (undocumented)- logger?: CrawleeLogger;- name?: string;- // (undocumented)- nativeBackend: FileSystemRequestQueueClient;-}-// (No @packageDocumentation comment for this package)```diff --git a/docs/public-api/crawlee-http.api.md b/docs/public-api/crawlee-http.api.mdindex a192462d5..f7e08622e 100644--- a/docs/public-api/crawlee-http.api.md+++ b/docs/public-api/crawlee-http.api.md@@ -5,19 +5,17 @@```tsimport type { Awaitable } from '@crawlee/types';-import { BaseHttpClient } from '@crawlee/http-client';import { BasicCrawler } from '@crawlee/basic';import { BasicCrawlerOptions } from '@crawlee/basic';-import { CheerioRoot } from '@crawlee/utils/internal';+import type { CheerioAPI } from 'cheerio';import { ConcurrencySystem } from '@crawlee/basic';import type { ConcurrencySystemOptions } from '@crawlee/basic';-import { Configuration } from '@crawlee/basic';import { ContextPipeline } from '@crawlee/basic';+import type { ContextPipeline as ContextPipeline_2 } from '@crawlee/core';import type { CrawlingContext } from '@crawlee/basic';import type { CrawlingContext as CrawlingContext_2 } from '@crawlee/core';import type { Dictionary } from '@crawlee/types';import { ErrorHandler } from '@crawlee/basic';-import { EventManager } from '@crawlee/basic';import { GetUserDataFromRequest } from '@crawlee/basic';import type { JsonValue } from 'type-fest';import { LoadedRequest } from '@crawlee/core';@@ -25,13 +23,11 @@ import { Request as Request_2 } from '@crawlee/basic';import type { Request as Request_3 } from '@crawlee/core';import { RequestHandler } from '@crawlee/basic';import type { RequireContextPipeline } from '@crawlee/basic';-import { ResponseWithUrl } from '@crawlee/http-client';import { RouterHandler } from '@crawlee/basic';import { RouterRoutes } from '@crawlee/basic';import type { RouteSchemas } from '@crawlee/basic';import type { RoutesFromSchemas } from '@crawlee/basic';import { Transform } from 'node:stream';-import { z } from 'zod';// @publicexport function ByteCounterStream(input: {@@ -62,15 +58,7 @@ export function createHttpRouter<Context extends HttpCrawlingContext = HttpCrawlexport class FileDownload extends BasicCrawler<FileDownloadCrawlingContext> {constructor(options?: BasicCrawlerOptions<FileDownloadCrawlingContext>);// (undocumented)- protected buildContextPipeline(): ContextPipeline<CrawlingContext_2<Dictionary>, CrawlingContext_2<Dictionary> & {- request: LoadedRequest<Request_3>;- response: ResponseWithUrl;- contentType: {- type: string;- encoding: BufferEncoding;- };- [kBodyDrained]: Promise<void>;- }>;+ protected buildContextPipeline(): ContextPipeline_2<CrawlingContext_2, FileDownloadCrawlingContext>;}// @public (undocumented)@@ -109,112 +97,6 @@ export class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any> =protected getNavigationTimeoutMillis(): number;// (undocumented)protected isRequestBlocked(crawlingContext: InternalHttpCrawlingContext): Promise<string | false>;- // (undocumented)- protected static optionsSchema: z.ZodObject<{- navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;- ignoreTlsErrors: z.ZodDefault<z.ZodBoolean>;- additionalMimeTypes: z.ZodDefault<z.ZodArray<z.ZodString>>;- suggestResponseEncoding: z.ZodOptional<z.ZodString>;- forceResponseEncoding: z.ZodOptional<z.ZodString>;- saveResponseCookies: z.ZodDefault<z.ZodBoolean>;- preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;- postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;- contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;- errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;- sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;- maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;- maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;- taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;- statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;- ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;- blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;- retryOnBlocked: z.ZodDefault<z.ZodBoolean>;- respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;- transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{- requestQueue: z.ZodOptional<z.ZodEnum<{- deferred: "deferred";- writeThrough: "writeThrough";- }>>;- }, z.core.$strict>]>>;- onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- httpClient: z.ZodOptional<z.ZodCustom<BaseHttpClient, BaseHttpClient>>;- configuration: z.ZodOptional<z.ZodCustom<Configuration, Configuration>>;- storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- eventManager: z.ZodOptional<z.ZodCustom<EventManager, EventManager>>;- logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;- maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;- maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;- keepAlive: z.ZodOptional<z.ZodBoolean>;- statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- id: z.ZodOptional<z.ZodString>;- }, z.core.$strict>;- // (undocumented)- protected static optionsShape: {- navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;- ignoreTlsErrors: z.ZodDefault<z.ZodBoolean>;- additionalMimeTypes: z.ZodDefault<z.ZodArray<z.ZodString>>;- suggestResponseEncoding: z.ZodOptional<z.ZodString>;- forceResponseEncoding: z.ZodOptional<z.ZodString>;- saveResponseCookies: z.ZodDefault<z.ZodBoolean>;- preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;- postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;- contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;- errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;- sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;- maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;- maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;- taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;- statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;- ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;- blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;- retryOnBlocked: z.ZodDefault<z.ZodBoolean>;- respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;- transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{- requestQueue: z.ZodOptional<z.ZodEnum<{- deferred: "deferred";- writeThrough: "writeThrough";- }>>;- }, z.core.$strict>]>>;- onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- httpClient: z.ZodOptional<z.ZodCustom<BaseHttpClient, BaseHttpClient>>;- configuration: z.ZodOptional<z.ZodCustom<Configuration, Configuration>>;- storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- eventManager: z.ZodOptional<z.ZodCustom<EventManager, EventManager>>;- logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;- maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;- maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;- keepAlive: z.ZodOptional<z.ZodBoolean>;- statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- id: z.ZodOptional<z.ZodString>;- };}// @public (undocumented)@@ -255,16 +137,13 @@ JSONData extends JsonValue = any> extends CrawlingContextWithResponse<UserData>encoding: BufferEncoding;};json: JSONData;- parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;+ parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;waitForSelector(selector: string, timeoutMs?: number): Promise<void>;}// @public (undocumented)export type InternalHttpHook<Context, ContextExtension = {}> = (crawlingContext: Context & ContextExtension) => Awaitable<void | Partial<Context>>;-// @public (undocumented)-const kBodyDrained: unique symbol;-// @publicexport function MinimumSpeedStream(input: {minSpeedKbps: number;diff --git a/docs/public-api/crawlee-jsdom.api.md b/docs/public-api/crawlee-jsdom.api.mdindex 82fbb21b7..6a4aef0fb 100644--- a/docs/public-api/crawlee-jsdom.api.md+++ b/docs/public-api/crawlee-jsdom.api.md@@ -5,17 +5,13 @@```tsimport type { AddRequestsBatchedResult } from '@crawlee/http';-import { BaseHttpClient } from '@crawlee/http-client';-import { CheerioAPI } from 'cheerio';-import { CheerioRoot } from '@crawlee/utils/internal';-import { Configuration } from '@crawlee/http';-import { ContextPipeline } from '@crawlee/http';+import type { CheerioAPI } from 'cheerio';+import type { ContextPipeline } from '@crawlee/http';import type { CrawlingContext } from '@crawlee/http';import type { Dictionary } from '@crawlee/types';import type { DOMWindow } from 'jsdom';import type { EnqueueLinksOptions } from '@crawlee/http';import type { ErrorHandler } from '@crawlee/http';-import { EventManager } from '@crawlee/http';import type { ExtractLinksOptions } from '@crawlee/http';import type { GetUserDataFromRequest } from '@crawlee/http';import { HttpCrawler } from '@crawlee/http';@@ -28,7 +24,6 @@ import type { RouterRoutes } from '@crawlee/http';import type { RouteSchemas } from '@crawlee/http';import type { RoutesFromSchemas } from '@crawlee/http';import { VirtualConsole } from 'jsdom';-import { z } from 'zod';// @publicexport function createJSDOMRouter<Context extends JSDOMCrawlingContext = JSDOMCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;@@ -43,127 +38,8 @@ export function createJSDOMRouter<Context extends JSDOMCrawlingContext = JSDOMCrexport class JSDOMCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<JSDOMCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends HttpCrawler<JSDOMCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {constructor(options?: JSDOMCrawlerOptions<ContextExtension, ExtendedContext, any, any, Routes, StatisticStateExtension>);// (undocumented)- protected buildContextPipeline(): ContextPipeline<CrawlingContext<Dictionary>, InternalHttpCrawlingContext<any, any> & {- readonly window: DOMWindow;- readonly body: string;- readonly document: Document;- } & {- extractLinks: (options?: ExtractLinksOptions) => Promise<string[]>;- enqueueLinks: (options?: EnqueueLinksOptions) => Promise<AddRequestsBatchedResult>;- waitForSelector(selector: string, timeoutMs?: number): Promise<void>;- parseWithCheerio(selector?: string, _timeoutMs?: number): Promise<CheerioAPI>;- }>;+ protected buildContextPipeline(): ContextPipeline<CrawlingContext, JSDOMCrawlingContext>;getVirtualConsole(): VirtualConsole;- // (undocumented)- protected static optionsSchema: z.ZodObject<{- runScripts: z.ZodOptional<z.ZodBoolean>;- hideInternalConsole: z.ZodOptional<z.ZodBoolean>;- navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;- ignoreTlsErrors: z.ZodDefault<z.ZodBoolean>;- additionalMimeTypes: z.ZodDefault<z.ZodArray<z.ZodString>>;- suggestResponseEncoding: z.ZodOptional<z.ZodString>;- forceResponseEncoding: z.ZodOptional<z.ZodString>;- saveResponseCookies: z.ZodDefault<z.ZodBoolean>;- preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;- postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;- contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;- errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;- sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;- maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;- maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;- taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;- statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;- ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;- blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;- retryOnBlocked: z.ZodDefault<z.ZodBoolean>;- respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;- transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{- requestQueue: z.ZodOptional<z.ZodEnum<{- deferred: "deferred";- writeThrough: "writeThrough";- }>>;- }, z.core.$strict>]>>;- onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- httpClient: z.ZodOptional<z.ZodCustom<BaseHttpClient, BaseHttpClient>>;- configuration: z.ZodOptional<z.ZodCustom<Configuration, Configuration>>;- storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- eventManager: z.ZodOptional<z.ZodCustom<EventManager, EventManager>>;- logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;- maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;- maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;- keepAlive: z.ZodOptional<z.ZodBoolean>;- statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- id: z.ZodOptional<z.ZodString>;- }, z.core.$strict>;- // (undocumented)- protected static optionsShape: {- runScripts: z.ZodOptional<z.ZodBoolean>;- hideInternalConsole: z.ZodOptional<z.ZodBoolean>;- navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;- ignoreTlsErrors: z.ZodDefault<z.ZodBoolean>;- additionalMimeTypes: z.ZodDefault<z.ZodArray<z.ZodString>>;- suggestResponseEncoding: z.ZodOptional<z.ZodString>;- forceResponseEncoding: z.ZodOptional<z.ZodString>;- saveResponseCookies: z.ZodDefault<z.ZodBoolean>;- preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;- postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;- contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;- errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;- sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;- maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;- maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;- taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;- statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;- ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;- blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;- retryOnBlocked: z.ZodDefault<z.ZodBoolean>;- respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;- transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{- requestQueue: z.ZodOptional<z.ZodEnum<{- deferred: "deferred";- writeThrough: "writeThrough";- }>>;- }, z.core.$strict>]>>;- onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- httpClient: z.ZodOptional<z.ZodCustom<BaseHttpClient, BaseHttpClient>>;- configuration: z.ZodOptional<z.ZodCustom<Configuration, Configuration>>;- storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- eventManager: z.ZodOptional<z.ZodCustom<EventManager, EventManager>>;- logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;- maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;- maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;- keepAlive: z.ZodOptional<z.ZodBoolean>;- statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- id: z.ZodOptional<z.ZodString>;- };}// @public (undocumented)@@ -183,7 +59,7 @@ JSONData extends Dictionary = any> extends InternalHttpCrawlingContext<UserData,document: Document;enqueueLinks(options?: EnqueueLinksOptions): Promise<AddRequestsBatchedResult>;extractLinks(options?: ExtractLinksOptions): Promise<string[]>;- parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;+ parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;waitForSelector(selector: string, timeoutMs?: number): Promise<void>;// (undocumented)window: DOMWindow;diff --git a/docs/public-api/crawlee-linkedom.api.md b/docs/public-api/crawlee-linkedom.api.mdindex 1908f3dc5..69a083fd1 100644--- a/docs/public-api/crawlee-linkedom.api.md+++ b/docs/public-api/crawlee-linkedom.api.md@@ -5,9 +5,8 @@```tsimport type { AddRequestsBatchedResult } from '@crawlee/http';-import * as cheerio from 'cheerio';-import { CheerioRoot } from '@crawlee/utils/internal';-import { ContextPipeline } from '@crawlee/http';+import type { CheerioAPI } from 'cheerio';+import type { ContextPipeline } from '@crawlee/http';import type { CrawlingContext } from '@crawlee/http';import type { Dictionary } from '@crawlee/types';import type { EnqueueLinksOptions } from '@crawlee/http';@@ -37,16 +36,7 @@ export function createLinkeDOMRouter<Context extends LinkeDOMCrawlingContext = Lexport class LinkeDOMCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<LinkeDOMCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends HttpCrawler<LinkeDOMCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {constructor(options?: LinkeDOMCrawlerOptions<ContextExtension, ExtendedContext, any, any, Routes, StatisticStateExtension>);// (undocumented)- protected buildContextPipeline(): ContextPipeline<CrawlingContext<Dictionary>, InternalHttpCrawlingContext<any, any> & {- readonly window: Window;- readonly body: string;- readonly document: Document;- } & {- extractLinks: (options?: ExtractLinksOptions) => Promise<string[]>;- enqueueLinks: (options?: EnqueueLinksOptions) => Promise<AddRequestsBatchedResult>;- waitForSelector(selector: string, timeoutMs?: number): Promise<void>;- parseWithCheerio(selector?: string, _timeoutMs?: number): Promise<cheerio.CheerioAPI>;- }>;+ protected buildContextPipeline(): ContextPipeline<CrawlingContext, LinkeDOMCrawlingContext>;}// @public (undocumented)@@ -62,7 +52,7 @@ JSONData extends Dictionary = any> extends InternalHttpCrawlingContext<UserData,document: Document;enqueueLinks(options?: EnqueueLinksOptions): Promise<AddRequestsBatchedResult>;extractLinks(options?: ExtractLinksOptions): Promise<string[]>;- parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;+ parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;waitForSelector(selector: string, timeoutMs?: number): Promise<void>;// (undocumented)window: Window;diff --git a/docs/public-api/crawlee-playwright.api.md b/docs/public-api/crawlee-playwright.api.mdindex c18b6cdb5..c7c33ba67 100644--- a/docs/public-api/crawlee-playwright.api.md+++ b/docs/public-api/crawlee-playwright.api.md@@ -6,10 +6,9 @@import type { AnyNode } from 'domhandler';import type { Awaitable } from '@crawlee/types';-import { BaseHttpClient } from '@crawlee/http-client';import { BasicCrawler } from '@crawlee/basic';import type { BasicCrawlerOptions } from '@crawlee/basic';-import { BatchAddRequestsResult } from '@crawlee/types';+import type { BatchAddRequestsResult } from '@crawlee/types';import type { Browser } from 'playwright';import { BrowserCrawler } from '@crawlee/browser';import { BrowserCrawlerOptions } from '@crawlee/browser';@@ -21,17 +20,15 @@ import type { BrowserPoolHooks } from '@crawlee/browser-pool';import type { BrowserPoolOptions } from '@crawlee/browser-pool';import type { BrowserType } from 'playwright';import { Cheerio } from 'cheerio';-import { CheerioAPI } from '@crawlee/browser';-import { CheerioRoot } from '@crawlee/utils/internal';+import { CheerioAPI } from 'cheerio';import { Configuration } from '@crawlee/browser';-import { ContextPipeline } from '@crawlee/browser';+import type { ContextPipeline } from '@crawlee/browser';import type { ContextPipeline as ContextPipeline_2 } from '@crawlee/core';-import { CrawlingContext } from '@crawlee/browser';+import type { CrawlingContext } from '@crawlee/browser';import type { CrawlingContext as CrawlingContext_2 } from '@crawlee/core';import { Dictionary } from '@crawlee/types';import type { Download } from 'playwright';import type { EnqueueLinksOptions } from '@crawlee/core';-import { EventManager } from '@crawlee/browser';import type { GetUserDataFromRequest } from '@crawlee/browser';import type { GetUserDataFromRequest as GetUserDataFromRequest_2 } from '@crawlee/core';import { IRequestManager } from '@crawlee/browser';@@ -76,16 +73,7 @@ interface AdaptiveHookContext extends Pick<AdaptivePlaywrightCrawlerContext, 'idexport class AdaptivePlaywrightCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest_2<AdaptivePlaywrightCrawlerContext['request']>>, StatisticStateExtension extends AdaptivePlaywrightCrawlerStatisticState = AdaptivePlaywrightCrawlerStatisticState> extends BasicCrawler<AdaptivePlaywrightCrawlerContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {constructor(options?: AdaptivePlaywrightCrawlerOptions<ContextExtension, ExtendedContext, Routes, StatisticStateExtension>);// (undocumented)- protected buildContextPipeline(): ContextPipeline_2<CrawlingContext_2<Dictionary>, CrawlingContext_2<Dictionary> & {- readonly request: LoadedRequest<Request_3<Dictionary>>;- readonly response: Response;- readonly page: Page;- readonly querySelector: AdaptivePlaywrightCrawlerContext["querySelector"];- readonly querySelectorAll: AdaptivePlaywrightCrawlerContext["querySelectorAll"];- readonly waitForSelector: AdaptivePlaywrightCrawlerContext["waitForSelector"];- readonly parseWithCheerio: AdaptivePlaywrightCrawlerContext["parseWithCheerio"];- readonly enqueueLinks: AdaptivePlaywrightCrawlerContext["enqueueLinks"];- }>;+ protected buildContextPipeline(): ContextPipeline_2<CrawlingContext_2, AdaptivePlaywrightCrawlerContext>;// (undocumented)protected init(): Promise<void>;// (undocumented)@@ -99,7 +87,7 @@ export interface AdaptivePlaywrightCrawlerContext<UserData extends Dictionary =// (undocumented)enqueueLinks(options?: EnqueueLinksOptions): Promise<unknown>;page: Page;- parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;+ parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;querySelector(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;querySelectorAll(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;// (undocumented)@@ -279,7 +267,7 @@ export interface IRenderingTypePredictor {export function launchPlaywright(launchContext?: PlaywrightLaunchContext, configuration?: Configuration): Promise<Browser>;// @public-function parseWithCheerio(page: Page, ignoreShadowRoots?: boolean, ignoreIframes?: boolean): Promise<CheerioRoot>;+function parseWithCheerio(page: Page, ignoreShadowRoots?: boolean, ignoreIframes?: boolean): Promise<CheerioAPI>;// @publicexport type PlaywrightBrowserPool = BrowserPool<{@@ -316,7 +304,7 @@ interface PlaywrightContextUtils {injectFile(filePath: string, options?: InjectFileOptions): Promise<unknown>;injectJQuery(): Promise<unknown>;listDownloads(): Promise<Download[]>;- parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;+ parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;saveSnapshot(options?: SaveSnapshotOptions): Promise<void>;waitForSelector(selector: string, timeoutMs?: number): Promise<void>;}@@ -325,136 +313,9 @@ interface PlaywrightContextUtils {export class PlaywrightCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PlaywrightCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawler<Page, Response_2, LaunchOptions, PlaywrightCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {constructor(options?: PlaywrightCrawlerOptions<ContextExtension, ExtendedContext, Routes, StatisticStateExtension>);// (undocumented)- protected buildContextPipeline(): ContextPipeline<CrawlingContext<Dictionary>, BrowserCrawlingContext<Page, Response_2, Dictionary, Dictionary> & {- injectFile: (filePath: string, options?: InjectFileOptions) => Promise<unknown>;- injectJQuery: () => Promise<void>;- blockRequests: (options?: BlockRequestsOptions) => Promise<void>;- waitForSelector: (selector: string, timeoutMs?: number) => Promise<void>;- parseWithCheerio: (selector?: string, timeoutMs?: number) => Promise<CheerioAPI>;- infiniteScroll: (options?: InfiniteScrollOptions) => Promise<void>;- listDownloads: () => Promise<Download[]>;- saveSnapshot: (options?: SaveSnapshotOptions) => Promise<void>;- enqueueLinksByClickingElements: (options: Omit<EnqueueLinksByClickingElementsOptions, "page" | "requestManager">) => Promise<BatchAddRequestsResult>;- compileScript: (scriptString: string, ctx?: Dictionary) => CompiledScriptFunction;- closeCookieModals: () => Promise<void>;- handleCloudflareChallenge: (options?: HandleCloudflareChallengeOptions) => Promise<Response_2 | undefined>;- }>;+ protected buildContextPipeline(): ContextPipeline<CrawlingContext, PlaywrightCrawlingContext>;// (undocumented)protected navigationHandler(crawlingContext: PlaywrightCrawlingContext, gotoOptions: PlaywrightDirectNavigationOptions): Promise<Response_2 | null>;- // (undocumented)- protected static optionsSchema: z.ZodObject<{- headless: z.ZodOptional<z.ZodBoolean>;- launcher: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;- preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;- postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;- launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;- browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- saveResponseCookies: z.ZodDefault<z.ZodBoolean>;- proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- ignoreIframes: z.ZodDefault<z.ZodBoolean>;- ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;- contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;- errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;- sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;- maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;- maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;- taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;- statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;- ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;- blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;- retryOnBlocked: z.ZodDefault<z.ZodBoolean>;- respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;- transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{- requestQueue: z.ZodOptional<z.ZodEnum<{- deferred: "deferred";- writeThrough: "writeThrough";- }>>;- }, z.core.$strict>]>>;- onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- httpClient: z.ZodOptional<z.ZodCustom<BaseHttpClient, BaseHttpClient>>;- configuration: z.ZodOptional<z.ZodCustom<Configuration, Configuration>>;- storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- eventManager: z.ZodOptional<z.ZodCustom<EventManager, EventManager>>;- logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;- maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;- maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;- keepAlive: z.ZodOptional<z.ZodBoolean>;- statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- id: z.ZodOptional<z.ZodString>;- }, z.core.$strict>;- // (undocumented)- protected static optionsShape: {- headless: z.ZodOptional<z.ZodBoolean>;- launcher: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;- preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;- postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;- launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;- browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- saveResponseCookies: z.ZodDefault<z.ZodBoolean>;- proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- ignoreIframes: z.ZodDefault<z.ZodBoolean>;- ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;- contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;- errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;- sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;- maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;- maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;- taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;- statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;- ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;- blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;- retryOnBlocked: z.ZodDefault<z.ZodBoolean>;- respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;- transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{- requestQueue: z.ZodOptional<z.ZodEnum<{- deferred: "deferred";- writeThrough: "writeThrough";- }>>;- }, z.core.$strict>]>>;- onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- httpClient: z.ZodOptional<z.ZodCustom<BaseHttpClient, BaseHttpClient>>;- configuration: z.ZodOptional<z.ZodCustom<Configuration, Configuration>>;- storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- eventManager: z.ZodOptional<z.ZodCustom<EventManager, EventManager>>;- logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;- maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;- maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;- keepAlive: z.ZodOptional<z.ZodBoolean>;- statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- id: z.ZodOptional<z.ZodString>;- };}// @public (undocumented)diff --git a/docs/public-api/crawlee-puppeteer.api.md b/docs/public-api/crawlee-puppeteer.api.mdindex a545068b3..857ab7669 100644--- a/docs/public-api/crawlee-puppeteer.api.md+++ b/docs/public-api/crawlee-puppeteer.api.md@@ -4,8 +4,7 @@```ts-import { BaseHttpClient } from '@crawlee/http-client';-import { BatchAddRequestsResult } from '@crawlee/types';+import type { BatchAddRequestsResult } from '@crawlee/types';import type { Browser } from 'puppeteer';import { BrowserCrawler } from '@crawlee/browser';import { BrowserCrawlerOptions } from '@crawlee/browser';@@ -15,14 +14,12 @@ import type { BrowserLaunchContext } from '@crawlee/browser';import type { BrowserPool } from '@crawlee/browser-pool';import type { BrowserPoolHooks } from '@crawlee/browser-pool';import type { BrowserPoolOptions } from '@crawlee/browser-pool';-import { CheerioAPI } from '@crawlee/browser';-import { CheerioRoot } from '@crawlee/utils/internal';+import type { CheerioAPI } from 'cheerio';import type { ClickOptions } from 'puppeteer';import { Configuration } from '@crawlee/browser';-import { ContextPipeline } from '@crawlee/browser';-import { CrawlingContext } from '@crawlee/browser';+import type { ContextPipeline } from '@crawlee/browser';+import type { CrawlingContext } from '@crawlee/browser';import { Dictionary } from '@crawlee/types';-import { EventManager } from '@crawlee/browser';import type { GetUserDataFromRequest } from '@crawlee/browser';import type { HTTPRequest } from 'puppeteer';import type { HTTPResponse } from 'puppeteer';@@ -42,7 +39,6 @@ import type { RoutesFromSchemas } from '@crawlee/browser';import type { SkippedRequestCallback } from '@crawlee/browser';import type { Target } from 'puppeteer';import type { UrlPatternInput } from '@crawlee/browser';-import { z } from 'zod';// @publicfunction addInterceptRequestHandler(page: Page, handler: InterceptHandler): Promise<void>;@@ -148,7 +144,7 @@ function isTargetRelevant(page: Page, target: Target): boolean;export function launchPuppeteer(launchContext?: PuppeteerLaunchContext, configuration?: Configuration): Promise<Browser>;// @public-function parseWithCheerio(page: Page, ignoreShadowRoots?: boolean, ignoreIframes?: boolean): Promise<CheerioRoot>;+function parseWithCheerio(page: Page, ignoreShadowRoots?: boolean, ignoreIframes?: boolean): Promise<CheerioAPI>;// @publicexport type PuppeteerBrowserPool = BrowserPool<{@@ -185,7 +181,7 @@ interface PuppeteerContextUtils {infiniteScroll(options?: InfiniteScrollOptions): Promise<void>;injectFile(filePath: string, options?: InjectFileOptions): Promise<unknown>;injectJQuery(): Promise<unknown>;- parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;+ parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;removeInterceptRequestHandler(handler: InterceptHandler): Promise<void>;saveSnapshot(options?: SaveSnapshotOptions): Promise<void>;waitForSelector(selector: string, timeoutMs?: number): Promise<void>;@@ -195,134 +191,9 @@ interface PuppeteerContextUtils {export class PuppeteerCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PuppeteerCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawler<Page, HTTPResponse, LaunchOptions, PuppeteerCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {constructor(options?: PuppeteerCrawlerOptions<ContextExtension, ExtendedContext, Routes, StatisticStateExtension>);// (undocumented)- protected buildContextPipeline(): ContextPipeline<CrawlingContext<Dictionary>, BrowserCrawlingContext<Page, HTTPResponse, Dictionary, Dictionary> & {- injectFile: (filePath: string, options?: InjectFileOptions) => Promise<unknown>;- injectJQuery: () => Promise<void>;- waitForSelector: (selector: string, timeoutMs?: number) => Promise<void>;- parseWithCheerio: (selector?: string, timeoutMs?: number) => Promise<CheerioAPI>;- enqueueLinksByClickingElements: (options: Omit<EnqueueLinksByClickingElementsOptions, "page" | "requestManager">) => Promise<BatchAddRequestsResult>;- blockRequests: (options?: BlockRequestsOptions) => Promise<void>;- compileScript: (scriptString: string, ctx?: Dictionary) => CompiledScriptFunction;- addInterceptRequestHandler: (handler: InterceptHandler) => Promise<void>;- removeInterceptRequestHandler: (handler: InterceptHandler) => Promise<void>;- infiniteScroll: (options?: InfiniteScrollOptions) => Promise<void>;- saveSnapshot: (options?: SaveSnapshotOptions) => Promise<void>;- closeCookieModals: () => Promise<void>;- }>;+ protected buildContextPipeline(): ContextPipeline<CrawlingContext, PuppeteerCrawlingContext>;// (undocumented)protected navigationHandler(crawlingContext: PuppeteerCrawlingContext, gotoOptions: PuppeteerDirectNavigationOptions): Promise<HTTPResponse | null>;- // (undocumented)- protected static optionsSchema: z.ZodObject<{- headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;- navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;- preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;- postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;- launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;- browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- saveResponseCookies: z.ZodDefault<z.ZodBoolean>;- proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- ignoreIframes: z.ZodDefault<z.ZodBoolean>;- ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;- contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;- errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;- sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;- maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;- maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;- taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;- statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;- ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;- blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;- retryOnBlocked: z.ZodDefault<z.ZodBoolean>;- respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;- transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{- requestQueue: z.ZodOptional<z.ZodEnum<{- deferred: "deferred";- writeThrough: "writeThrough";- }>>;- }, z.core.$strict>]>>;- onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- httpClient: z.ZodOptional<z.ZodCustom<BaseHttpClient, BaseHttpClient>>;- configuration: z.ZodOptional<z.ZodCustom<Configuration, Configuration>>;- storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- eventManager: z.ZodOptional<z.ZodCustom<EventManager, EventManager>>;- logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;- maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;- maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;- keepAlive: z.ZodOptional<z.ZodBoolean>;- statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- id: z.ZodOptional<z.ZodString>;- }, z.core.$strict>;- // (undocumented)- protected static optionsShape: {- headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;- navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;- preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;- postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;- launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;- browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- saveResponseCookies: z.ZodDefault<z.ZodBoolean>;- proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- ignoreIframes: z.ZodDefault<z.ZodBoolean>;- ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;- contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;- errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;- sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;- maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;- maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;- taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;- statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;- ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;- blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;- retryOnBlocked: z.ZodDefault<z.ZodBoolean>;- respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;- transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{- requestQueue: z.ZodOptional<z.ZodEnum<{- deferred: "deferred";- writeThrough: "writeThrough";- }>>;- }, z.core.$strict>]>>;- onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- httpClient: z.ZodOptional<z.ZodCustom<BaseHttpClient, BaseHttpClient>>;- configuration: z.ZodOptional<z.ZodCustom<Configuration, Configuration>>;- storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- eventManager: z.ZodOptional<z.ZodCustom<EventManager, EventManager>>;- logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;- maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;- maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;- keepAlive: z.ZodOptional<z.ZodBoolean>;- statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- id: z.ZodOptional<z.ZodString>;- };}// @public (undocumented)diff --git a/docs/public-api/crawlee-stagehand.api.md b/docs/public-api/crawlee-stagehand.api.mdindex e50a2ef0b..a5f66c106 100644--- a/docs/public-api/crawlee-stagehand.api.md+++ b/docs/public-api/crawlee-stagehand.api.md@@ -9,7 +9,6 @@ import { ActOptions } from '@browserbasehq/stagehand';import { ActResult } from '@browserbasehq/stagehand';import { AgentConfig } from '@browserbasehq/stagehand';import { AgentResult } from '@browserbasehq/stagehand';-import { BaseHttpClient } from '@crawlee/http-client';import type { Browser } from 'playwright';import type { BrowserController } from '@crawlee/browser-pool';import { BrowserCrawler } from '@crawlee/browser';@@ -27,7 +26,6 @@ import { Configuration } from '@crawlee/browser';import type { ContextPipeline } from '@crawlee/browser';import type { CrawlingContext } from '@crawlee/browser';import type { Dictionary } from '@crawlee/types';-import { EventManager } from '@crawlee/browser';import { ExtractOptions } from '@browserbasehq/stagehand';import type { GetUserDataFromRequest } from '@crawlee/browser';import type { LaunchContext } from '@crawlee/browser-pool';@@ -106,120 +104,6 @@ export class StagehandCrawler<ContextExtension = Dictionary<never>, ExtendedCont// (undocumented)protected buildContextPipeline(): ContextPipeline<CrawlingContext, StagehandCrawlingContext>;protected navigationHandler(crawlingContext: StagehandCrawlingContext, gotoOptions: StagehandGotoOptions): Promise<Response_2 | null>;- // (undocumented)- protected static optionsSchema: z.ZodObject<{- stagehandOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- headless: z.ZodOptional<z.ZodBoolean>;- navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;- preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;- postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;- launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;- browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- saveResponseCookies: z.ZodDefault<z.ZodBoolean>;- proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- ignoreIframes: z.ZodDefault<z.ZodBoolean>;- ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;- contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;- errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;- sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;- maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;- maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;- taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;- statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;- ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;- blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;- retryOnBlocked: z.ZodDefault<z.ZodBoolean>;- respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;- transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{- requestQueue: z.ZodOptional<z.ZodEnum<{- deferred: "deferred";- writeThrough: "writeThrough";- }>>;- }, z.core.$strict>]>>;- onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- httpClient: z.ZodOptional<z.ZodCustom<BaseHttpClient, BaseHttpClient>>;- configuration: z.ZodOptional<z.ZodCustom<Configuration, Configuration>>;- storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- eventManager: z.ZodOptional<z.ZodCustom<EventManager, EventManager>>;- logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;- maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;- maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;- keepAlive: z.ZodOptional<z.ZodBoolean>;- statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- id: z.ZodOptional<z.ZodString>;- }, z.core.$strict>;- // (undocumented)- protected static optionsShape: {- stagehandOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- headless: z.ZodOptional<z.ZodBoolean>;- navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;- preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;- postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;- launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;- browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- saveResponseCookies: z.ZodDefault<z.ZodBoolean>;- proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- ignoreIframes: z.ZodDefault<z.ZodBoolean>;- ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;- contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;- errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;- sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;- maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;- maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;- taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;- statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;- ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;- blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;- retryOnBlocked: z.ZodDefault<z.ZodBoolean>;- respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;- transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{- requestQueue: z.ZodOptional<z.ZodEnum<{- deferred: "deferred";- writeThrough: "writeThrough";- }>>;- }, z.core.$strict>]>>;- onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;- httpClient: z.ZodOptional<z.ZodCustom<BaseHttpClient, BaseHttpClient>>;- configuration: z.ZodOptional<z.ZodCustom<Configuration, Configuration>>;- storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- eventManager: z.ZodOptional<z.ZodCustom<EventManager, EventManager>>;- logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;- minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;- maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;- maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;- keepAlive: z.ZodOptional<z.ZodBoolean>;- statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;- id: z.ZodOptional<z.ZodString>;- };}// @publicdiff --git a/docs/public-api/crawlee-utils.api.md b/docs/public-api/crawlee-utils.api.mdindex 1f4935737..e9ab3d9c4 100644--- a/docs/public-api/crawlee-utils.api.md+++ b/docs/public-api/crawlee-utils.api.md@@ -17,10 +17,6 @@ export class ArgumentValidationError extends Error {readonly issues: z.ZodError['issues'];}-// Not exported by the entry point; reachable only as a referenced type.-// @public (undocumented)-type CheerioRoot = CheerioAPI;-// @publicconst DISCORD_REGEX: RegExp;@@ -85,7 +81,7 @@ const FACEBOOK_REGEX: RegExp;const FACEBOOK_REGEX_GLOBAL: RegExp;// @public-export function htmlToText(htmlOrCheerioElement: string | CheerioRoot): Promise<string>;+export function htmlToText(htmlOrCheerioElement: string | CheerioAPI): Promise<string>;// @publicconst INSTAGRAM_REGEX: RegExp;diff --git a/docs/upgrading/upgrading_v4.md b/docs/upgrading/upgrading_v4.mdindex e4f6fb37f..62f74fd28 100644--- a/docs/upgrading/upgrading_v4.md+++ b/docs/upgrading/upgrading_v4.md@@ -2005,7 +2005,9 @@ Besides the resource-detection helpers above, several other `@crawlee/utils` exp- **Removed URL helpers:** `filterUrl(target, origin, strategy)`, `matchesEnqueueStrategy(strategy, target, origin)`, and the `UNSUPPORTED_SCHEME_MESSAGE` constant. URL filtering by enqueue strategy is now internal to `enqueueLinks`. The related `filterRequestsByPatterns(requests, patterns?, onSkippedUrl?)` function (from `@crawlee/core`) was removed for the same reason — pattern-based request filtering now happens inside `enqueueLinks`.- **Relocated enums/types:** `EnqueueStrategy` now lives in `@crawlee/core` and `SearchParams` in `@crawlee/types`. They are no longer re-exported from `@crawlee/utils`, so `import { EnqueueStrategy } from '@crawlee/utils'` breaks — import them from `crawlee` (the meta-package) or from `@crawlee/core` / `@crawlee/types` instead.- **Removed `RobotsFile` alias:** `RobotsFile` was an alias for the `RobotsTxtFile` class and is removed. Rename any usage to `RobotsTxtFile`; the class itself is unchanged apart from the signature change described below.-- **Split into public and `/internal` entry points:** the main `@crawlee/utils` entry now exposes only the user-facing helpers (`sleep`, `htmlToText`, `extractUrls`, `downloadListOfUrls`, `expandShadowRoots`, the `social` namespace, the Open Graph parser, and the robots/sitemap utilities). Helpers that primarily serve the crawler packages - e.g. `URL_NO_COMMAS_REGEX`, `URL_WITH_COMMAS_REGEX`, `extractUrlsFromCheerio`, `tryAbsoluteURL`, the `CheerioRoot` type, and the blocked-detection and iterable helpers - moved to the `@crawlee/utils/internal` entry point. They keep working, but imports need updating: `import { URL_NO_COMMAS_REGEX } from '@crawlee/utils/internal'`. As the name suggests, the internal entry follows no semver guarantees.+- **Split into public and `/internal` entry points:** the main `@crawlee/utils` entry now exposes only the user-facing helpers (`sleep`, `htmlToText`, `extractUrls`, `downloadListOfUrls`, `expandShadowRoots`, the `social` namespace, the Open Graph parser, and the robots/sitemap utilities). Helpers that primarily serve the crawler packages - e.g. `URL_NO_COMMAS_REGEX`, `URL_WITH_COMMAS_REGEX`, `extractUrlsFromCheerio`, `tryAbsoluteURL`, and the blocked-detection and iterable helpers - moved to the `@crawlee/utils/internal` entry point. They keep working, but imports need updating: `import { URL_NO_COMMAS_REGEX } from '@crawlee/utils/internal'`. As the name suggests, the internal entry follows no semver guarantees.+- **Removed `CheerioRoot` and the cheerio type re-exports:** `CheerioRoot` was an alias for cheerio's own `CheerioAPI` and is gone; `parseWithCheerio()` and `htmlToText()` are typed with `CheerioAPI` directly. The crawler packages also no longer re-export `Cheerio`, `CheerioAPI` and `Element`, so `import type { CheerioAPI } from 'crawlee'` (or from `@crawlee/basic` / `@crawlee/puppeteer` / ...) breaks - import them from `cheerio` and `domhandler`, which are the packages that own them.+- **`@crawlee/core` no longer re-exports the internal helpers:** `parseArgument`, `schemas` and `tryAbsoluteURL` reached `@crawlee/core` (and through it `@crawlee/basic`, `@crawlee/http`, `@crawlee/browser` and `crawlee`) as public exports, which put symbols from the no-semver `/internal` entry point back into a semver-stable surface. Import them from `@crawlee/utils/internal` instead. `ArgumentValidationError` is unaffected and stays exported from `@crawlee/core`.#### `RobotsTxtFile.find` signature changed; sitemap options removeddiff --git a/packages/basic-crawler/src/index.ts b/packages/basic-crawler/src/index.tsindex 3af68daab..4b2eea9d6 100644--- a/packages/basic-crawler/src/index.ts+++ b/packages/basic-crawler/src/index.ts@@ -1,3 +1,2 @@export * from '@crawlee/core';export * from './internals/basic-crawler.js';-export type { CheerioRoot, CheerioAPI, Cheerio, Element } from '@crawlee/utils/internal';diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex 4111db951..7e9276cc5 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -63,7 +63,6 @@ import {NavigationSkippedError,NonRetryableError,OwnedOrInjected,- parseArgument,purgeDefaultStorages,RequestHandlerError,parseRetryAfterHeader,@@ -74,7 +73,6 @@ import {RetryRequestError,supportsDomainThrottling,Router,- schemas,ServiceLocator,serviceLocator,Session,@@ -96,7 +94,7 @@ import type {SetStatusMessageOptions,StorageBackend,} from '@crawlee/types';-import { isAsyncIterable, isIterable, ROTATE_PROXY_ERRORS } from '@crawlee/utils/internal';+import { isAsyncIterable, isIterable, parseArgument, ROTATE_PROXY_ERRORS, schemas } from '@crawlee/utils/internal';import { RobotsTxtFile } from '@crawlee/utils';import { getDomain } from 'tldts';import type { ReadonlyDeep } from 'type-fest';@@ -699,7 +697,7 @@ export class BasicCrawler<Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>,StatisticStateExtension extends object = {},> {- protected static readonly CRAWLEE_STATE_KEY = 'CRAWLEE_STATE';+ static readonly #CRAWLEE_STATE_KEY = 'CRAWLEE_STATE';/*** Tracks the number of crawler instances created. The first crawler uses the default@@ -847,15 +845,15 @@ export class BasicCrawler<}protected readonly requestHandler!: RequestHandler<ExtendedContext>;- protected readonly errorHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;- protected readonly failedRequestHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;+ readonly #errorHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;+ readonly #failedRequestHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;// kept as TS-private: tests read it at runtimeprivate requestHandlerTimeoutMillis!: number;protected readonly internalTimeoutMillis: number;- protected readonly maxRequestRetries: number;- protected readonly maxCrawlDepth?: number;+ readonly #maxRequestRetries: number;+ readonly #maxCrawlDepth?: number;#sameDomainDelaySecs: number;- protected readonly maxRequestsPerCrawl?: number;+ readonly #maxRequestsPerCrawl?: number;private get handledRequestsCount(): number {return this.statistics.state.requestsFinished + this.statistics.state.requestsFailed;@@ -881,16 +879,19 @@ export class BasicCrawler<readonly #transactionalStorageEnabled: boolean;/** The resolved per-storage-type write policy overrides forwarded to each request's transaction. */readonly #storageWritePolicy: Partial<StorageWritePolicy>;- protected readonly onSkippedRequest?: SkippedRequestCallback;+ readonly #onSkippedRequest?: SkippedRequestCallback;#closeEvents?: boolean;#loggedPerRun = new Set<string>();readonly #robotsTxtFileCache: LruCache<RobotsTxtFile>;- protected readonly identity: CrawlerIdentity;+ readonly #identity: CrawlerIdentity;readonly #contextPipelineOptions: {contextPipelineBuilder?: () => ContextPipeline<CrawlingContext, Context>;extendContext?: (context: CrawlingContext) => Awaitable<ContextExtension>;};+ /**+ * @internal+ */protected static optionsShape = {contextPipelineBuilder: schemas.anyObject.optional(),extendContext: schemas.anyFunction.optional(),@@ -949,7 +950,7 @@ export class BasicCrawler<id: z.string().optional(),};- protected static optionsSchema = z.strictObject(BasicCrawler.optionsShape);+ static #optionsSchema = z.strictObject(BasicCrawler.optionsShape);/*** All `BasicCrawler` parameters are passed via an options object.@@ -958,7 +959,7 @@ export class BasicCrawler<options: BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> &RequireContextPipeline<CrawlingContext, Context> = {} as any, // cast because the constructor logic handles missing `contextPipelineBuilder` - the type is just for DX) {- const parsedOptions = parseArgument(options, BasicCrawler.optionsSchema, 'BasicCrawlerOptions');+ const parsedOptions = parseArgument(options, BasicCrawler.#optionsSchema, 'BasicCrawlerOptions');const {// oxlint-disable-next-line typescript/no-deprecated -- still accepted and folded into `requestManager` for back-compat@@ -1060,7 +1061,7 @@ export class BasicCrawler<serviceLocator.getConfiguration();const instanceIndex = BasicCrawler.instanceCount++;- this.identity = { instanceIndex, hasExplicitId: id !== undefined, id: id ?? String(instanceIndex) };+ this.#identity = { instanceIndex, hasExplicitId: id !== undefined, id: id ?? String(instanceIndex) };if (requestManager !== undefined) {if (requestList !== undefined || requestQueue !== undefined) {@@ -1096,14 +1097,13 @@ export class BasicCrawler<this.#statusMessageLoggingInterval = statusMessageLoggingInterval;this.#statusMessageCallback = statusMessageCallback as StatusMessageCallback;this.#robotsTxtFileCache = new LruCache({ maxLength: 1000 });- this.handleSkippedRequest = this.handleSkippedRequest.bind(this);this.additionalHttpErrorStatusCodes = new Set([...additionalHttpErrorStatusCodes]);this.#ignoreHttpErrorStatusCodes = new Set([...ignoreHttpErrorStatusCodes]);this.requestHandler = requestHandler ?? this.router;- this.failedRequestHandler = failedRequestHandler;- this.errorHandler = errorHandler;+ this.#failedRequestHandler = failedRequestHandler;+ this.#errorHandler = errorHandler;if (requestHandlerTimeoutSecs) {this.requestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;@@ -1120,15 +1120,15 @@ export class BasicCrawler<| undefined;this.#transactionalStorageEnabled = transactionalStorageOption !== false;this.#storageWritePolicy = typeof transactionalStorageOption === 'object' ? transactionalStorageOption : {};- this.onSkippedRequest = onSkippedRequest;+ this.#onSkippedRequest = onSkippedRequest;// allow at least 5min for internal timeoutsthis.internalTimeoutMillis =serviceLocator.getConfiguration().internalTimeoutMillis ??Math.max(this.requestHandlerTimeoutMillis * 2, 300e3);- this.maxRequestRetries = maxRequestRetries;- this.maxCrawlDepth = maxCrawlDepth;+ this.#maxRequestRetries = maxRequestRetries;+ this.#maxCrawlDepth = maxCrawlDepth;this.#sameDomainDelaySecs = sameDomainDelaySecs;this.#statisticsDep = OwnedOrInjected.resolve<IStatistics<StatisticStateExtension>,@@ -1142,7 +1142,7 @@ export class BasicCrawler<new Statistics({logMessage: `${this.constructor.name} request statistics:`,log: this.log,- id: this.identity.id,+ id: this.#identity.id,}) as Statistics<StatisticStateExtension>,);@@ -1182,10 +1182,10 @@ export class BasicCrawler<this.internalTimeoutMillis = Math.min(this.internalTimeoutMillis, maxSignedInteger);- this.maxRequestsPerCrawl = maxRequestsPerCrawl;+ this.#maxRequestsPerCrawl = maxRequestsPerCrawl;const isMaxPagesExceeded = () =>- this.maxRequestsPerCrawl && this.maxRequestsPerCrawl <= this.handledRequestsCount;+ this.#maxRequestsPerCrawl && this.#maxRequestsPerCrawl <= this.handledRequestsCount;// eslint-disable-next-line prefer-constlet { isFinishedFunction, isTaskReadyFunction } = taskLoopOptions;@@ -1285,7 +1285,7 @@ export class BasicCrawler<this.logOncePerRun('shuttingDown','Crawler reached the maxRequestsPerCrawl limit of ' +- `${this.maxRequestsPerCrawl} requests and will shut down soon. Requests that are in progress will be allowed to finish.`,+ `${this.#maxRequestsPerCrawl} requests and will shut down soon. Requests that are in progress will be allowed to finish.`,);return false;}@@ -1304,7 +1304,7 @@ export class BasicCrawler<isFinishedFunction: async () => {if (isMaxPagesExceeded()) {this.log.info(- `Earlier, the crawler reached the maxRequestsPerCrawl limit of ${this.maxRequestsPerCrawl} requests ` ++ `Earlier, the crawler reached the maxRequestsPerCrawl limit of ${this.#maxRequestsPerCrawl} requests ` +'and all requests that were in progress at that time have now finished. ' +`In total, the crawler processed ${this.handledRequestsCount} requests and will shut down.`,);@@ -1400,7 +1400,7 @@ export class BasicCrawler<);request.state = RequestState.SKIPPED;request.noRetry = true;- await this.handleSkippedRequest({+ await this.#handleSkippedRequest({url: request.url,reason: 'robotsTxt',});@@ -1534,7 +1534,7 @@ export class BasicCrawler<request.noRetry = true;request.state = RequestState.SKIPPED;- await this.handleSkippedRequest({ url: request.url, reason: 'redirect' });+ await this.#handleSkippedRequest({ url: request.url, reason: 'redirect' });throw new ContextPipelineInterruptedError(message);}@@ -1574,7 +1574,7 @@ export class BasicCrawler<// Setting the status message is not a storage concern, so we intentionally don't route it// through the storage client anymore.serviceLocator.getEventManager().emit(EventType.STATUS_MESSAGE, {- crawlerId: this.identity.id,+ crawlerId: this.#identity.id,message,isStatusMessageTerminal: options.isStatusMessageTerminal,level: options.level,@@ -1850,7 +1850,7 @@ export class BasicCrawler<minCrawlDelaySecs: this.#sameDomainDelaySecs,// What `sameDomainDelaySecs` has always meant: one clock for a site, subdomains included.throttleBy: 'registrableDomain',- persistStateKey: `CRAWLEE_THROTTLED_DOMAINS_${this.identity.id}`,+ persistStateKey: `CRAWLEE_THROTTLED_DOMAINS_${this.#identity.id}`,});}@@ -1881,7 +1881,7 @@ export class BasicCrawler<private async openOwnedRequestQueue(): Promise<RequestQueue> {// The first crawler instance uses the default queue (null identifier);// subsequent instances get their own queue via a unique alias so they don't collide.- const identifier = this.identity.instanceIndex === 0 ? null : { alias: `__default_${this.identity.id}__` };+ const identifier = this.#identity.instanceIndex === 0 ? null : { alias: `__default_${this.#identity.id}__` };const requestQueue = await RequestQueue.open(identifier, { configuration: serviceLocator.getConfiguration() });return this.#ownedRequestQueue.set(requestQueue);@@ -1939,12 +1939,12 @@ export class BasicCrawler<async useState<State extends Dictionary = Dictionary>(defaultValue = {} as State): Promise<State> {const kvs = await KeyValueStore.open(null, { configuration: serviceLocator.getConfiguration() });- if (this.identity.hasExplicitId) {- const stateKey = `${BasicCrawler.CRAWLEE_STATE_KEY}_${this.identity.id}`;+ if (this.#identity.hasExplicitId) {+ const stateKey = `${BasicCrawler.#CRAWLEE_STATE_KEY}_${this.#identity.id}`;return kvs.getAutoSavedValue<State>(stateKey, defaultValue);}- BasicCrawler.#useStateAnonymousIndices.add(this.identity.instanceIndex);+ BasicCrawler.#useStateAnonymousIndices.add(this.#identity.instanceIndex);if (BasicCrawler.#useStateAnonymousIndices.size > 1) {serviceLocator@@ -1957,27 +1957,27 @@ export class BasicCrawler<);}- return kvs.getAutoSavedValue<State>(BasicCrawler.CRAWLEE_STATE_KEY, defaultValue);+ return kvs.getAutoSavedValue<State>(BasicCrawler.#CRAWLEE_STATE_KEY, defaultValue);}- protected async getPendingRequestCountApproximation(): Promise<number> {+ async #getPendingRequestCountApproximation(): Promise<number> {return (await this.requestManager?.getPendingCount()) ?? 0;}- protected async calculateEnqueuedRequestLimit(explicitLimit?: number): Promise<number | undefined> {- if (this.maxRequestsPerCrawl === undefined) {+ async #calculateEnqueuedRequestLimit(explicitLimit?: number): Promise<number | undefined> {+ if (this.#maxRequestsPerCrawl === undefined) {return explicitLimit;}const limit = Math.max(0,- this.maxRequestsPerCrawl - this.handledRequestsCount - (await this.getPendingRequestCountApproximation()),+ this.#maxRequestsPerCrawl - this.handledRequestsCount - (await this.#getPendingRequestCountApproximation()),);return Math.min(limit, explicitLimit ?? Infinity);}- protected async handleSkippedRequest(options: Parameters<SkippedRequestCallback>[0]): Promise<void> {+ async #handleSkippedRequest(options: Parameters<SkippedRequestCallback>[0]): Promise<void> {// A skipped request is a *successful* outcome, but the interrupt still unwinds through the// transaction scope, which rolls back - so the skip bookkeeping must write directly.await withDirectStorageAccess(async () => {@@ -1985,18 +1985,18 @@ export class BasicCrawler<this.logOncePerRun('maxRequestsPerCrawl','The number of requests enqueued by the crawler reached the maxRequestsPerCrawl limit of ' +- `${this.maxRequestsPerCrawl} requests and no further requests will be added.`,+ `${this.#maxRequestsPerCrawl} requests and no further requests will be added.`,);}if (options.reason === 'depth') {this.logOncePerRun('maxCrawlDepth',- `The crawler reached the maxCrawlDepth limit of ${this.maxCrawlDepth} and no further requests will be enqueued.`,+ `The crawler reached the maxCrawlDepth limit of ${this.#maxCrawlDepth} and no further requests will be enqueued.`,);}- await this.onSkippedRequest?.(options);+ await this.#onSkippedRequest?.(options);});}@@ -2044,7 +2044,7 @@ export class BasicCrawler<await this.validateRequestUserData({ label: options.label, userData: options.userData });}- const requestLimit = await this.calculateEnqueuedRequestLimit(options.limit);+ const requestLimit = await this.#calculateEnqueuedRequestLimit(options.limit);const strategy = options.strategy ?? EnqueueStrategy.All;const urlExcludePatternObjects: UrlPatternObject[] = options.exclude?.length@@ -2060,7 +2060,7 @@ export class BasicCrawler<: [];const isAllowedBasedOnRobotsTxtFile = this.isAllowedBasedOnRobotsTxtFile.bind(this);- const maxCrawlDepth = this.maxCrawlDepth;+ const maxCrawlDepth = this.#maxCrawlDepth;const validateRequestUserData = this.validateRequestUserData.bind(this);const allSkipped: { url: string; reason: SkippedRequestReason }[] = [];@@ -2163,7 +2163,7 @@ export class BasicCrawler<await Promise.all(allSkipped.map(async ({ url, reason }) => {- await this.handleSkippedRequest({ url, reason });+ await this.#handleSkippedRequest({ url, reason });await options.onSkippedRequest?.({ url, reason });}),);@@ -2291,7 +2291,7 @@ export class BasicCrawler<this.#autoscaledPool = new AutoscaledPool({...this.taskLoopOptions,concurrencySystem: this.#concurrencySystemDep.value,- consumer: this.identity,+ consumer: this.#identity,});await this.getRequestManager();@@ -2753,7 +2753,7 @@ export class BasicCrawler<if (shouldRetryRequest) {await this.statistics.errorTrackerRetry.addAsync(error, crawlingContext);- await this.errorHandler?.(+ await this.#errorHandler?.(crawlingContext as CrawlingContext & Partial<ExtendedContext>, // valid cast - ExtendedContext transitively extends CrawlingContexterror,);@@ -2811,8 +2811,8 @@ export class BasicCrawler<this.log.error(`Request failed and reached maximum retries. ${message}`, { id, url, method, uniqueKey });- if (this.failedRequestHandler) {- await this.failedRequestHandler?.(+ if (this.#failedRequestHandler) {+ await this.#failedRequestHandler?.(crawlingContext as CrawlingContext & Partial<ExtendedContext>, // valid cast - ExtendedContext transitively extends CrawlingContexterror,);@@ -2863,7 +2863,7 @@ export class BasicCrawler<}// Ensure there are more retries available for the request- const maxRequestRetries = request.maxRetries ?? this.maxRequestRetries;+ const maxRequestRetries = request.maxRetries ?? this.#maxRequestRetries;return request.retryCount < maxRequestRetries;}diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.tsindex 553ce7df6..aa531dda0 100644--- a/packages/browser-crawler/src/internals/browser-crawler.ts+++ b/packages/browser-crawler/src/internals/browser-crawler.ts@@ -21,20 +21,23 @@ import {EnqueueStrategy,NavigationSkippedError,OwnedOrInjected,- parseArgument,remainingNavigationWindowMillis,RequestState,RequestThrottledError,resolveBaseUrlForEnqueueLinksFiltering,- schemas,SessionError,toughCookieToBrowserPoolCookie,- tryAbsoluteURL,validators,} from '@crawlee/basic';import type { CommonPage, CrawlerRemoteBrowserOptions } from '@crawlee/browser-pool';import type { Awaitable, Cookie as CookieObject, Dictionary, IBrowserPool, ISession } from '@crawlee/types';-import { CLOUDFLARE_RETRY_CSS_SELECTORS, RETRY_CSS_SELECTORS } from '@crawlee/utils/internal';+import {+ CLOUDFLARE_RETRY_CSS_SELECTORS,+ parseArgument,+ RETRY_CSS_SELECTORS,+ schemas,+ tryAbsoluteURL,+} from '@crawlee/utils/internal';import { sleep } from '@crawlee/utils';import { z } from 'zod';@@ -384,6 +387,9 @@ export abstract class BrowserCrawler<readonly #postNavigationHooks: BrowserHook<Context>[];readonly #saveResponseCookies: boolean;+ /**+ * @internal+ */protected static override optionsShape = {...BasicCrawler.optionsShape,@@ -403,7 +409,8 @@ export abstract class BrowserCrawler<ignoreShadowRoots: z.boolean().default(false),};- protected static override optionsSchema = z.strictObject(BrowserCrawler.optionsShape);+ /** @internal */+ protected static optionsSchema = z.strictObject(BrowserCrawler.optionsShape);/*** All `BrowserCrawler` parameters are passed via an options object.diff --git a/packages/browser-crawler/src/internals/browser-launcher.ts b/packages/browser-crawler/src/internals/browser-launcher.tsindex 1e52aba5d..13dfc1cae 100644--- a/packages/browser-crawler/src/internals/browser-launcher.ts+++ b/packages/browser-crawler/src/internals/browser-launcher.ts@@ -2,7 +2,7 @@ import fs from 'node:fs';import { createRequire } from 'node:module';import os from 'node:os';-import { Configuration, schemas, serviceLocator } from '@crawlee/basic';+import { Configuration, serviceLocator } from '@crawlee/basic';import type {BrowserPlugin,BrowserPluginOptions,@@ -12,6 +12,7 @@ import type {} from '@crawlee/browser-pool';import { BrowserPool, RemoteBrowserPool } from '@crawlee/browser-pool';import type { Constructor, Dictionary } from '@crawlee/types';+import { schemas } from '@crawlee/utils/internal';import { z } from 'zod';const DEFAULT_VIEWPORT = {@@ -126,6 +127,9 @@ export abstract class BrowserLauncher<Plugin!: T;userAgent?: string;+ /**+ * @internal+ */protected static optionsShape = {proxyUrl: z.url().optional(),useChrome: z.boolean().optional(),@@ -137,6 +141,7 @@ export abstract class BrowserLauncher<userAgent: z.string().optional(),};+ /** @internal */protected static optionsSchema = z.strictObject(BrowserLauncher.optionsShape);static requireLauncherOrThrow<T>(launcher: string, apifyImageName: string): T {diff --git a/packages/browser-pool/package.json b/packages/browser-pool/package.jsonindex d6fa53377..20ba87f4a 100644--- a/packages/browser-pool/package.json+++ b/packages/browser-pool/package.json@@ -33,6 +33,7 @@"@apify/timeout": "^0.4.4","@crawlee/core": "workspace:*","@crawlee/types": "workspace:*",+ "@crawlee/utils": "workspace:*","fingerprint-generator": "^2.1.68","fingerprint-injector": "^2.1.68","lodash.merge": "^4.6.2",diff --git a/packages/browser-pool/src/browser-pool.ts b/packages/browser-pool/src/browser-pool.tsindex ef39cbe12..ef992ef7f 100644--- a/packages/browser-pool/src/browser-pool.ts+++ b/packages/browser-pool/src/browser-pool.ts@@ -1,7 +1,8 @@import { AsyncResource } from 'node:async_hooks';-import { type CrawleeLogger, parseArgument, schemas, SessionError, serviceLocator } from '@crawlee/core';+import { type CrawleeLogger, SessionError, serviceLocator } from '@crawlee/core';import type { IBrowserPool, NewPageOptions, PageState } from '@crawlee/types';+import { parseArgument, schemas } from '@crawlee/utils/internal';import type { BrowserFingerprintWithHeaders } from 'fingerprint-generator';import { FingerprintGenerator } from 'fingerprint-generator';import { FingerprintInjector } from 'fingerprint-injector';diff --git a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts b/packages/cheerio-crawler/src/internals/cheerio-crawler.tsindex f74ef217d..4688a9df8 100644--- a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts+++ b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts@@ -1,5 +1,6 @@import type {AddRequestsBatchedResult,+ ContextPipeline,CrawlingContext,EnqueueLinksOptions,ErrorHandler,@@ -22,7 +23,7 @@ import {Router,} from '@crawlee/http';import type { Dictionary } from '@crawlee/types';-import { type CheerioRoot, extractUrlsFromCheerio } from '@crawlee/utils/internal';+import { extractUrlsFromCheerio } from '@crawlee/utils/internal';import type { CheerioAPI, CheerioOptions } from 'cheerio';import * as cheerio from 'cheerio';import { parseDocument } from 'htmlparser2';@@ -96,7 +97,7 @@ export interface CheerioCrawlingContext<* });* ```*/- parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;+ parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;/*** Extracts URLs from the parsed HTML, without adding them to the request queue.@@ -216,7 +217,7 @@ export class CheerioCrawler<});}- protected override buildContextPipeline() {+ protected override buildContextPipeline(): ContextPipeline<CrawlingContext, CheerioCrawlingContext> {return super.buildContextPipeline().compose({diff --git a/packages/core/src/enqueue_links/shared.ts b/packages/core/src/enqueue_links/shared.tsindex 40798f770..f86fb9d47 100644--- a/packages/core/src/enqueue_links/shared.ts+++ b/packages/core/src/enqueue_links/shared.ts@@ -8,8 +8,6 @@ import type { RequestOptions } from '../request.js';import { schemas } from '../validators.js';import type { EnqueueStrategyOption } from './enqueue_links.js';-export { tryAbsoluteURL } from '@crawlee/utils/internal';-const MAX_ENQUEUE_LINKS_CACHE_SIZE = 1000;/**diff --git a/packages/core/src/index.ts b/packages/core/src/index.tsindex 29a05a552..da8c1f7d7 100644--- a/packages/core/src/index.ts+++ b/packages/core/src/index.ts@@ -15,7 +15,9 @@ export * from './serialization.js';export * from './session_pool/index.js';export * from './storages/index.js';export * from './memory-storage/index.js';-export * from './validators.js';+// Not `export *`: the rest of the module re-exports `@crawlee/utils/internal` symbols, which carry no+// semver guarantees and must not reach the public surface. Internal consumers import them directly.+export { ArgumentValidationError, validators } from './validators.js';export * from './cookie_utils.js';export * from './http.js';export * from './recoverable_state.js';diff --git a/packages/core/src/memory-storage/memory-storage.ts b/packages/core/src/memory-storage/memory-storage.tsindex 1cdd751a2..d946ddfec 100644--- a/packages/core/src/memory-storage/memory-storage.ts+++ b/packages/core/src/memory-storage/memory-storage.ts@@ -26,9 +26,9 @@ export class MemoryStorageBackend implements storage.StorageBackend {*/readonly #instanceCacheKey = `MemoryStorageBackend:${randomUUID()}`;- readonly keyValueStoreBackendCache: KeyValueStoreBackend[] = [];- readonly datasetBackendCache: DatasetBackend[] = [];- readonly requestQueueBackendCache: RequestQueueBackend[] = [];+ readonly #keyValueStoreBackendCache: KeyValueStoreBackend[] = [];+ readonly #datasetBackendCache: DatasetBackend[] = [];+ readonly #requestQueueBackendCache: RequestQueueBackend[] = [];constructor(options: MemoryStorageOptions = {}) {this.logger = options.logger;@@ -42,6 +42,42 @@ export class MemoryStorageBackend implements storage.StorageBackend {return this.#instanceCacheKey;}+ /**+ * Evict a cached backend so that a dropped storage is no longer resolved by `createXBackend`,+ * reported by `storageExists` or visited by `purge`. Returns whether the backend was cached, which+ * tells the caller whether it still owns in-memory state worth clearing.+ *+ * The resource clients own their own entry's lifetime but must not reach into the caches directly.+ * Because a client is only ever constructed by `createXBackend`, which caches it immediately, the+ * entry matching `id` is always the caller itself.+ * @internal+ */+ evictBackend(type: 'Dataset' | 'KeyValueStore' | 'RequestQueue', id: string): boolean {+ let cache: { id: string }[];++ switch (type) {+ case 'Dataset':+ cache = this.#datasetBackendCache;+ break;+ case 'KeyValueStore':+ cache = this.#keyValueStoreBackendCache;+ break;+ case 'RequestQueue':+ cache = this.#requestQueueBackendCache;+ break;+ }++ const index = cache.findIndex((entry) => entry.id === id);++ if (index === -1) {+ return false;+ }++ cache.splice(index, 1);++ return true;+ }+static #resolveStorageKey(options: { id?: string; name?: string; alias?: string }): {isAlias: boolean;cacheKey: string;@@ -60,7 +96,7 @@ export class MemoryStorageBackend implements storage.StorageBackend {async createDatasetBackend(options: storage.StorageIdentifier = {}): Promise<storage.DatasetBackend> {const { isAlias, cacheKey } = MemoryStorageBackend.#resolveStorageKey(options);- const found = this.datasetBackendCache.find(+ const found = this.#datasetBackendCache.find((store) =>store.id === cacheKey ||store.name?.toLowerCase() === cacheKey.toLowerCase() ||@@ -75,7 +111,7 @@ export class MemoryStorageBackend implements storage.StorageBackend {cacheKey,storageBackend: this,});- this.datasetBackendCache.push(newStore);+ this.#datasetBackendCache.push(newStore);return newStore;}@@ -83,7 +119,7 @@ export class MemoryStorageBackend implements storage.StorageBackend {async createKeyValueStoreBackend(options: storage.StorageIdentifier = {}): Promise<storage.KeyValueStoreBackend> {const { isAlias, cacheKey } = MemoryStorageBackend.#resolveStorageKey(options);- const found = this.keyValueStoreBackendCache.find(+ const found = this.#keyValueStoreBackendCache.find((store) =>store.id === cacheKey ||store.name?.toLowerCase() === cacheKey.toLowerCase() ||@@ -98,7 +134,7 @@ export class MemoryStorageBackend implements storage.StorageBackend {cacheKey,storageBackend: this,});- this.keyValueStoreBackendCache.push(newStore);+ this.#keyValueStoreBackendCache.push(newStore);return newStore;}@@ -106,7 +142,7 @@ export class MemoryStorageBackend implements storage.StorageBackend {async createRequestQueueBackend(options: storage.StorageIdentifier = {}): Promise<RequestQueueBackend> {const { isAlias, cacheKey } = MemoryStorageBackend.#resolveStorageKey(options);- const found = this.requestQueueBackendCache.find(+ const found = this.#requestQueueBackendCache.find((queue) =>queue.id === cacheKey ||queue.name?.toLowerCase() === cacheKey.toLowerCase() ||@@ -121,7 +157,7 @@ export class MemoryStorageBackend implements storage.StorageBackend {cacheKey,storageBackend: this,});- this.requestQueueBackendCache.push(newStore);+ this.#requestQueueBackendCache.push(newStore);return newStore;}@@ -131,13 +167,13 @@ export class MemoryStorageBackend implements storage.StorageBackend {switch (type) {case 'Dataset':- backends = this.datasetBackendCache;+ backends = this.#datasetBackendCache;break;case 'KeyValueStore':- backends = this.keyValueStoreBackendCache;+ backends = this.#keyValueStoreBackendCache;break;case 'RequestQueue':- backends = this.requestQueueBackendCache;+ backends = this.#requestQueueBackendCache;break;default:return false;@@ -168,11 +204,11 @@ export class MemoryStorageBackend implements storage.StorageBackend {await Promise.all([// Only the default store holds the run input, so it is the only one that keeps `INPUT`.- purgeRunScoped(this.keyValueStoreBackendCache, async (store) =>+ purgeRunScoped(this.#keyValueStoreBackendCache, async (store) =>isDefault(store) ? store.purgeExceptInput() : store.purge(),),- purgeRunScoped(this.datasetBackendCache, async (store) => store.purge()),- purgeRunScoped(this.requestQueueBackendCache, async (store) => store.purge()),+ purgeRunScoped(this.#datasetBackendCache, async (store) => store.purge()),+ purgeRunScoped(this.#requestQueueBackendCache, async (store) => store.purge()),]);}diff --git a/packages/core/src/memory-storage/resource-clients/dataset.ts b/packages/core/src/memory-storage/resource-clients/dataset.tsindex b5a5aa877..a7ed375de 100644--- a/packages/core/src/memory-storage/resource-clients/dataset.ts+++ b/packages/core/src/memory-storage/resource-clients/dataset.ts@@ -63,12 +63,9 @@ export class DatasetBackend<Data extends Dictionary = Dictionary>}async drop(): Promise<void> {- const storeIndex = this.storageBackend.datasetBackendCache.findIndex((store) => store.id === this.id);-- if (storeIndex !== -1) {- const [oldBackend] = this.storageBackend.datasetBackendCache.splice(storeIndex, 1);- oldBackend.itemCount = 0;- oldBackend.#datasetEntries.clear();+ if (this.storageBackend.evictBackend('Dataset', this.id)) {+ this.itemCount = 0;+ this.#datasetEntries.clear();}}diff --git a/packages/core/src/memory-storage/resource-clients/key-value-store.ts b/packages/core/src/memory-storage/resource-clients/key-value-store.tsindex 70f2d2240..72c26b43c 100644--- a/packages/core/src/memory-storage/resource-clients/key-value-store.ts+++ b/packages/core/src/memory-storage/resource-clients/key-value-store.ts@@ -80,11 +80,8 @@ export class KeyValueStoreBackend extends BaseClient implements storage.KeyValue}async drop(): Promise<void> {- const storeIndex = this.storageBackend.keyValueStoreBackendCache.findIndex((store) => store.id === this.id);-- if (storeIndex !== -1) {- const [oldBackend] = this.storageBackend.keyValueStoreBackendCache.splice(storeIndex, 1);- oldBackend.#keyValueEntries.clear();+ if (this.storageBackend.evictBackend('KeyValueStore', this.id)) {+ this.#keyValueEntries.clear();}}diff --git a/packages/core/src/memory-storage/resource-clients/request-queue.ts b/packages/core/src/memory-storage/resource-clients/request-queue.tsindex 4c098832a..5290d21b6 100644--- a/packages/core/src/memory-storage/resource-clients/request-queue.ts+++ b/packages/core/src/memory-storage/resource-clients/request-queue.ts@@ -88,17 +88,14 @@ export class RequestQueueBackend extends BaseClient implements storage.RequestQuawait this.#queueStateMutex.wait();try {- const storeIndex = this.storageBackend.requestQueueBackendCache.findIndex((queue) => queue.id === this.id);-- if (storeIndex !== -1) {- const [oldBackend] = this.storageBackend.requestQueueBackendCache.splice(storeIndex, 1);- oldBackend.pendingRequestCount = 0;+ if (this.storageBackend.evictBackend('RequestQueue', this.id)) {+ this.pendingRequestCount = 0;// Clear all in-memory state, consistent with `purge`. Clearing `requests` alone would// leave dangling ids in `forefrontRequestIds`/`inProgressRequestIds`, which a later head// scan would resolve to a missing request and dereference.- oldBackend.#requests.clear();- oldBackend.#forefrontRequestIds = [];- oldBackend.#inProgressRequestIds.clear();+ this.#requests.clear();+ this.#forefrontRequestIds = [];+ this.#inProgressRequestIds.clear();}} finally {this.#queueStateMutex.shift();diff --git a/packages/fs-storage/src/file-system-storage.ts b/packages/fs-storage/src/file-system-storage.tsindex 40e1cdd3a..339278b98 100644--- a/packages/fs-storage/src/file-system-storage.ts+++ b/packages/fs-storage/src/file-system-storage.ts@@ -77,9 +77,9 @@ export class FileSystemStorageBackend implements storage.StorageBackend {readonly logger?: CrawleeLogger;readonly requestQueueAccess: 'single' | 'shared';- readonly keyValueStoreBackendCache: KeyValueStoreBackend[] = [];- readonly datasetBackendCache: DatasetBackend[] = [];- readonly requestQueueBackendCache: RequestQueueBackend[] = [];+ readonly #keyValueStoreBackendCache: KeyValueStoreBackend[] = [];+ readonly #datasetBackendCache: DatasetBackend[] = [];+ readonly #requestQueueBackendCache: RequestQueueBackend[] = [];constructor(options: FileSystemStorageOptions) {const { logger, requestQueueAccess, localDataDirectory } = parseArgument(@@ -127,7 +127,7 @@ export class FileSystemStorageBackend implements storage.StorageBackend {async createDatasetBackend(options: storage.StorageIdentifier = {}): Promise<storage.DatasetBackend> {const { id, name, alias, cacheKey } = FileSystemStorageBackend.#resolveStorageKey(options);- const found = this.datasetBackendCache.find(+ const found = this.#datasetBackendCache.find((store) =>store.id === cacheKey ||store.name?.toLowerCase() === cacheKey.toLowerCase() ||@@ -146,7 +146,7 @@ export class FileSystemStorageBackend implements storage.StorageBackend {nativeBackend,logger: this.logger,});- this.datasetBackendCache.push(newStore);+ this.#datasetBackendCache.push(newStore);return newStore;}@@ -154,7 +154,7 @@ export class FileSystemStorageBackend implements storage.StorageBackend {async createKeyValueStoreBackend(options: storage.StorageIdentifier = {}): Promise<storage.KeyValueStoreBackend> {const { id, name, alias, cacheKey } = FileSystemStorageBackend.#resolveStorageKey(options);- const found = this.keyValueStoreBackendCache.find(+ const found = this.#keyValueStoreBackendCache.find((store) =>store.id === cacheKey ||store.name?.toLowerCase() === cacheKey.toLowerCase() ||@@ -173,7 +173,7 @@ export class FileSystemStorageBackend implements storage.StorageBackend {nativeBackend,logger: this.logger,});- this.keyValueStoreBackendCache.push(newStore);+ this.#keyValueStoreBackendCache.push(newStore);return newStore;}@@ -181,7 +181,7 @@ export class FileSystemStorageBackend implements storage.StorageBackend {async createRequestQueueBackend(options: storage.StorageIdentifier = {}): Promise<storage.RequestQueueBackend> {const { id, name, alias, cacheKey } = FileSystemStorageBackend.#resolveStorageKey(options);- const found = this.requestQueueBackendCache.find(+ const found = this.#requestQueueBackendCache.find((queue) =>queue.id === cacheKey ||queue.name?.toLowerCase() === cacheKey.toLowerCase() ||@@ -208,7 +208,7 @@ export class FileSystemStorageBackend implements storage.StorageBackend {nativeBackend,logger: this.logger,});- this.requestQueueBackendCache.push(newStore);+ this.#requestQueueBackendCache.push(newStore);return newStore;}@@ -219,15 +219,15 @@ export class FileSystemStorageBackend implements storage.StorageBackend {switch (type) {case 'Dataset':- backends = this.datasetBackendCache;+ backends = this.#datasetBackendCache;baseDir = this.datasetsDirectory;break;case 'KeyValueStore':- backends = this.keyValueStoreBackendCache;+ backends = this.#keyValueStoreBackendCache;baseDir = this.keyValueStoresDirectory;break;case 'RequestQueue':- backends = this.requestQueueBackendCache;+ backends = this.#requestQueueBackendCache;baseDir = this.requestQueuesDirectory;break;default:@@ -400,6 +400,6 @@ export class FileSystemStorageBackend implements storage.StorageBackend {* are not stuck (until their lock expires) for the next consumer of the same on-disk queue.*/async teardown(): Promise<void> {- await Promise.all(this.requestQueueBackendCache.map(async (queue) => queue.persistState()));+ await Promise.all(this.#requestQueueBackendCache.map(async (queue) => queue.persistState()));}}diff --git a/packages/http-crawler/src/internals/file-download.ts b/packages/http-crawler/src/internals/file-download.tsindex e68fce93b..28fe25003 100644--- a/packages/http-crawler/src/internals/file-download.ts+++ b/packages/http-crawler/src/internals/file-download.ts@@ -2,7 +2,7 @@ import { Transform } from 'node:stream';import type { BasicCrawlerOptions } from '@crawlee/basic';import { BasicCrawler } from '@crawlee/basic';-import type { CrawlingContext, LoadedRequest, Request } from '@crawlee/core';+import type { ContextPipeline, CrawlingContext, LoadedRequest, Request } from '@crawlee/core';import { ResponseWithUrl } from '@crawlee/http-client';import type { Dictionary } from '@crawlee/types';@@ -167,7 +167,7 @@ export class FileDownload extends BasicCrawler<FileDownloadCrawlingContext> {});}- protected override buildContextPipeline() {+ protected override buildContextPipeline(): ContextPipeline<CrawlingContext, FileDownloadCrawlingContext> {return super.buildContextPipeline().compose({action: async (context) => this.initiateDownload(context),cleanup: async (context) => {diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.tsindex 02872fe6a..5d0cb7e47 100644--- a/packages/http-crawler/src/internals/http-crawler.ts+++ b/packages/http-crawler/src/internals/http-crawler.ts@@ -26,16 +26,11 @@ import {Router,SessionError,} from '@crawlee/basic';-import {- type LoadedRequest,- RequestThrottledError,- getCookiesFromResponse,- parseArgument,- schemas,-} from '@crawlee/core';+import { type LoadedRequest, RequestThrottledError, getCookiesFromResponse } from '@crawlee/core';import { ResponseWithUrl } from '@crawlee/http-client';import type { Awaitable, Dictionary, ISession } from '@crawlee/types';-import { type CheerioRoot, RETRY_CSS_SELECTORS } from '@crawlee/utils/internal';+import { parseArgument, RETRY_CSS_SELECTORS, schemas } from '@crawlee/utils/internal';+import type { CheerioAPI } from 'cheerio';import type { RequestLike, ResponseLike } from 'content-type';import contentTypeParser from 'content-type';import iconv from 'iconv-lite';@@ -264,7 +259,7 @@ export interface InternalHttpCrawlingContext<* });* ```*/- parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;+ parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;}export interface HttpCrawlingContext<@@ -366,6 +361,9 @@ export class HttpCrawler<#forceResponseEncoding?: string;readonly #supportedMimeTypes: Set<string>;+ /**+ * @internal+ */protected static override optionsShape = {...BasicCrawler.optionsShape,@@ -380,7 +378,8 @@ export class HttpCrawler<postNavigationHooks: schemas.anyArray.default(() => []),};- protected static override optionsSchema = z.strictObject(HttpCrawler.optionsShape);+ /** @internal */+ protected static optionsSchema = z.strictObject(HttpCrawler.optionsShape);/*** All `HttpCrawlerOptions` parameters are passed via an options object.diff --git a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts b/packages/jsdom-crawler/src/internals/jsdom-crawler.tsindex 77677e789..bedbbe063 100644--- a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts+++ b/packages/jsdom-crawler/src/internals/jsdom-crawler.ts@@ -1,5 +1,6 @@import type {AddRequestsBatchedResult,+ ContextPipeline,CrawlingContext,EnqueueLinksOptions,ErrorHandler,@@ -18,14 +19,13 @@ import {EnqueueStrategy,HttpCrawler,NavigationSkippedError,- parseArgument,resolveBaseUrlForEnqueueLinksFiltering,Router,- tryAbsoluteURL,} from '@crawlee/http';import type { Dictionary } from '@crawlee/types';-import { type CheerioRoot } from '@crawlee/utils/internal';+import type { CheerioAPI } from 'cheerio';import { sleep } from '@crawlee/utils';+import { parseArgument, tryAbsoluteURL } from '@crawlee/utils/internal';import type { DOMWindow } from 'jsdom';import { JSDOM, ResourceLoader, VirtualConsole } from 'jsdom';import { z } from 'zod';@@ -103,7 +103,7 @@ export interface JSDOMCrawlingContext<* });* ```*/- parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;+ parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;/*** Extracts URLs from the parsed DOM, without adding them to the request queue.@@ -212,12 +212,16 @@ export class JSDOMCrawler<>,StatisticStateExtension extends object = {},> extends HttpCrawler<JSDOMCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {+ /**+ * @internal+ */protected static override optionsShape = {...HttpCrawler.optionsShape,runScripts: z.boolean().optional(),hideInternalConsole: z.boolean().optional(),};+ /** @internal */protected static override optionsSchema = z.strictObject(JSDOMCrawler.optionsShape);#runScripts: boolean;@@ -243,7 +247,7 @@ export class JSDOMCrawler<this.#hideInternalConsole = hideInternalConsole;}- protected override buildContextPipeline() {+ protected override buildContextPipeline(): ContextPipeline<CrawlingContext, JSDOMCrawlingContext> {return super.buildContextPipeline().compose({diff --git a/packages/linkedom-crawler/src/internals/linkedom-crawler.ts b/packages/linkedom-crawler/src/internals/linkedom-crawler.tsindex 06ba73d3a..adcd7b4ea 100644--- a/packages/linkedom-crawler/src/internals/linkedom-crawler.ts+++ b/packages/linkedom-crawler/src/internals/linkedom-crawler.ts@@ -1,5 +1,6 @@import type {AddRequestsBatchedResult,+ ContextPipeline,CrawlingContext,EnqueueLinksOptions,ErrorHandler,@@ -20,11 +21,11 @@ import {NavigationSkippedError,resolveBaseUrlForEnqueueLinksFiltering,Router,- tryAbsoluteURL,} from '@crawlee/http';import type { Dictionary } from '@crawlee/types';-import { type CheerioRoot } from '@crawlee/utils/internal';+import type { CheerioAPI } from 'cheerio';import { sleep } from '@crawlee/utils';+import { tryAbsoluteURL } from '@crawlee/utils/internal';import * as cheerio from 'cheerio';import { DOMParser } from 'linkedom/cached';@@ -93,7 +94,7 @@ export interface LinkeDOMCrawlingContext<* });* ```*/- parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;+ parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;/*** Extracts URLs from the parsed DOM, without adding them to the request queue.@@ -213,7 +214,7 @@ export class LinkeDOMCrawler<});}- protected override buildContextPipeline() {+ protected override buildContextPipeline(): ContextPipeline<CrawlingContext, LinkeDOMCrawlingContext> {return super.buildContextPipeline().compose({diff --git a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.tsindex ca6a5319c..53b321857 100644--- a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts+++ b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts@@ -31,15 +31,14 @@ import {createStorageTransaction,EnqueueStrategy,OwnedOrInjected,- parseArgument,RequestHandlerError,resolveBaseUrlForEnqueueLinksFiltering,Router,Statistics,} from '@crawlee/core';import type { Dictionary, Awaitable } from '@crawlee/types';-import { type CheerioRoot, extractUrlsFromCheerio } from '@crawlee/utils/internal';-import { type Cheerio } from 'cheerio';+import { extractUrlsFromCheerio, parseArgument } from '@crawlee/utils/internal';+import { type Cheerio, type CheerioAPI } from 'cheerio';import type { AnyNode } from 'domhandler';import type { Page } from 'playwright';import { z } from 'zod';@@ -138,7 +137,7 @@ export interface AdaptivePlaywrightCrawlerContext<* });* ```*/- parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;+ parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;enqueueLinks(options?: EnqueueLinksOptions): Promise<unknown>;}@@ -473,7 +472,7 @@ export class AdaptivePlaywrightCrawler<return await super.init();}- protected override buildContextPipeline() {+ protected override buildContextPipeline(): ContextPipeline<CrawlingContext, AdaptivePlaywrightCrawlerContext> {const errorMessage = (prop: string) =>`The \`${prop}\` property is not available on the outer context pipeline of AdaptivePlaywrightCrawler - it is provided by the inner (static/browser) pipelines`;diff --git a/packages/playwright-crawler/src/internals/enqueue-links/click-elements.ts b/packages/playwright-crawler/src/internals/enqueue-links/click-elements.tsindex 433a93147..155043fbd 100644--- a/packages/playwright-crawler/src/internals/enqueue-links/click-elements.ts+++ b/packages/playwright-crawler/src/internals/enqueue-links/click-elements.ts@@ -13,13 +13,12 @@ import {constructUrlPatternObjects,createRequestOptions,filterRequestOptionsByPatterns,- parseArgument,urlPatternSchema,Request as CrawleeRequest,- schemas,serviceLocator,} from '@crawlee/browser';import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';+import { parseArgument, schemas } from '@crawlee/utils/internal';import type { Frame, Page, Request, Route } from 'playwright';import { z } from 'zod';diff --git a/packages/playwright-crawler/src/internals/playwright-crawler.ts b/packages/playwright-crawler/src/internals/playwright-crawler.tsindex e38493ed3..0ccd7209b 100644--- a/packages/playwright-crawler/src/internals/playwright-crawler.ts+++ b/packages/playwright-crawler/src/internals/playwright-crawler.ts@@ -2,6 +2,8 @@ import type {BrowserCrawlerOptions,BrowserCrawlingContext,BrowserHook,+ ContextPipeline,+ CrawlingContext,GetUserDataFromRequest,RequestHandler,RouterHandler,@@ -9,16 +11,9 @@ import type {RouteSchemas,RoutesFromSchemas,} from '@crawlee/browser';-import {- assertBrowserPoolNotConfigured,- BrowserCrawler,- parseArgument,- RequestState,- Router,- schemas,- serviceLocator,-} from '@crawlee/browser';+import { assertBrowserPoolNotConfigured, BrowserCrawler, RequestState, Router, serviceLocator } from '@crawlee/browser';import type { Dictionary } from '@crawlee/types';+import { parseArgument, schemas } from '@crawlee/utils/internal';import type { Download, LaunchOptions, Page, Response } from 'playwright';import { z } from 'zod';@@ -224,12 +219,16 @@ export class PlaywrightCrawler<Routes,StatisticStateExtension> {+ /**+ * @internal+ */protected static override optionsShape = {...BrowserCrawler.optionsShape,headless: z.boolean().optional(),launcher: schemas.anyObject.optional(),};+ /** @internal */protected static override optionsSchema = z.strictObject(PlaywrightCrawler.optionsShape);/**@@ -275,7 +274,7 @@ export class PlaywrightCrawler<});}- protected override buildContextPipeline() {+ protected override buildContextPipeline(): ContextPipeline<CrawlingContext, PlaywrightCrawlingContext> {return super.buildContextPipeline().compose({ action: this.enhanceContext.bind(this) });}diff --git a/packages/playwright-crawler/src/internals/playwright-launcher.ts b/packages/playwright-crawler/src/internals/playwright-launcher.tsindex cbb279ded..009c499c6 100644--- a/packages/playwright-crawler/src/internals/playwright-launcher.ts+++ b/packages/playwright-crawler/src/internals/playwright-launcher.ts@@ -1,6 +1,7 @@import type { BrowserLaunchContext } from '@crawlee/browser';-import { BrowserLauncher, Configuration, parseArgument, schemas } from '@crawlee/browser';+import { BrowserLauncher, Configuration } from '@crawlee/browser';import { PlaywrightPlugin } from '@crawlee/browser-pool';+import { parseArgument, schemas } from '@crawlee/utils/internal';import type { Browser, BrowserType, LaunchOptions } from 'playwright';import { z } from 'zod';@@ -77,6 +78,9 @@ export interface PlaywrightLaunchContext extends BrowserLaunchContext<LaunchOpti* @ignore*/export class PlaywrightLauncher extends BrowserLauncher<PlaywrightPlugin> {+ /**+ * @internal+ */protected static override optionsShape = {...BrowserLauncher.optionsShape,// Passthrough schemas — the launcher module object must keep its prototype through parsing.@@ -84,6 +88,7 @@ export class PlaywrightLauncher extends BrowserLauncher<PlaywrightPlugin> {launchContextOptions: schemas.anyObject.optional(),};+ /** @internal */protected static override optionsSchema = z.strictObject(PlaywrightLauncher.optionsShape);/**diff --git a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts b/packages/playwright-crawler/src/internals/utils/playwright-utils.tsindex 5da70f3c2..bc7fcb903 100644--- a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts+++ b/packages/playwright-crawler/src/internals/utils/playwright-utils.ts@@ -22,19 +22,11 @@ import { readFile } from 'node:fs/promises';import { createRequire } from 'node:module';import vm from 'node:vm';-import {- Configuration,- KeyValueStore,- parseArgument,- type Request,- schemas,- serviceLocator,- SessionError,- validators,-} from '@crawlee/browser';+import { Configuration, KeyValueStore, type Request, serviceLocator, SessionError, validators } from '@crawlee/browser';import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';-import { type CheerioRoot } from '@crawlee/utils/internal';+import type { CheerioAPI } from 'cheerio';import { expandShadowRoots, sleep } from '@crawlee/utils';+import { parseArgument, schemas } from '@crawlee/utils/internal';import type { Download, Page, Response, Route } from 'playwright';import { z } from 'zod';@@ -589,7 +581,7 @@ export async function parseWithCheerio(page: Page,ignoreShadowRoots = false,ignoreIframes = false,-): Promise<CheerioRoot> {+): Promise<CheerioAPI> {parseArgument(page, validators.browserPage);const html = ignoreShadowRoots@@ -928,7 +920,7 @@ export interface PlaywrightContextUtils {* });* ```*/- parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;+ parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;/*** Scrolls to the bottom of a page, or until it times out.diff --git a/packages/puppeteer-crawler/src/internals/enqueue-links/click-elements.ts b/packages/puppeteer-crawler/src/internals/enqueue-links/click-elements.tsindex 4442e621f..5126c8d23 100644--- a/packages/puppeteer-crawler/src/internals/enqueue-links/click-elements.ts+++ b/packages/puppeteer-crawler/src/internals/enqueue-links/click-elements.ts@@ -13,13 +13,12 @@ import {constructUrlPatternObjects,createRequestOptions,filterRequestOptionsByPatterns,- parseArgument,urlPatternSchema,Request,- schemas,serviceLocator,} from '@crawlee/browser';import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';+import { parseArgument, schemas } from '@crawlee/utils/internal';// @ts-ignore This only throws when compiled against puppeteer 25+ (ESM only), we only import types, so its alllll goooooodimport type { ClickOptions, Frame, HTTPRequest as PuppeteerRequest, Page, Target } from 'puppeteer';import { z } from 'zod';diff --git a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.tsindex 9ac31a105..6b1b39f70 100644--- a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts+++ b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts@@ -2,6 +2,8 @@ import type {BrowserCrawlerOptions,BrowserCrawlingContext,BrowserHook,+ ContextPipeline,+ CrawlingContext,GetUserDataFromRequest,RouterHandler,RouterRoutes,@@ -9,8 +11,9 @@ import type {RoutesFromSchemas,} from '@crawlee/browser';import { assertBrowserPoolNotConfigured, BrowserCrawler, RequestState, Router } from '@crawlee/browser';-import { parseArgument, serviceLocator } from '@crawlee/core';+import { serviceLocator } from '@crawlee/core';import type { Dictionary } from '@crawlee/types';+import { parseArgument } from '@crawlee/utils/internal';// @ts-ignore This only throws when compiled against puppeteer 25+ (ESM only), we only import types, so its alllll goooooodimport type { HTTPResponse, LaunchOptions, Page } from 'puppeteer';import { z } from 'zod';@@ -193,6 +196,9 @@ export class PuppeteerCrawler<Routes,StatisticStateExtension> {+ /**+ * @internal+ */protected static override optionsShape = {...BrowserCrawler.optionsShape,// Deliberately looser than the declared type: Puppeteer's own accepted string values have moved over@@ -200,6 +206,7 @@ export class PuppeteerCrawler<headless: z.union([z.boolean(), z.string()]).optional(),};+ /** @internal */protected static override optionsSchema = z.strictObject(PuppeteerCrawler.optionsShape);/**@@ -255,7 +262,7 @@ export class PuppeteerCrawler<});}- protected override buildContextPipeline() {+ protected override buildContextPipeline(): ContextPipeline<CrawlingContext, PuppeteerCrawlingContext> {return super.buildContextPipeline().compose({ action: this.enhanceContext.bind(this) });}diff --git a/packages/puppeteer-crawler/src/internals/puppeteer-launcher.ts b/packages/puppeteer-crawler/src/internals/puppeteer-launcher.tsindex 4e1c7e874..4de77c462 100644--- a/packages/puppeteer-crawler/src/internals/puppeteer-launcher.ts+++ b/packages/puppeteer-crawler/src/internals/puppeteer-launcher.ts@@ -1,6 +1,7 @@import type { BrowserLaunchContext } from '@crawlee/browser';-import { BrowserLauncher, Configuration, parseArgument, schemas } from '@crawlee/browser';+import { BrowserLauncher, Configuration } from '@crawlee/browser';import { PuppeteerPlugin } from '@crawlee/browser-pool';+import { parseArgument, schemas } from '@crawlee/utils/internal';// @ts-ignore This only throws when compiled against puppeteer 25+ (ESM only), we only import types, so its alllll goooooodimport type { Browser } from 'puppeteer';import { z } from 'zod';@@ -73,11 +74,15 @@ export interface PuppeteerLaunchContext extends BrowserLaunchContext<PuppeteerPl* @ignore*/export class PuppeteerLauncher extends BrowserLauncher<PuppeteerPlugin, unknown> {+ /**+ * @internal+ */protected static override optionsShape = {...BrowserLauncher.optionsShape,launcher: schemas.anyObject.optional(),};+ /** @internal */protected static override optionsSchema = z.strictObject(PuppeteerLauncher.optionsShape);/**diff --git a/packages/puppeteer-crawler/src/internals/utils/puppeteer_request_interception.ts b/packages/puppeteer-crawler/src/internals/utils/puppeteer_request_interception.tsindex 1d3b6a064..5477016e7 100644--- a/packages/puppeteer-crawler/src/internals/utils/puppeteer_request_interception.ts+++ b/packages/puppeteer-crawler/src/internals/utils/puppeteer_request_interception.ts@@ -1,7 +1,8 @@import { EventEmitter } from 'node:events';-import { parseArgument, schemas, serviceLocator } from '@crawlee/browser';+import { serviceLocator } from '@crawlee/browser';import type { Dictionary } from '@crawlee/types';+import { parseArgument, schemas } from '@crawlee/utils/internal';// @ts-ignore This only throws when compiled against puppeteer 25+ (ESM only), we only import types, so its alllll goooooodimport type { HTTPRequest, HTTPRequest as PuppeteerRequest, Page } from 'puppeteer';diff --git a/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts b/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.tsindex 753bac4be..a69dc7ad5 100644--- a/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts+++ b/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts@@ -23,10 +23,11 @@ import { createRequire } from 'node:module';import vm from 'node:vm';import type { Request } from '@crawlee/browser';-import { Configuration, KeyValueStore, parseArgument, schemas, serviceLocator, validators } from '@crawlee/browser';+import { Configuration, KeyValueStore, serviceLocator, validators } from '@crawlee/browser';import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';-import { type CheerioRoot } from '@crawlee/utils/internal';+import type { CheerioAPI } from 'cheerio';import { expandShadowRoots, sleep } from '@crawlee/utils';+import { parseArgument, schemas } from '@crawlee/utils/internal';import type { ProtocolMapping } from 'devtools-protocol/types/protocol-mapping.js';// @ts-ignore This only throws when compiled against puppeteer 25+ (ESM only), we only import types, so its alllll goooooodimport type { HTTPRequest as PuppeteerRequest, HTTPResponse, Page, ResponseForRequest } from 'puppeteer';@@ -220,7 +221,7 @@ export async function parseWithCheerio(page: Page,ignoreShadowRoots = false,ignoreIframes = false,-): Promise<CheerioRoot> {+): Promise<CheerioAPI> {parseArgument(page, validators.browserPage);if (page.frames().length > 1 && !ignoreIframes) {@@ -872,7 +873,7 @@ export interface PuppeteerContextUtils {* });* ```*/- parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;+ parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;/*** The function finds elements matching a specific CSS selector in a Puppeteer page,diff --git a/packages/stagehand-crawler/src/internals/stagehand-crawler.ts b/packages/stagehand-crawler/src/internals/stagehand-crawler.tsindex 62d67b787..150d67ddc 100644--- a/packages/stagehand-crawler/src/internals/stagehand-crawler.ts+++ b/packages/stagehand-crawler/src/internals/stagehand-crawler.ts@@ -26,8 +26,9 @@ import type {RouteSchemas,RoutesFromSchemas,} from '@crawlee/browser';-import { assertBrowserPoolNotConfigured, BrowserCrawler, parseArgument, Router, schemas } from '@crawlee/browser';+import { assertBrowserPoolNotConfigured, BrowserCrawler, Router } from '@crawlee/browser';import type { Dictionary } from '@crawlee/types';+import { parseArgument, schemas } from '@crawlee/utils/internal';import type { LaunchOptions, Page, Response } from 'playwright';import { z } from 'zod';@@ -412,12 +413,16 @@ export class StagehandCrawler<Routes,StatisticStateExtension> {+ /**+ * @internal+ */protected static override optionsShape = {...BrowserCrawler.optionsShape,stagehandOptions: schemas.anyObject.optional(),headless: z.boolean().optional(),};+ /** @internal */protected static override optionsSchema = z.strictObject(StagehandCrawler.optionsShape);/**diff --git a/packages/stagehand-crawler/src/internals/stagehand-launcher.ts b/packages/stagehand-crawler/src/internals/stagehand-launcher.tsindex b54149f92..7f069ef57 100644--- a/packages/stagehand-crawler/src/internals/stagehand-launcher.ts+++ b/packages/stagehand-crawler/src/internals/stagehand-launcher.ts@@ -1,5 +1,6 @@import type { BrowserLaunchContext } from '@crawlee/browser';-import { BrowserLauncher, Configuration, parseArgument, schemas } from '@crawlee/browser';+import { BrowserLauncher, Configuration } from '@crawlee/browser';+import { parseArgument, schemas } from '@crawlee/utils/internal';import type { BrowserType, LaunchOptions } from 'playwright';import { z } from 'zod';@@ -63,6 +64,9 @@ export interface StagehandLaunchContext extends BrowserLaunchContext<LaunchOptio* @ignore*/export class StagehandLauncher extends BrowserLauncher<StagehandPlugin> {+ /**+ * @internal+ */protected static override optionsShape = {...BrowserLauncher.optionsShape,// Passthrough schemas — the launcher module object must keep its prototype through parsing.@@ -71,6 +75,7 @@ export class StagehandLauncher extends BrowserLauncher<StagehandPlugin> {stagehandOptions: schemas.anyObject.optional(),};+ /** @internal */protected static override optionsSchema = z.strictObject(StagehandLauncher.optionsShape);readonly #stagehandOptions: StagehandOptions;diff --git a/packages/utils/src/internal.ts b/packages/utils/src/internal.tsindex 91cc0cbc2..6f4af65e0 100644--- a/packages/utils/src/internal.ts+++ b/packages/utils/src/internal.ts@@ -1,5 +1,5 @@export * from './internals/blocked.js';-export type { CheerioRoot, CheerioAPI, Cheerio, Element } from './internals/cheerio.js';+export type { CheerioAPI, Cheerio, Element } from './internals/cheerio.js';export { extractUrlsFromCheerio } from './internals/cheerio.js';export { tryAbsoluteURL } from './internals/extract-urls.js';export { URL_NO_COMMAS_REGEX, URL_WITH_COMMAS_REGEX } from './internals/general.js';diff --git a/packages/utils/src/internals/cheerio.ts b/packages/utils/src/internals/cheerio.tsindex 149533abb..20f74819c 100644--- a/packages/utils/src/internals/cheerio.ts+++ b/packages/utils/src/internals/cheerio.ts@@ -3,7 +3,6 @@ import type { CheerioAPI } from 'cheerio';import { tryAbsoluteURL } from './extract-urls.js';-export type CheerioRoot = CheerioAPI;export type { CheerioAPI, Cheerio } from 'cheerio';export type { Element } from 'domhandler';@@ -39,7 +38,7 @@ const BLOCK_TAGS_REGEX =* @param htmlOrCheerioElement HTML text or parsed HTML represented using a [cheerio](https://www.npmjs.com/package/cheerio) function.* @return Plain text*/-export async function htmlToText(htmlOrCheerioElement: string | CheerioRoot): Promise<string> {+export async function htmlToText(htmlOrCheerioElement: string | CheerioAPI): Promise<string> {const { load } = await import('cheerio');if (!htmlOrCheerioElement) return '';diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 40e4b6d79..bfcfa8978 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -393,6 +393,9 @@ importers:'@crawlee/types':specifier: workspace:*version: link:../types+ '@crawlee/utils':+ specifier: workspace:*+ version: link:../utilsfingerprint-generator:specifier: ^2.1.68version: 2.1.82diff --git a/test/core/crawlers/playwright_crawler.test.ts b/test/core/crawlers/playwright_crawler.test.tsindex 109b21c9d..0e9ab7491 100644--- a/test/core/crawlers/playwright_crawler.test.ts+++ b/test/core/crawlers/playwright_crawler.test.ts@@ -11,7 +11,8 @@ import {RequestList,RequestValidationError,} from '@crawlee/playwright';-import type { Cheerio, CheerioAPI, CheerioRoot, Element } from '@crawlee/utils/internal';+import type { Cheerio, CheerioAPI } from 'cheerio';+import type { Element } from 'domhandler';import { sleep } from '@crawlee/utils';import express from 'express';import playwright from 'playwright';@@ -305,7 +306,6 @@ describe('PlaywrightCrawler', () => {// Checking that types are correctconst $ = await crawlingContext.parseWithCheerio();- const _cheerioRootType: CheerioRoot = $;const _apiType: CheerioAPI = $;const _cheerioElementType: Cheerio<Element> = $('div');};diff --git a/test/core/crawlers/puppeteer_crawler.test.ts b/test/core/crawlers/puppeteer_crawler.test.tsindex f90960644..293a35b83 100644--- a/test/core/crawlers/puppeteer_crawler.test.ts+++ b/test/core/crawlers/puppeteer_crawler.test.ts@@ -5,15 +5,9 @@ import type { AddressInfo } from 'node:net';import os from 'node:os';import { promisify } from 'node:util';-import type {- Cheerio,- CheerioAPI,- CheerioRoot,- Element,- PuppeteerCrawlingContext,- PuppeteerGoToOptions,- Request,-} from '@crawlee/puppeteer';+import type { PuppeteerCrawlingContext, PuppeteerGoToOptions, Request } from '@crawlee/puppeteer';+import type { Cheerio, CheerioAPI } from 'cheerio';+import type { Element } from 'domhandler';import {createPuppeteerRouter,ProxyConfiguration,@@ -434,7 +428,6 @@ describe('PuppeteerCrawler', () => {// Checking that types are correctconst $ = await crawlingContext.parseWithCheerio();- const _cheerioRootType: CheerioRoot = $;const _apiType: CheerioAPI = $;const _cheerioElementType: Cheerio<Element> = $('div');};diff --git a/test/core/storages/storage_manager.test.ts b/test/core/storages/storage_manager.test.tsindex 29f4d1115..0ed6566c2 100644--- a/test/core/storages/storage_manager.test.ts+++ b/test/core/storages/storage_manager.test.ts@@ -6,15 +6,12 @@ beforeEach(async () => {describe('StorageManager', () => {test('failed openStorage call does not block subsequent calls (#3661)', async () => {- const goodBackend = serviceLocator.getStorageBackend();- // Delegate to the real backend via the prototype chain so the mock still satisfies- // the StorageBackend interface validation.- const failingBackend = Object.assign(Object.create(goodBackend), {+ // A real instance, so every method the open path calls (`purge`, `getStorageBackendCacheKey`)+ // operates on its own state - only the failing call is stubbed.+ const failingBackend = Object.assign(new MemoryStorageBackend(), {createDatasetBackend: () => {throw new Error('boom');},- // The inherited method reads a private field that only the real instance carries.- getStorageBackendCacheKey: () => 'FailingBackend',});await expect(Dataset.open('will-fail', { storageBackend: failingBackend as any })).rejects.toThrow('boom');diff --git a/test/utils/cheerio.test.ts b/test/utils/cheerio.test.tsindex af2d0352d..83870daa3 100644--- a/test/utils/cheerio.test.ts+++ b/test/utils/cheerio.test.ts@@ -1,10 +1,10 @@-import type { CheerioRoot } from '@crawlee/utils/internal';import { htmlToText } from '@crawlee/utils';+import type { CheerioAPI } from 'cheerio';import * as cheerio from 'cheerio';import * as htmlToTextData from '../shared/data/html_to_text_test_data.js';-const checkHtmlToText = async (html: string | CheerioRoot, expectedText: string, hasBody = false) => {+const checkHtmlToText = async (html: string | CheerioAPI, expectedText: string, hasBody = false) => {const text1 = await htmlToText(html);expect(text1).toEqual(expectedText);