Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

docs: Autogenerated public API documentation (#3798)

Jan Buchara72033d4c202
diff --git a/.github/workflows/test-ci.yml b/.github/workflows/test-ci.yml
index be9381f18..ce4e3bb61 100644
--- a/.github/workflows/test-ci.yml
+++ b/.github/workflows/test-ci.yml
@@ -67,6 +67,37 @@ jobs:
- name: Tests
run: pnpm test
+ api_surface:
+ name: Public API surface
+ runs-on: ubuntu-22.04
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v6
+
+ - name: Use Node.js 24
+ uses: actions/setup-node@v6
+ with:
+ node-version: 24
+ package-manager-cache: false
+
+ - name: Turbo cache
+ id: turbo-cache
+ uses: actions/cache@v5
+ with:
+ path: .turbo
+ key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
+ restore-keys: |
+ turbo-${{ github.job }}-${{ github.ref_name }}-
+
+ - uses: apify/workflows/pnpm-install@main
+
+ - name: Build
+ run: pnpm ci:build
+
+ - name: Check public API surface
+ run: pnpm api:check
+
docs:
name: Docs build
if: (!contains(github.event.head_commit.message, '[skip ci]') && github.ref != 'refs/heads/master' && github.ref != 'refs/heads/v4')
diff --git a/.gitignore b/.gitignore
index 5e3d3b533..5b8eb9c4d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -33,3 +33,6 @@ test/e2e/**/packages
# Local vitest config overrides
vitest.config.local.mts
+
+# API Extractor intermediate reports (the committed reports live in docs/public-api/*.api.md)
+docs/public-api/temp
diff --git a/docs/public-api/README.md b/docs/public-api/README.md
new file mode 100644
index 000000000..da55871b7
--- /dev/null
+++ b/docs/public-api/README.md
@@ -0,0 +1,37 @@
+# Public API surface maps
+
+Each `*.api.md` file in this folder is a generated **map of the public, type-level
+interface** of one publishable `@crawlee/*` package — every exported class, method,
+property, function, and type, with full signatures. These reports define **where we
+promise backwards compatibility**.
+
+They are produced by [API Extractor](https://api-extractor.com/) from the built
+`dist/index.d.ts` of each package.
+
+## Workflow
+
+- After changing any package's public surface, regenerate the reports and commit them:
+
+ ```sh
+ pnpm build # the reports are generated from dist/
+ pnpm api:extract
+ ```
+
+- CI runs `pnpm api:check`, which fails if a committed report is out of date. A failing
+ check means you changed the public API: either that change is intentional (commit the
+ updated report — reviewers will see the surface diff) or it was accidental (fix it).
+
+## Notes
+
+- `docs/public-api/temp/` holds intermediate reports and is git-ignored.
+- `@crawlee/cli` and `@crawlee/templates` are deliberately excluded — they are tooling
+ (a CLI binary and project scaffolding), not an importable API where we promise BC. The
+ exclude list lives in `scripts/api-extractor/run.ts`.
+- The generator lives in `scripts/api-extractor/`. It temporarily strips the build's
+ injected `// @ts-ignore` comment lines from the `.d.ts` files (restoring them
+ afterwards) because API Extractor's AST walker trips over some of them; a small number
+ of packages additionally need a sanitized-mirror fallback. See the comments in
+ `scripts/api-extractor/run.ts` for details.
+- Symbols tagged `@internal` still show up here if they are exported. Shrinking these
+ reports (hiding internals, e.g. via `@internal` + a trimmed rollup, or by not exporting
+ them at all) is the goal tracked in issue #3109.
diff --git a/docs/public-api/crawlee-basic.api.md b/docs/public-api/crawlee-basic.api.md
new file mode 100644
index 000000000..df4d2721e
--- /dev/null
+++ b/docs/public-api/crawlee-basic.api.md
@@ -0,0 +1,335 @@
+## API Report File for "@crawlee/basic"
+
+> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
+
+```ts
+
+import type { AddRequestsBatchedOptions } from '@crawlee/core';
+import type { AddRequestsBatchedResult } from '@crawlee/core';
+import { AnyPredicate } from 'ow';
+import { ArrayPredicate } from 'ow';
+import { AutoscaledPool } from '@crawlee/core';
+import { AutoscaledPoolOptions } from '@crawlee/core';
+import type { Awaitable } from '@crawlee/types';
+import type { BaseHttpClient } from '@crawlee/types';
+import { BasePredicate } from 'ow';
+import type { BatchAddRequestsResult } from '@crawlee/types';
+import { BooleanPredicate } from 'ow';
+import { Cheerio } from '@crawlee/utils';
+import { CheerioAPI } from '@crawlee/utils';
+import { CheerioRoot } from '@crawlee/utils';
+import type { Configuration } from '@crawlee/core';
+import { ContextPipeline } from '@crawlee/core';
+import type { CrawleeLogger } from '@crawlee/core';
+import type { CrawlingContext } from '@crawlee/core';
+import { Dataset } from '@crawlee/core';
+import type { DatasetExportOptions } from '@crawlee/core';
+import type { Dictionary } from '@crawlee/types';
+import { Element as Element_2 } from '@crawlee/utils';
+import type { EnqueueLinksOptions } from '@crawlee/core';
+import type { EventManager } from '@crawlee/core';
+import type { FinalStatistics } from '@crawlee/core';
+import type { GetUserDataFromRequest } from '@crawlee/core';
+import { IRequestLoader } from '@crawlee/core';
+import { IRequestManager } from '@crawlee/core';
+import type { ISession } from '@crawlee/types';
+import type { ISessionPool } from '@crawlee/types';
+import { NumberPredicate } from 'ow';
+import { ObjectPredicate } from 'ow';
+import { Predicate } from 'ow';
+import { ProxyConfiguration } from '@crawlee/core';
+import type { ProxyInfo } from '@crawlee/types';
+import type { ReadonlyDeep } from 'type-fest';
+import { Request as Request_2 } from '@crawlee/core';
+import { RequestQueue } from '@crawlee/core';
+import type { RequestsLike } from '@crawlee/core';
+import { RobotsTxtFile } from '@crawlee/utils';
+import type { RouterHandler } from '@crawlee/core';
+import type { RouterRoutes } from '@crawlee/core';
+import type { SetRequired } from 'type-fest';
+import type { SetStatusMessageOptions } from '@crawlee/types';
+import type { SkippedRequestCallback } from '@crawlee/core';
+import type { Source } from '@crawlee/core';
+import { Statistics } from '@crawlee/core';
+import type { StatisticsOptions } from '@crawlee/core';
+import type { StatisticState } from '@crawlee/core';
+import type { StorageClient } from '@crawlee/types';
+import type { StorageIdentifier } from '@crawlee/core';
+import { StringPredicate } from 'ow';
+import { TimeoutError } from '@apify/timeout';
+
+// @public
+export class BasicCrawler<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension> {
+ constructor(options?: BasicCrawlerOptions<Context, ContextExtension, ExtendedContext> & RequireContextPipeline<CrawlingContext, Context>);
+ // @internal
+ protected addCrawlDepthRequestGenerator(requests: RequestsLike, newRequestDepth: number): AsyncGenerator<Source, void, undefined>;
+ // (undocumented)
+ protected additionalHttpErrorStatusCodes: Set<number>;
+ addRequests(requests: ReadonlyDeep<RequestsLike>, options?: CrawlerAddRequestsOptions): Promise<CrawlerAddRequestsResult>;
+ autoscaledPool?: AutoscaledPool;
+ // (undocumented)
+ protected autoscaledPoolOptions: AutoscaledPoolOptions;
+ get basicContextPipeline(): ContextPipeline<{
+ request: Request_2;
+ }, CrawlingContext>;
+ // (undocumented)
+ protected blockedStatusCodes: Set<number>;
+ protected buildBasicContextPipeline(): ContextPipeline<{
+ request: Request_2;
+ }, CrawlingContext>;
+ protected buildContextPipeline(): ContextPipeline<CrawlingContext, CrawlingContext>;
+ // (undocumented)
+ protected calculateEnqueuedRequestLimit(explicitLimit?: number): Promise<number | undefined>;
+ // (undocumented)
+ protected _canRequestBeRetried(request: Request_2, error: Error): boolean;
+ // (undocumented)
+ get contextPipeline(): ContextPipeline<CrawlingContext, ExtendedContext>;
+ // (undocumented)
+ protected static readonly CRAWLEE_STATE_KEY = "CRAWLEE_STATE";
+ protected _defaultIsFinishedFunction(): Promise<boolean>;
+ protected delayRequest(request: Request_2, source: IRequestManager): boolean;
+ // (undocumented)
+ protected domainAccessedTime: Map<string, number>;
+ // @internal
+ protected enqueueLinksWithCrawlDepth(options: SetRequired<EnqueueLinksOptions, 'urls'>, request: Request_2<Dictionary>, requestManager: IRequestManager): Promise<BatchAddRequestsResult>;
+ // (undocumented)
+ protected errorHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
+ exportData<Data>(path: string, format?: 'json' | 'csv', options?: DatasetExportOptions): Promise<Data[]>;
+ // (undocumented)
+ protected failedRequestHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
+ protected _fetchNextRequest(): Promise<Request_2<Dictionary> | null>;
+ // (undocumented)
+ protected _getCookieHeaderFromRequest(request: Request_2): string;
+ getData(...args: Parameters<Dataset['getData']>): ReturnType<Dataset['getData']>;
+ getDataset(identifier?: string | StorageIdentifier): Promise<Dataset>;
+ protected _getMessageFromError(error: Error, forceStack?: boolean): string | TimeoutError | undefined;
+ // (undocumented)
+ protected getPendingRequestCountApproximation(): Promise<number>;
+ getRequestManager(): Promise<IRequestManager>;
+ // @deprecated (undocumented)
+ getRequestQueue(): Promise<IRequestManager>;
+ // (undocumented)
+ protected getRobotsTxtFileForUrl(url: string): Promise<RobotsTxtFile | undefined>;
+ // (undocumented)
+ protected handledRequestsCount: number;
+ // (undocumented)
+ protected _handleFailedRequestHandler(crawlingContext: CrawlingContext, error: Error): Promise<void>;
+ protected handleRequest(crawlingContext: ExtendedContext, requestSource: IRequestManager, request: Request_2): Promise<void>;
+ // (undocumented)
+ protected handleSkippedRequest(options: Parameters<SkippedRequestCallback>[0]): Promise<void>;
+ // (undocumented)
+ hasFinishedBefore: boolean;
+ // (undocumented)
+ protected httpClient: BaseHttpClient;
+ // (undocumented)
+ protected ignoreHttpErrorStatusCodes: Set<number>;
+ protected _init(): Promise<void>;
+ // (undocumented)
+ protected internalTimeoutMillis: number;
+ protected isErrorStatusCode(status: number): boolean;
+ protected isProxyError(error: Error): boolean;
+ protected _isTaskReadyFunction(): Promise<boolean>;
+ protected _loadHandledRequestCount(): Promise<void>;
+ // (undocumented)
+ get log(): CrawleeLogger;
+ // (undocumented)
+ protected maxCrawlDepth?: number;
+ // (undocumented)
+ protected maxRequestRetries: number;
+ // (undocumented)
+ protected maxRequestsPerCrawl?: number;
+ // (undocumented)
+ protected onSkippedRequest?: SkippedRequestCallback;
+ // (undocumented)
+ protected static optionsShape: {
+ contextPipelineBuilder: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ extendContext: Predicate<Function> & BasePredicate<Function | undefined>;
+ requestList: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ requestQueue: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ requestHandler: Predicate<Function> & BasePredicate<Function | undefined>;
+ requestHandlerTimeoutSecs: NumberPredicate & BasePredicate<number | undefined>;
+ errorHandler: Predicate<Function> & BasePredicate<Function | undefined>;
+ failedRequestHandler: Predicate<Function> & BasePredicate<Function | undefined>;
+ maxRequestRetries: NumberPredicate & BasePredicate<number | undefined>;
+ sameDomainDelaySecs: NumberPredicate & BasePredicate<number | undefined>;
+ maxRequestsPerCrawl: NumberPredicate & BasePredicate<number | undefined>;
+ maxCrawlDepth: NumberPredicate & BasePredicate<number | undefined>;
+ autoscaledPoolOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ sessionPool: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ proxyConfiguration: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ statusMessageLoggingInterval: NumberPredicate & BasePredicate<number | undefined>;
+ statusMessageCallback: Predicate<Function> & BasePredicate<Function | undefined>;
+ additionalHttpErrorStatusCodes: ArrayPredicate<number>;
+ ignoreHttpErrorStatusCodes: ArrayPredicate<number>;
+ blockedStatusCodes: ArrayPredicate<number>;
+ retryOnBlocked: BooleanPredicate & BasePredicate<boolean | undefined>;
+ respectRobotsTxtFile: AnyPredicate<boolean | object>;
+ onSkippedRequest: Predicate<Function> & BasePredicate<Function | undefined>;
+ httpClient: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ configuration: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ storageClient: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ eventManager: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ logger: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ minConcurrency: NumberPredicate & BasePredicate<number | undefined>;
+ maxConcurrency: NumberPredicate & BasePredicate<number | undefined>;
+ maxRequestsPerMinute: NumberPredicate & BasePredicate<number | undefined>;
+ keepAlive: BooleanPredicate & BasePredicate<boolean | undefined>;
+ statisticsOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ id: StringPredicate & BasePredicate<string | undefined>;
+ };
+ // (undocumented)
+ protected _pauseOnMigration(): Promise<void>;
+ proxyConfiguration?: ProxyConfiguration;
+ pushData(data: Parameters<Dataset['pushData']>[0], datasetIdentifier?: string | StorageIdentifier): Promise<void>;
+ protected _requestFunctionErrorHandler(error: Error, crawlingContext: CrawlingContext, request: Request_2, source: IRequestManager): Promise<void>;
+ // (undocumented)
+ protected requestHandler: RequestHandler<ExtendedContext>;
+ // (undocumented)
+ protected requestHandlerTimeoutMillis: number;
+ protected requestManager?: IRequestManager;
+ // (undocumented)
+ protected respectRobotsTxtFile: boolean | {
+ userAgent?: string;
+ };
+ // (undocumented)
+ protected retryOnBlocked: boolean;
+ readonly router: RouterHandler<Context>;
+ run(requests?: RequestsLike, options?: CrawlerRunOptions): Promise<FinalStatistics>;
+ // (undocumented)
+ running: boolean;
+ // (undocumented)
+ protected runRequestHandler(crawlingContext: ExtendedContext): Promise<void>;
+ // (undocumented)
+ protected sameDomainDelayMillis: number;
+ sessionPool: ISessionPool;
+ setStatusMessage(message: string, options?: SetStatusMessageOptions): Promise<void>;
+ readonly stats: Statistics;
+ // (undocumented)
+ protected statusMessageCallback?: StatusMessageCallback;
+ // (undocumented)
+ protected statusMessageLoggingInterval: number;
+ stop(reason?: string): void;
+ // (undocumented)
+ protected _tagUserHandlerError<T>(cb: () => unknown): Promise<T>;
+ teardown(): Promise<void>;
+ protected _throwOnBlockedRequest(statusCode: number): void;
+ protected _timeoutAndRetry<T>(handler: () => Promise<T>, timeout: number, error: Error | string, maxRetries?: number, retried?: number): Promise<T>;
+ // (undocumented)
+ protected unexpectedStop: boolean;
+ // (undocumented)
+ useState<State extends Dictionary = Dictionary>(defaultValue?: State): Promise<State>;
+}
+
+// @public (undocumented)
+export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension> {
+ additionalHttpErrorStatusCodes?: number[];
+ autoscaledPoolOptions?: AutoscaledPoolOptions;
+ blockedStatusCodes?: number[];
+ configuration?: Configuration;
+ contextPipelineBuilder?: () => ContextPipeline<CrawlingContext, Context>;
+ errorHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
+ eventManager?: EventManager;
+ extendContext?: (context: Context) => Awaitable<ContextExtension>;
+ failedRequestHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
+ httpClient?: BaseHttpClient;
+ id?: string;
+ ignoreHttpErrorStatusCodes?: number[];
+ keepAlive?: boolean;
+ logger?: CrawleeLogger;
+ maxConcurrency?: number;
+ maxCrawlDepth?: number;
+ maxRequestRetries?: number;
+ maxRequestsPerCrawl?: number;
+ maxRequestsPerMinute?: number;
+ minConcurrency?: number;
+ onSkippedRequest?: SkippedRequestCallback;
+ proxyConfiguration?: ProxyConfiguration;
+ requestHandler?: RequestHandler<ExtendedContext>;
+ requestHandlerTimeoutSecs?: number;
+ // @deprecated
+ requestList?: IRequestLoader;
+ requestManager?: IRequestManager;
+ // @deprecated
+ requestQueue?: RequestQueue;
+ respectRobotsTxtFile?: boolean | {
+ userAgent?: string;
+ };
+ retryOnBlocked?: boolean;
+ sameDomainDelaySecs?: number;
+ sessionPool?: ISessionPool;
+ statisticsOptions?: StatisticsOptions;
+ statusMessageCallback?: StatusMessageCallback;
+ statusMessageLoggingInterval?: number;
+ storageClient?: StorageClient;
+}
+
+// @public (undocumented)
+export interface BasicCrawlingContext<UserData extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
+}
+
+export { Cheerio }
+
+export { CheerioAPI }
+
+export { CheerioRoot }
+
+// @public (undocumented)
+export interface CrawlerAddRequestsOptions extends AddRequestsBatchedOptions {
+}
+
+// @public (undocumented)
+export interface CrawlerAddRequestsResult extends AddRequestsBatchedResult {
+}
+
+// @public (undocumented)
+export interface CrawlerRunOptions extends CrawlerAddRequestsOptions {
+ purgeRequestQueue?: boolean;
+}
+
+// @public
+export function createBasicRouter<Context extends BasicCrawlingContext = BasicCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): RouterHandler<Context>;
+
+// @public (undocumented)
+export interface CreateContextOptions {
+ // (undocumented)
+ proxyInfo?: ProxyInfo;
+ // (undocumented)
+ request: Request_2;
+ // (undocumented)
+ session: ISession;
+}
+
+export { Element_2 as Element }
+
+// @public (undocumented)
+export type ErrorHandler<Context extends CrawlingContext = CrawlingContext, ExtendedContext extends Context = Context> = (inputs: Context & Partial<ExtendedContext>, error: Error) => Awaitable<void>;
+
+// @public (undocumented)
+export type RequestHandler<Context extends CrawlingContext = CrawlingContext> = (inputs: Context) => Awaitable<void>;
+
+// @public (undocumented)
+export type RequireContextPipeline<DefaultContextType extends CrawlingContext, FinalContextType extends DefaultContextType> = DefaultContextType extends FinalContextType ? {} : {
+ contextPipelineBuilder: () => ContextPipeline<CrawlingContext, FinalContextType>;
+};
+
+// @public (undocumented)
+export type StatusMessageCallback<Context extends CrawlingContext = BasicCrawlingContext, Crawler extends BasicCrawler<any> = BasicCrawler<Context>> = (params: StatusMessageCallbackParams<Context, Crawler>) => Awaitable<void>;
+
+// @public (undocumented)
+export interface StatusMessageCallbackParams<Context extends CrawlingContext = BasicCrawlingContext, Crawler extends BasicCrawler<any> = BasicCrawler<Context>> {
+ // (undocumented)
+ crawler: Crawler;
+ // (undocumented)
+ message: string;
+ // (undocumented)
+ previousState: StatisticState;
+ // (undocumented)
+ state: StatisticState;
+}
+
+
+export * from "@crawlee/core";
+
+// (No @packageDocumentation comment for this package)
+
+```
diff --git a/docs/public-api/crawlee-browser-pool.api.md b/docs/public-api/crawlee-browser-pool.api.md
new file mode 100644
index 000000000..d6c3fd809
--- /dev/null
+++ b/docs/public-api/crawlee-browser-pool.api.md
@@ -0,0 +1,536 @@
+## API Report File for "@crawlee/browser-pool"
+
+> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
+
+```ts
+
+import type { Browser } from 'playwright';
+import type { BrowserContext } from 'playwright';
+import type { BrowserFingerprintWithHeaders } from 'fingerprint-generator';
+import type { BrowserType } from 'playwright';
+import type { Cookie } from '@crawlee/types';
+import { CrawleeLogger } from '@crawlee/core';
+import { CriticalError } from '@crawlee/core';
+import type { Dictionary } from '@crawlee/types';
+import { EventEmitter } from 'node:events';
+import { FingerprintGenerator as FingerprintGenerator_2 } from 'fingerprint-generator';
+import type { FingerprintGeneratorOptions as FingerprintGeneratorOptions_2 } from 'fingerprint-generator';
+import { FingerprintInjector } from 'fingerprint-injector';
+import { IBrowserPool } from '@crawlee/types';
+import { NewPageOptions } from '@crawlee/types';
+import type { Page } from 'playwright';
+import type { PageState } from '@crawlee/types';
+import type Puppeteer from 'puppeteer';
+import type * as PuppeteerTypes from 'puppeteer';
+import QuickLRU from 'quick-lru';
+import { TypedEmitter } from 'tiny-typed-emitter';
+
+// @internal (undocumented)
+export const anonymizeProxySugar: (proxyUrl?: string, username?: string, password?: string, options?: AnonymizeProxySugarOptions) => Promise<[string | undefined, PromiseVoid]>;
+
+// @public (undocumented)
+export interface AnonymizeProxySugarOptions {
+ // (undocumented)
+ ignoreProxyCertificate?: boolean;
+}
+
+// @public (undocumented)
+export const enum BROWSER_CONTROLLER_EVENTS {
+ // (undocumented)
+ BROWSER_CLOSED = "browserClosed"
+}
+
+// @public (undocumented)
+export const enum BROWSER_POOL_EVENTS {
+ // (undocumented)
+ BROWSER_CLOSED = "browserClosed",
+ // (undocumented)
+ BROWSER_LAUNCHED = "browserLaunched",
+ // (undocumented)
+ BROWSER_RETIRED = "browserRetired",
+ // (undocumented)
+ PAGE_CLOSED = "pageClosed",
+ // (undocumented)
+ PAGE_CREATED = "pageCreated"
+}
+
+// @public
+export abstract class BrowserController<Library extends CommonLibrary = CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> extends TypedEmitter<BrowserControllerEvents<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>> implements IBrowserController<NewPageResult> {
+ constructor(browserPlugin: BrowserPlugin<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>);
+ activate(): void;
+ // (undocumented)
+ activePages: number;
+ // (undocumented)
+ assignBrowser(browser: LaunchResult, launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): void;
+ browser: LaunchResult;
+ browserPlugin: BrowserPlugin<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;
+ close(): Promise<void>;
+ // (undocumented)
+ protected abstract _close(): Promise<void>;
+ // (undocumented)
+ getCookies(page: NewPageResult): Promise<Cookie[]>;
+ // (undocumented)
+ protected abstract _getCookies(page: NewPageResult): Promise<Cookie[]>;
+ // (undocumented)
+ id: string;
+ // (undocumented)
+ isActive: boolean;
+ kill(): Promise<void>;
+ // (undocumented)
+ protected abstract _kill(): Promise<void>;
+ // (undocumented)
+ lastPageOpenedAt: number;
+ launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;
+ // (undocumented)
+ protected log: CrawleeLogger;
+ newPage(pageOptions?: NewPageOptions): Promise<NewPageResult>;
+ // (undocumented)
+ protected abstract _newPage(pageOptions?: NewPageOptions): Promise<NewPageResult>;
+ // (undocumented)
+ abstract normalizeProxyOptions(proxyUrl: string | undefined, pageOptions: any): Record<string, unknown>;
+ proxyUrl?: string;
+ // (undocumented)
+ setCookies(page: NewPageResult, cookies: Cookie[]): Promise<void>;
+ // (undocumented)
+ protected abstract _setCookies(page: NewPageResult, cookies: Cookie[]): Promise<void>;
+ // (undocumented)
+ totalPages: number;
+}
+
+// @public (undocumented)
+export interface BrowserControllerEvents<Library extends CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> {
+ // (undocumented)
+ [BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED]: (controller: BrowserController<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>) => void;
+}
+
+// @public (undocumented)
+export class BrowserLaunchError extends CriticalError {
+ constructor(...args: ConstructorParameters<typeof CriticalError>);
+}
+
+// @public (undocumented)
+export enum BrowserName {
+ // (undocumented)
+ chrome = "chrome",
+ // (undocumented)
+ edge = "edge",
+ // (undocumented)
+ firefox = "firefox",
+ // (undocumented)
+ safari = "safari"
+}
+
+// @public
+export abstract class BrowserPlugin<Library extends CommonLibrary = CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> {
+ constructor(library: Library, options?: BrowserPluginOptions<LibraryOptions>);
+ // (undocumented)
+ protected abstract _addProxyToLaunchOptions(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): Promise<void>;
+ // (undocumented)
+ browserPerProxy?: boolean;
+ // (undocumented)
+ abstract createController(): BrowserController<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;
+ createLaunchContext(options?: CreateLaunchContextOptions<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;
+ // (undocumented)
+ ignoreProxyCertificate?: boolean;
+ // (undocumented)
+ protected abstract _isChromiumBasedBrowser(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): boolean;
+ launch(launchContext?: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): Promise<LaunchResult>;
+ // (undocumented)
+ protected abstract _launch(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): Promise<LaunchResult>;
+ // (undocumented)
+ launchOptions: LibraryOptions;
+ // (undocumented)
+ library: Library;
+ // (undocumented)
+ protected log: CrawleeLogger;
+ // (undocumented)
+ name: string;
+ // (undocumented)
+ proxyUrl?: string;
+ // (undocumented)
+ protected _throwAugmentedLaunchError(cause: unknown, executablePath: string | undefined, dockerImage: string, moduleInstallCommand: string): never;
+ // (undocumented)
+ useIncognitoPages: boolean;
+ // (undocumented)
+ userDataDir?: string;
+}
+
+// @public (undocumented)
+export interface BrowserPluginOptions<LibraryOptions> {
+ browserPerProxy?: boolean;
+ ignoreProxyCertificate?: boolean;
+ launchOptions?: LibraryOptions;
+ proxyUrl?: string;
+ useIncognitoPages?: boolean;
+ userDataDir?: string;
+}
+
+// @public
+export class BrowserPool<Options extends BrowserPoolOptions = BrowserPoolOptions, BrowserPlugins extends BrowserPlugin[] = InferBrowserPluginArray<Options['browserPlugins']>, BrowserControllerReturn extends BrowserController = ReturnType<BrowserPlugins[number]['createController']>, LaunchContextReturn extends LaunchContext = ReturnType<BrowserPlugins[number]['createLaunchContext']>, PageOptions = Parameters<BrowserControllerReturn['newPage']>[0], PageReturn extends UnwrapPromise<ReturnType<BrowserControllerReturn['newPage']>> = UnwrapPromise<ReturnType<BrowserControllerReturn['newPage']>>> extends TypedEmitter<BrowserPoolEvents<BrowserControllerReturn, PageReturn>> implements IBrowserPool<PageReturn> {
+ constructor(options: Options & BrowserPoolHooks<BrowserControllerReturn, LaunchContextReturn, PageReturn>);
+ // (undocumented)
+ activeBrowserControllers: Set<BrowserControllerReturn>;
+ // (undocumented)
+ browserPlugins: BrowserPlugins;
+ closeAllBrowsers(): Promise<void>;
+ // (undocumented)
+ closeInactiveBrowserAfterMillis: number;
+ closePage(page: PageReturn, options?: {
+ error?: Error;
+ }): Promise<void>;
+ destroy(): Promise<void>;
+ extractPageState(page: PageReturn): Promise<PageState>;
+ // (undocumented)
+ fingerprintCache?: QuickLRU<string, BrowserFingerprintWithHeaders>;
+ // (undocumented)
+ fingerprintGenerator?: FingerprintGenerator_2;
+ // (undocumented)
+ fingerprintInjector?: FingerprintInjector;
+ // (undocumented)
+ fingerprintOptions: FingerprintOptions;
+ getBrowserControllerByPage(page: PageReturn): BrowserControllerReturn | undefined;
+ getPage(id: string): PageReturn | undefined;
+ getPageId(page: PageReturn): string | undefined;
+ injectPageState(page: PageReturn, state: PageState): Promise<void>;
+ // (undocumented)
+ maxOpenPagesPerBrowser: number;
+ newPage(options?: BrowserPoolNewPageOptions<PageOptions, BrowserPlugins[number]>): Promise<PageReturn>;
+ newPageInNewBrowser(options?: BrowserPoolNewPageInNewBrowserOptions<PageOptions, BrowserPlugins[number]>): Promise<PageReturn>;
+ newPageWithEachPlugin(optionsList?: Omit<BrowserPoolNewPageOptions<PageOptions, BrowserPlugins[number]>, 'browserPlugin'>[]): Promise<PageReturn[]>;
+ // (undocumented)
+ operationTimeoutMillis: number;
+ // (undocumented)
+ pageCounter: number;
+ // (undocumented)
+ pageIds: WeakMap<PageReturn, string>;
+ // (undocumented)
+ pages: Map<string, PageReturn>;
+ // (undocumented)
+ pageToBrowserController: WeakMap<PageReturn, BrowserControllerReturn>;
+ // (undocumented)
+ postLaunchHooks: PostLaunchHook<BrowserControllerReturn>[];
+ // (undocumented)
+ postPageCloseHooks: PostPageCloseHook<BrowserControllerReturn>[];
+ // (undocumented)
+ postPageCreateHooks: PostPageCreateHook<BrowserControllerReturn, PageReturn>[];
+ // (undocumented)
+ preLaunchHooks: PreLaunchHook<LaunchContextReturn>[];
+ // (undocumented)
+ prePageCloseHooks: PrePageCloseHook<BrowserControllerReturn, PageReturn>[];
+ // (undocumented)
+ prePageCreateHooks: PrePageCreateHook<BrowserControllerReturn, PageOptions>[];
+ retireAllBrowsers(): void;
+ // (undocumented)
+ retireBrowserAfterPageCount: number;
+ retireBrowserByPage(page: PageReturn): void;
+ retireBrowserController(browserController: BrowserControllerReturn): void;
+ // (undocumented)
+ retiredBrowserControllers: Set<BrowserControllerReturn>;
+ // (undocumented)
+ startingBrowserControllers: Set<BrowserControllerReturn>;
+ // (undocumented)
+ useFingerprints?: boolean;
+}
+
+// @public (undocumented)
+export interface BrowserPoolEvents<BC extends BrowserController, Page> {
+ // (undocumented)
+ [BROWSER_POOL_EVENTS.BROWSER_LAUNCHED]: (browserController: BC) => void | Promise<void>;
+ // (undocumented)
+ [BROWSER_POOL_EVENTS.BROWSER_RETIRED]: (browserController: BC) => void | Promise<void>;
+ // (undocumented)
+ [BROWSER_POOL_EVENTS.PAGE_CLOSED]: (page: Page) => void | Promise<void>;
+ // (undocumented)
+ [BROWSER_POOL_EVENTS.PAGE_CREATED]: (page: Page) => void | Promise<void>;
+}
+
+// @public (undocumented)
+export interface BrowserPoolHooks<BC extends BrowserController, LC extends LaunchContext, PR extends UnwrapPromise<ReturnType<BC['newPage']>> = UnwrapPromise<ReturnType<BC['newPage']>>> {
+ postLaunchHooks?: PostLaunchHook<BC>[];
+ postPageCloseHooks?: PostPageCloseHook<BC>[];
+ postPageCreateHooks?: PostPageCreateHook<BC, PR>[];
+ preLaunchHooks?: PreLaunchHook<LC>[];
+ prePageCloseHooks?: PrePageCloseHook<BC, PR>[];
+ prePageCreateHooks?: PrePageCreateHook<BC>[];
+}
+
+// @public (undocumented)
+export interface BrowserPoolNewPageInNewBrowserOptions<PageOptions, BP extends BrowserPlugin> {
+ browserPlugin?: BP;
+ id?: string;
+ launchOptions?: BP['launchOptions'];
+ pageOptions?: PageOptions;
+}
+
+// @public (undocumented)
+export interface BrowserPoolNewPageOptions<PageOptions, BP extends BrowserPlugin> extends NewPageOptions {
+ browserPlugin?: BP;
+ ignoreTlsErrors?: boolean;
+ pageOptions?: PageOptions;
+ proxyUrl?: string;
+}
+
+// @public (undocumented)
+export interface BrowserPoolOptions<Plugin extends BrowserPlugin = BrowserPlugin> {
+ browserPlugins: readonly Plugin[];
+ closeInactiveBrowserAfterSecs?: number;
+ // (undocumented)
+ fingerprintOptions?: FingerprintOptions;
+ maxOpenPagesPerBrowser?: number;
+ operationTimeoutSecs?: number;
+ retireBrowserAfterPageCount?: number;
+ retireInactiveBrowserAfterSecs?: number;
+ useFingerprints?: boolean;
+}
+
+// @public (undocumented)
+export interface BrowserSpecification {
+ httpVersion?: HttpVersion;
+ maxVersion?: number;
+ minVersion?: number;
+ name: BrowserName;
+}
+
+// @public
+export interface CommonLibrary {
+ // (undocumented)
+ launch(opts?: Dictionary): Promise<CommonBrowser>;
+ // (undocumented)
+ name?: () => string;
+ // (undocumented)
+ product?: string;
+}
+
+// @internal (undocumented)
+export interface CommonPage {
+ // (undocumented)
+ close(...args: unknown[]): Promise<unknown>;
+ // (undocumented)
+ url(): string | Promise<string>;
+}
+
+// @public (undocumented)
+export interface CreateLaunchContextOptions<Library extends CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> extends Partial<Omit<LaunchContextOptions<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>, 'browserPlugin'>> {
+}
+
+// @public
+export const DEFAULT_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36";
+
+// @public (undocumented)
+export const enum DeviceCategory {
+ desktop = "desktop",
+ mobile = "mobile"
+}
+
+// @public (undocumented)
+export interface FingerprintGenerator {
+ // (undocumented)
+ getFingerprint: (fingerprintGeneratorOptions?: FingerprintGeneratorOptions) => GetFingerprintReturn;
+}
+
+// @public (undocumented)
+export interface FingerprintGeneratorOptions extends Partial<FingerprintGeneratorOptions_2> {
+}
+
+// @public
+export interface FingerprintOptions {
+ fingerprintCacheSize?: number;
+ fingerprintGeneratorOptions?: FingerprintGeneratorOptions;
+ useFingerprintCache?: boolean;
+}
+
+// @public (undocumented)
+export interface GetFingerprintReturn {
+ // (undocumented)
+ fingerprint: BrowserFingerprintWithHeaders;
+}
+
+// @public
+export interface IBrowserController<Page = unknown> {
+ readonly browser: unknown;
+ close(): Promise<void>;
+ getCookies(page: Page): Promise<Cookie[]>;
+ readonly id: string;
+ readonly launchContext: IBrowserLaunchContext;
+ setCookies(page: Page, cookies: Cookie[]): Promise<void>;
+}
+
+// @public
+export interface IBrowserLaunchContext {
+ fingerprint?: unknown;
+ launchOptions?: Dictionary | undefined;
+ proxyUrl?: string;
+ useIncognitoPages?: boolean;
+}
+
+export { IBrowserPool }
+
+// @public (undocumented)
+export type InferBrowserPluginArray<Input extends readonly unknown[], Result extends BrowserPlugin[] = []> = Input extends readonly [infer FirstValue, ...infer Rest] | [infer FirstValue, ...infer Rest] ? FirstValue extends PlaywrightPlugin ? InferBrowserPluginArray<Rest, [...Result, PlaywrightPlugin]> : FirstValue extends PuppeteerPlugin ? InferBrowserPluginArray<Rest, [...Result, PuppeteerPlugin]> : never : Input extends [] ? Result : Input extends readonly (infer U)[] ? [
+U
+] extends [PuppeteerPlugin | PlaywrightPlugin] ? U[] : never : Result;
+
+// @public (undocumented)
+export class LaunchContext<Library extends CommonLibrary = CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> {
+ constructor(options: LaunchContextOptions<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>);
+ // (undocumented)
+ [K: PropertyKey]: unknown;
+ // (undocumented)
+ browserPerProxy?: boolean;
+ // (undocumented)
+ browserPlugin: BrowserPlugin<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;
+ extend<T extends Record<PropertyKey, unknown>>(fields: T): void;
+ // (undocumented)
+ fingerprint?: BrowserFingerprintWithHeaders;
+ // (undocumented)
+ id?: string;
+ // (undocumented)
+ ignoreProxyCertificate?: boolean;
+ // (undocumented)
+ launchOptions: LibraryOptions;
+ set proxyUrl(url: string | undefined);
+ get proxyUrl(): string | undefined;
+ // (undocumented)
+ useIncognitoPages: boolean;
+ // (undocumented)
+ userDataDir: string;
+}
+
+// @public
+export interface LaunchContextOptions<Library extends CommonLibrary = CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> {
+ browserPerProxy?: boolean;
+ browserPlugin: BrowserPlugin<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;
+ id?: string;
+ ignoreProxyCertificate?: boolean;
+ launchOptions: LibraryOptions;
+ // (undocumented)
+ proxyUrl?: string;
+ useIncognitoPages?: boolean;
+ userDataDir?: string;
+}
+
+export { NewPageOptions }
+
+// @public (undocumented)
+export const enum OperatingSystemsName {
+ android = "android",
+ ios = "ios",
+ // (undocumented)
+ linux = "linux",
+ // (undocumented)
+ macos = "macos",
+ // (undocumented)
+ windows = "windows"
+}
+
+// @public
+export class PlaywrightBrowser extends EventEmitter {
+ // (undocumented)
+ [Symbol.asyncDispose](): Promise<void>;
+ constructor(options: BrowserOptions);
+ // (undocumented)
+ browserType(): BrowserType;
+ // (undocumented)
+ close(): Promise<void>;
+ // (undocumented)
+ contexts(): BrowserContext[];
+ // (undocumented)
+ isConnected(): boolean;
+ // (undocumented)
+ newBrowserCDPSession(): Promise<never>;
+ // (undocumented)
+ newContext(): Promise<never>;
+ // (undocumented)
+ newPage(...args: Parameters<BrowserContext['newPage']>): ReturnType<BrowserContext['newPage']>;
+ // @internal (undocumented)
+ _setBrowserType(browserType: BrowserType): void;
+ // (undocumented)
+ startTracing(): Promise<never>;
+ // (undocumented)
+ stopTracing(): Promise<never>;
+ // (undocumented)
+ version(): string;
+}
+
+// @public (undocumented)
+export class PlaywrightController extends BrowserController<BrowserType, SafeParameters<BrowserType['launch']>[0], Browser> {
+ // (undocumented)
+ protected _close(): Promise<void>;
+ // (undocumented)
+ protected _getCookies(page: Page): Promise<Cookie[]>;
+ // (undocumented)
+ protected _kill(): Promise<void>;
+ // (undocumented)
+ protected _newPage(contextOptions?: SafeParameters<Browser['newPage']>[0]): Promise<Page>;
+ // (undocumented)
+ normalizeProxyOptions(proxyUrl: string | undefined, pageOptions: any): Record<string, unknown>;
+ // (undocumented)
+ protected _setCookies(page: Page, cookies: Cookie[]): Promise<void>;
+}
+
+// @public (undocumented)
+export class PlaywrightPlugin extends BrowserPlugin<BrowserType, SafeParameters<BrowserType['launch']>[0], Browser> {
+ // (undocumented)
+ protected _addProxyToLaunchOptions(launchContext: LaunchContext<BrowserType>): Promise<void>;
+ // (undocumented)
+ _containerProxyServer?: Awaited<ReturnType<typeof createProxyServerForContainers>>;
+ // (undocumented)
+ createController(): PlaywrightController;
+ // (undocumented)
+ protected _isChromiumBasedBrowser(): boolean;
+ // (undocumented)
+ protected _launch(launchContext: LaunchContext<BrowserType>): Promise<Browser>;
+}
+
+// @public
+export type PostLaunchHook<BC extends BrowserController> = (pageId: string, browserController: BC) => void | Promise<void>;
+
+// @public
+export type PostPageCloseHook<BC extends BrowserController> = (pageId: string, browserController: BC) => void | Promise<void>;
+
+// @public
+export type PostPageCreateHook<BC extends BrowserController, Page = UnwrapPromise<ReturnType<BC['newPage']>>> = (page: Page, browserController: BC) => void | Promise<void>;
+
+// @public
+export type PreLaunchHook<LC extends LaunchContext> = (pageId: string, launchContext: LC) => void | Promise<void>;
+
+// @public
+export type PrePageCloseHook<BC extends BrowserController, Page = UnwrapPromise<ReturnType<BC['newPage']>>> = (page: Page, browserController: BC) => void | Promise<void>;
+
+// @public
+export type PrePageCreateHook<BC extends BrowserController, PO = Parameters<BC['newPage']>[0]> = (pageId: string, browserController: BC, pageOptions?: PO) => void | Promise<void>;
+
+// @public (undocumented)
+export class PuppeteerController extends BrowserController<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions> {
+ // (undocumented)
+ protected _close(): Promise<void>;
+ // (undocumented)
+ protected _getCookies(page: PuppeteerTypes.Page): Promise<Cookie[]>;
+ // (undocumented)
+ protected _kill(): Promise<void>;
+ // (undocumented)
+ protected _newPage(contextOptions?: PuppeteerNewPageOptions): Promise<PuppeteerTypes.Page>;
+ // (undocumented)
+ normalizeProxyOptions(proxyUrl: string | undefined, pageOptions: any): Record<string, unknown>;
+ // (undocumented)
+ protected _setCookies(page: PuppeteerTypes.Page, cookies: Cookie[]): Promise<void>;
+}
+
+// @public (undocumented)
+export class PuppeteerPlugin extends BrowserPlugin<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions> {
+ // (undocumented)
+ protected _addProxyToLaunchOptions(_launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): Promise<void>;
+ // (undocumented)
+ createController(): PuppeteerController;
+ // (undocumented)
+ protected _isChromiumBasedBrowser(_launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): boolean;
+ // (undocumented)
+ protected _launch(launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): Promise<PuppeteerTypes.Browser>;
+}
+
+// @public (undocumented)
+export type UnwrapPromise<T> = T extends PromiseLike<infer R> ? UnwrapPromise<R> : T;
+
+// (No @packageDocumentation comment for this package)
+
+```
diff --git a/docs/public-api/crawlee-browser.api.md b/docs/public-api/crawlee-browser.api.md
new file mode 100644
index 000000000..1b0fa60ae
--- /dev/null
+++ b/docs/public-api/crawlee-browser.api.md
@@ -0,0 +1,14 @@
+## API Report File for "@crawlee/browser"
+
+> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
+
+```ts
+
+
+export * from "./internals/browser-crawler.js";
+export * from "./internals/browser-launcher.js";
+export * from "@crawlee/basic";
+
+// (No @packageDocumentation comment for this package)
+
+```
diff --git a/docs/public-api/crawlee-cheerio.api.md b/docs/public-api/crawlee-cheerio.api.md
new file mode 100644
index 000000000..5c52f2f8e
--- /dev/null
+++ b/docs/public-api/crawlee-cheerio.api.md
@@ -0,0 +1,81 @@
+## API Report File for "@crawlee/cheerio"
+
+> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
+
+```ts
+
+import type { BasicCrawlingContext } from '@crawlee/http';
+import type { BatchAddRequestsResult } from '@crawlee/types';
+import * as cheerio from 'cheerio';
+import type { CheerioAPI } from 'cheerio';
+import { CheerioRoot } from '@crawlee/utils';
+import { ContextPipeline } from '@crawlee/http';
+import { CrawlingContext } from '@crawlee/http';
+import type { Dictionary } from '@crawlee/types';
+import type { EnqueueLinksOptions } from '@crawlee/http';
+import type { ErrorHandler } from '@crawlee/http';
+import type { GetUserDataFromRequest } from '@crawlee/http';
+import { HttpCrawler } from '@crawlee/http';
+import type { HttpCrawlerOptions } from '@crawlee/http';
+import type { InternalHttpCrawlingContext } from '@crawlee/http';
+import type { InternalHttpHook } from '@crawlee/http';
+import { IRequestManager } from '@crawlee/http';
+import type { RequestHandler } from '@crawlee/http';
+import { RobotsTxtFile } from '@crawlee/utils';
+import { RouterHandler } from '@crawlee/http';
+import type { RouterRoutes } from '@crawlee/http';
+import type { SkippedRequestCallback } from '@crawlee/http';
+
+// @public
+export class CheerioCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends CheerioCrawlingContext = CheerioCrawlingContext & ContextExtension> extends HttpCrawler<CheerioCrawlingContext, ContextExtension, ExtendedContext> {
+ constructor(options?: CheerioCrawlerOptions<ContextExtension, ExtendedContext>);
+ // (undocumented)
+ protected buildContextPipeline(): ContextPipeline<CrawlingContext<Dictionary>, InternalHttpCrawlingContext<any, any> & {
+ readonly body: string;
+ readonly $: CheerioAPI;
+ } & {
+ enqueueLinks: (enqueueOptions?: EnqueueLinksOptions) => Promise<BatchAddRequestsResult>;
+ waitForSelector: (selector: string, _timeoutMs?: number) => Promise<void>;
+ parseWithCheerio: (selector?: string, timeoutMs?: number) => Promise<CheerioAPI>;
+ }>;
+}
+
+// @internal (undocumented)
+export function cheerioCrawlerEnqueueLinks(options: EnqueueLinksInternalOptions | BoundEnqueueLinksInternalOptions): Promise<unknown>;
+
+// @public (undocumented)
+export interface CheerioCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends CheerioCrawlingContext = CheerioCrawlingContext & ContextExtension, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
+JSONData extends Dictionary = any> extends HttpCrawlerOptions<CheerioCrawlingContext<UserData, JSONData>, ContextExtension, ExtendedContext> {
+}
+
+// @public (undocumented)
+export interface CheerioCrawlingContext<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
+JSONData extends Dictionary = any> extends InternalHttpCrawlingContext<UserData, JSONData> {
+ $: cheerio.CheerioAPI;
+ body: string;
+ enqueueLinks(options?: EnqueueLinksOptions): Promise<BatchAddRequestsResult>;
+ parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
+ waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
+}
+
+// @public (undocumented)
+export type CheerioErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
+JSONData extends Dictionary = any> = ErrorHandler<CheerioCrawlingContext<UserData, JSONData>>;
+
+// @public (undocumented)
+export type CheerioHook<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
+JSONData extends Dictionary = any> = InternalHttpHook<CheerioCrawlingContext<UserData, JSONData>>;
+
+// @public (undocumented)
+export type CheerioRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
+JSONData extends Dictionary = any> = RequestHandler<CheerioCrawlingContext<UserData, JSONData>>;
+
+// @public
+export function createCheerioRouter<Context extends CheerioCrawlingContext = CheerioCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): RouterHandler<Context>;
+
+
+export * from "@crawlee/http";
+
+// (No @packageDocumentation comment for this package)
+
+```
diff --git a/docs/public-api/crawlee-core.api.md b/docs/public-api/crawlee-core.api.md
new file mode 100644
index 000000000..228712277
--- /dev/null
+++ b/docs/public-api/crawlee-core.api.md
@@ -0,0 +1,1969 @@
+## API Report File for "@crawlee/core"
+
+> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
+
+```ts
+
+import { AllowedHttpMethods } from '@crawlee/types';
+import { AsyncEventEmitter } from '@vladfrangu/async_event_emitter';
+import { Awaitable } from '@crawlee/types';
+import type { BaseHttpClient } from '@crawlee/types';
+import type { BatchAddRequestsResult } from '@crawlee/types';
+import type { BetterIntervalID } from '@apify/utilities';
+import type { BinaryLike } from 'node:crypto';
+import { Constructor } from '@crawlee/types';
+import { Cookie } from '@crawlee/types';
+import { Cookie as Cookie_2 } from 'tough-cookie';
+import { CookieJar } from 'tough-cookie';
+import { CrawleeLogger } from '@crawlee/types';
+import type { CrawleeLoggerOptions } from '@crawlee/types';
+import type { DatasetClient } from '@crawlee/types';
+import type { DatasetInfo } from '@crawlee/types';
+import { Dictionary } from '@crawlee/types';
+import type { HttpRequestOptions } from '@crawlee/types';
+import type { ISession } from '@crawlee/types';
+import type { ISessionPool } from '@crawlee/types';
+import type { KeyValueStoreClient } from '@crawlee/types';
+import { Log } from '@apify/log';
+import log from '@apify/log';
+import { Logger } from '@apify/log';
+import { LoggerJson } from '@apify/log';
+import type { LoggerOptions } from '@apify/log';
+import { LoggerText } from '@apify/log';
+import { LogLevel } from '@apify/log';
+import { LruCache } from '@apify/datastructures';
+import { ParseSitemapOptions } from '@crawlee/utils';
+import type { ProcessedRequest } from '@crawlee/types';
+import type { ProxyInfo } from '@crawlee/types';
+import { PseudoUrl } from '@apify/pseudo_url';
+import { QueueOperationInfo } from '@crawlee/types';
+import { Readable } from 'node:stream';
+import type { ReadonlyDeep } from 'type-fest';
+import type { RequestQueueClient } from '@crawlee/types';
+import type { RequestQueueInfo } from '@crawlee/types';
+import { RobotsTxtFile } from '@crawlee/utils';
+import type { SendRequestOptions } from '@crawlee/types';
+import type { SessionFingerprint } from '@crawlee/types';
+import { SessionState } from '@crawlee/types';
+import type { SetRequired } from 'type-fest';
+import { StorageClient } from '@crawlee/types';
+import { StorageIdentifier } from '@crawlee/types';
+import { tryAbsoluteURL } from '@crawlee/utils';
+import { z } from 'zod';
+
+// @public (undocumented)
+export interface AddRequestsBatchedOptions extends RequestQueueOperationOptions {
+ batchSize?: number;
+ waitBetweenBatchesMillis?: number;
+ waitForAllRequestsToBeAdded?: boolean;
+}
+
+// @public (undocumented)
+export interface AddRequestsBatchedResult {
+ // (undocumented)
+ addedRequests: ProcessedRequest[];
+ waitForAllRequestsToBeAdded: Promise<ProcessedRequest[]>;
+}
+
+// @internal
+export const API_PROCESSED_REQUESTS_DELAY_MILLIS = 10000;
+
+// @public
+export class ApifyLogAdapter extends BaseCrawleeLogger {
+ constructor(apifyLog: Log, options?: Partial<CrawleeLoggerOptions>);
+ // (undocumented)
+ protected createChild(options: Partial<CrawleeLoggerOptions>): CrawleeLogger;
+ // (undocumented)
+ logWithLevel(level: number, message: string, data?: Record<string, unknown>): void;
+}
+
+// @internal
+export function applyRequestTransform(requestOptions: RequestOptions[], transformFn: RequestTransform, onSkipped?: (requestOptions: RequestOptions) => void): RequestOptions[];
+
+// @public
+export function assertJsonSerializable<T>(item: T, index?: number): void;
+
+// @public
+export class AutoscaledPool {
+ constructor(options: AutoscaledPoolOptions);
+ abort(): Promise<void>;
+ protected _autoscale(intervalCallback: () => void): void;
+ get currentConcurrency(): number;
+ get desiredConcurrency(): number;
+ set desiredConcurrency(value: number);
+ protected _destroy(): Promise<void>;
+ // (undocumented)
+ protected _incrementTasksDonePerSecond(intervalCallback: () => void): void;
+ // (undocumented)
+ protected get _isOverMaxRequestLimit(): boolean;
+ get maxConcurrency(): number;
+ set maxConcurrency(value: number);
+ protected _maybeFinish(): Promise<void>;
+ protected _maybeRunTask(intervalCallback?: () => void): Promise<void>;
+ get minConcurrency(): number;
+ set minConcurrency(value: number);
+ notify(): Promise<void>;
+ pause(timeoutSecs?: number): Promise<void>;
+ resume(): void;
+ run(): Promise<void>;
+ protected _scaleDown(systemStatus: SystemInfo): void;
+ protected _scaleUp(systemStatus: SystemInfo): void;
+}
+
+// @public (undocumented)
+export interface AutoscaledPoolOptions {
+ autoscaleIntervalSecs?: number;
+ desiredConcurrency?: number;
+ desiredConcurrencyRatio?: number;
+ isFinishedFunction?: () => Promise<boolean>;
+ isTaskReadyFunction?: () => Promise<boolean>;
+ // (undocumented)
+ log?: CrawleeLogger;
+ loggingIntervalSecs?: number | null;
+ maxConcurrency?: number;
+ maxTasksPerMinute?: number;
+ maybeRunIntervalSecs?: number;
+ minConcurrency?: number;
+ runTaskFunction?: () => Promise<unknown>;
+ scaleDownStepRatio?: number;
+ scaleUpStepRatio?: number;
+ snapshotterOptions?: SnapshotterOptions;
+ systemStatusOptions?: SystemStatusOptions;
+ taskTimeoutSecs?: number;
+}
+
+export { Awaitable }
+
+// @public
+export abstract class BaseCrawleeLogger implements CrawleeLogger {
+ constructor(options?: Partial<CrawleeLoggerOptions>);
+ // (undocumented)
+ child(options: Partial<CrawleeLoggerOptions>): CrawleeLogger;
+ protected abstract createChild(options: Partial<CrawleeLoggerOptions>): CrawleeLogger;
+ // (undocumented)
+ debug(message: string, data?: Record<string, unknown>): void;
+ // (undocumented)
+ deprecated(message: string): void;
+ // (undocumented)
+ error(message: string, data?: Record<string, unknown>): void;
+ // (undocumented)
+ exception(exception: Error, message: string, data?: Record<string, unknown>): void;
+ // (undocumented)
+ getOptions(): CrawleeLoggerOptions;
+ // (undocumented)
+ info(message: string, data?: Record<string, unknown>): void;
+ abstract logWithLevel(level: number, message: string, data?: Record<string, unknown>): void;
+ // (undocumented)
+ perf(message: string, data?: Record<string, unknown>): void;
+ // (undocumented)
+ setOptions(options: Partial<CrawleeLoggerOptions>): void;
+ // (undocumented)
+ softFail(message: string, data?: Record<string, unknown>): void;
+ // (undocumented)
+ warning(message: string, data?: Record<string, unknown>): void;
+ // (undocumented)
+ warningOnce(message: string): void;
+}
+
+// @internal
+export function bindMethodsToServiceLocator(serviceLocator: ServiceLocator, target: {}): {
+ run: <T>(fn: () => T) => T;
+ enterScope: () => void;
+ exitScope: () => void;
+};
+
+// @public (undocumented)
+export const BLOCKED_STATUS_CODES: number[];
+
+// @internal
+export function browserPoolCookieToToughCookie(cookieObject: Cookie, maxAgeSecs?: number): Cookie_2;
+
+// @public
+export const checkStorageAccess: () => void | undefined;
+
+// @public (undocumented)
+export interface ClientInfo {
+ // (undocumented)
+ actualRatio: number;
+ // (undocumented)
+ isOverloaded: boolean;
+ // (undocumented)
+ limitRatio: number;
+}
+
+// @public
+export const coerceBoolean: z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>;
+
+// @public (undocumented)
+export const coerceNumber: z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>;
+
+// @public (undocumented)
+export interface ConfigField<T extends z.ZodType = z.ZodType> {
+ // (undocumented)
+ envVar?: string | string[];
+ // (undocumented)
+ schema: T;
+}
+
+// @public (undocumented)
+export interface Configuration extends ResolvedConfigValues {
+}
+
+// @public
+export class Configuration {
+ constructor(options?: ConfigurationInput);
+ protected static fields: Record<string, ConfigField>;
+ static getGlobalConfig(): Configuration;
+}
+
+// @public (undocumented)
+export type ConfigurationInput = FieldsInput<typeof crawleeConfigFields>;
+
+// @public @deprecated (undocumented)
+export type ConfigurationOptions = ConfigurationInput;
+
+// @public
+export function constructGlobObjectsFromGlobs(globs: readonly GlobInput[]): GlobObject[];
+
+export { Constructor }
+
+// @public
+export function constructRegExpObjectsFromPseudoUrls(pseudoUrls: readonly PseudoUrlInput[]): RegExpObject[];
+
+// @public
+export function constructRegExpObjectsFromRegExps(regexps: readonly RegExpInput[]): RegExpObject[];
+
+// @public
+export interface ContextMiddleware<TCrawlingContext, TCrawlingContextExtension> {
+ action: (context: TCrawlingContext) => Awaitable<TCrawlingContextExtension>;
+ cleanup?: (context: TCrawlingContext & TCrawlingContextExtension, error?: unknown) => Awaitable<void>;
+}
+
+// @public
+export abstract class ContextPipeline<TContextBase, TCrawlingContext extends TContextBase> {
+ abstract call(crawlingContext: TContextBase, finalContextConsumer: (finalContext: TCrawlingContext) => Awaitable<unknown>): Promise<void>;
+ abstract chain<TFinalContext extends TCrawlingContext>(other: ContextPipeline<TCrawlingContext, TFinalContext>): ContextPipeline<TContextBase, TFinalContext>;
+ abstract compose<TCrawlingContextExtension>(middleware: ContextMiddleware<TCrawlingContext, TCrawlingContextExtension>): ContextPipeline<TContextBase, TCrawlingContext & TCrawlingContextExtension>;
+ static create<TContextBase>(): ContextPipeline<TContextBase, TContextBase>;
+}
+
+// @public (undocumented)
+export class ContextPipelineCleanupError extends CriticalError {
+ constructor(error: unknown, options?: ErrorOptions);
+}
+
+// @public (undocumented)
+export class ContextPipelineInitializationError extends Error {
+ constructor(error: unknown, options?: ErrorOptions);
+}
+
+// @public (undocumented)
+export class ContextPipelineInterruptedError extends Error {
+ constructor(message?: string);
+}
+
+export { Cookie }
+
+// @public (undocumented)
+export class CookieParseError extends Error {
+ constructor(cookieHeaderString: unknown);
+ // (undocumented)
+ readonly cookieHeaderString: unknown;
+}
+
+// @internal (undocumented)
+export function cookieStringToToughCookie(cookieString: string): Cookie | null;
+
+// @public (undocumented)
+export const crawleeConfigFields: {
+ defaultDatasetId: ConfigField<z.ZodDefault<z.ZodString>>;
+ purgeOnStart: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
+ defaultKeyValueStoreId: ConfigField<z.ZodDefault<z.ZodString>>;
+ defaultRequestQueueId: ConfigField<z.ZodDefault<z.ZodString>>;
+ maxUsedCpuRatio: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
+ availableMemoryRatio: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
+ memoryMbytes: ConfigField<z.ZodOptional<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
+ persistStateIntervalMillis: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
+ systemInfoIntervalMillis: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
+ inputKey: ConfigField<z.ZodDefault<z.ZodString>>;
+ headless: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
+ xvfb: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
+ chromeExecutablePath: ConfigField<z.ZodOptional<z.ZodString>>;
+ defaultBrowserPath: ConfigField<z.ZodOptional<z.ZodString>>;
+ disableBrowserSandbox: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
+ logLevel: ConfigField<z.ZodOptional<z.ZodPipe<z.ZodTransform<{} | null | undefined, unknown>, z.ZodEnum<typeof LogLevel>>>>;
+ persistStorage: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
+ containerized: ConfigField<z.ZodOptional<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
+};
+
+export { CrawleeLogger }
+
+export { CrawleeLoggerOptions }
+
+// @public (undocumented)
+export interface CrawlingContext<UserData extends Dictionary = Dictionary> extends RestrictedCrawlingContext<UserData> {
+ enqueueLinks(options: ReadonlyDeep<Omit<SetRequired<EnqueueLinksOptions, 'urls'>, 'requestManager' | 'robotsTxtFile'>> & Pick<EnqueueLinksOptions, 'requestManager' | 'robotsTxtFile'>): Promise<unknown>;
+ registerDeferredCleanup(cleanup: () => Promise<unknown>): void;
+ sendRequest: (requestOverrides?: Partial<HttpRequestOptions>, optionsOverrides?: SendRequestOptions) => Promise<Response>;
+}
+
+// @internal
+export function createDeserialize(compressedData: Buffer | Uint8Array): Readable;
+
+// @internal
+export function createDualIterable<TItem, TRawPage>(options: DualIterableOptions<TItem, TRawPage>): AsyncIterable<TItem> & Promise<TItem[]>;
+
+// @public (undocumented)
+export function createRequestOptions(sources: readonly (string | Record<string, unknown>)[], options?: Pick<EnqueueLinksOptions, 'label' | 'userData' | 'baseUrl' | 'skipNavigation' | 'sessionId' | 'strategy'>): RequestOptions[];
+
+// @public
+export interface CreateSession {
+ // (undocumented)
+ (options?: {
+ sessionOptions?: SessionOptions;
+ }): Session | Promise<Session>;
+}
+
+// @public
+export class CriticalError extends NonRetryableError {
+}
+
+// @public
+export class Dataset<Data extends Dictionary = Dictionary> {
+ [Symbol.asyncIterator](): AsyncGenerator<Data, void, undefined>;
+ // @internal
+ constructor(options: DatasetOptions, config?: Configuration);
+ // (undocumented)
+ client: DatasetClient<Data>;
+ // (undocumented)
+ readonly config: Configuration;
+ drop(): Promise<void>;
+ entries(options?: DatasetIteratorOptions): AsyncIterable<[number, Data]> & Promise<[number, Data][]>;
+ export(options?: DatasetExportOptions): Promise<Data[]>;
+ exportTo(key: string, options?: DatasetExportToOptions, contentType?: string): Promise<Data[]>;
+ exportToCSV(key: string, options?: Omit<DatasetExportToOptions, 'fromDataset'>): Promise<void>;
+ static exportToCSV(key: string, options?: DatasetExportToOptions): Promise<void>;
+ exportToJSON(key: string, options?: Omit<DatasetExportToOptions, 'fromDataset'>): Promise<void>;
+ static exportToJSON(key: string, options?: DatasetExportToOptions): Promise<void>;
+ forEach(iteratee: DatasetConsumer<Data>, options?: DatasetIteratorOptions, index?: number): Promise<void>;
+ getData(options?: DatasetDataOptions): Promise<DatasetContent<Data>>;
+ static getData<Data extends Dictionary = Dictionary>(options?: DatasetDataOptions): Promise<DatasetContent<Data>>;
+ getInfo(): Promise<DatasetInfo>;
+ // (undocumented)
+ id: string;
+ // (undocumented)
+ log: CrawleeLogger;
+ map<R>(iteratee: DatasetMapper<Data, R>, options?: DatasetIteratorOptions): Promise<R[]>;
+ // (undocumented)
+ name?: string;
+ static open<Data extends Dictionary = Dictionary>(identifier?: string | StorageIdentifier | null, options?: StorageOpenOptions): Promise<Dataset<Data>>;
+ pushData(data: Data | Data[]): Promise<void>;
+ static pushData<Data extends Dictionary = Dictionary>(item: Data | Data[]): Promise<void>;
+ reduce(iteratee: DatasetReducer<Data, Data>): Promise<Data | undefined>;
+ reduce(iteratee: DatasetReducer<Data, Data>, memo: undefined, options: DatasetIteratorOptions): Promise<Data | undefined>;
+ reduce<T>(iteratee: DatasetReducer<T, Data>, memo: T, options?: DatasetIteratorOptions): Promise<T>;
+ values(options?: DatasetIteratorOptions): AsyncIterable<Data> & Promise<Data[]>;
+}
+
+// @internal (undocumented)
+export const DATASET_ITERATORS_DEFAULT_LIMIT = 10000;
+
+// @public
+export interface DatasetConsumer<Data> {
+ // (undocumented)
+ (item: Data, index: number): Awaitable_2<void>;
+}
+
+// @public (undocumented)
+export interface DatasetContent<Data> {
+ count: number;
+ desc?: boolean;
+ items: Data[];
+ limit: number;
+ offset: number;
+ total: number;
+}
+
+// @public (undocumented)
+export interface DatasetDataOptions {
+ clean?: boolean;
+ desc?: boolean;
+ fields?: string[];
+ limit?: number;
+ offset?: number;
+ skipEmpty?: boolean;
+ skipHidden?: boolean;
+ unwind?: string;
+}
+
+// @public (undocumented)
+export interface DatasetExportOptions extends Omit<DatasetDataOptions, 'offset' | 'limit'> {
+ collectAllKeys?: boolean;
+}
+
+// @public (undocumented)
+export interface DatasetExportToOptions extends DatasetExportOptions {
+ // (undocumented)
+ fromDataset?: string | StorageIdentifier;
+ // (undocumented)
+ toKVS?: string | StorageIdentifier;
+}
+
+// @public (undocumented)
+export interface DatasetIteratorOptions extends Omit<DatasetDataOptions, 'offset' | 'limit' | 'clean' | 'skipHidden' | 'skipEmpty'> {
+ // @internal (undocumented)
+ clean?: boolean;
+ // @internal (undocumented)
+ format?: string;
+ // @internal
+ limit?: number;
+ // @internal (undocumented)
+ offset?: number;
+ // @internal (undocumented)
+ skipEmpty?: boolean;
+ // @internal (undocumented)
+ skipHidden?: boolean;
+}
+
+// @public
+export interface DatasetMapper<Data, R> {
+ (item: Data, index: number): Awaitable_2<R>;
+}
+
+// @public (undocumented)
+export interface DatasetOptions {
+ // (undocumented)
+ client: DatasetClient;
+ // (undocumented)
+ id: string;
+ // (undocumented)
+ name?: string;
+}
+
+// @public
+export interface DatasetReducer<T, Data> {
+ // (undocumented)
+ (memo: T, item: Data, index: number): Awaitable_2<T>;
+}
+
+// @public
+export interface DefaultStorageIdentifier {
+ // (undocumented)
+ alias?: never;
+ // (undocumented)
+ id?: never;
+ // (undocumented)
+ name?: never;
+}
+
+// @internal
+export function deserializeArray<T extends string | Buffer>(compressedData: Buffer | Uint8Array): Promise<T[]>;
+
+export { Dictionary }
+
+// @internal (undocumented)
+export interface DualIterableOptions<TItem, TRawPage> {
+ createPages: () => AsyncGenerator<TRawPage>;
+ extractItems: (page: TRawPage) => TItem[];
+}
+
+// @public
+export function enqueueLinks(options: SetRequired<Omit<EnqueueLinksOptions, 'requestManager'>, 'urls'> & {
+ requestManager: {
+ addRequestsBatched: (requests: Request_2<Dictionary>[], options: AddRequestsBatchedOptions) => Promise<AddRequestsBatchedResult>;
+ };
+}): Promise<BatchAddRequestsResult>;
+
+// @public (undocumented)
+export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
+ baseUrl?: string;
+ exclude?: readonly (GlobInput | RegExpInput)[];
+ globs?: readonly GlobInput[];
+ label?: string;
+ limit?: number;
+ onSkippedRequest?: SkippedRequestCallback;
+ // @deprecated
+ pseudoUrls?: readonly PseudoUrlInput[];
+ regexps?: readonly RegExpInput[];
+ requestManager?: IRequestManager;
+ robotsTxtFile?: Pick<RobotsTxtFile, 'isAllowed'>;
+ selector?: string;
+ sessionId?: string;
+ skipNavigation?: boolean;
+ strategy?: EnqueueStrategy | 'all' | 'same-domain' | 'same-hostname' | 'same-origin';
+ transformRequestFunction?: RequestTransform;
+ urls?: readonly string[];
+ userData?: Dictionary;
+ waitForAllRequestsToBeAdded?: boolean;
+}
+
+// @public
+export enum EnqueueStrategy {
+ All = "all",
+ SameDomain = "same-domain",
+ SameHostname = "same-hostname",
+ SameOrigin = "same-origin"
+}
+
+// @public
+export interface ErrnoException extends Error {
+ // (undocumented)
+ cause?: any;
+ // (undocumented)
+ code?: string | number;
+ // (undocumented)
+ errno?: number;
+ // (undocumented)
+ path?: string;
+ // (undocumented)
+ syscall?: string;
+}
+
+// @public
+export class ErrorSnapshotter {
+ // (undocumented)
+ static readonly BASE_MESSAGE = "An error occurred";
+ captureSnapshot(error: ErrnoException, context: CrawlingContext & SnapshottableProperties): Promise<ErrorSnapshot>;
+ contextCaptureSnapshot(context: BrowserCrawlingContext, fileName: string): Promise<SnapshotResult | undefined>;
+ generateFilename(error: ErrnoException): string;
+ // (undocumented)
+ static readonly MAX_ERROR_CHARACTERS = 30;
+ // (undocumented)
+ static readonly MAX_FILENAME_LENGTH = 250;
+ // (undocumented)
+ static readonly MAX_HASH_LENGTH = 30;
+ saveHTMLSnapshot(html: string, keyValueStore: Pick<KeyValueStore, 'setValue'>, fileName: string): Promise<string | undefined>;
+ // (undocumented)
+ static readonly SNAPSHOT_PREFIX = "ERROR_SNAPSHOT";
+}
+
+// @public
+export class ErrorTracker {
+ constructor(options?: Partial<ErrorTrackerOptions>);
+ // (undocumented)
+ add(error: ErrnoException): void;
+ addAsync(error: ErrnoException, context?: CrawlingContext): Promise<void>;
+ // (undocumented)
+ captureSnapshot(storage: Record<string, unknown>, error: ErrnoException, context: CrawlingContext & SnapshottableProperties): Promise<void>;
+ // (undocumented)
+ errorSnapshotter?: ErrorSnapshotter;
+ // (undocumented)
+ getMostPopularErrors(count: number): [number, string[]][];
+ // (undocumented)
+ getUniqueErrorCount(): number;
+ // (undocumented)
+ reset(): void;
+ // (undocumented)
+ result: Record<string, unknown>;
+ // (undocumented)
+ total: number;
+}
+
+// @public (undocumented)
+export interface ErrorTrackerOptions {
+ // (undocumented)
+ saveErrorSnapshots: boolean;
+ // (undocumented)
+ showErrorCode: boolean;
+ // (undocumented)
+ showErrorMessage: boolean;
+ // (undocumented)
+ showErrorName: boolean;
+ // (undocumented)
+ showFullMessage: boolean;
+ // (undocumented)
+ showFullStack: boolean;
+ // (undocumented)
+ showStackTrace: boolean;
+}
+
+// @public (undocumented)
+export abstract class EventManager {
+ constructor(options: EventManagerOptions);
+ close(): Promise<void>;
+ // (undocumented)
+ emit(event: EventTypeName, ...args: unknown[]): void;
+ // (undocumented)
+ protected events: AsyncEventEmitter<{}>;
+ init(): Promise<void>;
+ // (undocumented)
+ protected initialized: boolean;
+ // (undocumented)
+ protected intervals: Intervals;
+ // (undocumented)
+ isInitialized(): boolean;
+ // @internal (undocumented)
+ listenerCount(event: EventTypeName): number;
+ // @internal (undocumented)
+ listeners(event: EventTypeName): (() => Promise<unknown>)[];
+ // (undocumented)
+ protected log: CrawleeLogger;
+ // (undocumented)
+ off(event: EventTypeName, listener?: (...args: any[]) => any): void;
+ // (undocumented)
+ on(event: EventTypeName, listener: (...args: any[]) => any): void;
+ // @internal (undocumented)
+ waitForAllListenersToComplete(): Promise<boolean>;
+}
+
+// @public (undocumented)
+export interface EventManagerOptions {
+ persistStateIntervalMillis: number;
+}
+
+// @public (undocumented)
+export const enum EventType {
+ // (undocumented)
+ ABORTING = "aborting",
+ // (undocumented)
+ EXIT = "exit",
+ // (undocumented)
+ MIGRATING = "migrating",
+ // (undocumented)
+ PERSIST_STATE = "persistState",
+ // (undocumented)
+ SYSTEM_INFO = "systemInfo"
+}
+
+// @public (undocumented)
+export type EventTypeName = EventType | 'systemInfo' | 'persistState' | 'migrating' | 'aborting' | 'exit';
+
+// @public
+export type ExplicitStorageIdentifier = {
+ id: string;
+ name?: never;
+ alias?: never;
+} | {
+ id?: never;
+ name: string;
+ alias?: never;
+} | {
+ id?: never;
+ name?: never;
+ alias: string;
+};
+
+// @public (undocumented)
+export function field<T extends z.ZodType>(schema: T, envVar?: string | string[]): ConfigField<T>;
+
+// @public (undocumented)
+export type FieldsInput<F extends Record<string, ConfigField>> = {
+ [K in keyof F]?: z.output<F[K]['schema']>;
+};
+
+// @public (undocumented)
+export type FieldsOutput<F extends Record<string, ConfigField>> = {
+ [K in keyof F]: z.output<F[K]['schema']>;
+};
+
+// @public
+export function filterRequestOptionsByPatterns(requestOptions: RequestOptions[], includePatterns: UrlPatternObject[] | undefined, excludePatterns?: UrlPatternObject[], strategy?: EnqueueLinksOptions['strategy'], onSkippedUrl?: (url: string) => void): RequestOptions[];
+
+// @public (undocumented)
+export interface FinalStatistics {
+ // (undocumented)
+ crawlerRuntimeMillis: number;
+ // (undocumented)
+ requestAvgFailedDurationMillis: number;
+ // (undocumented)
+ requestAvgFinishedDurationMillis: number;
+ // (undocumented)
+ requestsFailed: number;
+ // (undocumented)
+ requestsFailedPerMinute: number;
+ // (undocumented)
+ requestsFinished: number;
+ // (undocumented)
+ requestsFinishedPerMinute: number;
+ // (undocumented)
+ requestsTotal: number;
+ // (undocumented)
+ requestTotalDurationMillis: number;
+ // (undocumented)
+ retryHistogram: number[];
+}
+
+// @internal (undocumented)
+export function getCookiesFromResponse(response: Response): Cookie_2[];
+
+// @internal
+export function getDefaultCookieExpirationDate(maxAgeSecs: number): Date;
+
+// @internal
+export function getRequestId(uniqueKey: string): string;
+
+// @public (undocumented)
+export type GetUserDataFromRequest<T> = T extends Request_2<infer Y> ? Y : never;
+
+// @public (undocumented)
+export type GlobInput = string | GlobObject;
+
+// @public (undocumented)
+export type GlobObject = {
+ glob: string;
+} & Pick<RequestOptions, 'method' | 'payload' | 'label' | 'userData' | 'headers'>;
+
+// @internal
+export function handleRequestTimeout(input: {
+ session?: ISession;
+ errorMessage: string;
+}): void;
+
+// @internal (undocumented)
+export interface InternalSource {
+ // (undocumented)
+ regex?: RegExp;
+ // (undocumented)
+ requestsFromUrl: string;
+}
+
+// @public
+export interface IRequestLoader {
+ [Symbol.asyncIterator](): AsyncGenerator<Request_2>;
+ fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request_2<T> | null>;
+ getHandledCount(): Promise<number>;
+ getPendingCount(): Promise<number>;
+ getTotalCount(): Promise<number>;
+ isEmpty(): Promise<boolean>;
+ isFinished(): Promise<boolean>;
+ markRequestAsHandled(request: Request_2): Promise<RequestQueueOperationInfo | void | null>;
+ persistState?(): Promise<void>;
+ toTandem?(requestManager?: IRequestManager): Promise<IRequestManager>;
+}
+
+// @public
+export interface IRequestManager extends IRequestLoader {
+ // (undocumented)
+ addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
+ // (undocumented)
+ addRequestsBatched(requests: RequestsLike, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
+ purge?(): Promise<void>;
+ reclaimRequest(request: Request_2, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;
+ setExpectedRequestProcessingTimeSecs?(secs: number): void;
+}
+
+// @internal (undocumented)
+export type IsAny<T> = 0 extends 1 & T ? true : false;
+
+// @public
+export interface IStorage {
+ // (undocumented)
+ id: string;
+ // (undocumented)
+ name?: string;
+}
+
+// @public
+export interface KeyConsumer {
+ // (undocumented)
+ (key: string, index: number, info: {
+ size: number;
+ }): Awaitable_2<void>;
+}
+
+// @public
+export class KeyValueStore {
+ [Symbol.asyncIterator]<T = unknown>(): AsyncGenerator<[string, T], void, undefined>;
+ // @internal
+ constructor(options: KeyValueStoreOptions, config?: Configuration);
+ // @internal (undocumented)
+ clearCache(): void;
+ // (undocumented)
+ readonly config: Configuration;
+ drop(): Promise<void>;
+ entries<T = unknown>(options?: KeyValueStoreIteratorOptions): AsyncIterable<[string, T]> & Promise<[string, T][]>;
+ forEachKey(iteratee: KeyConsumer, options?: KeyValueStoreIteratorOptions): Promise<void>;
+ // (undocumented)
+ getAutoSavedValue<T extends Dictionary = Dictionary>(key: string, defaultValue?: T): Promise<T>;
+ // (undocumented)
+ static getAutoSavedValue<T extends Dictionary = Dictionary>(key: string, defaultValue?: T): Promise<T>;
+ static getInput<T = Dictionary | string | Buffer>(): Promise<T | null>;
+ getPublicUrl(key: string): Promise<string | undefined>;
+ getRecord(key: string): Promise<KeyValueStoreRawRecord | null>;
+ static getRecord(key: string): Promise<KeyValueStoreRawRecord | null>;
+ getValue<T = unknown>(key: string): Promise<T | null>;
+ getValue<T = unknown>(key: string, defaultValue: T): Promise<T>;
+ static getValue<T = unknown>(key: string): Promise<T | null>;
+ static getValue<T = unknown>(key: string, defaultValue: T): Promise<T>;
+ // (undocumented)
+ readonly id: string;
+ keys(options?: KeyValueStoreIteratorOptions): AsyncIterable<string> & Promise<string[]>;
+ // (undocumented)
+ readonly name?: string;
+ static open(identifier?: string | StorageIdentifier | null, options?: StorageOpenOptions): Promise<KeyValueStore>;
+ recordExists(key: string): Promise<boolean>;
+ static recordExists(key: string): Promise<boolean>;
+ setValue<T>(key: string, value: T | null, options?: RecordOptions): Promise<void>;
+ static setValue<T>(key: string, value: T | null, options?: RecordOptions): Promise<void>;
+ values<T = unknown>(options?: KeyValueStoreIteratorOptions): AsyncIterable<T> & Promise<T[]>;
+}
+
+// @public (undocumented)
+export interface KeyValueStoreIteratorOptions {
+ prefix?: string;
+}
+
+// @public (undocumented)
+export interface KeyValueStoreOptions {
+ // (undocumented)
+ client: KeyValueStoreClient;
+ // (undocumented)
+ id: string;
+ // (undocumented)
+ name?: string;
+}
+
+// @public
+export interface KeyValueStoreRawRecord {
+ // (undocumented)
+ contentType: string | null;
+ // (undocumented)
+ value: Buffer | ArrayBuffer;
+}
+
+// @internal (undocumented)
+export type LoadedContext<Context extends RestrictedCrawlingContext> = IsAny<Context> extends true ? Context : {
+ request: LoadedRequest<Context['request']>;
+} & Omit<Context, 'request'>;
+
+// @public (undocumented)
+export type LoadedRequest<R extends Request_2> = WithRequired<R, 'id' | 'loadedUrl'>;
+
+// @public (undocumented)
+export class LocalEventManager extends EventManager {
+ constructor(options: LocalEventManagerOptions);
+ // (undocumented)
+ close(): Promise<void>;
+ // @internal (undocumented)
+ emitSystemInfoEvent(intervalCallback: () => unknown): Promise<void>;
+ static fromConfig(config?: Configuration): LocalEventManager;
+ init(): Promise<void>;
+ // @internal (undocumented)
+ isContainerizedWrapper(): Promise<boolean>;
+}
+
+// @public (undocumented)
+export interface LocalEventManagerOptions extends EventManagerOptions {
+ systemInfoIntervalMillis: number;
+}
+
+export { Log }
+
+export { log }
+
+export { Logger }
+
+export { LoggerJson }
+
+export { LoggerOptions }
+
+export { LoggerText }
+
+export { LogLevel }
+
+// @public (undocumented)
+export const MAX_POOL_SIZE = 1000;
+
+// @internal
+export const MAX_QUERIES_FOR_CONSISTENCY = 6;
+
+// @internal
+export function mergeCookies(url: string, sourceCookies: string[]): string;
+
+// @public (undocumented)
+export class MissingRouteError extends CriticalError {
+}
+
+// @public
+export class MissingSessionError extends Error {
+ constructor(sessionId?: string);
+}
+
+// @public
+export class NavigationSkippedError extends NonRetryableError {
+}
+
+// @public
+export class NonRetryableError extends Error {
+}
+
+// @public
+export function parseValue(body: Buffer | ArrayBuffer | string, contentTypeHeader: string | null): string | Buffer | ArrayBuffer | Record<string, unknown>;
+
+// @public (undocumented)
+export const PERSIST_STATE_KEY = "CRAWLEE_SESSION_POOL_STATE";
+
+// @public
+export interface PersistenceOptions {
+ enable?: boolean;
+}
+
+// @public
+export class ProxyConfiguration {
+ constructor(options?: ProxyConfigurationOptions);
+ protected _callNewUrlFunction(options?: {
+ request?: Request_2;
+ }): Promise<string | null>;
+ // (undocumented)
+ protected _handleProxyUrlsList(): string | null;
+ // (undocumented)
+ isManInTheMiddle: boolean;
+ // (undocumented)
+ protected log: CrawleeLogger;
+ newProxyInfo(options?: NewUrlOptions): Promise<ProxyInfo | undefined>;
+ newUrl(options?: NewUrlOptions): Promise<string | undefined>;
+ // (undocumented)
+ protected newUrlFunction?: ProxyConfigurationFunction;
+ // (undocumented)
+ protected nextCustomUrlIndex: number;
+ // (undocumented)
+ protected proxyUrls?: UrlList;
+ // (undocumented)
+ protected _throwCannotCombineCustomMethods(): never;
+ // (undocumented)
+ protected _throwNoOptionsProvided(): never;
+ // (undocumented)
+ protected usedProxyUrls: Map<string, string | null>;
+}
+
+// @public (undocumented)
+export interface ProxyConfigurationFunction {
+ // (undocumented)
+ (options?: {
+ request?: Request_2;
+ }): string | null | Promise<string | null>;
+}
+
+// @public (undocumented)
+export interface ProxyConfigurationOptions {
+ newUrlFunction?: ProxyConfigurationFunction;
+ proxyUrls?: UrlList;
+}
+
+export { PseudoUrl }
+
+// @public (undocumented)
+export type PseudoUrlInput = string | PseudoUrlObject;
+
+// @public (undocumented)
+export type PseudoUrlObject = {
+ purl: string;
+} & Pick<RequestOptions, 'method' | 'payload' | 'label' | 'userData' | 'headers'>;
+
+// @public
+export function purgeDefaultStorages(options?: PurgeDefaultStorageOptions): Promise<void>;
+
+// @public
+export function purgeDefaultStorages(config?: Configuration, client?: StorageClient): Promise<void>;
+
+// @public (undocumented)
+export interface PushErrorMessageOptions {
+ omitStack?: boolean;
+}
+
+// @internal (undocumented)
+export const QUERY_HEAD_BUFFER = 3;
+
+// @internal
+export const QUERY_HEAD_MIN_LENGTH = 100;
+
+export { QueueOperationInfo }
+
+// @public (undocumented)
+export interface RecordOptions {
+ contentType?: string;
+}
+
+// @public
+export class RecoverableState<TStateModel = Record<string, unknown>> {
+ constructor(options: RecoverableStateOptions<TStateModel>);
+ get currentValue(): TStateModel;
+ initialize(): Promise<TStateModel>;
+ persistState(eventData?: {
+ isMigrating: boolean;
+ }): Promise<void>;
+ reset(): Promise<void>;
+ teardown(): Promise<void>;
+}
+
+// @public
+export interface RecoverableStateOptions<TStateModel = Record<string, unknown>> extends RecoverableStatePersistenceOptions {
+ config?: Configuration;
+ defaultState: TStateModel;
+ deserialize?: (serializedState: string) => TStateModel;
+ logger?: CrawleeLogger;
+ serialize?: (state: TStateModel) => string;
+}
+
+// @public (undocumented)
+export interface RecoverableStatePersistenceOptions {
+ persistenceEnabled?: boolean;
+ persistStateKey: string;
+ persistStateKvsId?: string;
+ persistStateKvsName?: string;
+}
+
+// @public (undocumented)
+export type RegExpInput = RegExp | RegExpObject;
+
+// @public (undocumented)
+export type RegExpObject = {
+ regexp: RegExp;
+} & Pick<RequestOptions, 'method' | 'payload' | 'label' | 'userData' | 'headers'>;
+
+// @public
+class Request_2<UserData extends Dictionary = Dictionary> {
+ constructor(options: RequestOptions<UserData>);
+ // @internal (undocumented)
+ static computeUniqueKey(input: ComputeUniqueKeyOptions): string;
+ get crawlDepth(): number;
+ set crawlDepth(value: number);
+ errorMessages: string[];
+ handledAt?: string;
+ // @internal (undocumented)
+ static hashPayload(payload: BinaryLike): string;
+ headers?: Record<string, string>;
+ id?: string;
+ intoFetchAPIRequest(): Request;
+ get label(): string | undefined;
+ set label(value: string | undefined);
+ loadedUrl?: string;
+ get maxRetries(): number | undefined;
+ set maxRetries(value: number | undefined);
+ method: AllowedHttpMethods;
+ noRetry: boolean;
+ payload?: string;
+ pushErrorMessage(errorOrMessage: unknown, options?: PushErrorMessageOptions): void;
+ retryCount: number;
+ get sessionId(): string | undefined;
+ set sessionId(value: string | undefined);
+ get skipNavigation(): boolean;
+ set skipNavigation(value: boolean);
+ get skippedReason(): SkippedRequestReason | undefined;
+ set skippedReason(value: SkippedRequestReason | undefined);
+ get state(): RequestState;
+ set state(value: RequestState);
+ uniqueKey: string;
+ url: string;
+ userData: UserData;
+}
+export { Request_2 as Request }
+
+// @public (undocumented)
+export class RequestHandlerError extends Error {
+ constructor(error: unknown, options?: ErrorOptions);
+}
+
+// @public
+export class RequestHandlerResult {
+ constructor(config: Configuration, crawleeStateKey: string);
+ // (undocumented)
+ addRequests: RestrictedCrawlingContext['addRequests'];
+ get calls(): ReadonlyDeep<{
+ pushData: Parameters<RestrictedCrawlingContext['pushData']>[];
+ addRequests: Parameters<RestrictedCrawlingContext['addRequests']>[];
+ }>;
+ get datasetItems(): ReadonlyDeep<{
+ item: Dictionary;
+ datasetIdentifier?: string | StorageIdentifier;
+ }[]>;
+ get enqueuedUrlLists(): ReadonlyDeep<{
+ listUrl: string;
+ label?: string;
+ }[]>;
+ get enqueuedUrls(): ReadonlyDeep<{
+ url: string;
+ label?: string;
+ }[]>;
+ // (undocumented)
+ getKeyValueStore: RestrictedCrawlingContext['getKeyValueStore'];
+ get keyValueStoreChanges(): ReadonlyDeep<Record<string, Record<string, {
+ changedValue: unknown;
+ options?: RecordOptions;
+ }>>>;
+ // (undocumented)
+ pushData: RestrictedCrawlingContext['pushData'];
+ // (undocumented)
+ useState: RestrictedCrawlingContext['useState'];
+}
+
+// @public
+export class RequestList implements IRequestLoader {
+ // (undocumented)
+ [Symbol.asyncIterator](): AsyncGenerator<Request_2<Dictionary>, void, unknown>;
+ protected _addFetchedRequests(source: InternalSource, fetchedRequests: RequestOptions[]): Promise<void>;
+ protected _addPersistedRequests(persistedRequests: Buffer): Promise<void>;
+ protected _addRequest(source: RequestListSource): void;
+ protected _addRequestsFromSources(): Promise<void>;
+ // @internal
+ areRequestsPersisted: boolean;
+ protected _ensureInProgress(uniqueKey: string): void;
+ protected _ensureIsInitialized(): void;
+ protected _ensureUniqueKeyValid(uniqueKey: string): void;
+ // (undocumented)
+ fetchNextRequest(): Promise<Request_2 | null>;
+ protected _fetchRequestsFromUrl(source: InternalSource): Promise<RequestOptions[]>;
+ // (undocumented)
+ getHandledCount(): Promise<number>;
+ getPendingCount(): Promise<number>;
+ // (undocumented)
+ protected _getPersistedState<T>(key: string): Promise<T>;
+ getState(): RequestListState;
+ getTotalCount(): Promise<number>;
+ // @internal
+ inProgress: Set<string>;
+ // (undocumented)
+ isEmpty(): Promise<boolean>;
+ // (undocumented)
+ isFinished(): Promise<boolean>;
+ // @internal
+ isStatePersisted: boolean;
+ protected _loadStateAndPersistedRequests(): Promise<[RequestListState, Buffer]>;
+ // (undocumented)
+ markRequestAsHandled(request: Request_2): Promise<void>;
+ static open(listNameOrOptions: string | null | RequestListOptions, sources?: RequestListSource[], options?: RequestListOptions): Promise<RequestList>;
+ protected _persistRequests(): Promise<void>;
+ // (undocumented)
+ persistState(): Promise<void>;
+ // @internal
+ requests: (Request_2 | RequestOptions)[];
+ protected _restoreState(state?: RequestListState): void;
+ toTandem(requestManager?: IRequestManager): Promise<IRequestManager>;
+}
+
+// @public (undocumented)
+export interface RequestListOptions {
+ // @internal (undocumented)
+ config?: Configuration;
+ httpClient?: BaseHttpClient;
+ keepDuplicateUrls?: boolean;
+ persistRequestsKey?: string;
+ persistStateKey?: string;
+ proxyConfiguration?: ProxyConfiguration;
+ sources?: RequestListSource[];
+ sourcesFunction?: RequestListSourcesFunction;
+ state?: RequestListState;
+}
+
+// @public (undocumented)
+export type RequestListSourcesFunction = () => Promise<RequestListSource[]>;
+
+// @public
+export interface RequestListState {
+ inProgress: string[];
+ nextIndex: number;
+ nextUniqueKey: string | null;
+}
+
+// @public
+export class RequestManagerTandem implements IRequestManager {
+ // (undocumented)
+ [Symbol.asyncIterator](): AsyncGenerator<Request_2<Dictionary>, void, unknown>;
+ constructor(requestLoader: IRequestLoader, requestManager: IRequestManager | (() => IRequestManager | Promise<IRequestManager>));
+ // (undocumented)
+ addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
+ // (undocumented)
+ addRequestsBatched(requests: RequestsLike, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
+ fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request_2<T> | null>;
+ // (undocumented)
+ getHandledCount(): Promise<number>;
+ // (undocumented)
+ getPendingCount(): Promise<number>;
+ // (undocumented)
+ getTotalCount(): Promise<number>;
+ // (undocumented)
+ isEmpty(): Promise<boolean>;
+ // (undocumented)
+ isFinished(): Promise<boolean>;
+ // (undocumented)
+ markRequestAsHandled(request: Request_2): Promise<RequestQueueOperationInfo | void | null>;
+ persistState(): Promise<void>;
+ purge(): Promise<void>;
+ // (undocumented)
+ reclaimRequest(request: Request_2, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;
+ setExpectedRequestProcessingTimeSecs(secs: number): void;
+}
+
+// @public
+export interface RequestOptions<UserData extends Dictionary = Dictionary> {
+ crawlDepth?: number;
+ // @internal (undocumented)
+ enqueueStrategy?: EnqueueLinksOptions['strategy'];
+ // @internal (undocumented)
+ handledAt?: string;
+ headers?: Record<string, string>;
+ // @internal (undocumented)
+ id?: string;
+ keepUrlFragment?: boolean;
+ label?: string;
+ // @internal (undocumented)
+ lockExpiresAt?: Date;
+ maxRetries?: number;
+ method?: AllowedHttpMethods | Lowercase<AllowedHttpMethods>;
+ noRetry?: boolean;
+ payload?: string;
+ sessionId?: string;
+ skipNavigation?: boolean;
+ // @internal
+ skippedReason?: SkippedRequestReason;
+ uniqueKey?: string;
+ url: string;
+ useExtendedUniqueKey?: boolean;
+ userData?: UserData;
+}
+
+// @public
+export class RequestQueue implements IStorage, IRequestManager {
+ // (undocumented)
+ [Symbol.asyncIterator](): AsyncGenerator<Request_2<Dictionary>, void, unknown>;
+ // @internal
+ constructor(options: RequestQueueOptions, config?: Configuration);
+ protected _addFetchedRequests(source: InternalSource, fetchedRequests: RequestOptions[], options: RequestQueueOperationOptions): Promise<ProcessedRequest[]>;
+ addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
+ addRequests(requestsLike: RequestsLike, options?: RequestQueueOperationOptions): Promise<BatchAddRequestsResult>;
+ addRequestsBatched(requests: ReadonlyDeep<RequestsLike>, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
+ protected _cacheRequest(cacheKey: string, queueOperationInfo: RequestQueueOperationInfo): void;
+ // (undocumented)
+ client: RequestQueueClient;
+ // (undocumented)
+ clientKey: string;
+ // (undocumented)
+ protected readonly config: Configuration;
+ drop(): Promise<void>;
+ // (undocumented)
+ protected readonly events: EventManager;
+ protected expectedRequestProcessingSecs: number;
+ fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request_2<T> | null>;
+ protected _fetchRequestsFromUrl(source: InternalSource): Promise<RequestOptions[]>;
+ getHandledCount(): Promise<number>;
+ getInfo(): Promise<RequestQueueInfo>;
+ getPendingCount(): Promise<number>;
+ getRequest<T extends Dictionary = Dictionary>(uniqueKey: string): Promise<Request_2<T> | null>;
+ getTotalCount(): Promise<number>;
+ // (undocumented)
+ protected httpClient?: BaseHttpClient;
+ // (undocumented)
+ id: string;
+ // (undocumented)
+ protected inProgressRequestBatchCount: number;
+ isEmpty(): Promise<boolean>;
+ isFinished(): Promise<boolean>;
+ // (undocumented)
+ log: CrawleeLogger;
+ markRequestAsHandled(request: Request_2): Promise<RequestQueueOperationInfo | null>;
+ // (undocumented)
+ name?: string;
+ static open(identifier?: string | StorageIdentifier | null, options?: StorageOpenOptions): Promise<RequestQueue>;
+ // (undocumented)
+ protected proxyConfiguration?: ProxyConfiguration;
+ purge(): Promise<void>;
+ // (undocumented)
+ protected queuePausedForMigration: boolean;
+ reclaimRequest(request: Request_2, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;
+ // (undocumented)
+ protected requestCache: LruCache<RequestLruItem>;
+ setExpectedRequestProcessingTimeSecs(secs: number): void;
+ // (undocumented)
+ timeoutSecs: number;
+}
+
+// @internal (undocumented)
+export interface RequestQueueOperationInfo extends QueueOperationInfo {
+ // (undocumented)
+ forefront: boolean;
+ // (undocumented)
+ uniqueKey: string;
+}
+
+// @public (undocumented)
+export interface RequestQueueOperationOptions {
+ // @internal
+ cache?: boolean;
+ forefront?: boolean;
+}
+
+// @public (undocumented)
+export interface RequestQueueOptions {
+ // (undocumented)
+ client: RequestQueueClient;
+ // (undocumented)
+ id: string;
+ // (undocumented)
+ name?: string;
+ proxyConfiguration?: ProxyConfiguration;
+}
+
+// @internal (undocumented)
+export const REQUESTS_PERSISTENCE_KEY = "REQUEST_LIST_REQUESTS";
+
+// @public (undocumented)
+export type RequestsLike = AsyncIterable<Source | string> | Iterable<Source | string> | (Source | string)[];
+
+// @public (undocumented)
+export enum RequestState {
+ // (undocumented)
+ AFTER_NAV = 2,
+ // (undocumented)
+ BEFORE_NAV = 1,
+ // (undocumented)
+ DONE = 4,
+ // (undocumented)
+ ERROR = 6,
+ // (undocumented)
+ ERROR_HANDLER = 5,
+ // (undocumented)
+ REQUEST_HANDLER = 3,
+ // (undocumented)
+ SKIPPED = 7,
+ // (undocumented)
+ UNPROCESSED = 0
+}
+
+// @public
+export interface RequestTransform {
+ // (undocumented)
+ (original: RequestOptions): RequestOptions | false | undefined | null | 'skip' | 'unchanged';
+}
+
+// @internal (undocumented)
+export interface ResolveBaseUrl {
+ // (undocumented)
+ enqueueStrategy?: EnqueueLinksOptions['strategy'];
+ // (undocumented)
+ finalRequestUrl?: string;
+ // (undocumented)
+ originalRequestUrl: string;
+ // (undocumented)
+ userProvidedBaseUrl?: string;
+}
+
+// @internal
+export function resolveBaseUrlForEnqueueLinksFiltering(input: ResolveBaseUrl): string | undefined;
+
+// @public (undocumented)
+export type ResolvedConfigValues = FieldsOutput<typeof crawleeConfigFields>;
+
+// @public
+export function resolveStorageIdentifier(identifier: string | StorageIdentifier | null | undefined, client: StorageClient, storageType: 'Dataset' | 'KeyValueStore' | 'RequestQueue'): Promise<ExplicitStorageIdentifier>;
+
+// @public (undocumented)
+export interface ResponseLike {
+ // (undocumented)
+ headers?: Record<string, string | string[] | undefined> | (() => Record<string, string | string[] | undefined>);
+ // (undocumented)
+ url?: string | (() => string);
+}
+
+// @public (undocumented)
+export interface RestrictedCrawlingContext<UserData extends Dictionary = Dictionary> {
+ addRequests: (requestsLike: ReadonlyDeep<(string | Source)[]>, options?: ReadonlyDeep<RequestQueueOperationOptions>) => Promise<void>;
+ enqueueLinks: (options: ReadonlyDeep<Omit<SetRequired<EnqueueLinksOptions, 'urls'>, 'requestManager' | 'robotsTxtFile'>>) => Promise<unknown>;
+ getKeyValueStore: (identifier?: string | StorageIdentifier) => Promise<Pick<KeyValueStore, 'id' | 'name' | 'getValue' | 'getAutoSavedValue' | 'setValue' | 'getPublicUrl'>>;
+ // (undocumented)
+ id: string;
+ log: CrawleeLogger;
+ proxyInfo?: ProxyInfo;
+ pushData(data: ReadonlyDeep<Parameters<Dataset['pushData']>[0]>, datasetIdentifier?: string | StorageIdentifier): Promise<void>;
+ request: Request_2<UserData>;
+ // (undocumented)
+ session: ISession;
+ useState: <State extends Dictionary = Dictionary>(defaultValue?: State) => Promise<State>;
+}
+
+// @public
+export class RetryRequestError extends Error {
+ constructor(message?: string);
+}
+
+// @public
+export class Router<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'>> {
+ protected constructor();
+ addDefaultHandler<UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(handler: (ctx: Omit<Context, 'request'> & {
+ request: LoadedRequest<Request_2<UserData>>;
+ }) => Awaitable_2<void>): void;
+ addHandler<UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(label: string | symbol, handler: (ctx: Omit<Context, 'request'> & {
+ request: LoadedRequest<Request_2<UserData>>;
+ }) => Awaitable_2<void>): void;
+ static create<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'> = CrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): RouterHandler<Context>;
+ getHandler(label?: string | symbol): (ctx: Context) => Awaitable_2<void>;
+ use(middleware: (ctx: Context) => Awaitable_2<void>): void;
+}
+
+// @public (undocumented)
+export interface RouterHandler<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'> = CrawlingContext> extends Router<Context> {
+ // (undocumented)
+ (ctx: Context): Awaitable_2<void>;
+}
+
+// @public (undocumented)
+export type RouterRoutes<Context, UserData extends Dictionary> = {
+ [label in string | symbol]: (ctx: Omit<Context, 'request'> & {
+ request: Request_2<UserData>;
+ }) => Awaitable_2<void>;
+};
+
+// @internal
+export function serializeArray<T>(data: T[]): Promise<Buffer>;
+
+// @public
+export function serializeValue(value: unknown, contentType?: string): {
+ value: Buffer | ArrayBuffer | ArrayBufferView | string | NodeJS.ReadableStream | ReadableStream;
+ contentType: string;
+};
+
+// @public
+export class ServiceConflictError extends Error {
+ constructor(serviceName: string, newValue: unknown, existingValue: unknown);
+}
+
+// @public
+export class ServiceLocator implements ServiceLocatorInterface {
+ constructor(configuration?: Configuration, eventManager?: EventManager, storageClient?: StorageClient, logger?: CrawleeLogger);
+ // (undocumented)
+ getChildLog(prefix: string): CrawleeLogger;
+ // (undocumented)
+ getConfiguration(): Configuration;
+ // (undocumented)
+ getEventManager(): EventManager;
+ // (undocumented)
+ getLogger(): CrawleeLogger;
+ // (undocumented)
+ getStorageClient(): StorageClient;
+ // (undocumented)
+ getStorageInstanceManager(): StorageInstanceManager;
+ // (undocumented)
+ reset(): void;
+ // (undocumented)
+ setConfiguration(configuration: Configuration): void;
+ // (undocumented)
+ setEventManager(eventManager: EventManager): void;
+ // (undocumented)
+ setLogger(logger: CrawleeLogger): void;
+ // (undocumented)
+ setStorageClient(storageClient: StorageClient): void;
+}
+
+// @public (undocumented)
+export const serviceLocator: ServiceLocatorInterface;
+
+// @public
+export class Session implements ISession {
+ constructor(options?: SessionOptions);
+ // (undocumented)
+ get cookieJar(): CookieJar;
+ // (undocumented)
+ get createdAt(): Date;
+ // (undocumented)
+ get errorScore(): number;
+ // (undocumented)
+ get errorScoreDecrement(): number;
+ // (undocumented)
+ get expiresAt(): Date;
+ // (undocumented)
+ get fingerprint(): SessionFingerprint | undefined;
+ set fingerprint(fingerprint: SessionFingerprint | undefined);
+ getCookieString(url: string): string;
+ getState(): SessionState;
+ // (undocumented)
+ readonly id: string;
+ isBlocked(): boolean;
+ isExpired(): boolean;
+ isMaxUsageCountReached(): boolean;
+ isUsable(): boolean;
+ markBad(): void;
+ markGood(): void;
+ // (undocumented)
+ get maxErrorScore(): number;
+ // (undocumented)
+ get maxUsageCount(): number;
+ protected _maybeSelfRetire(): void;
+ // (undocumented)
+ get proxyInfo(): ProxyInfo | undefined;
+ retire(): void;
+ get retired(): boolean;
+ setCookie(rawCookie: string, url: string): void;
+ // (undocumented)
+ get usageCount(): number;
+ // (undocumented)
+ userData: Dictionary;
+}
+
+// @public
+export class SessionError extends Error {
+ constructor(message?: string);
+}
+
+// @public (undocumented)
+export interface SessionOptions {
+ // (undocumented)
+ cookieJar?: CookieJar;
+ createdAt?: Date;
+ // (undocumented)
+ errorScore?: number;
+ errorScoreDecrement?: number;
+ expiresAt?: Date;
+ fingerprint?: SessionFingerprint;
+ id?: string;
+ // (undocumented)
+ log?: CrawleeLogger;
+ maxAgeSecs?: number;
+ maxErrorScore?: number;
+ maxUsageCount?: number;
+ // (undocumented)
+ proxyInfo?: ProxyInfo;
+ retired?: boolean;
+ usageCount?: number;
+ userData?: Dictionary;
+}
+
+// @public
+export class SessionPool implements ISessionPool {
+ constructor(options?: SessionPoolOptions);
+ addSession(options?: Session | SessionOptions): Promise<void>;
+ protected _addSession(newSession: Session): void;
+ protected _createSession(): Promise<Session>;
+ // (undocumented)
+ protected createSessionFunction: CreateSession;
+ protected _defaultCreateSessionFunction(options?: {
+ sessionOptions?: SessionOptions;
+ }): Promise<Session>;
+ protected ensureInitialized(): Promise<void>;
+ // (undocumented)
+ protected events: EventManager;
+ protected _getRandomIndex(): number;
+ getSession(sessionId?: string): Promise<Session | undefined>;
+ getState(): Promise<{
+ usableSessionsCount: number;
+ retiredSessionsCount: number;
+ sessions: SessionState[];
+ }>;
+ protected _hasSpaceForSession(): boolean;
+ // (undocumented)
+ readonly id: string;
+ // (undocumented)
+ protected keyValueStore?: KeyValueStore;
+ // (undocumented)
+ protected _listener?: () => Promise<void>;
+ // (undocumented)
+ protected log: CrawleeLogger;
+ // (undocumented)
+ protected maxPoolSize: number;
+ protected _maybeLoadSessionPool(): Promise<void>;
+ newSession(sessionOptions?: SessionOptions): Promise<Session>;
+ // (undocumented)
+ protected persistenceOptions: PersistenceOptions;
+ persistState(options?: PersistenceOptions): Promise<void>;
+ // (undocumented)
+ protected persistStateKey: string;
+ // (undocumented)
+ protected persistStateKeyValueStoreId?: string;
+ protected _pickSession(): Session | undefined;
+ protected _removeRetiredSessions(): void;
+ // (undocumented)
+ resetStore(options?: PersistenceOptions): Promise<void>;
+ retiredSessionsCount(): Promise<number>;
+ // (undocumented)
+ protected sessionMap: Map<string, Session>;
+ // (undocumented)
+ protected sessionOptions: SessionOptions;
+ // (undocumented)
+ protected sessionReuseStrategy: SessionReuseStrategy;
+ // (undocumented)
+ protected sessions: Session[];
+ teardown(): Promise<void>;
+ usableSessionsCount(): Promise<number>;
+}
+
+// @public (undocumented)
+export interface SessionPoolOptions {
+ createSessionFunction?: CreateSession;
+ id?: string | number;
+ // @internal (undocumented)
+ log?: CrawleeLogger;
+ maxPoolSize?: number;
+ persistenceOptions?: PersistenceOptions;
+ persistStateKey?: string;
+ persistStateKeyValueStoreId?: string;
+ sessionOptions?: SessionOptions;
+ sessionReuseStrategy?: SessionReuseStrategy;
+}
+
+// @public (undocumented)
+export type SessionReuseStrategy = (typeof SESSION_REUSE_STRATEGIES)[number];
+
+// @public
+export class SitemapRequestLoader implements IRequestLoader {
+ // (undocumented)
+ [Symbol.asyncIterator](): AsyncGenerator<Request_2<Dictionary>, void, unknown>;
+ // (undocumented)
+ fetchNextRequest(): Promise<Request_2 | null>;
+ // (undocumented)
+ getHandledCount(): Promise<number>;
+ // (undocumented)
+ getPendingCount(): Promise<number>;
+ // (undocumented)
+ getTotalCount(): Promise<number>;
+ // @internal
+ inProgress: Set<string>;
+ // (undocumented)
+ isEmpty(): Promise<boolean>;
+ // (undocumented)
+ isFinished(): Promise<boolean>;
+ isSitemapFullyLoaded(): boolean;
+ // (undocumented)
+ markRequestAsHandled(request: Request_2): Promise<void>;
+ static open(options: SitemapRequestLoaderOptions): Promise<SitemapRequestLoader>;
+ // (undocumented)
+ persistState(): Promise<void>;
+ teardown(): Promise<void>;
+ toTandem(requestManager?: IRequestManager): Promise<IRequestManager>;
+}
+
+// @public (undocumented)
+export interface SitemapRequestLoaderOptions extends UrlConstraints {
+ httpClient?: BaseHttpClient;
+ maxBufferSize?: number;
+ parseSitemapOptions?: Omit<ParseSitemapOptions, 'emitNestedSitemaps' | 'maxDepth'>;
+ persistenceOptions?: {
+ enable?: boolean;
+ };
+ persistStateKey?: string;
+ proxyUrl?: string;
+ signal?: AbortSignal;
+ sitemapUrls: string[];
+ timeoutMillis?: number;
+}
+
+// @public (undocumented)
+export type SkippedRequestCallback = (args: {
+ url: string;
+ reason: SkippedRequestReason;
+}) => Awaitable<void>;
+
+// @public (undocumented)
+export type SkippedRequestReason = 'robotsTxt' | 'limit' | 'enqueueLimit' | 'filters' | 'transform' | 'redirect' | 'depth';
+
+// @public (undocumented)
+export interface SnapshotResult {
+ // (undocumented)
+ htmlFileName?: string;
+ // (undocumented)
+ screenshotFileName?: string;
+}
+
+// @public
+export class Snapshotter {
+ constructor(options?: SnapshotterOptions);
+ // (undocumented)
+ clientInterval: BetterIntervalID;
+ // (undocumented)
+ clientSnapshotIntervalMillis: number;
+ // (undocumented)
+ clientSnapshots: ClientSnapshot[];
+ // (undocumented)
+ cpuSnapshots: CpuSnapshot[];
+ // (undocumented)
+ eventLoopInterval: BetterIntervalID;
+ // (undocumented)
+ eventLoopSnapshotIntervalMillis: number;
+ // (undocumented)
+ eventLoopSnapshots: EventLoopSnapshot[];
+ getClientSample(sampleDurationMillis?: number): ClientSnapshot[];
+ getCpuSample(sampleDurationMillis?: number): CpuSnapshot[];
+ getEventLoopSample(sampleDurationMillis?: number): EventLoopSnapshot[];
+ getMemorySample(sampleDurationMillis?: number): MemorySnapshot[];
+ protected _getSample<T extends {
+ createdAt: Date;
+ }>(snapshots: T[], sampleDurationMillis?: number): T[];
+ // (undocumented)
+ lastLoggedCriticalMemoryOverloadAt: Date | null;
+ // (undocumented)
+ log: CrawleeLogger;
+ // (undocumented)
+ maxBlockedMillis: number;
+ // (undocumented)
+ maxClientErrors: number;
+ // (undocumented)
+ maxMemoryBytes: number;
+ // (undocumented)
+ maxUsedMemoryRatio: number;
+ protected _memoryOverloadWarning(systemInfo: SystemInfo): void;
+ // (undocumented)
+ memorySnapshots: MemorySnapshot[];
+ protected _pruneSnapshots(snapshots: MemorySnapshot[] | CpuSnapshot[] | EventLoopSnapshot[] | ClientSnapshot[], now: Date): void;
+ protected _snapshotClient(intervalCallback: () => unknown): void;
+ protected _snapshotCpu(systemInfo: SystemInfo): void;
+ protected _snapshotEventLoop(intervalCallback: () => unknown): void;
+ // (undocumented)
+ snapshotHistoryMillis: number;
+ protected _snapshotMemory(systemInfo: SystemInfo): void;
+ start(): Promise<void>;
+ stop(): Promise<void>;
+}
+
+// @public (undocumented)
+export interface SnapshotterOptions {
+ clientSnapshotIntervalSecs?: number;
+ eventLoopSnapshotIntervalSecs?: number;
+ // @internal (undocumented)
+ log?: CrawleeLogger;
+ maxBlockedMillis?: number;
+ maxClientErrors?: number;
+ maxUsedMemoryRatio?: number;
+ snapshotHistorySecs?: number;
+}
+
+// @public (undocumented)
+export type Source = (Partial<RequestOptions> & {
+ requestsFromUrl?: string;
+ regex?: RegExp;
+}) | Request_2;
+
+// @internal (undocumented)
+export const STATE_PERSISTENCE_KEY = "REQUEST_LIST_STATE";
+
+// @public
+export interface StatisticPersistedState extends Omit<StatisticState, 'statsPersistedAt'> {
+ // (undocumented)
+ crawlerLastStartTimestamp: number;
+ // (undocumented)
+ requestAvgFailedDurationMillis: number;
+ // (undocumented)
+ requestAvgFinishedDurationMillis: number;
+ // (undocumented)
+ requestRetryHistogram: number[];
+ // (undocumented)
+ requestsTotal: number;
+ // (undocumented)
+ requestTotalDurationMillis: number;
+ // (undocumented)
+ statsId: string;
+ // (undocumented)
+ statsPersistedAt: string;
+}
+
+// @public
+export class Statistics {
+ // @internal
+ constructor(options?: StatisticsOptions);
+ calculate(): {
+ requestAvgFailedDurationMillis: number;
+ requestAvgFinishedDurationMillis: number;
+ requestsFinishedPerMinute: number;
+ requestsFailedPerMinute: number;
+ requestTotalDurationMillis: number;
+ requestsTotal: number;
+ crawlerRuntimeMillis: number;
+ };
+ errorTracker: ErrorTracker;
+ errorTrackerRetry: ErrorTracker;
+ failJob(id: number | string, retryCount: number): void;
+ finishJob(id: number | string, retryCount: number): void;
+ readonly id: string;
+ // (undocumented)
+ protected keyValueStore?: KeyValueStore;
+ protected _maybeLoadStatistics(): Promise<void>;
+ persistState(options?: PersistenceOptions): Promise<void>;
+ // (undocumented)
+ protected persistStateKey: string;
+ registerStatusCode(code: number): void;
+ readonly requestRetryHistogram: number[];
+ reset(): void;
+ // (undocumented)
+ resetStore(options?: PersistenceOptions): Promise<void>;
+ // (undocumented)
+ protected _saveRetryCountForJob(retryCount: number): void;
+ startCapturing(): Promise<void>;
+ startJob(id: number | string): void;
+ state: StatisticState;
+ stopCapturing(): Promise<void>;
+ // (undocumented)
+ protected _teardown(): void;
+ toJSON(): StatisticPersistedState;
+}
+
+// @public
+export interface StatisticsOptions {
+ id?: string;
+ keyValueStore?: KeyValueStore;
+ log?: CrawleeLogger;
+ logIntervalSecs?: number;
+ logMessage?: string;
+ persistenceOptions?: PersistenceOptions;
+ saveErrorSnapshots?: boolean;
+}
+
+// @public
+export interface StatisticState {
+ // (undocumented)
+ crawlerFinishedAt: Date | string | null;
+ // (undocumented)
+ crawlerRuntimeMillis: number;
+ // (undocumented)
+ crawlerStartedAt: Date | string | null;
+ // (undocumented)
+ errors: Record<string, unknown>;
+ // (undocumented)
+ requestMaxDurationMillis: number;
+ // (undocumented)
+ requestMinDurationMillis: number;
+ // (undocumented)
+ requestsFailed: number;
+ // (undocumented)
+ requestsFailedPerMinute: number;
+ // (undocumented)
+ requestsFinished: number;
+ // (undocumented)
+ requestsFinishedPerMinute: number;
+ // (undocumented)
+ requestsRetries: number;
+ // (undocumented)
+ requestsWithStatusCode: Record<string, number>;
+ // (undocumented)
+ requestTotalFailedDurationMillis: number;
+ // (undocumented)
+ requestTotalFinishedDurationMillis: number;
+ // (undocumented)
+ retryErrors: Record<string, unknown>;
+ // (undocumented)
+ statsPersistedAt: Date | string | null;
+}
+
+// @internal
+export const STORAGE_CONSISTENCY_DELAY_MILLIS = 3000;
+
+export { StorageClient }
+
+export { StorageIdentifier }
+
+// @public
+export class StorageInstanceManager {
+ clearCache(): void;
+ openStorage<TStorage extends IStorage>(cls: Constructor_2<TStorage>, input: (ExplicitStorageIdentifier | DefaultStorageIdentifier) & {
+ clientOpener: () => Promise<DatasetClient | KeyValueStoreClient | RequestQueueClient>;
+ clientCacheKey: Hashable;
+ }): Promise<TStorage>;
+ removeFromCache(instance: IStorage): void;
+}
+
+// @public
+export interface StorageOpenOptions {
+ config?: Configuration;
+ httpClient?: BaseHttpClient;
+ proxyConfiguration?: ProxyConfiguration;
+ storageClient?: StorageClient;
+}
+
+// @public
+export interface SystemInfo {
+ // (undocumented)
+ clientInfo: ClientInfo;
+ // @internal
+ cpuCurrentUsage?: number;
+ // (undocumented)
+ cpuInfo: ClientInfo;
+ // @internal
+ createdAt?: Date;
+ // (undocumented)
+ eventLoopInfo: ClientInfo;
+ // @internal
+ isCpuOverloaded?: boolean;
+ isSystemIdle: boolean;
+ // (undocumented)
+ memCurrentBytes?: number;
+ // (undocumented)
+ memInfo: ClientInfo;
+}
+
+// @public
+export class SystemStatus {
+ constructor(options?: SystemStatusOptions);
+ getCurrentStatus(): SystemInfo;
+ getHistoricalStatus(): SystemInfo;
+ protected _isClientOverloaded(sampleDurationMillis?: number): ClientInfo;
+ protected _isCpuOverloaded(sampleDurationMillis?: number): ClientInfo;
+ protected _isEventLoopOverloaded(sampleDurationMillis?: number): ClientInfo;
+ protected _isMemoryOverloaded(sampleDurationMillis?: number): ClientInfo;
+ protected _isSampleOverloaded<T extends {
+ createdAt: Date;
+ isOverloaded: boolean;
+ }>(sample: T[], ratio: number): ClientInfo;
+ protected _isSystemIdle(sampleDurationMillis?: number): SystemInfo;
+}
+
+// @public (undocumented)
+export interface SystemStatusOptions {
+ currentHistorySecs?: number;
+ maxClientOverloadedRatio?: number;
+ maxCpuOverloadedRatio?: number;
+ maxEventLoopOverloadedRatio?: number;
+ maxMemoryOverloadedRatio?: number;
+ snapshotter?: Snapshotter;
+}
+
+// @internal
+export function toughCookieToBrowserPoolCookie(toughCookie: Cookie_2): Cookie;
+
+export { tryAbsoluteURL }
+
+// @public (undocumented)
+export function updateEnqueueLinksPatternCache(item: GlobInput | RegExpInput | PseudoUrlInput, pattern: RegExpObject | GlobObject): void;
+
+// @public (undocumented)
+export type UrlPatternObject = {
+ glob?: string;
+ regexp?: RegExp;
+} & Pick<RequestOptions, 'method' | 'payload' | 'label' | 'userData' | 'headers'>;
+
+// @public
+export function useState<State extends Dictionary = Dictionary>(name?: string, defaultValue?: State, options?: UseStateOptions): Promise<State>;
+
+// @public (undocumented)
+export interface UseStateOptions {
+ // (undocumented)
+ config?: Configuration;
+ keyValueStoreName?: string | null;
+}
+
+// @internal (undocumented)
+export function validateGlobPattern(glob: string): string;
+
+// @internal (undocumented)
+export const validators: {
+ browserPage: (value: Dictionary) => {
+ validator: boolean;
+ message: (label: string) => string;
+ };
+ proxyConfiguration: (value: Dictionary) => {
+ validator: boolean;
+ message: (label: string) => string;
+ };
+ requestList: (value: Dictionary) => {
+ validator: boolean;
+ message: (label: string) => string;
+ };
+ requestQueue: (value: Dictionary) => {
+ validator: boolean;
+ message: (label: string) => string;
+ };
+ browserPool: (value: Dictionary) => {
+ validator: boolean;
+ message: (label: string) => string;
+ };
+ sessionPool: (value: Dictionary) => {
+ validator: boolean;
+ message: (label: string) => string;
+ };
+};
+
+// @public
+export const withCheckedStorageAccess: <T>(checkFunction: () => void, callback: () => Awaitable_2<T>) => Promise<T>;
+
+// @internal (undocumented)
+export type WithRequired<T, K extends keyof T> = T & {
+ [P in K]-?: T[P];
+};
+
+// (No @packageDocumentation comment for this package)
+
+```
diff --git a/docs/public-api/crawlee-fs-storage.api.md b/docs/public-api/crawlee-fs-storage.api.md
new file mode 100644
index 000000000..f5d0c539d
--- /dev/null
+++ b/docs/public-api/crawlee-fs-storage.api.md
@@ -0,0 +1,56 @@
+## API Report File for "@crawlee/fs-storage"
+
+> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
+
+```ts
+
+import type { CrawleeLogger } from '@crawlee/types';
+import type { Dictionary } from '@crawlee/types';
+import type * as storage from '@crawlee/types';
+
+// @public (undocumented)
+export class FileSystemStorageClient implements storage.StorageClient {
+ constructor(options?: FileSystemStorageOptions);
+ // (undocumented)
+ createDatasetClient(options?: storage.CreateDatasetClientOptions): Promise<storage.DatasetClient>;
+ // (undocumented)
+ createKeyValueStoreClient(options?: storage.CreateKeyValueStoreClientOptions): Promise<storage.KeyValueStoreClient>;
+ // (undocumented)
+ createRequestQueueClient(options?: storage.CreateRequestQueueClientOptions): Promise<storage.RequestQueueClient>;
+ // (undocumented)
+ readonly datasetClientCache: DatasetClient[];
+ // (undocumented)
+ readonly datasetsDirectory: string;
+ getStorageClientCacheKey(): string;
+ // (undocumented)
+ readonly keyValueStoreCache: KeyValueStoreClient[];
+ // (undocumented)
+ readonly keyValueStoresDirectory: string;
+ // (undocumented)
+ readonly localDataDirectory: string;
+ // (undocumented)
+ readonly logger?: CrawleeLogger;
+ purge(): Promise<void>;
+ // (undocumented)
+ readonly requestQueueCache: RequestQueueClient[];
+ // (undocumented)
+ readonly requestQueuesDirectory: string;
+ // (undocumented)
+ setStatusMessage(message: string, options?: storage.SetStatusMessageOptions): Promise<void>;
+ // (undocumented)
+ storageExists(id: string, type: 'Dataset' | 'KeyValueStore' | 'RequestQueue'): Promise<boolean>;
+ teardown(): Promise<void>;
+ // (undocumented)
+ readonly writeMetadata: boolean;
+}
+
+// @public (undocumented)
+export interface FileSystemStorageOptions {
+ localDataDirectory?: string;
+ logger?: CrawleeLogger;
+ writeMetadata?: boolean;
+}
+
+// (No @packageDocumentation comment for this package)
+
+```
diff --git a/docs/public-api/crawlee-got-scraping-client.api.md b/docs/public-api/crawlee-got-scraping-client.api.md
new file mode 100644
index 000000000..5987bfffe
--- /dev/null
+++ b/docs/public-api/crawlee-got-scraping-client.api.md
@@ -0,0 +1,18 @@
+## API Report File for "@crawlee/got-scraping-client"
+
+> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
+
+```ts
+
+import { BaseHttpClient } from '@crawlee/http-client';
+import { CustomFetchOptions } from '@crawlee/http-client';
+
+// @public
+export class GotScrapingHttpClient extends BaseHttpClient {
+ // (undocumented)
+ fetch(request: Request, options?: RequestInit & CustomFetchOptions): Promise<Response>;
+}
+
+// (No @packageDocumentation comment for this package)
+
+```
diff --git a/docs/public-api/crawlee-http-client.api.md b/docs/public-api/crawlee-http-client.api.md
new file mode 100644
index 000000000..b4d3f8218
--- /dev/null
+++ b/docs/public-api/crawlee-http-client.api.md
@@ -0,0 +1,54 @@
+## API Report File for "@crawlee/http-client"
+
+> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
+
+```ts
+
+import type { BaseHttpClient as BaseHttpClient_2 } from '@crawlee/types';
+import { CookieJar } from 'tough-cookie';
+import type { CrawleeLogger } from '@crawlee/types';
+import type { SendRequestOptions } from '@crawlee/types';
+import type { SessionFingerprint } from '@crawlee/types';
+
+// @public
+export abstract class BaseHttpClient implements BaseHttpClient_2 {
+ constructor(options?: {
+ logger?: CrawleeLogger;
+ });
+ protected abstract fetch(input: Request, init?: RequestInit & CustomFetchOptions): Promise<Response>;
+ // (undocumented)
+ protected log?: CrawleeLogger;
+ sendRequest(initialRequest: Request, options?: SendRequestOptions): Promise<Response>;
+}
+
+// @public
+export interface CustomFetchOptions {
+ cookieJar?: CookieJar;
+ fingerprint?: SessionFingerprint;
+ proxyUrl?: string;
+}
+
+// @public
+export class FetchHttpClient extends BaseHttpClient {
+ // (undocumented)
+ fetch(request: Request, options?: RequestInit & CustomFetchOptions): Promise<Response>;
+}
+
+// @public (undocumented)
+export interface IResponseWithUrl extends Response {
+ // (undocumented)
+ url: string;
+}
+
+// @public
+export class ResponseWithUrl extends Response implements IResponseWithUrl {
+ constructor(body: BodyInit | null, init: ResponseInit & {
+ url?: string;
+ });
+ // (undocumented)
+ url: string;
+}
+
+// (No @packageDocumentation comment for this package)
+
+```
diff --git a/docs/public-api/crawlee-http.api.md b/docs/public-api/crawlee-http.api.md
new file mode 100644
index 000000000..9b76d3d54
--- /dev/null
+++ b/docs/public-api/crawlee-http.api.md
@@ -0,0 +1,250 @@
+## API Report File for "@crawlee/http"
+
+> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
+
+```ts
+
+import { AllowedHttpMethods } from '@crawlee/types';
+import { AnyPredicate } from 'ow';
+import { ArrayPredicate } from 'ow';
+import type { Awaitable } from '@crawlee/types';
+import { BasePredicate } from 'ow';
+import { BasicCrawler } from '@crawlee/basic';
+import type { BasicCrawlerOptions } from '@crawlee/basic';
+import { BooleanPredicate } from 'ow';
+import { CheerioRoot } from '@crawlee/utils';
+import { ContextPipeline } from '@crawlee/basic';
+import type { CrawlingContext } from '@crawlee/basic';
+import type { CrawlingContext as CrawlingContext_2 } from '@crawlee/core';
+import type { Dictionary } from '@crawlee/types';
+import { ErrorHandler } from '@crawlee/basic';
+import { GetUserDataFromRequest } from '@crawlee/basic';
+import type { ISession } from '@crawlee/types';
+import type { JsonValue } from 'type-fest';
+import { LoadedRequest } from '@crawlee/core';
+import { NumberPredicate } from 'ow';
+import { ObjectPredicate } from 'ow';
+import { Predicate } from 'ow';
+import { Request as Request_2 } from '@crawlee/basic';
+import type { Request as Request_3 } from '@crawlee/core';
+import { RequestHandler } from '@crawlee/basic';
+import type { RequestLike } from 'content-type';
+import type { RequireContextPipeline } from '@crawlee/basic';
+import type { ResponseLike } from 'content-type';
+import { ResponseWithUrl } from '@crawlee/http-client';
+import { RouterHandler } from '@crawlee/basic';
+import { RouterRoutes } from '@crawlee/basic';
+import { StringPredicate } from 'ow';
+import { Transform } from 'node:stream';
+
+// @public
+export function ByteCounterStream(input: {
+ logTransferredBytes: (transferredBytes: number) => void;
+ loggingInterval?: number;
+}): Transform;
+
+// @public
+export function createFileRouter<Context extends FileDownloadCrawlingContext = FileDownloadCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): RouterHandler<Context>;
+
+// @public
+export function createHttpRouter<Context extends HttpCrawlingContext = HttpCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): RouterHandler<Context>;
+
+// @public
+export class FileDownload extends BasicCrawler<FileDownloadCrawlingContext> {
+ constructor(options?: BasicCrawlerOptions<FileDownloadCrawlingContext>);
+ // (undocumented)
+ protected buildContextPipeline(): ContextPipeline<CrawlingContext_2<Dictionary>, CrawlingContext_2<Dictionary> & {
+ request: LoadedRequest<Request_3>;
+ response: ResponseWithUrl;
+ contentType: {
+ type: string;
+ encoding: BufferEncoding;
+ };
+ [kBodyDrained]: Promise<void>;
+ }>;
+}
+
+// @public (undocumented)
+export interface FileDownloadCrawlingContext<UserData extends Dictionary = any> extends CrawlingContext_2<UserData> {
+ // (undocumented)
+ contentType: {
+ type: string;
+ encoding: BufferEncoding;
+ };
+ // (undocumented)
+ request: LoadedRequest<Request_3<UserData>>;
+ // (undocumented)
+ response: Response;
+}
+
+// @public (undocumented)
+export type FileDownloadErrorHandler<UserData extends Dictionary = any> = ErrorHandler<FileDownloadCrawlingContext<UserData>>;
+
+// @public (undocumented)
+export type FileDownloadHook<UserData extends Dictionary = any> = InternalHttpHook<FileDownloadCrawlingContext<UserData>>;
+
+// @public (undocumented)
+export type FileDownloadRequestHandler<UserData extends Dictionary = any> = RequestHandler<FileDownloadCrawlingContext<UserData>>;
+
+// @public
+export class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any> = InternalHttpCrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension> extends BasicCrawler<Context, ContextExtension, ExtendedContext> {
+ constructor(options?: HttpCrawlerOptions<Context, ContextExtension, ExtendedContext> & RequireContextPipeline<InternalHttpCrawlingContext, Context>);
+ // (undocumented)
+ protected buildContextPipeline(): ContextPipeline<CrawlingContext, InternalHttpCrawlingContext>;
+ // (undocumented)
+ protected _encodeResponse(request: Request_2, response: Response, encoding: BufferEncoding): {
+ encoding: BufferEncoding;
+ response: Response;
+ };
+ protected _extendSupportedMimeTypes(additionalMimeTypes: (string | RequestLike | ResponseLike)[]): void;
+ // (undocumented)
+ protected forceResponseEncoding?: string;
+ protected _getRequestOptions(request: Request_2, session: ISession, proxyUrl?: string): {
+ url: string;
+ method: AllowedHttpMethods;
+ proxyUrl: string | undefined;
+ timeout: number;
+ sessionToken: ISession;
+ headers: Record<string, string> | undefined;
+ https: {
+ rejectUnauthorized: boolean;
+ };
+ body: string | undefined;
+ };
+ protected _handleRequestTimeout(session: ISession): void;
+ // (undocumented)
+ protected ignoreSslErrors: boolean;
+ // (undocumented)
+ protected isRequestBlocked(crawlingContext: InternalHttpCrawlingContext): Promise<string | false>;
+ // (undocumented)
+ protected navigationTimeoutMillis: number;
+ // (undocumented)
+ protected static optionsShape: {
+ navigationTimeoutSecs: NumberPredicate & BasePredicate<number | undefined>;
+ ignoreSslErrors: BooleanPredicate & BasePredicate<boolean | undefined>;
+ additionalMimeTypes: ArrayPredicate<string>;
+ suggestResponseEncoding: StringPredicate & BasePredicate<string | undefined>;
+ forceResponseEncoding: StringPredicate & BasePredicate<string | undefined>;
+ saveResponseCookies: BooleanPredicate & BasePredicate<boolean | undefined>;
+ preNavigationHooks: ArrayPredicate<unknown> & BasePredicate<unknown[] | undefined>;
+ postNavigationHooks: ArrayPredicate<unknown> & BasePredicate<unknown[] | undefined>;
+ contextPipelineBuilder: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ extendContext: Predicate<Function> & BasePredicate<Function | undefined>;
+ requestList: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ requestQueue: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ requestHandler: Predicate<Function> & BasePredicate<Function | undefined>;
+ requestHandlerTimeoutSecs: NumberPredicate & BasePredicate<number | undefined>;
+ errorHandler: Predicate<Function> & BasePredicate<Function | undefined>;
+ failedRequestHandler: Predicate<Function> & BasePredicate<Function | undefined>;
+ maxRequestRetries: NumberPredicate & BasePredicate<number | undefined>;
+ sameDomainDelaySecs: NumberPredicate & BasePredicate<number | undefined>;
+ maxRequestsPerCrawl: NumberPredicate & BasePredicate<number | undefined>;
+ maxCrawlDepth: NumberPredicate & BasePredicate<number | undefined>;
+ autoscaledPoolOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ sessionPool: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ proxyConfiguration: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ statusMessageLoggingInterval: NumberPredicate & BasePredicate<number | undefined>;
+ statusMessageCallback: Predicate<Function> & BasePredicate<Function | undefined>;
+ additionalHttpErrorStatusCodes: ArrayPredicate<number>;
+ ignoreHttpErrorStatusCodes: ArrayPredicate<number>;
+ blockedStatusCodes: ArrayPredicate<number>;
+ retryOnBlocked: BooleanPredicate & BasePredicate<boolean | undefined>;
+ respectRobotsTxtFile: AnyPredicate<boolean | object>;
+ onSkippedRequest: Predicate<Function> & BasePredicate<Function | undefined>;
+ httpClient: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ configuration: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ storageClient: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ eventManager: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ logger: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ minConcurrency: NumberPredicate & BasePredicate<number | undefined>;
+ maxConcurrency: NumberPredicate & BasePredicate<number | undefined>;
+ maxRequestsPerMinute: NumberPredicate & BasePredicate<number | undefined>;
+ keepAlive: BooleanPredicate & BasePredicate<boolean | undefined>;
+ statisticsOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ id: StringPredicate & BasePredicate<string | undefined>;
+ };
+ protected _parseResponse(request: Request_2, response: Response): Promise<{
+ response: Response;
+ contentType: {
+ type: string;
+ encoding: BufferEncoding;
+ };
+ body: string;
+ } | {
+ body: Buffer<ArrayBuffer>;
+ response: Response;
+ contentType: {
+ type: string;
+ encoding: BufferEncoding;
+ };
+ }>;
+ // (undocumented)
+ protected postNavigationHooks: ((crawlingContext: CrawlingContextWithResponse) => Awaitable<void | Partial<CrawlingContextWithResponse>>)[];
+ // (undocumented)
+ protected preNavigationHooks: InternalHttpHook<CrawlingContext>[];
+ protected _requestFunction(input: RequestFunctionOptions): Promise<Response>;
+ // (undocumented)
+ protected saveResponseCookies: boolean;
+ // (undocumented)
+ protected suggestResponseEncoding?: string;
+ // (undocumented)
+ protected readonly supportedMimeTypes: Set<string>;
+}
+
+// @public (undocumented)
+export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext = InternalHttpCrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension> extends BasicCrawlerOptions<Context, ContextExtension, ExtendedContext> {
+ additionalMimeTypes?: string[];
+ forceResponseEncoding?: string;
+ ignoreSslErrors?: boolean;
+ navigationTimeoutSecs?: number;
+ postNavigationHooks?: ((crawlingContext: CrawlingContextWithResponse) => Awaitable<void | Partial<CrawlingContextWithResponse>>)[];
+ preNavigationHooks?: InternalHttpHook<CrawlingContext>[];
+ saveResponseCookies?: boolean;
+ suggestResponseEncoding?: string;
+}
+
+// @public (undocumented)
+export interface HttpCrawlingContext<UserData extends Dictionary = any, JSONData extends JsonValue = any> extends InternalHttpCrawlingContext<UserData, JSONData> {
+}
+
+// @public (undocumented)
+export type HttpErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
+JSONData extends JsonValue = any> = ErrorHandler<HttpCrawlingContext<UserData, JSONData>>;
+
+// @public (undocumented)
+export type HttpHook<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
+JSONData extends JsonValue = any> = InternalHttpHook<HttpCrawlingContext<UserData, JSONData>>;
+
+// @public (undocumented)
+export type HttpRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
+JSONData extends JsonValue = any> = RequestHandler<HttpCrawlingContext<UserData, JSONData>>;
+
+// @internal (undocumented)
+export interface InternalHttpCrawlingContext<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
+JSONData extends JsonValue = any> extends CrawlingContextWithResponse<UserData> {
+ body: string | Buffer;
+ contentType: {
+ type: string;
+ encoding: BufferEncoding;
+ };
+ json: JSONData;
+ parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
+ waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
+}
+
+// @internal (undocumented)
+export type InternalHttpHook<Context> = (crawlingContext: Context) => Awaitable<void | Partial<Context>>;
+
+// @public
+export function MinimumSpeedStream(input: {
+ minSpeedKbps: number;
+ historyLengthMs?: number;
+ checkProgressInterval?: number;
+}): Transform;
+
+
+export * from "@crawlee/basic";
+
+// (No @packageDocumentation comment for this package)
+
+```
diff --git a/docs/public-api/crawlee-impit-client.api.md b/docs/public-api/crawlee-impit-client.api.md
new file mode 100644
index 000000000..a41a18ea1
--- /dev/null
+++ b/docs/public-api/crawlee-impit-client.api.md
@@ -0,0 +1,29 @@
+## API Report File for "@crawlee/impit-client"
+
+> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
+
+```ts
+
+import { BaseHttpClient } from '@crawlee/http-client';
+import type { CrawleeLogger } from '@crawlee/types';
+import type { CustomFetchOptions } from '@crawlee/http-client';
+import { ImpitOptions } from 'impit';
+
+// @public (undocumented)
+export const Browser: {
+ readonly Chrome: "chrome";
+ readonly Firefox: "firefox";
+};
+
+// @public
+export class ImpitHttpClient extends BaseHttpClient {
+ constructor(options?: Omit<ImpitOptions, 'proxyUrl' | 'timeout'> & {
+ logger?: CrawleeLogger;
+ });
+ // (undocumented)
+ fetch(request: Request, options?: RequestInit & CustomFetchOptions): Promise<Response>;
+}
+
+// (No @packageDocumentation comment for this package)
+
+```
diff --git a/docs/public-api/crawlee-jsdom.api.md b/docs/public-api/crawlee-jsdom.api.md
new file mode 100644
index 000000000..4f88c93e8
--- /dev/null
+++ b/docs/public-api/crawlee-jsdom.api.md
@@ -0,0 +1,149 @@
+## API Report File for "@crawlee/jsdom"
+
+> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
+
+```ts
+
+import { AnyPredicate } from 'ow';
+import { ArrayPredicate } from 'ow';
+import { BasePredicate } from 'ow';
+import type { BasicCrawlingContext } from '@crawlee/http';
+import { BooleanPredicate } from 'ow';
+import * as cheerio from 'cheerio';
+import { CheerioRoot } from '@crawlee/utils';
+import { ContextPipeline } from '@crawlee/http';
+import { CrawlingContext } from '@crawlee/http';
+import type { Dictionary } from '@crawlee/types';
+import type { DOMWindow } from 'jsdom';
+import type { EnqueueLinksOptions } from '@crawlee/http';
+import type { ErrorHandler } from '@crawlee/http';
+import type { GetUserDataFromRequest } from '@crawlee/http';
+import { HttpCrawler } from '@crawlee/http';
+import type { HttpCrawlerOptions } from '@crawlee/http';
+import type { InternalHttpCrawlingContext } from '@crawlee/http';
+import type { InternalHttpHook } from '@crawlee/http';
+import type { IRequestManager } from '@crawlee/http';
+import { NumberPredicate } from 'ow';
+import { ObjectPredicate } from 'ow';
+import { Predicate } from 'ow';
+import type { RequestHandler } from '@crawlee/http';
+import { RobotsTxtFile } from '@crawlee/utils';
+import { RouterHandler } from '@crawlee/http';
+import type { RouterRoutes } from '@crawlee/http';
+import type { SkippedRequestCallback } from '@crawlee/http';
+import { StringPredicate } from 'ow';
+import { VirtualConsole } from 'jsdom';
+
+// @public
+export function createJSDOMRouter<Context extends JSDOMCrawlingContext = JSDOMCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): RouterHandler<Context>;
+
+// @internal (undocumented)
+export function domCrawlerEnqueueLinks(options: EnqueueLinksInternalOptions | BoundEnqueueLinksInternalOptions): Promise<unknown>;
+
+// @public (undocumented)
+export class JSDOMCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext & ContextExtension> extends HttpCrawler<JSDOMCrawlingContext, ContextExtension, ExtendedContext> {
+ constructor(options?: JSDOMCrawlerOptions<ContextExtension, ExtendedContext>);
+ // (undocumented)
+ protected buildContextPipeline(): ContextPipeline<CrawlingContext<Dictionary>, InternalHttpCrawlingContext<any, any> & {
+ readonly window: DOMWindow;
+ readonly body: string;
+ readonly document: Document;
+ } & {
+ enqueueLinks: (enqueueOptions?: EnqueueLinksOptions) => Promise<unknown>;
+ waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
+ parseWithCheerio(selector?: string, _timeoutMs?: number): Promise<cheerio.CheerioAPI>;
+ }>;
+ getVirtualConsole(): VirtualConsole;
+ // (undocumented)
+ protected hideInternalConsole: boolean;
+ // (undocumented)
+ protected static optionsShape: {
+ runScripts: BooleanPredicate & BasePredicate<boolean | undefined>;
+ hideInternalConsole: BooleanPredicate & BasePredicate<boolean | undefined>;
+ navigationTimeoutSecs: NumberPredicate & BasePredicate<number | undefined>;
+ ignoreSslErrors: BooleanPredicate & BasePredicate<boolean | undefined>;
+ additionalMimeTypes: ArrayPredicate<string>;
+ suggestResponseEncoding: StringPredicate & BasePredicate<string | undefined>;
+ forceResponseEncoding: StringPredicate & BasePredicate<string | undefined>;
+ saveResponseCookies: BooleanPredicate & BasePredicate<boolean | undefined>;
+ preNavigationHooks: ArrayPredicate<unknown> & BasePredicate<unknown[] | undefined>;
+ postNavigationHooks: ArrayPredicate<unknown> & BasePredicate<unknown[] | undefined>;
+ contextPipelineBuilder: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ extendContext: Predicate<Function> & BasePredicate<Function | undefined>;
+ requestList: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ requestQueue: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ requestHandler: Predicate<Function> & BasePredicate<Function | undefined>;
+ requestHandlerTimeoutSecs: NumberPredicate & BasePredicate<number | undefined>;
+ errorHandler: Predicate<Function> & BasePredicate<Function | undefined>;
+ failedRequestHandler: Predicate<Function> & BasePredicate<Function | undefined>;
+ maxRequestRetries: NumberPredicate & BasePredicate<number | undefined>;
+ sameDomainDelaySecs: NumberPredicate & BasePredicate<number | undefined>;
+ maxRequestsPerCrawl: NumberPredicate & BasePredicate<number | undefined>;
+ maxCrawlDepth: NumberPredicate & BasePredicate<number | undefined>;
+ autoscaledPoolOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ sessionPool: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ proxyConfiguration: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ statusMessageLoggingInterval: NumberPredicate & BasePredicate<number | undefined>;
+ statusMessageCallback: Predicate<Function> & BasePredicate<Function | undefined>;
+ additionalHttpErrorStatusCodes: ArrayPredicate<number>;
+ ignoreHttpErrorStatusCodes: ArrayPredicate<number>;
+ blockedStatusCodes: ArrayPredicate<number>;
+ retryOnBlocked: BooleanPredicate & BasePredicate<boolean | undefined>;
+ respectRobotsTxtFile: AnyPredicate<boolean | object>;
+ onSkippedRequest: Predicate<Function> & BasePredicate<Function | undefined>;
+ httpClient: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ configuration: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ storageClient: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ eventManager: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ logger: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ minConcurrency: NumberPredicate & BasePredicate<number | undefined>;
+ maxConcurrency: NumberPredicate & BasePredicate<number | undefined>;
+ maxRequestsPerMinute: NumberPredicate & BasePredicate<number | undefined>;
+ keepAlive: BooleanPredicate & BasePredicate<boolean | undefined>;
+ statisticsOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ id: StringPredicate & BasePredicate<string | undefined>;
+ };
+ // (undocumented)
+ protected runScripts: boolean;
+ // (undocumented)
+ protected virtualConsole: VirtualConsole | null;
+}
+
+// @public (undocumented)
+export interface JSDOMCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext & ContextExtension, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
+JSONData extends Dictionary = any> extends HttpCrawlerOptions<JSDOMCrawlingContext<UserData, JSONData>, ContextExtension, ExtendedContext> {
+ hideInternalConsole?: boolean;
+ runScripts?: boolean;
+}
+
+// @public (undocumented)
+export interface JSDOMCrawlingContext<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
+JSONData extends Dictionary = any> extends InternalHttpCrawlingContext<UserData, JSONData> {
+ // (undocumented)
+ body: string;
+ // (undocumented)
+ document: Document;
+ parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
+ waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
+ // (undocumented)
+ window: DOMWindow;
+}
+
+// @public (undocumented)
+export type JSDOMErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
+JSONData extends Dictionary = any> = ErrorHandler<JSDOMCrawlingContext<UserData, JSONData>>;
+
+// @public (undocumented)
+export type JSDOMHook<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
+JSONData extends Dictionary = any> = InternalHttpHook<JSDOMCrawlingContext<UserData, JSONData>>;
+
+// @public (undocumented)
+export type JSDOMRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
+JSONData extends Dictionary = any> = RequestHandler<JSDOMCrawlingContext<UserData, JSONData>>;
+
+
+export * from "@crawlee/http";
+
+// (No @packageDocumentation comment for this package)
+
+```
diff --git a/docs/public-api/crawlee-linkedom.api.md b/docs/public-api/crawlee-linkedom.api.md
new file mode 100644
index 000000000..5de9540d9
--- /dev/null
+++ b/docs/public-api/crawlee-linkedom.api.md
@@ -0,0 +1,85 @@
+## API Report File for "@crawlee/linkedom"
+
+> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
+
+```ts
+
+import type { BasicCrawlingContext } from '@crawlee/http';
+import * as cheerio from 'cheerio';
+import { CheerioRoot } from '@crawlee/utils';
+import { ContextPipeline } from '@crawlee/http';
+import { CrawlingContext } from '@crawlee/http';
+import type { Dictionary } from '@crawlee/types';
+import type { EnqueueLinksOptions } from '@crawlee/http';
+import type { ErrorHandler } from '@crawlee/http';
+import type { GetUserDataFromRequest } from '@crawlee/http';
+import { HttpCrawler } from '@crawlee/http';
+import type { HttpCrawlerOptions } from '@crawlee/http';
+import type { InternalHttpCrawlingContext } from '@crawlee/http';
+import type { InternalHttpHook } from '@crawlee/http';
+import { IRequestManager } from '@crawlee/http';
+import type { RequestHandler } from '@crawlee/http';
+import { RobotsTxtFile } from '@crawlee/utils';
+import { RouterHandler } from '@crawlee/http';
+import type { RouterRoutes } from '@crawlee/http';
+import type { SkippedRequestCallback } from '@crawlee/http';
+
+// @public
+export function createLinkeDOMRouter<Context extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): RouterHandler<Context>;
+
+// @public
+export class LinkeDOMCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext & ContextExtension> extends HttpCrawler<LinkeDOMCrawlingContext, ContextExtension, ExtendedContext> {
+ constructor(options: LinkeDOMCrawlerOptions<ContextExtension, ExtendedContext>);
+ // (undocumented)
+ protected buildContextPipeline(): ContextPipeline<CrawlingContext<Dictionary>, InternalHttpCrawlingContext<any, any> & {
+ readonly window: Window;
+ readonly body: string;
+ readonly document: Document;
+ } & {
+ enqueueLinks: (enqueueOptions?: LinkeDOMCrawlerEnqueueLinksOptions) => Promise<unknown>;
+ waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
+ parseWithCheerio(selector?: string, _timeoutMs?: number): Promise<cheerio.CheerioAPI>;
+ }>;
+}
+
+// @internal (undocumented)
+export function linkedomCrawlerEnqueueLinks(options: EnqueueLinksInternalOptions | BoundEnqueueLinksInternalOptions): Promise<unknown>;
+
+// @public (undocumented)
+export interface LinkeDOMCrawlerEnqueueLinksOptions extends Omit<EnqueueLinksOptions, 'urls' | 'requestManager'> {
+}
+
+// @public (undocumented)
+export interface LinkeDOMCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext & ContextExtension, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
+JSONData extends Dictionary = any> extends HttpCrawlerOptions<LinkeDOMCrawlingContext<UserData, JSONData>, ContextExtension, ExtendedContext> {
+}
+
+// @public (undocumented)
+export interface LinkeDOMCrawlingContext<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
+JSONData extends Dictionary = any> extends InternalHttpCrawlingContext<UserData, JSONData> {
+ // (undocumented)
+ document: Document;
+ parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
+ waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
+ // (undocumented)
+ window: Window;
+}
+
+// @public (undocumented)
+export type LinkeDOMErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
+JSONData extends Dictionary = any> = ErrorHandler<LinkeDOMCrawlingContext<UserData, JSONData>>;
+
+// @public (undocumented)
+export type LinkeDOMHook<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
+JSONData extends Dictionary = any> = InternalHttpHook<LinkeDOMCrawlingContext<UserData, JSONData>>;
+
+// @public (undocumented)
+export type LinkeDOMRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
+JSONData extends Dictionary = any> = RequestHandler<LinkeDOMCrawlingContext<UserData, JSONData>>;
+
+
+export * from "@crawlee/http";
+
+// (No @packageDocumentation comment for this package)
+
+```
diff --git a/docs/public-api/crawlee-memory-storage.api.md b/docs/public-api/crawlee-memory-storage.api.md
new file mode 100644
index 000000000..1da99e83d
--- /dev/null
+++ b/docs/public-api/crawlee-memory-storage.api.md
@@ -0,0 +1,44 @@
+## API Report File for "@crawlee/memory-storage"
+
+> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
+
+```ts
+
+import type { CrawleeLogger } from '@crawlee/types';
+import type { Dictionary } from '@crawlee/types';
+import type * as storage from '@crawlee/types';
+
+// @public (undocumented)
+export class MemoryStorageClient implements storage.StorageClient {
+ constructor(options?: MemoryStorageOptions);
+ // (undocumented)
+ createDatasetClient(options?: storage.CreateDatasetClientOptions): Promise<storage.DatasetClient>;
+ // (undocumented)
+ createKeyValueStoreClient(options?: storage.CreateKeyValueStoreClientOptions): Promise<storage.KeyValueStoreClient>;
+ // (undocumented)
+ createRequestQueueClient(options?: storage.CreateRequestQueueClientOptions): Promise<storage.RequestQueueClient>;
+ // (undocumented)
+ readonly datasetClientCache: DatasetClient[];
+ getStorageClientCacheKey(): string;
+ // (undocumented)
+ readonly keyValueStoreCache: KeyValueStoreClient[];
+ // (undocumented)
+ readonly logger?: CrawleeLogger;
+ purge(): Promise<void>;
+ // (undocumented)
+ readonly requestQueueCache: RequestQueueClient[];
+ // (undocumented)
+ setStatusMessage(message: string, options?: storage.SetStatusMessageOptions): Promise<void>;
+ // (undocumented)
+ storageExists(id: string, type: 'Dataset' | 'KeyValueStore' | 'RequestQueue'): Promise<boolean>;
+ teardown(): Promise<void>;
+}
+
+// @public (undocumented)
+export interface MemoryStorageOptions {
+ logger?: CrawleeLogger;
+}
+
+// (No @packageDocumentation comment for this package)
+
+```
diff --git a/docs/public-api/crawlee-playwright.api.md b/docs/public-api/crawlee-playwright.api.md
new file mode 100644
index 000000000..4e8ddd47e
--- /dev/null
+++ b/docs/public-api/crawlee-playwright.api.md
@@ -0,0 +1,466 @@
+## API Report File for "@crawlee/playwright"
+
+> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
+
+```ts
+
+import type { AnyNode } from 'domhandler';
+import { AnyPredicate } from 'ow';
+import { ArrayPredicate } from 'ow';
+import { BasePredicate } from 'ow';
+import { BasicCrawler } from '@crawlee/basic';
+import type { BasicCrawlerOptions } from '@crawlee/browser';
+import { BatchAddRequestsResult } from '@crawlee/types';
+import { BooleanPredicate } from 'ow';
+import type { Browser } from 'playwright';
+import { BrowserCrawler } from '@crawlee/browser';
+import type { BrowserCrawlerOptions } from '@crawlee/browser';
+import type { BrowserCrawlingContext } from '@crawlee/browser';
+import type { BrowserHook } from '@crawlee/browser';
+import type { BrowserLaunchContext } from '@crawlee/browser';
+import { BrowserLauncher } from '@crawlee/browser';
+import type { BrowserType } from 'playwright';
+import { Cheerio } from 'cheerio';
+import { CheerioAPI } from '@crawlee/browser';
+import { CheerioRoot } from '@crawlee/utils';
+import { Configuration } from '@crawlee/browser';
+import { ContextPipeline } from '@crawlee/browser';
+import type { ContextPipeline as ContextPipeline_2 } from '@crawlee/core';
+import { CrawlingContext } from '@crawlee/browser';
+import type { CrawlingContext as CrawlingContext_2 } from '@crawlee/core';
+import { Dictionary } from '@crawlee/utils';
+import type { Dictionary as Dictionary_2 } from '@crawlee/types';
+import type { EnqueueLinksOptions } from '@crawlee/core';
+import type { GetUserDataFromRequest } from '@crawlee/browser';
+import type { GetUserDataFromRequest as GetUserDataFromRequest_2 } from '@crawlee/core';
+import type { GlobInput } from '@crawlee/browser';
+import { IRequestManager } from '@crawlee/browser';
+import type { LaunchOptions } from 'playwright';
+import type { LoadedRequest } from '@crawlee/browser';
+import { NumberPredicate } from 'ow';
+import { ObjectPredicate } from 'ow';
+import type { Page } from 'playwright';
+import { PlaywrightPlugin } from '@crawlee/browser-pool';
+import { Predicate } from 'ow';
+import type { PseudoUrlInput } from '@crawlee/browser';
+import type { RecoverableStatePersistenceOptions } from '@crawlee/core';
+import type { RegExpInput } from '@crawlee/browser';
+import type { Request as Request_2 } from '@crawlee/core';
+import { Request as Request_3 } from '@crawlee/browser';
+import type { RequestHandler } from '@crawlee/browser';
+import { RequestHandlerResult } from '@crawlee/core';
+import type { RequestTransform } from '@crawlee/browser';
+import type { Response as Response_2 } from 'playwright';
+import type { RestrictedCrawlingContext } from '@crawlee/core';
+import { RouterHandler } from '@crawlee/browser';
+import { RouterHandler as RouterHandler_2 } from '@crawlee/basic';
+import type { RouterRoutes } from '@crawlee/browser';
+import type { RouterRoutes as RouterRoutes_2 } from '@crawlee/core';
+import type { SetRequired } from 'type-fest';
+import type { SkippedRequestCallback } from '@crawlee/browser';
+import { Statistics } from '@crawlee/core';
+import type { StatisticsOptions } from '@crawlee/core';
+import type { StatisticState } from '@crawlee/core';
+import { StringPredicate } from 'ow';
+
+// @public
+export class AdaptivePlaywrightCrawler<ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext> extends BasicCrawler<AdaptivePlaywrightCrawlerContext, ExtendedContext> {
+ constructor(options?: AdaptivePlaywrightCrawlerOptions<ExtendedContext>);
+ // (undocumented)
+ protected allowStorageAccess<R, TArgs extends any[]>(func: (...args: TArgs) => Promise<R>): (...args: TArgs) => Promise<R>;
+ // (undocumented)
+ protected buildContextPipeline(): ContextPipeline_2<CrawlingContext_2<Dictionary_2>, CrawlingContext_2<Dictionary_2> & {
+ readonly request: LoadedRequest<Request_3<Dictionary_2>>;
+ readonly response: Response;
+ readonly page: Page;
+ readonly querySelector: AdaptivePlaywrightCrawlerContext["querySelector"];
+ readonly waitForSelector: AdaptivePlaywrightCrawlerContext["waitForSelector"];
+ readonly parseWithCheerio: AdaptivePlaywrightCrawlerContext["parseWithCheerio"];
+ }>;
+ // (undocumented)
+ protected commitResult(crawlingContext: CrawlingContext_2, input: RequestHandlerResult): Promise<void>;
+ // (undocumented)
+ protected enqueueLinks(options: SetRequired<EnqueueLinksOptions, 'urls'>, request: RestrictedCrawlingContext['request'], result: RequestHandlerResult): Promise<BatchAddRequestsResult>;
+ protected getPendingRequestCountApproximation(): Promise<number>;
+ // (undocumented)
+ protected _init(): Promise<void>;
+ // (undocumented)
+ protected runRequestHandler(crawlingContext: CrawlingContext_2): Promise<void>;
+ // (undocumented)
+ readonly stats: AdaptivePlaywrightCrawlerStatistics;
+ // (undocumented)
+ teardown(): Promise<void>;
+}
+
+// @public (undocumented)
+export interface AdaptivePlaywrightCrawlerContext<UserData extends Dictionary_2 = Dictionary_2> extends CrawlingContext_2<UserData> {
+ // (undocumented)
+ enqueueLinks(options?: EnqueueLinksOptions): Promise<unknown>;
+ page: Page;
+ parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
+ querySelector(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
+ // (undocumented)
+ request: LoadedRequest<Request_3<UserData>>;
+ response: Response;
+ waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
+}
+
+// @public (undocumented)
+export interface AdaptivePlaywrightCrawlerOptions<ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext> extends Omit<BasicCrawlerOptions<AdaptivePlaywrightCrawlerContext, ExtendedContext>, 'preNavigationHooks' | 'postNavigationHooks'> {
+ postNavigationHooks?: AdaptivePostNavigationHook[];
+ preNavigationHooks?: AdaptiveHook[];
+ preventDirectStorageAccess?: boolean;
+ renderingTypeDetectionRatio?: number;
+ renderingTypePredictor?: Pick<RenderingTypePredictor, 'predict' | 'storeResult' | 'initialize'>;
+ resultChecker?: (result: RequestHandlerResult) => boolean;
+ resultComparator?: (resultA: RequestHandlerResult, resultB: RequestHandlerResult) => boolean | 'equal' | 'different' | 'inconclusive';
+}
+
+// @public
+function blockRequests(page: Page, options?: BlockRequestsOptions): Promise<void>;
+
+// @public (undocumented)
+interface BlockRequestsOptions {
+ extraUrlPatterns?: string[];
+ urlPatterns?: string[];
+}
+
+// @public
+function clickElements(page: Page, selector: string, clickOptions?: ClickOptions): Promise<void>;
+
+// @public
+function clickElementsAndInterceptNavigationRequests(options: ClickElementsAndInterceptNavigationRequestsOptions): Promise<Dictionary_2[]>;
+
+// @public (undocumented)
+function closeCookieModals(page: Page): Promise<void>;
+
+// @public (undocumented)
+type CompiledScriptFunction = (params: CompiledScriptParams) => Promise<unknown>;
+
+// @public (undocumented)
+interface CompiledScriptParams {
+ // (undocumented)
+ page: Page;
+ // (undocumented)
+ request: Request_3;
+}
+
+// @public
+function compileScript(scriptString: string, context?: Dictionary): CompiledScriptFunction;
+
+// @public (undocumented)
+export function createAdaptivePlaywrightRouter<Context extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext, UserData extends Dictionary_2 = GetUserDataFromRequest_2<Context['request']>>(routes?: RouterRoutes_2<Context, UserData>): RouterHandler_2<Context>;
+
+// @public
+export function createPlaywrightRouter<Context extends PlaywrightCrawlingContext = PlaywrightCrawlingContext, UserData extends Dictionary_2 = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): RouterHandler<Context>;
+
+// @public
+function enqueueLinksByClickingElements(options: EnqueueLinksByClickingElementsOptions): Promise<BatchAddRequestsResult>;
+
+// @public (undocumented)
+interface EnqueueLinksByClickingElementsOptions {
+ clickOptions?: ClickOptions;
+ exclude?: readonly (GlobInput | RegExpInput)[];
+ forefront?: boolean;
+ globs?: GlobInput[];
+ label?: string;
+ maxWaitForPageIdleSecs?: number;
+ onSkippedRequest?: SkippedRequestCallback;
+ page: Page;
+ // @deprecated
+ pseudoUrls?: PseudoUrlInput[];
+ regexps?: RegExpInput[];
+ requestManager: IRequestManager;
+ selector: string;
+ skipNavigation?: boolean;
+ transformRequestFunction?: RequestTransform;
+ userData?: Dictionary_2;
+ waitForPageIdleSecs?: number;
+}
+
+// @public
+function gotoExtended(page: Page, request: Request_3, gotoOptions?: PlaywrightDirectNavigationOptions): Promise<Response_2 | null>;
+
+// @public
+export function handleCloudflareChallengeHook(options?: HandleCloudflareChallengeOptions): PlaywrightHook;
+
+// @public (undocumented)
+interface HandleCloudflareChallengeOptions {
+ clickCallback?: (page: Page, boundingBox: {
+ x: number;
+ y: number;
+ }) => Promise<void>;
+ clickPositionCallback?: (page: Page) => Promise<{
+ x: number;
+ y: number;
+ } | null>;
+ isBlockedCallback?: (page: Page) => Promise<boolean>;
+ isChallengeCallback?: (page: Page) => Promise<boolean>;
+ preChallengeSleepSecs?: number;
+ sleepSecs?: number;
+ verbose?: boolean;
+}
+
+// @public
+function infiniteScroll(page: Page, options?: InfiniteScrollOptions): Promise<void>;
+
+// @public (undocumented)
+interface InfiniteScrollOptions {
+ buttonSelector?: string;
+ maxScrollHeight?: number;
+ scrollDownAndUp?: boolean;
+ stopScrollCallback?: () => unknown | Promise<unknown>;
+ timeoutSecs?: number;
+ waitForSecs?: number;
+}
+
+// @public
+function injectFile(page: Page, filePath: string, options?: InjectFileOptions): Promise<unknown>;
+
+// @public (undocumented)
+interface InjectFileOptions {
+ surviveNavigations?: boolean;
+}
+
+// @public
+function injectJQuery(page: Page, options?: {
+ surviveNavigations?: boolean;
+}): Promise<unknown>;
+
+// @public
+export function launchPlaywright(launchContext?: PlaywrightLaunchContext, config?: Configuration): Promise<Browser>;
+
+// @public
+function parseWithCheerio(page: Page, ignoreShadowRoots?: boolean, ignoreIframes?: boolean): Promise<CheerioRoot>;
+
+declare namespace playwrightClickElements {
+ export {
+ enqueueLinksByClickingElements,
+ clickElementsAndInterceptNavigationRequests,
+ clickElements,
+ EnqueueLinksByClickingElementsOptions
+ }
+}
+
+// @internal (undocumented)
+interface PlaywrightContextUtils {
+ blockRequests(options?: BlockRequestsOptions): Promise<void>;
+ closeCookieModals(): Promise<void>;
+ compileScript(scriptString: string, ctx?: Dictionary): CompiledScriptFunction;
+ enqueueLinksByClickingElements(options: Omit<EnqueueLinksByClickingElementsOptions, 'page' | 'requestManager'>): Promise<BatchAddRequestsResult>;
+ handleCloudflareChallenge(options?: HandleCloudflareChallengeOptions): Promise<Response_2 | undefined>;
+ infiniteScroll(options?: InfiniteScrollOptions): Promise<void>;
+ injectFile(filePath: string, options?: InjectFileOptions): Promise<unknown>;
+ injectJQuery(): Promise<unknown>;
+ parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
+ saveSnapshot(options?: SaveSnapshotOptions): Promise<void>;
+ waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
+}
+
+// @public
+export class PlaywrightCrawler<ContextExtension = Dictionary_2<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension> extends BrowserCrawler<Page, Response_2, {
+ browserPlugins: [PlaywrightPlugin];
+}, LaunchOptions, PlaywrightCrawlingContext, ContextExtension, ExtendedContext> {
+ constructor(options?: PlaywrightCrawlerOptions<ExtendedContext>);
+ // (undocumented)
+ protected buildContextPipeline(): ContextPipeline<CrawlingContext<Dictionary_2>, BrowserCrawlingContext<Page, Response_2, Dictionary_2, Dictionary_2> & {
+ injectFile: (filePath: string, options?: InjectFileOptions) => Promise<unknown>;
+ injectJQuery: () => Promise<void>;
+ blockRequests: (options?: BlockRequestsOptions) => Promise<void>;
+ waitForSelector: (selector: string, timeoutMs?: number) => Promise<void>;
+ parseWithCheerio: (selector?: string, timeoutMs?: number) => Promise<CheerioAPI>;
+ infiniteScroll: (options?: InfiniteScrollOptions) => Promise<void>;
+ saveSnapshot: (options?: SaveSnapshotOptions) => Promise<void>;
+ enqueueLinksByClickingElements: (options: Omit<EnqueueLinksByClickingElementsOptions, "page" | "requestManager">) => Promise<BatchAddRequestsResult>;
+ compileScript: (scriptString: string, ctx?: Dictionary_2) => CompiledScriptFunction;
+ closeCookieModals: () => Promise<void>;
+ handleCloudflareChallenge: (options?: HandleCloudflareChallengeOptions) => Promise<Response_2 | undefined>;
+ }>;
+ // (undocumented)
+ protected _navigationHandler(crawlingContext: PlaywrightCrawlingContext, gotoOptions: PlaywrightDirectNavigationOptions): Promise<Response_2 | null>;
+ // (undocumented)
+ protected static optionsShape: {
+ browserPoolOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ launcher: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ ignoreIframes: BooleanPredicate & BasePredicate<boolean | undefined>;
+ ignoreShadowRoots: BooleanPredicate & BasePredicate<boolean | undefined>;
+ navigationTimeoutSecs: NumberPredicate & BasePredicate<number | undefined>;
+ preNavigationHooks: ArrayPredicate<unknown> & BasePredicate<unknown[] | undefined>;
+ postNavigationHooks: ArrayPredicate<unknown> & BasePredicate<unknown[] | undefined>;
+ launchContext: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ headless: AnyPredicate<string | boolean>;
+ browserPool: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ saveResponseCookies: BooleanPredicate & BasePredicate<boolean | undefined>;
+ proxyConfiguration: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ contextPipelineBuilder: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ extendContext: Predicate<Function> & BasePredicate<Function | undefined>;
+ requestList: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ requestQueue: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ requestHandler: Predicate<Function> & BasePredicate<Function | undefined>;
+ requestHandlerTimeoutSecs: NumberPredicate & BasePredicate<number | undefined>;
+ errorHandler: Predicate<Function> & BasePredicate<Function | undefined>;
+ failedRequestHandler: Predicate<Function> & BasePredicate<Function | undefined>;
+ maxRequestRetries: NumberPredicate & BasePredicate<number | undefined>;
+ sameDomainDelaySecs: NumberPredicate & BasePredicate<number | undefined>;
+ maxRequestsPerCrawl: NumberPredicate & BasePredicate<number | undefined>;
+ maxCrawlDepth: NumberPredicate & BasePredicate<number | undefined>;
+ autoscaledPoolOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ sessionPool: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ statusMessageLoggingInterval: NumberPredicate & BasePredicate<number | undefined>;
+ statusMessageCallback: Predicate<Function> & BasePredicate<Function | undefined>;
+ additionalHttpErrorStatusCodes: ArrayPredicate<number>;
+ ignoreHttpErrorStatusCodes: ArrayPredicate<number>;
+ blockedStatusCodes: ArrayPredicate<number>;
+ retryOnBlocked: BooleanPredicate & BasePredicate<boolean | undefined>;
+ respectRobotsTxtFile: AnyPredicate<boolean | object>;
+ onSkippedRequest: Predicate<Function> & BasePredicate<Function | undefined>;
+ httpClient: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ configuration: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ storageClient: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ eventManager: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ logger: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ minConcurrency: NumberPredicate & BasePredicate<number | undefined>;
+ maxConcurrency: NumberPredicate & BasePredicate<number | undefined>;
+ maxRequestsPerMinute: NumberPredicate & BasePredicate<number | undefined>;
+ keepAlive: BooleanPredicate & BasePredicate<boolean | undefined>;
+ statisticsOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ id: StringPredicate & BasePredicate<string | undefined>;
+ };
+}
+
+// @public (undocumented)
+export interface PlaywrightCrawlerOptions<ContextExtension = Dictionary_2<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension> extends BrowserCrawlerOptions<Page, Response_2, PlaywrightCrawlingContext, ContextExtension, ExtendedContext, {
+ browserPlugins: [PlaywrightPlugin];
+}> {
+ launchContext?: PlaywrightLaunchContext;
+ postNavigationHooks?: PlaywrightHook[];
+ preNavigationHooks?: PlaywrightHook[];
+ requestHandler?: RequestHandler<ExtendedContext>;
+}
+
+// @public (undocumented)
+export interface PlaywrightCrawlingContext<UserData extends Dictionary_2 = Dictionary_2> extends BrowserCrawlingContext<Page, Response_2, UserData, PlaywrightGotoOptions>, PlaywrightContextUtils {
+}
+
+// @public (undocumented)
+export interface PlaywrightDirectNavigationOptions {
+ referer?: string;
+ timeout?: number;
+ waitUntil?: 'domcontentloaded' | 'load' | 'networkidle';
+}
+
+// @public (undocumented)
+export type PlaywrightGotoOptions = NonNullable<Parameters<Page['goto']>[1]>;
+
+// @public (undocumented)
+export interface PlaywrightHook extends BrowserHook<PlaywrightCrawlingContext> {
+}
+
+// @public
+export interface PlaywrightLaunchContext extends BrowserLaunchContext<LaunchOptions, BrowserType> {
+ launcher?: BrowserType;
+ launchOptions?: LaunchOptions & Parameters<BrowserType['launchPersistentContext']>[1];
+ proxyUrl?: string;
+ useChrome?: boolean;
+ useIncognitoPages?: boolean;
+ userDataDir?: string;
+}
+
+// @public
+export class PlaywrightLauncher extends BrowserLauncher<PlaywrightPlugin> {
+ constructor(launchContext?: PlaywrightLaunchContext, config?: Configuration);
+ // (undocumented)
+ readonly config: Configuration;
+ // (undocumented)
+ protected static optionsShape: {
+ launcher: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ launchContextOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ proxyUrl: StringPredicate & BasePredicate<string | undefined>;
+ useChrome: BooleanPredicate & BasePredicate<boolean | undefined>;
+ useIncognitoPages: BooleanPredicate & BasePredicate<boolean | undefined>;
+ browserPerProxy: BooleanPredicate & BasePredicate<boolean | undefined>;
+ ignoreProxyCertificate: BooleanPredicate & BasePredicate<boolean | undefined>;
+ userDataDir: StringPredicate & BasePredicate<string | undefined>;
+ launchOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ userAgent: StringPredicate & BasePredicate<string | undefined>;
+ };
+}
+
+declare namespace playwrightUtils {
+ export {
+ injectFile,
+ injectJQuery,
+ gotoExtended,
+ blockRequests,
+ compileScript,
+ infiniteScroll,
+ saveSnapshot,
+ parseWithCheerio,
+ closeCookieModals,
+ InjectFileOptions,
+ BlockRequestsOptions,
+ PlaywrightDirectNavigationOptions as DirectNavigationOptions,
+ CompiledScriptParams,
+ CompiledScriptFunction,
+ InfiniteScrollOptions,
+ SaveSnapshotOptions,
+ HandleCloudflareChallengeOptions,
+ PlaywrightContextUtils,
+ enqueueLinksByClickingElements,
+ playwrightUtils_2 as playwrightUtils
+ }
+}
+
+// @internal (undocumented)
+const playwrightUtils_2: {
+ injectFile: typeof injectFile;
+ injectJQuery: typeof injectJQuery;
+ gotoExtended: typeof gotoExtended;
+ blockRequests: typeof blockRequests;
+ enqueueLinksByClickingElements: typeof enqueueLinksByClickingElements;
+ parseWithCheerio: typeof parseWithCheerio;
+ infiniteScroll: typeof infiniteScroll;
+ saveSnapshot: typeof saveSnapshot;
+ compileScript: typeof compileScript;
+ closeCookieModals: typeof closeCookieModals;
+ RenderingTypePredictor: typeof RenderingTypePredictor;
+ handleCloudflareChallenge: typeof handleCloudflareChallenge;
+};
+
+// @public (undocumented)
+export type RenderingType = 'clientOnly' | 'static';
+
+// @public
+export class RenderingTypePredictor {
+ constructor(input: RenderingTypePredictorOptions);
+ // (undocumented)
+ protected calculateFeatureVector(url: URLComponents, label: string | undefined): FeatureVector;
+ initialize(): Promise<void>;
+ predict(input: Request_2): {
+ renderingType: RenderingType;
+ detectionProbabilityRecommendation: number;
+ };
+ // (undocumented)
+ protected retrain(): void;
+ storeResult(requests: Request_2 | Request_2[], renderingType: RenderingType): void;
+}
+
+// @public
+function saveSnapshot(page: Page, options?: SaveSnapshotOptions): Promise<void>;
+
+// @public (undocumented)
+interface SaveSnapshotOptions {
+ config?: Configuration;
+ key?: string;
+ keyValueStoreName?: string | null;
+ saveHtml?: boolean;
+ saveScreenshot?: boolean;
+ screenshotQuality?: number;
+}
+
+
+export * from "@crawlee/browser";
+
+// (No @packageDocumentation comment for this package)
+
+```
diff --git a/docs/public-api/crawlee-puppeteer.api.md b/docs/public-api/crawlee-puppeteer.api.md
new file mode 100644
index 000000000..75c9c3bd8
--- /dev/null
+++ b/docs/public-api/crawlee-puppeteer.api.md
@@ -0,0 +1,387 @@
+## API Report File for "@crawlee/puppeteer"
+
+> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
+
+```ts
+
+import { AnyPredicate } from 'ow';
+import { ArrayPredicate } from 'ow';
+import { BasePredicate } from 'ow';
+import { BatchAddRequestsResult } from '@crawlee/types';
+import { BooleanPredicate } from 'ow';
+import type { Browser } from 'puppeteer';
+import { BrowserCrawler } from '@crawlee/browser';
+import type { BrowserCrawlerOptions } from '@crawlee/browser';
+import type { BrowserCrawlingContext } from '@crawlee/browser';
+import type { BrowserHook } from '@crawlee/browser';
+import type { BrowserLaunchContext } from '@crawlee/browser';
+import { BrowserLauncher } from '@crawlee/browser';
+import { CheerioAPI } from '@crawlee/browser';
+import { CheerioRoot } from '@crawlee/utils';
+import type { ClickOptions } from 'puppeteer';
+import { Configuration } from '@crawlee/browser';
+import { ContextPipeline } from '@crawlee/browser';
+import { CrawlingContext } from '@crawlee/browser';
+import type { Dictionary } from '@crawlee/types';
+import type { GetUserDataFromRequest } from '@crawlee/browser';
+import type { GlobInput } from '@crawlee/browser';
+import type { HTTPRequest } from 'puppeteer';
+import type { HTTPResponse } from 'puppeteer';
+import { IRequestManager } from '@crawlee/browser';
+import type { LaunchOptions } from 'puppeteer';
+import { NumberPredicate } from 'ow';
+import { ObjectPredicate } from 'ow';
+import type { Page } from 'puppeteer';
+import { Predicate } from 'ow';
+import type { ProtocolMapping } from 'devtools-protocol/types/protocol-mapping.js';
+import type { PseudoUrlInput } from '@crawlee/browser';
+import { PuppeteerPlugin } from '@crawlee/browser-pool';
+import type { RegExpInput } from '@crawlee/browser';
+import { Request as Request_2 } from '@crawlee/browser';
+import type { RequestTransform } from '@crawlee/browser';
+import type { ResponseForRequest } from 'puppeteer';
+import { RouterHandler } from '@crawlee/browser';
+import type { RouterRoutes } from '@crawlee/browser';
+import type { SkippedRequestCallback } from '@crawlee/browser';
+import { StringPredicate } from 'ow';
+import type { Target } from 'puppeteer';
+
+// @public
+function addInterceptRequestHandler(page: Page, handler: InterceptHandler): Promise<void>;
+
+// @public
+function blockRequests(page: Page, options?: BlockRequestsOptions): Promise<void>;
+
+// @public (undocumented)
+export interface BlockRequestsOptions {
+ extraUrlPatterns?: string[];
+ urlPatterns?: string[];
+}
+
+// @public @deprecated
+const blockResources: (page: Page, resourceTypes?: string[]) => Promise<void>;
+
+// @public @deprecated
+function cacheResponses(page: Page, cache: Dictionary<Partial<ResponseForRequest>>, responseUrlRules: (string | RegExp)[]): Promise<void>;
+
+// @public
+function clickElements(page: Page, selector: string, clickOptions?: ClickOptions): Promise<void>;
+
+// @public
+function clickElementsAndInterceptNavigationRequests(options: ClickElementsAndInterceptNavigationRequestsOptions): Promise<Dictionary[]>;
+
+// @public (undocumented)
+function closeCookieModals(page: Page): Promise<void>;
+
+// @public (undocumented)
+export type CompiledScriptFunction = (params: CompiledScriptParams) => Promise<unknown>;
+
+// @public (undocumented)
+export interface CompiledScriptParams {
+ // (undocumented)
+ page: Page;
+ // (undocumented)
+ request: Request_2;
+}
+
+// @public
+function compileScript(scriptString: string, context?: Dictionary): CompiledScriptFunction;
+
+// @public
+export function createPuppeteerRouter<Context extends PuppeteerCrawlingContext = PuppeteerCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): RouterHandler<Context>;
+
+// @public
+function enqueueLinksByClickingElements(options: EnqueueLinksByClickingElementsOptions): Promise<BatchAddRequestsResult>;
+
+// @public (undocumented)
+export interface EnqueueLinksByClickingElementsOptions {
+ clickOptions?: ClickOptions;
+ exclude?: readonly (GlobInput | RegExpInput)[];
+ forefront?: boolean;
+ globs?: GlobInput[];
+ label?: string;
+ maxWaitForPageIdleSecs?: number;
+ onSkippedRequest?: SkippedRequestCallback;
+ page: Page;
+ // @deprecated
+ pseudoUrls?: PseudoUrlInput[];
+ regexps?: RegExpInput[];
+ requestManager: IRequestManager;
+ selector: string;
+ skipNavigation?: boolean;
+ transformRequestFunction?: RequestTransform;
+ userData?: Dictionary;
+ waitForPageIdleSecs?: number;
+}
+
+// @public
+function gotoExtended(page: Page, request: Request_2, gotoOptions?: PuppeteerDirectNavigationOptions): Promise<HTTPResponse | null>;
+
+// @public
+function infiniteScroll(page: Page, options?: InfiniteScrollOptions): Promise<void>;
+
+// @public (undocumented)
+export interface InfiniteScrollOptions {
+ buttonSelector?: string;
+ maxScrollHeight?: number;
+ scrollDownAndUp?: boolean;
+ stopScrollCallback?: () => unknown | Promise<unknown>;
+ timeoutSecs?: number;
+ waitForSecs?: number;
+}
+
+// @public
+function injectFile(page: Page, filePath: string, options?: InjectFileOptions): Promise<unknown>;
+
+// @public (undocumented)
+export interface InjectFileOptions {
+ surviveNavigations?: boolean;
+}
+
+// @public
+function injectJQuery(page: Page, options?: {
+ surviveNavigations?: boolean;
+}): Promise<unknown>;
+
+// @public (undocumented)
+export type InterceptHandler = (request: HTTPRequest) => unknown;
+
+// @public
+function isTargetRelevant(page: Page, target: Target): boolean;
+
+// @public
+export function launchPuppeteer(launchContext?: PuppeteerLaunchContext, config?: Configuration): Promise<Browser>;
+
+// @public
+function parseWithCheerio(page: Page, ignoreShadowRoots?: boolean, ignoreIframes?: boolean): Promise<CheerioRoot>;
+
+declare namespace puppeteerClickElements {
+ export {
+ enqueueLinksByClickingElements,
+ clickElementsAndInterceptNavigationRequests,
+ isTargetRelevant,
+ clickElements,
+ EnqueueLinksByClickingElementsOptions
+ }
+}
+
+// @internal (undocumented)
+interface PuppeteerContextUtils {
+ addInterceptRequestHandler(handler: InterceptHandler): Promise<void>;
+ blockRequests(options?: BlockRequestsOptions): Promise<void>;
+ closeCookieModals(): Promise<void>;
+ compileScript(scriptString: string, ctx?: Dictionary): CompiledScriptFunction;
+ enqueueLinksByClickingElements(options: Omit<EnqueueLinksByClickingElementsOptions, 'page' | 'requestManager'>): Promise<BatchAddRequestsResult>;
+ infiniteScroll(options?: InfiniteScrollOptions): Promise<void>;
+ injectFile(filePath: string, options?: InjectFileOptions): Promise<unknown>;
+ injectJQuery(): Promise<unknown>;
+ parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
+ removeInterceptRequestHandler(handler: InterceptHandler): Promise<void>;
+ saveSnapshot(options?: SaveSnapshotOptions): Promise<void>;
+ waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
+}
+
+// @public
+export class PuppeteerCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension> extends BrowserCrawler<Page, HTTPResponse, {
+ browserPlugins: [PuppeteerPlugin];
+}, LaunchOptions, PuppeteerCrawlingContext, ContextExtension, ExtendedContext> {
+ constructor(options?: PuppeteerCrawlerOptions<ContextExtension, ExtendedContext>);
+ // (undocumented)
+ protected buildContextPipeline(): ContextPipeline<CrawlingContext<Dictionary>, BrowserCrawlingContext<Page, HTTPResponse, Dictionary, Dictionary> & {
+ injectFile: (filePath: string, options?: InjectFileOptions) => Promise<unknown>;
+ injectJQuery: () => Promise<void>;
+ waitForSelector: (selector: string, timeoutMs?: number) => Promise<void>;
+ parseWithCheerio: (selector?: string, timeoutMs?: number) => Promise<CheerioAPI>;
+ enqueueLinksByClickingElements: (options: Omit<EnqueueLinksByClickingElementsOptions, "page" | "requestManager">) => Promise<BatchAddRequestsResult>;
+ blockRequests: (options?: BlockRequestsOptions) => Promise<void>;
+ compileScript: (scriptString: string, ctx?: Dictionary) => CompiledScriptFunction;
+ addInterceptRequestHandler: (handler: InterceptHandler) => Promise<void>;
+ removeInterceptRequestHandler: (handler: InterceptHandler) => Promise<void>;
+ infiniteScroll: (options?: InfiniteScrollOptions) => Promise<void>;
+ saveSnapshot: (options?: SaveSnapshotOptions) => Promise<void>;
+ closeCookieModals: () => Promise<void>;
+ }>;
+ // (undocumented)
+ protected _navigationHandler(crawlingContext: PuppeteerCrawlingContext, gotoOptions: PuppeteerDirectNavigationOptions): Promise<HTTPResponse | null>;
+ // (undocumented)
+ protected static optionsShape: {
+ browserPoolOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ navigationTimeoutSecs: NumberPredicate & BasePredicate<number | undefined>;
+ preNavigationHooks: ArrayPredicate<unknown> & BasePredicate<unknown[] | undefined>;
+ postNavigationHooks: ArrayPredicate<unknown> & BasePredicate<unknown[] | undefined>;
+ launchContext: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ headless: AnyPredicate<string | boolean>;
+ browserPool: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ saveResponseCookies: BooleanPredicate & BasePredicate<boolean | undefined>;
+ proxyConfiguration: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ contextPipelineBuilder: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ extendContext: Predicate<Function> & BasePredicate<Function | undefined>;
+ requestList: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ requestQueue: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ requestHandler: Predicate<Function> & BasePredicate<Function | undefined>;
+ requestHandlerTimeoutSecs: NumberPredicate & BasePredicate<number | undefined>;
+ errorHandler: Predicate<Function> & BasePredicate<Function | undefined>;
+ failedRequestHandler: Predicate<Function> & BasePredicate<Function | undefined>;
+ maxRequestRetries: NumberPredicate & BasePredicate<number | undefined>;
+ sameDomainDelaySecs: NumberPredicate & BasePredicate<number | undefined>;
+ maxRequestsPerCrawl: NumberPredicate & BasePredicate<number | undefined>;
+ maxCrawlDepth: NumberPredicate & BasePredicate<number | undefined>;
+ autoscaledPoolOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ sessionPool: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ statusMessageLoggingInterval: NumberPredicate & BasePredicate<number | undefined>;
+ statusMessageCallback: Predicate<Function> & BasePredicate<Function | undefined>;
+ additionalHttpErrorStatusCodes: ArrayPredicate<number>;
+ ignoreHttpErrorStatusCodes: ArrayPredicate<number>;
+ blockedStatusCodes: ArrayPredicate<number>;
+ retryOnBlocked: BooleanPredicate & BasePredicate<boolean | undefined>;
+ respectRobotsTxtFile: AnyPredicate<boolean | object>;
+ onSkippedRequest: Predicate<Function> & BasePredicate<Function | undefined>;
+ httpClient: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ configuration: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ storageClient: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ eventManager: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ logger: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ minConcurrency: NumberPredicate & BasePredicate<number | undefined>;
+ maxConcurrency: NumberPredicate & BasePredicate<number | undefined>;
+ maxRequestsPerMinute: NumberPredicate & BasePredicate<number | undefined>;
+ keepAlive: BooleanPredicate & BasePredicate<boolean | undefined>;
+ statisticsOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ id: StringPredicate & BasePredicate<string | undefined>;
+ };
+}
+
+// @public (undocumented)
+export interface PuppeteerCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension> extends BrowserCrawlerOptions<Page, HTTPResponse, PuppeteerCrawlingContext, ContextExtension, ExtendedContext, {
+ browserPlugins: [PuppeteerPlugin];
+}> {
+ launchContext?: PuppeteerLaunchContext;
+ postNavigationHooks?: PuppeteerHook[];
+ preNavigationHooks?: PuppeteerHook[];
+}
+
+// @public (undocumented)
+export interface PuppeteerCrawlingContext<UserData extends Dictionary = Dictionary> extends BrowserCrawlingContext<Page, HTTPResponse, UserData, PuppeteerGoToOptions>, PuppeteerContextUtils {
+}
+
+// @public (undocumented)
+export interface PuppeteerDirectNavigationOptions {
+ referer?: string;
+ timeout?: number;
+ waitUntil?: 'domcontentloaded' | 'load' | 'networkidle' | 'networkidle0' | 'networkidle2';
+}
+
+// @public (undocumented)
+export type PuppeteerGoToOptions = NonNullable<Parameters<Page['goto']>[1]>;
+
+// @public (undocumented)
+export interface PuppeteerHook extends BrowserHook<PuppeteerCrawlingContext> {
+}
+
+// @public
+export interface PuppeteerLaunchContext extends BrowserLaunchContext<PuppeteerPlugin['launchOptions'], unknown> {
+ launcher?: unknown;
+ launchOptions?: PuppeteerPlugin['launchOptions'];
+ proxyUrl?: string;
+ useChrome?: boolean;
+ useIncognitoPages?: boolean;
+}
+
+// @public
+export class PuppeteerLauncher extends BrowserLauncher<PuppeteerPlugin, unknown> {
+ constructor(launchContext?: PuppeteerLaunchContext, config?: Configuration);
+ // (undocumented)
+ readonly config: Configuration;
+ // (undocumented)
+ protected _getDefaultHeadlessOption(): boolean;
+ // (undocumented)
+ protected static optionsShape: {
+ launcher: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ proxyUrl: StringPredicate & BasePredicate<string | undefined>;
+ useChrome: BooleanPredicate & BasePredicate<boolean | undefined>;
+ useIncognitoPages: BooleanPredicate & BasePredicate<boolean | undefined>;
+ browserPerProxy: BooleanPredicate & BasePredicate<boolean | undefined>;
+ ignoreProxyCertificate: BooleanPredicate & BasePredicate<boolean | undefined>;
+ userDataDir: StringPredicate & BasePredicate<string | undefined>;
+ launchOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ userAgent: StringPredicate & BasePredicate<string | undefined>;
+ };
+}
+
+declare namespace puppeteerRequestInterception {
+ export {
+ addInterceptRequestHandler,
+ removeInterceptRequestHandler,
+ InterceptHandler
+ }
+}
+
+declare namespace puppeteerUtils {
+ export {
+ injectFile,
+ injectJQuery,
+ parseWithCheerio,
+ blockRequests,
+ sendCDPCommand,
+ cacheResponses,
+ compileScript,
+ gotoExtended,
+ infiniteScroll,
+ saveSnapshot,
+ closeCookieModals,
+ PuppeteerDirectNavigationOptions as DirectNavigationOptions,
+ InjectFileOptions,
+ BlockRequestsOptions,
+ CompiledScriptParams,
+ CompiledScriptFunction,
+ blockResources,
+ InfiniteScrollOptions,
+ SaveSnapshotOptions,
+ PuppeteerContextUtils,
+ enqueueLinksByClickingElements,
+ addInterceptRequestHandler,
+ removeInterceptRequestHandler,
+ puppeteerUtils_2 as puppeteerUtils
+ }
+}
+
+// @internal (undocumented)
+const puppeteerUtils_2: {
+ injectFile: typeof injectFile;
+ injectJQuery: typeof injectJQuery;
+ enqueueLinksByClickingElements: typeof enqueueLinksByClickingElements;
+ blockRequests: typeof blockRequests;
+ compileScript: typeof compileScript;
+ gotoExtended: typeof gotoExtended;
+ addInterceptRequestHandler: typeof addInterceptRequestHandler;
+ removeInterceptRequestHandler: typeof removeInterceptRequestHandler;
+ infiniteScroll: typeof infiniteScroll;
+ saveSnapshot: typeof saveSnapshot;
+ parseWithCheerio: typeof parseWithCheerio;
+ closeCookieModals: typeof closeCookieModals;
+};
+
+// @public
+function removeInterceptRequestHandler(page: Page, handler: InterceptHandler): Promise<void>;
+
+// @public
+function saveSnapshot(page: Page, options?: SaveSnapshotOptions): Promise<void>;
+
+// @public (undocumented)
+export interface SaveSnapshotOptions {
+ config?: Configuration;
+ key?: string;
+ keyValueStoreName?: string | null;
+ saveHtml?: boolean;
+ saveScreenshot?: boolean;
+ screenshotQuality?: number;
+}
+
+// @internal (undocumented)
+function sendCDPCommand<T extends keyof ProtocolMapping.Commands>(page: Page, command: T, ...args: ProtocolMapping.Commands[T]['paramsType']): Promise<ProtocolMapping.Commands[T]['returnType']>;
+
+
+export * from "@crawlee/browser";
+
+// (No @packageDocumentation comment for this package)
+
+```
diff --git a/docs/public-api/crawlee-stagehand.api.md b/docs/public-api/crawlee-stagehand.api.md
new file mode 100644
index 000000000..fda999e31
--- /dev/null
+++ b/docs/public-api/crawlee-stagehand.api.md
@@ -0,0 +1,210 @@
+## API Report File for "@crawlee/stagehand"
+
+> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
+
+```ts
+
+import { Action } from '@browserbasehq/stagehand';
+import { ActOptions } from '@browserbasehq/stagehand';
+import { ActResult } from '@browserbasehq/stagehand';
+import { AgentConfig } from '@browserbasehq/stagehand';
+import { AgentResult } from '@browserbasehq/stagehand';
+import { AnyPredicate } from 'ow';
+import { ArrayPredicate } from 'ow';
+import { BasePredicate } from 'ow';
+import { BooleanPredicate } from 'ow';
+import type { Browser } from 'playwright';
+import type { BrowserController } from '@crawlee/browser-pool';
+import { BrowserCrawler } from '@crawlee/browser';
+import type { BrowserCrawlerOptions } from '@crawlee/browser';
+import type { BrowserCrawlingContext } from '@crawlee/browser';
+import type { BrowserHook } from '@crawlee/browser';
+import type { BrowserLaunchContext } from '@crawlee/browser';
+import { BrowserPlugin } from '@crawlee/browser-pool';
+import type { BrowserPluginOptions } from '@crawlee/browser-pool';
+import type { BrowserType } from 'playwright';
+import type { ContextPipeline } from '@crawlee/browser';
+import type { CrawlingContext } from '@crawlee/browser';
+import type { Dictionary } from '@crawlee/types';
+import { ExtractOptions } from '@browserbasehq/stagehand';
+import type { GetUserDataFromRequest } from '@crawlee/browser';
+import type { LaunchContext } from '@crawlee/browser-pool';
+import type { LaunchOptions } from 'playwright';
+import type { LLMClient } from '@browserbasehq/stagehand';
+import type { LoadedContext } from '@crawlee/browser';
+import { ModelConfiguration } from '@browserbasehq/stagehand';
+import type { NonStreamingAgentInstance } from '@browserbasehq/stagehand';
+import { NumberPredicate } from 'ow';
+import { ObjectPredicate } from 'ow';
+import { ObserveOptions } from '@browserbasehq/stagehand';
+import type { Page } from 'playwright';
+import { Predicate } from 'ow';
+import type { RequestHandler } from '@crawlee/browser';
+import type { Response as Response_2 } from 'playwright';
+import { RouterHandler } from '@crawlee/browser';
+import type { RouterRoutes } from '@crawlee/browser';
+import { Stagehand } from '@browserbasehq/stagehand';
+import type { StreamingAgentInstance } from '@browserbasehq/stagehand';
+import { StringPredicate } from 'ow';
+import type { z } from 'zod';
+
+export { Action }
+
+export { ActOptions }
+
+export { ActResult }
+
+export { AgentConfig }
+
+export { AgentResult }
+
+// @public
+export function createStagehandRouter<Context extends StagehandCrawlingContext = StagehandCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): RouterHandler<Context>;
+
+// @public
+function enhancePageWithStagehand(page: Page, stagehand: Stagehand): StagehandPage;
+
+export { ExtractOptions }
+
+export { ModelConfiguration }
+
+export { ObserveOptions }
+
+export { Stagehand }
+
+// @public
+export class StagehandCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends StagehandCrawlingContext = StagehandCrawlingContext & ContextExtension> extends BrowserCrawler<StagehandPage, Response_2, {
+ browserPlugins: [StagehandPlugin];
+}, LaunchOptions, StagehandCrawlingContext, ContextExtension, ExtendedContext> {
+ constructor(options?: StagehandCrawlerOptions<ContextExtension, ExtendedContext>);
+ // (undocumented)
+ protected buildContextPipeline(): ContextPipeline<CrawlingContext, StagehandCrawlingContext>;
+ protected _navigationHandler(crawlingContext: StagehandCrawlingContext, gotoOptions: StagehandGotoOptions): Promise<Response_2 | null>;
+ // (undocumented)
+ protected static optionsShape: {
+ stagehandOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ browserPoolOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ navigationTimeoutSecs: NumberPredicate & BasePredicate<number | undefined>;
+ preNavigationHooks: ArrayPredicate<unknown> & BasePredicate<unknown[] | undefined>;
+ postNavigationHooks: ArrayPredicate<unknown> & BasePredicate<unknown[] | undefined>;
+ launchContext: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ headless: AnyPredicate<string | boolean>;
+ browserPool: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ saveResponseCookies: BooleanPredicate & BasePredicate<boolean | undefined>;
+ proxyConfiguration: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ contextPipelineBuilder: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ extendContext: Predicate<Function> & BasePredicate<Function | undefined>;
+ requestList: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ requestQueue: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ requestHandler: Predicate<Function> & BasePredicate<Function | undefined>;
+ requestHandlerTimeoutSecs: NumberPredicate & BasePredicate<number | undefined>;
+ errorHandler: Predicate<Function> & BasePredicate<Function | undefined>;
+ failedRequestHandler: Predicate<Function> & BasePredicate<Function | undefined>;
+ maxRequestRetries: NumberPredicate & BasePredicate<number | undefined>;
+ sameDomainDelaySecs: NumberPredicate & BasePredicate<number | undefined>;
+ maxRequestsPerCrawl: NumberPredicate & BasePredicate<number | undefined>;
+ maxCrawlDepth: NumberPredicate & BasePredicate<number | undefined>;
+ autoscaledPoolOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ sessionPool: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ statusMessageLoggingInterval: NumberPredicate & BasePredicate<number | undefined>;
+ statusMessageCallback: Predicate<Function> & BasePredicate<Function | undefined>;
+ additionalHttpErrorStatusCodes: ArrayPredicate<number>;
+ ignoreHttpErrorStatusCodes: ArrayPredicate<number>;
+ blockedStatusCodes: ArrayPredicate<number>;
+ retryOnBlocked: BooleanPredicate & BasePredicate<boolean | undefined>;
+ respectRobotsTxtFile: AnyPredicate<boolean | object>;
+ onSkippedRequest: Predicate<Function> & BasePredicate<Function | undefined>;
+ httpClient: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ configuration: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ storageClient: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ eventManager: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ logger: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ minConcurrency: NumberPredicate & BasePredicate<number | undefined>;
+ maxConcurrency: NumberPredicate & BasePredicate<number | undefined>;
+ maxRequestsPerMinute: NumberPredicate & BasePredicate<number | undefined>;
+ keepAlive: BooleanPredicate & BasePredicate<boolean | undefined>;
+ statisticsOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;
+ id: StringPredicate & BasePredicate<string | undefined>;
+ };
+}
+
+// @public
+export interface StagehandCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends StagehandCrawlingContext = StagehandCrawlingContext & ContextExtension> extends BrowserCrawlerOptions<StagehandPage, Response_2, StagehandCrawlingContext, ContextExtension, ExtendedContext, {
+ browserPlugins: [StagehandPlugin];
+}> {
+ launchContext?: StagehandLaunchContext;
+ postNavigationHooks?: StagehandHook[];
+ preNavigationHooks?: StagehandHook[];
+ requestHandler?: StagehandRequestHandler;
+ stagehandOptions?: StagehandOptions;
+}
+
+// @public (undocumented)
+export interface StagehandCrawlingContext<UserData extends Dictionary = Dictionary> extends BrowserCrawlingContext<StagehandPage, Response_2, UserData, StagehandGotoOptions> {
+ page: StagehandPage;
+ stagehand: Stagehand;
+}
+
+// @public
+export type StagehandGotoOptions = NonNullable<Parameters<Page['goto']>[1]>;
+
+// @public
+export interface StagehandHook extends BrowserHook<StagehandCrawlingContext> {
+}
+
+// @public
+export interface StagehandLaunchContext extends BrowserLaunchContext<LaunchOptions, BrowserType> {
+ launcher?: BrowserType;
+ launchOptions?: LaunchOptions & Parameters<BrowserType['launchPersistentContext']>[1];
+ proxyUrl?: string;
+ stagehandOptions?: StagehandOptions;
+ useChrome?: boolean;
+ useIncognitoPages?: boolean;
+ userDataDir?: string;
+}
+
+// @public
+export interface StagehandOptions {
+ apiKey?: string;
+ cacheDir?: string;
+ domSettleTimeout?: number;
+ env?: 'LOCAL' | 'BROWSERBASE';
+ llmClient?: LLMClient;
+ logInferenceToFile?: boolean;
+ model?: ModelConfiguration;
+ projectId?: string;
+ selfHeal?: boolean;
+ systemPrompt?: string;
+ verbose?: 0 | 1 | 2;
+}
+
+// @public
+export interface StagehandPage extends Page {
+ act(instruction: string, options?: Omit<ActOptions, 'page'>): Promise<ActResult>;
+ agent(config: AgentConfig & {
+ stream: true;
+ }): StreamingAgentInstance;
+ // (undocumented)
+ agent(config?: AgentConfig & {
+ stream?: false;
+ }): NonStreamingAgentInstance;
+ extract<T>(instruction: string, schema: z.ZodType<T>, options?: Omit<ExtractOptions, 'page'>): Promise<T>;
+ observe(options?: Omit<ObserveOptions, 'page'>): Promise<Action[]>;
+}
+
+// @public
+export interface StagehandRequestHandler extends RequestHandler<LoadedContext<StagehandCrawlingContext>> {
+}
+
+declare namespace stagehandUtils {
+ export {
+ enhancePageWithStagehand
+ }
+}
+
+
+export * from "@crawlee/browser";
+
+// (No @packageDocumentation comment for this package)
+
+```
diff --git a/docs/public-api/crawlee-types.api.md b/docs/public-api/crawlee-types.api.md
new file mode 100644
index 000000000..76819e4fe
--- /dev/null
+++ b/docs/public-api/crawlee-types.api.md
@@ -0,0 +1,587 @@
+## API Report File for "@crawlee/types"
+
+> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
+
+```ts
+
+import type { CookieJar } from 'tough-cookie';
+import type { Readable } from 'node:stream';
+import type { SerializedCookieJar } from 'tough-cookie';
+
+// @public (undocumented)
+export type AllowedHttpMethods = 'GET' | 'HEAD' | 'POST' | 'PUT' | 'DELETE' | 'TRACE' | 'OPTIONS' | 'CONNECT' | 'PATCH' | 'get' | 'head' | 'post' | 'put' | 'delete' | 'trace' | 'options' | 'connect' | 'patch';
+
+// @public (undocumented)
+export type Awaitable<T> = T | PromiseLike<T>;
+
+// @public
+export interface BaseHttpClient {
+ sendRequest(request: Request, options?: SendRequestOptions): Promise<Response>;
+}
+
+// @public (undocumented)
+export interface BatchAddRequestsResult {
+ // (undocumented)
+ processedRequests: ProcessedRequest[];
+ // (undocumented)
+ unprocessedRequests: UnprocessedRequest[];
+}
+
+// @public (undocumented)
+export interface BrowserLikeResponse {
+ // (undocumented)
+ headers(): Dictionary<string | string[]>;
+ // (undocumented)
+ url(): string;
+}
+
+// @public (undocumented)
+export type Constructor<T = unknown> = new (...args: any[]) => T;
+
+// @public (undocumented)
+export interface Cookie {
+ domain?: string;
+ expires?: number;
+ httpOnly?: boolean;
+ name: string;
+ path?: string;
+ priority?: 'Low' | 'Medium' | 'High';
+ sameParty?: boolean;
+ sameSite?: 'Strict' | 'Lax' | 'None';
+ secure?: boolean;
+ sourcePort?: number;
+ sourceScheme?: 'Unset' | 'NonSecure' | 'Secure';
+ url?: string;
+ value: string;
+}
+
+// @public
+export interface CrawleeLogger {
+ child(options: Partial<CrawleeLoggerOptions>): CrawleeLogger;
+ debug(message: string, data?: Record<string, unknown>): void;
+ deprecated(message: string): void;
+ error(message: string, data?: Record<string, unknown>): void;
+ exception(exception: Error, message: string, data?: Record<string, unknown>): void;
+ getOptions(): CrawleeLoggerOptions;
+ info(message: string, data?: Record<string, unknown>): void;
+ logWithLevel(level: number, message: string, data?: Record<string, unknown>): void;
+ perf(message: string, data?: Record<string, unknown>): void;
+ setOptions(options: Partial<CrawleeLoggerOptions>): void;
+ softFail(message: string, data?: Record<string, unknown>): void;
+ warning(message: string, data?: Record<string, unknown>): void;
+ warningOnce(message: string): void;
+}
+
+// @public
+export interface CrawleeLoggerOptions {
+ prefix?: string | null;
+}
+
+// @public
+export type CreateDatasetClientOptions = StorageIdentifier;
+
+// @public
+export type CreateKeyValueStoreClientOptions = StorageIdentifier;
+
+// @public
+export type CreateRequestQueueClientOptions = StorageIdentifier & {
+ clientKey?: string;
+ timeoutSecs?: number;
+};
+
+// @public (undocumented)
+export interface DatasetClient<Data extends Dictionary = Dictionary> {
+ drop(): Promise<void>;
+ getData(options?: DatasetClientListOptions): Promise<PaginatedList<Data>>;
+ getMetadata(): Promise<DatasetInfo>;
+ purge(): Promise<void>;
+ pushData(items: Data[]): Promise<void>;
+}
+
+// @public (undocumented)
+export interface DatasetClientListOptions {
+ // (undocumented)
+ desc?: boolean;
+ // (undocumented)
+ limit?: number;
+ // (undocumented)
+ offset?: number;
+}
+
+// @public (undocumented)
+export interface DatasetInfo {
+ // (undocumented)
+ accessedAt: Date;
+ // (undocumented)
+ actId?: string;
+ // (undocumented)
+ actRunId?: string;
+ // (undocumented)
+ createdAt: Date;
+ // (undocumented)
+ id: string;
+ // (undocumented)
+ itemCount: number;
+ // (undocumented)
+ modifiedAt: Date;
+ // (undocumented)
+ name?: string;
+}
+
+// @public (undocumented)
+export interface DatasetStats {
+ // (undocumented)
+ deleteCount?: number;
+ // (undocumented)
+ readCount?: number;
+ // (undocumented)
+ storageBytes?: number;
+ // (undocumented)
+ writeCount?: number;
+}
+
+// @public (undocumented)
+export type Dictionary<T = any> = Record<PropertyKey, T>;
+
+// @public
+export interface HttpRequest {
+ // (undocumented)
+ body?: Readable;
+ // (undocumented)
+ cookieJar?: CookieJar;
+ // (undocumented)
+ encoding?: BufferEncoding;
+ // (undocumented)
+ followRedirect?: boolean | ((response: any) => boolean);
+ // (undocumented)
+ headerGenerator?: {
+ getHeaders: (options: Record<string, unknown>) => Record<string, string>;
+ };
+ // (undocumented)
+ headerGeneratorOptions?: Record<string, unknown>;
+ // (undocumented)
+ headers?: Headers;
+ // (undocumented)
+ insecureHTTPParser?: boolean;
+ // (undocumented)
+ maxRedirects?: number;
+ // (undocumented)
+ method?: AllowedHttpMethods;
+ // (undocumented)
+ proxyUrl?: string;
+ // (undocumented)
+ sessionToken?: object;
+ // (undocumented)
+ signal?: AbortSignal;
+ // (undocumented)
+ throwHttpErrors?: boolean;
+ // (undocumented)
+ timeout?: number;
+ // (undocumented)
+ url: string | URL;
+ // (undocumented)
+ useHeaderGenerator?: boolean;
+}
+
+// @public
+export interface HttpRequestOptions extends HttpRequest {
+ form?: Record<string, string>;
+ json?: unknown;
+ password?: string;
+ searchParams?: SearchParams;
+ username?: string;
+}
+
+// @public
+export interface IBrowserPool<Page = unknown> {
+ closePage(page: Page, options?: {
+ error?: Error;
+ }): Promise<void>;
+ extractPageState(page: Page): Promise<PageState>;
+ injectPageState(page: Page, state: PageState): Promise<void>;
+ newPage(options?: NewPageOptions): Promise<Page>;
+}
+
+// @public
+export interface ISession {
+ // (undocumented)
+ cookieJar: CookieJar;
+ // (undocumented)
+ fingerprint?: SessionFingerprint;
+ // (undocumented)
+ readonly id: string;
+ isUsable(): boolean;
+ markBad(): void;
+ markGood(): void;
+ // (undocumented)
+ proxyInfo?: ProxyInfo;
+ retire(): void;
+}
+
+// @public
+export interface ISessionPool {
+ getSession(sessionId?: string): Promise<ISession | undefined>;
+}
+
+// @public
+export interface KeyValueStoreClient {
+ deleteValue(key: string): Promise<void>;
+ drop(): Promise<void>;
+ getMetadata(): Promise<KeyValueStoreInfo>;
+ getPublicUrl(key: string): Promise<string | undefined>;
+ getValue(key: string): Promise<KeyValueStoreRecord | undefined>;
+ listKeys(options?: KeyValueStoreListKeysOptions): Promise<KeyValueStoreItemData[]>;
+ purge(): Promise<void>;
+ recordExists(key: string): Promise<boolean>;
+ setValue(record: KeyValueStoreInputRecord): Promise<void>;
+}
+
+// @public (undocumented)
+export interface KeyValueStoreInfo {
+ // (undocumented)
+ accessedAt: Date;
+ // (undocumented)
+ actId?: string;
+ // (undocumented)
+ actRunId?: string;
+ // (undocumented)
+ createdAt: Date;
+ // (undocumented)
+ id: string;
+ // (undocumented)
+ modifiedAt: Date;
+ // (undocumented)
+ name?: string;
+ // (undocumented)
+ stats?: KeyValueStoreStats;
+ // (undocumented)
+ userId?: string;
+}
+
+// @public
+export interface KeyValueStoreInputRecord {
+ // (undocumented)
+ contentType?: string;
+ // (undocumented)
+ key: string;
+ // (undocumented)
+ value: KeyValueStoreRecordInputValue;
+}
+
+// @public (undocumented)
+export interface KeyValueStoreItemData {
+ // (undocumented)
+ key: string;
+ // (undocumented)
+ size: number;
+}
+
+// @public (undocumented)
+export interface KeyValueStoreListKeysOptions {
+ exclusiveStartKey?: string;
+ limit?: number;
+ prefix?: string;
+}
+
+// @public
+export interface KeyValueStoreRecord {
+ // (undocumented)
+ contentType?: string;
+ // (undocumented)
+ key: string;
+ // (undocumented)
+ value: Buffer | ArrayBuffer;
+}
+
+// @public
+export type KeyValueStoreRecordInputValue = Buffer | ArrayBuffer | ArrayBufferView | string | NodeJS.ReadableStream | ReadableStream;
+
+// @public (undocumented)
+export interface KeyValueStoreStats {
+ // (undocumented)
+ deleteCount?: number;
+ // (undocumented)
+ listCount?: number;
+ // (undocumented)
+ readCount?: number;
+ // (undocumented)
+ storageBytes?: number;
+ // (undocumented)
+ writeCount?: number;
+}
+
+// @public
+export interface NewPageOptions {
+ id?: string;
+ session?: ISession;
+}
+
+// @public
+export interface PageState {
+ cookies: Cookie[];
+}
+
+// @public (undocumented)
+export interface PaginatedList<Data> {
+ count: number;
+ desc?: boolean;
+ items: Data[];
+ limit: number;
+ offset: number;
+ total: number;
+}
+
+// @public (undocumented)
+export interface ProcessedRequest {
+ // (undocumented)
+ requestId: string;
+ // (undocumented)
+ uniqueKey: string;
+ // (undocumented)
+ wasAlreadyHandled: boolean;
+ // (undocumented)
+ wasAlreadyPresent: boolean;
+}
+
+// @public
+export interface ProxyInfo {
+ hostname: string;
+ ignoreTlsErrors?: boolean;
+ password: string;
+ port: number | string;
+ url: string;
+ username?: string;
+}
+
+// @public
+export interface QueueOperationInfo {
+ requestId: string;
+ wasAlreadyHandled: boolean;
+ wasAlreadyPresent: boolean;
+}
+
+// @public
+export type RedirectHandler = (redirectResponse: Response, updatedRequest: {
+ url?: string | URL;
+ headers: Headers;
+}) => void;
+
+// @public (undocumented)
+export interface RequestOptions {
+ // (undocumented)
+ [k: string]: unknown;
+ // (undocumented)
+ forefront?: boolean;
+}
+
+// @public
+export interface RequestQueueClient {
+ addBatchOfRequests(requests: RequestSchema[], options?: RequestOptions): Promise<BatchAddRequestsResult>;
+ drop(): Promise<void>;
+ fetchNextRequest(): Promise<RequestOptions | null>;
+ getMetadata(): Promise<RequestQueueInfo>;
+ getRequest(uniqueKey: string): Promise<RequestOptions | undefined>;
+ isEmpty(): Promise<boolean>;
+ isFinished(): Promise<boolean>;
+ markRequestAsHandled(request: UpdateRequestSchema): Promise<QueueOperationInfo | null>;
+ purge(): Promise<void>;
+ reclaimRequest(request: UpdateRequestSchema, options?: RequestOptions): Promise<QueueOperationInfo | null>;
+ setExpectedRequestProcessingTimeSecs?(secs: number): void;
+}
+
+// @public (undocumented)
+export interface RequestQueueInfo {
+ // (undocumented)
+ accessedAt: Date;
+ // (undocumented)
+ actId?: string;
+ // (undocumented)
+ actRunId?: string;
+ // (undocumented)
+ createdAt: Date;
+ // (undocumented)
+ expireAt?: string;
+ // (undocumented)
+ hadMultipleClients?: boolean;
+ // (undocumented)
+ handledRequestCount: number;
+ // (undocumented)
+ id: string;
+ // (undocumented)
+ modifiedAt: Date;
+ // (undocumented)
+ name?: string;
+ // (undocumented)
+ pendingRequestCount: number;
+ // (undocumented)
+ stats?: RequestQueueStats;
+ // (undocumented)
+ totalRequestCount: number;
+ // (undocumented)
+ userId?: string;
+}
+
+// @public (undocumented)
+export interface RequestQueueStats {
+ // (undocumented)
+ deleteCount?: number;
+ // (undocumented)
+ headItemReadCount?: number;
+ // (undocumented)
+ readCount?: number;
+ // (undocumented)
+ storageBytes?: number;
+ // (undocumented)
+ writeCount?: number;
+}
+
+// @public (undocumented)
+export interface RequestSchema {
+ // (undocumented)
+ errorMessages?: string[];
+ // (undocumented)
+ handledAt?: string;
+ // (undocumented)
+ headers?: Dictionary<string>;
+ // (undocumented)
+ id?: string;
+ // (undocumented)
+ loadedUrl?: string;
+ // (undocumented)
+ method?: AllowedHttpMethods;
+ // (undocumented)
+ noRetry?: boolean;
+ // (undocumented)
+ payload?: string;
+ // (undocumented)
+ retryCount?: number;
+ // (undocumented)
+ uniqueKey: string;
+ // (undocumented)
+ url: string;
+ // (undocumented)
+ userData?: Dictionary;
+}
+
+// @public (undocumented)
+export type SearchParams = string | URLSearchParams | Record<string, string | number | boolean | null | undefined>;
+
+// @public (undocumented)
+export interface SendRequestOptions {
+ // (undocumented)
+ cookieJar?: CookieJar;
+ proxyUrl?: string;
+ // (undocumented)
+ session?: ISession;
+ signal?: AbortSignal;
+ timeoutMillis?: number;
+}
+
+// @public
+export interface SessionFingerprint {
+ browser?: 'chrome' | 'firefox' | 'safari' | 'edge';
+ device?: 'desktop' | 'mobile';
+ platform?: 'windows' | 'macos' | 'linux' | 'android' | 'ios';
+}
+
+// @public
+export interface SessionState {
+ // (undocumented)
+ cookieJar: SerializedCookieJar;
+ // (undocumented)
+ createdAt: string;
+ // (undocumented)
+ errorScore: number;
+ // (undocumented)
+ errorScoreDecrement: number;
+ // (undocumented)
+ expiresAt: string;
+ // (undocumented)
+ fingerprint?: SessionFingerprint;
+ // (undocumented)
+ id: string;
+ // (undocumented)
+ maxErrorScore: number;
+ // (undocumented)
+ maxUsageCount: number;
+ // (undocumented)
+ proxyInfo?: ProxyInfo;
+ // (undocumented)
+ retired: boolean;
+ // (undocumented)
+ usageCount: number;
+ // (undocumented)
+ userData: object;
+}
+
+// @public (undocumented)
+export interface SetStatusMessageOptions {
+ // (undocumented)
+ isStatusMessageTerminal?: boolean;
+ // (undocumented)
+ level?: 'DEBUG' | 'INFO' | 'WARNING' | 'ERROR';
+}
+
+// @public
+export interface StorageClient {
+ createDatasetClient(options?: CreateDatasetClientOptions): Promise<DatasetClient>;
+ createKeyValueStoreClient(options?: CreateKeyValueStoreClientOptions): Promise<KeyValueStoreClient>;
+ createRequestQueueClient(options?: CreateRequestQueueClientOptions): Promise<RequestQueueClient>;
+ getStorageClientCacheKey?(): string;
+ // (undocumented)
+ purge?(): Promise<void>;
+ // (undocumented)
+ setStatusMessage?(message: string, options?: SetStatusMessageOptions): Promise<void>;
+ // (undocumented)
+ stats?: {
+ rateLimitErrors: number[];
+ };
+ storageExists?(id: string, type: 'Dataset' | 'KeyValueStore' | 'RequestQueue'): Promise<boolean>;
+ // (undocumented)
+ teardown?(): Promise<void>;
+}
+
+// @public
+export type StorageIdentifier = {
+ id: string;
+ name?: never;
+ alias?: never;
+} | {
+ id?: never;
+ name: string;
+ alias?: never;
+} | {
+ id?: never;
+ name?: never;
+ alias: string;
+} | {
+ id?: never;
+ name?: never;
+ alias?: never;
+};
+
+// @public (undocumented)
+export interface StreamOptions extends SendRequestOptions {
+ // (undocumented)
+ onRedirect?: RedirectHandler;
+}
+
+// @public (undocumented)
+export interface UnprocessedRequest {
+ // (undocumented)
+ method?: AllowedHttpMethods;
+ // (undocumented)
+ uniqueKey: string;
+ // (undocumented)
+ url: string;
+}
+
+// @public (undocumented)
+export interface UpdateRequestSchema extends RequestSchema {
+ // (undocumented)
+ id: string;
+}
+
+// (No @packageDocumentation comment for this package)
+
+```
diff --git a/docs/public-api/crawlee-utils.api.md b/docs/public-api/crawlee-utils.api.md
new file mode 100644
index 000000000..31849b2f1
--- /dev/null
+++ b/docs/public-api/crawlee-utils.api.md
@@ -0,0 +1,400 @@
+## API Report File for "@crawlee/utils"
+
+> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
+
+```ts
+
+import type { AllowedHttpMethods } from '@crawlee/types';
+import { Awaitable } from '@crawlee/types';
+import type { BaseHttpClient } from '@crawlee/types';
+import { Cheerio } from 'cheerio';
+import { CheerioAPI } from 'cheerio';
+import { Constructor } from '@crawlee/types';
+import type { CrawleeLogger } from '@crawlee/types';
+import { Dictionary } from '@crawlee/types';
+import { Element as Element_2 } from 'domhandler';
+import type { IncomingMessage } from 'node:http';
+import type { SearchParams } from '@crawlee/types';
+
+// @internal
+export function applySearchParams(url: URL, searchParams: SearchParams | undefined): void;
+
+// @internal
+export function asyncifyIterable<T>(iterable: Iterable<T> | AsyncIterable<T>): AsyncIterable<T>;
+
+export { Awaitable }
+
+export { Cheerio }
+
+export { CheerioAPI }
+
+// @public (undocumented)
+export type CheerioRoot = CheerioAPI;
+
+// @public (undocumented)
+export function chunk<T>(array: readonly T[], chunkSize: number): T[][];
+
+// @internal
+export function chunkedAsyncIterable<T>(iterable: AsyncIterable<T> | Iterable<T>, chunkSize: number): AsyncIterable<T[]>;
+
+// @public (undocumented)
+export const CLOUDFLARE_RETRY_CSS_SELECTORS: string[];
+
+export { Constructor }
+
+// @public
+export interface CpuSample {
+ // (undocumented)
+ containerUsage: number;
+ // (undocumented)
+ systemUsage: number;
+}
+
+// @public
+export function createRequestDebugInfo(request: Request_2, response?: IncomingMessage | Partial<BrowserResponseLike>, additionalFields?: Dictionary): Dictionary;
+
+export { Dictionary }
+
+// @public
+const DISCORD_REGEX: RegExp;
+
+// @public
+const DISCORD_REGEX_GLOBAL: RegExp;
+
+// @public
+export function discoverValidSitemaps(urls: string[], options?: {
+ proxyUrl?: string;
+ timeoutMillis?: number;
+ signal?: AbortSignal;
+ requestTimeoutMillis?: number;
+ httpClient?: BaseHttpClient;
+ logger?: CrawleeLogger;
+}): AsyncIterable<string>;
+
+// @public
+export function downloadListOfUrls(options: DownloadListOfUrlsOptions): Promise<string[]>;
+
+// @public (undocumented)
+export interface DownloadListOfUrlsOptions {
+ encoding?: BufferEncoding;
+ httpClient?: BaseHttpClient;
+ proxyUrl?: string;
+ url: string;
+ urlRegExp?: RegExp;
+}
+
+export { Element_2 as Element }
+
+// @public
+const EMAIL_REGEX: RegExp;
+
+// @public
+const EMAIL_REGEX_GLOBAL: RegExp;
+
+// @public
+function emailsFromText(text: string): string[];
+
+// @public
+function emailsFromUrls(urls: string[]): string[];
+
+// @public (undocumented)
+export function entries<T extends {}>(obj: T): [keyof T, T[keyof T]][];
+
+// @public
+export function expandShadowRoots(document: Document): string;
+
+// @public
+export function extractUrls(options: ExtractUrlsOptions): string[];
+
+// @public
+export function extractUrlsFromCheerio($: CheerioAPI, selector?: string, baseUrl?: string): string[];
+
+// @public (undocumented)
+export interface ExtractUrlsOptions {
+ string: string;
+ urlRegExp?: RegExp;
+}
+
+// @public
+const FACEBOOK_REGEX: RegExp;
+
+// @public
+const FACEBOOK_REGEX_GLOBAL: RegExp;
+
+// @public
+export function getCgroupsVersion(forceReset?: boolean): Promise<"V1" | "V2" | null>;
+
+// @internal
+export function getCurrentCpuTicksV2(options?: {
+ containerized?: boolean;
+ logger?: CrawleeLogger;
+}): Promise<number>;
+
+// @internal
+export function getMemoryInfo(options?: {
+ containerized?: boolean;
+ logger?: CrawleeLogger;
+}): Promise<MemoryInfo>;
+
+// @public (undocumented)
+export function getObjectType(value: unknown): string;
+
+// @public
+export function htmlToText(htmlOrCheerioElement: string | CheerioRoot): string;
+
+// @internal
+export function inspectValue(value: unknown): string;
+
+// @public
+const INSTAGRAM_REGEX: RegExp;
+
+// @public
+const INSTAGRAM_REGEX_GLOBAL: RegExp;
+
+// @internal
+export function isAsyncIterable<T>(value: unknown): value is AsyncIterable<T>;
+
+// @public
+export function isBuffer(value: unknown): value is Buffer | ArrayBuffer | ArrayBufferView;
+
+// @public
+export function isContainerized(): Promise<boolean>;
+
+// @public
+export function isDocker(forceReset?: boolean): Promise<boolean>;
+
+// @internal
+export function isIterable<T>(value: unknown): value is Iterable<T>;
+
+// @public (undocumented)
+export function isLambda(): boolean;
+
+// @public
+export function isStream(value: unknown): value is NodeJS.ReadableStream | ReadableStream;
+
+// @public (undocumented)
+export function keys<T extends {}>(obj: T): (keyof T)[];
+
+// @public
+const LINKEDIN_REGEX: RegExp;
+
+// @public
+const LINKEDIN_REGEX_GLOBAL: RegExp;
+
+// @public
+export interface MemoryInfo {
+ childProcessesBytes: number;
+ freeBytes: number;
+ mainProcessBytes: number;
+ totalBytes: number;
+ usedBytes: number;
+}
+
+// @public
+export function mergeAsyncIterables<T>(...iterables: AsyncIterable<T>[]): AsyncIterable<T>;
+
+// @public (undocumented)
+export interface OpenGraphProperty {
+ // (undocumented)
+ children: OpenGraphProperty[];
+ // (undocumented)
+ name: string;
+ // (undocumented)
+ outputName: string;
+}
+
+// @public
+function parseHandlesFromHtml(html: string, data?: Record<string, unknown> | null): SocialHandles;
+
+// @public
+export function parseOpenGraph(raw: string, additionalProperties?: OpenGraphProperty[]): Dictionary<OpenGraphResult>;
+
+// @public (undocumented)
+export function parseOpenGraph($: CheerioAPI, additionalProperties?: OpenGraphProperty[]): Dictionary<OpenGraphResult>;
+
+// @public (undocumented)
+export function parseSitemap<T extends ParseSitemapOptions>(initialSources: SitemapSource[], proxyUrl?: string, options?: T): AsyncIterable<T['emitNestedSitemaps'] extends true ? SitemapUrl | NestedSitemap : SitemapUrl>;
+
+// @public (undocumented)
+export interface ParseSitemapOptions {
+ emitNestedSitemaps?: true | false;
+ httpClient?: BaseHttpClient;
+ logger?: CrawleeLogger;
+ maxDepth?: number;
+ reportNetworkErrors?: boolean;
+ sitemapRetries?: number;
+ timeoutMillis?: number;
+}
+
+// @internal
+export interface PeekableAsyncIterable<T> extends AsyncIterable<T> {
+ // (undocumented)
+ [Symbol.asyncIterator](): PeekableAsyncIterator<T>;
+}
+
+// @internal
+export function peekableAsyncIterable<T>(iterable: AsyncIterable<T> | Iterable<T>): PeekableAsyncIterable<T>;
+
+// @internal
+export interface PeekableAsyncIterator<T> extends AsyncIterator<T>, AsyncIterable<T> {
+ peek(): Promise<T | undefined>;
+}
+
+// @public
+function phonesFromText(text: string): string[];
+
+// @public
+function phonesFromUrls(urls: string[]): string[];
+
+// @public
+const PINTEREST_REGEX: RegExp;
+
+// @public
+const PINTEREST_REGEX_GLOBAL: RegExp;
+
+// @public
+export const RETRY_CSS_SELECTORS: string[];
+
+// @public
+class RobotsTxtFile {
+ static find(url: string, options?: {
+ signal?: AbortSignal;
+ timeoutMillis?: number;
+ proxyUrl?: string;
+ httpClient?: BaseHttpClient;
+ logger?: CrawleeLogger;
+ }): Promise<RobotsTxtFile>;
+ static from(url: string, content: string, proxyUrl?: string): RobotsTxtFile;
+ getSitemaps(): string[];
+ isAllowed(url: string, userAgent?: string): boolean;
+ // (undocumented)
+ protected static load(url: string, options?: {
+ signal?: AbortSignal;
+ timeoutMillis?: number;
+ proxyUrl?: string;
+ httpClient?: BaseHttpClient;
+ logger?: CrawleeLogger;
+ }): Promise<RobotsTxtFile>;
+ parseSitemaps(): Promise<Sitemap>;
+ parseUrlsFromSitemaps(): Promise<string[]>;
+}
+export { RobotsTxtFile as RobotsFile }
+export { RobotsTxtFile }
+
+// @public
+export const ROTATE_PROXY_ERRORS: string[];
+
+// @public
+export class Sitemap {
+ constructor(urls: string[]);
+ static fromXmlString(content: string, proxyUrl?: string, parseSitemapOptions?: ParseSitemapOptions): Promise<Sitemap>;
+ static load(urls: string | string[], proxyUrl?: string, parseSitemapOptions?: ParseSitemapOptions): Promise<Sitemap>;
+ // (undocumented)
+ protected static parse(sources: SitemapSource[], proxyUrl?: string, parseSitemapOptions?: ParseSitemapOptions): Promise<Sitemap>;
+ static tryCommonNames(url: string, proxyUrl?: string, parseSitemapOptions?: ParseSitemapOptions): Promise<Sitemap>;
+ // (undocumented)
+ readonly urls: string[];
+}
+
+// @public (undocumented)
+export type SitemapUrl = SitemapUrlData & {
+ originSitemapUrl: string;
+};
+
+// @public
+export function sleep(millis?: number): Promise<void>;
+
+// @public
+export function snakeCaseToCamelCase(snakeCaseStr: string): string;
+
+declare namespace social {
+ export {
+ emailsFromText,
+ emailsFromUrls,
+ phonesFromText,
+ phonesFromUrls,
+ parseHandlesFromHtml,
+ EMAIL_REGEX,
+ EMAIL_REGEX_GLOBAL,
+ SocialHandles,
+ LINKEDIN_REGEX,
+ LINKEDIN_REGEX_GLOBAL,
+ INSTAGRAM_REGEX,
+ INSTAGRAM_REGEX_GLOBAL,
+ TWITTER_REGEX,
+ TWITTER_REGEX_GLOBAL,
+ FACEBOOK_REGEX,
+ FACEBOOK_REGEX_GLOBAL,
+ YOUTUBE_REGEX,
+ YOUTUBE_REGEX_GLOBAL,
+ TIKTOK_REGEX,
+ TIKTOK_REGEX_GLOBAL,
+ PINTEREST_REGEX,
+ PINTEREST_REGEX_GLOBAL,
+ DISCORD_REGEX,
+ DISCORD_REGEX_GLOBAL
+ }
+}
+
+// @public
+interface SocialHandles {
+ // (undocumented)
+ discords: string[];
+ // (undocumented)
+ emails: string[];
+ // (undocumented)
+ facebooks: string[];
+ // (undocumented)
+ instagrams: string[];
+ // (undocumented)
+ linkedIns: string[];
+ // (undocumented)
+ phones: string[];
+ // (undocumented)
+ phonesUncertain: string[];
+ // (undocumented)
+ pinterests: string[];
+ // (undocumented)
+ tiktoks: string[];
+ // (undocumented)
+ twitters: string[];
+ // (undocumented)
+ youtubes: string[];
+}
+
+// @public
+const TIKTOK_REGEX: RegExp;
+
+// @public
+const TIKTOK_REGEX_GLOBAL: RegExp;
+
+// @public
+export function toBuffer(value: Buffer | ArrayBuffer | ArrayBufferView): Buffer;
+
+// @public
+export function tryAbsoluteURL(href: string, baseUrl: string): string | undefined;
+
+// @public
+const TWITTER_REGEX: RegExp;
+
+// @public
+const TWITTER_REGEX_GLOBAL: RegExp;
+
+// @public
+export const URL_NO_COMMAS_REGEX: RegExp;
+
+// @public
+export const URL_WITH_COMMAS_REGEX: RegExp;
+
+// @public
+export function weightedAvg(arrValues: number[], arrWeights: number[]): number;
+
+// @public
+const YOUTUBE_REGEX: RegExp;
+
+// @public
+const YOUTUBE_REGEX_GLOBAL: RegExp;
+
+// (No @packageDocumentation comment for this package)
+
+```
diff --git a/docs/public-api/crawlee.api.md b/docs/public-api/crawlee.api.md
new file mode 100644
index 000000000..f722c9cef
--- /dev/null
+++ b/docs/public-api/crawlee.api.md
@@ -0,0 +1,45 @@
+## API Report File for "crawlee"
+
+> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
+
+```ts
+
+import { downloadListOfUrls } from '@crawlee/utils';
+import { enqueueLinks } from '@crawlee/core';
+import { Log } from '@apify/log';
+import { parseOpenGraph } from '@crawlee/utils';
+import { playwrightUtils } from '@crawlee/playwright';
+import { puppeteerUtils } from '@crawlee/puppeteer';
+import { sleep } from '@crawlee/utils';
+import { social } from '@crawlee/utils';
+
+// @public (undocumented)
+export const utils: {
+ puppeteer: typeof puppeteerUtils;
+ playwright: typeof playwrightUtils;
+ log: Log;
+ enqueueLinks: typeof enqueueLinks;
+ social: typeof social;
+ sleep: typeof sleep;
+ downloadListOfUrls: typeof downloadListOfUrls;
+ parseOpenGraph: typeof parseOpenGraph;
+};
+
+
+export * from "@crawlee/basic";
+export * from "@crawlee/browser";
+export * from "@crawlee/browser-pool";
+export * from "@crawlee/cheerio";
+export * from "@crawlee/core";
+export * from "@crawlee/fs-storage";
+export * from "@crawlee/http";
+export * from "@crawlee/jsdom";
+export * from "@crawlee/linkedom";
+export * from "@crawlee/memory-storage";
+export * from "@crawlee/playwright";
+export * from "@crawlee/puppeteer";
+export * from "@crawlee/utils";
+
+// (No @packageDocumentation comment for this package)
+
+```
diff --git a/package.json b/package.json
index 1c3722a36..5a14ef1a9 100644
--- a/package.json
+++ b/package.json
@@ -45,6 +45,8 @@
"publish:prod": "lerna publish from-package --contents dist --force-publish --yes",
"release:prod": "pnpm build && pnpm publish:prod",
"release:pin-versions": "turbo run copy -- -- --pin-versions",
+ "api:extract": "tsx scripts/api-extractor/run.ts",
+ "api:check": "tsx scripts/api-extractor/run.ts --verify",
"lint": "oxlint packages test docs --tsconfig=tsconfig.json --type-aware",
"lint:fix": "oxlint packages test docs --tsconfig=tsconfig.json --type-aware --fix",
"format": "oxfmt packages test docs --write",
@@ -56,6 +58,7 @@
"@apify/oxlint-config": "^0.2.5",
"@apify/tsconfig": "^0.2.0",
"@commitlint/config-conventional": "^21.0.0",
+ "@microsoft/api-extractor": "^7.58.9",
"@playwright/browser-chromium": "1.61.1",
"@playwright/browser-firefox": "1.61.1",
"@playwright/browser-webkit": "1.61.1",
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 86056940d..dcc960ac9 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -30,6 +30,9 @@ importers:
'@commitlint/config-conventional':
specifier: ^20.0.0
version: 20.5.0
+ '@microsoft/api-extractor':
+ specifier: ^7.58.9
+ version: 7.58.9(@types/[email protected])
'@playwright/browser-chromium':
specifier: 1.60.0
version: 1.60.0
@@ -3166,6 +3169,19 @@ packages:
'@mermaid-js/[email protected]':
resolution: {integrity: sha512-gxK9ZX2+Fex5zu8LhRQoMeMPEHbc73UKZ0FQ54YrQtUxE1VVhMwzeNtKRPAu5aXks4FasbMe4xB4bWrmq6Jlxw==}
+ '@microsoft/[email protected]':
+ resolution: {integrity: sha512-aIcoQggPyer3B6Ze3usz0YWC/oBwUHfRH5ETUsr+oT2BRA6SfTJl7IKPcPZkX4UR+PohowzW4uMxsvjrn8vm+w==}
+
+ '@microsoft/[email protected]':
+ resolution: {integrity: sha512-S2UF4yza5GoxCmf7hJQNxJNZN9ltOVuOQv8Dy+Z21aol5ERoBNMdWcQHm4MJMPPItW4H/4rZD906iaf4mUojJA==}
+ hasBin: true
+
+ '@microsoft/[email protected]':
+ resolution: {integrity: sha512-9brPoVdfN9k9g0dcWkFeA7IH9bbcttzDJlXvkf8b2OBzd5MueR1V2wkKBL0abn0otvmkHJC6aapBOTJDDeMCZg==}
+
+ '@microsoft/[email protected]':
+ resolution: {integrity: sha512-xgAyonlVVS+q7Vc7qLW0UrJU7rSFcETRWsqdXZtjzRU8dF+6CkozTK4V4y1LwOX7j8r/vHphjDeMeGI4tNGeGA==}
+
'@modelcontextprotocol/[email protected]':
resolution: {integrity: sha512-zo37mZA9hJWpULgkRpowewez1y6ML5GsXJPY8FI0tBBCd77HEvza4jDqRKOXgHNn867PVGCyTdzqpz0izu5ZjQ==}
engines: {node: '>=18'}
@@ -3997,6 +4013,36 @@ packages:
resolution: {integrity: sha512-zt6OdqaDoOnJ1ZYsCYGt9YmWzDXl4vQdKTyJev62gFhRGKdx7mcT54V9KIjg+d2wi9EXsPvAPKe7i7WjfVWB8g==}
+ '@rushstack/[email protected]':
+ resolution: {integrity: sha512-wlKmIKIYCKuCASbITvOxLZXepPbwXvrv7S6ig6XNWFchSyhL/E2txmVXspHY49Wu2dzf7nI27a2k/yV5BA3EiA==}
+ peerDependencies:
+ '@types/node': '*'
+ peerDependenciesMeta:
+ '@types/node':
+ optional: true
+
+ '@rushstack/[email protected]':
+ resolution: {integrity: sha512-gulfhBs6n+I5b7DvjKRfhMGyUejtSgOHTclF/eONr8hcgF1APEDjhxIsfdUYYMzC3rvLwGluqLjbwCFZ8nxrog==}
+ peerDependencies:
+ '@types/node': '*'
+ peerDependenciesMeta:
+ '@types/node':
+ optional: true
+
+ '@rushstack/[email protected]':
+ resolution: {integrity: sha512-aAA518n6wxxjCfnTAOjQnm7ngNE0FVHxHAw2pxKlIhxrMn0XQjGcXKF0oKWpjBgJOmsaJpVob/v+zr3zxgPWuA==}
+
+ '@rushstack/[email protected]':
+ resolution: {integrity: sha512-8ZQS4MMaGsv27EXCBiH7WMPkRZrffeDoIevs6z9TM5dzqiY6+Hn4evfK/G+gvgBTjfvfkHIZPQQmalmI2sM4TQ==}
+ peerDependencies:
+ '@types/node': '*'
+ peerDependenciesMeta:
+ '@types/node':
+ optional: true
+
+ '@rushstack/[email protected]':
+ resolution: {integrity: sha512-fwI076HYknC0IrMXdY6UmjDv+PH7NHhNJX3/pY2UblSE5XrXgndXZPiOe/6ZtuFpn6DvVDVNhtkIzQ+Qu/MhVQ==}
+
'@sapphire/[email protected]':
resolution: {integrity: sha512-cvGzxbba6sav2zZkH8GPf2oGk9yYoD5qrNWdu9fRehifgnFZJMV+nuy2nON2roRO4yQQ+v7MK/Pktl/HgfsUXg==}
engines: {node: '>=v14.0.0', npm: '>=7.0.0'}
@@ -4436,6 +4482,9 @@ packages:
resolution: {integrity: sha512-6+7nlbMVX/PVDCwaIQ8nTOPveOcFLSt8GcXdx8hD0bt39uWxYT88uXzqTd4fTvqta7oeUJqudepapKNt2DYJFw==}
+ '@types/[email protected]':
+ resolution: {integrity: sha512-ebDJ9b0e702Yr7pWgB0jzm+CX4Srzz8RcXtLJDJB+BSccqMa36uyH/zUsSYao5+BD1ytv3k3rPYCq4mAE1hsXA==}
+
resolution: {integrity: sha512-HLFeCYgz89uk22N5Qg3dvGvsv46B8GLvKKo1zKG4NybA8U2DiEO3w9lqGg29t/tfLRJpJ6iQxnVw4OnB7MoM9g==}
@@ -5135,6 +5184,14 @@ packages:
peerDependencies:
zod: ^3.25.76 || ^4.1.8
+ resolution: {integrity: sha512-mv00Te6nmYbRp5DCwclxtt7yV/joXJPGS7nM+97GdxvuttCOfgI3K4U25zboyeX0O+myI8ERluxQe5wljMmVIw==}
+ peerDependencies:
+ ajv: ^8.5.0
+ peerDependenciesMeta:
+ ajv:
+ optional: true
+
resolution: {integrity: sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==}
peerDependencies:
@@ -6619,6 +6676,10 @@ packages:
resolution: {integrity: sha512-hp32aOyalF3vGZiTPA3CiKCgpIH7QOj+S0dRVMmEReagtmWQYigiCQWp7OGWrgaW2IHNlSzCwfDneZe16DlBCQ==}
+ resolution: {integrity: sha512-DPi0FmjiSU5EvQV0++GFDOJ9ASQUVFh5kD+OzOnYdi7n3Wpm9hWWGfB/O2blfHcMVTL5WkQXSnRiK9makhrcnw==}
+ engines: {node: '>=0.3.1'}
+
resolution: {integrity: sha512-kqag/Nl+f3GwyK25fhUMYj81BUOrZ9IuJsjIcDE5icNM9FJHAVm3VcUDxdLPoQtTuUylWm6ZIknYJwwaPxsUzg==}
@@ -8495,6 +8556,9 @@ packages:
resolution: {integrity: sha512-ekilCSN1jwRvIbgeg/57YFh8qQDNbwDb9xT/qu2DAHbFFZUicIl4ygVaAvzveMhMVr3LnpSKTNnwt8PoOfmKhQ==}
hasBin: true
+ resolution: {integrity: sha512-8wb9Yw966OSxApiCt0K3yNJL8pnNeIv+OEq2YMidz4FKP6nonSRoOXc80iXY4JaN2FC11B9qsNmDsm+ZOfMROA==}
+
resolution: {integrity: sha512-otDA4ldcIx+ZXsKHWmp0YizCweVRZG96J10b0FevjfuncLO1oX59THoAmHkNubYJ+9gWsYsp5k8v4ib6oDv1fA==}
@@ -15881,6 +15945,41 @@ snapshots:
dependencies:
langium: 4.2.2
+ '@microsoft/[email protected](@types/[email protected])':
+ dependencies:
+ '@microsoft/tsdoc': 0.16.0
+ '@microsoft/tsdoc-config': 0.18.1
+ '@rushstack/node-core-library': 5.23.1(@types/[email protected])
+ transitivePeerDependencies:
+ - '@types/node'
+
+ '@microsoft/[email protected](@types/[email protected])':
+ dependencies:
+ '@microsoft/api-extractor-model': 7.33.8(@types/[email protected])
+ '@microsoft/tsdoc': 0.16.0
+ '@microsoft/tsdoc-config': 0.18.1
+ '@rushstack/node-core-library': 5.23.1(@types/[email protected])
+ '@rushstack/rig-package': 0.7.3
+ '@rushstack/terminal': 0.24.0(@types/[email protected])
+ '@rushstack/ts-command-line': 5.3.10(@types/[email protected])
+ diff: 8.0.4
+ minimatch: 9.0.9
+ resolve: 1.22.12
+ semver: 7.7.4
+ source-map: 0.6.1
+ typescript: 5.9.3
+ transitivePeerDependencies:
+ - '@types/node'
+
+ '@microsoft/[email protected]':
+ dependencies:
+ '@microsoft/tsdoc': 0.16.0
+ ajv: 8.18.0
+ jju: 1.4.0
+ resolve: 1.22.12
+
+ '@microsoft/[email protected]': {}
+
'@modelcontextprotocol/[email protected](@cfworker/[email protected])([email protected])':
dependencies:
'@hono/node-server': 1.19.14([email protected])
@@ -16635,6 +16734,45 @@ snapshots:
'@rtsao/[email protected]': {}
+ '@rushstack/[email protected](@types/[email protected])':
+ dependencies:
+ ajv: 8.18.0
+ ajv-draft-04: 1.0.0([email protected])
+ ajv-formats: 3.0.1([email protected])
+ fs-extra: 11.3.4
+ import-lazy: 4.0.0
+ jju: 1.4.0
+ resolve: 1.22.12
+ semver: 7.7.4
+ optionalDependencies:
+ '@types/node': 24.12.2
+
+ '@rushstack/[email protected](@types/[email protected])':
+ optionalDependencies:
+ '@types/node': 24.12.2
+
+ '@rushstack/[email protected]':
+ dependencies:
+ jju: 1.4.0
+ resolve: 1.22.12
+
+ '@rushstack/[email protected](@types/[email protected])':
+ dependencies:
+ '@rushstack/node-core-library': 5.23.1(@types/[email protected])
+ '@rushstack/problem-matcher': 0.2.1(@types/[email protected])
+ supports-color: 8.1.1
+ optionalDependencies:
+ '@types/node': 24.12.2
+
+ '@rushstack/[email protected](@types/[email protected])':
+ dependencies:
+ '@rushstack/terminal': 0.24.0(@types/[email protected])
+ '@types/argparse': 1.0.38
+ argparse: 1.0.10
+ string-argv: 0.3.2
+ transitivePeerDependencies:
+ - '@types/node'
+
'@sapphire/[email protected]': {}
'@sapphire/[email protected]':
@@ -17048,6 +17186,8 @@ snapshots:
dependencies:
tslib: 2.8.1
+ '@types/[email protected]': {}
+
dependencies:
'@types/connect': 3.4.38
@@ -17819,6 +17959,10 @@ snapshots:
'@opentelemetry/api': 1.9.0
zod: 4.3.6
+ optionalDependencies:
+ ajv: 8.18.0
+
optionalDependencies:
ajv: 8.18.0
@@ -19535,6 +19679,8 @@ snapshots:
+
dependencies:
bn.js: 4.12.3
@@ -21859,6 +22005,8 @@ snapshots:
+
dependencies:
'@hapi/hoek': 9.3.0
diff --git a/scripts/api-extractor/api-extractor.base.json b/scripts/api-extractor/api-extractor.base.json
new file mode 100644
index 000000000..9db9c06a2
--- /dev/null
+++ b/scripts/api-extractor/api-extractor.base.json
@@ -0,0 +1,54 @@
+{
+ "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json",
+
+ "newlineKind": "lf",
+
+ "compiler": {
+ "overrideTsconfig": {
+ "compilerOptions": {
+ "module": "NodeNext",
+ "moduleResolution": "NodeNext",
+ "target": "ESNext",
+ "lib": ["DOM", "ES2023", "ES2024", "DOM.AsyncIterable"],
+ "skipLibCheck": true,
+ "strict": true
+ }
+ }
+ },
+
+ "apiReport": {
+ "enabled": true,
+ "reportFolder": "<projectFolder>/../../docs/public-api",
+ "reportTempFolder": "<projectFolder>/../../docs/public-api/temp"
+ },
+
+ "docModel": {
+ "enabled": false
+ },
+
+ "dtsRollup": {
+ "enabled": false
+ },
+
+ "tsdocMetadata": {
+ "enabled": false
+ },
+
+ "messages": {
+ "compilerMessageReporting": {
+ "default": {
+ "logLevel": "none"
+ }
+ },
+ "extractorMessageReporting": {
+ "default": {
+ "logLevel": "none"
+ }
+ },
+ "tsdocMessageReporting": {
+ "default": {
+ "logLevel": "none"
+ }
+ }
+ }
+}
diff --git a/scripts/api-extractor/run.ts b/scripts/api-extractor/run.ts
new file mode 100644
index 000000000..0599b8c75
--- /dev/null
+++ b/scripts/api-extractor/run.ts
@@ -0,0 +1,188 @@
+/* eslint-disable no-console */
+import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { dirname, relative, resolve } from 'node:path';
+
+import { Extractor, ExtractorConfig, type IConfigFile } from '@microsoft/api-extractor';
+import { globbySync } from 'globby';
+
+/**
+ * Generates (`--verify` to check) a per-package map of the public type-level interface of
+ * each publishable `@crawlee/*` package, committed to `docs/public-api/<package>.api.md`.
+ * These reports define where we promise backwards compatibility; changes must be reviewed.
+ *
+ * The build (`scripts/typescript_fixes.mjs`) injects `// @ts-ignore` comment lines into the
+ * `.d.ts` files that crash API Extractor's AST walker, so we strip them for the duration of
+ * the run and restore them afterwards. A few packages re-export such a member across a
+ * package boundary and crash anyway; those are retried against a sanitized mirror of the
+ * dist tree with `@crawlee/*` deps remapped via tsconfig `paths`, which dodges the bug.
+ *
+ * When running under GitHub Actions (or with `--github`), failures are additionally emitted
+ * as workflow commands (`::error::`) so they show up as inline annotations in the CI run.
+ */
+
+const root = resolve(import.meta.dirname, '..', '..');
+const baseConfigPath = resolve(import.meta.dirname, 'api-extractor.base.json');
+const baseConfig = JSON.parse(readFileSync(baseConfigPath, 'utf8')) as IConfigFile;
+const reportFolder = resolve(root, 'docs', 'public-api');
+const mirrorRoot = resolve(root, 'node_modules', '.cache', 'api-extractor-dts');
+const verify = process.argv.includes('--verify');
+
+// Emit GitHub Actions workflow commands (annotations) when running in CI, so out-of-date
+// reports and crashes surface as inline warnings/errors. Opt in with `--github` or force
+// off with `--no-github` (auto-detected via the runner-set GITHUB_ACTIONS env var otherwise).
+const github = process.argv.includes('--github')
+ || (process.env.GITHUB_ACTIONS === 'true' && !process.argv.includes('--no-github'));
+
+// GitHub workflow commands must escape `%`, `\r` and `\n` in the message. See
+// https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-commands
+const ghEscape = (message: string) => message.replace(/%/g, '%25').replace(/\r/g, '%0D').replace(/\n/g, '%0A');
+const ghCommand = (kind: 'error' | 'warning', message: string) => {
+ if (github) console.log(`::${kind}::${ghEscape(message)}`);
+};
+
+const TS_IGNORE_LINE = /^\s*\/\/ @ts-ignore optional peer dependency or compatibility with es2022\s*$/;
+// CLI binary and project scaffolding are tooling, not an importable API where we promise BC.
+const EXCLUDED = new Set(['@crawlee/cli', '@crawlee/templates']);
+
+interface PackageManifest {
+ name: string;
+ private?: boolean;
+ types?: string;
+ exports?: Record<string, string | { types?: string }>;
+}
+
+const packageJsonPaths = globbySync('packages/*/package.json', { cwd: root, absolute: true }).sort();
+
+function manifest(pkgJsonPath: string): PackageManifest {
+ return JSON.parse(readFileSync(pkgJsonPath, 'utf8')) as PackageManifest;
+}
+
+function dtsEntry(pkgDir: string, pkg: PackageManifest): string | undefined {
+ const dot = pkg.exports?.['.'];
+ const candidate = (typeof dot === 'object' ? dot.types : undefined) ?? pkg.types ?? './dist/index.d.ts';
+ const full = resolve(pkgDir, candidate);
+ return existsSync(full) ? full : undefined;
+}
+
+const stripTsIgnore = (content: string) =>
+ content
+ .split('\n')
+ .filter((line) => !TS_IGNORE_LINE.test(line))
+ .join('\n');
+
+const reportFileName = (name: string) => `${name.replace('@', '').replace('/', '-')}.api.md`;
+
+/** Lazily built sanitized mirror of the dist tree, with a `@crawlee/*` -> mirror paths map. */
+let mirror: { packages: string; paths: Record<string, string[]> } | undefined;
+function getMirror() {
+ if (mirror) return mirror;
+ rmSync(mirrorRoot, { recursive: true, force: true });
+ for (const file of globbySync('packages/*/dist/**/*.d.ts', { cwd: root, absolute: true })) {
+ const target = resolve(mirrorRoot, relative(root, file));
+ mkdirSync(dirname(target), { recursive: true });
+ writeFileSync(target, stripTsIgnore(readFileSync(file, 'utf8')));
+ }
+ const packages = resolve(mirrorRoot, 'packages');
+ const paths: Record<string, string[]> = {};
+ for (const pkgJsonPath of packageJsonPaths) {
+ const dir = resolve(packages, relative(resolve(root, 'packages'), dirname(pkgJsonPath)));
+ if (existsSync(resolve(dir, 'dist/index.d.ts'))) paths[manifest(pkgJsonPath).name] = [resolve(dir, 'dist/index.d.ts')];
+ }
+ mirror = { packages, paths };
+ return mirror;
+}
+
+function extract(pkgDir: string, pkgJsonPath: string, entry: string, paths?: Record<string, string[]>) {
+ const config = ExtractorConfig.prepare({
+ configObjectFullPath: baseConfigPath,
+ packageJsonFullPath: pkgJsonPath,
+ configObject: {
+ ...baseConfig,
+ projectFolder: pkgDir,
+ mainEntryPointFilePath: entry,
+ compiler: paths
+ ? { overrideTsconfig: { compilerOptions: { baseUrl: root, paths } } }
+ : baseConfig.compiler,
+ apiReport: {
+ enabled: true,
+ reportFileName: reportFileName(manifest(pkgJsonPath).name),
+ reportFolder,
+ reportTempFolder: resolve(reportFolder, 'temp'),
+ },
+ },
+ });
+ return Extractor.invoke(config, { localBuild: !verify, showVerboseMessages: false });
+}
+
+function main() {
+ let failed = 0;
+
+ // The build injects `// @ts-ignore` lines into the `.d.ts` files that crash API
+ // Extractor's AST walker, so strip them for the duration of the run and restore after.
+ const originals = new Map<string, string>();
+ for (const file of globbySync('packages/*/dist/**/*.d.ts', { cwd: root, absolute: true })) {
+ const content = readFileSync(file, 'utf8');
+ if (content.includes('@ts-ignore optional peer dependency')) {
+ originals.set(file, content);
+ writeFileSync(file, stripTsIgnore(content));
+ }
+ }
+
+ try {
+ for (const pkgJsonPath of packageJsonPaths) {
+ const pkg = manifest(pkgJsonPath);
+ if (pkg.private || EXCLUDED.has(pkg.name)) continue;
+
+ const pkgDir = dirname(pkgJsonPath);
+ const entry = dtsEntry(pkgDir, pkg);
+ if (!entry) {
+ const message = `${pkg.name}: no built dist/index.d.ts — run "pnpm build" first`;
+ console.error(`✗ ${message}`);
+ ghCommand('error', message);
+ failed++;
+ continue;
+ }
+
+ // Up to date iff the committed report didn't change. Extractor warnings are
+ // diagnostics, not BC-surface changes, so we key success on apiReportChanged.
+ const ok = (result: { apiReportChanged: boolean }, via = '') => {
+ if (verify && result.apiReportChanged) {
+ const message = `${pkg.name}: report out of date${via} — run "pnpm api:extract" and commit the changes in docs/public-api/`;
+ console.error(`✗ ${pkg.name}: report out of date${via}`);
+ ghCommand('error', message);
+ failed++;
+ } else {
+ console.log(`✓ ${pkg.name}${via}`);
+ }
+ };
+
+ try {
+ ok(extract(pkgDir, pkgJsonPath, entry));
+ } catch {
+ // Fallback: retry against the sanitized mirror (dodges an API Extractor crash
+ // on cross-package re-exports of comment-injected members, e.g. @crawlee/browser).
+ try {
+ const { packages, paths } = getMirror();
+ const mirrorEntry = resolve(packages, relative(resolve(root, 'packages'), pkgDir), relative(pkgDir, entry));
+ const { [pkg.name]: _self, ...deps } = paths;
+ ok(extract(pkgDir, pkgJsonPath, mirrorEntry, deps), ' (via mirror)');
+ } catch (err) {
+ const message = `${pkg.name}: api-extractor crashed: ${(err as Error).message}`;
+ console.error(`✗ ${message}`);
+ ghCommand('error', message);
+ failed++;
+ }
+ }
+ }
+ } finally {
+ for (const [file, content] of originals) writeFileSync(file, content);
+ rmSync(mirrorRoot, { recursive: true, force: true });
+ }
+
+ if (failed > 0) {
+ if (verify) console.error('\nRun "pnpm api:extract" and commit the changes in docs/public-api/.');
+ process.exit(1);
+ }
+}
+
+main();