Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

5.4 KB
Raw Blame

Public API Report File for "@crawlee/browser"

Do not edit this file. It is a report generated by API Extractor.


import type { AddRequestsBatchedResult } from '@crawlee/basic';
import type { Awaitable } from '@crawlee/types';
import { BasicCrawler } from '@crawlee/basic';
import { BasicCrawlerOptions } from '@crawlee/basic';
import { BrowserPluginOptions } from '@crawlee/browser-pool';
import type { CommonPage } from '@crawlee/browser-pool';
import { ContextPipeline } from '@crawlee/basic';
import type { CrawlingContext } from '@crawlee/basic';
import { CrawlingRequest } from '@crawlee/basic';
import type { Dictionary } from '@crawlee/types';
import type { EnqueueLinksOptions } from '@crawlee/basic';
import type { ErrorHandler } from '@crawlee/basic';
import type { ExtractLinksOptions } from '@crawlee/basic';
import type { GetUserDataFromRequest } from '@crawlee/basic';
import type { IBrowserPool } from '@crawlee/types';
import type { LoadedRequest } from '@crawlee/basic';
import { RemoteBrowserOptions } from '@crawlee/browser-pool';
import type { RequestHandler } from '@crawlee/basic';
import type { RouterHandler } from '@crawlee/basic';

// Not exported by the entry point; reachable only as a referenced type.
// @public (undocumented)
interface BaseResponse {
    headers?(): Record<string, string>;
    // (undocumented)
    status(): number;
}

// @public
export abstract class BrowserCrawler<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, Context extends BrowserCrawlingContext<Page, Response> = BrowserCrawlingContext<Page, Response>, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}, GoToOptions extends Dictionary = Dictionary> extends BasicCrawler<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
    get browserPool(): IBrowserPool<Page>;
    // (undocumented)
    protected buildContextPipeline(): ContextPipeline<CrawlingContext, BrowserCrawlingContext<Page, Response, Dictionary>>;
    // (undocumented)
    destroy(): Promise<void>;
    // (undocumented)
    protected readonly ignoreIframes: boolean;
    // (undocumented)
    protected readonly ignoreShadowRoots: boolean;
    // (undocumented)
    protected abstract navigationHandler(crawlingContext: BrowserCrawlingContext<Page, Response>, gotoOptions: GoToOptions): Promise<Context['response'] | null | undefined>;
    teardown(): Promise<void>;
}

// @public (undocumented)
export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, Context extends BrowserCrawlingContext<Page, Response> = BrowserCrawlingContext<Page, Response>, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> extends Omit<BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>, 'requestHandler' | 'failedRequestHandler' | 'errorHandler' | 'contextPipelineBuilder'> {
    browserPool?: IBrowserPool<Page>;
    errorHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
    failedRequestHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
    ignoreIframes?: boolean;
    ignoreShadowRoots?: boolean;
    navigationTimeoutSecs?: number;
    postNavigationHooks?: BrowserHook<Context, ContextExtension>[];
    preNavigationHooks?: BrowserHook<Context, ContextExtension>[];
    remoteBrowser?: CrawlerRemoteBrowserOptions;
    requestHandler?: RouterHandler<ExtendedContext, Routes> | RequestHandler<ExtendedContext>;
    saveResponseCookies?: boolean;
}

// @public (undocumented)
export interface BrowserCrawlingContext<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
GoToOptions extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
    enqueueLinks: (options?: EnqueueLinksOptions) => Promise<AddRequestsBatchedResult>;
    extractLinks: (options?: ExtractLinksOptions) => Promise<string[]>;
    gotoOptions: GoToOptions;
    page: Page;
    request: LoadedRequest<CrawlingRequest<UserData>>;
    response: Response;
}

// @public (undocumented)
export type BrowserHook<Context = BrowserCrawlingContext, ContextExtension = {}> = (crawlingContext: Context & ContextExtension) => Awaitable<void | Partial<Context>>;

// @public (undocumented)
export interface BrowserLaunchContext<TOptions, Launcher> extends BrowserPluginOptions<TOptions> {
    browserPerProxy?: boolean;
    ignoreProxyCertificate?: boolean;
    launcher?: Launcher;
    proxyUrl?: string;
    useChrome?: boolean;
    useIncognitoPages?: boolean;
    userAgent?: string;
    userDataDir?: string;
}

// @public
export interface CrawlerRemoteBrowserOptions extends RemoteBrowserOptions {
    maxOpenBrowsers?: number;
}

// @public
export type OwnedBrowserPool<Page> = IBrowserPool<Page> & {
    releaseAllBrowsers: () => Promise<void>;
    destroy: () => Promise<void>;
};


export * from "@crawlee/basic";

// (No @packageDocumentation comment for this package)