Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

10.5 KB
Raw Blame

Public API Report File for "@crawlee/puppeteer"

Do not edit this file. It is a report generated by API Extractor.


import type { AnyNode } from 'domhandler';
import type { BatchAddRequestsResult } from '@crawlee/types';
import type { Browser } from 'puppeteer';
import { BrowserCrawler } from '@crawlee/browser';
import { BrowserCrawlerOptions } from '@crawlee/browser';
import type { BrowserCrawlingContext } from '@crawlee/browser';
import type { BrowserHook } from '@crawlee/browser';
import type { BrowserLaunchContext } from '@crawlee/browser';
import type { BrowserPool } from '@crawlee/browser-pool';
import type { BrowserPoolHooks } from '@crawlee/browser-pool';
import type { BrowserPoolOptions } from '@crawlee/browser-pool';
import type { Cheerio } from 'cheerio';
import type { CheerioAPI } from 'cheerio';
import type { ClickOptions } from 'puppeteer';
import { Configuration } from '@crawlee/browser';
import type { CrawlerRemoteBrowserOptions } from '@crawlee/browser';
import { Dictionary } from '@crawlee/types';
import type { GetUserDataFromRequest } from '@crawlee/browser';
import type { HTTPRequest } from 'puppeteer';
import type { HTTPResponse } from 'puppeteer';
import { IRequestManager } from '@crawlee/browser';
import type { Page } from 'puppeteer';
import { PuppeteerPlugin } from '@crawlee/browser-pool';
import { Request as Request_2 } from '@crawlee/browser';
import type { RequestTransform } from '@crawlee/browser';
import type { RouterHandler } from '@crawlee/browser';
import type { RouterRoutes } from '@crawlee/browser';
import type { RouteSchemas } from '@crawlee/browser';
import type { RoutesFromSchemas } from '@crawlee/browser';
import type { SkippedRequestCallback } from '@crawlee/browser';
import type { UrlPatternInput } from '@crawlee/browser';

// @public
function addInterceptRequestHandler(page: Page, handler: InterceptHandler): Promise<void>;

// @public
function blockRequests(page: Page, options?: BlockRequestsOptions): Promise<void>;

// @public (undocumented)
interface BlockRequestsOptions {
    extraUrlPatterns?: string[];
    urlPatterns?: string[];
}

// @public (undocumented)
type CompiledScriptFunction = (params: CompiledScriptParams) => Promise<unknown>;

// @public (undocumented)
interface CompiledScriptParams {
    // (undocumented)
    page: Page;
    // (undocumented)
    request: Request_2;
}

// @public
function compileScript(scriptString: string, context?: Dictionary): CompiledScriptFunction;

// @public
export function createPuppeteerRouter<Context extends PuppeteerCrawlingContext = PuppeteerCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;

// @public (undocumented)
export function createPuppeteerRouter<Context extends PuppeteerCrawlingContext = PuppeteerCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;

// @public (undocumented)
export function createPuppeteerRouter<Context extends PuppeteerCrawlingContext = PuppeteerCrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;

// @public
function enqueueLinksByClickingElements(options: EnqueueLinksByClickingElementsOptions): Promise<BatchAddRequestsResult>;

// @public (undocumented)
export interface EnqueueLinksByClickingElementsOptions {
    clickOptions?: ClickOptions;
    exclude?: readonly UrlPatternInput[];
    forefront?: boolean;
    include?: UrlPatternInput[];
    label?: string;
    maxWaitForPageIdleSecs?: number;
    onSkippedRequest?: SkippedRequestCallback;
    page: Page;
    requestManager: IRequestManager;
    selector: string;
    skipNavigation?: boolean;
    transformRequestFunction?: RequestTransform;
    userData?: Dictionary;
    waitForPageIdleSecs?: number;
}

// @public
function gotoExtended(page: Page, request: Request_2, gotoOptions?: PuppeteerDirectNavigationOptions): Promise<HTTPResponse | null>;

// @public
function infiniteScroll(page: Page, options?: InfiniteScrollOptions): Promise<void>;

// @public (undocumented)
interface InfiniteScrollOptions {
    buttonSelector?: string;
    maxScrollHeight?: number;
    scrollDownAndUp?: boolean;
    stopScrollCallback?: () => unknown | Promise<unknown>;
    timeoutSecs?: number;
    waitForSecs?: number;
}

// @public
function injectFile(page: Page, filePath: string, options?: InjectFileOptions): Promise<unknown>;

// @public (undocumented)
interface InjectFileOptions {
    surviveNavigations?: boolean;
}

// @public
function injectJQuery(page: Page, options?: {
    surviveNavigations?: boolean;
}): Promise<unknown>;

// @public (undocumented)
export type InterceptHandler = (request: HTTPRequest) => unknown;

// @public
export function launchPuppeteer(launchContext?: PuppeteerLaunchContext, configuration?: Configuration): Promise<Browser>;

// @public
function parseWithCheerio(page: Page, ignoreShadowRoots?: boolean, ignoreIframes?: boolean): Promise<CheerioAPI>;

// @public
export type PuppeteerBrowserPool = BrowserPool<{
    browserPlugins: [PuppeteerPlugin];
}, [PuppeteerPlugin]>;

// @public
export function puppeteerBrowserPool(options?: PuppeteerBrowserPoolOptions): PuppeteerBrowserPool;

// @public (undocumented)
export interface PuppeteerBrowserPoolOptions extends Omit<BrowserPoolOptions, 'browserPlugins'>, BrowserPoolHooks<ReturnType<PuppeteerPlugin['createController']>, ReturnType<PuppeteerPlugin['createLaunchContext']>, Page> {
    configuration?: Configuration;
    headless?: boolean | 'new' | 'old';
    launchContext?: PuppeteerLaunchContext;
    remoteBrowser?: CrawlerRemoteBrowserOptions;
}

// @public (undocumented)
interface PuppeteerContextUtils {
    addInterceptRequestHandler(handler: InterceptHandler): Promise<void>;
    blockRequests(options?: BlockRequestsOptions): Promise<void>;
    compileScript(scriptString: string, ctx?: Dictionary): CompiledScriptFunction;
    enqueueLinksByClickingElements(options: Omit<EnqueueLinksByClickingElementsOptions, 'page' | 'requestManager'>): Promise<BatchAddRequestsResult>;
    infiniteScroll(options?: InfiniteScrollOptions): Promise<void>;
    injectFile(filePath: string, options?: InjectFileOptions): Promise<unknown>;
    injectJQuery(): Promise<unknown>;
    parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;
    querySelectorAll(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
    querySelectorOne(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
    removeInterceptRequestHandler(handler: InterceptHandler): Promise<void>;
    saveSnapshot(options?: SaveSnapshotOptions): Promise<void>;
    waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
}

// @public
export class PuppeteerCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PuppeteerCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawler<Page, HTTPResponse, PuppeteerCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
    constructor(options?: PuppeteerCrawlerOptions<ContextExtension, ExtendedContext, Routes, StatisticStateExtension>);
    // (undocumented)
    protected navigationHandler(crawlingContext: PuppeteerCrawlingContext, gotoOptions: PuppeteerDirectNavigationOptions): Promise<HTTPResponse | null>;
}

// @public (undocumented)
export interface PuppeteerCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PuppeteerCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawlerOptions<Page, HTTPResponse, PuppeteerCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
    headless?: boolean | 'new' | 'old';
    launchContext?: PuppeteerLaunchContext;
    postNavigationHooks?: BrowserHook<PuppeteerCrawlingContext<GetUserDataFromRequest<ExtendedContext['request']>>, ContextExtension>[];
    preNavigationHooks?: BrowserHook<PuppeteerCrawlingContext<GetUserDataFromRequest<ExtendedContext['request']>>, ContextExtension>[];
}

// @public (undocumented)
export interface PuppeteerCrawlingContext<UserData extends Dictionary = any> extends BrowserCrawlingContext<Page, HTTPResponse, UserData, PuppeteerGoToOptions>, PuppeteerContextUtils {
}

// @public (undocumented)
export interface PuppeteerDirectNavigationOptions {
    referer?: string;
    timeout?: number;
    waitUntil?: 'domcontentloaded' | 'load' | 'networkidle' | 'networkidle0' | 'networkidle2';
}

// @public (undocumented)
export type PuppeteerGoToOptions = NonNullable<Parameters<Page['goto']>[1]>;

// @public (undocumented)
export type PuppeteerHook<UserData extends Dictionary = any> = BrowserHook<PuppeteerCrawlingContext<UserData>>;

// @public
export interface PuppeteerLaunchContext extends BrowserLaunchContext<PuppeteerPlugin['launchOptions'], unknown> {
    launcher?: unknown;
    launchOptions?: PuppeteerPlugin['launchOptions'];
    proxyUrl?: string;
    useChrome?: boolean;
    useIncognitoPages?: boolean;
}

declare namespace puppeteerUtils {
    export {
        injectFile,
        injectJQuery,
        parseWithCheerio,
        blockRequests,
        compileScript,
        gotoExtended,
        infiniteScroll,
        saveSnapshot,
        PuppeteerDirectNavigationOptions as DirectNavigationOptions,
        InjectFileOptions,
        BlockRequestsOptions,
        CompiledScriptParams,
        CompiledScriptFunction,
        InfiniteScrollOptions,
        SaveSnapshotOptions,
        PuppeteerContextUtils,
        enqueueLinksByClickingElements,
        addInterceptRequestHandler,
        removeInterceptRequestHandler
    }
}

// @public
function removeInterceptRequestHandler(page: Page, handler: InterceptHandler): Promise<void>;

// @public
function saveSnapshot(page: Page, options?: SaveSnapshotOptions): Promise<void>;

// @public (undocumented)
interface SaveSnapshotOptions {
    configuration?: Configuration;
    key?: string;
    keyValueStoreName?: string | null;
    saveHtml?: boolean;
    saveScreenshot?: boolean;
    screenshotQuality?: number;
}


export * from "@crawlee/browser";

// (No @packageDocumentation comment for this package)