Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

2.1 KB
Raw Blame
import type { BrowserPoolHooks, BrowserPoolOptions, PuppeteerPlugin } from '@crawlee/browser-pool';
import { BrowserPool, RemotePuppeteerPlugin } from '@crawlee/browser-pool';
import type {
    BrowserCrawlerOptions,
    BrowserCrawlingContext,
    PuppeteerCrawlingContext,
    PuppeteerGoToOptions,
} from '@crawlee/puppeteer';
import { BrowserCrawler } from '@crawlee/puppeteer';
import type { Dictionary } from '@crawlee/types';
import puppeteer, { type HTTPResponse, type Page } from 'puppeteer';

export type TestCrawlingContext = BrowserCrawlingContext<Page, HTTPResponse, Dictionary>;

type TestBrowserPoolOptions = BrowserPoolOptions<PuppeteerPlugin> &
    BrowserPoolHooks<
        ReturnType<PuppeteerPlugin['createController']>,
        ReturnType<PuppeteerPlugin['createLaunchContext']>,
        Page
    >;

export class BrowserCrawlerTest extends BrowserCrawler<Page, HTTPResponse, TestCrawlingContext> {
    constructor(
        options: Partial<BrowserCrawlerOptions<Page, HTTPResponse, TestCrawlingContext>> & {
            /**
             * The concrete crawlers derive their plugin from a `launchContext`; this bare test subclass has no
             * launcher, so tests hand it the pool options - `browserPlugins` included - directly.
             */
            browserPoolOptions?: TestBrowserPoolOptions;
        } = {},
    ) {
        const { browserPoolOptions, ...browserCrawlerOptions } = options;

        super({
            ...browserCrawlerOptions,
            browserPoolBuilder: (remoteBrowser) =>
                remoteBrowser
                    ? new BrowserPool({
                          maxOpenBrowsers: remoteBrowser.maxOpenBrowsers,
                          browserPlugins: [new RemotePuppeteerPlugin(puppeteer, remoteBrowser)],
                      })
                    : new BrowserPool(browserPoolOptions!),
            contextPipelineBuilder: () => this.buildContextPipeline(),
        });
    }

    protected async navigationHandler(
        ctx: PuppeteerCrawlingContext,
        gotoOptions: PuppeteerGoToOptions,
    ): Promise<HTTPResponse | null | undefined> {
        return ctx.page.goto(ctx.request.url, gotoOptions);
    }
}