Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
import type { BrowserPoolHooks, BrowserPoolOptions, PuppeteerPlugin } from '@crawlee/browser-pool';
import { BrowserPool, RemotePuppeteerPlugin } from '@crawlee/browser-pool';
import type {
BrowserCrawlerOptions,
BrowserCrawlingContext,
PuppeteerCrawlingContext,
PuppeteerGoToOptions,
} from '@crawlee/puppeteer';
import { BrowserCrawler } from '@crawlee/puppeteer';
import type { Dictionary } from '@crawlee/types';
import puppeteer, { type HTTPResponse, type Page } from 'puppeteer';
export type TestCrawlingContext = BrowserCrawlingContext<Page, HTTPResponse, Dictionary>;
type TestBrowserPoolOptions = BrowserPoolOptions<PuppeteerPlugin> &
BrowserPoolHooks<
ReturnType<PuppeteerPlugin['createController']>,
ReturnType<PuppeteerPlugin['createLaunchContext']>,
Page
>;
export class BrowserCrawlerTest extends BrowserCrawler<Page, HTTPResponse, TestCrawlingContext> {
constructor(
options: Partial<BrowserCrawlerOptions<Page, HTTPResponse, TestCrawlingContext>> & {
/**
* The concrete crawlers derive their plugin from a `launchContext`; this bare test subclass has no
* launcher, so tests hand it the pool options - `browserPlugins` included - directly.
*/
browserPoolOptions?: TestBrowserPoolOptions;
} = {},
) {
const { browserPoolOptions, ...browserCrawlerOptions } = options;
super({
...browserCrawlerOptions,
browserPoolBuilder: (remoteBrowser) =>
remoteBrowser
? new BrowserPool({
maxOpenBrowsers: remoteBrowser.maxOpenBrowsers,
browserPlugins: [new RemotePuppeteerPlugin(puppeteer, remoteBrowser)],
})
: new BrowserPool(browserPoolOptions!),
contextPipelineBuilder: () => this.buildContextPipeline(),
});
}
protected async navigationHandler(
ctx: PuppeteerCrawlingContext,
gotoOptions: PuppeteerGoToOptions,
): Promise<HTTPResponse | null | undefined> {
return ctx.page.goto(ctx.request.url, gotoOptions);
}
}