Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
/**
* AI-powered web crawling with Stagehand integration for Crawlee.
*
* This package provides {@apilink StagehandCrawler}, which extends {@apilink BrowserCrawler}
* with natural language browser automation capabilities powered by Browserbase's Stagehand library.
*
* ## Key Features
*
* - **Natural Language Actions**: Use `page.act()` to perform actions with plain English instructions
* - **Structured Data Extraction**: Use `page.extract()` with Zod schemas for type-safe data extraction
* - **Action Discovery**: Use `page.observe()` to get AI-suggested actions
* - **Autonomous Agents**: Use `page.agent()` for complex multi-step workflows
* - **Anti-Blocking**: Automatic browser fingerprinting and Cloudflare bypass
* - **Browserbase Integration**: Optional cloud browser support
*
* @example
* ```typescript
* import { StagehandCrawler } from '@crawlee/stagehand';
* import { z } from 'zod';
*
* const crawler = new StagehandCrawler({
* stagehandOptions: {
* env: 'LOCAL',
* model: 'openai/gpt-4.1-mini',
* },
* async requestHandler({ page, request, log }) {
* log.info(`Processing ${request.url}`);
*
* // Use natural language to interact
* await page.act('Click the Products link');
*
* // Extract structured data
* const products = await page.extract(
* 'Get all products',
* z.object({
* items: z.array(z.object({
* name: z.string(),
* price: z.number(),
* })),
* })
* );
*
* await Dataset.pushData(products);
* },
* });
*
* await crawler.run(['https://example.com']);
* ```
*
* @module @crawlee/stagehand
*/
// Re-export everything from @crawlee/browser for convenience
export * from '@crawlee/browser';
// Export main crawler class
export { StagehandCrawler, createStagehandRouter } from './internals/stagehand-crawler.js';
// Export the browser pool factories, which are how a pool with non-default options reaches the crawler
export { stagehandBrowserPool } from './internals/stagehand-browser-pool.js';
export type { StagehandBrowserPool, StagehandBrowserPoolOptions } from './internals/stagehand-browser-pool.js';
// Export types
export type {
StagehandOptions,
StagehandPage,
StagehandCrawlingContext,
StagehandHook,
StagehandGotoOptions,
StagehandCrawlerOptions,
} from './internals/stagehand-crawler.js';
export type { StagehandLaunchContext } from './internals/stagehand-launcher.js';
// Re-export key types from Stagehand for convenience
export type {
ActOptions,
ActResult,
Action,
AgentConfig,
ExtractOptions,
ModelConfiguration,
ObserveOptions,
Stagehand,
} from '@browserbasehq/stagehand';