Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: Migrate StagehandCrawler to ContextPipeline
Jan Bucharb821af2fb2ef
diff --git a/packages/memory-storage/src/resource-clients/dataset.ts b/packages/memory-storage/src/resource-clients/dataset.tsindex 976a30a05..d069c101e 100644--- a/packages/memory-storage/src/resource-clients/dataset.ts+++ b/packages/memory-storage/src/resource-clients/dataset.ts@@ -155,9 +155,9 @@ export class DatasetClient<Data extends Dictionary = Dictionary>): AsyncIterable<[number, Data]> & Promise<storage.PaginatedList<[number, Data]>> {const { desc, limit, offset } = s.object({- desc: s.boolean.optional,- limit: s.number.int.optional,- offset: s.number.int.optional,+ desc: s.boolean().optional(),+ limit: s.number().int().optional(),+ offset: s.number().int().optional(),}).parse(options);diff --git a/packages/memory-storage/src/resource-clients/key-value-store.ts b/packages/memory-storage/src/resource-clients/key-value-store.tsindex f7d66f145..53bc17404 100644--- a/packages/memory-storage/src/resource-clients/key-value-store.ts+++ b/packages/memory-storage/src/resource-clients/key-value-store.ts@@ -155,10 +155,10 @@ export class KeyValueStoreClient extends BaseClient {): AsyncIterable<string> & Promise<storage.KeyValueStoreClientListData> {const { limit, exclusiveStartKey, prefix } = s.object({- limit: s.number.greaterThan(0).optional,- exclusiveStartKey: s.string.optional,- collection: s.string.optional,- prefix: s.string.optional,+ limit: s.number().greaterThan(0).optional(),+ exclusiveStartKey: s.string().optional(),+ collection: s.string().optional(),+ prefix: s.string().optional(),}).parse(options);diff --git a/packages/stagehand-crawler/package.json b/packages/stagehand-crawler/package.jsonindex 6df5db631..92bdff384 100644--- a/packages/stagehand-crawler/package.json+++ b/packages/stagehand-crawler/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/stagehand",- "version": "3.18.1",+ "version": "3.17.0","description": "AI-powered web crawling with Stagehand integration for Crawlee - enables natural language browser automation with act(), extract(), and observe() methods.","engines": {"node": ">=16.0.0"@@ -56,15 +56,15 @@"access": "public"},"dependencies": {- "@apify/log": "^2.4.0",- "@apify/timeout": "^0.4.0",- "@crawlee/browser": "3.18.1",- "@crawlee/browser-pool": "3.18.1",- "@crawlee/core": "3.18.1",- "@crawlee/types": "3.18.1",- "@crawlee/utils": "3.18.1",- "ow": "^0.28.1",- "tslib": "^2.4.0"+ "@apify/log": "^2.5.18",+ "@apify/timeout": "^0.3.2",+ "@crawlee/browser": "4.0.0",+ "@crawlee/browser-pool": "4.0.0",+ "@crawlee/core": "4.0.0",+ "@crawlee/types": "4.0.0",+ "@crawlee/utils": "4.0.0",+ "ow": "^2.0.0",+ "tslib": "^2.8.1"},"peerDependencies": {"@browserbasehq/stagehand": "^3.0.0",diff --git a/packages/stagehand-crawler/src/internals/stagehand-crawler.ts b/packages/stagehand-crawler/src/internals/stagehand-crawler.tsindex f4011fce3..596ee82e7 100644--- a/packages/stagehand-crawler/src/internals/stagehand-crawler.ts+++ b/packages/stagehand-crawler/src/internals/stagehand-crawler.ts@@ -15,15 +15,18 @@ import type {BrowserCrawlerOptions,BrowserCrawlingContext,BrowserHook,- BrowserRequestHandler,+ ContextPipeline,+ CrawlingContext,GetUserDataFromRequest,LoadedContext,+ RequestHandler,RouterHandler,RouterRoutes,RouteSchemas,RoutesFromSchemas,} from '@crawlee/browser';-import { BrowserCrawler, Configuration, Router } from '@crawlee/browser';+import { BrowserCrawler, Router } from '@crawlee/browser';+import type { BrowserPoolOptions } from '@crawlee/browser-pool';import type { Dictionary } from '@crawlee/types';import ow from 'ow';import type { LaunchOptions, Page, Response } from 'playwright';@@ -208,7 +211,7 @@ export interface StagehandPage extends Page {* Crawling context for StagehandCrawler with enhanced page object.*/export interface StagehandCrawlingContext<UserData extends Dictionary = Dictionary>- extends BrowserCrawlingContext<StagehandCrawler, StagehandPage, Response, StagehandController, UserData> {+ extends BrowserCrawlingContext<StagehandPage, Response, StagehandController, UserData> {/*** Enhanced Playwright page with Stagehand AI methods.* Use page.act(), page.extract(), page.observe(), page.agent() for AI-powered operations.@@ -230,7 +233,7 @@ export interface StagehandHook extends BrowserHook<StagehandCrawlingContext, Sta/*** Request handler for StagehandCrawler.*/-export interface StagehandRequestHandler extends BrowserRequestHandler<LoadedContext<StagehandCrawlingContext>> {}+export interface StagehandRequestHandler extends RequestHandler<LoadedContext<StagehandCrawlingContext>> {}/*** Goto options for StagehandCrawler navigation.@@ -240,8 +243,18 @@ export type StagehandGotoOptions = Dictionary & Parameters<Page['goto']>[1];/*** Options for StagehandCrawler.*/-export interface StagehandCrawlerOptions- extends BrowserCrawlerOptions<StagehandCrawlingContext, { browserPlugins: [StagehandPlugin] }> {+export interface StagehandCrawlerOptions<+ ContextExtension = Dictionary<never>,+ ExtendedContext extends StagehandCrawlingContext = StagehandCrawlingContext & ContextExtension,+> extends BrowserCrawlerOptions<+ StagehandPage,+ Response,+ StagehandController,+ StagehandCrawlingContext,+ ContextExtension,+ ExtendedContext,+ { browserPlugins: [StagehandPlugin] }+ > {/*** Stagehand-specific configuration options.* These options configure the AI behavior and Browserbase integration.@@ -300,11 +313,6 @@ export interface StagehandCrawlerOptions*/requestHandler?: StagehandRequestHandler;- /**- * Function called when request handling fails after all retries.- */- failedRequestHandler?: StagehandRequestHandler;-/*** Async functions that are sequentially evaluated before the navigation.*/@@ -365,10 +373,18 @@ export interface StagehandCrawlerOptions* await crawler.run(['https://example.com']);* ```*/-export class StagehandCrawler extends BrowserCrawler<+export class StagehandCrawler<+ ContextExtension = Dictionary<never>,+ ExtendedContext extends StagehandCrawlingContext = StagehandCrawlingContext & ContextExtension,+> extends BrowserCrawler<+ StagehandPage,+ Response,+ StagehandController,{ browserPlugins: [StagehandPlugin] },LaunchOptions,- StagehandCrawlingContext+ StagehandCrawlingContext,+ ContextExtension,+ ExtendedContext> {protected static override optionsShape = {...BrowserCrawler.optionsShape,@@ -388,87 +404,49 @@ export class StagehandCrawler extends BrowserCrawler<** @param options - Crawler configuration options*/- constructor(- options: StagehandCrawlerOptions = {},- override readonly config = Configuration.getGlobalConfig(),- ) {- const {- stagehandOptions = {},- launchContext = {},- browserPoolOptions = {},- ...browserCrawlerOptions- } = options;-- // Validate options+ constructor(options: StagehandCrawlerOptions<ContextExtension, ExtendedContext> = {}) {ow(options, 'StagehandCrawlerOptions', ow.object.exactShape(StagehandCrawler.optionsShape));+ const { stagehandOptions = {}, launchContext = {}, contextPipelineBuilder, ...browserCrawlerOptions } = options;++ const browserPoolOptions = {+ ...options.browserPoolOptions,+ } as BrowserPoolOptions;+// Create launcher with Stagehand plugin- const launcher = new StagehandLauncher(- {- ...launchContext,- stagehandOptions,- },- config,- );+ const launcher = new StagehandLauncher({+ ...launchContext,+ stagehandOptions,+ });++ browserPoolOptions.browserPlugins = [launcher.createBrowserPlugin()];// Initialize BrowserCrawler with Stagehand plugin and fingerprinting enabled- super(- {- ...browserCrawlerOptions,- launchContext,- browserPoolOptions: {- ...browserPoolOptions,- browserPlugins: [launcher.createBrowserPlugin()],- // Enable fingerprinting by default for anti-blocking- useFingerprints: browserPoolOptions.useFingerprints ?? true,- },- },- config,- );+ super({+ ...(browserCrawlerOptions as StagehandCrawlerOptions<ContextExtension, ExtendedContext>),+ launchContext,+ browserPoolOptions,+ contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),+ });+ }++ protected override buildContextPipeline(): ContextPipeline<CrawlingContext, StagehandCrawlingContext> {+ return super.buildContextPipeline().compose({ action: this.setUpStagehand.bind(this) });}/**- * Overrides the request handler to enhance the page with Stagehand AI methods.- *- * The pattern here is:- * 1. Store the original userProvidedRequestHandler- * 2. Replace it with a wrapper that enhances the page first- * 3. Call super (which creates page/browserController, then calls our wrapper)- * 4. Our wrapper enhances the page and calls the original handler- * 5. Restore the original handler- *- * This is similar to how PlaywrightCrawler adds utility methods via registerUtilsToContext,- * but we need to actually transform the page object to add Stagehand AI methods.+ * Enhance the page with Stagehand AI methods.*/- protected override async _runRequestHandler(crawlingContext: StagehandCrawlingContext): Promise<void> {- // Store the original handler (could be this.requestHandler or this.router)- const originalHandler = this.userProvidedRequestHandler!;-- // Keep `userRequestHandler` pointing at the unwrapped handler: the swap below is live while the user's- // handler runs, which is exactly when `context.addRequests`/`enqueueLinks` resolve router metadata.- this.unwrappedRequestHandler = originalHandler;-- // Replace with a wrapper that enhances the page before calling the user's handler- this.userProvidedRequestHandler = async (ctx: any) => {- // Get Stagehand instance from controller- const stagehand = (ctx.browserController as StagehandController).getStagehand();- ctx.stagehand = stagehand;-- // Enhance page with AI methods (page.act(), page.extract(), etc.)- ctx.page = enhancePageWithStagehand(ctx.page, stagehand) as StagehandPage;-- // Call the original user handler- return originalHandler(ctx);+ private async setUpStagehand(crawlingContext: {+ browserController: StagehandController;+ page: Page;+ }): Promise<{ stagehand: Stagehand; page: StagehandPage }> {+ const stagehand = crawlingContext.browserController.getStagehand();++ return {+ stagehand,+ page: enhancePageWithStagehand(crawlingContext.page, stagehand),};-- try {- // Call parent - this creates the page and eventually calls our wrapped handler- await super._runRequestHandler(crawlingContext);- } finally {- // Restore original handler- this.userProvidedRequestHandler = originalHandler;- this.unwrappedRequestHandler = undefined;- }}/**diff --git a/packages/stagehand-crawler/src/internals/stagehand-plugin.ts b/packages/stagehand-crawler/src/internals/stagehand-plugin.tsindex 2cdc9bec9..5488a0f61 100644--- a/packages/stagehand-crawler/src/internals/stagehand-plugin.ts+++ b/packages/stagehand-crawler/src/internals/stagehand-plugin.ts@@ -1,13 +1,12 @@+import log from '@apify/log';import type { Stagehand, V3Options } from '@browserbasehq/stagehand';import type { BrowserController, BrowserPluginOptions, LaunchContext } from '@crawlee/browser-pool';import { anonymizeProxySugar, BrowserPlugin } from '@crawlee/browser-pool';-import type { Browser as PlaywrightBrowser, BrowserType, LaunchOptions } from 'playwright';+import type { BrowserType, LaunchOptions, Browser as PlaywrightBrowser } from 'playwright';// Stagehand is built on CDP (Chrome DevTools Protocol), which only works with Chromium-based browsers.// Firefox and WebKit are not supported by Stagehand.import { chromium } from 'playwright';-import log from '@apify/log';-import { StagehandController } from './stagehand-controller';import type { StagehandOptions } from './stagehand-crawler';@@ -145,7 +144,7 @@ export class StagehandPlugin extends BrowserPlugin<BrowserType, LaunchOptions, P/*** Creates a controller for the Stagehand browser.*/- protected _createController(): BrowserController<BrowserType, LaunchOptions, PlaywrightBrowser> {+ override createController(): BrowserController<BrowserType, LaunchOptions, PlaywrightBrowser> {return new StagehandController(this, this.stagehandInstances) as any;}