Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat: add user data generics to browser navigation hooks (#3970)

Martin Adámek28e24b4152dd
diff --git a/docs/public-api/crawlee-browser.api.md b/docs/public-api/crawlee-browser.api.md
index 7b53887da..aa7e4ea22 100644
--- a/docs/public-api/crawlee-browser.api.md
+++ b/docs/public-api/crawlee-browser.api.md
@@ -45,7 +45,7 @@ interface BaseResponse {
}
// @public
-export abstract class BrowserCrawler<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions, LaunchOptions extends Dictionary | undefined = Dictionary, Context extends BrowserCrawlingContext<Page, Response, Dictionary> = BrowserCrawlingContext<Page, Response, Dictionary>, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, GoToOptions extends Dictionary = Dictionary> extends BasicCrawler<Context, ContextExtension, ExtendedContext, Routes> {
+export abstract class BrowserCrawler<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions, LaunchOptions extends Dictionary | undefined = Dictionary, Context extends BrowserCrawlingContext<Page, Response> = BrowserCrawlingContext<Page, Response>, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, GoToOptions extends Dictionary = Dictionary> extends BasicCrawler<Context, ContextExtension, ExtendedContext, Routes> {
protected constructor(options: BrowserCrawlerOptions<Page, Response, Context, ContextExtension, ExtendedContext> & {
contextPipelineBuilder: () => ContextPipeline<CrawlingContext, Context>;
});
@@ -113,7 +113,7 @@ export abstract class BrowserCrawler<Page extends CommonPage = CommonPage, Respo
}
// @public (undocumented)
-export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, Context extends BrowserCrawlingContext<Page, Response, Dictionary> = BrowserCrawlingContext<Page, Response, Dictionary>, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, __BrowserPlugins extends BrowserPlugin[] = InferBrowserPluginArray<InternalBrowserPoolOptions['browserPlugins']>, __BrowserControllerReturn extends BrowserController = ReturnType<__BrowserPlugins[number]['createController']>, __LaunchContextReturn extends LaunchContext = ReturnType<__BrowserPlugins[number]['createLaunchContext']>> extends Omit<BasicCrawlerOptions<Context, ContextExtension, ExtendedContext>, 'requestHandler' | 'failedRequestHandler' | 'errorHandler'> {
+export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, Context extends BrowserCrawlingContext<Page, Response> = BrowserCrawlingContext<Page, Response>, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, __BrowserPlugins extends BrowserPlugin[] = InferBrowserPluginArray<InternalBrowserPoolOptions['browserPlugins']>, __BrowserControllerReturn extends BrowserController = ReturnType<__BrowserPlugins[number]['createController']>, __LaunchContextReturn extends LaunchContext = ReturnType<__BrowserPlugins[number]['createLaunchContext']>> extends Omit<BasicCrawlerOptions<Context, ContextExtension, ExtendedContext>, 'requestHandler' | 'failedRequestHandler' | 'errorHandler'> {
browserPool?: IBrowserPool<Page>;
browserPoolOptions?: Partial<BrowserPoolOptions> & Partial<BrowserPoolHooks<__BrowserControllerReturn, __LaunchContextReturn>>;
errorHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
@@ -132,7 +132,8 @@ export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Res
}
// @public (undocumented)
-export interface BrowserCrawlingContext<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, UserData extends Dictionary = Dictionary, GoToOptions extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
+export interface BrowserCrawlingContext<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
+GoToOptions extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
enqueueLinks: (options?: EnqueueLinksOptions) => Promise<BatchAddRequestsResult>;
gotoOptions: GoToOptions;
page: Page;
diff --git a/docs/public-api/crawlee-http.api.md b/docs/public-api/crawlee-http.api.md
index 07a7d75dc..f08ecdc5d 100644
--- a/docs/public-api/crawlee-http.api.md
+++ b/docs/public-api/crawlee-http.api.md
@@ -168,7 +168,7 @@ export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext
ignoreSslErrors?: boolean;
navigationTimeoutSecs?: number;
postNavigationHooks?: ((crawlingContext: CrawlingContextWithResponse & ContextExtension) => Awaitable<void | Partial<CrawlingContextWithResponse>>)[];
- preNavigationHooks?: InternalHttpHook<CrawlingContext, ContextExtension>[];
+ preNavigationHooks?: InternalHttpHook<CrawlingContext<any>, ContextExtension>[];
saveResponseCookies?: boolean;
suggestResponseEncoding?: string;
}
diff --git a/docs/public-api/crawlee-playwright.api.md b/docs/public-api/crawlee-playwright.api.md
index f54730fa8..9f8601c9f 100644
--- a/docs/public-api/crawlee-playwright.api.md
+++ b/docs/public-api/crawlee-playwright.api.md
@@ -100,7 +100,7 @@ export class AdaptivePlaywrightCrawler<ContextExtension = Dictionary<never>, Ext
}
// @public (undocumented)
-export interface AdaptivePlaywrightCrawlerContext<UserData extends Dictionary = Dictionary> extends CrawlingContext_2<UserData> {
+export interface AdaptivePlaywrightCrawlerContext<UserData extends Dictionary = any> extends CrawlingContext_2<UserData> {
// (undocumented)
enqueueLinks(options?: EnqueueLinksOptions): Promise<unknown>;
page: Page;
@@ -400,13 +400,13 @@ export interface PlaywrightCrawlerOptions<ContextExtension = Dictionary<never>,
browserPlugins: [PlaywrightPlugin];
}, Routes> {
launchContext?: PlaywrightLaunchContext;
- postNavigationHooks?: BrowserHook<PlaywrightCrawlingContext, ContextExtension>[];
- preNavigationHooks?: BrowserHook<PlaywrightCrawlingContext, ContextExtension>[];
+ postNavigationHooks?: BrowserHook<PlaywrightCrawlingContext<GetUserDataFromRequest<ExtendedContext['request']>>, ContextExtension>[];
+ preNavigationHooks?: BrowserHook<PlaywrightCrawlingContext<GetUserDataFromRequest<ExtendedContext['request']>>, ContextExtension>[];
requestHandler?: RouterHandler<ExtendedContext, Routes> | RequestHandler<ExtendedContext>;
}
// @public (undocumented)
-export interface PlaywrightCrawlingContext<UserData extends Dictionary = Dictionary> extends BrowserCrawlingContext<Page, Response_2, UserData, PlaywrightGotoOptions>, PlaywrightContextUtils {
+export interface PlaywrightCrawlingContext<UserData extends Dictionary = any> extends BrowserCrawlingContext<Page, Response_2, UserData, PlaywrightGotoOptions>, PlaywrightContextUtils {
}
// @public (undocumented)
@@ -420,8 +420,7 @@ export interface PlaywrightDirectNavigationOptions {
export type PlaywrightGotoOptions = NonNullable<Parameters<Page['goto']>[1]>;
// @public (undocumented)
-export interface PlaywrightHook extends BrowserHook<PlaywrightCrawlingContext> {
-}
+export type PlaywrightHook<UserData extends Dictionary = any> = BrowserHook<PlaywrightCrawlingContext<UserData>>;
// @public
export interface PlaywrightLaunchContext extends BrowserLaunchContext<LaunchOptions, BrowserType> {
diff --git a/docs/public-api/crawlee-puppeteer.api.md b/docs/public-api/crawlee-puppeteer.api.md
index 6d244e8c4..f26626465 100644
--- a/docs/public-api/crawlee-puppeteer.api.md
+++ b/docs/public-api/crawlee-puppeteer.api.md
@@ -252,12 +252,12 @@ export interface PuppeteerCrawlerOptions<ContextExtension = Dictionary<never>, E
browserPlugins: [PuppeteerPlugin];
}, Routes> {
launchContext?: PuppeteerLaunchContext;
- postNavigationHooks?: BrowserHook<PuppeteerCrawlingContext, ContextExtension>[];
- preNavigationHooks?: BrowserHook<PuppeteerCrawlingContext, ContextExtension>[];
+ postNavigationHooks?: BrowserHook<PuppeteerCrawlingContext<GetUserDataFromRequest<ExtendedContext['request']>>, ContextExtension>[];
+ preNavigationHooks?: BrowserHook<PuppeteerCrawlingContext<GetUserDataFromRequest<ExtendedContext['request']>>, ContextExtension>[];
}
// @public (undocumented)
-export interface PuppeteerCrawlingContext<UserData extends Dictionary = Dictionary> extends BrowserCrawlingContext<Page, HTTPResponse, UserData, PuppeteerGoToOptions>, PuppeteerContextUtils {
+export interface PuppeteerCrawlingContext<UserData extends Dictionary = any> extends BrowserCrawlingContext<Page, HTTPResponse, UserData, PuppeteerGoToOptions>, PuppeteerContextUtils {
}
// @public (undocumented)
@@ -271,8 +271,7 @@ export interface PuppeteerDirectNavigationOptions {
export type PuppeteerGoToOptions = NonNullable<Parameters<Page['goto']>[1]>;
// @public (undocumented)
-export interface PuppeteerHook extends BrowserHook<PuppeteerCrawlingContext> {
-}
+export type PuppeteerHook<UserData extends Dictionary = any> = BrowserHook<PuppeteerCrawlingContext<UserData>>;
// @public
export interface PuppeteerLaunchContext extends BrowserLaunchContext<PuppeteerPlugin['launchOptions'], unknown> {
diff --git a/docs/public-api/crawlee-stagehand.api.md b/docs/public-api/crawlee-stagehand.api.md
index 8581e2086..75d59b0a8 100644
--- a/docs/public-api/crawlee-stagehand.api.md
+++ b/docs/public-api/crawlee-stagehand.api.md
@@ -140,14 +140,14 @@ export interface StagehandCrawlerOptions<ContextExtension = Dictionary<never>, E
browserPlugins: [StagehandPlugin];
}, Routes> {
launchContext?: StagehandLaunchContext;
- postNavigationHooks?: StagehandHook[];
- preNavigationHooks?: StagehandHook[];
+ postNavigationHooks?: BrowserHook<StagehandCrawlingContext<GetUserDataFromRequest<ExtendedContext['request']>>, ContextExtension>[];
+ preNavigationHooks?: BrowserHook<StagehandCrawlingContext<GetUserDataFromRequest<ExtendedContext['request']>>, ContextExtension>[];
requestHandler?: RouterHandler<ExtendedContext, Routes> | RequestHandler<ExtendedContext>;
stagehandOptions?: StagehandOptions;
}
// @public (undocumented)
-export interface StagehandCrawlingContext<UserData extends Dictionary = Dictionary> extends BrowserCrawlingContext<StagehandPage, Response_2, UserData, StagehandGotoOptions> {
+export interface StagehandCrawlingContext<UserData extends Dictionary = any> extends BrowserCrawlingContext<StagehandPage, Response_2, UserData, StagehandGotoOptions> {
page: StagehandPage;
stagehand: Stagehand;
}
@@ -156,8 +156,7 @@ export interface StagehandCrawlingContext<UserData extends Dictionary = Dictiona
export type StagehandGotoOptions = NonNullable<Parameters<Page['goto']>[1]>;
// @public
-export interface StagehandHook extends BrowserHook<StagehandCrawlingContext> {
-}
+export type StagehandHook<UserData extends Dictionary = any> = BrowserHook<StagehandCrawlingContext<UserData>>;
// @public
export interface StagehandLaunchContext extends BrowserLaunchContext<LaunchOptions, BrowserType> {
diff --git a/docs/upgrading/upgrading_v4.md b/docs/upgrading/upgrading_v4.md
index c77732a5d..dc7905254 100644
--- a/docs/upgrading/upgrading_v4.md
+++ b/docs/upgrading/upgrading_v4.md
@@ -662,6 +662,10 @@ To support the new `ContextPipeline` / `extendContext` composition, the crawler
The exported handler types were reshaped accordingly. `ErrorHandler` and `RequestHandler` no longer wrap their context in `LoadedContext<...>`; `ErrorHandler` now takes two type parameters (`ErrorHandler<BaseContext, ExtendedContext>`), receiving `inputs: BaseContext & Partial<ExtendedContext>`. The `RestrictedCrawlingContext` and `LoadedContext` types are no longer exported from `@crawlee/basic`. If you imported or annotated these directly, update the references; if you only used the crawler options' `requestHandler` / `errorHandler` / `failedRequestHandler` callbacks with inferred parameter types, nothing changes.
+### Navigation hook types are now generic
+
+`PlaywrightHook`, `PuppeteerHook` and `StagehandHook` are now type aliases (previously interfaces) generic over the request's `userData` type. A hook that types its context — via the generic (e.g. `PlaywrightHook<MyUserData>`) or an explicit context annotation — is now assignable to the `preNavigationHooks` / `postNavigationHooks` options of an untyped crawler. If you extended one of these interfaces, use an intersection type instead.
+
### The `log` property is typed as `CrawleeLogger`
The `log` property exposed throughout the public API (on the crawling context, `Statistics`, `EventManager`, `SessionOptions`, `Dataset`, etc.) is now typed as the `CrawleeLogger` interface (from `@crawlee/types`) rather than the concrete `Log` class from `@apify/log`. If you consume it structurally — calling `log.info(...)`, `log.debug(...)`, `log.child(...)` — nothing changes. You only need to act if you explicitly annotated a variable or parameter with the `Log` type from `@apify/log` and assigned `context.log` to it; type it as `CrawleeLogger` instead.
diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.ts
index c954349ce..54f30f698 100644
--- a/packages/browser-crawler/src/internals/browser-crawler.ts
+++ b/packages/browser-crawler/src/internals/browser-crawler.ts
@@ -73,7 +73,7 @@ type ContextDifference<T, U> = Omit<U, keyof T> & Partial<U>;
export interface BrowserCrawlingContext<
Page extends CommonPage = CommonPage,
Response extends BaseResponse = BaseResponse,
- UserData extends Dictionary = Dictionary,
+ UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
GoToOptions extends Dictionary = Dictionary,
> extends CrawlingContext<UserData> {
/**
@@ -114,11 +114,7 @@ const readContextField = <T>(ctx: object, key: symbol): T => (ctx as Record<symb
export interface BrowserCrawlerOptions<
Page extends CommonPage = CommonPage,
Response extends BaseResponse = BaseResponse,
- Context extends BrowserCrawlingContext<Page, Response, Dictionary> = BrowserCrawlingContext<
- Page,
- Response,
- Dictionary
- >,
+ Context extends BrowserCrawlingContext<Page, Response> = BrowserCrawlingContext<Page, Response>,
ContextExtension = Dictionary<never>,
ExtendedContext extends Context = Context & ContextExtension,
InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions,
@@ -357,11 +353,7 @@ export abstract class BrowserCrawler<
Response extends BaseResponse = BaseResponse,
InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions,
LaunchOptions extends Dictionary | undefined = Dictionary,
- Context extends BrowserCrawlingContext<Page, Response, Dictionary> = BrowserCrawlingContext<
- Page,
- Response,
- Dictionary
- >,
+ Context extends BrowserCrawlingContext<Page, Response> = BrowserCrawlingContext<Page, Response>,
ContextExtension = Dictionary<never>,
ExtendedContext extends Context = Context & ContextExtension,
Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>,
diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.ts
index eed6d49b2..9b606e384 100644
--- a/packages/http-crawler/src/internals/http-crawler.ts
+++ b/packages/http-crawler/src/internals/http-crawler.ts
@@ -114,7 +114,7 @@ export interface HttpCrawlerOptions<
* ]
* ```
*/
- preNavigationHooks?: InternalHttpHook<CrawlingContext, ContextExtension>[];
+ preNavigationHooks?: InternalHttpHook<CrawlingContext<any>, ContextExtension>[];
/**
* Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
diff --git a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts
index 8f177f195..c4aa91de2 100644
--- a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts
+++ b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts
@@ -116,7 +116,7 @@ class AdaptivePlaywrightCrawlerStatistics extends Statistics {
}
export interface AdaptivePlaywrightCrawlerContext<
- UserData extends Dictionary = Dictionary,
+ UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
> extends CrawlingContext<UserData> {
request: LoadedRequest<Request<UserData>>;
/**
diff --git a/packages/playwright-crawler/src/internals/playwright-crawler.ts b/packages/playwright-crawler/src/internals/playwright-crawler.ts
index 996a06d4e..24aafe029 100644
--- a/packages/playwright-crawler/src/internals/playwright-crawler.ts
+++ b/packages/playwright-crawler/src/internals/playwright-crawler.ts
@@ -31,9 +31,13 @@ import { gotoExtended, playwrightUtils } from './utils/playwright-utils.js';
export type PlaywrightGotoOptions = NonNullable<Parameters<Page['goto']>[1]>;
-export interface PlaywrightCrawlingContext<UserData extends Dictionary = Dictionary>
+export interface PlaywrightCrawlingContext<
+ UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
+>
extends BrowserCrawlingContext<Page, Response, UserData, PlaywrightGotoOptions>, PlaywrightContextUtils {}
-export interface PlaywrightHook extends BrowserHook<PlaywrightCrawlingContext> {}
+export type PlaywrightHook<
+ UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
+> = BrowserHook<PlaywrightCrawlingContext<UserData>>;
export interface PlaywrightCrawlerOptions<
ContextExtension = Dictionary<never>,
@@ -96,7 +100,10 @@ export interface PlaywrightCrawlerOptions<
* ]
* ```
*/
- preNavigationHooks?: BrowserHook<PlaywrightCrawlingContext, ContextExtension>[];
+ preNavigationHooks?: BrowserHook<
+ PlaywrightCrawlingContext<GetUserDataFromRequest<ExtendedContext['request']>>,
+ ContextExtension
+ >[];
/**
* Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
@@ -114,7 +121,10 @@ export interface PlaywrightCrawlerOptions<
* ]
* ```
*/
- postNavigationHooks?: BrowserHook<PlaywrightCrawlingContext, ContextExtension>[];
+ postNavigationHooks?: BrowserHook<
+ PlaywrightCrawlingContext<GetUserDataFromRequest<ExtendedContext['request']>>,
+ ContextExtension
+ >[];
}
/**
diff --git a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts
index 62b0bc51c..c7165ba11 100644
--- a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts
+++ b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts
@@ -32,9 +32,13 @@ import { gotoExtended, puppeteerUtils } from './utils/puppeteer_utils.js';
export type PuppeteerGoToOptions = NonNullable<Parameters<Page['goto']>[1]>;
-export interface PuppeteerCrawlingContext<UserData extends Dictionary = Dictionary>
+export interface PuppeteerCrawlingContext<
+ UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
+>
extends BrowserCrawlingContext<Page, HTTPResponse, UserData, PuppeteerGoToOptions>, PuppeteerContextUtils {}
-export interface PuppeteerHook extends BrowserHook<PuppeteerCrawlingContext> {}
+export type PuppeteerHook<
+ UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
+> = BrowserHook<PuppeteerCrawlingContext<UserData>>;
export interface PuppeteerCrawlerOptions<
ContextExtension = Dictionary<never>,
@@ -73,7 +77,10 @@ export interface PuppeteerCrawlerOptions<
* ]
* ```
*/
- preNavigationHooks?: BrowserHook<PuppeteerCrawlingContext, ContextExtension>[];
+ preNavigationHooks?: BrowserHook<
+ PuppeteerCrawlingContext<GetUserDataFromRequest<ExtendedContext['request']>>,
+ ContextExtension
+ >[];
/**
* Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
@@ -91,7 +98,10 @@ export interface PuppeteerCrawlerOptions<
* ]
* ```
*/
- postNavigationHooks?: BrowserHook<PuppeteerCrawlingContext, ContextExtension>[];
+ postNavigationHooks?: BrowserHook<
+ PuppeteerCrawlingContext<GetUserDataFromRequest<ExtendedContext['request']>>,
+ ContextExtension
+ >[];
}
/**
diff --git a/packages/stagehand-crawler/src/internals/stagehand-crawler.ts b/packages/stagehand-crawler/src/internals/stagehand-crawler.ts
index b54dfc58d..74120644e 100644
--- a/packages/stagehand-crawler/src/internals/stagehand-crawler.ts
+++ b/packages/stagehand-crawler/src/internals/stagehand-crawler.ts
@@ -215,12 +215,9 @@ export interface StagehandPage extends Page {
*/
export type StagehandGotoOptions = NonNullable<Parameters<Page['goto']>[1]>;
-export interface StagehandCrawlingContext<UserData extends Dictionary = Dictionary> extends BrowserCrawlingContext<
- StagehandPage,
- Response,
- UserData,
- StagehandGotoOptions
-> {
+export interface StagehandCrawlingContext<
+ UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
+> extends BrowserCrawlingContext<StagehandPage, Response, UserData, StagehandGotoOptions> {
/**
* Enhanced Playwright page with Stagehand AI methods.
* Use page.act(), page.extract(), page.observe(), page.agent() for AI-powered operations.
@@ -237,7 +234,9 @@ export interface StagehandCrawlingContext<UserData extends Dictionary = Dictiona
/**
* Hook function for StagehandCrawler.
*/
-export interface StagehandHook extends BrowserHook<StagehandCrawlingContext> {}
+export type StagehandHook<
+ UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
+> = BrowserHook<StagehandCrawlingContext<UserData>>;
/**
* Request handler for StagehandCrawler.
@@ -327,12 +326,18 @@ export interface StagehandCrawlerOptions<
/**
* Async functions that are sequentially evaluated before the navigation.
*/
- preNavigationHooks?: StagehandHook[];
+ preNavigationHooks?: BrowserHook<
+ StagehandCrawlingContext<GetUserDataFromRequest<ExtendedContext['request']>>,
+ ContextExtension
+ >[];
/**
* Async functions that are sequentially evaluated after the navigation.
*/
- postNavigationHooks?: StagehandHook[];
+ postNavigationHooks?: BrowserHook<
+ StagehandCrawlingContext<GetUserDataFromRequest<ExtendedContext['request']>>,
+ ContextExtension
+ >[];
}
/**
diff --git a/test/core/crawlers/adaptive_playwright_types.test.ts b/test/core/crawlers/adaptive_playwright_types.test.ts
new file mode 100644
index 000000000..957daf72b
--- /dev/null
+++ b/test/core/crawlers/adaptive_playwright_types.test.ts
@@ -0,0 +1,22 @@
+import type { AdaptivePlaywrightCrawlerContext, AdaptivePlaywrightCrawlerOptions } from '@crawlee/playwright';
+import type { Dictionary } from '@crawlee/types';
+
+/**
+ * Type-level regression test: a request handler typed with custom user data must stay assignable
+ * to an untyped `AdaptivePlaywrightCrawlerOptions`, like in the HTTP-based crawlers (issue #2063).
+ */
+
+interface OrderUserData extends Dictionary {
+ label: string;
+}
+
+describe('AdaptivePlaywrightCrawler option types (#2063)', () => {
+ test('request handler typed with custom user data stays assignable', () => {
+ const requestHandler = async ({ request }: AdaptivePlaywrightCrawlerContext<OrderUserData>) =>
+ void request.userData.label;
+
+ const options: AdaptivePlaywrightCrawlerOptions = { requestHandler };
+
+ expect(options).toBeTruthy();
+ });
+});
diff --git a/test/core/crawlers/navigation_hook_types.test.ts b/test/core/crawlers/navigation_hook_types.test.ts
new file mode 100644
index 000000000..7ae0d7bcf
--- /dev/null
+++ b/test/core/crawlers/navigation_hook_types.test.ts
@@ -0,0 +1,185 @@
+import type { CrawlingContext } from '@crawlee/basic';
+import type { CheerioCrawlerOptions } from '@crawlee/cheerio';
+import type { InternalHttpHook } from '@crawlee/http';
+import type { PlaywrightCrawlerOptions, PlaywrightCrawlingContext, PlaywrightHook } from '@crawlee/playwright';
+import type { PuppeteerCrawlerOptions, PuppeteerCrawlingContext, PuppeteerHook } from '@crawlee/puppeteer';
+import type { StagehandCrawlerOptions, StagehandCrawlingContext, StagehandHook } from '@crawlee/stagehand';
+import type { Dictionary } from '@crawlee/types';
+
+/**
+ * Type-level regression test for https://github.com/apify/crawlee/issues/2063.
+ */
+
+interface OrderUserData extends Dictionary {
+ label: string;
+}
+
+describe('navigation hook option types (#2063)', () => {
+ test('puppeteer - hooks with explicitly typed context stay assignable', () => {
+ const preNavigationHook = async ({ request, gotoOptions }: PuppeteerCrawlingContext<OrderUserData>) => {
+ void request.userData.label;
+ gotoOptions.timeout = 60_000;
+ };
+ const postNavigationHook = async ({ request }: PuppeteerCrawlingContext<OrderUserData>) =>
+ void request.userData.label;
+
+ const options: PuppeteerCrawlerOptions = {
+ preNavigationHooks: [preNavigationHook],
+ postNavigationHooks: [postNavigationHook],
+ };
+
+ expect(options).toBeTruthy();
+ });
+
+ test('puppeteer - hooks typed via the PuppeteerHook generic', () => {
+ const hook: PuppeteerHook<OrderUserData> = async ({ request }) => void request.userData.label;
+
+ const options: PuppeteerCrawlerOptions = {
+ preNavigationHooks: [hook],
+ postNavigationHooks: [hook],
+ };
+
+ expect(options).toBeTruthy();
+ });
+
+ test('playwright - hooks with explicitly typed context stay assignable', () => {
+ const preNavigationHook = async ({ request, gotoOptions }: PlaywrightCrawlingContext<OrderUserData>) => {
+ void request.userData.label;
+ gotoOptions.timeout = 60_000;
+ };
+ const postNavigationHook = async ({ request }: PlaywrightCrawlingContext<OrderUserData>) =>
+ void request.userData.label;
+
+ const options: PlaywrightCrawlerOptions = {
+ preNavigationHooks: [preNavigationHook],
+ postNavigationHooks: [postNavigationHook],
+ };
+
+ expect(options).toBeTruthy();
+ });
+
+ test('playwright - hooks typed via the PlaywrightHook generic', () => {
+ const hook: PlaywrightHook<OrderUserData> = async ({ request }) => void request.userData.label;
+
+ const options: PlaywrightCrawlerOptions = {
+ preNavigationHooks: [hook],
+ postNavigationHooks: [hook],
+ };
+
+ expect(options).toBeTruthy();
+ });
+
+ test('stagehand - hooks with explicitly typed context stay assignable', () => {
+ const preNavigationHook = async ({ request, gotoOptions }: StagehandCrawlingContext<OrderUserData>) => {
+ void request.userData.label;
+ gotoOptions.timeout = 60_000;
+ };
+ const postNavigationHook = async ({ request }: StagehandCrawlingContext<OrderUserData>) =>
+ void request.userData.label;
+
+ const options: StagehandCrawlerOptions = {
+ preNavigationHooks: [preNavigationHook],
+ postNavigationHooks: [postNavigationHook],
+ };
+
+ expect(options).toBeTruthy();
+ });
+
+ test('stagehand - hooks typed via the StagehandHook generic', () => {
+ const hook: StagehandHook<OrderUserData> = async ({ request }) => void request.userData.label;
+
+ const options: StagehandCrawlerOptions = {
+ preNavigationHooks: [hook],
+ postNavigationHooks: [hook],
+ };
+
+ expect(options).toBeTruthy();
+ });
+
+ test('cheerio - pre-navigation hook typed with custom user data stays assignable', () => {
+ const hook: InternalHttpHook<CrawlingContext<OrderUserData>> = async ({ request }) =>
+ void request.userData.label;
+
+ const options: CheerioCrawlerOptions = {
+ preNavigationHooks: [hook],
+ };
+
+ expect(options).toBeTruthy();
+ });
+});
+
+describe('request handler option types (#2063)', () => {
+ test('playwright - request handler typed with custom user data stays assignable', () => {
+ const requestHandler = async ({ request }: PlaywrightCrawlingContext<OrderUserData>) =>
+ void request.userData.label;
+
+ const options: PlaywrightCrawlerOptions = { requestHandler };
+
+ expect(options).toBeTruthy();
+ });
+
+ test('puppeteer - request handler typed with custom user data stays assignable', () => {
+ const requestHandler = async ({ request }: PuppeteerCrawlingContext<OrderUserData>) =>
+ void request.userData.label;
+
+ const options: PuppeteerCrawlerOptions = { requestHandler };
+
+ expect(options).toBeTruthy();
+ });
+
+ test('stagehand - request handler typed with custom user data stays assignable', () => {
+ const requestHandler = async ({ request }: StagehandCrawlingContext<OrderUserData>) =>
+ void request.userData.label;
+
+ const options: StagehandCrawlerOptions = { requestHandler };
+
+ expect(options).toBeTruthy();
+ });
+});
+
+describe('hooks with a crawler-level typed context (#2063)', () => {
+ test('playwright - hooks receive the user data typed via the options generic', () => {
+ const options: PlaywrightCrawlerOptions<Dictionary<never>, PlaywrightCrawlingContext<OrderUserData>> = {
+ requestHandler: async ({ request }) => {
+ expectTypeOf(request.userData).toEqualTypeOf<OrderUserData>();
+ },
+ preNavigationHooks: [
+ async ({ request, gotoOptions }) => {
+ expectTypeOf(request.userData).toEqualTypeOf<OrderUserData>();
+ void gotoOptions;
+ },
+ ],
+ postNavigationHooks: [
+ async ({ request }) => {
+ expectTypeOf(request.userData).toEqualTypeOf<OrderUserData>();
+ },
+ ],
+ };
+
+ expect(options).toBeTruthy();
+ });
+
+ test('puppeteer - hooks receive the user data typed via the options generic', () => {
+ const options: PuppeteerCrawlerOptions<Dictionary<never>, PuppeteerCrawlingContext<OrderUserData>> = {
+ preNavigationHooks: [
+ async ({ request }) => {
+ expectTypeOf(request.userData).toEqualTypeOf<OrderUserData>();
+ },
+ ],
+ };
+
+ expect(options).toBeTruthy();
+ });
+
+ test('stagehand - hooks receive the user data typed via the options generic', () => {
+ const options: StagehandCrawlerOptions<Dictionary<never>, StagehandCrawlingContext<OrderUserData>> = {
+ preNavigationHooks: [
+ async ({ request }) => {
+ expectTypeOf(request.userData).toEqualTypeOf<OrderUserData>();
+ },
+ ],
+ };
+
+ expect(options).toBeTruthy();
+ });
+});
diff --git a/test/tsconfig.json b/test/tsconfig.json
index fdbd3b4fc..6a60be6a1 100644
--- a/test/tsconfig.json
+++ b/test/tsconfig.json
@@ -19,6 +19,7 @@
"@crawlee/cheerio": ["../packages/cheerio-crawler/src/index.ts"],
"@crawlee/playwright": ["../packages/playwright-crawler/src/index.ts"],
"@crawlee/puppeteer": ["../packages/puppeteer-crawler/src/index.ts"],
+ "@crawlee/stagehand": ["../packages/stagehand-crawler/src/index.ts"],
"@crawlee/*": ["../packages/*/src/index.ts"]
}
}