Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat: infer router route map on the crawler & type request inputs (#3748)

Martin Adámek2070c102f7b1
diff --git a/docs/public-api/crawlee-basic.api.md b/docs/public-api/crawlee-basic.api.md
index e079855b7..5d8358fb4 100644
--- a/docs/public-api/crawlee-basic.api.md
+++ b/docs/public-api/crawlee-basic.api.md
@@ -57,13 +57,14 @@ import type { StorageBackend } from '@crawlee/types';
import type { StorageIdentifier } from '@crawlee/core';
import { StringPredicate } from 'ow';
import { TimeoutError } from '@apify/timeout';
+import type { TypedRequestsLike } from '@crawlee/core';
// @public (undocumented)
-export class BasicCrawler<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension> {
- constructor(options?: BasicCrawlerOptions<Context, ContextExtension, ExtendedContext> & RequireContextPipeline<CrawlingContext, Context>);
+export class BasicCrawler<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> {
+ constructor(options?: BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes> & RequireContextPipeline<CrawlingContext, Context>);
// (undocumented)
protected readonly additionalHttpErrorStatusCodes: Set<number>;
- addRequests(requests: ReadonlyDeep<RequestsLike>, options?: CrawlerAddRequestsOptions): Promise<CrawlerAddRequestsResult>;
+ addRequests(requests: ReadonlyDeep<TypedRequestsLike<Routes>>, options?: CrawlerAddRequestsOptions): Promise<CrawlerAddRequestsResult>;
autoscaledPool?: AutoscaledPool;
get basicContextPipeline(): ContextPipeline<{
request: Request_2;
@@ -161,8 +162,8 @@ export class BasicCrawler<Context extends CrawlingContext = CrawlingContext, Con
protected requestManager?: IRequestManager;
// (undocumented)
protected readonly retryOnBlocked: boolean;
- readonly router: RouterHandler<Context>;
- run(requests?: RequestsLike, options?: CrawlerRunOptions): Promise<FinalStatistics>;
+ readonly router: RouterHandler<Context, Routes>;
+ run(requests?: TypedRequestsLike<Routes>, options?: CrawlerRunOptions): Promise<FinalStatistics>;
// (undocumented)
running: boolean;
// (undocumented)
@@ -178,7 +179,7 @@ export class BasicCrawler<Context extends CrawlingContext = CrawlingContext, Con
}
// @public (undocumented)
-export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension> {
+export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> {
additionalHttpErrorStatusCodes?: number[];
autoscaledPoolOptions?: AutoscaledPoolOptions;
blockedStatusCodes?: number[];
@@ -201,7 +202,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
minConcurrency?: number;
onSkippedRequest?: SkippedRequestCallback;
proxyConfiguration?: IProxyConfiguration;
- requestHandler?: RequestHandler<ExtendedContext>;
+ requestHandler?: RouterHandler<ExtendedContext, Routes> | RequestHandler<ExtendedContext>;
requestHandlerTimeoutSecs?: number;
// @deprecated
requestList?: IRequestLoader;
@@ -273,10 +274,10 @@ export type RequireContextPipeline<DefaultContextType extends CrawlingContext, F
};
// @public (undocumented)
-export type StatusMessageCallback<Context extends CrawlingContext = BasicCrawlingContext, Crawler extends BasicCrawler<any> = BasicCrawler<Context>> = (params: StatusMessageCallbackParams<Context, Crawler>) => Awaitable<void>;
+export type StatusMessageCallback<Context extends CrawlingContext = BasicCrawlingContext, Crawler extends BasicCrawler<any, any, any, any> = BasicCrawler<Context>> = (params: StatusMessageCallbackParams<Context, Crawler>) => Awaitable<void>;
// @public (undocumented)
-export interface StatusMessageCallbackParams<Context extends CrawlingContext = BasicCrawlingContext, Crawler extends BasicCrawler<any> = BasicCrawler<Context>> {
+export interface StatusMessageCallbackParams<Context extends CrawlingContext = BasicCrawlingContext, Crawler extends BasicCrawler<any, any, any, any> = BasicCrawler<Context>> {
// (undocumented)
crawler: Crawler;
// (undocumented)
diff --git a/docs/public-api/crawlee-browser.api.md b/docs/public-api/crawlee-browser.api.md
index 2cc12e4fa..4e6525bda 100644
--- a/docs/public-api/crawlee-browser.api.md
+++ b/docs/public-api/crawlee-browser.api.md
@@ -28,6 +28,7 @@ import type { Dictionary } from '@crawlee/basic';
import type { Dictionary as Dictionary_2 } from '@crawlee/utils';
import type { EnqueueLinksOptions } from '@crawlee/basic';
import type { ErrorHandler } from '@crawlee/basic';
+import type { GetUserDataFromRequest } from '@crawlee/basic';
import type { IBrowserPool } from '@crawlee/types';
import type { InferBrowserPluginArray } from '@crawlee/browser-pool';
import { IRequestManager } from '@crawlee/basic';
@@ -40,11 +41,12 @@ import type { ReadonlyDeep } from 'type-fest';
import { Request as Request_2 } from '@crawlee/basic';
import type { RequestHandler } from '@crawlee/basic';
import type { RobotsTxtFile } from '@crawlee/utils';
+import type { RouterHandler } from '@crawlee/basic';
import type { SkippedRequestCallback } from '@crawlee/basic';
import { StringPredicate } from 'ow';
// @public
-export abstract class BrowserCrawler<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions, LaunchOptions extends Dictionary | undefined = Dictionary, Context extends BrowserCrawlingContext<Page, Response, Dictionary> = BrowserCrawlingContext<Page, Response, Dictionary>, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, GoToOptions extends Dictionary = Dictionary> extends BasicCrawler<Context, ContextExtension, ExtendedContext> {
+export abstract class BrowserCrawler<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions, LaunchOptions extends Dictionary | undefined = Dictionary, Context extends BrowserCrawlingContext<Page, Response, Dictionary> = BrowserCrawlingContext<Page, Response, Dictionary>, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, GoToOptions extends Dictionary = Dictionary> extends BasicCrawler<Context, ContextExtension, ExtendedContext, Routes> {
protected constructor(options: BrowserCrawlerOptions<Page, Response, Context, ContextExtension, ExtendedContext> & {
contextPipelineBuilder: () => ContextPipeline<CrawlingContext, Context>;
});
@@ -108,7 +110,7 @@ export abstract class BrowserCrawler<Page extends CommonPage = CommonPage, Respo
}
// @public (undocumented)
-export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, Context extends BrowserCrawlingContext<Page, Response, Dictionary> = BrowserCrawlingContext<Page, Response, Dictionary>, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions, __BrowserPlugins extends BrowserPlugin[] = InferBrowserPluginArray<InternalBrowserPoolOptions['browserPlugins']>, __BrowserControllerReturn extends BrowserController = ReturnType<__BrowserPlugins[number]['createController']>, __LaunchContextReturn extends LaunchContext = ReturnType<__BrowserPlugins[number]['createLaunchContext']>> extends Omit<BasicCrawlerOptions<Context, ContextExtension, ExtendedContext>, 'requestHandler' | 'failedRequestHandler' | 'errorHandler'> {
+export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, Context extends BrowserCrawlingContext<Page, Response, Dictionary> = BrowserCrawlingContext<Page, Response, Dictionary>, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, __BrowserPlugins extends BrowserPlugin[] = InferBrowserPluginArray<InternalBrowserPoolOptions['browserPlugins']>, __BrowserControllerReturn extends BrowserController = ReturnType<__BrowserPlugins[number]['createController']>, __LaunchContextReturn extends LaunchContext = ReturnType<__BrowserPlugins[number]['createLaunchContext']>> extends Omit<BasicCrawlerOptions<Context, ContextExtension, ExtendedContext>, 'requestHandler' | 'failedRequestHandler' | 'errorHandler'> {
browserPool?: IBrowserPool<Page>;
browserPoolOptions?: Partial<BrowserPoolOptions> & Partial<BrowserPoolHooks<__BrowserControllerReturn, __LaunchContextReturn>>;
errorHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
@@ -122,7 +124,7 @@ export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Res
postNavigationHooks?: BrowserHook<Context, ContextExtension>[];
preNavigationHooks?: BrowserHook<Context, ContextExtension>[];
remoteBrowser?: CrawlerRemoteBrowserOptions;
- requestHandler?: RequestHandler<ExtendedContext>;
+ requestHandler?: RouterHandler<ExtendedContext, Routes> | RequestHandler<ExtendedContext>;
saveResponseCookies?: boolean;
}
diff --git a/docs/public-api/crawlee-cheerio.api.md b/docs/public-api/crawlee-cheerio.api.md
index 8611d6759..a3e818c9d 100644
--- a/docs/public-api/crawlee-cheerio.api.md
+++ b/docs/public-api/crawlee-cheerio.api.md
@@ -29,8 +29,8 @@ import type { RoutesFromSchemas } from '@crawlee/http';
import type { SkippedRequestCallback } from '@crawlee/http';
// @public
-export class CheerioCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends CheerioCrawlingContext = CheerioCrawlingContext & ContextExtension> extends HttpCrawler<CheerioCrawlingContext, ContextExtension, ExtendedContext> {
- constructor(options?: CheerioCrawlerOptions<ContextExtension, ExtendedContext>);
+export class CheerioCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends CheerioCrawlingContext = CheerioCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<CheerioCrawlingContext['request']>>> extends HttpCrawler<CheerioCrawlingContext, ContextExtension, ExtendedContext, Routes> {
+ constructor(options?: CheerioCrawlerOptions<ContextExtension, ExtendedContext, any, any, Routes>);
// (undocumented)
protected buildContextPipeline(): ContextPipeline<CrawlingContext<Dictionary>, InternalHttpCrawlingContext<any, any> & {
readonly body: string;
@@ -44,7 +44,8 @@ export class CheerioCrawler<ContextExtension = Dictionary<never>, ExtendedContex
// @public (undocumented)
export interface CheerioCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends CheerioCrawlingContext = CheerioCrawlingContext & ContextExtension, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
-JSONData extends Dictionary = any> extends HttpCrawlerOptions<CheerioCrawlingContext<UserData, JSONData>, ContextExtension, ExtendedContext> {
+JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
+Routes extends Record<keyof Routes, Dictionary> = Record<string, UserData>> extends HttpCrawlerOptions<CheerioCrawlingContext<UserData, JSONData>, ContextExtension, ExtendedContext, Routes> {
}
// @public (undocumented)
diff --git a/docs/public-api/crawlee-core.api.md b/docs/public-api/crawlee-core.api.md
index 2820f4e36..edbe4cbf8 100644
--- a/docs/public-api/crawlee-core.api.md
+++ b/docs/public-api/crawlee-core.api.md
@@ -1343,9 +1343,9 @@ export class RetryRequestError extends Error {
// @public
export class Router<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'>, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> {
- addDefaultHandler<UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(handler: (ctx: RouterHandlerContext<Context, UserData>) => Awaitable_2<void>): void;
- addHandler<Label extends keyof Routes & string>(label: Label, handler: (ctx: RouterHandlerContext<Context, Routes[Label]>) => Awaitable_2<void>): void;
- addHandler<UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(label: RouterLabel<Routes>, handler: (ctx: RouterHandlerContext<Context, UserData>) => Awaitable_2<void>): void;
+ addDefaultHandler<UserData extends Dictionary = DefaultRouteUserData<Routes, GetUserDataFromRequest<Context['request']>>>(handler: (ctx: RouterHandlerContext<Context, UserData, Routes>) => Awaitable_2<void>): void;
+ addHandler<Label extends keyof Routes & string>(label: Label, handler: (ctx: RouterHandlerContext<Context, Routes[Label], Routes>) => Awaitable_2<void>): void;
+ addHandler<UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(label: RouterLabel<Routes>, handler: (ctx: RouterHandlerContext<Context, UserData, Routes>) => Awaitable_2<void>): void;
static create<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'> = CrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
// (undocumented)
static create<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'> = CrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
@@ -1362,9 +1362,14 @@ export interface RouterHandler<Context extends Omit<RestrictedCrawlingContext, '
}
// @public
-export type RouterHandlerContext<Context, UserData extends Dictionary> = Omit<Context, 'request'> & {
+export type RouterHandlerContext<Context, UserData extends Dictionary, Routes extends Record<keyof Routes, Dictionary>> = Omit<Context, 'request' | 'addRequests' | 'enqueueLinks'> & {
request: LoadedRequest<Request_2<UserData>>;
-};
+ addRequests: TypedContextAddRequests<Routes>;
+} & (Context extends {
+ enqueueLinks: infer EnqueueLinks;
+} ? {
+ enqueueLinks: TypedContextEnqueueLinks<EnqueueLinks, Routes>;
+} : {});
// @public
export type RouterLabel<Routes extends Record<keyof Routes, Dictionary>> = string extends keyof Routes ? string | symbol : (keyof Routes & string) | symbol;
@@ -1383,8 +1388,12 @@ export type RouteSchemas = Record<string, StandardSchemaV1> & {
// @public
export type RoutesFromSchemas<Schemas extends RouteSchemas> = {
- [Label in Extract<keyof Schemas, string>]: StandardSchemaV1.InferOutput<Schemas[Label]> extends Dictionary ? StandardSchemaV1.InferOutput<Schemas[Label]> : Dictionary;
-};
+ [Label in Extract<keyof Schemas, string>]: SchemaUserData<Schemas[Label]>;
+} & (Schemas extends {
+ [defaultRoute]: StandardSchemaV1;
+} ? {
+ [defaultRoute]: SchemaUserData<Schemas[typeof defaultRoute]>;
+} : {});
// @public
export function serializeValue(value: unknown, contentType?: string): {
diff --git a/docs/public-api/crawlee-http.api.md b/docs/public-api/crawlee-http.api.md
index 2917e5cff..09bf5f072 100644
--- a/docs/public-api/crawlee-http.api.md
+++ b/docs/public-api/crawlee-http.api.md
@@ -92,7 +92,7 @@ export type FileDownloadHook<UserData extends Dictionary = any> = InternalHttpHo
export type FileDownloadRequestHandler<UserData extends Dictionary = any> = RequestHandler<FileDownloadCrawlingContext<UserData>>;
// @public
-export class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any> = InternalHttpCrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension> extends BasicCrawler<Context, ContextExtension, ExtendedContext> {
+export class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any> = InternalHttpCrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> extends BasicCrawler<Context, ContextExtension, ExtendedContext, Routes> {
constructor(options?: HttpCrawlerOptions<Context, ContextExtension, ExtendedContext> & RequireContextPipeline<InternalHttpCrawlingContext, Context>);
// (undocumented)
protected buildContextPipeline(): ContextPipeline<CrawlingContext, InternalHttpCrawlingContext>;
@@ -146,7 +146,7 @@ export class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any> =
}
// @public (undocumented)
-export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext = InternalHttpCrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension> extends BasicCrawlerOptions<Context, ContextExtension, ExtendedContext> {
+export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext = InternalHttpCrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> extends BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes> {
additionalMimeTypes?: string[];
forceResponseEncoding?: string;
ignoreSslErrors?: boolean;
diff --git a/docs/public-api/crawlee-jsdom.api.md b/docs/public-api/crawlee-jsdom.api.md
index 27dcf81e8..bdeb78cec 100644
--- a/docs/public-api/crawlee-jsdom.api.md
+++ b/docs/public-api/crawlee-jsdom.api.md
@@ -46,8 +46,8 @@ export function createJSDOMRouter<Context extends JSDOMCrawlingContext = JSDOMCr
export function createJSDOMRouter<Context extends JSDOMCrawlingContext = JSDOMCrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;
// @public (undocumented)
-export class JSDOMCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext & ContextExtension> extends HttpCrawler<JSDOMCrawlingContext, ContextExtension, ExtendedContext> {
- constructor(options?: JSDOMCrawlerOptions<ContextExtension, ExtendedContext>);
+export class JSDOMCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<JSDOMCrawlingContext['request']>>> extends HttpCrawler<JSDOMCrawlingContext, ContextExtension, ExtendedContext, Routes> {
+ constructor(options?: JSDOMCrawlerOptions<ContextExtension, ExtendedContext, any, any, Routes>);
// (undocumented)
protected buildContextPipeline(): ContextPipeline<CrawlingContext<Dictionary>, InternalHttpCrawlingContext<any, any> & {
readonly window: DOMWindow;
@@ -110,7 +110,8 @@ export class JSDOMCrawler<ContextExtension = Dictionary<never>, ExtendedContext
// @public (undocumented)
export interface JSDOMCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext & ContextExtension, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
-JSONData extends Dictionary = any> extends HttpCrawlerOptions<JSDOMCrawlingContext<UserData, JSONData>, ContextExtension, ExtendedContext> {
+JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
+Routes extends Record<keyof Routes, Dictionary> = Record<string, UserData>> extends HttpCrawlerOptions<JSDOMCrawlingContext<UserData, JSONData>, ContextExtension, ExtendedContext, Routes> {
hideInternalConsole?: boolean;
runScripts?: boolean;
}
diff --git a/docs/public-api/crawlee-linkedom.api.md b/docs/public-api/crawlee-linkedom.api.md
index 77051c2b5..fd54fe8a4 100644
--- a/docs/public-api/crawlee-linkedom.api.md
+++ b/docs/public-api/crawlee-linkedom.api.md
@@ -36,8 +36,8 @@ export function createLinkeDOMRouter<Context extends LinkeDOMCrawlingContext = L
export function createLinkeDOMRouter<Context extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;
// @public
-export class LinkeDOMCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext & ContextExtension> extends HttpCrawler<LinkeDOMCrawlingContext, ContextExtension, ExtendedContext> {
- constructor(options: LinkeDOMCrawlerOptions<ContextExtension, ExtendedContext>);
+export class LinkeDOMCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<LinkeDOMCrawlingContext['request']>>> extends HttpCrawler<LinkeDOMCrawlingContext, ContextExtension, ExtendedContext, Routes> {
+ constructor(options: LinkeDOMCrawlerOptions<ContextExtension, ExtendedContext, any, any, Routes>);
// (undocumented)
protected buildContextPipeline(): ContextPipeline<CrawlingContext<Dictionary>, InternalHttpCrawlingContext<any, any> & {
readonly window: Window;
@@ -56,7 +56,8 @@ export interface LinkeDOMCrawlerEnqueueLinksOptions extends Omit<EnqueueLinksOpt
// @public (undocumented)
export interface LinkeDOMCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext & ContextExtension, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
-JSONData extends Dictionary = any> extends HttpCrawlerOptions<LinkeDOMCrawlingContext<UserData, JSONData>, ContextExtension, ExtendedContext> {
+JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
+Routes extends Record<keyof Routes, Dictionary> = Record<string, UserData>> extends HttpCrawlerOptions<LinkeDOMCrawlingContext<UserData, JSONData>, ContextExtension, ExtendedContext, Routes> {
}
// @public (undocumented)
diff --git a/docs/public-api/crawlee-playwright.api.md b/docs/public-api/crawlee-playwright.api.md
index 2e3075bae..d8c101d9e 100644
--- a/docs/public-api/crawlee-playwright.api.md
+++ b/docs/public-api/crawlee-playwright.api.md
@@ -65,8 +65,8 @@ import type { StatisticState } from '@crawlee/core';
import { StringPredicate } from 'ow';
// @public
-export class AdaptivePlaywrightCrawler<ContextExtension = Dictionary_2<never>, ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext & ContextExtension> extends BasicCrawler<AdaptivePlaywrightCrawlerContext, ContextExtension, ExtendedContext> {
- constructor(options?: AdaptivePlaywrightCrawlerOptions<ContextExtension, ExtendedContext>);
+export class AdaptivePlaywrightCrawler<ContextExtension = Dictionary_2<never>, ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary_2> = Record<string, GetUserDataFromRequest_2<AdaptivePlaywrightCrawlerContext['request']>>> extends BasicCrawler<AdaptivePlaywrightCrawlerContext, ContextExtension, ExtendedContext, Routes> {
+ constructor(options?: AdaptivePlaywrightCrawlerOptions<ContextExtension, ExtendedContext, Routes>);
// (undocumented)
protected buildContextPipeline(): ContextPipeline_2<CrawlingContext_2<Dictionary_2>, CrawlingContext_2<Dictionary_2> & {
readonly request: LoadedRequest<Request_3<Dictionary_2>>;
@@ -103,7 +103,7 @@ export interface AdaptivePlaywrightCrawlerContext<UserData extends Dictionary_2
}
// @public (undocumented)
-export interface AdaptivePlaywrightCrawlerOptions<ContextExtension = Dictionary_2<never>, ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext & ContextExtension> extends Omit<BasicCrawlerOptions<AdaptivePlaywrightCrawlerContext, ContextExtension, ExtendedContext>, 'preNavigationHooks' | 'postNavigationHooks'> {
+export interface AdaptivePlaywrightCrawlerOptions<ContextExtension = Dictionary_2<never>, ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary_2> = Record<string, GetUserDataFromRequest_2<AdaptivePlaywrightCrawlerContext['request']>>> extends Omit<BasicCrawlerOptions<AdaptivePlaywrightCrawlerContext, ContextExtension, ExtendedContext, Routes>, 'preNavigationHooks' | 'postNavigationHooks'> {
postNavigationHooks?: AdaptivePostNavigationHook<ContextExtension>[];
preNavigationHooks?: AdaptiveHook<ContextExtension>[];
preventDirectStorageAccess?: boolean;
@@ -250,10 +250,10 @@ declare namespace playwrightClickElements {
}
// @public
-export class PlaywrightCrawler<ContextExtension = Dictionary_2<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension> extends BrowserCrawler<Page, Response_2, {
+export class PlaywrightCrawler<ContextExtension = Dictionary_2<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary_2> = Record<string, GetUserDataFromRequest<PlaywrightCrawlingContext['request']>>> extends BrowserCrawler<Page, Response_2, {
browserPlugins: [PlaywrightPlugin];
-}, LaunchOptions, PlaywrightCrawlingContext, ContextExtension, ExtendedContext> {
- constructor(options?: PlaywrightCrawlerOptions<ContextExtension, ExtendedContext>);
+}, LaunchOptions, PlaywrightCrawlingContext, ContextExtension, ExtendedContext, Routes> {
+ constructor(options?: PlaywrightCrawlerOptions<ContextExtension, ExtendedContext, Routes>);
// (undocumented)
protected buildContextPipeline(): ContextPipeline<CrawlingContext<Dictionary_2>, BrowserCrawlingContext<Page, Response_2, Dictionary_2, Dictionary_2> & {
injectFile: (filePath: string, options?: InjectFileOptions) => Promise<unknown>;
@@ -323,13 +323,13 @@ export class PlaywrightCrawler<ContextExtension = Dictionary_2<never>, ExtendedC
}
// @public (undocumented)
-export interface PlaywrightCrawlerOptions<ContextExtension = Dictionary_2<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension> extends BrowserCrawlerOptions<Page, Response_2, PlaywrightCrawlingContext, ContextExtension, ExtendedContext, {
+export interface PlaywrightCrawlerOptions<ContextExtension = Dictionary_2<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary_2> = Record<string, GetUserDataFromRequest<PlaywrightCrawlingContext['request']>>> extends BrowserCrawlerOptions<Page, Response_2, PlaywrightCrawlingContext, ContextExtension, ExtendedContext, {
browserPlugins: [PlaywrightPlugin];
-}> {
+}, Routes> {
launchContext?: PlaywrightLaunchContext;
postNavigationHooks?: BrowserHook<PlaywrightCrawlingContext, ContextExtension>[];
preNavigationHooks?: BrowserHook<PlaywrightCrawlingContext, ContextExtension>[];
- requestHandler?: RequestHandler<ExtendedContext>;
+ requestHandler?: RouterHandler<ExtendedContext, Routes> | RequestHandler<ExtendedContext>;
}
// @public (undocumented)
diff --git a/docs/public-api/crawlee-puppeteer.api.md b/docs/public-api/crawlee-puppeteer.api.md
index 7e7e1fa99..f4ec52733 100644
--- a/docs/public-api/crawlee-puppeteer.api.md
+++ b/docs/public-api/crawlee-puppeteer.api.md
@@ -168,10 +168,10 @@ declare namespace puppeteerClickElements {
}
// @public
-export class PuppeteerCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension> extends BrowserCrawler<Page, HTTPResponse, {
+export class PuppeteerCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PuppeteerCrawlingContext['request']>>> extends BrowserCrawler<Page, HTTPResponse, {
browserPlugins: [PuppeteerPlugin];
-}, LaunchOptions, PuppeteerCrawlingContext, ContextExtension, ExtendedContext> {
- constructor(options?: PuppeteerCrawlerOptions<ContextExtension, ExtendedContext>);
+}, LaunchOptions, PuppeteerCrawlingContext, ContextExtension, ExtendedContext, Routes> {
+ constructor(options?: PuppeteerCrawlerOptions<ContextExtension, ExtendedContext, Routes>);
// (undocumented)
protected buildContextPipeline(): ContextPipeline<CrawlingContext<Dictionary>, BrowserCrawlingContext<Page, HTTPResponse, Dictionary, Dictionary> & {
injectFile: (filePath: string, options?: InjectFileOptions) => Promise<unknown>;
@@ -238,9 +238,9 @@ export class PuppeteerCrawler<ContextExtension = Dictionary<never>, ExtendedCont
}
// @public (undocumented)
-export interface PuppeteerCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension> extends BrowserCrawlerOptions<Page, HTTPResponse, PuppeteerCrawlingContext, ContextExtension, ExtendedContext, {
+export interface PuppeteerCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PuppeteerCrawlingContext['request']>>> extends BrowserCrawlerOptions<Page, HTTPResponse, PuppeteerCrawlingContext, ContextExtension, ExtendedContext, {
browserPlugins: [PuppeteerPlugin];
-}> {
+}, Routes> {
launchContext?: PuppeteerLaunchContext;
postNavigationHooks?: BrowserHook<PuppeteerCrawlingContext, ContextExtension>[];
preNavigationHooks?: BrowserHook<PuppeteerCrawlingContext, ContextExtension>[];
diff --git a/docs/public-api/crawlee-stagehand.api.md b/docs/public-api/crawlee-stagehand.api.md
index 177f3bb02..f35f7c262 100644
--- a/docs/public-api/crawlee-stagehand.api.md
+++ b/docs/public-api/crawlee-stagehand.api.md
@@ -78,10 +78,10 @@ export { ObserveOptions }
export { Stagehand }
// @public
-export class StagehandCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends StagehandCrawlingContext = StagehandCrawlingContext & ContextExtension> extends BrowserCrawler<StagehandPage, Response_2, {
+export class StagehandCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends StagehandCrawlingContext = StagehandCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<StagehandCrawlingContext['request']>>> extends BrowserCrawler<StagehandPage, Response_2, {
browserPlugins: [StagehandPlugin];
-}, LaunchOptions, StagehandCrawlingContext, ContextExtension, ExtendedContext> {
- constructor(options?: StagehandCrawlerOptions<ContextExtension, ExtendedContext>);
+}, LaunchOptions, StagehandCrawlingContext, ContextExtension, ExtendedContext, Routes> {
+ constructor(options?: StagehandCrawlerOptions<ContextExtension, ExtendedContext, Routes>);
// (undocumented)
protected buildContextPipeline(): ContextPipeline<CrawlingContext, StagehandCrawlingContext>;
protected _navigationHandler(crawlingContext: StagehandCrawlingContext, gotoOptions: StagehandGotoOptions): Promise<Response_2 | null>;
@@ -135,13 +135,13 @@ export class StagehandCrawler<ContextExtension = Dictionary<never>, ExtendedCont
}
// @public
-export interface StagehandCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends StagehandCrawlingContext = StagehandCrawlingContext & ContextExtension> extends BrowserCrawlerOptions<StagehandPage, Response_2, StagehandCrawlingContext, ContextExtension, ExtendedContext, {
+export interface StagehandCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends StagehandCrawlingContext = StagehandCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<StagehandCrawlingContext['request']>>> extends BrowserCrawlerOptions<StagehandPage, Response_2, StagehandCrawlingContext, ContextExtension, ExtendedContext, {
browserPlugins: [StagehandPlugin];
-}> {
+}, Routes> {
launchContext?: StagehandLaunchContext;
postNavigationHooks?: StagehandHook[];
preNavigationHooks?: StagehandHook[];
- requestHandler?: StagehandRequestHandler;
+ requestHandler?: RouterHandler<ExtendedContext, Routes> | RequestHandler<ExtendedContext>;
stagehandOptions?: StagehandOptions;
}
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index edc07ee3c..6125ac8b7 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -27,6 +27,7 @@ import type {
StatisticsOptions,
StatisticState,
StorageIdentifier,
+ TypedRequestsLike,
} from '@crawlee/core';
import {
AutoscaledPool,
@@ -143,7 +144,7 @@ export type ErrorHandler<
export interface StatusMessageCallbackParams<
Context extends CrawlingContext = BasicCrawlingContext,
- Crawler extends BasicCrawler<any> = BasicCrawler<Context>,
+ Crawler extends BasicCrawler<any, any, any, any> = BasicCrawler<Context>,
> {
state: StatisticState;
crawler: Crawler;
@@ -153,7 +154,7 @@ export interface StatusMessageCallbackParams<
export type StatusMessageCallback<
Context extends CrawlingContext = BasicCrawlingContext,
- Crawler extends BasicCrawler<any> = BasicCrawler<Context>,
+ Crawler extends BasicCrawler<any, any, any, any> = BasicCrawler<Context>,
> = (params: StatusMessageCallbackParams<Context, Crawler>) => Awaitable<void>;
export type RequireContextPipeline<
@@ -167,6 +168,7 @@ export interface BasicCrawlerOptions<
Context extends CrawlingContext = CrawlingContext,
ContextExtension = Dictionary<never>,
ExtendedContext extends Context = Context & ContextExtension,
+ Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>,
> {
/**
* User-provided function that performs the logic of the crawler. It is called for each URL to crawl.
@@ -185,7 +187,7 @@ export interface BasicCrawlerOptions<
* The exceptions are logged to the request using the
* {@apilink Request.pushErrorMessage|`Request.pushErrorMessage()`} function.
*/
- requestHandler?: RequestHandler<ExtendedContext>;
+ requestHandler?: RouterHandler<ExtendedContext, Routes> | RequestHandler<ExtendedContext>;
/**
* Allows the user to extend the crawling context with custom functionality (helpers, references, etc.).
@@ -570,6 +572,7 @@ export class BasicCrawler<
Context extends CrawlingContext = CrawlingContext,
ContextExtension = Dictionary<never>,
ExtendedContext extends Context = Context & ContextExtension,
+ Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>,
> {
protected static readonly CRAWLEE_STATE_KEY = 'CRAWLEE_STATE';
@@ -642,7 +645,10 @@ export class BasicCrawler<
* Default {@apilink Router} instance that will be used if we don't specify any {@apilink BasicCrawlerOptions.requestHandler|`requestHandler`}.
* See {@apilink Router.addHandler|`router.addHandler()`} and {@apilink Router.addDefaultHandler|`router.addDefaultHandler()`}.
*/
- readonly router: RouterHandler<Context> = Router.create<Context>();
+ readonly router: RouterHandler<Context, Routes> = Router.create<Context>() as unknown as RouterHandler<
+ Context,
+ Routes
+ >;
private _basicContextPipeline?: ContextPipeline<{ request: Request }, CrawlingContext>;
@@ -769,7 +775,7 @@ export class BasicCrawler<
* All `BasicCrawler` parameters are passed via an options object.
*/
constructor(
- options: BasicCrawlerOptions<Context, ContextExtension, ExtendedContext> &
+ options: BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes> &
RequireContextPipeline<CrawlingContext, Context> = {} as any, // cast because the constructor logic handles missing `contextPipelineBuilder` - the type is just for DX
) {
ow(options, 'BasicCrawlerOptions', ow.object.exactShape(BasicCrawler.optionsShape));
@@ -1357,7 +1363,7 @@ export class BasicCrawler<
* @param [requests] The requests to add.
* @param [options] Options for the request queue.
*/
- async run(requests?: RequestsLike, options?: CrawlerRunOptions): Promise<FinalStatistics> {
+ async run(requests?: TypedRequestsLike<Routes>, options?: CrawlerRunOptions): Promise<FinalStatistics> {
if (this.running) {
throw new Error(
'This crawler instance is already running, you can add more requests to it via `crawler.addRequests()`.',
@@ -1662,7 +1668,7 @@ export class BasicCrawler<
* @param options Options for the request queue
*/
async addRequests(
- requests: ReadonlyDeep<RequestsLike>,
+ requests: ReadonlyDeep<TypedRequestsLike<Routes>>,
options: CrawlerAddRequestsOptions = {},
): Promise<CrawlerAddRequestsResult> {
await this.getRequestManager();
diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.ts
index 9ccd68c07..6d206340e 100644
--- a/packages/browser-crawler/src/internals/browser-crawler.ts
+++ b/packages/browser-crawler/src/internals/browser-crawler.ts
@@ -7,10 +7,12 @@ import type {
Dictionary,
EnqueueLinksOptions,
ErrorHandler,
+ GetUserDataFromRequest,
IRequestManager,
LoadedRequest,
Request,
RequestHandler,
+ RouterHandler,
SkippedRequestCallback,
} from '@crawlee/basic';
import {
@@ -115,6 +117,7 @@ export interface BrowserCrawlerOptions<
ContextExtension = Dictionary<never>,
ExtendedContext extends Context = Context & ContextExtension,
InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions,
+ Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>,
__BrowserPlugins extends BrowserPlugin[] = InferBrowserPluginArray<InternalBrowserPoolOptions['browserPlugins']>,
__BrowserControllerReturn extends BrowserController = ReturnType<__BrowserPlugins[number]['createController']>,
__LaunchContextReturn extends LaunchContext = ReturnType<__BrowserPlugins[number]['createLaunchContext']>,
@@ -171,7 +174,7 @@ export interface BrowserCrawlerOptions<
* The exceptions are logged to the request using the
* {@apilink Request.pushErrorMessage|`Request.pushErrorMessage()`} function.
*/
- requestHandler?: RequestHandler<ExtendedContext>;
+ requestHandler?: RouterHandler<ExtendedContext, Routes> | RequestHandler<ExtendedContext>;
/**
* User-provided function that allows modifying the request object before it gets retried by the crawler.
@@ -340,8 +343,9 @@ export abstract class BrowserCrawler<
>,
ContextExtension = Dictionary<never>,
ExtendedContext extends Context = Context & ContextExtension,
+ Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>,
GoToOptions extends Dictionary = Dictionary,
-> extends BasicCrawler<Context, ContextExtension, ExtendedContext> {
+> extends BasicCrawler<Context, ContextExtension, ExtendedContext, Routes> {
/** Backs the {@apilink BrowserCrawler.browserPool|`browserPool`} getter. */
private browserPoolDep: OwnedOrInjected<IBrowserPool<Page>, OwnedBrowserPool<Page>>;
diff --git a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
index 6954d7d1f..1f3febb97 100644
--- a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
+++ b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
@@ -39,7 +39,8 @@ export interface CheerioCrawlerOptions<
ExtendedContext extends CheerioCrawlingContext = CheerioCrawlingContext & ContextExtension,
UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
-> extends HttpCrawlerOptions<CheerioCrawlingContext<UserData, JSONData>, ContextExtension, ExtendedContext> {}
+ Routes extends Record<keyof Routes, Dictionary> = Record<string, UserData>,
+> extends HttpCrawlerOptions<CheerioCrawlingContext<UserData, JSONData>, ContextExtension, ExtendedContext, Routes> {}
export type CheerioHook<
UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
@@ -184,11 +185,15 @@ export type CheerioRequestHandler<
export class CheerioCrawler<
ContextExtension = Dictionary<never>,
ExtendedContext extends CheerioCrawlingContext = CheerioCrawlingContext & ContextExtension,
-> extends HttpCrawler<CheerioCrawlingContext, ContextExtension, ExtendedContext> {
+ Routes extends Record<keyof Routes, Dictionary> = Record<
+ string,
+ GetUserDataFromRequest<CheerioCrawlingContext['request']>
+ >,
+> extends HttpCrawler<CheerioCrawlingContext, ContextExtension, ExtendedContext, Routes> {
/**
* All `CheerioCrawler` parameters are passed via an options object.
*/
- constructor(options?: CheerioCrawlerOptions<ContextExtension, ExtendedContext>) {
+ constructor(options?: CheerioCrawlerOptions<ContextExtension, ExtendedContext, any, any, Routes>) {
const { contextPipelineBuilder, ...rest } = options ?? {};
super({
diff --git a/packages/core/src/crawlers/crawler_commons.ts b/packages/core/src/crawlers/crawler_commons.ts
index 7f0c921e4..ccc331e41 100644
--- a/packages/core/src/crawlers/crawler_commons.ts
+++ b/packages/core/src/crawlers/crawler_commons.ts
@@ -4,7 +4,7 @@ import type { ReadonlyDeep, SetRequired } from 'type-fest';
import type { Configuration } from '../configuration.js';
import type { EnqueueLinksOptions } from '../enqueue_links/enqueue_links.js';
import type { CrawleeLogger } from '../log.js';
-import type { Request, Source } from '../request.js';
+import type { Request, RequestOptions, Source } from '../request.js';
import type { Dataset } from '../storages/dataset.js';
import { KeyValueStore, type RecordOptions } from '../storages/key_value_store.js';
import type { RequestQueueOperationOptions } from '../storages/request_queue.js';
@@ -13,6 +13,77 @@ import type { StorageIdentifier } from '../storages/storage_instance_manager.js'
/** @internal */
export type IsAny<T> = 0 extends 1 & T ? true : false;
+/**
+ * A request input (URL string, request-options object, or {@apilink Request}) whose `userData` is typed
+ * according to its `label`, based on a router's route map.
+ *
+ * When the route map is open (the default `Record<string, ...>`), this is just the regular loose
+ * {@apilink Source} input. When the map declares concrete labels, providing a `label` requires the matching
+ * `userData` shape and rejects labels not present in the map; unlabeled requests keep loose `userData`.
+ * @internal
+ */
+export type LabeledSource<Routes extends Record<keyof Routes, Dictionary>> = string extends keyof Routes
+ ? string | Source
+ :
+ | string
+ | Request
+ | ({ requestsFromUrl?: string; regex?: RegExp } & (
+ | {
+ [Label in keyof Routes & string]: Omit<Partial<RequestOptions<Routes[Label]>>, 'label'> & {
+ label: Label;
+ };
+ }[keyof Routes & string]
+ | (Omit<Partial<RequestOptions>, 'label'> & { label?: undefined })
+ ));
+
+/**
+ * The iterable/array of {@apilink LabeledSource} inputs accepted by the label-aware `addRequests`/`run`
+ * methods of a crawler bound to a typed router.
+ * @internal
+ */
+export type TypedRequestsLike<Routes extends Record<keyof Routes, Dictionary>> =
+ | AsyncIterable<LabeledSource<Routes>>
+ | Iterable<LabeledSource<Routes>>
+ | LabeledSource<Routes>[];
+
+/**
+ * The label-aware `addRequests` method signature exposed on a request handler's context when the crawler is
+ * bound to a typed router. Mirrors {@apilink RestrictedCrawlingContext.addRequests} with typed sources.
+ * @internal
+ */
+export type TypedContextAddRequests<Routes extends Record<keyof Routes, Dictionary>> = (
+ requestsLike: ReadonlyDeep<LabeledSource<Routes>[]>,
+ options?: ReadonlyDeep<RequestQueueOperationOptions>,
+) => Promise<void>;
+
+/**
+ * An `enqueueLinks`-options object with its `label`/`userData` retyped according to a router's route map: a
+ * declared `label` requires the matching `userData` shape (unknown labels are rejected), while unlabeled
+ * calls keep loose `userData`. Returns the options unchanged when the route map is open (the default).
+ */
+type TypedEnqueueLinksOptions<Options, Routes extends Record<keyof Routes, Dictionary>> = string extends keyof Routes
+ ? Options
+ : Omit<Options, 'label' | 'userData'> &
+ (
+ | { [Label in keyof Routes & string]: { label: Label; userData?: Routes[Label] } }[keyof Routes & string]
+ | { label?: undefined; userData?: Dictionary }
+ );
+
+/**
+ * Transforms a context's existing `enqueueLinks` method so that the `label`/`userData` in its options follow
+ * the router's route map, while preserving everything else about the signature (argument optionality and
+ * return type, which differ between crawler types).
+ * @internal
+ */
+export type TypedContextEnqueueLinks<
+ EnqueueLinks,
+ Routes extends Record<keyof Routes, Dictionary>,
+> = EnqueueLinks extends (options?: infer Options) => infer Result
+ ? (options?: TypedEnqueueLinksOptions<Options, Routes>) => Result
+ : EnqueueLinks extends (options: infer Options) => infer Result
+ ? (options: TypedEnqueueLinksOptions<Options, Routes>) => Result
+ : EnqueueLinks;
+
/** @internal */
export type WithRequired<T, K extends keyof T> = T & { [P in K]-?: T[P] };
diff --git a/packages/core/src/router.ts b/packages/core/src/router.ts
index 91fe8ed3f..73f5d78aa 100644
--- a/packages/core/src/router.ts
+++ b/packages/core/src/router.ts
@@ -1,7 +1,13 @@
import type { Dictionary } from '@crawlee/types';
import type { StandardSchemaV1 } from '@standard-schema/spec';
-import type { CrawlingContext, LoadedRequest, RestrictedCrawlingContext } from './crawlers/crawler_commons.js';
+import type {
+ CrawlingContext,
+ LoadedRequest,
+ RestrictedCrawlingContext,
+ TypedContextAddRequests,
+ TypedContextEnqueueLinks,
+} from './crawlers/crawler_commons.js';
import { MissingRouteError, RequestValidationError } from './errors.js';
import type { Request } from './request.js';
import type { Awaitable } from './typedefs.js';
@@ -14,11 +20,20 @@ import type { Awaitable } from './typedefs.js';
export const defaultRoute: unique symbol = Symbol('default-route');
/**
- * The crawling context received by a route handler, with `request.userData` narrowed to `UserData`.
+ * The crawling context received by a route handler, with `request.userData` narrowed to `UserData`, and
+ * `addRequests`/`enqueueLinks` typed according to the router's route map (`Routes`) so that enqueuing a
+ * request under a declared label requires the matching `userData` shape.
*/
-export type RouterHandlerContext<Context, UserData extends Dictionary> = Omit<Context, 'request'> & {
+export type RouterHandlerContext<
+ Context,
+ UserData extends Dictionary,
+ Routes extends Record<keyof Routes, Dictionary>,
+> = Omit<Context, 'request' | 'addRequests' | 'enqueueLinks'> & {
request: LoadedRequest<Request<UserData>>;
-};
+ addRequests: TypedContextAddRequests<Routes>;
+} & (Context extends { enqueueLinks: infer EnqueueLinks }
+ ? { enqueueLinks: TypedContextEnqueueLinks<EnqueueLinks, Routes> }
+ : {});
/**
* A map of request labels to a [Standard Schema](https://standardschema.dev) (Zod, Valibot, ArkType, …)
@@ -38,7 +53,7 @@ type SchemaUserData<Schema extends StandardSchemaV1> =
* Derives a route map (label → `userData` type) from a {@apilink RouteSchemas} map by inferring each schema's
* output type. Outputs that are not object-shaped fall back to a plain {@apilink Dictionary}. The
* {@apilink defaultRoute} schema is kept under its symbol key so {@apilink Router.addDefaultHandler} can pick it
- * up; string labels (the ones {@apilink Router.addHandler} accepts) ignore it.
+ * up; string labels (the ones {@apilink Router.addHandler} and the crawler-level typing accept) ignore it.
*/
export type RoutesFromSchemas<Schemas extends RouteSchemas> = {
[Label in Extract<keyof Schemas, string>]: SchemaUserData<Schemas[Label]>;
@@ -49,6 +64,7 @@ export type RoutesFromSchemas<Schemas extends RouteSchemas> = {
/**
* The `userData` type of the default route: inferred from the {@apilink defaultRoute} schema when the route map
* carries one, otherwise the provided `Fallback`.
+ * @internal
*/
export type DefaultRouteUserData<Routes, Fallback extends Dictionary> = Routes extends {
[defaultRoute]: infer DefaultUserData extends Dictionary;
@@ -246,7 +262,7 @@ export class Router<
*/
addHandler<Label extends keyof Routes & string>(
label: Label,
- handler: (ctx: RouterHandlerContext<Context, Routes[Label]>) => Awaitable<void>,
+ handler: (ctx: RouterHandlerContext<Context, Routes[Label], Routes>) => Awaitable<void>,
): void;
/**
@@ -256,7 +272,7 @@ export class Router<
*/
addHandler<UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(
label: RouterLabel<Routes>,
- handler: (ctx: RouterHandlerContext<Context, UserData>) => Awaitable<void>,
+ handler: (ctx: RouterHandlerContext<Context, UserData, Routes>) => Awaitable<void>,
): void;
addHandler(label: string | symbol, handler: (ctx: any) => Awaitable<void>): void {
@@ -272,7 +288,7 @@ export class Router<
*/
addDefaultHandler<
UserData extends Dictionary = DefaultRouteUserData<Routes, GetUserDataFromRequest<Context['request']>>,
- >(handler: (ctx: RouterHandlerContext<Context, UserData>) => Awaitable<void>) {
+ >(handler: (ctx: RouterHandlerContext<Context, UserData, Routes>) => Awaitable<void>) {
this.validate(defaultRoute);
this.routes.set(defaultRoute, handler);
}
diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.ts
index c25e64716..e1047427f 100644
--- a/packages/http-crawler/src/internals/http-crawler.ts
+++ b/packages/http-crawler/src/internals/http-crawler.ts
@@ -64,7 +64,8 @@ export interface HttpCrawlerOptions<
Context extends InternalHttpCrawlingContext = InternalHttpCrawlingContext,
ContextExtension = Dictionary<never>,
ExtendedContext extends Context = Context & ContextExtension,
-> extends BasicCrawlerOptions<Context, ContextExtension, ExtendedContext> {
+ Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>,
+> extends BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes> {
/**
* Timeout in which the HTTP request to the resource needs to finish, given in seconds.
*/
@@ -322,7 +323,8 @@ export class HttpCrawler<
Context extends InternalHttpCrawlingContext<any, any> = InternalHttpCrawlingContext,
ContextExtension = Dictionary<never>,
ExtendedContext extends Context = Context & ContextExtension,
-> extends BasicCrawler<Context, ContextExtension, ExtendedContext> {
+ Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>,
+> extends BasicCrawler<Context, ContextExtension, ExtendedContext, Routes> {
// Internal storage uses the base (non-extended) context types. The public option types are
// extension-aware for consumer DX, but internally the pipeline composes hooks against the
// concrete crawling context, which does not statically carry `ContextExtension`. The members
diff --git a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts b/packages/jsdom-crawler/src/internals/jsdom-crawler.ts
index e7640d186..3e6a43aa4 100644
--- a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts
+++ b/packages/jsdom-crawler/src/internals/jsdom-crawler.ts
@@ -42,7 +42,8 @@ export interface JSDOMCrawlerOptions<
ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext & ContextExtension,
UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
-> extends HttpCrawlerOptions<JSDOMCrawlingContext<UserData, JSONData>, ContextExtension, ExtendedContext> {
+ Routes extends Record<keyof Routes, Dictionary> = Record<string, UserData>,
+> extends HttpCrawlerOptions<JSDOMCrawlingContext<UserData, JSONData>, ContextExtension, ExtendedContext, Routes> {
/**
* Download and run scripts.
*/
@@ -187,7 +188,11 @@ const resources = new ResourceLoader({
export class JSDOMCrawler<
ContextExtension = Dictionary<never>,
ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext & ContextExtension,
-> extends HttpCrawler<JSDOMCrawlingContext, ContextExtension, ExtendedContext> {
+ Routes extends Record<keyof Routes, Dictionary> = Record<
+ string,
+ GetUserDataFromRequest<JSDOMCrawlingContext['request']>
+ >,
+> extends HttpCrawler<JSDOMCrawlingContext, ContextExtension, ExtendedContext, Routes> {
protected static override optionsShape = {
...HttpCrawler.optionsShape,
runScripts: ow.optional.boolean,
@@ -198,7 +203,7 @@ export class JSDOMCrawler<
private hideInternalConsole: boolean;
private virtualConsole: VirtualConsole | null = null;
- constructor(options: JSDOMCrawlerOptions<ContextExtension, ExtendedContext> = {}) {
+ constructor(options: JSDOMCrawlerOptions<ContextExtension, ExtendedContext, any, any, Routes> = {}) {
const { runScripts = false, hideInternalConsole = false, contextPipelineBuilder, ...httpOptions } = options;
super({
diff --git a/packages/linkedom-crawler/src/internals/linkedom-crawler.ts b/packages/linkedom-crawler/src/internals/linkedom-crawler.ts
index 4d3b60b3f..cf5f2dec3 100644
--- a/packages/linkedom-crawler/src/internals/linkedom-crawler.ts
+++ b/packages/linkedom-crawler/src/internals/linkedom-crawler.ts
@@ -39,7 +39,8 @@ export interface LinkeDOMCrawlerOptions<
ExtendedContext extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext & ContextExtension,
UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
-> extends HttpCrawlerOptions<LinkeDOMCrawlingContext<UserData, JSONData>, ContextExtension, ExtendedContext> {}
+ Routes extends Record<keyof Routes, Dictionary> = Record<string, UserData>,
+> extends HttpCrawlerOptions<LinkeDOMCrawlingContext<UserData, JSONData>, ContextExtension, ExtendedContext, Routes> {}
export interface LinkeDOMCrawlerEnqueueLinksOptions extends Omit<EnqueueLinksOptions, 'urls' | 'requestManager'> {}
@@ -171,10 +172,14 @@ export type LinkeDOMRequestHandler<
export class LinkeDOMCrawler<
ContextExtension = Dictionary<never>,
ExtendedContext extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext & ContextExtension,
-> extends HttpCrawler<LinkeDOMCrawlingContext, ContextExtension, ExtendedContext> {
+ Routes extends Record<keyof Routes, Dictionary> = Record<
+ string,
+ GetUserDataFromRequest<LinkeDOMCrawlingContext['request']>
+ >,
+> extends HttpCrawler<LinkeDOMCrawlingContext, ContextExtension, ExtendedContext, Routes> {
private static parser = new DOMParser();
- constructor(options: LinkeDOMCrawlerOptions<ContextExtension, ExtendedContext>) {
+ constructor(options: LinkeDOMCrawlerOptions<ContextExtension, ExtendedContext, any, any, Routes>) {
const { contextPipelineBuilder, ...rest } = options;
super({
diff --git a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts
index 4b118ef1c..8152e448d 100644
--- a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts
+++ b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts
@@ -184,8 +184,12 @@ type AdaptivePostNavigationHook<ContextExtension = Dictionary<never>> = BrowserH
export interface AdaptivePlaywrightCrawlerOptions<
ContextExtension = Dictionary<never>,
ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext & ContextExtension,
+ Routes extends Record<keyof Routes, Dictionary> = Record<
+ string,
+ GetUserDataFromRequest<AdaptivePlaywrightCrawlerContext['request']>
+ >,
> extends Omit<
- BasicCrawlerOptions<AdaptivePlaywrightCrawlerContext, ContextExtension, ExtendedContext>,
+ BasicCrawlerOptions<AdaptivePlaywrightCrawlerContext, ContextExtension, ExtendedContext, Routes>,
'preNavigationHooks' | 'postNavigationHooks'
> {
/**
@@ -306,7 +310,11 @@ type LogProxyCall = [log: CrawleeLogger, method: (typeof proxyLogMethods)[number
export class AdaptivePlaywrightCrawler<
ContextExtension = Dictionary<never>,
ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext & ContextExtension,
-> extends BasicCrawler<AdaptivePlaywrightCrawlerContext, ContextExtension, ExtendedContext> {
+ Routes extends Record<keyof Routes, Dictionary> = Record<
+ string,
+ GetUserDataFromRequest<AdaptivePlaywrightCrawlerContext['request']>
+ >,
+> extends BasicCrawler<AdaptivePlaywrightCrawlerContext, ContextExtension, ExtendedContext, Routes> {
private renderingTypePredictor: NonNullable<AdaptivePlaywrightCrawlerOptions['renderingTypePredictor']>;
private resultChecker: NonNullable<AdaptivePlaywrightCrawlerOptions['resultChecker']>;
private shouldPropagateError: NonNullable<AdaptivePlaywrightCrawlerOptions['shouldPropagateError']>;
@@ -321,7 +329,7 @@ export class AdaptivePlaywrightCrawler<
private teardownHooks: (() => Promise<unknown>)[] = [];
- constructor(options: AdaptivePlaywrightCrawlerOptions<ContextExtension, ExtendedContext> = {}) {
+ constructor(options: AdaptivePlaywrightCrawlerOptions<ContextExtension, ExtendedContext, Routes> = {}) {
const {
requestHandler,
renderingTypeDetectionRatio = 0.1,
diff --git a/packages/playwright-crawler/src/internals/playwright-crawler.ts b/packages/playwright-crawler/src/internals/playwright-crawler.ts
index 6afe9a397..9290e05ef 100644
--- a/packages/playwright-crawler/src/internals/playwright-crawler.ts
+++ b/packages/playwright-crawler/src/internals/playwright-crawler.ts
@@ -38,13 +38,18 @@ export interface PlaywrightHook extends BrowserHook<PlaywrightCrawlingContext> {
export interface PlaywrightCrawlerOptions<
ContextExtension = Dictionary<never>,
ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension,
+ Routes extends Record<keyof Routes, Dictionary> = Record<
+ string,
+ GetUserDataFromRequest<PlaywrightCrawlingContext['request']>
+ >,
> extends BrowserCrawlerOptions<
Page,
Response,
PlaywrightCrawlingContext,
ContextExtension,
ExtendedContext,
- { browserPlugins: [PlaywrightPlugin] }
+ { browserPlugins: [PlaywrightPlugin] },
+ Routes
> {
/**
* The same options as used by {@apilink launchPlaywright}.
@@ -73,7 +78,7 @@ export interface PlaywrightCrawlerOptions<
* The exceptions are logged to the request using the
* {@apilink Request.pushErrorMessage} function.
*/
- requestHandler?: RequestHandler<ExtendedContext>;
+ requestHandler?: RouterHandler<ExtendedContext, Routes> | RequestHandler<ExtendedContext>;
/**
* Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
@@ -180,6 +185,10 @@ export interface PlaywrightCrawlerOptions<
export class PlaywrightCrawler<
ContextExtension = Dictionary<never>,
ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension,
+ Routes extends Record<keyof Routes, Dictionary> = Record<
+ string,
+ GetUserDataFromRequest<PlaywrightCrawlingContext['request']>
+ >,
> extends BrowserCrawler<
Page,
Response,
@@ -187,7 +196,8 @@ export class PlaywrightCrawler<
LaunchOptions,
PlaywrightCrawlingContext,
ContextExtension,
- ExtendedContext
+ ExtendedContext,
+ Routes
> {
protected static override optionsShape = {
...BrowserCrawler.optionsShape,
@@ -200,7 +210,7 @@ export class PlaywrightCrawler<
/**
* All `PlaywrightCrawler` parameters are passed via an options object.
*/
- constructor(options: PlaywrightCrawlerOptions<ContextExtension, ExtendedContext> = {}) {
+ constructor(options: PlaywrightCrawlerOptions<ContextExtension, ExtendedContext, Routes> = {}) {
ow(options, 'PlaywrightCrawlerOptions', ow.object.exactShape(PlaywrightCrawler.optionsShape));
const { launchContext = {}, headless, contextPipelineBuilder, ...browserCrawlerOptions } = options;
diff --git a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts
index b9538ce78..7e663de8c 100644
--- a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts
+++ b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts
@@ -39,13 +39,18 @@ export interface PuppeteerHook extends BrowserHook<PuppeteerCrawlingContext> {}
export interface PuppeteerCrawlerOptions<
ContextExtension = Dictionary<never>,
ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension,
+ Routes extends Record<keyof Routes, Dictionary> = Record<
+ string,
+ GetUserDataFromRequest<PuppeteerCrawlingContext['request']>
+ >,
> extends BrowserCrawlerOptions<
Page,
HTTPResponse,
PuppeteerCrawlingContext,
ContextExtension,
ExtendedContext,
- { browserPlugins: [PuppeteerPlugin] }
+ { browserPlugins: [PuppeteerPlugin] },
+ Routes
> {
/**
* Options used by {@apilink launchPuppeteer} to start new Puppeteer instances.
@@ -157,6 +162,10 @@ export interface PuppeteerCrawlerOptions<
export class PuppeteerCrawler<
ContextExtension = Dictionary<never>,
ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension,
+ Routes extends Record<keyof Routes, Dictionary> = Record<
+ string,
+ GetUserDataFromRequest<PuppeteerCrawlingContext['request']>
+ >,
> extends BrowserCrawler<
Page,
HTTPResponse,
@@ -164,7 +173,8 @@ export class PuppeteerCrawler<
LaunchOptions,
PuppeteerCrawlingContext,
ContextExtension,
- ExtendedContext
+ ExtendedContext,
+ Routes
> {
protected static override optionsShape = {
...BrowserCrawler.optionsShape,
@@ -174,7 +184,7 @@ export class PuppeteerCrawler<
/**
* All `PuppeteerCrawler` parameters are passed via an options object.
*/
- constructor(options: PuppeteerCrawlerOptions<ContextExtension, ExtendedContext> = {}) {
+ constructor(options: PuppeteerCrawlerOptions<ContextExtension, ExtendedContext, Routes> = {}) {
ow(options, 'PuppeteerCrawlerOptions', ow.object.exactShape(PuppeteerCrawler.optionsShape));
const {
diff --git a/packages/stagehand-crawler/src/internals/stagehand-crawler.ts b/packages/stagehand-crawler/src/internals/stagehand-crawler.ts
index 8b3907d2d..457764db2 100644
--- a/packages/stagehand-crawler/src/internals/stagehand-crawler.ts
+++ b/packages/stagehand-crawler/src/internals/stagehand-crawler.ts
@@ -250,13 +250,18 @@ export interface StagehandRequestHandler extends RequestHandler<LoadedContext<St
export interface StagehandCrawlerOptions<
ContextExtension = Dictionary<never>,
ExtendedContext extends StagehandCrawlingContext = StagehandCrawlingContext & ContextExtension,
+ Routes extends Record<keyof Routes, Dictionary> = Record<
+ string,
+ GetUserDataFromRequest<StagehandCrawlingContext['request']>
+ >,
> extends BrowserCrawlerOptions<
StagehandPage,
Response,
StagehandCrawlingContext,
ContextExtension,
ExtendedContext,
- { browserPlugins: [StagehandPlugin] }
+ { browserPlugins: [StagehandPlugin] },
+ Routes
> {
/**
* Stagehand-specific configuration options.
@@ -313,7 +318,11 @@ export interface StagehandCrawlerOptions<
* }
* ```
*/
- requestHandler?: StagehandRequestHandler;
+ // Both union members must share the exact same call signature, otherwise TS cannot contextually type
+ // an inline `requestHandler({ page, request, ... })`. `StagehandRequestHandler` wraps the context in
+ // `LoadedContext`, while the router member uses `ExtendedContext`; since `StagehandCrawlingContext`
+ // already carries a `LoadedRequest`, using `ExtendedContext` for both keeps the signatures identical.
+ requestHandler?: RouterHandler<ExtendedContext, Routes> | RequestHandler<ExtendedContext>;
/**
* Async functions that are sequentially evaluated before the navigation.
@@ -378,6 +387,10 @@ export interface StagehandCrawlerOptions<
export class StagehandCrawler<
ContextExtension = Dictionary<never>,
ExtendedContext extends StagehandCrawlingContext = StagehandCrawlingContext & ContextExtension,
+ Routes extends Record<keyof Routes, Dictionary> = Record<
+ string,
+ GetUserDataFromRequest<StagehandCrawlingContext['request']>
+ >,
> extends BrowserCrawler<
StagehandPage,
Response,
@@ -385,7 +398,8 @@ export class StagehandCrawler<
LaunchOptions,
StagehandCrawlingContext,
ContextExtension,
- ExtendedContext
+ ExtendedContext,
+ Routes
> {
protected static override optionsShape = {
...BrowserCrawler.optionsShape,
@@ -398,7 +412,7 @@ export class StagehandCrawler<
*
* @param options - Crawler configuration options
*/
- constructor(options: StagehandCrawlerOptions<ContextExtension, ExtendedContext> = {}) {
+ constructor(options: StagehandCrawlerOptions<ContextExtension, ExtendedContext, Routes> = {}) {
ow(options, 'StagehandCrawlerOptions', ow.object.exactShape(StagehandCrawler.optionsShape));
const { stagehandOptions = {}, launchContext = {}, contextPipelineBuilder, ...browserCrawlerOptions } = options;
diff --git a/test/core/crawlers/adaptive_playwright_crawler.test.ts b/test/core/crawlers/adaptive_playwright_crawler.test.ts
index e9988f392..300912734 100644
--- a/test/core/crawlers/adaptive_playwright_crawler.test.ts
+++ b/test/core/crawlers/adaptive_playwright_crawler.test.ts
@@ -806,7 +806,7 @@ describe('AdaptivePlaywrightCrawler', () => {
const crawler = new AdaptivePlaywrightCrawler({ requestHandler: router });
await expect(
- crawler.addRequests([{ url: 'https://example.com/a', label: 'DETAIL', userData: { id: 123 } }]),
+ crawler.addRequests([{ url: 'https://example.com/a', label: 'DETAIL', userData: { id: 123 } }] as never),
).rejects.toThrow(RequestValidationError);
});
});
diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts
index eeeaf8189..fa2359d11 100644
--- a/test/core/crawlers/basic_crawler.test.ts
+++ b/test/core/crawlers/basic_crawler.test.ts
@@ -2703,7 +2703,9 @@ describe('BasicCrawler', () => {
const crawler = makeCrawler();
await expect(
- crawler.addRequests([{ url: 'https://example.com/a', label: 'DETAIL', userData: { id: 123 } }]),
+ crawler.addRequests([
+ { url: 'https://example.com/a', label: 'DETAIL', userData: { id: 123 } },
+ ] as never),
).rejects.toThrow(RequestValidationError);
});
@@ -2711,7 +2713,7 @@ describe('BasicCrawler', () => {
const crawler = makeCrawler();
await expect(
- crawler.run([{ url: 'https://example.com/a', label: 'DETAIL', userData: { id: 123 } }]),
+ crawler.run([{ url: 'https://example.com/a', label: 'DETAIL', userData: { id: 123 } }] as never),
).rejects.toThrow(RequestValidationError);
});
@@ -2746,7 +2748,9 @@ describe('BasicCrawler', () => {
const crawler = new BasicCrawler({ requestHandler: router });
// the label is not part of the source's `userData`, yet a schema declaring it still validates
- await crawler.addRequests([{ url: 'https://example.com/l', label: 'DETAIL', userData: { id: 'ok' } }]);
+ await crawler.addRequests([
+ { url: 'https://example.com/l', label: 'DETAIL', userData: { id: 'ok' } },
+ ] as never);
const queue = await crawler.getRequestQueue();
expect((await queue.fetchNextRequest())?.userData).toMatchObject({ label: 'DETAIL', id: 'ok' });
@@ -2761,7 +2765,7 @@ describe('BasicCrawler', () => {
// the label matches, so the only reported issue must be the bad `id` — not a spurious label one
const error = await crawler
- .addRequests([{ url: 'https://example.com/m', label: 'DETAIL', userData: { id: 123 } }])
+ .addRequests([{ url: 'https://example.com/m', label: 'DETAIL', userData: { id: 123 } }] as never)
.catch((err: Error) => err);
expect(error).toBeInstanceOf(RequestValidationError);
@@ -2778,7 +2782,7 @@ describe('BasicCrawler', () => {
await crawler.addRequests([
{ url: 'https://example.com/c', label: 'DETAIL', userData: { id: 'ok', price: '42' } },
- ]);
+ ] as never);
const queue = await crawler.getRequestQueue();
const request = await queue.fetchNextRequest();
@@ -2798,14 +2802,16 @@ describe('BasicCrawler', () => {
// an unregistered label is validated against the default-route schema on add
await expect(
- crawler.addRequests([{ url: 'https://example.com/l', label: 'LIST', userData: { page: 'nope' } }]),
+ crawler.addRequests([
+ { url: 'https://example.com/l', label: 'LIST', userData: { page: 'nope' } },
+ ] as never),
).rejects.toThrow(RequestValidationError);
// a registered label uses its own schema, and a matching default-route request is accepted
await crawler.addRequests([
{ url: 'https://example.com/d', label: 'DETAIL', userData: { id: 'ok' } },
{ url: 'https://example.com/p', label: 'LIST', userData: { page: 2 } },
- ]);
+ ] as never);
const queue = await crawler.getRequestQueue();
expect(await queue.isEmpty()).toBe(false);
});
@@ -2815,7 +2821,11 @@ describe('BasicCrawler', () => {
let caught: unknown;
router.addDefaultHandler(async ({ enqueueLinks }) => {
try {
- await enqueueLinks({ urls: ['https://example.com/x'], label: 'DETAIL', userData: { id: 123 } });
+ await enqueueLinks({
+ urls: ['https://example.com/x'],
+ label: 'DETAIL',
+ userData: { id: 123 },
+ } as never);
} catch (err) {
caught = err;
}
@@ -2830,7 +2840,9 @@ describe('BasicCrawler', () => {
test('requests with a label that has no registered schema are not validated', async () => {
const crawler = makeCrawler();
- await crawler.addRequests([{ url: 'https://example.com/d', label: 'OTHER', userData: { whatever: true } }]);
+ await crawler.addRequests([
+ { url: 'https://example.com/d', label: 'OTHER', userData: { whatever: true } },
+ ] as never);
const queue = await crawler.getRequestQueue();
expect(await queue.isEmpty()).toBe(false);
diff --git a/test/core/crawlers/playwright_crawler.test.ts b/test/core/crawlers/playwright_crawler.test.ts
index 7b962f206..d6d0eb993 100644
--- a/test/core/crawlers/playwright_crawler.test.ts
+++ b/test/core/crawlers/playwright_crawler.test.ts
@@ -291,7 +291,9 @@ describe('PlaywrightCrawler', () => {
const crawler = new PlaywrightCrawler({ requestHandler: router });
await expect(
- crawler.addRequests([{ url: `http://${HOSTNAME}:${port}/`, label: 'DETAIL', userData: { id: 123 } }]),
+ crawler.addRequests([
+ { url: `http://${HOSTNAME}:${port}/`, label: 'DETAIL', userData: { id: 123 } },
+ ] as never),
).rejects.toThrow(RequestValidationError);
});
});
diff --git a/test/core/crawlers/puppeteer_crawler.test.ts b/test/core/crawlers/puppeteer_crawler.test.ts
index e6b971bfa..e7d170d18 100644
--- a/test/core/crawlers/puppeteer_crawler.test.ts
+++ b/test/core/crawlers/puppeteer_crawler.test.ts
@@ -456,7 +456,7 @@ describe('PuppeteerCrawler', () => {
const crawler = new PuppeteerCrawler({ requestHandler: router });
await expect(
- crawler.addRequests([{ url: 'https://example.com/a', label: 'DETAIL', userData: { id: 123 } }]),
+ crawler.addRequests([{ url: 'https://example.com/a', label: 'DETAIL', userData: { id: 123 } }] as never),
).rejects.toThrow(RequestValidationError);
});
});
diff --git a/test/core/router.test.ts b/test/core/router.test.ts
index a22b85735..073841f3c 100644
--- a/test/core/router.test.ts
+++ b/test/core/router.test.ts
@@ -2,10 +2,15 @@ import { BasicCrawler } from '@crawlee/basic';
import type { CrawlingContext } from '@crawlee/core';
import { defaultRoute, MissingRouteError, Request, RequestValidationError, Router } from '@crawlee/core';
import {
+ CheerioCrawler,
type CheerioCrawlingContext,
createCheerioRouter,
createPlaywrightRouter,
+ createPuppeteerRouter,
+ PlaywrightCrawler,
type PlaywrightCrawlingContext,
+ PuppeteerCrawler,
+ type PuppeteerCrawlingContext,
} from 'crawlee';
import { z } from 'zod';
@@ -404,4 +409,90 @@ describe('Router', () => {
expect(typeof typeOnly).toBe('function');
});
+
+ test('crawler infers the route map from a typed requestHandler and types addRequests/context', () => {
+ // type-level only: the block is never executed, it just has to type-check
+ const typeOnly = async () => {
+ interface Routes {
+ PRODUCT: { sku: string; price: number };
+ CATEGORY: { categoryId: string };
+ }
+
+ const router = createCheerioRouter<CheerioCrawlingContext, Routes>();
+
+ router.addHandler('PRODUCT', async ({ addRequests, enqueueLinks }) => {
+ // context methods are typed from the route map
+ await addRequests([{ url: 'https://e.com/c', label: 'CATEGORY', userData: { categoryId: 'c1' } }]);
+ await enqueueLinks({ urls: ['https://e.com/p'], label: 'PRODUCT', userData: { sku: 's', price: 1 } });
+ // @ts-expect-error wrong userData shape for the label
+ await addRequests([{ url: 'https://e.com/p', label: 'PRODUCT', userData: { categoryId: 'x' } }]);
+ // @ts-expect-error label not present in the route map
+ await addRequests([{ url: 'https://e.com/x', label: 'NOPE' }]);
+ });
+
+ // the crawler infers `Routes` from the typed router passed as `requestHandler`
+ const crawler = new CheerioCrawler({ requestHandler: router });
+
+ await crawler.addRequests([{ url: 'https://e.com/p', label: 'PRODUCT', userData: { sku: 's', price: 1 } }]);
+ await crawler.run([
+ 'https://e.com',
+ { url: 'https://e.com/c', label: 'CATEGORY', userData: { categoryId: 'c1' } },
+ ]);
+ // @ts-expect-error wrong userData shape for the label
+ await crawler.addRequests([{ url: 'https://e.com/p', label: 'PRODUCT', userData: { categoryId: 'x' } }]);
+ // @ts-expect-error label not present in the route map
+ await crawler.addRequests([{ url: 'https://e.com/x', label: 'NOPE' }]);
+ };
+
+ expect(typeof typeOnly).toBe('function');
+ });
+
+ test('browser crawler also infers the route map from a typed requestHandler', () => {
+ // type-level only: never executed
+ const typeOnly = async () => {
+ interface Routes {
+ PRODUCT: { sku: string };
+ }
+
+ const router = createPlaywrightRouter<PlaywrightCrawlingContext, Routes>();
+
+ router.addHandler('PRODUCT', async ({ addRequests }) => {
+ await addRequests([{ url: 'https://e.com/p', label: 'PRODUCT', userData: { sku: 's' } }]);
+ // @ts-expect-error wrong userData shape for the label
+ await addRequests([{ url: 'https://e.com/p', label: 'PRODUCT', userData: { sku: 1 } }]);
+ // @ts-expect-error label not present in the route map
+ await addRequests([{ url: 'https://e.com/x', label: 'NOPE' }]);
+ });
+
+ const crawler = new PlaywrightCrawler({ requestHandler: router });
+
+ await crawler.addRequests([{ url: 'https://e.com/p', label: 'PRODUCT', userData: { sku: 's' } }]);
+ // @ts-expect-error wrong userData shape for the label
+ await crawler.addRequests([{ url: 'https://e.com/p', label: 'PRODUCT', userData: { sku: 1 } }]);
+ // @ts-expect-error label not present in the route map
+ await crawler.addRequests([{ url: 'https://e.com/x', label: 'NOPE' }]);
+ };
+
+ expect(typeof typeOnly).toBe('function');
+ });
+
+ test('puppeteer crawler infers the route map too (inherited requestHandler path)', () => {
+ // type-level only: never executed
+ const typeOnly = async () => {
+ interface Routes {
+ PRODUCT: { sku: string };
+ }
+
+ const router = createPuppeteerRouter<PuppeteerCrawlingContext, Routes>();
+ const crawler = new PuppeteerCrawler({ requestHandler: router });
+
+ await crawler.addRequests([{ url: 'https://e.com/p', label: 'PRODUCT', userData: { sku: 's' } }]);
+ // @ts-expect-error wrong userData shape for the label
+ await crawler.addRequests([{ url: 'https://e.com/p', label: 'PRODUCT', userData: { sku: 1 } }]);
+ // @ts-expect-error label not present in the route map
+ await crawler.addRequests([{ url: 'https://e.com/x', label: 'NOPE' }]);
+ };
+
+ expect(typeof typeOnly).toBe('function');
+ });
});