Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat(router): allow inline router definition (#1877)
Pavel 'Strajk' Dolecek2d241c9f8896
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex f60ca0b69..193593770 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -9,12 +9,14 @@ import type {EnqueueLinksOptions,EventManager,FinalStatistics,+ GetUserDataFromRequest,ProxyInfo,Request,RequestList,RequestOptions,RequestQueueOperationOptions,RouterHandler,+ RouterRoutes,Session,SessionPoolOptions,} from '@crawlee/core';@@ -1399,6 +1401,9 @@ interface HandlePropertyNameChangeData<New, Old> {* await crawler.run();* ```*/-export function createBasicRouter<Context extends BasicCrawlingContext = BasicCrawlingContext>() {- return Router.create<Context>();+export function createBasicRouter<+ Context extends BasicCrawlingContext = BasicCrawlingContext,+ UserData extends Dictionary = GetUserDataFromRequest<Context['request']>,+>(routes?: RouterRoutes<Context, UserData>) {+ return Router.create<Context>(routes);}diff --git a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts b/packages/cheerio-crawler/src/internals/cheerio-crawler.tsindex 46f41e911..472d6c207 100644--- a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts+++ b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts@@ -1,12 +1,14 @@import type {+ Configuration,+ EnqueueLinksOptions,+ ErrorHandler,+ GetUserDataFromRequest,HttpCrawlerOptions,InternalHttpCrawlingContext,InternalHttpHook,- ErrorHandler,RequestHandler,- EnqueueLinksOptions,RequestQueue,- Configuration,+ RouterRoutes,} from '@crawlee/http';import { HttpCrawler, enqueueLinks, Router, resolveBaseUrlForEnqueueLinksFiltering, tryAbsoluteURL } from '@crawlee/http';import type { Dictionary } from '@crawlee/types';@@ -275,6 +277,9 @@ function extractUrlsFromCheerio($: cheerio.CheerioAPI, selector: string, baseUrl* await crawler.run();* ```*/-export function createCheerioRouter<Context extends CheerioCrawlingContext = CheerioCrawlingContext>() {- return Router.create<Context>();+export function createCheerioRouter<+ Context extends CheerioCrawlingContext = CheerioCrawlingContext,+ UserData extends Dictionary = GetUserDataFromRequest<Context['request']>,+>(routes?: RouterRoutes<Context, UserData>) {+ return Router.create<Context>(routes);}diff --git a/packages/core/src/router.ts b/packages/core/src/router.tsindex 41c213640..1e652533f 100644--- a/packages/core/src/router.ts+++ b/packages/core/src/router.ts@@ -10,7 +10,11 @@ export interface RouterHandler<Context extends CrawlingContext = CrawlingContext(ctx: Context): Awaitable<void>;}-type GetUserDataFromRequest<T> = T extends Request<infer Y> ? Y : never;+export type GetUserDataFromRequest<T> = T extends Request<infer Y> ? Y : never;++export type RouterRoutes<Context, UserData extends Dictionary> = {+ [label in string | symbol]: (ctx: Omit<Context, 'request'> & { request: Request<UserData> }) => Awaitable<void>;+}/*** Simple router that works based on request labels. This instance can then serve as a `requestHandler` of your crawler.@@ -54,6 +58,19 @@ type GetUserDataFromRequest<T> = T extends Request<infer Y> ? Y : never;* await crawler.run();* ```*+ * For convenience, we can also define the routes right when creating the router:+ *+ * ```ts+ * import { CheerioCrawler, createCheerioRouter } from 'crawlee';+ * const crawler = new CheerioCrawler({+ * requestHandler: createCheerioRouter({+ * 'label-a': async (ctx) => { ... },+ * 'label-b': async (ctx) => { ... },+ * })},+ * });+ * await crawler.run();+ * ```+ ** Middlewares are also supported via the `router.use` method. There can be multiple* middlewares for a single router, they will be executed sequentially in the same* order as they were registered.@@ -62,6 +79,7 @@ type GetUserDataFromRequest<T> = T extends Request<infer Y> ? Y : never;* crawler.router.use(async (ctx) => {* ctx.log.info('...');* });+ * ```*/export class Router<Context extends CrawlingContext> {private readonly routes: Map<string | symbol, (ctx: Context) => Awaitable<void>> = new Map();@@ -153,7 +171,10 @@ export class Router<Context extends CrawlingContext> {* await crawler.run();* ```*/- static create<Context extends CrawlingContext = CrawlingContext>(): RouterHandler<Context> {+ static create<+ Context extends CrawlingContext = CrawlingContext,+ UserData extends Dictionary = GetUserDataFromRequest<Context['request']>,+ >(routes?: RouterRoutes<Context, UserData>): RouterHandler<Context> {const router = new Router<Context>();const obj = Object.create(Function.prototype);@@ -162,6 +183,10 @@ export class Router<Context extends CrawlingContext> {obj.getHandler = router.getHandler.bind(router);obj.use = router.use.bind(router);+ for (const [label, handler] of Object.entries(routes ?? {})) {+ router.addHandler(label, handler);+ }+const func = async function (context: Context) {const { url, loadedUrl, label } = context.request;context.log.debug('Page opened.', { label, url: loadedUrl ?? url });diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.tsindex 82cac7403..61c5dbce2 100644--- a/packages/http-crawler/src/internals/http-crawler.ts+++ b/packages/http-crawler/src/internals/http-crawler.ts@@ -3,11 +3,13 @@ import { concatStreamToBuffer, readStreamToString } from '@apify/utilities';import type {AutoscaledPoolOptions,BasicCrawlerOptions,- ErrorHandler,- RequestHandler,CrawlingContext,+ ErrorHandler,+ GetUserDataFromRequest,ProxyConfiguration,Request,+ RequestHandler,+ RouterRoutes,Session,} from '@crawlee/basic';import {@@ -870,6 +872,9 @@ function parseContentTypeFromResponse(response: IncomingMessage): { type: string* await crawler.run();* ```*/-export function createHttpRouter<Context extends HttpCrawlingContext = HttpCrawlingContext>() {- return Router.create<Context>();+export function createHttpRouter<+ Context extends HttpCrawlingContext = HttpCrawlingContext,+ UserData extends Dictionary = GetUserDataFromRequest<Context['request']>,+>(routes?: RouterRoutes<Context, UserData>) {+ return Router.create<Context>(routes);}diff --git a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts b/packages/jsdom-crawler/src/internals/jsdom-crawler.tsindex a56991ca2..95cf67058 100644--- a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts+++ b/packages/jsdom-crawler/src/internals/jsdom-crawler.ts@@ -1,14 +1,16 @@import ow from 'ow';import * as cheerio from 'cheerio';import type {+ Configuration,+ EnqueueLinksOptions,+ ErrorHandler,+ GetUserDataFromRequest,HttpCrawlerOptions,InternalHttpCrawlingContext,InternalHttpHook,- ErrorHandler,RequestHandler,- EnqueueLinksOptions,RequestQueue,- Configuration,+ RouterRoutes,} from '@crawlee/http';import { HttpCrawler, enqueueLinks, Router, resolveBaseUrlForEnqueueLinksFiltering, tryAbsoluteURL } from '@crawlee/http';import type { Dictionary } from '@crawlee/types';@@ -351,6 +353,9 @@ function extractUrlsFromWindow(window: DOMWindow, selector: string, baseUrl: str* await crawler.run();* ```*/-export function createJSDOMRouter<Context extends JSDOMCrawlingContext = JSDOMCrawlingContext>() {- return Router.create<Context>();+export function createJSDOMRouter<+ Context extends JSDOMCrawlingContext = JSDOMCrawlingContext,+ UserData extends Dictionary = GetUserDataFromRequest<Context['request']>,+>(routes?: RouterRoutes<Context, UserData>) {+ return Router.create<Context>(routes);}diff --git a/packages/playwright-crawler/src/internals/playwright-crawler.ts b/packages/playwright-crawler/src/internals/playwright-crawler.tsindex 50a6e22f4..0a0028ed4 100644--- a/packages/playwright-crawler/src/internals/playwright-crawler.ts+++ b/packages/playwright-crawler/src/internals/playwright-crawler.ts@@ -1,7 +1,14 @@import ow from 'ow';import type { LaunchOptions, Page, Response } from 'playwright';import type { BrowserPoolOptions, PlaywrightController, PlaywrightPlugin } from '@crawlee/browser-pool';-import type { BrowserCrawlerOptions, BrowserCrawlingContext, BrowserRequestHandler, BrowserHook } from '@crawlee/browser';+import type {+ BrowserCrawlerOptions,+ BrowserCrawlingContext,+ BrowserHook,+ BrowserRequestHandler,+ GetUserDataFromRequest,+ RouterRoutes,+} from '@crawlee/browser';import { BrowserCrawler, Configuration, Router } from '@crawlee/browser';import type { Dictionary } from '@crawlee/types';import type { PlaywrightLaunchContext } from './playwright-launcher';@@ -262,6 +269,9 @@ export class PlaywrightCrawler extends BrowserCrawler<{ browserPlugins: [Playwri* await crawler.run();* ```*/-export function createPlaywrightRouter<Context extends PlaywrightCrawlingContext = PlaywrightCrawlingContext>() {- return Router.create<Context>();+export function createPlaywrightRouter<+ Context extends PlaywrightCrawlingContext = PlaywrightCrawlingContext,+ UserData extends Dictionary = GetUserDataFromRequest<Context['request']>,+>(routes?: RouterRoutes<Context, UserData>) {+ return Router.create<Context>(routes);}diff --git a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.tsindex b15c8716b..95cea01f0 100644--- a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts+++ b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts@@ -1,8 +1,10 @@import type {- BrowserRequestHandler,BrowserCrawlerOptions,BrowserCrawlingContext,BrowserHook,+ BrowserRequestHandler,+ GetUserDataFromRequest,+ RouterRoutes,} from '@crawlee/browser';import { BrowserCrawler, Configuration, Router } from '@crawlee/browser';import type { BrowserPoolOptions, PuppeteerController, PuppeteerPlugin } from '@crawlee/browser-pool';@@ -212,6 +214,9 @@ export class PuppeteerCrawler extends BrowserCrawler<{ browserPlugins: [Puppetee* await crawler.run();* ```*/-export function createPuppeteerRouter<Context extends PuppeteerCrawlingContext = PuppeteerCrawlingContext>() {- return Router.create<Context>();+export function createPuppeteerRouter<+ Context extends PuppeteerCrawlingContext = PuppeteerCrawlingContext,+ UserData extends Dictionary = GetUserDataFromRequest<Context['request']>,+>(routes?: RouterRoutes<Context, UserData>) {+ return Router.create<Context>(routes);}diff --git a/test/core/router.test.ts b/test/core/router.test.tsindex 2ba884585..7ace0269b 100644--- a/test/core/router.test.ts+++ b/test/core/router.test.ts@@ -89,6 +89,38 @@ describe('Router', () => {]);});+ test('should be possible to define routes when creating router', async () => {+ const logs: string[] = [];+ // it should be possible to define router inline when creating router+ const router = Router.create({+ 'A': async (ctx) => {+ logs.push(`label A handled with url ${ctx.request.loadedUrl}`);+ },+ 'B': async (ctx) => {+ logs.push(`label B handled with url ${ctx.request.loadedUrl}`);+ },+ });+ // and it's still possible to attach handlers later+ router.addHandler('C', async (ctx) => {+ logs.push(`label C handled with url ${ctx.request.loadedUrl}`);+ });+ router.addDefaultHandler(async (ctx) => {+ logs.push(`default handled with url ${ctx.request.loadedUrl}`);+ });+ const log = { info: jest.fn(), warn: jest.fn(), debug: jest.fn() };+ await router({ request: { loadedUrl: 'https://example.com/A', label: 'A' }, log } as any);+ await router({ request: { loadedUrl: 'https://example.com/B', label: 'B' }, log } as any);+ await router({ request: { loadedUrl: 'https://example.com/C', label: 'C' }, log } as any);+ await router({ request: { loadedUrl: 'https://example.com/' }, log } as any);++ expect(logs).toEqual([+ 'label A handled with url https://example.com/A',+ 'label B handled with url https://example.com/B',+ 'label C handled with url https://example.com/C',+ 'default handled with url https://example.com/',+ ]);+ });+test('validation', async () => {const router = Router.create();router.addHandler('A', async (ctx) => {});