Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat: add support for middlewares to the `Router` via `use` method (#1431)
Martin Adámek6fee20d8a2a3
diff --git a/packages/core/src/router.ts b/packages/core/src/router.tsindex 98c138075..4ba52e890 100644--- a/packages/core/src/router.ts+++ b/packages/core/src/router.ts@@ -49,9 +49,19 @@ export interface RouterHandler<Context extends CrawlingContext = CrawlingContext** await crawler.run();* ```+ *+ * Middlewares are also supported via the `router.use` method. There can be multiple+ * middlewares for a single router, they will be executed sequentially in the same+ * order as they were registered.+ *+ * ```ts+ * crawler.router.use(async (ctx) => {+ * ctx.log.info('...');+ * });*/export class Router<Context extends CrawlingContext> {private readonly routes: Map<string | symbol, (ctx: Context) => Awaitable<void>> = new Map();+ private readonly middlewares: ((ctx: Context) => Awaitable<void>)[] = [];/*** use Router.create() instead!@@ -75,6 +85,14 @@ export class Router<Context extends CrawlingContext> {this.routes.set(defaultRoute, handler);}+ /**+ * Registers a middleware that will be fired before the matching route handler.+ * Multiple middlewares can be registered, they will be fired in the same order.+ */+ use(middleware: (ctx: Context) => Awaitable<void>) {+ this.middlewares.push(middleware);+ }+/*** Returns route handler for given label. If no label is provided, the default request handler will be returned.*/@@ -134,10 +152,16 @@ export class Router<Context extends CrawlingContext> {obj.addHandler = router.addHandler.bind(router);obj.addDefaultHandler = router.addDefaultHandler.bind(router);obj.getHandler = router.getHandler.bind(router);+ obj.use = router.use.bind(router);- const func = function (context: Context) {+ const func = async function (context: Context) {const { url, loadedUrl, label } = context.request;context.log.debug('Page opened.', { label, url: loadedUrl ?? url });++ for (const middleware of router.middlewares) {+ await middleware(context);+ }+return router.getHandler(label)(context);};diff --git a/test/core/router.test.ts b/test/core/router.test.tsindex dcff1c17a..026a4751f 100644--- a/test/core/router.test.ts+++ b/test/core/router.test.ts@@ -18,6 +18,9 @@ describe('Router', () => {router.addDefaultHandler(async (ctx) => {logs.push(`default handled with url ${ctx.request.loadedUrl}`);});+ router.use(({ request }) => void logs.push(`middleware 1: ${request.loadedUrl}`));+ router.use(({ request }) => void logs.push(`middleware 2: ${request.loadedUrl}`));+ router.use(({ request }) => void logs.push(`middleware 3: ${request.loadedUrl}`));const log = { info: jest.fn(), warn: jest.fn(), debug: jest.fn() };await router({ request: { loadedUrl: 'https://example.com/A', label: 'A' }, log } as any);@@ -34,17 +37,53 @@ describe('Router', () => {await router({ request: { loadedUrl: 'https://example.com/C', label: 'C' }, log } as any);expect(logs).toEqual([+ 'middleware 1: https://example.com/A',+ 'middleware 2: https://example.com/A',+ 'middleware 3: https://example.com/A','label A handled with url https://example.com/A',+ 'middleware 1: https://example.com/A',+ 'middleware 2: https://example.com/A',+ 'middleware 3: https://example.com/A','label A handled with url https://example.com/A',+ 'middleware 1: https://example.com/C',+ 'middleware 2: https://example.com/C',+ 'middleware 3: https://example.com/C','label C handled with url https://example.com/C',+ 'middleware 1: https://example.com/A',+ 'middleware 2: https://example.com/A',+ 'middleware 3: https://example.com/A','label A handled with url https://example.com/A',+ 'middleware 1: https://example.com/B',+ 'middleware 2: https://example.com/B',+ 'middleware 3: https://example.com/B','label B handled with url https://example.com/B',+ 'middleware 1: https://example.com/B',+ 'middleware 2: https://example.com/B',+ 'middleware 3: https://example.com/B','label B handled with url https://example.com/B',+ 'middleware 1: https://example.com/',+ 'middleware 2: https://example.com/',+ 'middleware 3: https://example.com/','default handled with url https://example.com/',+ 'middleware 1: https://example.com/A',+ 'middleware 2: https://example.com/A',+ 'middleware 3: https://example.com/A','label A handled with url https://example.com/A',+ 'middleware 1: https://example.com/C',+ 'middleware 2: https://example.com/C',+ 'middleware 3: https://example.com/C','label C handled with url https://example.com/C',+ 'middleware 1: https://example.com/C',+ 'middleware 2: https://example.com/C',+ 'middleware 3: https://example.com/C','label C handled with url https://example.com/C',+ 'middleware 1: https://example.com/D',+ 'middleware 2: https://example.com/D',+ 'middleware 3: https://example.com/D','default handled with url https://example.com/D',+ 'middleware 1: https://example.com/C',+ 'middleware 2: https://example.com/C',+ 'middleware 3: https://example.com/C','label C handled with url https://example.com/C',]);});@@ -54,7 +93,7 @@ describe('Router', () => {router.addHandler('A', async (ctx) => {});expect(() => router.addHandler('A', async (ctx) => {})).toThrow();const log = { info: jest.fn(), warn: jest.fn(), debug: jest.fn() };- expect(() => router({ request: { loadedUrl: 'https://example.com/C', label: 'C' }, log } as any)).toThrow(MissingRouteError);+ await expect(router({ request: { loadedUrl: 'https://example.com/C', label: 'C' }, log } as any)).rejects.toThrow(MissingRouteError);router.addDefaultHandler(async (ctx) => {});expect(() => router.addDefaultHandler(async (ctx) => {})).toThrow();});