Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat: `createAdaptivePlaywrightRouter` utility (#2415)

Jan Bucharcee477814e49
diff --git a/packages/core/src/router.ts b/packages/core/src/router.ts
index 6bcab6f00..4abc05dc1 100644
--- a/packages/core/src/router.ts
+++ b/packages/core/src/router.ts
@@ -1,13 +1,13 @@
import type { Dictionary } from '@crawlee/types';
-import type { CrawlingContext } from './crawlers/crawler_commons';
+import type { CrawlingContext, RestrictedCrawlingContext } from './crawlers/crawler_commons';
import { MissingRouteError } from './errors';
import type { Request } from './request';
import type { Awaitable } from './typedefs';
const defaultRoute = Symbol('default-route');
-export interface RouterHandler<Context extends CrawlingContext = CrawlingContext> extends Router<Context> {
+export interface RouterHandler<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'> = CrawlingContext> extends Router<Context> {
(ctx: Context): Awaitable<void>;
}
@@ -82,7 +82,7 @@ export type RouterRoutes<Context, UserData extends Dictionary> = {
* });
* ```
*/
-export class Router<Context extends CrawlingContext> {
+export class Router<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'>> {
private readonly routes: Map<string | symbol, (ctx: Context) => Awaitable<void>> = new Map();
private readonly middlewares: ((ctx: Context) => Awaitable<void>)[] = [];
@@ -173,7 +173,7 @@ export class Router<Context extends CrawlingContext> {
* ```
*/
static create<
- Context extends CrawlingContext = CrawlingContext,
+ Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'> = CrawlingContext,
UserData extends Dictionary = GetUserDataFromRequest<Context['request']>,
>(routes?: RouterRoutes<Context, UserData>): RouterHandler<Context> {
const router = new Router<Context>();
diff --git a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts
index 1fcc41617..607fecfd1 100644
--- a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts
+++ b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts
@@ -1,7 +1,7 @@
import { addTimeoutToPromise } from '@apify/timeout';
import { extractUrlsFromPage } from '@crawlee/browser';
-import type { RestrictedCrawlingContext, StatisticState, StatisticPersistedState } from '@crawlee/core';
-import { Configuration, RequestHandlerResult, Statistics, withCheckedStorageAccess } from '@crawlee/core';
+import type { RestrictedCrawlingContext, StatisticState, StatisticPersistedState, GetUserDataFromRequest, RouterRoutes } from '@crawlee/core';
+import { Configuration, RequestHandlerResult, Router, Statistics, withCheckedStorageAccess } from '@crawlee/core';
import type { Awaitable, Dictionary } from '@crawlee/types';
import { extractUrlsFromCheerio } from '@crawlee/utils';
import { load, type Cheerio, type Element } from 'cheerio';
@@ -64,7 +64,7 @@ class AdaptivePlaywrightCrawlerStatistics extends Statistics {
}
}
-interface AdaptivePlaywrightCrawlerContext extends RestrictedCrawlingContext {
+export interface AdaptivePlaywrightCrawlerContext extends RestrictedCrawlingContext {
/**
* Wait for an element matching the selector to appear and return a Cheerio object of matched elements.
*/
@@ -356,3 +356,10 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {
}
}
}
+
+export function createAdaptivePlaywrightRouter<
+ Context extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext,
+ UserData extends Dictionary = GetUserDataFromRequest<Context['request']>,
+>(routes?: RouterRoutes<Context, UserData>) {
+ return Router.create<Context>(routes);
+}