Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: ensure CrawlingContext interface is inferred correctly in route handlers

Martin Adámekaa84633b1a20
diff --git a/packages/core/src/crawlers/crawler_commons.ts b/packages/core/src/crawlers/crawler_commons.ts
index fcf3e1b07..dba824fe4 100644
--- a/packages/core/src/crawlers/crawler_commons.ts
+++ b/packages/core/src/crawlers/crawler_commons.ts
@@ -7,7 +7,8 @@ import type { ProxyInfo } from '../proxy_configuration';
import type { Request } from '../request';
import type { Session } from '../session_pool/session';
-export interface CrawlingContext<Crawler = unknown, UserData extends Dictionary = Dictionary> extends Record<PropertyKey, unknown> {
+// eslint-disable-next-line @typescript-eslint/ban-types
+export interface CrawlingContext<Crawler = unknown, UserData extends Dictionary = Dictionary> extends Record<string & {}, unknown> {
id: string;
/**
* The original {@apilink Request} object.
diff --git a/packages/core/src/except.ts b/packages/core/src/except.ts
deleted file mode 100644
index 7472721f8..000000000
--- a/packages/core/src/except.ts
+++ /dev/null
@@ -1,13 +0,0 @@
-// Do not convert this file to .d.ts
-
-type IsEqual<T, U> =
- (<G>() => G extends T ? 1 : 2) extends
- (<G>() => G extends U ? 1 : 2)
- ? true
- : false;
-
-type Filter<KeyType, ExcludeType> = IsEqual<KeyType, ExcludeType> extends true ? never : (KeyType extends ExcludeType ? never : KeyType);
-
-export type Except<ObjectType, KeysType extends keyof ObjectType> = {
- [KeyType in keyof ObjectType as Filter<KeyType, KeysType>]: ObjectType[KeyType];
-};
diff --git a/packages/core/src/router.ts b/packages/core/src/router.ts
index 74c50c5d8..41c213640 100644
--- a/packages/core/src/router.ts
+++ b/packages/core/src/router.ts
@@ -2,7 +2,6 @@ import type { Dictionary } from '@crawlee/types';
import type { CrawlingContext } from './crawlers/crawler_commons';
import type { Awaitable } from './typedefs';
import type { Request } from './request';
-import type { Except } from './except';
import { MissingRouteError } from './errors';
const defaultRoute = Symbol('default-route');
@@ -79,7 +78,7 @@ export class Router<Context extends CrawlingContext> {
*/
addHandler<UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(
label: string | symbol,
- handler: (ctx: Except<Context, 'request'> & {request: Request<UserData>}) => Awaitable<void>,
+ handler: (ctx: Omit<Context, 'request'> & { request: Request<UserData> }) => Awaitable<void>,
) {
this.validate(label);
this.routes.set(label, handler);
@@ -89,7 +88,7 @@ export class Router<Context extends CrawlingContext> {
* Registers default route handler.
*/
addDefaultHandler<UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(
- handler: (ctx: Except<Context, 'request'> & {request: Request<UserData>}) => Awaitable<void>,
+ handler: (ctx: Omit<Context, 'request'> & { request: Request<UserData> }) => Awaitable<void>,
) {
this.validate(defaultRoute);
this.routes.set(defaultRoute, handler);
diff --git a/test/core/crawlers/cheerio_crawler.test.ts b/test/core/crawlers/cheerio_crawler.test.ts
index e61eaba4a..0c6170457 100644
--- a/test/core/crawlers/cheerio_crawler.test.ts
+++ b/test/core/crawlers/cheerio_crawler.test.ts
@@ -125,9 +125,9 @@ describe('CheerioCrawler', () => {
},
});
- cheerioCrawler.router.addDefaultHandler(({ $, body, request }) => {
+ cheerioCrawler.router.addDefaultHandler<{ title: string; body: string }>(({ $, body, request }) => {
request.userData.title = $('title').text();
- request.userData.body = body;
+ request.userData.body = body.toString();
processed.push(request);
});