Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: ensure CrawlingContext interface is inferred correctly in route handlers
Martin Adámekaa84633b1a20
diff --git a/packages/core/src/crawlers/crawler_commons.ts b/packages/core/src/crawlers/crawler_commons.tsindex fcf3e1b07..dba824fe4 100644--- a/packages/core/src/crawlers/crawler_commons.ts+++ b/packages/core/src/crawlers/crawler_commons.ts@@ -7,7 +7,8 @@ import type { ProxyInfo } from '../proxy_configuration';import type { Request } from '../request';import type { Session } from '../session_pool/session';-export interface CrawlingContext<Crawler = unknown, UserData extends Dictionary = Dictionary> extends Record<PropertyKey, unknown> {+// eslint-disable-next-line @typescript-eslint/ban-types+export interface CrawlingContext<Crawler = unknown, UserData extends Dictionary = Dictionary> extends Record<string & {}, unknown> {id: string;/*** The original {@apilink Request} object.diff --git a/packages/core/src/except.ts b/packages/core/src/except.tsdeleted file mode 100644index 7472721f8..000000000--- a/packages/core/src/except.ts+++ /dev/null@@ -1,13 +0,0 @@-// Do not convert this file to .d.ts--type IsEqual<T, U> =- (<G>() => G extends T ? 1 : 2) extends- (<G>() => G extends U ? 1 : 2)- ? true- : false;--type Filter<KeyType, ExcludeType> = IsEqual<KeyType, ExcludeType> extends true ? never : (KeyType extends ExcludeType ? never : KeyType);--export type Except<ObjectType, KeysType extends keyof ObjectType> = {- [KeyType in keyof ObjectType as Filter<KeyType, KeysType>]: ObjectType[KeyType];-};diff --git a/packages/core/src/router.ts b/packages/core/src/router.tsindex 74c50c5d8..41c213640 100644--- a/packages/core/src/router.ts+++ b/packages/core/src/router.ts@@ -2,7 +2,6 @@ import type { Dictionary } from '@crawlee/types';import type { CrawlingContext } from './crawlers/crawler_commons';import type { Awaitable } from './typedefs';import type { Request } from './request';-import type { Except } from './except';import { MissingRouteError } from './errors';const defaultRoute = Symbol('default-route');@@ -79,7 +78,7 @@ export class Router<Context extends CrawlingContext> {*/addHandler<UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(label: string | symbol,- handler: (ctx: Except<Context, 'request'> & {request: Request<UserData>}) => Awaitable<void>,+ handler: (ctx: Omit<Context, 'request'> & { request: Request<UserData> }) => Awaitable<void>,) {this.validate(label);this.routes.set(label, handler);@@ -89,7 +88,7 @@ export class Router<Context extends CrawlingContext> {* Registers default route handler.*/addDefaultHandler<UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(- handler: (ctx: Except<Context, 'request'> & {request: Request<UserData>}) => Awaitable<void>,+ handler: (ctx: Omit<Context, 'request'> & { request: Request<UserData> }) => Awaitable<void>,) {this.validate(defaultRoute);this.routes.set(defaultRoute, handler);diff --git a/test/core/crawlers/cheerio_crawler.test.ts b/test/core/crawlers/cheerio_crawler.test.tsindex e61eaba4a..0c6170457 100644--- a/test/core/crawlers/cheerio_crawler.test.ts+++ b/test/core/crawlers/cheerio_crawler.test.ts@@ -125,9 +125,9 @@ describe('CheerioCrawler', () => {},});- cheerioCrawler.router.addDefaultHandler(({ $, body, request }) => {+ cheerioCrawler.router.addDefaultHandler<{ title: string; body: string }>(({ $, body, request }) => {request.userData.title = $('title').text();- request.userData.body = body;+ request.userData.body = body.toString();processed.push(request);});