Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: using explicitly typed router with `CheerioCrawler`

Martin Adámek07b7e69e1a7b
diff --git a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
index 200b4770c..2f5608332 100644
--- a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
+++ b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
@@ -36,14 +36,14 @@ const CHEERIO_OPTIMIZED_AUTOSCALED_POOL_OPTIONS = {
};
export type CheerioErrorHandler<
- UserData extends Dictionary = Dictionary,
+ UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
JSONData extends Dictionary = Dictionary,
-> = ErrorHandler<CheerioCrawlingContext<UserData, JSONData>>;
+ > = ErrorHandler<CheerioCrawlingContext<UserData, JSONData>>;
export interface CheerioCrawlerOptions<
- UserData extends Dictionary = Dictionary,
+ UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
JSONData extends Dictionary = Dictionary,
-> extends Omit<BasicCrawlerOptions<CheerioCrawlingContext<UserData, JSONData>>,
+ > extends Omit<BasicCrawlerOptions<CheerioCrawlingContext<UserData, JSONData>>,
// Overridden with cheerio context
| 'requestHandler'
| 'handleRequestFunction'
@@ -53,7 +53,7 @@ export interface CheerioCrawlerOptions<
| 'handleRequestTimeoutSecs'
| 'errorHandler'
-> {
+ > {
/**
* User-provided function that performs the logic of the crawler. It is called for each page
* loaded and parsed by the crawler.
@@ -243,17 +243,17 @@ export interface CheerioCrawlerOptions<
}
export type CheerioHook<
- UserData extends Dictionary = Dictionary,
+ UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
JSONData extends Dictionary = Dictionary,
-> = (
+ > = (
crawlingContext: CheerioCrawlingContext<UserData, JSONData>,
gotOptions: OptionsInit,
) => Awaitable<void>;
export interface CheerioCrawlingContext<
- UserData extends Dictionary = Dictionary,
+ UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
JSONData extends Dictionary = Dictionary,
-> extends CrawlingContext<UserData> {
+ > extends CrawlingContext<UserData> {
/**
* The [Cheerio](https://cheerio.js.org/) object with parsed HTML.
*/
@@ -280,9 +280,9 @@ export interface CheerioCrawlingContext<
}
export type CheerioRequestHandler<
- UserData extends Dictionary = Dictionary,
+ UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
JSONData extends Dictionary = Dictionary,
-> = RequestHandler<CheerioCrawlingContext<UserData, JSONData>>;
+ > = RequestHandler<CheerioCrawlingContext<UserData, JSONData>>;
export interface CheerioCrawlerEnqueueLinksOptions extends Omit<EnqueueLinksOptions, 'urls' | 'requestQueue'> {}
/**
@@ -575,9 +575,9 @@ export class CheerioCrawler extends BasicCrawler<CheerioCrawlingContext> {
Object.defineProperty(crawlingContext, 'body', {
get() {
- // NOTE: For XML/HTML documents, we don't store the original body and only reconstruct it from Cheerio's DOM.
- // This is to save memory for high-concurrency crawls. The downside is that changes
- // made to DOM are reflected in the HTML, but we can live with that...
+ // NOTE: For XML/HTML documents, we don't store the original body and only reconstruct it from Cheerio's DOM.
+ // This is to save memory for high-concurrency crawls. The downside is that changes
+ // made to DOM are reflected in the HTML, but we can live with that...
if (dom) {
return isXml ? $!.xml() : $!.html({ decodeEntities: false });
}
diff --git a/test/core/crawlers/cheerio_crawler.test.ts b/test/core/crawlers/cheerio_crawler.test.ts
index 68e055947..51803eeb9 100644
--- a/test/core/crawlers/cheerio_crawler.test.ts
+++ b/test/core/crawlers/cheerio_crawler.test.ts
@@ -14,6 +14,7 @@ import {
AutoscaledPool,
CheerioCrawler,
CrawlerExtension,
+ createCheerioRouter,
mergeCookies,
ProxyConfiguration,
Request,
@@ -234,7 +235,7 @@ describe('CheerioCrawler', () => {
});
});
- test('should work with router', async () => {
+ test('should work with implicit router', async () => {
const requestList = await getRequestListForMirror(port);
const processed: Request[] = [];
const failed: Request[] = [];
@@ -267,6 +268,42 @@ describe('CheerioCrawler', () => {
});
});
+ test('should work with explcit router', async () => {
+ const requestList = await getRequestListForMirror(port);
+ const processed: Request[] = [];
+ const failed: Request[] = [];
+
+ const router = createCheerioRouter<CheerioCrawlingContext<{ title: string; body: string }>>();
+
+ const cheerioCrawler = new CheerioCrawler({
+ requestHandler: router,
+ requestList,
+ minConcurrency: 2,
+ maxConcurrency: 2,
+ failedRequestHandler: ({ request }) => {
+ failed.push(request);
+ },
+ });
+
+ router.addDefaultHandler(({ $, body, request }) => {
+ request.userData.title = $('title').text();
+ request.userData.body = body.toString();
+ processed.push(request);
+ });
+
+ await cheerioCrawler.run();
+
+ expect(cheerioCrawler.autoscaledPool.minConcurrency).toBe(2);
+ expect(processed).toHaveLength(4);
+ expect(failed).toHaveLength(0);
+
+ processed.forEach((request) => {
+ expect(request.userData.title).toBe('Title');
+ expect(typeof request.userData.body).toBe('string');
+ expect((request.userData.body as string).length).not.toBe(0);
+ });
+ });
+
test('should throw when no requestHandler nor default route provided', async () => {
const requestList = await getRequestListForMirror(port);