Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: ensure default route is not ignored in `CheerioCrawler` (#1411)

Martin Adámek6475628eb0fa
diff --git a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
index 1186cbcb5..8a9ea0e54 100644
--- a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
+++ b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
@@ -417,7 +417,7 @@ export class CheerioCrawler extends BasicCrawler<CheerioCrawlingContext> {
super({
...basicCrawlerOptions,
// Will be overridden below
- requestHandler: () => {},
+ requestHandler,
autoscaledPoolOptions,
// We need to add some time for internal functions to finish,
// but not too much so that we would stall the crawler.
diff --git a/test/core/crawlers/cheerio_crawler.test.ts b/test/core/crawlers/cheerio_crawler.test.ts
index 3d5fbd8c6..b7e5687b1 100644
--- a/test/core/crawlers/cheerio_crawler.test.ts
+++ b/test/core/crawlers/cheerio_crawler.test.ts
@@ -235,6 +235,39 @@ describe('CheerioCrawler', () => {
});
});
+ test('should work with router', async () => {
+ const requestList = await getRequestListForMirror(port);
+ const processed: Request[] = [];
+ const failed: Request[] = [];
+
+ const cheerioCrawler = new CheerioCrawler({
+ requestList,
+ minConcurrency: 2,
+ maxConcurrency: 2,
+ failedRequestHandler: ({ request }) => {
+ failed.push(request);
+ },
+ });
+
+ cheerioCrawler.router.addDefaultHandler(({ $, body, request }) => {
+ request.userData.title = $('title').text();
+ request.userData.body = body;
+ processed.push(request);
+ });
+
+ await cheerioCrawler.run();
+
+ expect(cheerioCrawler.autoscaledPool.minConcurrency).toBe(2);
+ expect(processed).toHaveLength(4);
+ expect(failed).toHaveLength(0);
+
+ processed.forEach((request) => {
+ expect(request.userData.title).toBe('Title');
+ expect(typeof request.userData.body).toBe('string');
+ expect((request.userData.body as string).length).not.toBe(0);
+ });
+ });
+
test('should ignore ssl by default', async () => {
const sources = [
{ url: 'http://example.com/?q=1' },