Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat(HttpCrawler): add `parseWithCheerio` helper to `HttpCrawler` (#1906)
Pavel 'Strajk' Dolecekff5f76f9336c
diff --git a/packages/http-crawler/package.json b/packages/http-crawler/package.jsonindex 25f1dfb30..7005ece21 100644--- a/packages/http-crawler/package.json+++ b/packages/http-crawler/package.json@@ -58,6 +58,7 @@"@crawlee/basic": "^3.3.1","@crawlee/types": "^3.3.1","@types/content-type": "^1.1.5",+ "cheerio": "^1.0.0-rc.12","content-type": "^1.0.4","got-scraping": "^3.2.9","iconv-lite": "^0.6.3",diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.tsindex 61c5dbce2..9c8cbf914 100644--- a/packages/http-crawler/src/internals/http-crawler.ts+++ b/packages/http-crawler/src/internals/http-crawler.ts@@ -24,6 +24,7 @@ import {} from '@crawlee/basic';import type { Awaitable, Dictionary } from '@crawlee/types';import type { RequestLike, ResponseLike } from 'content-type';+import * as cheerio from 'cheerio';import contentTypeParser from 'content-type';import mime from 'mime-types';import type { OptionsInit, Method, Request as GotRequest, Options } from 'got-scraping';@@ -191,6 +192,8 @@ export interface InternalHttpCrawlingContext<*/contentType: { type: string; encoding: BufferEncoding };response: IncomingMessage;++ parseWithCheerio(): Promise<cheerio.CheerioAPI>;}export interface HttpCrawlingContext<UserData extends Dictionary = any, JSONData extends JsonValue = any>@@ -437,6 +440,9 @@ export class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any, Hconst contentType = parsed.contentType!;tryCancel();+ // `??=` because descendant classes may already set optimized version+ crawlingContext.parseWithCheerio ??= async () => cheerio.load(parsed.body!.toString());+if (this.useSessionPool) {this._throwOnBlockedRequest(session!, response.statusCode!);}diff --git a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts b/packages/jsdom-crawler/src/internals/jsdom-crawler.tsindex 95cf67058..0e6403eda 100644--- a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts+++ b/packages/jsdom-crawler/src/internals/jsdom-crawler.ts@@ -274,9 +274,7 @@ export class JSDOMCrawler extends HttpCrawler<JSDOMCrawlingContext> {}override async _runRequestHandler(context: JSDOMCrawlingContext) {- context.parseWithCheerio = async () => {- return cheerio.load(context.body);- };+ context.parseWithCheerio = () => Promise.resolve(cheerio.load(context.body));await super._runRequestHandler(context);}}diff --git a/test/core/crawlers/http_crawler.test.ts b/test/core/crawlers/http_crawler.test.tsindex b071d0930..7f54fb7ce 100644--- a/test/core/crawlers/http_crawler.test.ts+++ b/test/core/crawlers/http_crawler.test.ts@@ -94,6 +94,22 @@ test('works', async () => {expect(results[0].includes('Example Domain')).toBeTruthy();});+test('parseWithCheerio works', async () => {+ const results: string[] = [];++ const crawler = new HttpCrawler({+ maxRequestRetries: 0,+ requestHandler: async ({ parseWithCheerio }) => {+ const $ = await parseWithCheerio();+ results.push($('title').text());+ },+ });++ await crawler.run([`${url}/hello.html`]);++ expect(results).toStrictEqual(['Example Domain']);+});+test('should parse content type from header', async () => {const results: { type: string; encoding: BufferEncoding }[] = [];diff --git a/yarn.lock b/yarn.lockindex 479ca6d77..bf48f6993 100644--- a/yarn.lock+++ b/yarn.lock@@ -858,6 +858,7 @@ __metadata:"@crawlee/basic": ^3.3.1"@crawlee/types": ^3.3.1"@types/content-type": ^1.1.5+ cheerio: ^1.0.0-rc.12content-type: ^1.0.4got-scraping: ^3.2.9iconv-lite: ^0.6.3