Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat(jsdom): add `parseWithCheerio` context helper

Martin Adámekc8f0796aebc0
diff --git a/packages/cheerio-crawler/package.json b/packages/cheerio-crawler/package.json
index a82beebb4..29a466436 100644
--- a/packages/cheerio-crawler/package.json
+++ b/packages/cheerio-crawler/package.json
@@ -55,7 +55,7 @@
"dependencies": {
"@crawlee/http": "^3.3.0",
"@crawlee/types": "^3.3.0",
- "cheerio": "1.0.0-rc.12",
+ "cheerio": "^1.0.0-rc.12",
"htmlparser2": "^8.0.1",
"tslib": "^2.4.0"
}
diff --git a/packages/jsdom-crawler/package.json b/packages/jsdom-crawler/package.json
index 1e419769a..9e7bc2c17 100644
--- a/packages/jsdom-crawler/package.json
+++ b/packages/jsdom-crawler/package.json
@@ -57,6 +57,7 @@
"@crawlee/http": "^3.3.0",
"@crawlee/types": "^3.3.0",
"@types/jsdom": "^21.0.0",
+ "cheerio": "^1.0.0-rc.12",
"jsdom": "^21.0.0",
"ow": "^0.28.2",
"tslib": "^2.4.0"
diff --git a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts b/packages/jsdom-crawler/src/internals/jsdom-crawler.ts
index a26c9a31c..3929c1831 100644
--- a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts
+++ b/packages/jsdom-crawler/src/internals/jsdom-crawler.ts
@@ -1,4 +1,5 @@
import ow from 'ow';
+import * as cheerio from 'cheerio';
import type {
HttpCrawlerOptions,
InternalHttpCrawlingContext,
@@ -10,7 +11,7 @@ import type {
Configuration,
} from '@crawlee/http';
import { HttpCrawler, enqueueLinks, Router, resolveBaseUrlForEnqueueLinksFiltering, tryAbsoluteURL } from '@crawlee/http';
-import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
+import type { Dictionary } from '@crawlee/types';
import { concatStreamToBuffer } from '@apify/utilities';
import type { DOMWindow } from 'jsdom';
import { JSDOM, ResourceLoader, VirtualConsole } from 'jsdom';
@@ -46,7 +47,18 @@ export interface JSDOMCrawlingContext<
> extends InternalHttpCrawlingContext<UserData, JSONData, JSDOMCrawler> {
window: DOMWindow;
- enqueueLinks: (options?: EnqueueLinksOptions) => Promise<BatchAddRequestsResult>;
+ /**
+ * Returns Cheerio handle, allowing to work with the data same way as with {@apilink CheerioCrawler}.
+ *
+ * **Example usage:**
+ * ```javascript
+ * async requestHandler({ parseWithCheerio }) {
+ * const $ = await parseWithCheerio();
+ * const title = $('title').text();
+ * });
+ * ```
+ */
+ parseWithCheerio(): Promise<cheerio.CheerioAPI>;
}
export type JSDOMRequestHandler<
@@ -220,6 +232,13 @@ export class JSDOMCrawler extends HttpCrawler<JSDOMCrawlingContext> {
},
};
}
+
+ override async _runRequestHandler(context: JSDOMCrawlingContext) {
+ context.parseWithCheerio = async () => {
+ return cheerio.load(context.body);
+ };
+ await super._runRequestHandler(context);
+ }
}
interface EnqueueLinksInternalOptions {
diff --git a/yarn.lock b/yarn.lock
index 9d37a20dc..0f6591a69 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -797,7 +797,7 @@ __metadata:
dependencies:
"@crawlee/http": ^3.3.0
"@crawlee/types": ^3.3.0
- cheerio: 1.0.0-rc.12
+ cheerio: ^1.0.0-rc.12
htmlparser2: ^8.0.1
tslib: ^2.4.0
languageName: unknown
@@ -875,6 +875,7 @@ __metadata:
"@crawlee/http": ^3.3.0
"@crawlee/types": ^3.3.0
"@types/jsdom": ^21.0.0
+ cheerio: ^1.0.0-rc.12
jsdom: ^21.0.0
ow: ^0.28.2
tslib: ^2.4.0
@@ -3775,7 +3776,7 @@ __metadata:
languageName: node
linkType: hard
-"cheerio@npm:1.0.0-rc.12":
+"cheerio@npm:1.0.0-rc.12, cheerio@npm:^1.0.0-rc.12":
version: 1.0.0-rc.12
resolution: "cheerio@npm:1.0.0-rc.12"
dependencies: