Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat: add `parseWithCheerio` context helper to cheerio crawler

Martin Adámekb336a739117a
diff --git a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
index aa940e85d..46f41e911 100644
--- a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
+++ b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
@@ -40,6 +40,21 @@ export interface CheerioCrawlingContext<
* Cheerio is available only for HTML and XML content types.
*/
$: cheerio.CheerioAPI;
+
+ /**
+ * Returns Cheerio handle, this is here to unify the crawler API, so they all have this handy method.
+ * It has the same return type as the `$` context property, use it only if you are abstracting your workflow to
+ * support different context types in one handler.
+ *
+ * **Example usage:**
+ * ```javascript
+ * async requestHandler({ parseWithCheerio }) {
+ * const $ = await parseWithCheerio();
+ * const title = $('title').text();
+ * });
+ * ```
+ */
+ parseWithCheerio(): Promise<cheerio.CheerioAPI>;
}
export type CheerioRequestHandler<
@@ -175,6 +190,11 @@ export class CheerioCrawler extends HttpCrawler<CheerioCrawlingContext> {
.pipe(parser);
});
}
+
+ protected override async _runRequestHandler(context: CheerioCrawlingContext) {
+ context.parseWithCheerio = () => Promise.resolve(context.$);
+ await super._runRequestHandler(context);
+ }
}
interface EnqueueLinksInternalOptions {