Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Add cheerio

Ondra Urbanc24d78174a65
diff --git a/package.json b/package.json
index b48e1883f..8fdd6ecf4 100644
--- a/package.json
+++ b/package.json
@@ -91,6 +91,7 @@
},
"optionalDependencies": {
"puppeteer": "^1.6.0",
- "selenium-webdriver": "^3.6.0"
+ "selenium-webdriver": "^3.6.0",
+ "cheerio": "^1.0.0-rc.2"
}
}
diff --git a/src/cheerio_crawler.js b/src/cheerio_crawler.js
index 7a31b550b..b0787920e 100644
--- a/src/cheerio_crawler.js
+++ b/src/cheerio_crawler.js
@@ -1,10 +1,11 @@
import rp from 'request-promise';
import _ from 'underscore';
+import cheerio from 'cheerio';
import { checkParamOrThrow } from 'apify-client/build/utils';
import BasicCrawler from './basic_crawler';
const DEFAULT_OPTIONS = {
- requestFunction: ({ request }) => rp({ url: request.url, method: request.method }),
+ requestFunction: ({ request }) => rp({ url: request.url, method: request.method, headers: request.headers }),
pageOpsTimeoutMillis: 300000,
};
@@ -75,14 +76,10 @@ export default class CheerioCrawler {
*/
_handleRequestFunction({ request }) {
const handlePagePromise = this.requestFunction({ request })
- .then((response) => {
- const promise = this.handlePageFunction({
- request,
- response,
- });
-
+ .then((html) => {
+ const $ = cheerio.load(html);
+ const promise = this.handlePageFunction({ $, html, request });
if (!isPromise(promise)) throw new Error('User provided handlePageFunction must return a Promise.');
-
return promise;
});
diff --git a/src/index.js b/src/index.js
index f0620a7cc..94ae49488 100644
--- a/src/index.js
+++ b/src/index.js
@@ -3,6 +3,7 @@ import { ENV_VARS } from './constants';
import { main, readyFreddy, getEnv, call, getApifyProxyUrl } from './actor';
import AutoscaledPool from './autoscaled_pool';
import BasicCrawler from './basic_crawler';
+import CheerioCrawler from './cheerio_crawler';
import { pushData, openDataset } from './dataset';
import events, { initializeEvents, stopEvents } from './events';
import { getValue, setValue, openKeyValueStore } from './key_value_store';
@@ -55,6 +56,9 @@ module.exports = {
// Basic crawler
BasicCrawler,
+ // Cheerio crawler
+ CheerioCrawler,
+
// Dataset
pushData,
openDataset,