Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Add cheerio
Ondra Urbanc24d78174a65
diff --git a/package.json b/package.jsonindex b48e1883f..8fdd6ecf4 100644--- a/package.json+++ b/package.json@@ -91,6 +91,7 @@},"optionalDependencies": {"puppeteer": "^1.6.0",- "selenium-webdriver": "^3.6.0"+ "selenium-webdriver": "^3.6.0",+ "cheerio": "^1.0.0-rc.2"}}diff --git a/src/cheerio_crawler.js b/src/cheerio_crawler.jsindex 7a31b550b..b0787920e 100644--- a/src/cheerio_crawler.js+++ b/src/cheerio_crawler.js@@ -1,10 +1,11 @@import rp from 'request-promise';import _ from 'underscore';+import cheerio from 'cheerio';import { checkParamOrThrow } from 'apify-client/build/utils';import BasicCrawler from './basic_crawler';const DEFAULT_OPTIONS = {- requestFunction: ({ request }) => rp({ url: request.url, method: request.method }),+ requestFunction: ({ request }) => rp({ url: request.url, method: request.method, headers: request.headers }),pageOpsTimeoutMillis: 300000,};@@ -75,14 +76,10 @@ export default class CheerioCrawler {*/_handleRequestFunction({ request }) {const handlePagePromise = this.requestFunction({ request })- .then((response) => {- const promise = this.handlePageFunction({- request,- response,- });-+ .then((html) => {+ const $ = cheerio.load(html);+ const promise = this.handlePageFunction({ $, html, request });if (!isPromise(promise)) throw new Error('User provided handlePageFunction must return a Promise.');-return promise;});diff --git a/src/index.js b/src/index.jsindex f0620a7cc..94ae49488 100644--- a/src/index.js+++ b/src/index.js@@ -3,6 +3,7 @@ import { ENV_VARS } from './constants';import { main, readyFreddy, getEnv, call, getApifyProxyUrl } from './actor';import AutoscaledPool from './autoscaled_pool';import BasicCrawler from './basic_crawler';+import CheerioCrawler from './cheerio_crawler';import { pushData, openDataset } from './dataset';import events, { initializeEvents, stopEvents } from './events';import { getValue, setValue, openKeyValueStore } from './key_value_store';@@ -55,6 +56,9 @@ module.exports = {// Basic crawlerBasicCrawler,+ // Cheerio crawler+ CheerioCrawler,+// DatasetpushData,openDataset,