Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Initial commit

Ondra Urbana8774a50bc2d
diff --git a/src/cheerio_crawler.js b/src/cheerio_crawler.js
new file mode 100644
index 000000000..7a31b550b
--- /dev/null
+++ b/src/cheerio_crawler.js
@@ -0,0 +1,92 @@
+import rp from 'request-promise';
+import _ from 'underscore';
+import { checkParamOrThrow } from 'apify-client/build/utils';
+import BasicCrawler from './basic_crawler';
+
+const DEFAULT_OPTIONS = {
+ requestFunction: ({ request }) => rp({ url: request.url, method: request.method }),
+ pageOpsTimeoutMillis: 300000,
+};
+
+export default class CheerioCrawler {
+ constructor(opts) {
+ const {
+ requestFunction,
+ handlePageFunction,
+ pageOpsTimeoutMillis,
+ requestOptions,
+
+ // Autoscaled pool options
+ maxMemoryMbytes,
+ maxConcurrency,
+ minConcurrency,
+ minFreeMemoryRatio,
+ isFinishedFunction,
+
+ // Basic crawler options
+ requestList,
+ requestQueue,
+ maxRequestRetries,
+ maxRequestsPerCrawl,
+ handleFailedRequestFunction,
+ } = _.defaults(opts, DEFAULT_OPTIONS);
+
+ checkParamOrThrow(requestFunction, 'opts.requestFunction', 'Function');
+ checkParamOrThrow(handlePageFunction, 'opts.handlePageFunction', 'Function');
+ checkParamOrThrow(handleFailedRequestFunction, 'opts.handleFailedRequestFunction', 'Maybe Function');
+ checkParamOrThrow(requestOptions, 'opts.requestOptions', 'Maybe Object');
+
+ this.handlePageFunction = handlePageFunction;
+ this.requestFunction = requestFunction;
+ this.pageOpsTimeoutMillis = pageOpsTimeoutMillis;
+
+ this.basicCrawler = new BasicCrawler({
+ // Basic crawler options.
+ requestList,
+ requestQueue,
+ maxRequestRetries,
+ maxRequestsPerCrawl,
+ handleRequestFunction: (...args) => this._handleRequestFunction(...args),
+ handleFailedRequestFunction,
+
+ // Autoscaled pool options.
+ maxMemoryMbytes,
+ maxConcurrency,
+ minConcurrency,
+ minFreeMemoryRatio,
+ isFinishedFunction,
+ ignoreMainProcess: true,
+ });
+ }
+
+ /**
+ * Runs the crawler. Returns promise that gets resolved once all the requests got processed.
+ *
+ * @return {Promise}
+ */
+ run() {
+ return this.basicCrawler.run();
+ }
+
+ /**
+ * Wrapper around handlePageFunction that opens and closes pages etc.
+ *
+ * @ignore
+ */
+ _handleRequestFunction({ request }) {
+ const handlePagePromise = this.requestFunction({ request })
+ .then((response) => {
+ const promise = this.handlePageFunction({
+ request,
+ response,
+ });
+
+ if (!isPromise(promise)) throw new Error('User provided handlePageFunction must return a Promise.');
+
+ return promise;
+ });
+
+ return handlePagePromise
+ .timeout(this.pageOpsTimeoutMillis, 'CheerioCrawler: handlePageFunction timed out.');
+ }
+}