Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Switch CheerioCrawler to async-await; fix tests; implement PR comments
Ondra Urbanafdc47139f19
diff --git a/src/basic_crawler.js b/src/basic_crawler.jsindex aaef02c83..f6dfb2132 100644--- a/src/basic_crawler.js+++ b/src/basic_crawler.js@@ -68,7 +68,10 @@ const DEFAULT_OPTIONS = {* Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.* @param {Function} [options.handleRequestFunction]* Function that processes a single `Request` object. It must return a promise.- * @param {Function} [options.handleFailedRequestFunction=({ request, error }) => log.error('Request failed', _.pick(request, 'url', 'uniqueKey'))`]+ * @param {Function} [options.handleFailedRequestFunction=({ request }) => {+ const details = _.pick(request, 'id', 'url', 'method', 'uniqueKey');++ log.error('BasicCrawler: Request failed and reached maximum retries', details);]* Function that handles requests that failed more then `option.maxRequestRetries` times.* @param {Number} [options.maxRequestRetries=3]* How many times the request is retried if `handleRequestFunction` failed.diff --git a/src/cheerio_crawler.js b/src/cheerio_crawler.jsindex d643ded7a..7dab85b5f 100644--- a/src/cheerio_crawler.js+++ b/src/cheerio_crawler.js@@ -1,25 +1,142 @@import rp from 'request-promise';import _ from 'underscore';import cheerio from 'cheerio';+import log from 'apify-shared/log';import { checkParamOrThrow } from 'apify-client/build/utils';import BasicCrawler from './basic_crawler';-import { isPromise } from './utils';+import { createTimeoutPromise } from './utils';const DEFAULT_OPTIONS = {- requestFunction: ({ request }) => rp({ url: request.url, method: request.method, headers: request.headers }),- pageOpsTimeoutMillis: 300000,- requestOptions: {- ignoreSslErrors: true,+ handlePageTimeoutSecs: 300,+ ignoreSslErrors: false,+ handleFailedRequestFunction: ({ request }) => {+ const details = _.pick(request, 'id', 'url', 'method', 'uniqueKey');++ log.error('CheerioCrawler: Request failed and reached maximum retries', details);},};++/**+ * Builds upon the `BasicCrawler` with functionality specifically designed for parsing raw HTML+ * of web pages. It uses the `cheerio` npm package to provide the user with a pre-parsed HTML+ * document, that may be manipulated in a same way one would manipulate the DOM in the browser+ * with jQuery.+ *+ * Unlike `BasicCrawler` which uses a `handleRequestFunction`, the `CheerioCrawler` uses a `handlePageFunction`.+ * This function gets invoked only after the HTTP response is received and the page's HTML has been parsed.+ * To customize the HTTP request that is used to fetch the raw HTML, see the `requestFunction` option.+ *+ * `CheerioCrawler` invokes `handlePageFunction` for each `Request` object fetched from `options.requestList` or `options.requestQueue`,+ * as long as none of them is empty. New requests are only handled if there is enough free CPU and memory available,+ * using the functionality provided by the `AutoscaledPool` class.+ * Note that all `AutoscaledPool` configuration options can be passed to `options` parameter of the `CheerioCrawler` constructor.+ *+ * If both `requestList` and `requestQueue` are used, the instance first+ * processes URLs from the `requestList` and automatically enqueues all of them to `requestQueue` before it starts+ * their processing. This guarantees that a single URL is not crawled multiple times.+ *+ * Example usage:+ *+ * ```javascript+ * const rp = require('request-promise');+ *+ * // Prepare a list of URLs to crawl+ * const requestList = new Apify.RequestList({+ * sources: [+ * { url: 'http://www.example.com/page-1' },+ * { url: 'http://www.example.com/page-2' },+ * ],+ * });+ * await requestList.initialize();+ *+ * // Crawl the URLs+ * const crawler = new Apify.CheerioCrawler({+ * requestList,+ * handlePageFunction: async ({ $, html, request }) => {+ *+ * const data = [];+ *+ * // Do some data extraction from the page with Cheerio.+ * $('.some-collection').each((index, el) => {+ * data.push({ title: $(el).find('.some-title').text() });+ * });+ *+ * // Save the data to dataset.+ * await Apify.pushData({+ * url: request.url,+ * html,+ * data,+ * })+ * },+ * });+ *+ * await crawler.run();+ * ```+ *+ * @param {Object} options+ * @param {RequestList} [options.requestList]+ * Static list of URLs to be processed.+ * @param {RequestQueue} [options.requestQueue]+ * Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.+ * @param {Function} [options.handlePageFunction]+ * A function that receives three arguments: the Cheerio object `$`, the raw HTML and the `Request` object and does all the document manipulation.+ * If it returns a promise, it is awaited.+ * @param {Number} [options.handlePageTimeoutSecs=300]+ * Timeout in which the function passed as `options.handlePageFunction` needs to finish, given in seconds.+ * @param {Function} [options.requestFunction=({ request }) => {+ * return rp({ // request-promise npm package+ * url: request.url,+ * method: request.method,+ * headers: request.headers,+ * strictSSL: !this.ignoreSslErrors,+ * });+ * }]+ * Overrides the function that performs the HTTP request to get the raw HTML needed for Cheerio.+ * @param {Boolean} [options.ignoreSslErrors=false]+ * If set to true, SSL certificate errors will be ignored. This is dependent on using the default+ * request function. If using a custom request function, user needs to implement this functionality.+ * @param {Function} [options.handleFailedRequestFunction=({ request }) => {+ * const details = _.pick(request, 'id', 'url', 'method', 'uniqueKey');+ * log.error('BasicCrawler: Request failed and reached maximum retries', details);]+ * Function that handles requests that failed more then `option.maxRequestRetries` times.+ * @param {Number} [options.maxRequestRetries=3]+ * How many times the request is retried if either `requestFunction` or `handlePageFunction` failed.+ * @param {Number} [options.maxRequestsPerCrawl]+ * Maximum number of pages that the crawler will open. The crawl will stop when this limit is reached.+ * Always set this value in order to prevent infinite loops in misconfigured crawlers.+ * Note that in cases of parallel crawling, the actual number of pages visited might be slightly higher than this value.+ * @param {Number} [options.maxMemoryMbytes]+ * Maximum memory available in the system+ * See `AutoscaledPool` for details.+ * @param {Number} [options.minConcurrency=1]+ * Minimum number of request to process in parallel.+ * See `AutoscaledPool` for details.+ * @param {Number} [options.maxConcurrency=1000]+ * Maximum number of request to process in parallel.+ * See `AutoscaledPool` for details.+ * @param {Number} [options.minFreeMemoryRatio=0.2]+ * Minimum ratio of free memory kept in the system.+ * See `AutoscaledPool` for details.+ * @param {Function} [opts.isFinishedFunction]+ * By default CheerioCrawler finishes when all the requests have been processed.+ * You can override this behaviour by providing custom `isFinishedFunction`.+ * This function that is called every time there are no requests being processed.+ * If it resolves to `true` then the crawler's run finishes.+ * See `AutoscaledPool` for details.+ * @param {Boolean} [options.ignoreMainProcess=false]+ * If set to `true` then the auto-scaling manager does not consider memory consumption+ * of the main Node.js process when scaling the pool up or down.+ * This is mainly useful when tasks are running as separate processes (e.g. web browsers).+ * See `AutoscaledPool` for details.+ */export default class CheerioCrawler {constructor(opts) {const {requestFunction,handlePageFunction,- pageOpsTimeoutMillis,- requestOptions,+ handlePageTimeoutSecs,+ ignoreSslErrors,// Autoscaled pool optionsmaxMemoryMbytes,@@ -36,16 +153,15 @@ export default class CheerioCrawler {handleFailedRequestFunction,} = _.defaults(opts, DEFAULT_OPTIONS);- checkParamOrThrow(requestFunction, 'opts.requestFunction', 'Function');checkParamOrThrow(handlePageFunction, 'opts.handlePageFunction', 'Function');+ checkParamOrThrow(requestFunction, 'opts.requestFunction', 'Maybe Function');checkParamOrThrow(handleFailedRequestFunction, 'opts.handleFailedRequestFunction', 'Maybe Function');- checkParamOrThrow(requestOptions, 'opts.requestOptions', 'Object');- if (requestOptions.ignoreSslErrors) process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0';+ this.ignoreSslErrors = ignoreSslErrors;this.handlePageFunction = handlePageFunction;- this.requestFunction = requestFunction;- this.pageOpsTimeoutMillis = pageOpsTimeoutMillis;+ this.requestFunction = requestFunction || this._defaultRequestFunction;+ this.handlePageTimeoutSecs = handlePageTimeoutSecs;this.basicCrawler = new BasicCrawler({// Basic crawler options.@@ -71,7 +187,7 @@ export default class CheerioCrawler {** @return {Promise}*/- run() {+ async run() {return this.basicCrawler.run();}@@ -80,16 +196,25 @@ export default class CheerioCrawler {** @ignore*/- _handleRequestFunction({ request }) {- const handlePagePromise = this.requestFunction({ request })- .then((html) => {- const $ = cheerio.load(html);- const promise = this.handlePageFunction({ $, html, request });- if (!isPromise(promise)) throw new Error('User provided handlePageFunction must return a Promise.');- return promise;- });+ async _handleRequestFunction({ request }) {+ const html = await this.requestFunction({ request });+ const $ = cheerio.load(html);+ await Promise.race([+ this.handlePageFunction({ $, html, request }),+ createTimeoutPromise(this.handlePageTimeoutSecs * 1000, 'CheerioCrawler: handlePageFunction timed out.')+ ]);+ }- return handlePagePromise- .timeout(this.pageOpsTimeoutMillis, 'CheerioCrawler: handlePageFunction timed out.');+ /**+ * Default request function to be used.+ * @ignore+ */+ async _defaultRequestFunction({ request }) {+ return rp({+ url: request.url,+ method: request.method,+ headers: request.headers,+ strictSSL: !this.ignoreSslErrors,+ });}}diff --git a/src/constants.js b/src/constants.jsindex 919827f38..28edf94a5 100644--- a/src/constants.js+++ b/src/constants.js@@ -103,6 +103,4 @@ consts.USER_AGENT_LIST = ['Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:61.0) Gecko/20100101 Firefox/61.0',];-consts.LOCAL_USER_ID = 'local_user_id';-module.exports = consts;diff --git a/src/dataset.js b/src/dataset.jsindex 62f73b312..2c452b629 100644--- a/src/dataset.js+++ b/src/dataset.js@@ -6,7 +6,7 @@ import Promise from 'bluebird';import { leftpad } from 'apify-shared/utilities';import LruCache from 'apify-shared/lru_cache';import { checkParamOrThrow } from 'apify-client/build/utils';-import { ENV_VARS, LOCAL_EMULATION_SUBDIRS, MAX_PAYLOAD_SIZE_BYTES, LOCAL_USER_ID } from './constants';+import { ENV_VARS, LOCAL_EMULATION_SUBDIRS, MAX_PAYLOAD_SIZE_BYTES } from './constants';import { apifyClient, ensureDirExists } from './utils';export const LOCAL_EMULATION_SUBDIR = LOCAL_EMULATION_SUBDIRS.datasets;@@ -204,7 +204,7 @@ export class Dataset {}/**- * Returns a dataset object containing general information about the dataset.+ * Returns an object containing general information about the dataset.** @example* {@@ -444,7 +444,7 @@ export class DatasetLocal {return {id,name,- userId: LOCAL_USER_ID,+ userId: process.env[ENV_VARS.USER_ID] || null,createdAt: this.createdAt,modifiedAt: this.modifiedAt,accessedAt: this.accessedAt,diff --git a/test/dataset.js b/test/dataset.jsindex fce29b769..610e8d096 100644--- a/test/dataset.js+++ b/test/dataset.js@@ -5,7 +5,7 @@ import fs from 'fs-extra';import path from 'path';import sinon from 'sinon';import { leftpad, delayPromise } from 'apify-shared/utilities';-import { ENV_VARS, MAX_PAYLOAD_SIZE_BYTES, LOCAL_USER_ID } from '../build/constants';+import { ENV_VARS, MAX_PAYLOAD_SIZE_BYTES } from '../build/constants';import { LOCAL_FILENAME_DIGITS, Dataset, DatasetLocal, LOCAL_EMULATION_SUBDIR,LOCAL_GET_ITEMS_DEFAULT_LIMIT, checkAndSerialize, chunkBySize } from '../build/dataset';import { apifyClient } from '../build/utils';@@ -108,22 +108,44 @@ describe('dataset', () => {});it('getInfo() should work', async () => {- const dataset = await getLocalDataset([+ const datasetName = 'stats-dataset';+ const dataset = new DatasetLocal(datasetName, LOCAL_EMULATION_DIR);+ await Apify.utils.sleep(2);++ // Save orig env var since it persists over tests.+ const original = process.env[ENV_VARS.USER_ID];+ // Try empty ID+ delete process.env[ENV_VARS.USER_ID];++ let info = await dataset.getInfo();+ expect(info).to.be.an('object');+ expect(info.id).to.be.eql(datasetName);+ expect(info.name).to.be.eql(datasetName);+ expect(info.userId).to.be.eql(null);+ expect(info.itemsCount).to.be.eql(0);++ const cTime = info.createdAt.getTime();+ let mTime = info.modifiedAt.getTime();++ expect(cTime).to.be.below(Date.now());+ expect(cTime).to.be.eql(mTime);++ await dataset.pushData([{ foo: 'a' },{ foo: 'b' },{ foo: 'c' },{ foo: 'd' },]);+ await Apify.utils.sleep(2);- let info = await dataset.getInfo();+ info = await dataset.getInfo();expect(info).to.be.an('object');- expect(info.id).to.be.eql('my-dataset');- expect(info.name).to.be.eql('my-dataset');- expect(info.userId).to.be.eql(LOCAL_USER_ID);+ expect(info.id).to.be.eql(datasetName);+ expect(info.name).to.be.eql(datasetName);+ expect(info.userId).to.be.eql(null);expect(info.itemsCount).to.be.eql(4);- const cTime = info.createdAt.getTime();- let mTime = info.modifiedAt.getTime();+ mTime = info.modifiedAt.getTime();let aTime = info.accessedAt.getTime();expect(cTime).to.be.below(Date.now());@@ -131,16 +153,19 @@ describe('dataset', () => {expect(mTime).to.be.eql(aTime);await dataset.getData();-await Apify.utils.sleep(2);const now = Date.now();await Apify.utils.sleep(2);+ // Try setting an ID+ const userId = 'some_ID';+ process.env[ENV_VARS.USER_ID] = userId;+info = await dataset.getInfo();expect(info).to.be.an('object');- expect(info.id).to.be.eql('my-dataset');- expect(info.name).to.be.eql('my-dataset');- expect(info.userId).to.be.eql(LOCAL_USER_ID);+ expect(info.id).to.be.eql(datasetName);+ expect(info.name).to.be.eql(datasetName);+ expect(info.userId).to.be.eql(userId);expect(info.itemsCount).to.be.eql(4);const cTime2 = info.createdAt.getTime();@@ -151,6 +176,9 @@ describe('dataset', () => {expect(mTime).to.be.below(aTime);expect(mTime).to.be.below(now);expect(aTime).to.be.below(now);++ // Restore.+ process.env[ENV_VARS.USER_ID] = original;});it('forEach() should work', async () => {