Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Implement proxy support in CheerioCrawler, add tests
Ondra Urbanccc8b5b520be
diff --git a/src/autoscaling/autoscaled_pool.js b/src/autoscaling/autoscaled_pool.jsindex 33f4f7a75..d1b53c8a5 100644--- a/src/autoscaling/autoscaled_pool.js+++ b/src/autoscaling/autoscaled_pool.js@@ -128,7 +128,7 @@ class AutoscaledPool {isTaskReadyFunction,systemStatusOptions,snapshotterOptions,- } = _.defaults(options, DEFAULT_OPTIONS);+ } = _.defaults({}, options, DEFAULT_OPTIONS);checkParamOrThrow(maxConcurrency, 'options.maxConcurrency', 'Number');checkParamOrThrow(minConcurrency, 'options.minConcurrency', 'Number');diff --git a/src/autoscaling/snapshotter.js b/src/autoscaling/snapshotter.jsindex 75132cee9..36aa923cc 100644--- a/src/autoscaling/snapshotter.js+++ b/src/autoscaling/snapshotter.js@@ -59,7 +59,7 @@ class Snapshotter {snapshotHistorySecs,maxBlockedMillis,maxUsedMemoryRatio,- } = _.defaults(options, DEFAULT_OPTIONS);+ } = _.defaults({}, options, DEFAULT_OPTIONS);checkParamOrThrow(eventLoopSnapshotIntervalSecs, 'options.eventLoopSnapshotIntervalSecs', 'Number');checkParamOrThrow(memorySnapshotIntervalSecs, 'options.memorySnapshotIntervalSecs', 'Number');diff --git a/src/autoscaling/system_status.js b/src/autoscaling/system_status.jsindex 74c118ac1..a1427013e 100644--- a/src/autoscaling/system_status.js+++ b/src/autoscaling/system_status.js@@ -54,7 +54,7 @@ class SystemStatus {maxEventLoopOverloadedRatio,maxCpuOverloadedRatio,snapshotter,- } = _.defaults(options, DEFAULT_OPTIONS);+ } = _.defaults({}, options, DEFAULT_OPTIONS);checkParamOrThrow(currentHistorySecs, 'options.currentHistorySecs', 'Number');checkParamOrThrow(maxMemoryOverloadedRatio, 'options.maxMemoryOverloadedRatio', 'Number');diff --git a/src/basic_crawler.js b/src/basic_crawler.jsindex 188fcf2f6..4ee61d9bc 100644--- a/src/basic_crawler.js+++ b/src/basic_crawler.js@@ -121,7 +121,7 @@ class BasicCrawler {// AutoscaledPool shorthandsminConcurrency,maxConcurrency,- } = _.defaults(options, DEFAULT_OPTIONS);+ } = _.defaults({}, options, DEFAULT_OPTIONS);checkParamPrototypeOrThrow(requestList, 'options.requestList', RequestList, 'Apify.RequestList', true);checkParamPrototypeOrThrow(requestQueue, 'options.requestQueue', [RequestQueue, RequestQueueLocal], 'Apify.RequestQueue', true);@@ -167,7 +167,7 @@ class BasicCrawler {},};- this.autoscaledPoolOptions = _.defaults(basicCrawlerAutoscaledPoolConfiguration, autoscaledPoolOptions);+ this.autoscaledPoolOptions = _.defaults({}, basicCrawlerAutoscaledPoolConfiguration, autoscaledPoolOptions);}/**diff --git a/src/cheerio_crawler.js b/src/cheerio_crawler.jsindex b25924491..7fe1bfc0b 100644--- a/src/cheerio_crawler.js+++ b/src/cheerio_crawler.js@@ -5,11 +5,11 @@ import log from 'apify-shared/log';import { checkParamOrThrow } from 'apify-client/build/utils';import BasicCrawler from './basic_crawler';import { createTimeoutPromise } from './utils';+import { getApifyProxyUrl } from './actor';const DEFAULT_OPTIONS = {requestTimeoutSecs: 30,handlePageTimeoutSecs: 300,- ignoreSslErrors: false,handleFailedRequestFunction: ({ request }) => {const details = _.pick(request, 'id', 'url', 'method', 'uniqueKey');@@ -22,7 +22,7 @@ const DEFAULT_OPTIONS = {* <a href="https://www.npmjs.com/package/cheerio" target="_blank">cheerio</a> HTML parser.** `CheerioCrawler` downloads each URL using a plain HTTP request,- * parses the HTML content using <a href="https://www.npmjs.com/package/cheerio" target="_blank">cheerio</a>+ * parses the HTML content using <a href="https://www.npmjs.com/package/cheerio" target="_blank">Cheerio</a>* and then invokes the user-provided [`handlePageFunction()`](#new_CheerioCrawler_new) to extract page data* using a <a href="https://jquery.com/" target="_blank">jQuery</a>-like interface to the parsed HTML DOM.*@@ -38,7 +38,8 @@ const DEFAULT_OPTIONS = {** By default, `CheerioCrawler` downloads HTML using the* <a href="https://www.npmjs.com/package/request-promise" target="_blank">request-promise</a> NPM package.- * You can override this behavior by setting the `requestFunction` option.+ * You can override this behavior by setting the `requestFunction` option. If you want to keep `request-promise`,+ * but use different than default options, use the `requestOptions` parameter.** New requests are only dispatched when there is enough free CPU and memory available,* using the functionality provided by the {@link AutoscaledPool} class.@@ -105,8 +106,13 @@ const DEFAULT_OPTIONS = {* Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.* Either {@link RequestList} or {@link RequestQueue} must be provided.* @param {Function} [options.requestFunction]- * Overrides the function that performs the HTTP request to get the raw HTML needed for Cheerio.+ * Overrides the default function that performs the HTTP request to get the raw HTML needed for Cheerio.* See source code on <a href="https://github.com/apifytech/apify-js/blob/master/src/cheerio_crawler.js#L264">GitHub</a> for default behavior.+ * @param {Object} [options.requestOptions]+ * Represents the options passed to the `requestFunction`, which are essentially the options passed to+ * <a href="https://www.npmjs.com/package/request-promise" target="_blank">request-promise</a> to make the HTTP call.+ * Provided `requestOptions` are merged with defaults so if you only need to add a parameter, there's no need to duplicate+ * the whole object.* @param {Number} [options.handlePageTimeoutSecs=300]* Timeout in which the function passed as `options.handlePageFunction` needs to finish, given in seconds.* @param {Number} [options.requestTimeoutSecs=30]@@ -114,6 +120,25 @@ const DEFAULT_OPTIONS = {* @param {Boolean} [options.ignoreSslErrors=false]* If set to true, SSL certificate errors will be ignored. This is dependent on using the default* request function. If using a custom `options.requestFunction`, user needs to implement this functionality.+ * @param {Boolean} [useApifyProxy=false]+ * If set to `true`, `CheerioCrawler` will be configured to use+ * <a href="https://my.apify.com/proxy" target="_blank">Apify Proxy</a> for all connections.+ * For more information, see the <a href="https://www.apify.com/docs/proxy" target="_blank">documentation</a>+ * @param {String[]} [apifyProxyGroups]+ * An array of proxy groups to be used+ * by the <a href="https://www.apify.com/docs/proxy" target="_blank">Apify Proxy</a>.+ * Only applied if the `useApifyProxy` option is `true`.+ * @param {String} [apifyProxySession]+ * Apify Proxy session identifier to be used with requests made by `CheerioCrawler`.+ * All HTTP requests going through the proxy with the same session identifier+ * will use the same target proxy server (i.e. the same IP address).+ * The identifier can only contain the following characters: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.+ * Only applied if the `useApifyProxy` option is `true`.+ * @param {String[]} [options.proxyUrls]+ * An array of custom proxy URLs to be used by the `CheerioCrawler` instance.+ * The provided custom proxies' order will be randomized and the resulting list rotated.+ * Custom proxies are not compatible with Apify Proxy and an attempt to use both+ * configuration options will cause an error to be thrown on startup.* @param {Function} [options.handleFailedRequestFunction]* Function that handles requests that failed more then `option.maxRequestRetries` times.* See source code on <a href="https://github.com/apifytech/apify-js/blob/master/src/cheerio_crawler.js#L13">GitHub</a> for default behavior.@@ -136,10 +161,15 @@ class CheerioCrawler {constructor(options = {}) {const {requestFunction,+ requestOptions,handlePageFunction,requestTimeoutSecs,handlePageTimeoutSecs,ignoreSslErrors,+ useApifyProxy,+ apifyProxyGroups,+ apifyProxySession,+ proxyUrls,// Autoscaled pool shorthandsminConcurrency,@@ -152,15 +182,21 @@ class CheerioCrawler {maxRequestsPerCrawl,handleFailedRequestFunction,autoscaledPoolOptions,- } = _.defaults(options, DEFAULT_OPTIONS);+ } = _.defaults({}, options, DEFAULT_OPTIONS);checkParamOrThrow(handlePageFunction, 'options.handlePageFunction', 'Function');checkParamOrThrow(requestFunction, 'options.requestFunction', 'Maybe Function');+ checkParamOrThrow(requestOptions, 'options.requestOptions', 'Maybe Object');checkParamOrThrow(requestTimeoutSecs, 'options.requestTimeoutSecs', 'Number');checkParamOrThrow(handlePageTimeoutSecs, 'options.handlePageTimeoutSecs', 'Number');- checkParamOrThrow(ignoreSslErrors, 'options.ignoreSslErrors', 'Boolean');-- this.ignoreSslErrors = ignoreSslErrors;+ checkParamOrThrow(ignoreSslErrors, 'options.ignoreSslErrors', 'Maybe Boolean');+ checkParamOrThrow(useApifyProxy, 'options.useApifyProxy', 'Maybe Boolean');+ checkParamOrThrow(apifyProxyGroups, 'options.apifyProxyGroups', 'Maybe Array');+ checkParamOrThrow(apifyProxySession, 'options.apifyProxySession', 'Maybe String');+ checkParamOrThrow(proxyUrls, 'options.proxyUrls', 'Maybe Array');+ // Enforce valid proxy configuration+ if (proxyUrls && !proxyUrls.length) throw new Error('Parameter "options.proxyUrls" of type Array must not be empty');+ if (useApifyProxy && proxyUrls) throw new Error('Cannot combine "options.useApifyProxy" with "options.proxyUrls"!');this.requestFunction = async ({ request }) => {if (!this.isRunning) throw new Error('CheerioCrawler is stopped.');@@ -174,9 +210,16 @@ class CheerioCrawler {this.rejectOnAbortPromise,]);};+ this.requestOptions = requestOptions;this.handlePageFunction = handlePageFunction;this.handlePageTimeoutMillis = handlePageTimeoutSecs * 1000;this.requestTimeoutMillis = requestTimeoutSecs * 1000;+ this.ignoreSslErrors = ignoreSslErrors;+ this.useApifyProxy = useApifyProxy;+ this.apifyProxyGroups = apifyProxyGroups;+ this.apifyProxySession = apifyProxySession;+ this.proxyUrls = _.shuffle(proxyUrls);+ this.lastUsedProxyUrlIndex = 0;this.basicCrawler = new BasicCrawler({// Basic crawler options.@@ -260,14 +303,45 @@ class CheerioCrawler {* @ignore*/async _defaultRequestFunction({ request }) {- return rp({+ return rp(this._getRequestOptions(request));+ }++ /**+ * Combines the provided `requestOptions` with default values.+ * @param {Request} request+ * @ignore+ */+ _getRequestOptions(request) {+ const defaultRequestOptions = {url: request.url,method: request.method,headers: request.headers,strictSSL: !this.ignoreSslErrors,resolveWithFullResponse: true,simple: false,- });+ proxy: this._getProxyUrl(),+ };+ return _.defaults({}, this.requestOptions, defaultRequestOptions);+ }++ /**+ * Enables the use of a proxy by returning a proxy URL+ * based on configured options or null if no proxy is used.+ * @ignore+ */+ _getProxyUrl() {+ if (this.useApifyProxy) {+ return getApifyProxyUrl({+ groups: [...this.apifyProxyGroups],+ session: this.apifyProxySession,+ groupsParamName: 'options.apifyProxyGroups',+ sessionParamName: 'options.apifyProxySession',+ });+ }+ if (this.proxyUrls) {+ return this.proxyUrls[this.lastUsedProxyUrlIndex++ % this.proxyUrls.length];+ }+ return null;}}diff --git a/src/puppeteer_crawler.js b/src/puppeteer_crawler.jsindex e8be5268a..df19390c8 100644--- a/src/puppeteer_crawler.js+++ b/src/puppeteer_crawler.js@@ -139,6 +139,11 @@ const PAGE_CLOSE_TIMEOUT_MILLIS = 30000;* the whole browser is closed too. This parameter defines a time limit for inactivity* after which the browser is closed even if there are pending tabs. See* `killInstanceAfterMillis` parameter of {@link PuppeteerPool}.+ * @param {String[]} [options.proxyUrls]+ * An array of custom proxy URLs to be used by the {@link PuppeteerPool} instance.+ * The provided custom proxies' order will be randomized and the resulting list rotated.+ * Custom proxies are not compatible with Apify Proxy and an attempt to use both+ * configuration options will cause an error to be thrown on startup.* @param {Function} [options.launchPuppeteerFunction]* Overrides the default function to launch a new Puppeteer instance.* See `launchPuppeteerFunction` parameter of {@link PuppeteerPool}.@@ -189,9 +194,10 @@ class PuppeteerCrawler {retireInstanceAfterRequestCount,instanceKillerIntervalMillis,killInstanceAfterMillis,+ proxyUrls,launchPuppeteerFunction,launchPuppeteerOptions,- } = _.defaults(options, DEFAULT_OPTIONS);+ } = _.defaults({}, options, DEFAULT_OPTIONS);checkParamOrThrow(handlePageFunction, 'options.handlePageFunction', 'Function');checkParamOrThrow(handleFailedRequestFunction, 'options.handleFailedRequestFunction', 'Maybe Function');@@ -208,6 +214,7 @@ class PuppeteerCrawler {retireInstanceAfterRequestCount,instanceKillerIntervalMillis,killInstanceAfterMillis,+ proxyUrls,launchPuppeteerFunction,launchPuppeteerOptions,};diff --git a/src/puppeteer_pool.js b/src/puppeteer_pool.jsindex 1ca78b0b3..9934f0027 100644--- a/src/puppeteer_pool.js+++ b/src/puppeteer_pool.js@@ -137,9 +137,10 @@ class PuppeteerInstance {** The `options.recycleDiskCache` setting should not be used together with `--disk-cache-dir` argument in `options.launchPuppeteerOptions.args`.* @param {String[]} [options.proxyUrls]- * An array of custom proxy URLs to be used by the `PuppeteerPool` instance. Custom proxies are not compatible- * with Apify Proxy. Therefore, by using the `proxyUrls` parameter, all Apify Proxy groups will be automatically disabled- * including the `auto` group. The provided custom proxies' order will be randomized and the resulting list rotated.+ * An array of custom proxy URLs to be used by the `PuppeteerPool` instance.+ * The provided custom proxies' order will be randomized and the resulting list rotated.+ * Custom proxies are not compatible with Apify Proxy and an attempt to use both+ * configuration options will cause an error to be thrown on startup.*/class PuppeteerPool {constructor(options = {}) {@@ -160,7 +161,7 @@ class PuppeteerPool {launchPuppeteerOptions,recycleDiskCache,proxyUrls,- } = _.defaults(options, DEFAULT_OPTIONS);+ } = _.defaults({}, options, DEFAULT_OPTIONS);checkParamOrThrow(maxOpenPagesPerInstance, 'options.maxOpenPagesPerInstance', 'Number');checkParamOrThrow(retireInstanceAfterRequestCount, 'options.retireInstanceAfterRequestCount', 'Number');@@ -204,8 +205,7 @@ class PuppeteerPool {// Rotate custom proxyUrls.if (this.proxyUrls) {- opts.proxyUrl = this.proxyUrls[this.lastUsedProxyUrlIndex % this.proxyUrls.length];- this.lastUsedProxyUrlIndex++;+ opts.proxyUrl = this.proxyUrls[this.lastUsedProxyUrlIndex++ % this.proxyUrls.length];}const browser = await launchPuppeteerFunction(opts);diff --git a/test/cheerio_crawler.js b/test/cheerio_crawler.jsindex c422a5b57..7eca2ca1e 100644--- a/test/cheerio_crawler.js+++ b/test/cheerio_crawler.js@@ -3,7 +3,8 @@ import chaiAsPromised from 'chai-as-promised';import log from 'apify-shared/log';import _ from 'underscore';import { delayPromise } from 'apify-shared/utilities';-import * as Apify from '../build/index';+import { ENV_VARS } from 'apify-shared/consts';+import Apify from '../build/index';chai.use(chaiAsPromised);@@ -180,7 +181,7 @@ describe('CheerioCrawler', () => {log.setLevel(ll);});- it('should timeout after requestTimeoutSecs', async () => {+ it('after requestTimeoutSecs', async () => {const sources = [{ url: 'http://example.com/?q=0' },{ url: 'http://example.com/?q=1' },@@ -221,7 +222,7 @@ describe('CheerioCrawler', () => {});});- it('should timeout after handlePageTimeoutSecs', async () => {+ it('after handlePageTimeoutSecs', async () => {const sources = [{ url: 'http://example.com/?q=0' },{ url: 'http://example.com/?q=1' },@@ -258,4 +259,123 @@ describe('CheerioCrawler', () => {});});});++ describe('proxy', () => {+ let requestList;+ beforeEach(async () => {+ requestList = new Apify.RequestList({+ sources: [+ { url: 'http://example.com/?q=0' },+ { url: 'http://example.com/?q=1' },+ { url: 'http://example.com/?q=2' },+ { url: 'http://example.com/?q=3' },+ ],+ });+ await requestList.initialize();+ });++ afterEach(() => {+ requestList = null;+ });++ /* eslint-disable no-underscore-dangle */+ it('should work with proxyUrls array', async () => {+ const proxies = [];+ const crawler = new Apify.CheerioCrawler({+ requestList,+ handlePageFunction: async () => {},+ requestFunction: async ({ request }) => {+ const opts = crawler._getRequestOptions(request);+ proxies.push(opts.proxy);+ // it needs to return something valid+ return 'html';+ },+ proxyUrls: ['http://proxy.com:1111', 'http://proxy.com:2222', 'http://proxy.com:3333'],+ });++ const shuffled = crawler.proxyUrls;+ await crawler.run();++ expect(proxies).to.have.lengthOf(4);+ expect(proxies[0]).to.be.eql(shuffled[0]);+ expect(proxies[1]).to.be.eql(shuffled[1]);+ expect(proxies[2]).to.be.eql(shuffled[2]);+ expect(proxies[3]).to.be.eql(shuffled[0]);+ });++ it('should work with useApifyProxy', async () => {+ process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';+ const proxies = [];+ const useApifyProxy = true;+ const apifyProxyGroups = ['GROUP1', 'GROUP2'];+ const apifyProxySession = 'session';++ const proxy = Apify.getApifyProxyUrl({+ groups: apifyProxyGroups,+ session: apifyProxySession,+ });++ const crawler = new Apify.CheerioCrawler({+ requestList,+ handlePageFunction: async () => {},+ requestFunction: async ({ request }) => {+ const opts = crawler._getRequestOptions(request);+ proxies.push(opts.proxy);+ // it needs to return something valid+ return 'html';+ },+ useApifyProxy,+ apifyProxyGroups,+ apifyProxySession,+ });++ await crawler.run();+ delete process.env[ENV_VARS.PROXY_PASSWORD];++ // expect(proxies).to.have.lengthOf(1);+ expect(proxies[0]).to.be.eql(proxy);+ expect(proxies[1]).to.be.eql(proxy);+ expect(proxies[2]).to.be.eql(proxy);+ expect(proxies[3]).to.be.eql(proxy);+ });++ describe('throws', () => {+ /* eslint-disable no-new */+ beforeEach(() => {+ log.setLevel(log.LEVELS.OFF);+ });+ afterEach(async () => {+ log.setLevel(log.LEVELS.ERROR);+ });++ it('when proxyUrls is used together with useApifyProxy', async () => {+ try {+ new Apify.CheerioCrawler({+ requestList,+ handlePageFunction: async () => {},+ requestFunction: async () => {},+ proxyUrls: ['http://proxy.com:1111', 'http://proxy.com:2222', 'http://proxy.com:3333'],+ useApifyProxy: true,+ });+ throw new Error('Invalid error.');+ } catch (err) {+ expect(err.message).to.include('useApifyProxy');+ }+ });++ it('when proxyUrls array is empty', async () => {+ try {+ new Apify.CheerioCrawler({+ requestList,+ handlePageFunction: async () => {},+ requestFunction: async () => {},+ proxyUrls: [],+ });+ throw new Error('Invalid error.');+ } catch (err) {+ expect(err.message).to.include('must not be empty');+ }+ });+ });+ });});