Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Added tests. Rename function.
Petr Patekf92c602182e1
diff --git a/src/cheerio_crawler.js b/src/cheerio_crawler.jsindex e1025b382..cd19fde29 100644--- a/src/cheerio_crawler.js+++ b/src/cheerio_crawler.js@@ -29,7 +29,7 @@ const DEFAULT_OPTIONS = {maxEventLoopOverloadedRatio: 0.7,},},- beforeRequestFunction: ({ request }) => request,+ prepareRequestFunction: ({ request }) => request,};/**@@ -226,7 +226,7 @@ class CheerioCrawler {maxRequestsPerCrawl,handleFailedRequestFunction,autoscaledPoolOptions,- beforeRequestFunction,+ prepareRequestFunction,} = _.defaults({}, options, DEFAULT_OPTIONS);checkParamOrThrow(handlePageFunction, 'options.handlePageFunction', 'Function');@@ -238,7 +238,7 @@ class CheerioCrawler {checkParamOrThrow(apifyProxyGroups, 'options.apifyProxyGroups', 'Maybe [String]');checkParamOrThrow(apifyProxySession, 'options.apifyProxySession', 'Maybe String');checkParamOrThrow(proxyUrls, 'options.proxyUrls', 'Maybe [String]');- checkParamOrThrow(handlePageFunction, 'options.beforeRequestFunction', 'Maybe Function');+ checkParamOrThrow(handlePageFunction, 'options.prepareRequestFunction', 'Maybe Function');// Enforce valid proxy configurationif (proxyUrls && !proxyUrls.length) throw new Error('Parameter "options.proxyUrls" of type Array must not be empty');if (useApifyProxy && proxyUrls) throw new Error('Cannot combine "options.useApifyProxy" with "options.proxyUrls"!');@@ -253,7 +253,7 @@ class CheerioCrawler {this.apifyProxySession = apifyProxySession;this.proxyUrls = _.shuffle(proxyUrls);this.lastUsedProxyUrlIndex = 0;- this.beforeRequestFunction = beforeRequestFunction;+ this.prepareRequestFunction = prepareRequestFunction;this.basicCrawler = new BasicCrawler({// Basic crawler options.@@ -296,7 +296,7 @@ class CheerioCrawler {* @ignore*/async _handleRequestFunction({ request, autoscaledPool }) {- const modifiedRequest = await this.beforeRequestFunction({ request });+ const modifiedRequest = await this.prepareRequestFunction({ request });const response = await addTimeoutToPromise(this._requestFunction({ request: modifiedRequest }),this.requestTimeoutMillis,diff --git a/test/cheerio_crawler.js b/test/cheerio_crawler.jsindex 2e9597bf0..e71e1a2d9 100644--- a/test/cheerio_crawler.js+++ b/test/cheerio_crawler.js@@ -79,6 +79,34 @@ describe('CheerioCrawler', () => {});});+ it('should trigger prepareRequestFunction', async () => {+ const MODIFIED_URL = 'http://example.com/?q=2';+ const sources = [+ { url: 'http://example.com/' },++ ];+ let failed = null;+ let success;+ const requestList = new Apify.RequestList({ sources });+ const handlePageFunction = async ({ request }) => { success = request; };+ await requestList.initialize();++ const cheerioCrawler = new Apify.CheerioCrawler({+ requestList,+ handlePageFunction,+ handleFailedRequestFunction: ({ request }) => {+ failed = request;+ },+ prepareRequestFunction: async ({ request }) => {+ request.url = MODIFIED_URL;+ return request;+ },+ });+ await cheerioCrawler.run();+ expect(failed).to.be.eql(null);+ expect(success.url).to.be.eql(MODIFIED_URL);+ });+describe('should timeout', () => {let ll;before(() => {