Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Rename request.retry to noRetry
Ondra Urban322f826eba5b
diff --git a/src/basic_crawler.js b/src/basic_crawler.jsindex 73dc562f2..86814e87e 100644--- a/src/basic_crawler.js+++ b/src/basic_crawler.js@@ -301,7 +301,7 @@ class BasicCrawler {request.pushErrorMessage(error);// Reclaim and retry request if flagged as retriable and retryCount is not exceeded.- if (request.retry && request.retryCount < this.maxRequestRetries) {+ if (!request.noRetry && request.retryCount < this.maxRequestRetries) {request.retryCount++;log.exception(error, 'BasicCrawler: handleRequestFunction failed, reclaiming failed request back to the list or queue', { // eslint-disable-line max-lenurl: request.url,diff --git a/src/request.js b/src/request.jsindex 3944df6a6..9a2bf4f52 100644--- a/src/request.js+++ b/src/request.js@@ -102,7 +102,7 @@ class Request {uniqueKey,method = 'GET',payload = null,- retry = true,+ noRetry = false,retryCount = 0,errorMessages = null,headers = {},@@ -117,7 +117,7 @@ class Request {checkParamOrThrow(uniqueKey, 'uniqueKey', 'Maybe String');checkParamOrThrow(method, 'method', 'String');checkParamOrThrow(payload, 'payload', 'Maybe Buffer | String');- checkParamOrThrow(retry, 'retry', 'Boolean');+ checkParamOrThrow(noRetry, 'retry', 'Boolean');checkParamOrThrow(retryCount, 'retryCount', 'Number');checkParamOrThrow(errorMessages, 'errorMessages', 'Maybe Array');checkParamOrThrow(headers, 'headers', 'Object');@@ -134,7 +134,7 @@ class Request {this.uniqueKey = uniqueKey || computeUniqueKey(url, keepUrlFragment) || url;this.method = method;this.payload = payload;- this.retry = retry;+ this.noRetry = noRetry;this.retryCount = retryCount;this.errorMessages = errorMessages;this.headers = headers;@@ -200,7 +200,7 @@ class Request {* @param {String} [message]*/doNotRetry(message) {- this.retry = false;+ this.noRetry = true;if (message) throw new Error(message);}}diff --git a/test/basic_crawler.js b/test/basic_crawler.jsindex 9c7ed2c6c..75aed4f0e 100644--- a/test/basic_crawler.js+++ b/test/basic_crawler.js@@ -136,11 +136,19 @@ describe('BasicCrawler', () => {expect(await requestList.isEmpty()).to.be.eql(true);});- it('should not retry requests with retry set to false ', async () => {+ it('should not retry requests with noRetry set to true ', async () => {+ const noRetryRequest = new Apify.Request({ url: 'http://example.com/3' });+ try {+ noRetryRequest.doNotRetry('no retry');+ throw new Error('wrong error');+ } catch (err) {+ expect(err.message).to.be.eql('no retry');+ }+const sources = [- { url: 'http://example.com/1', retry: false },+ { url: 'http://example.com/1', noRetry: true },{ url: 'http://example.com/2' },- { url: 'http://example.com/3', retry: false },+ noRetryRequest,];const processed = {};const requestList = new Apify.RequestList({ sources });diff --git a/test/puppeteer_utils.js b/test/puppeteer_utils.jsindex dff3ff833..f8ff0af10 100644--- a/test/puppeteer_utils.js+++ b/test/puppeteer_utils.js@@ -10,6 +10,15 @@ chai.use(chaiAsPromised);/* global process, describe, it */describe('Apify.utils.puppeteer', () => {+ let ll;+ before(() => {+ ll = log.getLevel();+ log.setLevel(log.LEVELS.ERROR);+ });+ after(() => {+ log.setLevel(ll);+ });+it('injectJQuery()', async () => {const browser = await Apify.launchPuppeteer({ headless: true });