Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Rename request.retry to noRetry

Ondra Urban322f826eba5b
diff --git a/src/basic_crawler.js b/src/basic_crawler.js
index 73dc562f2..86814e87e 100644
--- a/src/basic_crawler.js
+++ b/src/basic_crawler.js
@@ -301,7 +301,7 @@ class BasicCrawler {
request.pushErrorMessage(error);
// Reclaim and retry request if flagged as retriable and retryCount is not exceeded.
- if (request.retry && request.retryCount < this.maxRequestRetries) {
+ if (!request.noRetry && request.retryCount < this.maxRequestRetries) {
request.retryCount++;
log.exception(error, 'BasicCrawler: handleRequestFunction failed, reclaiming failed request back to the list or queue', { // eslint-disable-line max-len
url: request.url,
diff --git a/src/request.js b/src/request.js
index 3944df6a6..9a2bf4f52 100644
--- a/src/request.js
+++ b/src/request.js
@@ -102,7 +102,7 @@ class Request {
uniqueKey,
method = 'GET',
payload = null,
- retry = true,
+ noRetry = false,
retryCount = 0,
errorMessages = null,
headers = {},
@@ -117,7 +117,7 @@ class Request {
checkParamOrThrow(uniqueKey, 'uniqueKey', 'Maybe String');
checkParamOrThrow(method, 'method', 'String');
checkParamOrThrow(payload, 'payload', 'Maybe Buffer | String');
- checkParamOrThrow(retry, 'retry', 'Boolean');
+ checkParamOrThrow(noRetry, 'retry', 'Boolean');
checkParamOrThrow(retryCount, 'retryCount', 'Number');
checkParamOrThrow(errorMessages, 'errorMessages', 'Maybe Array');
checkParamOrThrow(headers, 'headers', 'Object');
@@ -134,7 +134,7 @@ class Request {
this.uniqueKey = uniqueKey || computeUniqueKey(url, keepUrlFragment) || url;
this.method = method;
this.payload = payload;
- this.retry = retry;
+ this.noRetry = noRetry;
this.retryCount = retryCount;
this.errorMessages = errorMessages;
this.headers = headers;
@@ -200,7 +200,7 @@ class Request {
* @param {String} [message]
*/
doNotRetry(message) {
- this.retry = false;
+ this.noRetry = true;
if (message) throw new Error(message);
}
}
diff --git a/test/basic_crawler.js b/test/basic_crawler.js
index 9c7ed2c6c..75aed4f0e 100644
--- a/test/basic_crawler.js
+++ b/test/basic_crawler.js
@@ -136,11 +136,19 @@ describe('BasicCrawler', () => {
expect(await requestList.isEmpty()).to.be.eql(true);
});
- it('should not retry requests with retry set to false ', async () => {
+ it('should not retry requests with noRetry set to true ', async () => {
+ const noRetryRequest = new Apify.Request({ url: 'http://example.com/3' });
+ try {
+ noRetryRequest.doNotRetry('no retry');
+ throw new Error('wrong error');
+ } catch (err) {
+ expect(err.message).to.be.eql('no retry');
+ }
+
const sources = [
- { url: 'http://example.com/1', retry: false },
+ { url: 'http://example.com/1', noRetry: true },
{ url: 'http://example.com/2' },
- { url: 'http://example.com/3', retry: false },
+ noRetryRequest,
];
const processed = {};
const requestList = new Apify.RequestList({ sources });
diff --git a/test/puppeteer_utils.js b/test/puppeteer_utils.js
index dff3ff833..f8ff0af10 100644
--- a/test/puppeteer_utils.js
+++ b/test/puppeteer_utils.js
@@ -10,6 +10,15 @@ chai.use(chaiAsPromised);
/* global process, describe, it */
describe('Apify.utils.puppeteer', () => {
+ let ll;
+ before(() => {
+ ll = log.getLevel();
+ log.setLevel(log.LEVELS.ERROR);
+ });
+ after(() => {
+ log.setLevel(ll);
+ });
+
it('injectJQuery()', async () => {
const browser = await Apify.launchPuppeteer({ headless: true });