Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Remove request.ignoreErrors in favor of req.doNotRetry()
Ondra Urban482ac55ac121
diff --git a/src/basic_crawler.js b/src/basic_crawler.jsindex ae57ff2cc..1c964ccd1 100644--- a/src/basic_crawler.js+++ b/src/basic_crawler.js@@ -297,21 +297,11 @@ class BasicCrawler {// All running requests are reclaimed and will be retried.if (!this.isRunning) return source.reclaimRequest(request);- // If we use the ignore errors option, we mark request as handled and do not retry.- if (request.ignoreErrors) {- log.exception(error, 'BasicCrawler: handleRequestFunction failed, request.ignoreErrors=true so marking the request as handled', { // eslint-disable-line max-len- url: request.url,- retryCount: request.retryCount,- });- this.handledRequestsCount++;- return source.markRequestHandled(request);- }-- // If we got here, it means we actually want to handle the error.+ // If we got here, it means we actually want to process the error.request.pushErrorMessage(error);- // Reclaim and retry request if request is not skipped and retryCount is not exceeded.- if (!request.skipped && request.retryCount < this.maxRequestRetries) {+ // Reclaim and retry request if flagged as retriable and retryCount is not exceeded.+ if (request.retry && request.retryCount < this.maxRequestRetries) {request.retryCount++;log.exception(error, 'BasicCrawler: handleRequestFunction failed, reclaiming failed request back to the list or queue', { // eslint-disable-line max-lenurl: request.url,@@ -320,7 +310,8 @@ class BasicCrawler {return source.reclaimRequest(request);}- // This is the final fallback. If we get here, the request failed more than retryCount times and will not be retried anymore.+ // If we get here, the request is either not retriable+ // or failed more than retryCount times and will not be retried anymore.// Mark the request as failed and do not retry.this.handledRequestsCount++;await source.markRequestHandled(request);diff --git a/src/cheerio_crawler.js b/src/cheerio_crawler.jsindex d8dc4a602..f50bbbfed 100644--- a/src/cheerio_crawler.js+++ b/src/cheerio_crawler.js@@ -362,27 +362,28 @@ class CheerioCrawler {// Handle situations where the server explicitly states that// it will not serve the resource as text/html by skipping.if (status === 406) {- request.skip();+ request.doNotRetry();res.destroy();return reject(new Error(`CheerioCrawler: Resource ${request.url} is not available in HTML format. Skipping resource.`));}// Other 200-499 responses are considered OK, but first check the content type.- if (type === 'text/html') {- try {- res.body = await this._readStreamIntoString(res, encoding);- resolve(res);- } catch (err) {- // Error in reading the body.- reject(err);- }- } else {- request.skip();+ if (type !== 'text/html') {+ request.doNotRetry();res.destroy();reject(new Error(`CheerioCrawler: Resource ${request.url} served Content-Type ${type} instead of text/html. Skipping resource.`,));}++ // Content-Type is fine. Read the body and respond.+ try {+ res.body = await this._readStreamIntoString(res, encoding);+ resolve(res);+ } catch (err) {+ // Error in reading the body.+ reject(err);+ }});});}diff --git a/src/request.js b/src/request.jsindex 58363d6f7..3944df6a6 100644--- a/src/request.js+++ b/src/request.js@@ -66,9 +66,6 @@ export const computeUniqueKey = (url, keepUrlFragment) => normalizeUrl(url, keep* For example, this causes the `http://www.example.com#foo` and `http://www.example.com#bar` URLs* to have the same `uniqueKey` of `http://www.example.com` and thus the URLs are considered equal.* Note that this option only has an effect if `uniqueKey` is not set.- * @param {String} [options.ignoreErrors=false]- * If `true` then errors in processing of this request will be ignored.- * For example, the request won't be retried in a case of an error.** @property {String} id* Request ID@@ -81,6 +78,8 @@ export const computeUniqueKey = (url, keepUrlFragment) => normalizeUrl(url, keep* HTTP method, e.g. `GET` or `POST`.* @property {String} payload* HTTP request payload, e.g. for POST requests.+ * @property {Boolean} retry+ * Indicates whether the request will be automatically retried or not.* @property {Number} retryCount* Indicates the number of times the crawling of the request has been retried on error.* @property {String[]} errorMessages@@ -89,9 +88,6 @@ export const computeUniqueKey = (url, keepUrlFragment) => normalizeUrl(url, keep* Object with HTTP headers. Key is header name, value is the value.* @property {Object} userData* Custom user data assigned to the request.- * @property {Boolean} ignoreErrors- * If `true` then errors in processing of this request are ignored.- * For example, the request won't be retried in a case of an error.* @property {Date} handledAt* Indicates the time when the request has been processed.* Is `null` if the request has not been crawled yet.@@ -106,12 +102,12 @@ class Request {uniqueKey,method = 'GET',payload = null,+ retry = true,retryCount = 0,errorMessages = null,headers = {},userData = {},keepUrlFragment = false,- ignoreErrors = false,handledAt = null,} = options;@@ -121,11 +117,11 @@ class Request {checkParamOrThrow(uniqueKey, 'uniqueKey', 'Maybe String');checkParamOrThrow(method, 'method', 'String');checkParamOrThrow(payload, 'payload', 'Maybe Buffer | String');+ checkParamOrThrow(retry, 'retry', 'Boolean');checkParamOrThrow(retryCount, 'retryCount', 'Number');checkParamOrThrow(errorMessages, 'errorMessages', 'Maybe Array');checkParamOrThrow(headers, 'headers', 'Object');checkParamOrThrow(userData, 'userData', 'Object');- checkParamOrThrow(ignoreErrors, 'ignoreErrors', 'Boolean');checkParamOrThrow(handledAt, 'handledAt', 'Maybe String | Date');if (method === 'GET' && payload) throw new Error('Request with GET method cannot have a payload.');@@ -138,13 +134,12 @@ class Request {this.uniqueKey = uniqueKey || computeUniqueKey(url, keepUrlFragment) || url;this.method = method;this.payload = payload;+ this.retry = retry;this.retryCount = retryCount;this.errorMessages = errorMessages;this.headers = headers;this.userData = userData;- this.ignoreErrors = ignoreErrors;this.handledAt = handledAt && new Date(handledAt);- this.skipped = false;}/**@@ -195,7 +190,7 @@ class Request {}/**- * Marks the request as skipped which in turn prevents {@link BasicCrawler}+ * Flags the request with no retry which prevents {@link BasicCrawler}* (and {@PuppeteerCrawler} + {@CheerioCrawler}, since they use {@BasicCrawler} internally)* from retrying the request after an error occurs.*@@ -204,8 +199,8 @@ class Request {** @param {String} [message]*/- skip(message) {- this.skipped = true;+ doNotRetry(message) {+ this.retry = false;if (message) throw new Error(message);}}diff --git a/test/basic_crawler.js b/test/basic_crawler.jsindex 8151e8a46..9c7ed2c6c 100644--- a/test/basic_crawler.js+++ b/test/basic_crawler.js@@ -136,11 +136,11 @@ describe('BasicCrawler', () => {expect(await requestList.isEmpty()).to.be.eql(true);});- it('should not retry requests with ignoreErrors set to true ', async () => {+ it('should not retry requests with retry set to false ', async () => {const sources = [- { url: 'http://example.com/1', ignoreErrors: true },+ { url: 'http://example.com/1', retry: false },{ url: 'http://example.com/2' },- { url: 'http://example.com/3', ignoreErrors: true },+ { url: 'http://example.com/3', retry: false },];const processed = {};const requestList = new Apify.RequestList({ sources });@@ -170,17 +170,17 @@ describe('BasicCrawler', () => {await basicCrawler.run();expect(processed['http://example.com/1'].userData.foo).to.be.eql('bar');- expect(processed['http://example.com/1'].errorMessages).to.be.a('null');+ expect(processed['http://example.com/1'].errorMessages).to.have.lengthOf(1);expect(processed['http://example.com/1'].retryCount).to.be.eql(0);expect(processed['http://example.com/3'].userData.foo).to.be.eql('bar');- expect(processed['http://example.com/3'].errorMessages).to.be.a('null');+ expect(processed['http://example.com/3'].errorMessages).to.have.lengthOf(1);expect(processed['http://example.com/3'].retryCount).to.be.eql(0);expect(processed['http://example.com/2'].userData.foo).to.be.eql('bar');expect(processed['http://example.com/2'].errorMessages).to.have.lengthOf(11);expect(processed['http://example.com/2'].retryCount).to.be.eql(10);- expect(handleFailedRequestFunctionCalls).to.be.eql(1);+ expect(handleFailedRequestFunctionCalls).to.be.eql(3);expect(await requestList.isFinished()).to.be.eql(true);expect(await requestList.isEmpty()).to.be.eql(true);