Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Feature/ignore error (#97)

Marek Trunkátd8187df50d23
diff --git a/src/basic_crawler.js b/src/basic_crawler.js
index b422e1c84..2b4f6fb06 100644
--- a/src/basic_crawler.js
+++ b/src/basic_crawler.js
@@ -184,6 +184,15 @@ export default class BasicCrawler {
return handlePromise
.then(() => source.markRequestHandled(request))
.catch((error) => {
+ if (request.ignoreErrors) {
+ log.exception(error, 'BasicCrawler: handleRequestFunction failed, request.ignoreErrors=true so marking request handled', {
+ url: request.url,
+ retryCount: request.retryCount,
+ });
+
+ return source.markRequestHandled(request);
+ }
+
request.pushErrorMessage(error);
// Retry request.
diff --git a/src/request.js b/src/request.js
index 75093edc7..d8a4f99fb 100644
--- a/src/request.js
+++ b/src/request.js
@@ -35,6 +35,8 @@ export const computeUniqueKey = (url, keepUrlFragment) => normalizeUrl(url, keep
* @param {String} [opts.headers={}] HTTP headers.
* @param {Object} [opts.userData={}] Custom data that user can assign to request.
* @param {Boolean} [opts.keepUrlFragment=false] If false then hash part is removed from url when computing `uniqueKey`.
+ * @param {String} [opts.ignoreErrors=false] If set to `true` then errors in processing of this will be ignored (request won't be
+ * retried in a case of an error for example).
*/
export default class Request {
constructor(opts = {}) {
@@ -51,6 +53,7 @@ export default class Request {
headers = {},
userData = {},
keepUrlFragment = false,
+ ignoreErrors = false,
} = opts;
@@ -63,6 +66,7 @@ export default class Request {
checkParamOrThrow(errorMessages, 'errorMessages', 'Maybe Array');
checkParamOrThrow(headers, 'headers', 'Object');
checkParamOrThrow(userData, 'userData', 'Object');
+ checkParamOrThrow(ignoreErrors, 'ignoreErrors', 'Boolean');
if (method === 'GET' && payload) throw new Error('Request with GET method cannot have a payload.');
@@ -75,6 +79,7 @@ export default class Request {
this.errorMessages = errorMessages;
this.headers = headers;
this.userData = userData;
+ this.ignoreErrors = ignoreErrors;
}
/**
diff --git a/test/basic_crawler.js b/test/basic_crawler.js
index 15ad1ac5d..9c79746af 100644
--- a/test/basic_crawler.js
+++ b/test/basic_crawler.js
@@ -11,7 +11,7 @@ import { LOCAL_EMULATION_DIR } from './_helper';
chai.use(chaiAsPromised);
describe('BasicCrawler', () => {
- it('runs in parallel thru all the requests', async () => {
+ it('should run in parallel thru all the requests', async () => {
const startedAt = Date.now();
const sources = _.range(0, 500).map(index => ({ url: `https://example.com/${index}` }));
@@ -38,7 +38,7 @@ describe('BasicCrawler', () => {
expect(await requestList.isEmpty()).to.be.eql(true);
});
- it('retries failed requests', async () => {
+ it('should retry failed requests', async () => {
const sources = [
{ url: 'http://example.com/1' },
{ url: 'http://example.com/2' },
@@ -84,6 +84,56 @@ describe('BasicCrawler', () => {
expect(await requestList.isEmpty()).to.be.eql(true);
});
+ it('should not retry requests with ignoreErrors set to true ', async () => {
+ const sources = [
+ { url: 'http://example.com/1', ignoreErrors: true },
+ { url: 'http://example.com/2' },
+ { url: 'http://example.com/3', ignoreErrors: true },
+ ];
+ const processed = {};
+ const requestList = new Apify.RequestList({ sources });
+
+ const handleRequestFunction = async ({ request }) => {
+ await delayPromise(10);
+ processed[request.url] = request;
+ request.userData.foo = 'bar';
+ throw Error(`This is ${request.retryCount}th error!`);
+ };
+
+ let handleFailedRequestFunctionCalls = 0;
+ const handleFailedRequestFunction = () => {
+ handleFailedRequestFunctionCalls++;
+ };
+
+ const basicCrawler = new Apify.BasicCrawler({
+ requestList,
+ maxRequestRetries: 10,
+ minConcurrency: 3,
+ maxConcurrency: 3,
+ handleRequestFunction,
+ handleFailedRequestFunction,
+ });
+
+ await requestList.initialize();
+ await basicCrawler.run();
+
+ expect(processed['http://example.com/1'].userData.foo).to.be.eql('bar');
+ expect(processed['http://example.com/1'].errorMessages).to.be.a('null');
+ expect(processed['http://example.com/1'].retryCount).to.be.eql(0);
+ expect(processed['http://example.com/3'].userData.foo).to.be.eql('bar');
+ expect(processed['http://example.com/3'].errorMessages).to.be.a('null');
+ expect(processed['http://example.com/3'].retryCount).to.be.eql(0);
+
+ expect(processed['http://example.com/2'].userData.foo).to.be.eql('bar');
+ expect(processed['http://example.com/2'].errorMessages).to.have.lengthOf(11);
+ expect(processed['http://example.com/2'].retryCount).to.be.eql(10);
+
+ expect(handleFailedRequestFunctionCalls).to.be.eql(1);
+
+ expect(await requestList.isFinished()).to.be.eql(true);
+ expect(await requestList.isEmpty()).to.be.eql(true);
+ });
+
it('should allow to handle failed requests', async () => {
const sources = [
{ url: 'http://example.com/1' },