Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Basic Crawler: add error to handleFailedRequestFunction (#50)

Jakub Drobník0d9615006232
diff --git a/src/basic_crawler.js b/src/basic_crawler.js
index 3f6ce4af0..1baebc036 100644
--- a/src/basic_crawler.js
+++ b/src/basic_crawler.js
@@ -39,7 +39,7 @@ const DEFAULT_OPTIONS = {
* @param {Object} options
* @param {RequestList} options.requestList List of the requests to be processed.
* @param {Function} options.handleRequestFunction Function that processes a request. It must return a promise.
- * @param {Function} [options.handleFailedRequestFunction=({ request }) => log.error('Request failed', _.pick(request, 'url', 'uniqueKey'))`]
+ * @param {Function} [options.handleFailedRequestFunction=({ request, error }) => log.error('Request failed', _.pick(request, 'url', 'uniqueKey'))`]
* Function to handle requests that failed more then option.maxRequestRetries times.
* @param {Number} [options.maxRequestRetries=3] How many times request is retried if handleRequestFunction failed.
* @param {Number} [options.maxMemoryMbytes] Maximal memory available in the system (see `maxMemoryMbytes` parameter of `Apify.AutoscaledPool`).
@@ -108,8 +108,8 @@ export default class BasicCrawler {
.then(() => {
this.requestList.markRequestHandled(request);
})
- .catch((err) => {
- request.pushErrorMessage(err);
+ .catch((error) => {
+ request.pushErrorMessage(error);
// Retry request.
if (request.retryCount < this.maxRequestRetries) {
@@ -121,7 +121,7 @@ export default class BasicCrawler {
// Mark as failed.
this.requestList.markRequestHandled(request);
- return this.handleFailedRequestFunction({ request });
+ return this.handleFailedRequestFunction({ request, error });
});
}
}
diff --git a/test/basic_crawler.js b/test/basic_crawler.js
index 12511ec0f..35321d496 100644
--- a/test/basic_crawler.js
+++ b/test/basic_crawler.js
@@ -89,6 +89,7 @@ describe('BasicCrawler', () => {
];
const processed = {};
const failed = {};
+ const errors = [];
const requestList = new Apify.RequestList({ sources });
const handleRequestFunction = async ({ request }) => {
@@ -96,8 +97,9 @@ describe('BasicCrawler', () => {
processed[request.url] = request;
};
- const handleFailedRequestFunction = async ({ request }) => {
+ const handleFailedRequestFunction = async ({ request, error }) => {
failed[request.url] = request;
+ errors.push(error);
};
const basicCrawler = new Apify.BasicCrawler({
@@ -119,5 +121,6 @@ describe('BasicCrawler', () => {
expect(_.values(processed)).to.have.length.of(0);
expect(requestList.isFinished()).to.be.eql(true);
expect(requestList.isEmpty()).to.be.eql(true);
+ errors.forEach(error => expect(error).to.be.an('error'));
});
});
diff --git a/test/puppeteer_crawler.js b/test/puppeteer_crawler.js
index a90994408..81eeddecb 100644
--- a/test/puppeteer_crawler.js
+++ b/test/puppeteer_crawler.js
@@ -65,6 +65,8 @@ describe('PuppeteerCrawler', () => {
{ url: 'http://example.com/?q=1' },
];
const failed = [];
+ const errors = [];
+ const expectErrorMsgString = 'timed out';
const requestList = new Apify.RequestList({ sources });
const handlePageFunction = async () => {
await delayPromise(1000);
@@ -73,7 +75,10 @@ describe('PuppeteerCrawler', () => {
const puppeteerCrawler = new Apify.PuppeteerCrawler({
requestList,
handlePageFunction,
- handleFailedRequestFunction: ({ request }) => failed.push(request),
+ handleFailedRequestFunction: ({ request, error }) => {
+ failed.push(request);
+ errors.push(error);
+ },
disableProxy: true,
pageOpsTimeoutMillis: 900,
});
@@ -86,7 +91,11 @@ describe('PuppeteerCrawler', () => {
expect(failed[0].errorMessages).to.have.lengthOf(4);
failed[0].errorMessages.forEach((error) => {
expect(error).to.be.a('string');
- expect(error).to.include('timed out');
+ expect(error).to.include(expectErrorMsgString);
+ });
+ errors.forEach((error) => {
+ expect(error).to.be.an('error');
+ expect(error.message).to.include(expectErrorMsgString);
});
});
});