Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Changing maxEventLoopOverloadedRatio to 20%

Marek Trunkata4b300e433f0
diff --git a/src/autoscaling/system_status.js b/src/autoscaling/system_status.js
index 85f23f123..1af5c09d5 100644
--- a/src/autoscaling/system_status.js
+++ b/src/autoscaling/system_status.js
@@ -6,7 +6,7 @@ import { weightedAvg } from '../utils';
const DEFAULT_OPTIONS = {
currentHistorySecs: 5, // TODO this should be something like "nowDurationSecs" but it's weird, ideas?
maxMemoryOverloadedRatio: 0.2,
- maxEventLoopOverloadedRatio: 0.02,
+ maxEventLoopOverloadedRatio: 0.2,
maxCpuOverloadedRatio: 0.1,
};
diff --git a/src/basic_crawler.js b/src/basic_crawler.js
index 619679ad5..d0d4bc3c8 100644
--- a/src/basic_crawler.js
+++ b/src/basic_crawler.js
@@ -319,14 +319,9 @@ export default class BasicCrawler {
}
// This is the final fallback. If we get here, the request failed more than retryCount times and will not be retried anymore.
- log.exception(error, 'BasicCrawler: handleRequestFunction failed, marking failed request as handled', {
- url: request.url,
- retryCount: request.retryCount,
- });
-
// Mark the request as failed and do not retry.
this.handledRequestsCount++;
await source.markRequestHandled(request);
- return this.handleFailedRequestFunction({ request, error });
+ return this.handleFailedRequestFunction({ request, error }); // This function prints an error message.
}
}
diff --git a/src/puppeteer_crawler.js b/src/puppeteer_crawler.js
index 539a687b3..ce5956ebc 100644
--- a/src/puppeteer_crawler.js
+++ b/src/puppeteer_crawler.js
@@ -10,7 +10,6 @@ const DEFAULT_OPTIONS = {
handlePageTimeoutSecs: 300,
handleFailedRequestFunction: ({ request }) => {
const details = _.pick(request, 'id', 'url', 'method', 'uniqueKey');
-
log.error('PuppeteerCrawler: Request failed and reached maximum retries', details);
},
};