Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Improve logging for BasicCrawler error handler exceptions
Ondra Urban30bdc03b1ca2
diff --git a/src/basic_crawler.js b/src/basic_crawler.jsindex 88d012b94..1fce54e16 100644--- a/src/basic_crawler.js+++ b/src/basic_crawler.js@@ -260,7 +260,21 @@ class BasicCrawler {await source.markRequestHandled(request);this.handledRequestsCount++;} catch (err) {- await this._requestFunctionErrorHandler(err, request, source);+ try {+ // TODO Errors thrown from within the error handler will in most cases terminate+ // the crawler because runTaskFunction errors kill autoscaled pool+ // which is correct, since in this case, RequestQueue is probably in an unknown+ // state. However, it's also troublesome when RequestQueue is overloaded+ // since it may actually cause the crawlers to crash.+ await this._requestFunctionErrorHandler(err, request, source);+ } catch (secondaryError) {+ log.exception(secondaryError, 'BasicCrawler: runTaskFunction error handler threw an exception. '+ + 'This places the RequestQueue into an unknown state and crawling will be terminated. '+ + 'This most likely happened due to RequestQueue being overloaded and unable to handle '+ + 'Request updates even after exponential backoff. Try limiting the concurrency '+ + 'of the run by using the maxConcurrency option.');+ throw secondaryError;+ }}}