Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Adding requestId to logs
Marek Trunkatc41b4acb6042
diff --git a/src/basic_crawler.js b/src/basic_crawler.jsindex ef54f60d1..7618c80eb 100644--- a/src/basic_crawler.js+++ b/src/basic_crawler.js@@ -383,10 +383,11 @@ class BasicCrawler {// Reclaim and retry request if flagged as retriable and retryCount is not exceeded.if (!request.noRetry && request.retryCount < this.maxRequestRetries) {request.retryCount++;- log.exception(error, 'BasicCrawler: handleRequestFunction failed, reclaiming failed request back to the list or queue', { // eslint-disable-line max-len- url: request.url,- retryCount: request.retryCount,- });+ log.exception(+ error,+ 'BasicCrawler: handleRequestFunction failed, reclaiming failed request back to the list or queue',+ _.pick(request, 'url', 'retryCount', 'id'),+ );return source.reclaimRequest(request);}