Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix(basic-crawler): handle `request.noRetry` after `errorHandler` (#1542)
Matt Stephens2a2040e13209
diff --git a/package-lock.json b/package-lock.jsonindex 55967fb3c..e8272abd2 100644--- a/package-lock.json+++ b/package-lock.json@@ -17815,7 +17815,7 @@"@types/tough-cookie": "^4.0.2","@vladfrangu/async_event_emitter": "^2.0.0","fs-extra": "^10.1.0",- "json5": "*",+ "json5": "^2.2.1","minimatch": "^5.1.0","ow": "^0.28.1","stream-chain": "^2.2.5",diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex a78027710..cc27e608a 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -982,35 +982,38 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextconst shouldRetryRequest = this._canRequestBeRetried(request, error);if (shouldRetryRequest) {- request.retryCount++;-this.stats.errorTrackerRetry.add(error);await this._tagUserHandlerError(() => this.errorHandler?.(this._augmentContextWithDeprecatedError(crawlingContext, error), error));- const { url, retryCount, id } = request;-- // We don't want to see the stack trace in the logs by default, when we are going to retry the request.- // Thus, we print the full stack trace only when CRAWLEE_VERBOSE_LOG environment variable is set to true.- const message = this._getMessageFromError(error);- this.log.warning(- `Reclaiming failed request back to the list or queue. ${message}`,- { id, url, retryCount },- );+ if (!request.noRetry) {+ request.retryCount++;- await source.reclaimRequest(request);- } else {- this.stats.errorTracker.add(error);+ const { url, retryCount, id } = request;- // If we get here, the request is either not retryable- // or failed more than retryCount times and will not be retried anymore.- // Mark the request as failed and do not retry.- this.handledRequestsCount++;- await source.markRequestHandled(request);- this.stats.failJob(request.id || request.uniqueKey);+ // We don't want to see the stack trace in the logs by default, when we are going to retry the request.+ // Thus, we print the full stack trace only when CRAWLEE_VERBOSE_LOG environment variable is set to true.+ const message = this._getMessageFromError(error);+ this.log.warning(+ `Reclaiming failed request back to the list or queue. ${message}`,+ { id, url, retryCount },+ );- await this._handleFailedRequestHandler(crawlingContext, error); // This function prints an error message.+ await source.reclaimRequest(request);+ return;+ }}++ this.stats.errorTracker.add(error);++ // If we get here, the request is either not retryable+ // or failed more than retryCount times and will not be retried anymore.+ // Mark the request as failed and do not retry.+ this.handledRequestsCount++;+ await source.markRequestHandled(request);+ this.stats.failJob(request.id || request.uniqueKey);++ await this._handleFailedRequestHandler(crawlingContext, error); // This function prints an error message.}protected async _tagUserHandlerError<T>(cb: () => unknown): Promise<T> {diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.tsindex 3f6d90e45..bf475c333 100644--- a/test/core/crawlers/basic_crawler.test.ts+++ b/test/core/crawlers/basic_crawler.test.ts@@ -390,6 +390,29 @@ describe('BasicCrawler', () => {errors.forEach((error) => expect(error).toBeInstanceOf(NonRetryableError));});+ test('noRetry after calling errorHandler', async () => {+ const sources = [{ url: `http://example.com` }];+ const requestList = await RequestList.open(null, sources);++ let request: Request<Dictionary<any>>;++ const crawler = new BasicCrawler({+ requestList,+ errorHandler: (context, error) => {+ request = context.request;+ context.request.noRetry = true;+ },+ maxRequestRetries: 3,+ requestHandler: () => {+ throw new Error('Failure');+ },+ });++ await crawler.run();++ expect(request.retryCount).toBe(0);+ });+test('should crash on CriticalError', async () => {const sources = [{ url: 'http://example.com/1' },