Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: suppress info message for undefined maxRequestsPerCrawl (#3237)
Rafael Laidlawf3d9a7967da1
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex 2f52f075f..a9eeb1461 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -1144,6 +1144,13 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextthis.shouldLogMaxEnqueuedRequestsExceeded = false;}+ if (options.reason === 'enqueueLimit') {+ const enqueuedRequestLimit = this.calculateEnqueuedRequestLimit();+ if (enqueuedRequestLimit === undefined || enqueuedRequestLimit !== 0) {+ this.log.info('The number of requests enqueued by the crawler reached the enqueueLinks limit.');+ }+ }+await this.onSkippedRequest?.(options);}diff --git a/packages/core/src/enqueue_links/enqueue_links.ts b/packages/core/src/enqueue_links/enqueue_links.tsindex 0c3dd0256..5d6d2fce0 100644--- a/packages/core/src/enqueue_links/enqueue_links.ts+++ b/packages/core/src/enqueue_links/enqueue_links.ts@@ -490,7 +490,7 @@ export async function enqueueLinks(let requests = await createFilteredRequests();if (typeof limit === 'number' && limit < requests.length) {- await reportSkippedRequests(requests.slice(limit), 'limit');+ await reportSkippedRequests(requests.slice(limit), 'enqueueLimit');requests = requests.slice(0, limit);}diff --git a/packages/core/src/enqueue_links/shared.ts b/packages/core/src/enqueue_links/shared.tsindex 831e24f7e..eae760313 100644--- a/packages/core/src/enqueue_links/shared.ts+++ b/packages/core/src/enqueue_links/shared.ts@@ -47,7 +47,7 @@ export type RegExpObject = { regexp: RegExp } & Pick<export type RegExpInput = RegExp | RegExpObject;-export type SkippedRequestReason = 'robotsTxt' | 'limit' | 'filters' | 'redirect' | 'depth';+export type SkippedRequestReason = 'robotsTxt' | 'limit' | 'enqueueLimit' | 'filters' | 'redirect' | 'depth';export type SkippedRequestCallback = (args: { url: string; reason: SkippedRequestReason }) => Awaitable<void>;