Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: suppress info message for undefined maxRequestsPerCrawl (#3237)

Rafael Laidlawf3d9a7967da1
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index 2f52f075f..a9eeb1461 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -1144,6 +1144,13 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
this.shouldLogMaxEnqueuedRequestsExceeded = false;
}
+ if (options.reason === 'enqueueLimit') {
+ const enqueuedRequestLimit = this.calculateEnqueuedRequestLimit();
+ if (enqueuedRequestLimit === undefined || enqueuedRequestLimit !== 0) {
+ this.log.info('The number of requests enqueued by the crawler reached the enqueueLinks limit.');
+ }
+ }
+
await this.onSkippedRequest?.(options);
}
diff --git a/packages/core/src/enqueue_links/enqueue_links.ts b/packages/core/src/enqueue_links/enqueue_links.ts
index 0c3dd0256..5d6d2fce0 100644
--- a/packages/core/src/enqueue_links/enqueue_links.ts
+++ b/packages/core/src/enqueue_links/enqueue_links.ts
@@ -490,7 +490,7 @@ export async function enqueueLinks(
let requests = await createFilteredRequests();
if (typeof limit === 'number' && limit < requests.length) {
- await reportSkippedRequests(requests.slice(limit), 'limit');
+ await reportSkippedRequests(requests.slice(limit), 'enqueueLimit');
requests = requests.slice(0, limit);
}
diff --git a/packages/core/src/enqueue_links/shared.ts b/packages/core/src/enqueue_links/shared.ts
index 831e24f7e..eae760313 100644
--- a/packages/core/src/enqueue_links/shared.ts
+++ b/packages/core/src/enqueue_links/shared.ts
@@ -47,7 +47,7 @@ export type RegExpObject = { regexp: RegExp } & Pick<
export type RegExpInput = RegExp | RegExpObject;
-export type SkippedRequestReason = 'robotsTxt' | 'limit' | 'filters' | 'redirect' | 'depth';
+export type SkippedRequestReason = 'robotsTxt' | 'limit' | 'enqueueLimit' | 'filters' | 'redirect' | 'depth';
export type SkippedRequestCallback = (args: { url: string; reason: SkippedRequestReason }) => Awaitable<void>;