Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Implement comments

Ondra Urban3155f5e8b458
diff --git a/src/basic_crawler.js b/src/basic_crawler.js
index 2f779138b..91cd4eeee 100644
--- a/src/basic_crawler.js
+++ b/src/basic_crawler.js
@@ -168,7 +168,6 @@ class BasicCrawler {
const isMaxPagesExceeded = () => maxRequestsPerCrawl && maxRequestsPerCrawl <= this.handledRequestsCount;
const { isFinishedFunction } = autoscaledPoolOptions;
- this.isFinishedFunction = isFinishedFunction || this._defaultIsFinishedFunction;
const basicCrawlerAutoscaledPoolConfiguration = {
minConcurrency,
@@ -191,14 +190,17 @@ class BasicCrawler {
return true;
}
- const isFinished = await this.isFinishedFunction();
+ const isFinished = isFinishedFunction
+ ? isFinishedFunction()
+ : this._defaultIsFinishedFunction();
+
if (isFinished) {
- if (isFinishedFunction) {
- log.info('BasicCrawler: Crawler\'s custom isFinishedFunction() returned true, the Crawler will shut down.');
- } else {
- log.info('BasicCrawler: All the Crawler\'s storages are finished (empty and processed), the Crawler will shut down.');
- }
+ const reason = isFinishedFunction
+ ? 'BasicCrawler: Crawler\'s custom isFinishedFunction() returned true, the Crawler will shut down.'
+ : 'BasicCrawler: All the requests from request list and/or request queue have been processed, the Crawler will shut down.';
+ log.info(reason);
}
+
return isFinished;
},
};