Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

initial commit

Ondra Urban8dbafe22198e
diff --git a/src/basic_crawler.js b/src/basic_crawler.js
index aaef02c83..b24f44063 100644
--- a/src/basic_crawler.js
+++ b/src/basic_crawler.js
@@ -144,14 +144,18 @@ export default class BasicCrawler {
maxConcurrency,
minConcurrency,
minFreeMemoryRatio,
- runTaskFunction: () => this._runTaskFunction(),
+ runTaskFunction: () => {
+ if (this.isStopped) return null;
+
+ return this._runTaskFunction();
+ },
isTaskReadyFunction: () => {
- if (isMaxPagesExceeded()) return Promise.resolve(false);
+ if (isMaxPagesExceeded() || this.isStopped) return Promise.resolve(false);
return this._isTaskReadyFunction();
},
isFinishedFunction: () => {
- if (isMaxPagesExceeded()) return Promise.resolve(true);
+ if (isMaxPagesExceeded() || this.isStopped) return Promise.resolve(true);
return isFinishedFunction
? isFinishedFunction()
@@ -167,8 +171,15 @@ export default class BasicCrawler {
* @return {Promise}
*/
run() {
+ this.isStopped = false;
return this.autoscaledPool.run();
}
+ /**
+ * Stops the crawler by preventing crawls of additional pages. Pages already running are not terminated.
+ */
+ stop() {
+ this.isStopped = true;
+ }
/**
* Fetches request from either RequestList or RequestQueue. If request comes from a RequestList