Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
initial commit
Ondra Urban8dbafe22198e
diff --git a/src/basic_crawler.js b/src/basic_crawler.jsindex aaef02c83..b24f44063 100644--- a/src/basic_crawler.js+++ b/src/basic_crawler.js@@ -144,14 +144,18 @@ export default class BasicCrawler {maxConcurrency,minConcurrency,minFreeMemoryRatio,- runTaskFunction: () => this._runTaskFunction(),+ runTaskFunction: () => {+ if (this.isStopped) return null;++ return this._runTaskFunction();+ },isTaskReadyFunction: () => {- if (isMaxPagesExceeded()) return Promise.resolve(false);+ if (isMaxPagesExceeded() || this.isStopped) return Promise.resolve(false);return this._isTaskReadyFunction();},isFinishedFunction: () => {- if (isMaxPagesExceeded()) return Promise.resolve(true);+ if (isMaxPagesExceeded() || this.isStopped) return Promise.resolve(true);return isFinishedFunction? isFinishedFunction()@@ -167,8 +171,15 @@ export default class BasicCrawler {* @return {Promise}*/run() {+ this.isStopped = false;return this.autoscaledPool.run();}+ /**+ * Stops the crawler by preventing crawls of additional pages. Pages already running are not terminated.+ */+ stop() {+ this.isStopped = true;+ }/*** Fetches request from either RequestList or RequestQueue. If request comes from a RequestList