Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Refactor .stop() to .abort()
Ondra Urbaneccd8aa7f3dc
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 9dd3cefa5..6775a9256 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -3,7 +3,7 @@ xxxxxxxxxxxxxxxxxxx- Added `getInfo()` method to Dataset to get meta-information about a dataset.- Added CheerioCrawler, a specialized class for crawling the web using `cheerio`.- Added `keepDuplicateUrls` option to RequestList to allow duplicate URLs.-- Added `.stop()` method to all Crawler classes to enable stopping the crawl programmatically.+- Added `.abort()` method to all Crawler classes to enable stopping the crawl programmatically.- Deprecated `pageOpsTimeoutMillis` option. Use `handlePageTimeoutSecs`.- Bluebird promises are being phased out of `apify` in favor of `async-await`.- Added `log` to `Apify.utils` to improve logging experience.diff --git a/src/autoscaled_pool.js b/src/autoscaled_pool.jsindex d934bfffd..574078e0f 100644--- a/src/autoscaled_pool.js+++ b/src/autoscaled_pool.js@@ -229,7 +229,7 @@ export default class AutoscaledPool {** @return {Promise}*/- async stop() {+ async abort() {if (this.resolve) this.resolve();// _destroy gets called 10 lines above so no need to do it here}diff --git a/src/basic_crawler.js b/src/basic_crawler.jsindex 654a58245..5657e5a7e 100644--- a/src/basic_crawler.js+++ b/src/basic_crawler.js@@ -187,13 +187,13 @@ export default class BasicCrawler {}/**- * Stops the crawler by preventing crawls of additional pages. Pages already running are NOT terminated.+ * Stops the crawler by preventing additional requests and terminating the running ones.** @return {Promise}*/- async stop() {+ async abort() {this.isRunning = false;- await this.autoscaledPool.stop();+ await this.autoscaledPool.abort();this.rejectOnStop(new Error('BasicCrawler: .stop() function has been called. Stopping the crawler.'));}diff --git a/src/cheerio_crawler.js b/src/cheerio_crawler.jsindex dd83637c0..e598900f9 100644--- a/src/cheerio_crawler.js+++ b/src/cheerio_crawler.js@@ -209,13 +209,13 @@ export default class CheerioCrawler {}/**- * Stops the crawler by preventing crawls of additional pages. Pages already running are terminated.+ * Stops the crawler by preventing crawls of additional pages and terminating the running ones.** @return {Promise}*/- async stop() {+ async abort() {this.isRunning = false;- await this.basicCrawler.stop();+ await this.basicCrawler.abort();this.rejectOnStop(new Error('CheerioCrawler: .stop() function has been called. Stopping the crawler.'));}diff --git a/src/puppeteer_crawler.js b/src/puppeteer_crawler.jsindex 325bb8792..555aef574 100644--- a/src/puppeteer_crawler.js+++ b/src/puppeteer_crawler.js@@ -227,13 +227,13 @@ export default class PuppeteerCrawler {}/**- * Stops the crawler by preventing crawls of additional pages. Pages already running are terminated.+ * Stops the crawler by preventing crawls of additional pages and terminating the running ones.** @return {Promise}*/- async stop() {+ async abort() {this.isRunning = false;- await this.basicCrawler.stop();+ await this.basicCrawler.abort();this.rejectOnStop(new Error('PuppeteerCrawler: .stop() function has been called. Stopping the crawler.'));}diff --git a/test/basic_crawler.js b/test/basic_crawler.jsindex bad335fd1..64b4036a6 100644--- a/test/basic_crawler.js+++ b/test/basic_crawler.js@@ -50,7 +50,7 @@ describe('BasicCrawler', () => {expect(await requestList.isEmpty()).to.be.eql(true);});- it('should run, stop and then finish', async () => {+ it('should abort and resume', async () => {const sources = _.range(500).map(index => ({ url: `https://example.com/${index + 1}` }));let basicCrawler;@@ -59,7 +59,7 @@ describe('BasicCrawler', () => {const requestList = new Apify.RequestList({ sources });const handleRequestFunction = async ({ request }) => {if (request.url.endsWith('200') && !isStopped) {- await basicCrawler.stop();+ await basicCrawler.abort();isStopped = true;} else {await delayPromise(10);diff --git a/test/cheerio_crawler.js b/test/cheerio_crawler.jsindex 63946934a..5609853cc 100644--- a/test/cheerio_crawler.js+++ b/test/cheerio_crawler.js@@ -65,7 +65,7 @@ describe('CheerioCrawler', () => {});});- it('should stop and resume', async () => {+ it('should abort and resume', async () => {const sources = _.range(30).map(index => ({ url: `https://example.com/?q=${index + 1}` }));let cheerioCrawler;let isStopped = false;@@ -74,7 +74,7 @@ describe('CheerioCrawler', () => {const requestList = new Apify.RequestList({ sources });const handlePageFunction = async ({ $, html, request }) => {if (request.url.endsWith('15') && !isStopped) {- cheerioCrawler.stop();+ cheerioCrawler.abort();isStopped = true;} else {request.userData.title = $('title').text();diff --git a/test/puppeteer_crawler.js b/test/puppeteer_crawler.jsindex 16cbecc87..1c24da5a9 100644--- a/test/puppeteer_crawler.js+++ b/test/puppeteer_crawler.js@@ -78,7 +78,7 @@ describe('PuppeteerCrawler', () => {const requestList = new Apify.RequestList({ sources });const handlePageFunction = async ({ page, request, response }) => {if (request.url.endsWith('15') && !isStopped) {- await puppeteerCrawler.stop();+ await puppeteerCrawler.abort();isStopped = true;} else {await page.waitFor('title');