Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Implement feature
Ondra Urban4c6d7809e7f4
diff --git a/src/autoscaling/autoscaled_pool.js b/src/autoscaling/autoscaled_pool.jsindex 851221f99..4894c703e 100644--- a/src/autoscaling/autoscaled_pool.js+++ b/src/autoscaling/autoscaled_pool.js@@ -161,6 +161,7 @@ class AutoscaledPool {this.isTaskReadyFunction = isTaskReadyFunction;// Internal properties.+ this.isPaused = false;this.desiredConcurrency = this.minConcurrency;this.currentConcurrency = 0;this.lastLoggingTime = 0;@@ -238,6 +239,49 @@ class AutoscaledPool {}}+ /**+ * Prevents the auto-scaled pool from starting new tasks, but allows the running ones to finish+ * (unlike abort, which terminates them). Used together with [`resume()`](#AutoscaledPool+resume)+ *+ * The function's promise will resolve once all running tasks have completed and the pool+ * is effectively idle. If the `timeoutSecs` argument is provided, the promise will reject+ * with a timeout error after the `timeoutSecs` seconds.+ * @param {number} [timeoutSecs]+ * @return {Promise}+ */+ async pause(timeoutSecs) {+ this.isPaused = true;+ return new Promise((resolve, reject) => {+ let timeout;+ if (timeoutSecs) {+ timeout = setTimeout(() => {+ const err = new Error('AutoscaledPool: The pool\'s running tasks did not finish'+ + `in ${timeoutSecs} secs after pool.pause() invocation.`);+ reject(err);+ }, timeoutSecs);+ }++ const interval = setInterval(() => {+ if (this.currentConcurrency <= 0) {+ // Clean up timeout and interval to prevent process hanging.+ if (timeout) clearTimeout(timeout);+ clearInterval(interval);+ resolve();+ }+ }, this.maybeRunIntervalMillis);+ });+ }++ /**+ * Resumes the operation of the autoscaled-pool by allowing more tasks to be run.+ * Used together with [`pause()`](#AutoscaledPool+pause)+ *+ * Tasks will automatically start running again in `options.maybeRunIntervalSecs`.+ */+ resume() {+ this.isPaused = false;+ }+/*** Starts a new task* if the number of running tasks (current concurrency) is lower than desired concurrency@@ -253,6 +297,8 @@ class AutoscaledPool {const done = intervalCallback || (() => {});// Prevent starting a new task if:+ // - the pool is paused+ if (this.isPaused) return done();// - we are already querying for a task.if (this.queryingIsTaskReady) return done();// - we would exceed desired concurrency.@@ -318,6 +364,9 @@ class AutoscaledPool {* @ignore*/_autoscale(intervalCallback) {+ // Don't scale if paused.+ if (this.isPaused) return intervalCallback();+// Only scale up if:// - system has not been overloaded lately.const systemStatus = this.systemStatus.getHistoricalStatus();diff --git a/src/basic_crawler.js b/src/basic_crawler.jsindex 8706c4db1..e7b2e89d2 100644--- a/src/basic_crawler.js+++ b/src/basic_crawler.js@@ -1,10 +1,25 @@import { checkParamOrThrow } from 'apify-client/build/utils';import _ from 'underscore';import log from 'apify-shared/log';+import { ACTOR_EVENT_NAMES } from 'apify-shared/consts';import { checkParamPrototypeOrThrow } from 'apify-shared/utilities';import AutoscaledPool from './autoscaling/autoscaled_pool';import { RequestList } from './request_list';import { RequestQueue, RequestQueueLocal } from './request_queue';+import events from './events';++/**+ * Since there's no set number of seconds before the container is terminated after+ * a migration event, we need some reasonable number to use for RequestList persistence.+ * Once a migration event is received, the Crawler will be paused and it will wait for+ * this long before persisting the RequestList state. This should allow most healthy+ * requests to finish and be marked as handled, thus lowering the amount of duplicate+ * results after migration.+ *+ * @type {number}+ * @ignore+ */+const SAFE_MIGRATION_WAIT_MILLIS = 20000;const DEFAULT_OPTIONS = {maxRequestRetries: 3,@@ -206,6 +221,9 @@ class BasicCrawler {};this.autoscaledPoolOptions = _.defaults({}, basicCrawlerAutoscaledPoolConfiguration, autoscaledPoolOptions);++ // Attach a listener to handle migration events gracefully.+ events.on(ACTOR_EVENT_NAMES.MIGRATING, this._pauseOnMigration.bind(this));}/**@@ -241,6 +259,23 @@ class BasicCrawler {this.rejectOnAbort(new Error('BasicCrawler: .abort() function has been called. Aborting the crawler.'));}+ async _pauseOnMigration() {+ if (!this.isRunning) return;+ await this.autoscaledPool.pause(SAFE_MIGRATION_WAIT_MILLIS)+ .catch(() => {+ log.error('BasicCrawler: The crawler was paused due to migration to another host, '+ + 'but some requests did not finish in time. Those requests\' results may be duplicated.');+ });+ if (this.requestList) {+ await this.requestList.persistState()+ .catch(() => {+ log.error('BasicCrawler: The crawler attempted to persist it\'s request list\'s state and failed due to invalid config. '+ + 'Make sure to use either Apify.openRequestList() or the "stateKeyPrefix" option of RequestList constructor '+ + 'to ensure your crawling state is persisted through host migrations and restarts.');+ });+ }+ }+/*** Fetches request from either RequestList or RequestQueue. If request comes from a RequestList* and RequestQueue is present then enqueues it to the queue first.