Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Added desiredConcurrency option to AutoscaledPool constructor
Jan Curn8e6e002f6f63
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex a1ecca4de..844062c88 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -2,6 +2,8 @@ xxx====================- Bugfix in BasicCrawler: async calls in `isFinishedFunction` were not awaited- Better logging of memory overload errors+- Added `desiredConcurrency` option to `AutoscaledPool` constructor, removed+ unnecessary bound check from the setter property- Upgraded `apify-client` to 0.5.220.14.15 / 2019-05-31diff --git a/src/autoscaling/autoscaled_pool.js b/src/autoscaling/autoscaled_pool.jsindex f46238f69..2ad7b97c3 100644--- a/src/autoscaling/autoscaled_pool.js+++ b/src/autoscaling/autoscaled_pool.js@@ -89,6 +89,10 @@ const DEFAULT_OPTIONS = {* If you're not sure, just keep the default value and the concurrency will scale up automatically.* @param {Number} [options.maxConcurrency=1000]* The maximum number of tasks running in parallel.+ * @param {Number} [options.desiredConcurrency]+ * The desired number of tasks that should be running parallel on the start of the pool,+ * if there is a large enough supply of them.+ * By default, it is `options.minConcurrency`.* @param {Number} [options.desiredConcurrencyRatio=0.95]* Minimum level of desired concurrency to reach before more scaling up is allowed.* @param {Number} [options.scaleUpStepRatio=0.05]@@ -120,6 +124,7 @@ class AutoscaledPool {const {maxConcurrency,minConcurrency,+ desiredConcurrency,desiredConcurrencyRatio,scaleUpStepRatio,scaleDownStepRatio,@@ -135,6 +140,7 @@ class AutoscaledPool {checkParamOrThrow(maxConcurrency, 'options.maxConcurrency', 'Number');checkParamOrThrow(minConcurrency, 'options.minConcurrency', 'Number');+ checkParamOrThrow(desiredConcurrency, 'options.desiredConcurrency', 'Maybe Number');checkParamOrThrow(desiredConcurrencyRatio, 'options.desiredConcurrencyRatio', 'Number');checkParamOrThrow(scaleUpStepRatio, 'options.scaleUpStepRatio', 'Number');checkParamOrThrow(scaleDownStepRatio, 'options.scaleDownStepRatio', 'Number');@@ -159,9 +165,9 @@ class AutoscaledPool {this.isTaskReadyFunction = isTaskReadyFunction;// Internal properties.- this._maxConcurrency = maxConcurrency;this._minConcurrency = minConcurrency;- this._desiredConcurrency = this._minConcurrency;+ this._maxConcurrency = maxConcurrency;+ this._desiredConcurrency = typeof desiredConcurrency === 'number' ? desiredConcurrency : minConcurrency;this._currentConcurrency = 0;this.isStopped = false;this.lastLoggingTime = 0;@@ -213,6 +219,7 @@ class AutoscaledPool {* @param {number} value*/set minConcurrency(value) {+ checkParamOrThrow(value, 'value', 'Number');this._minConcurrency = value;}@@ -231,6 +238,7 @@ class AutoscaledPool {* @param {number} value*/set maxConcurrency(value) {+ checkParamOrThrow(value, 'value', 'Number');this._maxConcurrency = value;}@@ -251,9 +259,7 @@ class AutoscaledPool {* @param {number} value*/set desiredConcurrency(value) {- if (!(this._minConcurrency <= value && value <= this._maxConcurrency)) {- throw new Error(`The desired concurrency must be an integer from ${this._minConcurrency} to ${this._maxConcurrency}`);- }+ checkParamOrThrow(value, 'value', 'Number');this._desiredConcurrency = value;}diff --git a/src/crawlers/basic_crawler.js b/src/crawlers/basic_crawler.jsindex e08a40254..5d13d44d8 100644--- a/src/crawlers/basic_crawler.js+++ b/src/crawlers/basic_crawler.js@@ -256,8 +256,13 @@ class BasicCrawler {async run() {if (this.isRunningPromise) return this.isRunningPromise;- await this._loadHandledRequestCount();+ // Initialize AutoscaledPool before awaiting _loadHandledRequestCount(),+ // so that the caller can get a reference to it before awaiting the promise returned from run()+ // (otherwise there would be no way)this.autoscaledPool = new AutoscaledPool(this.autoscaledPoolOptions);++ await this._loadHandledRequestCount();+this.isRunningPromise = this.autoscaledPool.run();this.stats.startLogging();try {diff --git a/test/autoscaling/autoscaled_pool.js b/test/autoscaling/autoscaled_pool.jsindex d6f1ed72f..760e68583 100644--- a/test/autoscaling/autoscaled_pool.js+++ b/test/autoscaling/autoscaled_pool.js@@ -81,15 +81,55 @@ describe('AutoscaledPool', () => {const startedAt = Date.now();await pool.run();- // Test setDesiredConcurrency()- pool.desiredConcurrency = 10;- expect(pool.desiredConcurrency).to.be.eql(10);- expect(() => {- pool.desiredConcurrency = 100;- }).to.throw(Error);- expect(() => {- pool.desiredConcurrency = -100;- }).to.throw(Error);+ expect(result).to.be.eql(_.range(0, 100));+ expect(Date.now() - startedAt).to.be.within(50, 200);+ });++ it('enables setting concurrency', async () => {+ const range = _.range(0, 100);+ const result = [];++ let isFinished = false;++ const runTaskFunction = () => {+ if (range.length === 1) {+ isFinished = true;+ }++ return new Promise((resolve) => {+ const item = range.shift();+ result.push(item);+ setTimeout(resolve, 5);+ });+ };++ const pool = new AutoscaledPool({+ // Test initial concurrency setting+ minConcurrency: 3,+ maxConcurrency: 13,+ desiredConcurrency: 9,+ runTaskFunction,+ isFinishedFunction: () => Promise.resolve(isFinished),+ isTaskReadyFunction: () => Promise.resolve(!isFinished),+ });++ expect(pool.minConcurrency).to.be.eql(3);+ expect(pool.maxConcurrency).to.be.eql(13);+ expect(pool.desiredConcurrency).to.be.eql(9);++ const startedAt = Date.now();+ const promise = await pool.run();++ // Test setting concurrency+ pool.minConcurrency = 4;+ pool.maxConcurrency = 14;+ pool.desiredConcurrency = 7;++ expect(pool.minConcurrency).to.be.eql(4);+ expect(pool.maxConcurrency).to.be.eql(14);+ expect(pool.desiredConcurrency).to.be.eql(7);++ await promise;expect(result).to.be.eql(_.range(0, 100));expect(Date.now() - startedAt).to.be.within(50, 200);