Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Allow AutoscaledPool to run at minConcurrency when overloaded

Ondra Urban485b8c352e4e
diff --git a/CHANGELOG.md b/CHANGELOG.md
index fbcc6a7ba..ce2a820a6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,7 @@
+xxx
+==================
+- Allow AutoscaledPool to run tasks up to minConcurrency even when the system is overloaded.
+
0.8.11 / 2018-09-27
==================
- Upgraded @apify/ps-tree depedency (fixes "Error: spawn ps ENFILE"), upgraded other NPM packages
diff --git a/src/autoscaling/autoscaled_pool.js b/src/autoscaling/autoscaled_pool.js
index 5e5fd05fa..9e6d0e5cd 100644
--- a/src/autoscaling/autoscaled_pool.js
+++ b/src/autoscaling/autoscaled_pool.js
@@ -228,15 +228,15 @@ export default class AutoscaledPool {
// Check if the function was invoked by the maybeRunInterval and use an empty function if not.
const done = intervalCallback || (() => {});
- // Only run task if:
- // - we're not already querying for a task
+ // Prevent starting a new task if:
+ // - we are already querying for a task.
if (this.queryingIsTaskReady) return done();
- // - we will not exceed desired concurrency.
+ // - we would exceed desired concurrency.
if (this.currentConcurrency >= this.desiredConcurrency) return done();
- // - system is not overloaded now
+ // - system is overloaded now and we are at or above minConcurrency
const currentStatus = this.systemStatus.getCurrentStatus();
const { isSystemIdle } = currentStatus;
- if (!isSystemIdle) {
+ if (!isSystemIdle && this.currentConcurrency >= this.minConcurrency) {
log.debug('AutoscaledPool: Task will not be run. System is overloaded.', currentStatus);
return done();
}
diff --git a/test/autoscaling/autoscaled_pool.js b/test/autoscaling/autoscaled_pool.js
index 7cedbeb32..d44e99037 100644
--- a/test/autoscaling/autoscaled_pool.js
+++ b/test/autoscaling/autoscaled_pool.js
@@ -103,9 +103,9 @@ describe('AutoscaledPool', () => {
pool = new AutoscaledPool({
minConcurrency: 1,
maxConcurrency: 100,
- runTaskFunction: () => Promise.resolve(),
- isFinishedFunction: () => Promise.resolve(false),
- isTaskReadyFunction: () => Promise.resolve(true),
+ runTaskFunction: async () => {},
+ isFinishedFunction: async () => false,
+ isTaskReadyFunction: async () => true,
});
pool.systemStatus = systemStatus;
});
@@ -151,6 +151,44 @@ describe('AutoscaledPool', () => {
pool._autoscale(cb);
expect(pool.desiredConcurrency).to.be.eql(newConcurrency);
});
+
+ it('works at minConcurrency when currently overloaded', async () => {
+ let limit = 5;
+ let concurrencyLog = [];
+ let count = 0;
+ pool.systemStatus.okNow = false;
+ pool.runTaskFunction = async () => {
+ await delayPromise(10);
+ count++;
+ };
+ pool.isFinishedFunction = async () => count >= limit;
+ pool.isTaskReadyFunction = async () => count < limit;
+ pool.desiredConcurrency = 10;
+ pool._currentConcurrency = pool.currentConcurrency;
+
+ Object.defineProperty(pool, 'currentConcurrency', {
+ get() {
+ return this._currentConcurrency;
+ },
+ set(v) {
+ concurrencyLog.push(v);
+ this._currentConcurrency = v;
+ },
+ });
+
+ expect(pool.currentConcurrency).to.be.eql(0);
+
+ await pool.run();
+ expect(concurrencyLog.some(n => n > 1)).to.be.eql(false);
+
+ limit = 50;
+ concurrencyLog = [];
+ count = 0;
+ pool.minConcurrency = 5;
+
+ await pool.run();
+ expect(concurrencyLog.some(n => n > 5)).to.be.eql(false);
+ });
});
describe('should throw', () => {
diff --git a/test/autoscaling/snapshotter.js b/test/autoscaling/snapshotter.js
index f653ee508..1b18e4a68 100644
--- a/test/autoscaling/snapshotter.js
+++ b/test/autoscaling/snapshotter.js
@@ -137,7 +137,7 @@ describe('Snapshotter', () => {
expect(ss.exceededMillis).to.be.eql(0);
}
});
- expect(overloadedCount).to.be.eql(1);
+ expect(overloadedCount).to.be.above(0);
});
it('correctly marks memoryOverloaded', async () => {