Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Scaling pool down less aggresively (#56)

Marek Trunkát94cf79c88ee5
diff --git a/src/autoscaled_pool.js b/src/autoscaled_pool.js
index fd04a52f7..a5d8a0221 100644
--- a/src/autoscaled_pool.js
+++ b/src/autoscaled_pool.js
@@ -6,9 +6,6 @@ import { getMemoryInfo, isPromise } from './utils';
import { events } from './actor';
import { ACTOR_EVENT_NAMES } from './constants';
-// TODO: we should scale down less aggressively - ie. scale down only if CPU or memory
-// gets exceeded for some longer period of time ...
-
const MEM_CHECK_INTERVAL_MILLIS = 200; // This is low to have at least.
const MIN_FREE_MEMORY_RATIO = 0.1; // Minimum amount of memory that we keep free.
const DEFAULT_OPTIONS = {
@@ -139,7 +136,7 @@ export default class AutoscaledPool {
this.concurrency = minConcurrency;
this.runningCount = 0;
this.freeBytesSnapshots = [];
- this.isCpuOverloaded = false;
+ this.isCpuOverloadedSnapshots = [false];
this.queryingIsTaskReady = false;
this.queryingIsFinished = false;
@@ -156,7 +153,9 @@ export default class AutoscaledPool {
// Connect to actor events for CPU info.
// TODO: This doesn't work on local machine!
this.cpuInfoListener = (data) => {
- this.isCpuOverloaded = data.isCpuOverloaded;
+ this.isCpuOverloadedSnapshots = this.isCpuOverloadedSnapshots
+ .concat(data.isCpuOverloaded)
+ .slice(-SCALE_DOWN_INTERVAL);
};
events.on(ACTOR_EVENT_NAMES.CPU_INFO, this.cpuInfoListener);
}
@@ -214,33 +213,65 @@ export default class AutoscaledPool {
this.intervalCounter++;
this.freeBytesSnapshots = this.freeBytesSnapshots.concat(freeBytes).slice(-SCALE_UP_INTERVAL);
- // Maybe scale down.
- if (
- this.intervalCounter % SCALE_DOWN_INTERVAL === 0
- && this.concurrency > this.minConcurrency
- && (this.isCpuOverloaded || freeBytes / totalBytes < this.minFreeMemoryRatio)
- ) {
- this.concurrency--;
- log.debug('AutoscaledPool: scaling down', { concurrency: this.concurrency });
-
- // Maybe scale up.
- } else if (
- this.intervalCounter % SCALE_UP_INTERVAL === 0
- && this.concurrency < this.maxConcurrency
- ) {
- const spaceForInstances = this._computeSpaceForInstances(totalBytes, this.intervalCounter % LOG_INFO_INTERVAL);
-
- if (spaceForInstances > 0) {
- const increaseBy = Math.min(spaceForInstances, SCALE_UP_MAX_STEP);
- const oldConcurrency = this.concurrency;
- this.concurrency = Math.min(this.concurrency + increaseBy, this.maxConcurrency);
- log.debug('AutoscaledPool: scaling up', { oldConcurrency, newConcurrency: this.concurrency });
- }
- }
+ const scaledDown = this._maybeScaleDown(totalBytes);
+
+ if (!scaledDown) this._maybeScaleUp(totalBytes);
})
.catch(err => log.exception(err, 'AutoscaledPool._autoscale() function failed'));
}
+ /**
+ * Scales pool down if there is enough memory and CPU.
+ *
+ * @return true if concurrency was changed
+ * @ignore
+ */
+ _maybeScaleDown(totalBytes) {
+ if (this.intervalCounter % SCALE_DOWN_INTERVAL !== 0 || this.concurrency <= this.minConcurrency) return false;
+
+ const snapshots = this.freeBytesSnapshots.slice(-SCALE_DOWN_INTERVAL);
+ const averageFreeBytes = snapshots.reduce((sum, c) => sum + c, 0) / snapshots.length;
+ const isMemoryOverloaded = averageFreeBytes / totalBytes < this.minFreeMemoryRatio;
+ const isCpuOverloaded = _.all(this.isCpuOverloadedSnapshots);
+
+ if (!isCpuOverloaded && !isMemoryOverloaded) return false;
+
+ this.concurrency--;
+ log.debug('AutoscaledPool: scaling down', {
+ oldConcurrency: this.concurrency - 1,
+ newConcurrency: this.concurrency,
+ isMemoryOverloaded,
+ isCpuOverloaded,
+ });
+
+ return true;
+ }
+
+ /**
+ * Scales pool up if there is enough memory and CPU.
+ *
+ * @return true if concurrency was changed
+ * @ignore
+ */
+ _maybeScaleUp(totalBytes) {
+ if (this.intervalCounter % SCALE_UP_INTERVAL !== 0 || this.concurrency >= this.maxConcurrency) return false;
+
+ const spaceForInstances = this._computeSpaceForInstances(totalBytes, this.intervalCounter % LOG_INFO_INTERVAL);
+
+ if (spaceForInstances <= 0) return false;
+
+ const increaseBy = Math.min(spaceForInstances, SCALE_UP_MAX_STEP);
+ const oldConcurrency = this.concurrency;
+
+ this.concurrency = Math.min(this.concurrency + increaseBy, this.maxConcurrency);
+ log.debug('AutoscaledPool: scaling up', {
+ oldConcurrency,
+ newConcurrency: this.concurrency,
+ });
+
+ return true;
+ }
+
/**
* Gets memory info and computes how much we can scale pool
* to avoid exceeding the maximum memory.
diff --git a/test/autoscaled_pool.js b/test/autoscaled_pool.js
index 412adb9ac..3d367a9dc 100644
--- a/test/autoscaled_pool.js
+++ b/test/autoscaled_pool.js
@@ -6,7 +6,7 @@ import sinon from 'sinon';
import { delayPromise } from 'apify-shared/utilities';
import * as Apify from '../build/index';
import { ACTOR_EVENT_NAMES } from '../build/constants';
-import { LOG_INFO_INTERVAL, SCALE_UP_MAX_STEP } from '../build/autoscaled_pool';
+import { LOG_INFO_INTERVAL, SCALE_UP_MAX_STEP, SCALE_DOWN_INTERVAL } from '../build/autoscaled_pool';
import * as utils from '../build/utils';
chai.use(chaiAsPromised);
@@ -131,7 +131,7 @@ describe('AutoscaledPool', () => {
await pool._autoscale();
expect(pool.concurrency).to.be.eql(1 + SCALE_UP_MAX_STEP);
- // Should not do anything.
+ // Should not do anything when there is right amount of memory used.
pool.concurrency = 10;
pool.runningCount = 10;
pool.intervalCounter = LOG_INFO_INTERVAL - 1;
@@ -141,13 +141,29 @@ describe('AutoscaledPool', () => {
await pool._autoscale();
expect(pool.concurrency).to.be.eql(10);
- // Should scale down.
+ // Now there is not enough of memory but average from last SCALE_DOWN_INTERVAL snapshots is still ok.
+ pool.concurrency = 10;
+ pool.runningCount = 10;
+ mock.expects('getMemoryInfo')
+ .exactly(3)
+ .returns(Promise.resolve({ freeBytes: toBytes(0.9), totalBytes: toBytes(10) }));
+ let promise = Promise.resolve();
+ for (let i = 0; i < 3; i++) {
+ promise = promise.then(() => {
+ pool.intervalCounter = LOG_INFO_INTERVAL - 1;
+ return pool._autoscale();
+ });
+ }
+ await promise;
+ expect(pool.concurrency).to.be.eql(10);
+
+ // Now the average is below threshold so pool scales down.
pool.concurrency = 10;
pool.runningCount = 10;
- pool.intervalCounter = LOG_INFO_INTERVAL - 1;
mock.expects('getMemoryInfo')
.once()
.returns(Promise.resolve({ freeBytes: toBytes(0.9), totalBytes: toBytes(10) }));
+ pool.intervalCounter = LOG_INFO_INTERVAL - 1;
await pool._autoscale();
expect(pool.concurrency).to.be.eql(9);
@@ -209,6 +225,11 @@ describe('AutoscaledPool', () => {
});
const mock = sinon.mock(utils);
+ // Emit CPU overloaded events but not required emount so that we can still scale up.
+ for (let i = 0; i < SCALE_DOWN_INTERVAL - 1; i++) {
+ Apify.events.emit(ACTOR_EVENT_NAMES.CPU_INFO, { isCpuOverloaded: true });
+ }
+
// Should scale up.
pool.intervalCounter = LOG_INFO_INTERVAL - 1;
pool.runningCount = pool.concurrency;
@@ -227,7 +248,7 @@ describe('AutoscaledPool', () => {
await pool._autoscale();
expect(pool.concurrency).to.be.eql(1 + (2 * SCALE_UP_MAX_STEP));
- // Emit CPU overloaded = true event.
+ // Emit final CPU overloaded = true event.
Apify.events.emit(ACTOR_EVENT_NAMES.CPU_INFO, { isCpuOverloaded: true });
// Should scale up but because of CPU overloaded event it always scales down.
@@ -360,7 +381,6 @@ describe('AutoscaledPool', () => {
* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
*/
-
it('[DEPRECATED] should work with concurrency 1', async () => {
const range = _.range(0, 10);
const result = [];
@@ -463,7 +483,7 @@ describe('AutoscaledPool', () => {
await pool._autoscale();
expect(pool.concurrency).to.be.eql(1 + SCALE_UP_MAX_STEP);
- // Should not do anything.
+ // Should not do anything when there is right amount of memory used.
pool.concurrency = 10;
pool.runningCount = 10;
pool.intervalCounter = LOG_INFO_INTERVAL - 1;
@@ -473,13 +493,29 @@ describe('AutoscaledPool', () => {
await pool._autoscale();
expect(pool.concurrency).to.be.eql(10);
- // Should scale down.
+ // Now there is not enough of memory but average from last SCALE_DOWN_INTERVAL snapshots is still ok.
+ pool.concurrency = 10;
+ pool.runningCount = 10;
+ mock.expects('getMemoryInfo')
+ .exactly(3)
+ .returns(Promise.resolve({ freeBytes: toBytes(0.9), totalBytes: toBytes(10) }));
+ let promise = Promise.resolve();
+ for (let i = 0; i < 3; i++) {
+ promise = promise.then(() => {
+ pool.intervalCounter = LOG_INFO_INTERVAL - 1;
+ return pool._autoscale();
+ });
+ }
+ await promise;
+ expect(pool.concurrency).to.be.eql(10);
+
+ // Now the average is below threshold so pool scales down.
pool.concurrency = 10;
pool.runningCount = 10;
- pool.intervalCounter = LOG_INFO_INTERVAL - 1;
mock.expects('getMemoryInfo')
.once()
.returns(Promise.resolve({ freeBytes: toBytes(0.9), totalBytes: toBytes(10) }));
+ pool.intervalCounter = LOG_INFO_INTERVAL - 1;
await pool._autoscale();
expect(pool.concurrency).to.be.eql(9);
@@ -521,6 +557,11 @@ describe('AutoscaledPool', () => {
});
const mock = sinon.mock(utils);
+ // Emit CPU overloaded events but not required emount so that we can still scale up.
+ for (let i = 0; i < SCALE_DOWN_INTERVAL - 1; i++) {
+ Apify.events.emit(ACTOR_EVENT_NAMES.CPU_INFO, { isCpuOverloaded: true });
+ }
+
// Should scale up.
pool.intervalCounter = LOG_INFO_INTERVAL - 1;
pool.runningCount = pool.concurrency;
@@ -539,7 +580,7 @@ describe('AutoscaledPool', () => {
await pool._autoscale();
expect(pool.concurrency).to.be.eql(1 + (2 * SCALE_UP_MAX_STEP));
- // Emit CPU overloaded = true event.
+ // Emit final CPU overloaded = true event.
Apify.events.emit(ACTOR_EVENT_NAMES.CPU_INFO, { isCpuOverloaded: true });
// Should scale up but because of CPU overloaded event it always scales down.