Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Feature/autoscaling (#82)

Marek Trunkát1e9abde55397
diff --git a/CHANGELOG.md b/CHANGELOG.md
index a363d4d4e..4fecb112d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,7 @@
+xxxxxxxxxxxxxxxxxxx
+===================
+- Added `ignoreMainProcess` parameter to `AutoscaledPool`. Check documentation for more info.
+
0.5.26 / 2018-04-27
===================
- Parameters `session` and `groups` of `getApifyProxyUrl()` renamed to `apifyProxySession` and `apifyProxyGroups` to match
diff --git a/package.json b/package.json
index 376c62273..2ffee8ae1 100644
--- a/package.json
+++ b/package.json
@@ -49,7 +49,9 @@
"bluebird": "^3.5.0",
"content-type": "^1.0.3",
"fs-extra": "^5.0.0",
+ "pidusage": "^2.0.6",
"proxy-chain": "^0.1.26",
+ "ps-tree": "^1.1.0",
"request-promise": "^4.2.2",
"underscore": "^1.9.0",
"ws": "^5.1.0"
diff --git a/src/autoscaled_pool.js b/src/autoscaled_pool.js
index d3b8e81c0..dabb009c8 100644
--- a/src/autoscaled_pool.js
+++ b/src/autoscaled_pool.js
@@ -2,7 +2,7 @@ import Promise from 'bluebird';
import log from 'apify-shared/log';
import _ from 'underscore';
import { checkParamOrThrow } from 'apify-client/build/utils';
-import { getMemoryInfo, isPromise } from './utils';
+import { getMemoryInfo, isPromise, avg } from './utils';
import events from './events';
import { ACTOR_EVENT_NAMES } from './constants';
@@ -14,6 +14,7 @@ const DEFAULT_OPTIONS = {
minFreeMemoryRatio: 0.2,
maybeRunIntervalMillis: 500,
finishWhenEmpty: true,
+ ignoreMainProcess: false,
};
// These constants defines that in Nth execution memCheckInterval we do:
@@ -77,6 +78,9 @@ const humanReadable = bytes => `${Math.round(bytes / 1024 / 1024)} MB`;
* @param {Number} [options.minFreeMemoryRatio=0.2] Minimum ratio of free memory kept in the system.
* @param {number} [options.maybeRunIntervalMillis=1000] Indicates how often should the pool try to call
* `opts.runTaskFunction` to start a new task.
+ * @param {Boolean} [options.ignoreMainProcess=false] If set to `true` then autoscaling doesn't consider memory consumption
+ * of the main NodeJS process when autoscaling the pool up/down. This is mainly useful when
+ * tasks are running as separate processes (for example web browser).
*/
export default class AutoscaledPool {
constructor(opts) {
@@ -109,6 +113,7 @@ export default class AutoscaledPool {
runTaskFunction,
isFinishedFunction,
isTaskReadyFunction,
+ ignoreMainProcess,
} = _.defaults(opts, DEFAULT_OPTIONS);
checkParamOrThrow(maxConcurrency, 'opts.maxConcurrency', 'Number');
@@ -119,6 +124,7 @@ export default class AutoscaledPool {
checkParamOrThrow(runTaskFunction, 'opts.runTaskFunction', 'Function');
checkParamOrThrow(isFinishedFunction, 'opts.isFinishedFunction', 'Maybe Function');
checkParamOrThrow(isTaskReadyFunction, 'opts.isTaskReadyFunction', 'Maybe Function');
+ checkParamOrThrow(ignoreMainProcess, 'opts.ignoreMainProcess', 'Boolean');
// Configuration.
this.maxMemoryMbytes = maxMemoryMbytes;
@@ -129,6 +135,7 @@ export default class AutoscaledPool {
this.runTaskFunction = runTaskFunction;
this.isFinishedFunction = isFinishedFunction;
this.isTaskReadyFunction = isTaskReadyFunction;
+ this.ignoreMainProcess = ignoreMainProcess;
// State.
this.promiseCounter = 0;
@@ -202,19 +209,26 @@ export default class AutoscaledPool {
* - SCALE_UP_INTERVAL-th call checks memory and possibly scales UP
* - MEM_INFO_INTERVAL-th call logs statistics about memory.
*
+ * NOTE: Parameter this.ignoreMainProcess=true solves the situation where for example we have 3GB memory,
+ * node process takes 1.5GB and Chrome 500MB so autoscaling things that we 1 concurrency allocates
+ * 2/3 of memory and therefore it doesn't scales up.
+ *
+ * Ignoring the main process we get total memory decreased to 1.5GB so AutoscaledPool considers that
+ * one Chrome process to be correctly taking that 1/3 of memory.
+ *
* @ignore
*/
_autoscale() {
// Returning promise so that we can await it in unit tests.
return getMemoryInfo()
- .then(({ freeBytes, totalBytes }) => {
+ .then(({ freeBytes, totalBytes, mainProcessBytes }) => {
+ if (this.ignoreMainProcess) totalBytes -= mainProcessBytes;
if (this.maxMemoryMbytes) totalBytes = Math.min(totalBytes, this.maxMemoryMbytes);
this.intervalCounter++;
this.freeBytesSnapshots = this.freeBytesSnapshots.concat(freeBytes).slice(-SCALE_UP_INTERVAL);
const scaledDown = this._maybeScaleDown(totalBytes);
-
if (!scaledDown) this._maybeScaleUp(totalBytes);
})
.catch(err => log.exception(err, 'AutoscaledPool._autoscale() function failed'));
@@ -230,7 +244,7 @@ export default class AutoscaledPool {
if (this.intervalCounter % SCALE_DOWN_INTERVAL !== 0 || this.concurrency <= this.minConcurrency) return false;
const snapshots = this.freeBytesSnapshots.slice(-SCALE_DOWN_INTERVAL);
- const averageFreeBytes = snapshots.reduce((sum, c) => sum + c, 0) / snapshots.length;
+ const averageFreeBytes = avg(snapshots);
const isMemoryOverloaded = averageFreeBytes / totalBytes < this.minFreeMemoryRatio;
const isCpuOverloaded = _.all(this.isCpuOverloadedSnapshots);
diff --git a/src/basic_crawler.js b/src/basic_crawler.js
index 0e57bd96c..8f32532ab 100644
--- a/src/basic_crawler.js
+++ b/src/basic_crawler.js
@@ -111,6 +111,7 @@ export default class BasicCrawler {
? isFinishedFunction()
: this._defaultIsFinishedFunction()
),
+ ignoreMainProcess: true,
});
}
diff --git a/src/utils.js b/src/utils.js
index 7d31cb353..82c062e89 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -4,9 +4,15 @@ import os from 'os';
import fs from 'fs';
import fsExtra from 'fs-extra';
import ApifyClient from 'apify-client';
+import psTree from 'ps-tree';
+import pidusage from 'pidusage';
+import _ from 'underscore';
import { ENV_VARS } from './constants';
+export const PID_USAGE_NOT_FOUND_ERROR = 'No maching pid found';
+
const ensureDirPromised = Promise.promisify(fsExtra.ensureDir);
+const psTreePromised = Promise.promisify(psTree);
/**
* Creates an instance of ApifyClient using options as defined in the environment variables.
@@ -80,7 +86,7 @@ export const addCharsetToContentType = (contentType) => {
return contentTypeParser.format(parsed);
};
-let isDockerPromise;
+let isDockerPromiseCache;
const createIsDockerPromise = () => {
const promise1 = Promise
.promisify(fs.stat)('/.dockerenv')
@@ -109,11 +115,31 @@ const createIsDockerPromise = () => {
*/
export const isDocker = (forceReset) => {
// Parameter forceReset is just internal for unit tests.
- if (!isDockerPromise || forceReset) isDockerPromise = createIsDockerPromise();
+ if (!isDockerPromiseCache || forceReset) isDockerPromiseCache = createIsDockerPromise();
- return isDockerPromise;
+ return isDockerPromiseCache;
};
+/**
+ * Sums an array of numbers.
+ *
+ * @param {[Number]} arr An array of numbers.
+ * @return {Number} Sum of the numbers.
+ *
+ * @ignore
+ */
+export const sum = arr => arr.reduce((total, c) => total + c, 0);
+
+/**
+ * Computes an average of an array of numbers.
+ *
+ * @param {[Number]} arr An array of numbers.
+ * @return {Number} Average value.
+ *
+ * @ignore
+ */
+export const avg = arr => sum(arr) / arr.length;
+
/**
* Returns memory statistics of the container, which is an object with the following properties:
*
@@ -127,6 +153,10 @@ export const isDocker = (forceReset) => {
* &nbsp;
* // Amount of memory used (= totalBytes - freeBytes)
* usedBytes: Number,
+ * // Amount of memory used by main NodeJS process
+ * mainProcessBytes: Number,
+ * // Amount of memory used by child processes of main NodeJS process
+ * childProcessesBytes: Number,
* }
* ```
*
@@ -139,13 +169,43 @@ export const isDocker = (forceReset) => {
*/
export const getMemoryInfo = () => {
// module.exports must be here so that we can mock it.
- return module.exports.isDocker()
- .then((isDockerVar) => {
+ const isDockerPromise = module.exports.isDocker();
+
+ const childProcessesUsagePromise = psTreePromised(process.pid)
+ .then((childProcesses) => {
+ const pids = _.pluck(childProcesses, 'PID');
+
+ const promises = pids.map((pid) => {
+ return pidusage(pid)
+ .then(info => info.memory)
+ .catch((err) => {
+ if (err.message === PID_USAGE_NOT_FOUND_ERROR) return 0;
+
+ throw err;
+ });
+ });
+
+ return Promise.all(promises).then(infos => sum(infos));
+ });
+
+ return Promise
+ .all([
+ isDockerPromise,
+ childProcessesUsagePromise,
+ ])
+ .then(([isDockerVar, childProcessesBytes]) => {
if (!isDockerVar) {
const freeBytes = os.freemem();
const totalBytes = os.totalmem();
+ const usedBytes = totalBytes - freeBytes;
- return Promise.resolve({ totalBytes, freeBytes, usedBytes: totalBytes - freeBytes });
+ return Promise.resolve({
+ totalBytes,
+ freeBytes,
+ usedBytes,
+ mainProcessBytes: usedBytes - childProcessesBytes,
+ childProcessesBytes,
+ });
}
// This must be promisified here so that we can Mock it.
@@ -160,7 +220,13 @@ export const getMemoryInfo = () => {
const totalBytes = parseInt(totalBytesStr, 10);
const usedBytes = parseInt(usedBytesStr, 10);
- return { totalBytes, freeBytes: totalBytes - usedBytes, usedBytes };
+ return {
+ totalBytes,
+ freeBytes: totalBytes - usedBytes,
+ usedBytes,
+ mainProcessBytes: usedBytes - childProcessesBytes,
+ childProcessesBytes,
+ };
});
});
};
diff --git a/test/autoscaled_pool.js b/test/autoscaled_pool.js
index 3d367a9dc..76197b587 100644
--- a/test/autoscaled_pool.js
+++ b/test/autoscaled_pool.js
@@ -78,7 +78,7 @@ describe('AutoscaledPool', () => {
expect(Date.now() - startedAt).to.be.within(100, 150);
});
- it('correctly computed space for instances using', async () => {
+ it('correctly computed space for instances', async () => {
const pool = new Apify.AutoscaledPool({
minConcurrency: 1,
maxConcurrency: 100,
@@ -171,6 +171,50 @@ describe('AutoscaledPool', () => {
mock.restore();
});
+ it('should autoscale correctly when ignoring main process', async () => {
+ const pool = new Apify.AutoscaledPool({
+ ignoreMainProcess: true,
+ minConcurrency: 1,
+ maxConcurrency: 100,
+ minFreeMemoryRatio: 0.1,
+ runTaskFunction: () => Promise.resolve(),
+ isFinishedFunction: () => Promise.resolve(false),
+ isTaskReadyFunction: () => Promise.resolve(true),
+ });
+ const mock = sinon.mock(utils);
+
+ // Should not scale up.
+ pool.concurrency = 1;
+ pool.runningCount = 1;
+ pool.intervalCounter = LOG_INFO_INTERVAL - 1;
+ mock.expects('getMemoryInfo')
+ .once()
+ .returns(Promise.resolve({
+ freeBytes: toBytes(5),
+ totalBytes: toBytes(10),
+ mainProcessBytes: 0,
+ }));
+ await pool._autoscale();
+ expect(pool.concurrency).to.be.eql(1);
+
+ // Should scale up.
+ pool.concurrency = 1;
+ pool.runningCount = 1;
+ pool.intervalCounter = LOG_INFO_INTERVAL - 1;
+ mock.expects('getMemoryInfo')
+ .once()
+ .returns(Promise.resolve({
+ freeBytes: toBytes(5),
+ totalBytes: toBytes(10),
+ mainProcessBytes: toBytes(4),
+ }));
+ await pool._autoscale();
+ expect(pool.concurrency).to.be.eql(5);
+
+ mock.verify();
+ mock.restore();
+ });
+
it('should throw when some of the promises throws', async () => {
let counter = 0;
const runTaskFunction = () => {
diff --git a/test/utils.js b/test/utils.js
index 5e368d03d..0a37ce484 100644
--- a/test/utils.js
+++ b/test/utils.js
@@ -1,9 +1,15 @@
-import { expect } from 'chai';
+import chai, { expect } from 'chai';
+import chaiAsPromised from 'chai-as-promised';
import sinon from 'sinon';
import fs from 'fs';
import os from 'os';
+import pidusage from 'pidusage';
+import Promise from 'bluebird';
import * as utils from '../build/utils';
import Apify from '../build/index';
+import { ENV_VARS } from '../build/constants';
+
+chai.use(chaiAsPromised);
/* global process, describe, it */
@@ -102,7 +108,7 @@ describe('utils.isDocker()', () => {
});
describe('utils.getMemoryInfo()', () => {
- it('works outside the container', () => {
+ it('works WITHOUT child process outside the container', () => {
const osMock = sinon.mock(os);
const utilsMock = sinon.mock(utils);
@@ -128,6 +134,8 @@ describe('utils.getMemoryInfo()', () => {
totalBytes: 333,
freeBytes: 222,
usedBytes: 111,
+ mainProcessBytes: 111,
+ childProcessesBytes: 0,
});
utilsMock.restore();
@@ -135,7 +143,7 @@ describe('utils.getMemoryInfo()', () => {
});
});
- it('works inside the container', () => {
+ it('works WITHOUT child process inside the container', () => {
const utilsMock = sinon.mock(utils);
utilsMock
@@ -158,12 +166,83 @@ describe('utils.getMemoryInfo()', () => {
totalBytes: 333,
freeBytes: 222,
usedBytes: 111,
+ mainProcessBytes: 111,
+ childProcessesBytes: 0,
});
utilsMock.restore();
fs.readFile.restore();
});
});
+
+ it('works WITH child process outside the container', () => {
+ const osMock = sinon.mock(os);
+ const utilsMock = sinon.mock(utils);
+ process.env[ENV_VARS.HEADLESS] = '1';
+
+ utilsMock
+ .expects('isDocker')
+ .once()
+ .returns(Promise.resolve(false));
+
+ osMock
+ .expects('freemem')
+ .once()
+ .returns(222);
+
+ osMock
+ .expects('totalmem')
+ .once()
+ .returns(333);
+
+ return Apify.launchPuppeteer()
+ .then((browser) => {
+ return Apify
+ .getMemoryInfo()
+ .then((data) => {
+ expect(data.childProcessesBytes).to.be.above(0);
+ expect(data.usedBytes).to.be.above(0);
+ expect(data.mainProcessBytes).to.be.eql(data.usedBytes - data.childProcessesBytes);
+ utilsMock.restore();
+ osMock.restore();
+ delete process.env[ENV_VARS.HEADLESS];
+ })
+ .then(() => browser.close());
+ });
+ });
+
+ it('works WITH child process inside the container', () => {
+ const utilsMock = sinon.mock(utils);
+ process.env[ENV_VARS.HEADLESS] = '1';
+
+ utilsMock
+ .expects('isDocker')
+ .once()
+ .returns(Promise.resolve(true));
+
+ sinon
+ .stub(fs, 'readFile')
+ .callsFake((path, callback) => {
+ if (path === '/sys/fs/cgroup/memory/memory.limit_in_bytes') callback(null, '333');
+ else if (path === '/sys/fs/cgroup/memory/memory.usage_in_bytes') callback(null, '111');
+ else throw new Error('Invalid path');
+ });
+
+ return Apify.launchPuppeteer()
+ .then((browser) => {
+ return Apify
+ .getMemoryInfo()
+ .then((data) => {
+ expect(data.childProcessesBytes).to.be.above(0);
+ expect(data.usedBytes).to.be.above(0);
+ expect(data.mainProcessBytes).to.be.eql(data.usedBytes - data.childProcessesBytes);
+ utilsMock.restore();
+ fs.readFile.restore();
+ delete process.env[ENV_VARS.HEADLESS];
+ })
+ .then(() => browser.close());
+ });
+ });
});
describe('utils.isPromise()', () => {
@@ -187,3 +266,28 @@ describe('utils.newPromise()', () => {
if (!utils.isPromise(utils.newPromise())) throw new Error('utils.newPromise() must return a promise!');
});
});
+
+describe('pidusage NPM package', () => {
+ it('throws correct error message when process not found', () => {
+ const NONEXISTING_PID = 9999;
+ const promise = Promise.promisify(pidusage)(NONEXISTING_PID);
+
+ return expect(promise).to.be.rejectedWith(utils.PID_USAGE_NOT_FOUND_ERROR);
+ });
+});
+
+describe('utils.sum', () => {
+ it('works', () => {
+ expect(utils.sum([1, 2, 3, 1.2])).to.be.eql(7.2);
+ expect(utils.sum([])).to.be.eql(0);
+ expect(utils.sum([9])).to.be.eql(9);
+ });
+});
+
+describe('utils.avg', () => {
+ it('works', () => {
+ expect(utils.avg([1, 2, 3, 1.2])).to.be.eql(7.2 / 4);
+ expect(utils.avg([])).to.be.eql(NaN);
+ expect(utils.avg([9])).to.be.eql(9);
+ });
+});