Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Adding this.minFreeMemoryRatio configuration to AutoscaledPool

Marek Trunkatbe023598b182
diff --git a/package.json b/package.json
index acd493193..dca1b234c 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
{
"name": "apify",
- "version": "0.5.1",
+ "version": "0.5.2",
"description": "Apify Actor runtime for JavaScript",
"main": "build/index.js",
"keywords": [
diff --git a/src/autoscaled_pool.js b/src/autoscaled_pool.js
index 063eca346..c0847a482 100644
--- a/src/autoscaled_pool.js
+++ b/src/autoscaled_pool.js
@@ -10,6 +10,7 @@ const MIN_FREE_MEMORY_RATIO = 0.1; // Minimum amount of memory that we keep free
const DEFAULT_OPTIONS = {
maxConcurrency: 1000,
minConcurrency: 1,
+ minFreeMemoryRatio: 0.2,
};
// These constants defines that in Nth execution memCheckInterval we do:
@@ -49,14 +50,16 @@ const humanReadable = bytes => `${Math.round(bytes / 1024 / 1024)} MB`;
* @param {Number} [options.maxConcurrency=1000] Maximal concurrency.
* @param {Number} [options.minConcurrency=1] Minimal concurrency.
* @param {Number} [options.maxMemoryMbytes] Maximum memory available in the system. By default uses the totalMemory from Apify.getMemoryInfo().
+ * @param {Number} [options.minFreeMemoryRatio=0.2] Minumum ratio of free memory kept in the system.
*/
export default class AutoscaledPool {
constructor(opts) {
- const { maxMemoryMbytes, maxConcurrency, minConcurrency, workerFunction } = _.defaults(opts, DEFAULT_OPTIONS);
+ const { maxMemoryMbytes, maxConcurrency, minConcurrency, workerFunction, minFreeMemoryRatio } = _.defaults(opts, DEFAULT_OPTIONS);
checkParamOrThrow(maxMemoryMbytes, 'opts.maxMemoryMbytes', 'Maybe Number');
checkParamOrThrow(maxConcurrency, 'opts.maxConcurrency', 'Number');
checkParamOrThrow(minConcurrency, 'opts.minConcurrency', 'Number');
+ checkParamOrThrow(minFreeMemoryRatio, 'opts.minFreeMemoryRatio', 'Number');
checkParamOrThrow(workerFunction, 'opts.workerFunction', 'Function');
// Configuration.
@@ -64,6 +67,7 @@ export default class AutoscaledPool {
this.maxConcurrency = maxConcurrency;
this.minConcurrency = Math.min(minConcurrency, maxConcurrency);
this.workerFunction = workerFunction;
+ this.minFreeMemoryRatio = minFreeMemoryRatio;
// State.
this.promiseCounter = 0;
@@ -135,15 +139,14 @@ export default class AutoscaledPool {
// Maybe scale down.
if (
this.intervalCounter % SCALE_DOWN_INTERVAL === 0
- && (freeBytes / totalBytes < MIN_FREE_MEMORY_RATIO)
+ && (freeBytes / totalBytes < this.minFreeMemoryRatio)
&& this.concurrency > this.minConcurrency
) {
this.concurrency--;
log.debug('AutoscaledPool: scaling down', { concurrency: this.concurrency });
- }
// Maybe scale up.
- if (
+ } else if (
this.intervalCounter % SCALE_UP_INTERVAL === 0
&& this.concurrency < this.maxConcurrency
) {
diff --git a/src/basic_crawler.js b/src/basic_crawler.js
index 4d752c522..6c9f93766 100644
--- a/src/basic_crawler.js
+++ b/src/basic_crawler.js
@@ -45,6 +45,7 @@ const DEFAULT_OPTIONS = {
* @param {Number} [options.maxMemoryMbytes] Maximal memory available in the system (see `maxMemoryMbytes` parameter of `Apify.AutoscaledPool`).
* @param {Number} [options.maxConcurrency=1] Minimal concurrency of requests processing (see `maxConcurrency` parameter of `Apify.AutoscaledPool`).
* @param {Number} [options.minConcurrency=1000] Maximal concurrency of request processing (see `minConcurrency` parameter of `Apify.AutoscaledPool`).
+ * @param {Number} [options.minFreeMemoryRatio=0.2] Minumum ratio of free memory kept in the system.
*/
export default class BasicCrawler {
constructor(opts) {
@@ -58,6 +59,7 @@ export default class BasicCrawler {
maxMemoryMbytes,
maxConcurrency,
minConcurrency,
+ minFreeMemoryRatio,
} = _.defaults(opts, DEFAULT_OPTIONS);
checkParamOrThrow(handleRequestFunction, 'opts.handleRequestFunction', 'Function');
@@ -76,6 +78,7 @@ export default class BasicCrawler {
maxMemoryMbytes,
maxConcurrency,
minConcurrency,
+ minFreeMemoryRatio,
});
}
diff --git a/src/puppeteer_crawler.js b/src/puppeteer_crawler.js
index f3e998e27..bfaaed237 100644
--- a/src/puppeteer_crawler.js
+++ b/src/puppeteer_crawler.js
@@ -59,6 +59,7 @@ const DEFAULT_OPTIONS = {
* @param {Number} [options.maxMemoryMbytes] Maximal memory available in the system (See `maxMemoryMbytes` parameter of `Apify.AutoscaledPool`).
* @param {Number} [options.maxConcurrency=1] Minimal concurrency of requests processing (See `maxConcurrency` parameter of `Apify.AutoscaledPool`).
* @param {Number} [options.minConcurrency=1000] Maximal concurrency of request processing (See `minConcurrency` parameter of `Apify.AutoscaledPool`).
+ * @param {Number} [options.minFreeMemoryRatio=0.2] Minumum ratio of free memory kept in the system.
*
* @param {Number} [options.maxOpenPagesPerInstance=100] Maximal number of opened tabs per browser. If limit is reached then the new
* browser gets started. (See `maxOpenPagesPerInstance` parameter of `Apify.PuppeteerPool`)
@@ -90,6 +91,7 @@ export default class PuppeteerCrawler {
maxMemoryMbytes,
maxConcurrency,
minConcurrency,
+ minFreeMemoryRatio,
// Basic crawler options
requestList,
@@ -138,6 +140,7 @@ export default class PuppeteerCrawler {
maxMemoryMbytes,
maxConcurrency,
minConcurrency,
+ minFreeMemoryRatio,
});
}