Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Disabling autoscaling when running out of Apify platform (#92)

Marek Trunkát3229016faf8e
diff --git a/src/autoscaled_pool.js b/src/autoscaled_pool.js
index dabb009c8..66f7e2eda 100644
--- a/src/autoscaled_pool.js
+++ b/src/autoscaled_pool.js
@@ -2,7 +2,7 @@ import Promise from 'bluebird';
import log from 'apify-shared/log';
import _ from 'underscore';
import { checkParamOrThrow } from 'apify-client/build/utils';
-import { getMemoryInfo, isPromise, avg } from './utils';
+import { getMemoryInfo, isPromise, avg, isAtHome } from './utils';
import events from './events';
import { ACTOR_EVENT_NAMES } from './constants';
@@ -188,7 +188,13 @@ export default class AutoscaledPool {
// This interval checks memory and in each call saves current memory stats and in every
// SCALE_UP_INTERVAL-th/SCALE_DOWN_INTERVAL-th call it may scale up/down based on memory.
- this.memCheckInterval = setInterval(() => this._autoscale(), MEM_CHECK_INTERVAL_MILLIS);
+ if (isAtHome()) {
+ this.memCheckInterval = setInterval(() => this._autoscale(), MEM_CHECK_INTERVAL_MILLIS);
+ } else {
+ log.warning('Autoscaling feature is currently available only when running at Apify platform!');
+ log.warning('Use `minConcurrency` parameter if you need to test multiple requests in parallel.');
+ log.warning('This feature will be enabled soon!');
+ }
return this.poolPromise
.then(() => {
diff --git a/test/autoscaled_pool.js b/test/autoscaled_pool.js
index 76197b587..44f19762b 100644
--- a/test/autoscaled_pool.js
+++ b/test/autoscaled_pool.js
@@ -5,7 +5,7 @@ import _ from 'underscore';
import sinon from 'sinon';
import { delayPromise } from 'apify-shared/utilities';
import * as Apify from '../build/index';
-import { ACTOR_EVENT_NAMES } from '../build/constants';
+import { ACTOR_EVENT_NAMES, ENV_VARS } from '../build/constants';
import { LOG_INFO_INTERVAL, SCALE_UP_MAX_STEP, SCALE_DOWN_INTERVAL } from '../build/autoscaled_pool';
import * as utils from '../build/utils';
@@ -13,6 +13,15 @@ chai.use(chaiAsPromised);
const toBytes = x => x * 1024 * 1024;
describe('AutoscaledPool', () => {
+ // This is here to enable autoscaling that is temporarily disabled when running locally.
+ before(() => {
+ process.env[ENV_VARS.IS_AT_HOME] = 1;
+ });
+
+ after(() => {
+ delete process.env[ENV_VARS.IS_AT_HOME];
+ });
+
it('should work with concurrency 1', async () => {
const range = _.range(0, 10);
const result = [];