Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Add memory env var to snapshotter
Ondra Urbane6b5ac0a1454
diff --git a/jsdoc/conf.json b/jsdoc/conf.jsonindex a2d2f3b47..e90719e02 100644--- a/jsdoc/conf.json+++ b/jsdoc/conf.json@@ -22,6 +22,7 @@"opts": {"template": "./node_modules/apify-jsdoc-template","encoding": "utf8",- "destination": ".docs"+ "destination": ".docs",+ "recurse": true}}diff --git a/src/autoscaling/snapshotter.js b/src/autoscaling/snapshotter.jsindex 138c5eea7..96fa2e7b7 100644--- a/src/autoscaling/snapshotter.js+++ b/src/autoscaling/snapshotter.js@@ -1,7 +1,7 @@import _ from 'underscore';import { betterSetInterval, betterClearInterval } from 'apify-shared/utilities';import log from 'apify-shared/log';-import { ACTOR_EVENT_NAMES } from 'apify-shared/consts';+import { ACTOR_EVENT_NAMES, ENV_VARS } from 'apify-shared/consts';import { checkParamOrThrow } from 'apify-client/build/utils';import { getMemoryInfo, isAtHome } from '../utils';import events from '../events';@@ -49,8 +49,6 @@ const DEFAULT_OPTIONS = {* @param {Number} [options.snapshotHistorySecs=60]* Sets the interval in seconds for which a history of resource snapshots* will be kept. Increasing this to very high numbers will affect performance.- * @param {Number} [options.maxMemoryMbytes]- * Sets the maximum amount of memory to be used in megabytes.* @ignore*/export default class Snapshotter {@@ -61,7 +59,6 @@ export default class Snapshotter {snapshotHistorySecs,maxBlockedMillis,maxUsedMemoryRatio,- maxMemoryMbytes,} = _.defaults(options, DEFAULT_OPTIONS);checkParamOrThrow(eventLoopSnapshotIntervalSecs, 'options.eventLoopSnapshotIntervalSecs', 'Number');@@ -69,14 +66,13 @@ export default class Snapshotter {checkParamOrThrow(snapshotHistorySecs, 'options.snapshotHistorySecs', 'Number');checkParamOrThrow(maxBlockedMillis, 'options.maxBlockedMillis', 'Number');checkParamOrThrow(maxUsedMemoryRatio, 'options.maxUsedMemoryRatio', 'Number');- checkParamOrThrow(maxMemoryMbytes, 'options.maxMemoryMbytes', 'Maybe Number');this.eventLoopSnapshotIntervalMillis = eventLoopSnapshotIntervalSecs * 1000;this.memorySnapshotIntervalMillis = memorySnapshotIntervalSecs * 1000;this.snapshotHistoryMillis = snapshotHistorySecs * 1000;this.maxBlockedMillis = maxBlockedMillis;this.maxUsedMemoryRatio = maxUsedMemoryRatio;- this.maxMemoryBytes = maxMemoryMbytes ? maxMemoryMbytes * 1024 * 1024 : null;+ this.maxMemoryBytes = parseInt(process.env[ENV_VARS.MEMORY_MBYTES], 10) * 1024 * 1024;this.cpuSnapshots = [];this.eventLoopSnapshots = [];@@ -90,13 +86,13 @@ export default class Snapshotter {*/async start() {// Ensure max memory is correctly computed.- if (!this.maxMemoryBytes) {+ if (!this.maxMemoryBytes || Number.isNaN(this.maxMemoryBytes)) {const { totalBytes } = await getMemoryInfo();if (isAtHome()) {this.maxMemoryBytes = totalBytes;} else {this.maxMemoryBytes = Math.ceil(totalBytes / 4);- log.info(`Snapshotter: Setting max memory of this run to ${Math.round(this.maxMemoryBytes / 1024 / 1024)} MB.`);+ log.info(`Snapshotter: Setting max memory of this run to ${Math.round(this.maxMemoryBytes / 1024 / 1024)} MB. Use the ${ENV_VARS.MEMORY_MBYTES} environment variable to override.`); // eslint-disable-line max-len}}diff --git a/test/autoscaling/snapshotter.js b/test/autoscaling/snapshotter.jsindex 3822b4d92..f653ee508 100644--- a/test/autoscaling/snapshotter.js+++ b/test/autoscaling/snapshotter.js@@ -141,9 +141,9 @@ describe('Snapshotter', () => {});it('correctly marks memoryOverloaded', async () => {+ process.env[ENV_VARS.MEMORY_MBYTES] = 20;const options = {memorySnapshotIntervalSecs: 0.1,- maxMemoryMbytes: 20,};const snapshotter = new Snapshotter(options);@@ -158,6 +158,7 @@ describe('Snapshotter', () => {expect(memorySnapshots[0].isOverloaded).to.be.eql(true);expect(memorySnapshots[1].isOverloaded).to.be.eql(true);expect(memorySnapshots[2].isOverloaded).to.be.eql(false);+ delete process.env[ENV_VARS.MEMORY_MBYTES];});it('.get...Sample limits amount of samples', async () => {