Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Better logging of memory overload errors

Jan Curn21cb53ee86f6
diff --git a/CHANGELOG.md b/CHANGELOG.md
index aa166c70f..c89c7a821 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,7 @@
xxx
====================
- Bugfix in BasicCrawler: async calls in `isFinishedFunction` were not awaited
+- Better logging of memory overload errors
0.14.15 / 2019-05-31
====================
diff --git a/src/autoscaling/snapshotter.js b/src/autoscaling/snapshotter.js
index ebe1a7f81..59bfef24a 100644
--- a/src/autoscaling/snapshotter.js
+++ b/src/autoscaling/snapshotter.js
@@ -125,7 +125,7 @@ class Snapshotter {
this.clientInterval = null;
this.cpuInterval = null;
- this.lastLoggedCriticalMemoryOverloadAt = 0;
+ this.lastLoggedCriticalMemoryOverloadAt = null;
// We need to pre-bind those functions to be able to successfully remove listeners.
this._snapshotCpuOnPlatform = this._snapshotCpuOnPlatform.bind(this);
@@ -285,14 +285,14 @@ class Snapshotter {
*/
_memoryOverloadWarning({ memCurrentBytes }) {
const now = new Date();
- if (now < this.lastLoggedCriticalMemoryOverloadAt + CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS) return;
+ if (this.lastLoggedCriticalMemoryOverloadAt && now < this.lastLoggedCriticalMemoryOverloadAt + CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS) return;
const maxDesiredMemoryBytes = this.maxUsedMemoryRatio * this.maxMemoryBytes;
const reserveMemory = this.maxMemoryBytes * (1 - this.maxUsedMemoryRatio) * RESERVE_MEMORY_RATIO;
const criticalOverloadBytes = maxDesiredMemoryBytes + reserveMemory;
const isCriticalOverload = memCurrentBytes > criticalOverloadBytes;
if (isCriticalOverload) {
- const usedPercentage = memCurrentBytes / this.maxMemoryBytes * 100;
+ const usedPercentage = Math.round((memCurrentBytes / this.maxMemoryBytes) * 100);
const toMb = bytes => Math.round(bytes / (1024 ** 2));
log.warning('Memory is critically overloaded. '
+ `Used ${toMb(memCurrentBytes)} MB of ${toMb(this.maxMemoryBytes)} MB (${usedPercentage}%)`);