Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Fall back when memory cgroups are unavailable (#694)
Ondra Urband61204ac06ba
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex f23af0381..2a2b97a0c 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -27,6 +27,8 @@ in avoiding bot detection.in `Puppeteer` and `Cheerio` crawlers.- Fixed a bug where intercepted requests would never continue.- Fixed a bug where `Apify.utils.requestAsBrowser()` would get into redirect loops.+- Fix `Apify.utils.getMemoryInfo()` crashing the process on AWS Lambda and on systems+ running in Docker without memory cgroups enabled.0.20.4 / 2020-05-11diff --git a/src/utils.js b/src/utils.jsindex 68d50cc96..c4095994a 100644--- a/src/utils.js+++ b/src/utils.js@@ -285,23 +285,35 @@ export const getMemoryInfo = async () => {freeBytes = totalBytes - usedBytes;log.debug(`lambda size of ${totalBytes} with ${freeBytes} free bytes`);- } else if (!isDockerVar) {- totalBytes = os.totalmem();- freeBytes = os.freemem();- usedBytes = totalBytes - freeBytes;- } else {+ } else if (isDockerVar) {// When running inside Docker container, use container memory limits// This must be promisified here so that we can mock it.const readPromised = util.promisify(fs.readFile);- const [totalBytesStr, usedBytesStr] = await Promise.all([- readPromised('/sys/fs/cgroup/memory/memory.limit_in_bytes'),- readPromised('/sys/fs/cgroup/memory/memory.usage_in_bytes'),- ]);-- totalBytes = parseInt(totalBytesStr, 10);- usedBytes = parseInt(usedBytesStr, 10);- freeBytes = totalBytes - usedBytes;+ try {+ const [totalBytesStr, usedBytesStr] = await Promise.all([+ readPromised('/sys/fs/cgroup/memory/memory.limit_in_bytes'),+ readPromised('/sys/fs/cgroup/memory/memory.usage_in_bytes'),+ ]);+ totalBytes = parseInt(totalBytesStr, 10);+ // https://unix.stackexchange.com/q/420906+ const containerRunsWithUnlimitedMemory = totalBytes > Number.MAX_SAFE_INTEGER;+ if (containerRunsWithUnlimitedMemory) totalBytes = os.totalmem();+ usedBytes = parseInt(usedBytesStr, 10);+ freeBytes = totalBytes - usedBytes;+ } catch (err) {+ // log.deprecated logs a warning only once+ log.deprecated('Your environment is Docker, but your system does not support memory cgroups. '+ + 'If you\'re running containers with limited memory, memory auto-scaling will not work properly.\n\n'+ + `Cause: ${err.message}`);+ totalBytes = os.totalmem();+ freeBytes = os.freemem();+ usedBytes = totalBytes - freeBytes;+ }+ } else {+ totalBytes = os.totalmem();+ freeBytes = os.freemem();+ usedBytes = totalBytes - freeBytes;}return {