Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Adding getMemoryInfo()

Marek Trunkat6a47f6f981f8
diff --git a/src/actor.js b/src/actor.js
index 08ce1c373..d91db764f 100644
--- a/src/actor.js
+++ b/src/actor.js
@@ -1,11 +1,12 @@
import fs from 'fs';
import path from 'path';
import _ from 'underscore';
+import os from 'os';
import contentTypeParser from 'content-type';
import Promise from 'bluebird';
import { checkParamOrThrow } from 'apify-client/build/utils';
import { ENV_VARS, EXIT_CODES, ACT_TASK_TERMINAL_STATUSES } from './constants';
-import { getPromisePrototype, newPromise, nodeifyPromise, newClient, addCharsetToContentType } from './utils';
+import { getPromisePrototype, newPromise, nodeifyPromise, newClient, addCharsetToContentType, isDocker } from './utils';
/* global process, Buffer */
@@ -602,3 +603,39 @@ export const call = (actId, input, opts = {}, callback) => {
return nodeifyPromise(promise, callback);
};
+
+
+/**
+ * @memberof module:Apify
+ * @function
+ * @description Returns memory statistics of the container.
+ *
+ * @returns {Promise} Returns a promise unless `callback` was supplied.
+ */
+export const getMemoryInfo = (callback) => {
+ // This must be promisified here so that we can Mock it.
+ const readPromised = Promise.promisify(fs.readFile);
+ const promise = isDocker()
+ .then((isDockerVar) => {
+ if (!isDockerVar) {
+ const freeBytes = os.freemem();
+ const totalBytes = os.totalmem();
+
+ return Promise.resolve({ totalBytes, freeBytes, usedBytes: totalBytes - freeBytes });
+ }
+
+ return Promise
+ .all([
+ readPromised('/sys/fs/cgroup/memory/memory.limit_in_bytes'),
+ readPromised('/sys/fs/cgroup/memory/memory.usage_in_bytes'),
+ ])
+ .then(([totalBytesStr, usedBytesStr]) => {
+ const totalBytes = parseInt(totalBytesStr, 10);
+ const usedBytes = parseInt(usedBytesStr, 10);
+
+ return { totalBytes, freeBytes: totalBytes - usedBytes, usedBytes };
+ });
+ });
+
+ return nodeifyPromise(promise, callback);
+};
diff --git a/src/index.js b/src/index.js
index 874a8e85f..8d5f248b1 100644
--- a/src/index.js
+++ b/src/index.js
@@ -1,7 +1,7 @@
import EventEmitter from 'events';
import { setPromisesDependency, getPromisesDependency } from './utils';
-import { main, readyFreddy, getEnv, getValue, setValue, apifyClient, call } from './actor';
+import { main, readyFreddy, getEnv, getValue, setValue, apifyClient, call, getMemoryInfo } from './actor';
import { launchPuppeteer } from './puppeteer';
import { browse, launchWebDriver } from './webdriver';
@@ -17,6 +17,7 @@ const Apify = {
readyFreddy,
setPromisesDependency,
getPromisesDependency,
+ getMemoryInfo,
browse,
launchWebDriver,
launchPuppeteer,
diff --git a/src/utils.js b/src/utils.js
index fb89f0b17..dd15c419e 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -1,5 +1,6 @@
import Promise from 'bluebird';
import contentTypeParser from 'content-type';
+import fs from 'fs';
import ApifyClient from 'apify-client';
import { ENV_VARS } from './constants';
@@ -120,3 +121,33 @@ export const addCharsetToContentType = (contentType) => {
return contentTypeParser.format(parsed);
};
+
+/**
+ * Returns promise that resolves to true if the code is running in Docker container.
+ * See https://github.com/sindresorhus/is-docker
+ *
+ * Param forceReset is just internal for unit tests.
+ *
+ * @return Promise
+ */
+let isDockerPromise;
+const createIsDockerPromise = () => {
+ const promise1 = Promise
+ .promisify(fs.stat)('/.dockerenv')
+ .then(() => true)
+ .catch(() => false);
+
+ const promise2 = Promise
+ .promisify(fs.readFile)('/proc/self/cgroup', 'utf8')
+ .then(content => content.indexOf('docker') !== -1)
+ .catch(() => false);
+
+ return Promise
+ .all([promise1, promise2])
+ .then(([result1, result2]) => result1 || result2);
+};
+export const isDocker = (forceReset) => {
+ if (!isDockerPromise || forceReset) isDockerPromise = createIsDockerPromise();
+
+ return isDockerPromise;
+};
diff --git a/test/actor.js b/test/actor.js
index 04373f4e8..d86331e4d 100644
--- a/test/actor.js
+++ b/test/actor.js
@@ -3,10 +3,12 @@ import pathModule from 'path';
import _ from 'underscore';
import { expect, assert } from 'chai';
import sinon from 'sinon';
+import os from 'os';
import tmp from 'tmp';
import rimraf from 'rimraf';
import Promise from 'bluebird';
import { ACT_TASK_STATUSES } from '../build/constants';
+import * as utils from '../build/utils';
// NOTE: test use of require() here because this is how its done in acts
const Apify = require('../build/index');
@@ -1234,3 +1236,100 @@ describe('Apify.call()', () => {
});
});
});
+
+describe('Apify.getMemoryInfo()', () => {
+ it('works in docker container', () => {
+ const osMock = sinon.mock(os);
+ const utilsMock = sinon.mock(utils);
+
+ utilsMock
+ .expects('isDocker')
+ .once()
+ .returns(Promise.resolve(false));
+
+ osMock
+ .expects('freemem')
+ .once()
+ .returns(222);
+
+ osMock
+ .expects('totalmem')
+ .once()
+ .returns(333);
+
+ return Apify
+ .getMemoryInfo()
+ .then((data) => {
+ expect(data).to.be.eql({
+ totalBytes: 333,
+ freeBytes: 222,
+ usedBytes: 111,
+ });
+
+ utilsMock.restore();
+ osMock.restore();
+ });
+ });
+
+ it('works with callback', (done) => {
+ const osMock = sinon.mock(os);
+ const utilsMock = sinon.mock(utils);
+
+ utilsMock
+ .expects('isDocker')
+ .once()
+ .returns(Promise.resolve(false));
+
+ osMock
+ .expects('freemem')
+ .once()
+ .returns(222);
+
+ osMock
+ .expects('totalmem')
+ .once()
+ .returns(333);
+
+ Apify.getMemoryInfo((err, data) => {
+ expect(data).to.be.eql({
+ totalBytes: 333,
+ freeBytes: 222,
+ usedBytes: 111,
+ });
+
+ utilsMock.restore();
+ osMock.restore();
+ done();
+ });
+ });
+
+ it('works outside the container', () => {
+ const utilsMock = sinon.mock(utils);
+
+ utilsMock
+ .expects('isDocker')
+ .once()
+ .returns(Promise.resolve(true));
+
+ sinon
+ .stub(fs, 'readFile')
+ .callsFake((path, callback) => {
+ if (path === '/sys/fs/cgroup/memory/memory.limit_in_bytes') callback(null, '333');
+ else if (path === '/sys/fs/cgroup/memory/memory.usage_in_bytes') callback(null, '111');
+ else throw new Error('Invalid path');
+ });
+
+ return Apify
+ .getMemoryInfo()
+ .then((data) => {
+ expect(data).to.be.eql({
+ totalBytes: 333,
+ freeBytes: 222,
+ usedBytes: 111,
+ });
+
+ utilsMock.restore();
+ fs.readFile.restore();
+ });
+ });
+});
diff --git a/test/utils.js b/test/utils.js
index cc0a3b939..aad6e435c 100644
--- a/test/utils.js
+++ b/test/utils.js
@@ -1,5 +1,7 @@
import BluebirdPromise from 'bluebird';
import { expect } from 'chai';
+import sinon from 'sinon';
+import fs from 'fs';
import * as utils from '../build/utils';
import Apify from '../build/index';
@@ -72,3 +74,57 @@ describe('utils.addCharsetToContentType()', () => {
expect(utils.addCharsetToContentType(undefined)).to.eql(undefined);
});
});
+
+describe('utils.isDocker()', () => {
+ it('works for dockerenv && cgroup', () => {
+ sinon.stub(fs, 'stat').callsFake((path, callback) => callback(null));
+ sinon.stub(fs, 'readFile').callsFake((path, encoding, callback) => callback(null, 'something ... docker ... something'));
+
+ return utils
+ .isDocker(true)
+ .then((is) => {
+ expect(is).to.be.eql(true);
+ fs.stat.restore();
+ fs.readFile.restore();
+ });
+ });
+
+ it('works for dockerenv', () => {
+ sinon.stub(fs, 'stat').callsFake((path, callback) => callback(null));
+ sinon.stub(fs, 'readFile').callsFake((path, encoding, callback) => callback(null, 'something ... ... something'));
+
+ return utils
+ .isDocker(true)
+ .then((is) => {
+ expect(is).to.be.eql(true);
+ fs.stat.restore();
+ fs.readFile.restore();
+ });
+ });
+
+ it('works for cgroup', () => {
+ sinon.stub(fs, 'stat').callsFake((path, callback) => callback(new Error()));
+ sinon.stub(fs, 'readFile').callsFake((path, encoding, callback) => callback(null, 'something ... docker ... something'));
+
+ return utils
+ .isDocker(true)
+ .then((is) => {
+ expect(is).to.be.eql(true);
+ fs.stat.restore();
+ fs.readFile.restore();
+ });
+ });
+
+ it('works for nothing', () => {
+ sinon.stub(fs, 'stat').callsFake((path, callback) => callback(new Error()));
+ sinon.stub(fs, 'readFile').callsFake((path, encoding, callback) => callback(null, 'something ... ... something'));
+
+ return utils
+ .isDocker(true)
+ .then((is) => {
+ expect(is).to.be.eql(false);
+ fs.stat.restore();
+ fs.readFile.restore();
+ });
+ });
+});