Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

add support for cgroups v2 (#1177)

Ondra Urbanf174531f3145
diff --git a/src/utils.js b/src/utils.js
index c8cff436c..87256a9b9 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -42,6 +42,20 @@ const URL_NO_COMMAS_REGEX = RegExp('https?://(www\\.)?[\\p{L}0-9][-\\p{L}0-9@:%.
*/
const URL_WITH_COMMAS_REGEX = RegExp('https?://(www\\.)?[\\p{L}0-9][-\\p{L}0-9@:%._\\+~#=]{0,254}[\\p{L}0-9]\\.[a-z]{2,63}(:\\d{1,5})?(/[-\\p{L}0-9@:%_\\+,.~#?&//=\\(\\)]*)?', 'giu'); // eslint-disable-line
+const MEMORY_FILE_PATHS = {
+ TOTAL: {
+ V1: '/sys/fs/cgroup/memory/memory.limit_in_bytes',
+ V2: '/sys/fs/cgroup/memory.max',
+ },
+ USED: {
+ V1: '/sys/fs/cgroup/memory/memory.usage_in_bytes',
+ V2: '/sys/fs/cgroup/memory.current',
+ },
+};
+
+// Set encoding to utf-8 so fs.readFile returns string instead of buffer
+const MEMORY_FILE_ENCODING = 'utf-8';
+
const psTreePromised = util.promisify(psTree);
/**
@@ -261,16 +275,38 @@ export const getMemoryInfo = async () => {
// When running inside Docker container, use container memory limits
// This must be promisified here so that we can mock it.
const readPromised = util.promisify(fs.readFile);
+ const accessPromised = util.promisify(fs.access);
+ // Check wheter cgroups V1 or V2 is used
+ let cgroupsVersion = 'V1';
try {
- const [totalBytesStr, usedBytesStr] = await Promise.all([
- readPromised('/sys/fs/cgroup/memory/memory.limit_in_bytes'),
- readPromised('/sys/fs/cgroup/memory/memory.usage_in_bytes'),
+ // If this directory does not exists, assume docker is using cgroups V2
+ await accessPromised('/sys/fs/cgroup/memory/', fs.constants.R_OK);
+ } catch (err) {
+ cgroupsVersion = 'V2';
+ }
+
+ try {
+ let [totalBytesStr, usedBytesStr] = await Promise.all([
+ readPromised(MEMORY_FILE_PATHS.TOTAL[cgroupsVersion], MEMORY_FILE_ENCODING),
+ readPromised(MEMORY_FILE_PATHS.USED[cgroupsVersion], MEMORY_FILE_ENCODING),
]);
- totalBytes = parseInt(totalBytesStr, 10);
- // https://unix.stackexchange.com/q/420906
- const containerRunsWithUnlimitedMemory = totalBytes > Number.MAX_SAFE_INTEGER;
- if (containerRunsWithUnlimitedMemory) totalBytes = os.totalmem();
+
+ // Cgroups V2 files contains newline character. Getting rid of it for better handling in later part of the code.
+ totalBytesStr = totalBytesStr.replace(/[^a-zA-Z0-9 ]/g, '');
+ usedBytesStr = usedBytesStr.replace(/[^a-zA-Z0-9 ]/g, '');
+
+ // Cgroups V2 contains 'max' string if memory is not limited
+ // See https://git.kernel.org/pub/scm/linux/kernel/git/tj/cgroup.git/tree/Documentation/admin-guide/cgroup-v2.rst (see "memory.max")
+ if (totalBytesStr === 'max') {
+ totalBytes = os.totalmem();
+ // Cgroups V1 is set to number related to platform and page size if memory is not limited
+ // See https://unix.stackexchange.com/q/420906
+ } else {
+ totalBytes = parseInt(totalBytesStr, 10);
+ const containerRunsWithUnlimitedMemory = totalBytes > Number.MAX_SAFE_INTEGER;
+ if (containerRunsWithUnlimitedMemory) totalBytes = os.totalmem();
+ }
usedBytes = parseInt(usedBytesStr, 10);
freeBytes = totalBytes - usedBytes;
} catch (err) {
diff --git a/test/utils.test.js b/test/utils.test.js
index accbf78c9..d80bf129f 100644
--- a/test/utils.test.js
+++ b/test/utils.test.js
@@ -141,9 +141,13 @@ describe('utils.getMemoryInfo()', () => {
.once()
.returns(Promise.resolve(true));
+ sinon
+ .stub(fs, 'access')
+ .callsFake((file, mode, callback) => callback(null));
+
sinon
.stub(fs, 'readFile')
- .callsFake((filePath, callback) => {
+ .callsFake((filePath, encoding, callback) => {
if (filePath === '/sys/fs/cgroup/memory/memory.limit_in_bytes') callback(null, '333');
else if (filePath === '/sys/fs/cgroup/memory/memory.usage_in_bytes') callback(null, '111');
else throw new Error('Invalid path');
@@ -160,6 +164,7 @@ describe('utils.getMemoryInfo()', () => {
} finally {
utilsMock.verify();
fs.readFile.restore();
+ fs.access.restore();
}
});
@@ -211,9 +216,13 @@ describe('utils.getMemoryInfo()', () => {
.once()
.returns(Promise.resolve(true));
+ sinon
+ .stub(fs, 'access')
+ .callsFake((file, mode, callback) => callback(null));
+
sinon
.stub(fs, 'readFile')
- .callsFake((filePath, callback) => {
+ .callsFake((filePath, encoding, callback) => {
if (filePath === '/sys/fs/cgroup/memory/memory.limit_in_bytes') callback(null, '333');
else if (filePath === '/sys/fs/cgroup/memory/memory.usage_in_bytes') callback(null, '111');
else throw new Error('Invalid path');
@@ -233,10 +242,161 @@ describe('utils.getMemoryInfo()', () => {
} finally {
utilsMock.verify();
fs.readFile.restore();
+ fs.access.restore();
delete process.env[ENV_VARS.HEADLESS];
if (browser) browser.close();
}
});
+
+ test('works with cgroup V1 with LIMITED memory', async () => {
+ const utilsMock = sinon.mock(utils);
+
+ utilsMock
+ .expects('isDocker')
+ .once()
+ .returns(Promise.resolve(true));
+
+ sinon
+ .stub(fs, 'access')
+ .callsFake((file, mode, callback) => callback(null));
+
+ sinon
+ .stub(fs, 'readFile')
+ .callsFake((filePath, encoding, callback) => {
+ if (filePath === '/sys/fs/cgroup/memory/memory.limit_in_bytes') callback(null, '333');
+ else if (filePath === '/sys/fs/cgroup/memory/memory.usage_in_bytes') callback(null, '111');
+ else throw new Error('Invalid path');
+ });
+
+ try {
+ const data = await Apify.getMemoryInfo();
+ expect(data).toMatchObject({
+ totalBytes: 333,
+ freeBytes: 222,
+ usedBytes: 111,
+ });
+ } finally {
+ utilsMock.verify();
+ fs.readFile.restore();
+ fs.access.restore();
+ }
+ });
+
+ test('works with cgroup V1 with UNLIMITED memory', async () => {
+ const utilsMock = sinon.mock(utils);
+ const osMock = sinon.mock(os);
+
+ utilsMock
+ .expects('isDocker')
+ .once()
+ .returns(Promise.resolve(true));
+
+ sinon
+ .stub(fs, 'access')
+ .callsFake((file, mode, callback) => callback(null));
+
+ sinon
+ .stub(fs, 'readFile')
+ .callsFake((filePath, encoding, callback) => {
+ if (filePath === '/sys/fs/cgroup/memory/memory.limit_in_bytes') callback(null, '9223372036854771712');
+ else if (filePath === '/sys/fs/cgroup/memory/memory.usage_in_bytes') callback(null, '111');
+ else throw new Error('Invalid path');
+ });
+
+ osMock
+ .expects('totalmem')
+ .once()
+ .returns(333);
+
+ try {
+ const data = await Apify.getMemoryInfo();
+ expect(data).toMatchObject({
+ totalBytes: 333,
+ freeBytes: 222,
+ usedBytes: 111,
+ });
+ } finally {
+ utilsMock.verify();
+ osMock.verify();
+ fs.readFile.restore();
+ fs.access.restore();
+ }
+ });
+
+ test('works with cgroup V2 with LIMITED memory', async () => {
+ const utilsMock = sinon.mock(utils);
+
+ utilsMock
+ .expects('isDocker')
+ .once()
+ .returns(Promise.resolve(true));
+
+ sinon
+ .stub(fs, 'access')
+ .callsFake((file, mode, callback) => callback('error'));
+
+ sinon
+ .stub(fs, 'readFile')
+ .callsFake((filePath, encoding, callback) => {
+ if (filePath === '/sys/fs/cgroup/memory.max') callback(null, '333\n');
+ else if (filePath === '/sys/fs/cgroup/memory.current') callback(null, '111\n');
+ else throw new Error('Invalid path');
+ });
+
+ try {
+ const data = await Apify.getMemoryInfo();
+ expect(data).toMatchObject({
+ totalBytes: 333,
+ freeBytes: 222,
+ usedBytes: 111,
+ });
+ } finally {
+ utilsMock.verify();
+ fs.readFile.restore();
+ fs.access.restore();
+ }
+ });
+
+ test('works with cgroup V2 with UNLIMITED memory', async () => {
+ const utilsMock = sinon.mock(utils);
+ const osMock = sinon.mock(os);
+
+ utilsMock
+ .expects('isDocker')
+ .once()
+ .returns(Promise.resolve(true));
+
+ sinon
+ .stub(fs, 'access')
+ .callsFake((file, mode, callback) => callback('error'));
+
+ sinon
+ .stub(fs, 'readFile')
+ .callsFake((filePath, encoding, callback) => {
+ if (filePath === '/sys/fs/cgroup/memory.max') callback(null, 'max\n');
+ else if (filePath === '/sys/fs/cgroup/memory.current') callback(null, '111\n');
+ else throw new Error('Invalid path');
+ });
+
+ osMock
+ .expects('totalmem')
+ .once()
+ .returns(333);
+
+ try {
+ const data = await Apify.getMemoryInfo();
+ expect(data).toMatchObject({
+ totalBytes: 333,
+ freeBytes: 222,
+ usedBytes: 111,
+ });
+ } finally {
+ utilsMock.verify();
+ osMock.verify();
+ fs.readFile.restore();
+ fs.access.restore();
+ }
+ });
});
describe('utils.isAtHome()', () => {