Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat: move autoscaling helpers to `@crawlee/core` (#3835)

Jindřich Bäre614738e6544
diff --git a/docs/public-api/crawlee-utils.api.md b/docs/public-api/crawlee-utils.api.md
index adb9389f2..d7d10e194 100644
--- a/docs/public-api/crawlee-utils.api.md
+++ b/docs/public-api/crawlee-utils.api.md
@@ -42,14 +42,6 @@ export const CLOUDFLARE_RETRY_CSS_SELECTORS: string[];
export { Constructor }
-// @public
-export interface CpuSample {
- // (undocumented)
- containerUsage: number;
- // (undocumented)
- systemUsage: number;
-}
-
// @public
export function createRequestDebugInfo(request: Request_2, response?: IncomingMessage | Partial<BrowserResponseLike>, additionalFields?: Dictionary): Dictionary;
@@ -121,21 +113,6 @@ const FACEBOOK_REGEX: RegExp;
// @public
const FACEBOOK_REGEX_GLOBAL: RegExp;
-// @public
-export function getCgroupsVersion(forceReset?: boolean): Promise<"V1" | "V2" | null>;
-
-// @internal
-export function getCurrentCpuTicksV2(options?: {
- containerized?: boolean;
- logger?: CrawleeLogger;
-}): Promise<number>;
-
-// @internal
-export function getMemoryInfo(options?: {
- containerized?: boolean;
- logger?: CrawleeLogger;
-}): Promise<MemoryInfo>;
-
// @public (undocumented)
export function getObjectType(value: unknown): string;
@@ -157,18 +134,9 @@ export function isAsyncIterable<T>(value: unknown): value is AsyncIterable<T>;
// @public
export function isBuffer(value: unknown): value is Buffer | ArrayBuffer | ArrayBufferView;
-// @public
-export function isContainerized(): Promise<boolean>;
-
-// @public
-export function isDocker(forceReset?: boolean): Promise<boolean>;
-
// @internal
export function isIterable<T>(value: unknown): value is Iterable<T>;
-// @public (undocumented)
-export function isLambda(): boolean;
-
// @public
export function isStream(value: unknown): value is NodeJS.ReadableStream | ReadableStream;
@@ -181,15 +149,6 @@ const LINKEDIN_REGEX: RegExp;
// @public
const LINKEDIN_REGEX_GLOBAL: RegExp;
-// @public
-export interface MemoryInfo {
- childProcessesBytes: number;
- freeBytes: number;
- mainProcessBytes: number;
- totalBytes: number;
- usedBytes: number;
-}
-
// @public
export function mergeAsyncIterables<T>(...iterables: AsyncIterable<T>[]): AsyncIterable<T>;
diff --git a/packages/core/src/autoscaling/memory_load_signal.ts b/packages/core/src/autoscaling/memory_load_signal.ts
index b88463b88..1e8fca502 100644
--- a/packages/core/src/autoscaling/memory_load_signal.ts
+++ b/packages/core/src/autoscaling/memory_load_signal.ts
@@ -1,10 +1,10 @@
-import { getMemoryInfo, isContainerized } from '@crawlee/utils';
-
import type { Configuration } from '../configuration.js';
import type { EventManager } from '../events/event_manager.js';
import { EventType } from '../events/event_manager.js';
import type { CrawleeLogger } from '../log.js';
import { serviceLocator } from '../service_locator.js';
+import { getMemoryInfo } from '../system-info/memory-info.js';
+import { isContainerized } from '../system-info/runtime.js';
import type { LoadSignal, LoadSnapshot } from './load_signal.js';
import { SnapshotStore } from './load_signal.js';
import type { SystemInfo } from './system_status.js';
diff --git a/packages/core/src/events/local_event_manager.ts b/packages/core/src/events/local_event_manager.ts
index d69211bea..7bb2c9ffb 100644
--- a/packages/core/src/events/local_event_manager.ts
+++ b/packages/core/src/events/local_event_manager.ts
@@ -76,7 +76,7 @@ export class LocalEventManager extends EventManager {
* @internal
*/
async isContainerizedWrapper() {
- const { isContainerized } = await import('@crawlee/utils');
+ const { isContainerized } = await import('../system-info/runtime.js');
return serviceLocator.getConfiguration().containerized ?? (await isContainerized());
}
@@ -92,7 +92,7 @@ export class LocalEventManager extends EventManager {
}
private async createCpuInfo(options: { maxUsedCpuRatio: number }) {
- const { getCurrentCpuTicksV2 } = await import('@crawlee/utils');
+ const { getCurrentCpuTicksV2 } = await import('../system-info/cpu-info.js');
const usedCpuRatio = await getCurrentCpuTicksV2({
containerized: await this.isContainerizedWrapper(),
logger: serviceLocator.getLogger(),
@@ -117,7 +117,7 @@ export class LocalEventManager extends EventManager {
}
private async getMemoryInfo() {
- const { getMemoryInfo } = await import('@crawlee/utils');
+ const { getMemoryInfo } = await import('../system-info/memory-info.js');
return getMemoryInfo({
containerized: await this.isContainerizedWrapper(),
logger: serviceLocator.getLogger(),
diff --git a/packages/utils/src/internals/system-info/cpu-info.ts b/packages/core/src/system-info/cpu-info.ts
similarity index 99%
rename from packages/utils/src/internals/system-info/cpu-info.ts
rename to packages/core/src/system-info/cpu-info.ts
index a2f3f5289..c8e3fbaa0 100644
--- a/packages/utils/src/internals/system-info/cpu-info.ts
+++ b/packages/core/src/system-info/cpu-info.ts
@@ -4,7 +4,7 @@ import os from 'node:os';
import type { CrawleeLogger } from '@crawlee/types';
-import { getCgroupsVersion } from '../general.js';
+import { getCgroupsVersion } from './runtime.js';
const CPU_FILE_PATHS = {
STAT: {
diff --git a/packages/utils/src/internals/system-info/memory-info.ts b/packages/core/src/system-info/memory-info.ts
similarity index 98%
rename from packages/utils/src/internals/system-info/memory-info.ts
rename to packages/core/src/system-info/memory-info.ts
index bbbf24f23..a1c7326b9 100644
--- a/packages/utils/src/internals/system-info/memory-info.ts
+++ b/packages/core/src/system-info/memory-info.ts
@@ -4,7 +4,7 @@ import { freemem, totalmem } from 'node:os';
import type { CrawleeLogger } from '@crawlee/types';
-import { getCgroupsVersion, isLambda } from '../general.js';
+import { getCgroupsVersion, isLambda } from './runtime.js';
import { psTree } from './ps-tree.js';
const MEMORY_FILE_PATHS = {
diff --git a/packages/utils/src/internals/system-info/ps-tree.ts b/packages/core/src/system-info/ps-tree.ts
similarity index 100%
rename from packages/utils/src/internals/system-info/ps-tree.ts
rename to packages/core/src/system-info/ps-tree.ts
diff --git a/packages/core/src/system-info/runtime.ts b/packages/core/src/system-info/runtime.ts
new file mode 100644
index 000000000..63dfdefe4
--- /dev/null
+++ b/packages/core/src/system-info/runtime.ts
@@ -0,0 +1,93 @@
+import fs from 'node:fs/promises';
+
+let isDockerPromiseCache: Promise<boolean> | undefined;
+
+async function createIsDockerPromise() {
+ const promise1 = fs
+ .stat('/.dockerenv')
+ .then(() => true)
+ .catch(() => false);
+
+ const promise2 = fs
+ .readFile('/proc/self/cgroup', 'utf8')
+ .then((content) => content.includes('docker'))
+ .catch(() => false);
+
+ const [result1, result2] = await Promise.all([promise1, promise2]);
+
+ return result1 || result2;
+}
+
+/**
+ * Returns a `Promise` that resolves to true if the code is running in a Docker container.
+ */
+export async function isDocker(forceReset?: boolean): Promise<boolean> {
+ // Parameter forceReset is just internal for unit tests.
+ if (!isDockerPromiseCache || forceReset) isDockerPromiseCache = createIsDockerPromise();
+
+ return isDockerPromiseCache;
+}
+
+let isContainerizedResult: boolean | undefined;
+
+/**
+ * Detects if crawlee is running in a containerized environment.
+ */
+export async function isContainerized() {
+ // Value is very unlikley to change. Cache the result after the first execution.
+ if (isContainerizedResult !== undefined) {
+ return isContainerizedResult;
+ }
+
+ // return false if running in aws lambda
+ if (isLambda()) {
+ isContainerizedResult = false;
+ return isContainerizedResult;
+ }
+
+ const dockerenvCheck = fs
+ .stat('/.dockerenv')
+ .then(() => true)
+ .catch(() => false);
+
+ const cgroupCheck = fs
+ .readFile('/proc/self/cgroup', 'utf8')
+ .then((content) => content.includes('docker'))
+ .catch(() => false);
+
+ const [dockerenvResult, cgroupResult] = await Promise.all([dockerenvCheck, cgroupCheck]);
+
+ isContainerizedResult = dockerenvResult || cgroupResult || !!process.env.KUBERNETES_SERVICE_HOST;
+ return isContainerizedResult;
+}
+
+export function isLambda() {
+ return !!process.env.AWS_LAMBDA_FUNCTION_MEMORY_SIZE;
+}
+
+let _cgroupsVersion: null | 'V1' | 'V2';
+/**
+ * gets the cgroup version by checking for a file at /sys/fs/cgroup/memory
+ * @returns "V1" or "V2" for the version of cgroup or null if cgroup is not found.
+ */
+export async function getCgroupsVersion(forceReset?: boolean) {
+ // Parameter forceReset is just internal for unit tests.
+ if (_cgroupsVersion !== undefined && !forceReset) {
+ return _cgroupsVersion;
+ }
+ try {
+ // If this directory does not exists, cgroups are not available
+ await fs.access('/sys/fs/cgroup/');
+ } catch (e) {
+ _cgroupsVersion = null;
+ return null;
+ }
+ _cgroupsVersion = 'V1';
+ try {
+ // If this directory does not exists, assume the container is using cgroups V2
+ await fs.access('/sys/fs/cgroup/memory/');
+ } catch (e) {
+ _cgroupsVersion = 'V2';
+ }
+ return _cgroupsVersion;
+}
diff --git a/packages/utils/src/index.ts b/packages/utils/src/index.ts
index eadf94dc5..517c14e6c 100644
--- a/packages/utils/src/index.ts
+++ b/packages/utils/src/index.ts
@@ -14,9 +14,4 @@ export * from './internals/robots.js';
export * from './internals/sitemap.js';
export * from './internals/url.js';
-export type { CpuSample } from './internals/system-info/cpu-info.js';
-export { getCurrentCpuTicksV2 } from './internals/system-info/cpu-info.js';
-export type { MemoryInfo } from './internals/system-info/memory-info.js';
-export { getMemoryInfo } from './internals/system-info/memory-info.js';
-
export type { Dictionary, Awaitable, Constructor } from '@crawlee/types';
diff --git a/packages/utils/src/internals/general.ts b/packages/utils/src/internals/general.ts
index 3158e8899..7d59c1b3b 100644
--- a/packages/utils/src/internals/general.ts
+++ b/packages/utils/src/internals/general.ts
@@ -1,4 +1,3 @@
-import fs from 'node:fs/promises';
import { setTimeout } from 'node:timers/promises';
/**
@@ -15,98 +14,6 @@ export const URL_NO_COMMAS_REGEX =
export const URL_WITH_COMMAS_REGEX =
/https?:\/\/(www\.)?([\p{L}0-9]|[\p{L}0-9][-\p{L}0-9@:%._+~#=]{0,254}[\p{L}0-9])\.[a-z]{2,63}(:\d{1,5})?(\/[-\p{L}0-9@:%_+,.~#?&/=()'*]*)?/giu;
-let isDockerPromiseCache: Promise<boolean> | undefined;
-
-async function createIsDockerPromise() {
- const promise1 = fs
- .stat('/.dockerenv')
- .then(() => true)
- .catch(() => false);
-
- const promise2 = fs
- .readFile('/proc/self/cgroup', 'utf8')
- .then((content) => content.includes('docker'))
- .catch(() => false);
-
- const [result1, result2] = await Promise.all([promise1, promise2]);
-
- return result1 || result2;
-}
-
-/**
- * Returns a `Promise` that resolves to true if the code is running in a Docker container.
- */
-export async function isDocker(forceReset?: boolean): Promise<boolean> {
- // Parameter forceReset is just internal for unit tests.
- if (!isDockerPromiseCache || forceReset) isDockerPromiseCache = createIsDockerPromise();
-
- return isDockerPromiseCache;
-}
-
-let isContainerizedResult: boolean | undefined;
-
-/**
- * Detects if crawlee is running in a containerized environment.
- */
-export async function isContainerized() {
- // Value is very unlikley to change. Cache the result after the first execution.
- if (isContainerizedResult !== undefined) {
- return isContainerizedResult;
- }
-
- // return false if running in aws lambda
- if (isLambda()) {
- isContainerizedResult = false;
- return isContainerizedResult;
- }
-
- const dockerenvCheck = fs
- .stat('/.dockerenv')
- .then(() => true)
- .catch(() => false);
-
- const cgroupCheck = fs
- .readFile('/proc/self/cgroup', 'utf8')
- .then((content) => content.includes('docker'))
- .catch(() => false);
-
- const [dockerenvResult, cgroupResult] = await Promise.all([dockerenvCheck, cgroupCheck]);
-
- isContainerizedResult = dockerenvResult || cgroupResult || !!process.env.KUBERNETES_SERVICE_HOST;
- return isContainerizedResult;
-}
-
-export function isLambda() {
- return !!process.env.AWS_LAMBDA_FUNCTION_MEMORY_SIZE;
-}
-
-let _cgroupsVersion: null | 'V1' | 'V2';
-/**
- * gets the cgroup version by checking for a file at /sys/fs/cgroup/memory
- * @returns "V1" or "V2" for the version of cgroup or null if cgroup is not found.
- */
-export async function getCgroupsVersion(forceReset?: boolean) {
- // Parameter forceReset is just internal for unit tests.
- if (_cgroupsVersion !== undefined && !forceReset) {
- return _cgroupsVersion;
- }
- try {
- // If this directory does not exists, cgroups are not available
- await fs.access('/sys/fs/cgroup/');
- } catch (e) {
- _cgroupsVersion = null;
- return null;
- }
- _cgroupsVersion = 'V1';
- try {
- // If this directory does not exists, assume the container is using cgroups V2
- await fs.access('/sys/fs/cgroup/memory/');
- } catch (e) {
- _cgroupsVersion = 'V2';
- }
- return _cgroupsVersion;
-}
-
/**
* Computes a weighted average of an array of numbers, complemented by an array of weights.
* @ignore
diff --git a/test/utils/cpu-infoV2.test.ts b/test/core/autoscaling/cpu-infoV2.test.ts
similarity index 97%
rename from test/utils/cpu-infoV2.test.ts
rename to test/core/autoscaling/cpu-infoV2.test.ts
index 8786e3c3e..37a198be5 100644
--- a/test/utils/cpu-infoV2.test.ts
+++ b/test/core/autoscaling/cpu-infoV2.test.ts
@@ -1,7 +1,7 @@
import { readFile } from 'node:fs/promises';
import os from 'node:os';
-import { getCgroupsVersion } from '@crawlee/utils';
+import { getCgroupsVersion } from '../../../packages/core/src/system-info/runtime.js';
import {
getContainerCpuUsage,
@@ -11,10 +11,10 @@ import {
getCurrentCpuTicksV2,
getSystemCpuUsage,
sampleCpuUsage,
-} from '../../packages/utils/src/internals/system-info/cpu-info.js';
+} from '../../../packages/core/src/system-info/cpu-info.js';
-vitest.mock('@crawlee/utils/src/internals/general', async (importActual) => {
- const original: typeof import('@crawlee/utils') = await importActual();
+vitest.mock('../../../packages/core/src/system-info/runtime.js', async (importActual) => {
+ const original: typeof import('../../../packages/core/src/system-info/runtime.js') = await importActual();
return {
...original,
getCgroupsVersion: vitest.fn(),
diff --git a/test/utils/fixtures/child.js b/test/core/autoscaling/fixtures/child.js
similarity index 100%
rename from test/utils/fixtures/child.js
rename to test/core/autoscaling/fixtures/child.js
diff --git a/test/utils/fixtures/parent.js b/test/core/autoscaling/fixtures/parent.js
similarity index 59%
rename from test/utils/fixtures/parent.js
rename to test/core/autoscaling/fixtures/parent.js
index 19e0c7f5b..e77f6f701 100644
--- a/test/utils/fixtures/parent.js
+++ b/test/core/autoscaling/fixtures/parent.js
@@ -1,5 +1,5 @@
import { exec } from 'node:child_process';
for (let count = 1; count < 10; count++) {
- exec('node ./test/utils/fixtures/child.js');
+ exec('node ./test/core/autoscaling/fixtures/child.js');
}
diff --git a/test/utils/memory-infoV2.test.ts b/test/core/autoscaling/memory-infoV2.test.ts
similarity index 96%
rename from test/utils/memory-infoV2.test.ts
rename to test/core/autoscaling/memory-infoV2.test.ts
index 9094cd4ee..52fc0961e 100644
--- a/test/utils/memory-infoV2.test.ts
+++ b/test/core/autoscaling/memory-infoV2.test.ts
@@ -2,7 +2,8 @@ import { access, readFile } from 'node:fs/promises';
import { freemem, totalmem } from 'node:os';
import { launchPuppeteer } from '@crawlee/puppeteer';
-import { getCgroupsVersion, getMemoryInfo } from '@crawlee/utils';
+import { getMemoryInfo } from '../../../packages/core/src/system-info/memory-info.js';
+import { getCgroupsVersion } from '../../../packages/core/src/system-info/runtime.js';
vitest.mock('node:os', async (importActual) => {
const originalOs: typeof import('node:os') = await importActual();
@@ -13,8 +14,8 @@ vitest.mock('node:os', async (importActual) => {
};
});
-vitest.mock('@crawlee/utils/src/internals/general', async (importActual) => {
- const original: typeof import('@crawlee/utils') = await importActual();
+vitest.mock('../../../packages/core/src/system-info/runtime.js', async (importActual) => {
+ const original: typeof import('../../../packages/core/src/system-info/runtime.js') = await importActual();
return {
...original,
diff --git a/test/utils/psTree.test.ts b/test/core/autoscaling/psTree.test.ts
similarity index 96%
rename from test/utils/psTree.test.ts
rename to test/core/autoscaling/psTree.test.ts
index 1f24d9088..ad52da2d1 100644
--- a/test/utils/psTree.test.ts
+++ b/test/core/autoscaling/psTree.test.ts
@@ -1,7 +1,7 @@
import { exec } from 'node:child_process';
import path from 'node:path';
-import { psTree } from '../../packages/utils/src/internals/system-info/ps-tree.js';
+import { psTree } from '../../../packages/core/src/system-info/ps-tree.js';
const scripts = {
parent: path.join(import.meta.dirname, 'fixtures', 'parent.js'),
diff --git a/test/core/autoscaling/runtime.test.ts b/test/core/autoscaling/runtime.test.ts
new file mode 100644
index 000000000..bc5269dff
--- /dev/null
+++ b/test/core/autoscaling/runtime.test.ts
@@ -0,0 +1,145 @@
+import asyncFs from 'node:fs/promises';
+
+import { getCgroupsVersion, isDocker, isLambda } from '../../../packages/core/src/system-info/runtime.js';
+
+describe('isDocker()', () => {
+ test('works for dockerenv && cgroup', async () => {
+ const statMock = vitest.spyOn(asyncFs, 'stat').mockImplementationOnce(async () => null as any);
+ const readMock = vitest
+ .spyOn(asyncFs, 'readFile')
+ .mockImplementationOnce(async () => Promise.resolve('something ... docker ... something'));
+
+ const is = await isDocker(true);
+
+ expect(is).toBe(true);
+ });
+
+ test('works for dockerenv', async () => {
+ const statMock = vitest.spyOn(asyncFs, 'stat').mockImplementationOnce(async () => null as any);
+ const readMock = vitest
+ .spyOn(asyncFs, 'readFile')
+ .mockImplementationOnce(async () => Promise.resolve('something ... ... something'));
+
+ const is = await isDocker(true);
+
+ expect(is).toBe(true);
+ });
+
+ test('works for cgroup', async () => {
+ const statMock = vitest
+ .spyOn(asyncFs, 'stat')
+ .mockImplementationOnce(async () => Promise.reject(new Error('no.')));
+ const readMock = vitest
+ .spyOn(asyncFs, 'readFile')
+ .mockImplementationOnce(async () => Promise.resolve('something ... docker ... something'));
+
+ const is = await isDocker(true);
+
+ expect(is).toBe(true);
+ });
+
+ test('works for nothing', async () => {
+ const statMock = vitest
+ .spyOn(asyncFs, 'stat')
+ .mockImplementationOnce(async () => Promise.reject(new Error('no.')));
+ const readMock = vitest
+ .spyOn(asyncFs, 'readFile')
+ .mockImplementationOnce(async () => Promise.resolve('something ... ... something'));
+
+ const is = await isDocker(true);
+
+ expect(is).toBe(false);
+ });
+});
+
+describe('isContainerized()', () => {
+ afterEach(() => {
+ delete process.env.KUBERNETES_SERVICE_HOST;
+ delete process.env.CRAWLEE_CONTAINERIZED;
+ // resets the `isContainerizedResult` module scoped variable
+ vi.resetModules();
+ });
+
+ test('returns true when a "/.dockerenv" file exists', async () => {
+ // @ts-ignore flaky linting of dynamic import. Some environments throw ts(2307), others not.
+ const { isContainerized } = await import('../../../packages/core/src/system-info/runtime.js');
+ const statMock = vitest.spyOn(asyncFs, 'stat').mockImplementationOnce(async () => null as any);
+ const result = await isContainerized();
+ expect(result).toBe(true);
+ });
+
+ test('returns false when isLambda is true', async () => {
+ // @ts-ignore flaky linting of dynamic import. Some environments throw ts(2307), others not.
+ const utils = await import('../../../packages/core/src/system-info/runtime.js');
+ const lambdaMock = vitest.spyOn(utils, 'isLambda').mockReturnValue(true);
+ const result = await utils.isContainerized();
+ expect(result).toBe(false);
+ });
+
+ test('returns true when a "/proc/stat/cgroup" file contains "docker"', async () => {
+ // @ts-ignore flaky linting of dynamic import. Some environments throw ts(2307), others not.
+ const { isContainerized } = await import('../../../packages/core/src/system-info/runtime.js');
+ const readFileMock = vitest
+ .spyOn(asyncFs, 'readFile')
+ .mockResolvedValue("'something ... docker ... something'");
+ const result = await isContainerized();
+ expect(result).toBe(true);
+ });
+
+ test('returns true when KUBERNETES_SERVICE_HOST environment variable is set', async () => {
+ // @ts-ignore flaky linting of dynamic import. Some environments throw ts(2307), others not.
+ const { isContainerized } = await import('../../../packages/core/src/system-info/runtime.js');
+ process.env.KUBERNETES_SERVICE_HOST = 'some-host';
+ const result = await isContainerized();
+ expect(result).toBe(true);
+ });
+
+ test('returns false when no other conditions are met', async () => {
+ // @ts-ignore flaky linting of dynamic import. Some environments throw ts(2307), others not.
+ const { isContainerized } = await import('../../../packages/core/src/system-info/runtime.js');
+ const result = await isContainerized();
+ expect(result).toBe(false);
+ });
+});
+
+describe('isLambda()', () => {
+ afterEach(() => {
+ delete process.env.AWS_LAMBDA_FUNCTION_MEMORY_SIZE;
+ });
+
+ test('returns true when AWS_LAMBDA_FUNCTION_MEMORY_SIZE environment variable is set', () => {
+ process.env.AWS_LAMBDA_FUNCTION_MEMORY_SIZE = '12345';
+ const result = isLambda();
+ expect(result).toBe(true);
+ });
+});
+
+describe('getCgroupsVersion()', () => {
+ // Reset the module cache so that _cgroupsVersion is not retained across tests.
+ beforeEach(() => {
+ vitest.resetModules();
+ });
+
+ test('returns null when access to /sys/fs/cgroup/ fails', async () => {
+ vitest.spyOn(asyncFs, 'access').mockRejectedValue(new Error('not found'));
+ const version = await getCgroupsVersion(true);
+ expect(version).toBe(null);
+ });
+
+ test('returns V2 when access to /sys/fs/cgroup/memory/ fails', async () => {
+ vitest.spyOn(asyncFs, 'access').mockImplementation(async (path) => {
+ if (path === '/sys/fs/cgroup/') {
+ return;
+ }
+ throw new Error('not found');
+ });
+ const version = await getCgroupsVersion(true);
+ expect(version).toBe('V2');
+ });
+
+ test('returns V1 when access to /sys/fs/cgroup/memory/ succeeds', async () => {
+ vitest.spyOn(asyncFs, 'access').mockResolvedValue();
+ const version = await getCgroupsVersion(true);
+ expect(version).toBe('V1');
+ });
+});
diff --git a/test/core/autoscaling/snapshotter.test.ts b/test/core/autoscaling/snapshotter.test.ts
index 243048e29..6f805c164 100644
--- a/test/core/autoscaling/snapshotter.test.ts
+++ b/test/core/autoscaling/snapshotter.test.ts
@@ -1,8 +1,8 @@
import os from 'node:os';
import { Configuration, EventType, LocalEventManager, serviceLocator, Snapshotter } from '@crawlee/core';
-import type { MemoryInfo } from '@crawlee/utils';
-import * as utils from '@crawlee/utils';
+import type { MemoryInfo } from '../../../packages/core/src/system-info/memory-info.js';
+import * as utils from '../../../packages/core/src/system-info/memory-info.js';
import { sleep } from '@crawlee/utils';
import log from '@apify/log';
diff --git a/test/utils/general.test.ts b/test/utils/general.test.ts
index 6b7212790..1267f76f9 100644
--- a/test/utils/general.test.ts
+++ b/test/utils/general.test.ts
@@ -1,148 +1,4 @@
-import asyncFs from 'node:fs/promises';
-
-import { getCgroupsVersion, isDocker, isLambda, sleep, snakeCaseToCamelCase, weightedAvg } from '@crawlee/utils';
-
-describe('isDocker()', () => {
- test('works for dockerenv && cgroup', async () => {
- const statMock = vitest.spyOn(asyncFs, 'stat').mockImplementationOnce(async () => null as any);
- const readMock = vitest
- .spyOn(asyncFs, 'readFile')
- .mockImplementationOnce(async () => Promise.resolve('something ... docker ... something'));
-
- const is = await isDocker(true);
-
- expect(is).toBe(true);
- });
-
- test('works for dockerenv', async () => {
- const statMock = vitest.spyOn(asyncFs, 'stat').mockImplementationOnce(async () => null as any);
- const readMock = vitest
- .spyOn(asyncFs, 'readFile')
- .mockImplementationOnce(async () => Promise.resolve('something ... ... something'));
-
- const is = await isDocker(true);
-
- expect(is).toBe(true);
- });
-
- test('works for cgroup', async () => {
- const statMock = vitest
- .spyOn(asyncFs, 'stat')
- .mockImplementationOnce(async () => Promise.reject(new Error('no.')));
- const readMock = vitest
- .spyOn(asyncFs, 'readFile')
- .mockImplementationOnce(async () => Promise.resolve('something ... docker ... something'));
-
- const is = await isDocker(true);
-
- expect(is).toBe(true);
- });
-
- test('works for nothing', async () => {
- const statMock = vitest
- .spyOn(asyncFs, 'stat')
- .mockImplementationOnce(async () => Promise.reject(new Error('no.')));
- const readMock = vitest
- .spyOn(asyncFs, 'readFile')
- .mockImplementationOnce(async () => Promise.resolve('something ... ... something'));
-
- const is = await isDocker(true);
-
- expect(is).toBe(false);
- });
-});
-
-describe('isContainerized()', () => {
- afterEach(() => {
- delete process.env.KUBERNETES_SERVICE_HOST;
- delete process.env.CRAWLEE_CONTAINERIZED;
- // resets the `isContainerizedResult` module scoped variable
- vi.resetModules();
- });
-
- test('returns true when a "/.dockerenv" file exists', async () => {
- // @ts-ignore flaky linting of dynamic import. Some environments throw ts(2307), others not.
- const { isContainerized } = await import('@crawlee/utils');
- const statMock = vitest.spyOn(asyncFs, 'stat').mockImplementationOnce(async () => null as any);
- const result = await isContainerized();
- expect(result).toBe(true);
- });
-
- test('returns false when isLambda is true', async () => {
- // @ts-ignore flaky linting of dynamic import. Some environments throw ts(2307), others not.
- const utils = await import('@crawlee/utils');
- const lambdaMock = vitest.spyOn(utils, 'isLambda').mockReturnValue(true);
- const result = await utils.isContainerized();
- expect(result).toBe(false);
- });
-
- test('returns true when a "/proc/stat/cgroup" file contains "docker"', async () => {
- // @ts-ignore flaky linting of dynamic import. Some environments throw ts(2307), others not.
- const { isContainerized } = await import('@crawlee/utils');
- const readFileMock = vitest
- .spyOn(asyncFs, 'readFile')
- .mockResolvedValue("'something ... docker ... something'");
- const result = await isContainerized();
- expect(result).toBe(true);
- });
-
- test('returns true when KUBERNETES_SERVICE_HOST environment variable is set', async () => {
- // @ts-ignore flaky linting of dynamic import. Some environments throw ts(2307), others not.
- const { isContainerized } = await import('@crawlee/utils');
- process.env.KUBERNETES_SERVICE_HOST = 'some-host';
- const result = await isContainerized();
- expect(result).toBe(true);
- });
-
- test('returns false when no other conditions are met', async () => {
- // @ts-ignore flaky linting of dynamic import. Some environments throw ts(2307), others not.
- const { isContainerized } = await import('@crawlee/utils');
- const result = await isContainerized();
- expect(result).toBe(false);
- });
-});
-
-describe('isLambda()', () => {
- afterEach(() => {
- delete process.env.AWS_LAMBDA_FUNCTION_MEMORY_SIZE;
- });
-
- test('returns true when AWS_LAMBDA_FUNCTION_MEMORY_SIZE environment variable is set', () => {
- process.env.AWS_LAMBDA_FUNCTION_MEMORY_SIZE = '12345';
- const result = isLambda();
- expect(result).toBe(true);
- });
-});
-
-describe('getCgroupsVersion()', () => {
- // Reset the module cache so that _cgroupsVersion is not retained across tests.
- beforeEach(() => {
- vitest.resetModules();
- });
-
- test('returns null when access to /sys/fs/cgroup/ fails', async () => {
- vitest.spyOn(asyncFs, 'access').mockRejectedValue(new Error('not found'));
- const version = await getCgroupsVersion(true);
- expect(version).toBe(null);
- });
-
- test('returns V2 when access to /sys/fs/cgroup/memory/ fails', async () => {
- vitest.spyOn(asyncFs, 'access').mockImplementation(async (path) => {
- if (path === '/sys/fs/cgroup/') {
- return;
- }
- throw new Error('not found');
- });
- const version = await getCgroupsVersion(true);
- expect(version).toBe('V2');
- });
-
- test('returns V1 when access to /sys/fs/cgroup/memory/ succeeds', async () => {
- vitest.spyOn(asyncFs, 'access').mockResolvedValue();
- const version = await getCgroupsVersion(true);
- expect(version).toBe('V1');
- });
-});
+import { sleep, snakeCaseToCamelCase, weightedAvg } from '@crawlee/utils';
describe('weightedAvg()', () => {
test('works', () => {