Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat: move autoscaling helpers to `@crawlee/core` (#3835)
Jindřich Bäre614738e6544
diff --git a/docs/public-api/crawlee-utils.api.md b/docs/public-api/crawlee-utils.api.mdindex adb9389f2..d7d10e194 100644--- a/docs/public-api/crawlee-utils.api.md+++ b/docs/public-api/crawlee-utils.api.md@@ -42,14 +42,6 @@ export const CLOUDFLARE_RETRY_CSS_SELECTORS: string[];export { Constructor }-// @public-export interface CpuSample {- // (undocumented)- containerUsage: number;- // (undocumented)- systemUsage: number;-}-// @publicexport function createRequestDebugInfo(request: Request_2, response?: IncomingMessage | Partial<BrowserResponseLike>, additionalFields?: Dictionary): Dictionary;@@ -121,21 +113,6 @@ const FACEBOOK_REGEX: RegExp;// @publicconst FACEBOOK_REGEX_GLOBAL: RegExp;-// @public-export function getCgroupsVersion(forceReset?: boolean): Promise<"V1" | "V2" | null>;--// @internal-export function getCurrentCpuTicksV2(options?: {- containerized?: boolean;- logger?: CrawleeLogger;-}): Promise<number>;--// @internal-export function getMemoryInfo(options?: {- containerized?: boolean;- logger?: CrawleeLogger;-}): Promise<MemoryInfo>;-// @public (undocumented)export function getObjectType(value: unknown): string;@@ -157,18 +134,9 @@ export function isAsyncIterable<T>(value: unknown): value is AsyncIterable<T>;// @publicexport function isBuffer(value: unknown): value is Buffer | ArrayBuffer | ArrayBufferView;-// @public-export function isContainerized(): Promise<boolean>;--// @public-export function isDocker(forceReset?: boolean): Promise<boolean>;-// @internalexport function isIterable<T>(value: unknown): value is Iterable<T>;-// @public (undocumented)-export function isLambda(): boolean;-// @publicexport function isStream(value: unknown): value is NodeJS.ReadableStream | ReadableStream;@@ -181,15 +149,6 @@ const LINKEDIN_REGEX: RegExp;// @publicconst LINKEDIN_REGEX_GLOBAL: RegExp;-// @public-export interface MemoryInfo {- childProcessesBytes: number;- freeBytes: number;- mainProcessBytes: number;- totalBytes: number;- usedBytes: number;-}-// @publicexport function mergeAsyncIterables<T>(...iterables: AsyncIterable<T>[]): AsyncIterable<T>;diff --git a/packages/core/src/autoscaling/memory_load_signal.ts b/packages/core/src/autoscaling/memory_load_signal.tsindex b88463b88..1e8fca502 100644--- a/packages/core/src/autoscaling/memory_load_signal.ts+++ b/packages/core/src/autoscaling/memory_load_signal.ts@@ -1,10 +1,10 @@-import { getMemoryInfo, isContainerized } from '@crawlee/utils';-import type { Configuration } from '../configuration.js';import type { EventManager } from '../events/event_manager.js';import { EventType } from '../events/event_manager.js';import type { CrawleeLogger } from '../log.js';import { serviceLocator } from '../service_locator.js';+import { getMemoryInfo } from '../system-info/memory-info.js';+import { isContainerized } from '../system-info/runtime.js';import type { LoadSignal, LoadSnapshot } from './load_signal.js';import { SnapshotStore } from './load_signal.js';import type { SystemInfo } from './system_status.js';diff --git a/packages/core/src/events/local_event_manager.ts b/packages/core/src/events/local_event_manager.tsindex d69211bea..7bb2c9ffb 100644--- a/packages/core/src/events/local_event_manager.ts+++ b/packages/core/src/events/local_event_manager.ts@@ -76,7 +76,7 @@ export class LocalEventManager extends EventManager {* @internal*/async isContainerizedWrapper() {- const { isContainerized } = await import('@crawlee/utils');+ const { isContainerized } = await import('../system-info/runtime.js');return serviceLocator.getConfiguration().containerized ?? (await isContainerized());}@@ -92,7 +92,7 @@ export class LocalEventManager extends EventManager {}private async createCpuInfo(options: { maxUsedCpuRatio: number }) {- const { getCurrentCpuTicksV2 } = await import('@crawlee/utils');+ const { getCurrentCpuTicksV2 } = await import('../system-info/cpu-info.js');const usedCpuRatio = await getCurrentCpuTicksV2({containerized: await this.isContainerizedWrapper(),logger: serviceLocator.getLogger(),@@ -117,7 +117,7 @@ export class LocalEventManager extends EventManager {}private async getMemoryInfo() {- const { getMemoryInfo } = await import('@crawlee/utils');+ const { getMemoryInfo } = await import('../system-info/memory-info.js');return getMemoryInfo({containerized: await this.isContainerizedWrapper(),logger: serviceLocator.getLogger(),diff --git a/packages/utils/src/internals/system-info/cpu-info.ts b/packages/core/src/system-info/cpu-info.tssimilarity index 99%rename from packages/utils/src/internals/system-info/cpu-info.tsrename to packages/core/src/system-info/cpu-info.tsindex a2f3f5289..c8e3fbaa0 100644--- a/packages/utils/src/internals/system-info/cpu-info.ts+++ b/packages/core/src/system-info/cpu-info.ts@@ -4,7 +4,7 @@ import os from 'node:os';import type { CrawleeLogger } from '@crawlee/types';-import { getCgroupsVersion } from '../general.js';+import { getCgroupsVersion } from './runtime.js';const CPU_FILE_PATHS = {STAT: {diff --git a/packages/utils/src/internals/system-info/memory-info.ts b/packages/core/src/system-info/memory-info.tssimilarity index 98%rename from packages/utils/src/internals/system-info/memory-info.tsrename to packages/core/src/system-info/memory-info.tsindex bbbf24f23..a1c7326b9 100644--- a/packages/utils/src/internals/system-info/memory-info.ts+++ b/packages/core/src/system-info/memory-info.ts@@ -4,7 +4,7 @@ import { freemem, totalmem } from 'node:os';import type { CrawleeLogger } from '@crawlee/types';-import { getCgroupsVersion, isLambda } from '../general.js';+import { getCgroupsVersion, isLambda } from './runtime.js';import { psTree } from './ps-tree.js';const MEMORY_FILE_PATHS = {diff --git a/packages/utils/src/internals/system-info/ps-tree.ts b/packages/core/src/system-info/ps-tree.tssimilarity index 100%rename from packages/utils/src/internals/system-info/ps-tree.tsrename to packages/core/src/system-info/ps-tree.tsdiff --git a/packages/core/src/system-info/runtime.ts b/packages/core/src/system-info/runtime.tsnew file mode 100644index 000000000..63dfdefe4--- /dev/null+++ b/packages/core/src/system-info/runtime.ts@@ -0,0 +1,93 @@+import fs from 'node:fs/promises';++let isDockerPromiseCache: Promise<boolean> | undefined;++async function createIsDockerPromise() {+ const promise1 = fs+ .stat('/.dockerenv')+ .then(() => true)+ .catch(() => false);++ const promise2 = fs+ .readFile('/proc/self/cgroup', 'utf8')+ .then((content) => content.includes('docker'))+ .catch(() => false);++ const [result1, result2] = await Promise.all([promise1, promise2]);++ return result1 || result2;+}++/**+ * Returns a `Promise` that resolves to true if the code is running in a Docker container.+ */+export async function isDocker(forceReset?: boolean): Promise<boolean> {+ // Parameter forceReset is just internal for unit tests.+ if (!isDockerPromiseCache || forceReset) isDockerPromiseCache = createIsDockerPromise();++ return isDockerPromiseCache;+}++let isContainerizedResult: boolean | undefined;++/**+ * Detects if crawlee is running in a containerized environment.+ */+export async function isContainerized() {+ // Value is very unlikley to change. Cache the result after the first execution.+ if (isContainerizedResult !== undefined) {+ return isContainerizedResult;+ }++ // return false if running in aws lambda+ if (isLambda()) {+ isContainerizedResult = false;+ return isContainerizedResult;+ }++ const dockerenvCheck = fs+ .stat('/.dockerenv')+ .then(() => true)+ .catch(() => false);++ const cgroupCheck = fs+ .readFile('/proc/self/cgroup', 'utf8')+ .then((content) => content.includes('docker'))+ .catch(() => false);++ const [dockerenvResult, cgroupResult] = await Promise.all([dockerenvCheck, cgroupCheck]);++ isContainerizedResult = dockerenvResult || cgroupResult || !!process.env.KUBERNETES_SERVICE_HOST;+ return isContainerizedResult;+}++export function isLambda() {+ return !!process.env.AWS_LAMBDA_FUNCTION_MEMORY_SIZE;+}++let _cgroupsVersion: null | 'V1' | 'V2';+/**+ * gets the cgroup version by checking for a file at /sys/fs/cgroup/memory+ * @returns "V1" or "V2" for the version of cgroup or null if cgroup is not found.+ */+export async function getCgroupsVersion(forceReset?: boolean) {+ // Parameter forceReset is just internal for unit tests.+ if (_cgroupsVersion !== undefined && !forceReset) {+ return _cgroupsVersion;+ }+ try {+ // If this directory does not exists, cgroups are not available+ await fs.access('/sys/fs/cgroup/');+ } catch (e) {+ _cgroupsVersion = null;+ return null;+ }+ _cgroupsVersion = 'V1';+ try {+ // If this directory does not exists, assume the container is using cgroups V2+ await fs.access('/sys/fs/cgroup/memory/');+ } catch (e) {+ _cgroupsVersion = 'V2';+ }+ return _cgroupsVersion;+}diff --git a/packages/utils/src/index.ts b/packages/utils/src/index.tsindex eadf94dc5..517c14e6c 100644--- a/packages/utils/src/index.ts+++ b/packages/utils/src/index.ts@@ -14,9 +14,4 @@ export * from './internals/robots.js';export * from './internals/sitemap.js';export * from './internals/url.js';-export type { CpuSample } from './internals/system-info/cpu-info.js';-export { getCurrentCpuTicksV2 } from './internals/system-info/cpu-info.js';-export type { MemoryInfo } from './internals/system-info/memory-info.js';-export { getMemoryInfo } from './internals/system-info/memory-info.js';-export type { Dictionary, Awaitable, Constructor } from '@crawlee/types';diff --git a/packages/utils/src/internals/general.ts b/packages/utils/src/internals/general.tsindex 3158e8899..7d59c1b3b 100644--- a/packages/utils/src/internals/general.ts+++ b/packages/utils/src/internals/general.ts@@ -1,4 +1,3 @@-import fs from 'node:fs/promises';import { setTimeout } from 'node:timers/promises';/**@@ -15,98 +14,6 @@ export const URL_NO_COMMAS_REGEX =export const URL_WITH_COMMAS_REGEX =/https?:\/\/(www\.)?([\p{L}0-9]|[\p{L}0-9][-\p{L}0-9@:%._+~#=]{0,254}[\p{L}0-9])\.[a-z]{2,63}(:\d{1,5})?(\/[-\p{L}0-9@:%_+,.~#?&/=()'*]*)?/giu;-let isDockerPromiseCache: Promise<boolean> | undefined;--async function createIsDockerPromise() {- const promise1 = fs- .stat('/.dockerenv')- .then(() => true)- .catch(() => false);-- const promise2 = fs- .readFile('/proc/self/cgroup', 'utf8')- .then((content) => content.includes('docker'))- .catch(() => false);-- const [result1, result2] = await Promise.all([promise1, promise2]);-- return result1 || result2;-}--/**- * Returns a `Promise` that resolves to true if the code is running in a Docker container.- */-export async function isDocker(forceReset?: boolean): Promise<boolean> {- // Parameter forceReset is just internal for unit tests.- if (!isDockerPromiseCache || forceReset) isDockerPromiseCache = createIsDockerPromise();-- return isDockerPromiseCache;-}--let isContainerizedResult: boolean | undefined;--/**- * Detects if crawlee is running in a containerized environment.- */-export async function isContainerized() {- // Value is very unlikley to change. Cache the result after the first execution.- if (isContainerizedResult !== undefined) {- return isContainerizedResult;- }-- // return false if running in aws lambda- if (isLambda()) {- isContainerizedResult = false;- return isContainerizedResult;- }-- const dockerenvCheck = fs- .stat('/.dockerenv')- .then(() => true)- .catch(() => false);-- const cgroupCheck = fs- .readFile('/proc/self/cgroup', 'utf8')- .then((content) => content.includes('docker'))- .catch(() => false);-- const [dockerenvResult, cgroupResult] = await Promise.all([dockerenvCheck, cgroupCheck]);-- isContainerizedResult = dockerenvResult || cgroupResult || !!process.env.KUBERNETES_SERVICE_HOST;- return isContainerizedResult;-}--export function isLambda() {- return !!process.env.AWS_LAMBDA_FUNCTION_MEMORY_SIZE;-}--let _cgroupsVersion: null | 'V1' | 'V2';-/**- * gets the cgroup version by checking for a file at /sys/fs/cgroup/memory- * @returns "V1" or "V2" for the version of cgroup or null if cgroup is not found.- */-export async function getCgroupsVersion(forceReset?: boolean) {- // Parameter forceReset is just internal for unit tests.- if (_cgroupsVersion !== undefined && !forceReset) {- return _cgroupsVersion;- }- try {- // If this directory does not exists, cgroups are not available- await fs.access('/sys/fs/cgroup/');- } catch (e) {- _cgroupsVersion = null;- return null;- }- _cgroupsVersion = 'V1';- try {- // If this directory does not exists, assume the container is using cgroups V2- await fs.access('/sys/fs/cgroup/memory/');- } catch (e) {- _cgroupsVersion = 'V2';- }- return _cgroupsVersion;-}-/*** Computes a weighted average of an array of numbers, complemented by an array of weights.* @ignorediff --git a/test/utils/cpu-infoV2.test.ts b/test/core/autoscaling/cpu-infoV2.test.tssimilarity index 97%rename from test/utils/cpu-infoV2.test.tsrename to test/core/autoscaling/cpu-infoV2.test.tsindex 8786e3c3e..37a198be5 100644--- a/test/utils/cpu-infoV2.test.ts+++ b/test/core/autoscaling/cpu-infoV2.test.ts@@ -1,7 +1,7 @@import { readFile } from 'node:fs/promises';import os from 'node:os';-import { getCgroupsVersion } from '@crawlee/utils';+import { getCgroupsVersion } from '../../../packages/core/src/system-info/runtime.js';import {getContainerCpuUsage,@@ -11,10 +11,10 @@ import {getCurrentCpuTicksV2,getSystemCpuUsage,sampleCpuUsage,-} from '../../packages/utils/src/internals/system-info/cpu-info.js';+} from '../../../packages/core/src/system-info/cpu-info.js';-vitest.mock('@crawlee/utils/src/internals/general', async (importActual) => {- const original: typeof import('@crawlee/utils') = await importActual();+vitest.mock('../../../packages/core/src/system-info/runtime.js', async (importActual) => {+ const original: typeof import('../../../packages/core/src/system-info/runtime.js') = await importActual();return {...original,getCgroupsVersion: vitest.fn(),diff --git a/test/utils/fixtures/child.js b/test/core/autoscaling/fixtures/child.jssimilarity index 100%rename from test/utils/fixtures/child.jsrename to test/core/autoscaling/fixtures/child.jsdiff --git a/test/utils/fixtures/parent.js b/test/core/autoscaling/fixtures/parent.jssimilarity index 59%rename from test/utils/fixtures/parent.jsrename to test/core/autoscaling/fixtures/parent.jsindex 19e0c7f5b..e77f6f701 100644--- a/test/utils/fixtures/parent.js+++ b/test/core/autoscaling/fixtures/parent.js@@ -1,5 +1,5 @@import { exec } from 'node:child_process';for (let count = 1; count < 10; count++) {- exec('node ./test/utils/fixtures/child.js');+ exec('node ./test/core/autoscaling/fixtures/child.js');}diff --git a/test/utils/memory-infoV2.test.ts b/test/core/autoscaling/memory-infoV2.test.tssimilarity index 96%rename from test/utils/memory-infoV2.test.tsrename to test/core/autoscaling/memory-infoV2.test.tsindex 9094cd4ee..52fc0961e 100644--- a/test/utils/memory-infoV2.test.ts+++ b/test/core/autoscaling/memory-infoV2.test.ts@@ -2,7 +2,8 @@ import { access, readFile } from 'node:fs/promises';import { freemem, totalmem } from 'node:os';import { launchPuppeteer } from '@crawlee/puppeteer';-import { getCgroupsVersion, getMemoryInfo } from '@crawlee/utils';+import { getMemoryInfo } from '../../../packages/core/src/system-info/memory-info.js';+import { getCgroupsVersion } from '../../../packages/core/src/system-info/runtime.js';vitest.mock('node:os', async (importActual) => {const originalOs: typeof import('node:os') = await importActual();@@ -13,8 +14,8 @@ vitest.mock('node:os', async (importActual) => {};});-vitest.mock('@crawlee/utils/src/internals/general', async (importActual) => {- const original: typeof import('@crawlee/utils') = await importActual();+vitest.mock('../../../packages/core/src/system-info/runtime.js', async (importActual) => {+ const original: typeof import('../../../packages/core/src/system-info/runtime.js') = await importActual();return {...original,diff --git a/test/utils/psTree.test.ts b/test/core/autoscaling/psTree.test.tssimilarity index 96%rename from test/utils/psTree.test.tsrename to test/core/autoscaling/psTree.test.tsindex 1f24d9088..ad52da2d1 100644--- a/test/utils/psTree.test.ts+++ b/test/core/autoscaling/psTree.test.ts@@ -1,7 +1,7 @@import { exec } from 'node:child_process';import path from 'node:path';-import { psTree } from '../../packages/utils/src/internals/system-info/ps-tree.js';+import { psTree } from '../../../packages/core/src/system-info/ps-tree.js';const scripts = {parent: path.join(import.meta.dirname, 'fixtures', 'parent.js'),diff --git a/test/core/autoscaling/runtime.test.ts b/test/core/autoscaling/runtime.test.tsnew file mode 100644index 000000000..bc5269dff--- /dev/null+++ b/test/core/autoscaling/runtime.test.ts@@ -0,0 +1,145 @@+import asyncFs from 'node:fs/promises';++import { getCgroupsVersion, isDocker, isLambda } from '../../../packages/core/src/system-info/runtime.js';++describe('isDocker()', () => {+ test('works for dockerenv && cgroup', async () => {+ const statMock = vitest.spyOn(asyncFs, 'stat').mockImplementationOnce(async () => null as any);+ const readMock = vitest+ .spyOn(asyncFs, 'readFile')+ .mockImplementationOnce(async () => Promise.resolve('something ... docker ... something'));++ const is = await isDocker(true);++ expect(is).toBe(true);+ });++ test('works for dockerenv', async () => {+ const statMock = vitest.spyOn(asyncFs, 'stat').mockImplementationOnce(async () => null as any);+ const readMock = vitest+ .spyOn(asyncFs, 'readFile')+ .mockImplementationOnce(async () => Promise.resolve('something ... ... something'));++ const is = await isDocker(true);++ expect(is).toBe(true);+ });++ test('works for cgroup', async () => {+ const statMock = vitest+ .spyOn(asyncFs, 'stat')+ .mockImplementationOnce(async () => Promise.reject(new Error('no.')));+ const readMock = vitest+ .spyOn(asyncFs, 'readFile')+ .mockImplementationOnce(async () => Promise.resolve('something ... docker ... something'));++ const is = await isDocker(true);++ expect(is).toBe(true);+ });++ test('works for nothing', async () => {+ const statMock = vitest+ .spyOn(asyncFs, 'stat')+ .mockImplementationOnce(async () => Promise.reject(new Error('no.')));+ const readMock = vitest+ .spyOn(asyncFs, 'readFile')+ .mockImplementationOnce(async () => Promise.resolve('something ... ... something'));++ const is = await isDocker(true);++ expect(is).toBe(false);+ });+});++describe('isContainerized()', () => {+ afterEach(() => {+ delete process.env.KUBERNETES_SERVICE_HOST;+ delete process.env.CRAWLEE_CONTAINERIZED;+ // resets the `isContainerizedResult` module scoped variable+ vi.resetModules();+ });++ test('returns true when a "/.dockerenv" file exists', async () => {+ // @ts-ignore flaky linting of dynamic import. Some environments throw ts(2307), others not.+ const { isContainerized } = await import('../../../packages/core/src/system-info/runtime.js');+ const statMock = vitest.spyOn(asyncFs, 'stat').mockImplementationOnce(async () => null as any);+ const result = await isContainerized();+ expect(result).toBe(true);+ });++ test('returns false when isLambda is true', async () => {+ // @ts-ignore flaky linting of dynamic import. Some environments throw ts(2307), others not.+ const utils = await import('../../../packages/core/src/system-info/runtime.js');+ const lambdaMock = vitest.spyOn(utils, 'isLambda').mockReturnValue(true);+ const result = await utils.isContainerized();+ expect(result).toBe(false);+ });++ test('returns true when a "/proc/stat/cgroup" file contains "docker"', async () => {+ // @ts-ignore flaky linting of dynamic import. Some environments throw ts(2307), others not.+ const { isContainerized } = await import('../../../packages/core/src/system-info/runtime.js');+ const readFileMock = vitest+ .spyOn(asyncFs, 'readFile')+ .mockResolvedValue("'something ... docker ... something'");+ const result = await isContainerized();+ expect(result).toBe(true);+ });++ test('returns true when KUBERNETES_SERVICE_HOST environment variable is set', async () => {+ // @ts-ignore flaky linting of dynamic import. Some environments throw ts(2307), others not.+ const { isContainerized } = await import('../../../packages/core/src/system-info/runtime.js');+ process.env.KUBERNETES_SERVICE_HOST = 'some-host';+ const result = await isContainerized();+ expect(result).toBe(true);+ });++ test('returns false when no other conditions are met', async () => {+ // @ts-ignore flaky linting of dynamic import. Some environments throw ts(2307), others not.+ const { isContainerized } = await import('../../../packages/core/src/system-info/runtime.js');+ const result = await isContainerized();+ expect(result).toBe(false);+ });+});++describe('isLambda()', () => {+ afterEach(() => {+ delete process.env.AWS_LAMBDA_FUNCTION_MEMORY_SIZE;+ });++ test('returns true when AWS_LAMBDA_FUNCTION_MEMORY_SIZE environment variable is set', () => {+ process.env.AWS_LAMBDA_FUNCTION_MEMORY_SIZE = '12345';+ const result = isLambda();+ expect(result).toBe(true);+ });+});++describe('getCgroupsVersion()', () => {+ // Reset the module cache so that _cgroupsVersion is not retained across tests.+ beforeEach(() => {+ vitest.resetModules();+ });++ test('returns null when access to /sys/fs/cgroup/ fails', async () => {+ vitest.spyOn(asyncFs, 'access').mockRejectedValue(new Error('not found'));+ const version = await getCgroupsVersion(true);+ expect(version).toBe(null);+ });++ test('returns V2 when access to /sys/fs/cgroup/memory/ fails', async () => {+ vitest.spyOn(asyncFs, 'access').mockImplementation(async (path) => {+ if (path === '/sys/fs/cgroup/') {+ return;+ }+ throw new Error('not found');+ });+ const version = await getCgroupsVersion(true);+ expect(version).toBe('V2');+ });++ test('returns V1 when access to /sys/fs/cgroup/memory/ succeeds', async () => {+ vitest.spyOn(asyncFs, 'access').mockResolvedValue();+ const version = await getCgroupsVersion(true);+ expect(version).toBe('V1');+ });+});diff --git a/test/core/autoscaling/snapshotter.test.ts b/test/core/autoscaling/snapshotter.test.tsindex 243048e29..6f805c164 100644--- a/test/core/autoscaling/snapshotter.test.ts+++ b/test/core/autoscaling/snapshotter.test.ts@@ -1,8 +1,8 @@import os from 'node:os';import { Configuration, EventType, LocalEventManager, serviceLocator, Snapshotter } from '@crawlee/core';-import type { MemoryInfo } from '@crawlee/utils';-import * as utils from '@crawlee/utils';+import type { MemoryInfo } from '../../../packages/core/src/system-info/memory-info.js';+import * as utils from '../../../packages/core/src/system-info/memory-info.js';import { sleep } from '@crawlee/utils';import log from '@apify/log';diff --git a/test/utils/general.test.ts b/test/utils/general.test.tsindex 6b7212790..1267f76f9 100644--- a/test/utils/general.test.ts+++ b/test/utils/general.test.ts@@ -1,148 +1,4 @@-import asyncFs from 'node:fs/promises';--import { getCgroupsVersion, isDocker, isLambda, sleep, snakeCaseToCamelCase, weightedAvg } from '@crawlee/utils';--describe('isDocker()', () => {- test('works for dockerenv && cgroup', async () => {- const statMock = vitest.spyOn(asyncFs, 'stat').mockImplementationOnce(async () => null as any);- const readMock = vitest- .spyOn(asyncFs, 'readFile')- .mockImplementationOnce(async () => Promise.resolve('something ... docker ... something'));-- const is = await isDocker(true);-- expect(is).toBe(true);- });-- test('works for dockerenv', async () => {- const statMock = vitest.spyOn(asyncFs, 'stat').mockImplementationOnce(async () => null as any);- const readMock = vitest- .spyOn(asyncFs, 'readFile')- .mockImplementationOnce(async () => Promise.resolve('something ... ... something'));-- const is = await isDocker(true);-- expect(is).toBe(true);- });-- test('works for cgroup', async () => {- const statMock = vitest- .spyOn(asyncFs, 'stat')- .mockImplementationOnce(async () => Promise.reject(new Error('no.')));- const readMock = vitest- .spyOn(asyncFs, 'readFile')- .mockImplementationOnce(async () => Promise.resolve('something ... docker ... something'));-- const is = await isDocker(true);-- expect(is).toBe(true);- });-- test('works for nothing', async () => {- const statMock = vitest- .spyOn(asyncFs, 'stat')- .mockImplementationOnce(async () => Promise.reject(new Error('no.')));- const readMock = vitest- .spyOn(asyncFs, 'readFile')- .mockImplementationOnce(async () => Promise.resolve('something ... ... something'));-- const is = await isDocker(true);-- expect(is).toBe(false);- });-});--describe('isContainerized()', () => {- afterEach(() => {- delete process.env.KUBERNETES_SERVICE_HOST;- delete process.env.CRAWLEE_CONTAINERIZED;- // resets the `isContainerizedResult` module scoped variable- vi.resetModules();- });-- test('returns true when a "/.dockerenv" file exists', async () => {- // @ts-ignore flaky linting of dynamic import. Some environments throw ts(2307), others not.- const { isContainerized } = await import('@crawlee/utils');- const statMock = vitest.spyOn(asyncFs, 'stat').mockImplementationOnce(async () => null as any);- const result = await isContainerized();- expect(result).toBe(true);- });-- test('returns false when isLambda is true', async () => {- // @ts-ignore flaky linting of dynamic import. Some environments throw ts(2307), others not.- const utils = await import('@crawlee/utils');- const lambdaMock = vitest.spyOn(utils, 'isLambda').mockReturnValue(true);- const result = await utils.isContainerized();- expect(result).toBe(false);- });-- test('returns true when a "/proc/stat/cgroup" file contains "docker"', async () => {- // @ts-ignore flaky linting of dynamic import. Some environments throw ts(2307), others not.- const { isContainerized } = await import('@crawlee/utils');- const readFileMock = vitest- .spyOn(asyncFs, 'readFile')- .mockResolvedValue("'something ... docker ... something'");- const result = await isContainerized();- expect(result).toBe(true);- });-- test('returns true when KUBERNETES_SERVICE_HOST environment variable is set', async () => {- // @ts-ignore flaky linting of dynamic import. Some environments throw ts(2307), others not.- const { isContainerized } = await import('@crawlee/utils');- process.env.KUBERNETES_SERVICE_HOST = 'some-host';- const result = await isContainerized();- expect(result).toBe(true);- });-- test('returns false when no other conditions are met', async () => {- // @ts-ignore flaky linting of dynamic import. Some environments throw ts(2307), others not.- const { isContainerized } = await import('@crawlee/utils');- const result = await isContainerized();- expect(result).toBe(false);- });-});--describe('isLambda()', () => {- afterEach(() => {- delete process.env.AWS_LAMBDA_FUNCTION_MEMORY_SIZE;- });-- test('returns true when AWS_LAMBDA_FUNCTION_MEMORY_SIZE environment variable is set', () => {- process.env.AWS_LAMBDA_FUNCTION_MEMORY_SIZE = '12345';- const result = isLambda();- expect(result).toBe(true);- });-});--describe('getCgroupsVersion()', () => {- // Reset the module cache so that _cgroupsVersion is not retained across tests.- beforeEach(() => {- vitest.resetModules();- });-- test('returns null when access to /sys/fs/cgroup/ fails', async () => {- vitest.spyOn(asyncFs, 'access').mockRejectedValue(new Error('not found'));- const version = await getCgroupsVersion(true);- expect(version).toBe(null);- });-- test('returns V2 when access to /sys/fs/cgroup/memory/ fails', async () => {- vitest.spyOn(asyncFs, 'access').mockImplementation(async (path) => {- if (path === '/sys/fs/cgroup/') {- return;- }- throw new Error('not found');- });- const version = await getCgroupsVersion(true);- expect(version).toBe('V2');- });-- test('returns V1 when access to /sys/fs/cgroup/memory/ succeeds', async () => {- vitest.spyOn(asyncFs, 'access').mockResolvedValue();- const version = await getCgroupsVersion(true);- expect(version).toBe('V1');- });-});+import { sleep, snakeCaseToCamelCase, weightedAvg } from '@crawlee/utils';describe('weightedAvg()', () => {test('works', () => {