Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
import { EventEmitter } from 'node:events';
import { existsSync, unlinkSync, writeFileSync } from 'node:fs';
import { join } from 'node:path';
import { coerceBoolean, Configuration, crawleeConfigFields, field, LogLevel } from '@crawlee/core';
describe('Configuration', () => {
const envBackup: Record<string, string | undefined> = {};
function setEnv(key: string, value: string) {
envBackup[key] ??= process.env[key];
process.env[key] = value;
}
beforeEach(() => {
// Clean all CRAWLEE_ env vars so tests are isolated
for (const key of Object.keys(process.env)) {
if (key.startsWith('CRAWLEE_')) {
envBackup[key] ??= process.env[key];
delete process.env[key];
}
}
});
afterEach(() => {
// Restore env vars
for (const [key, val] of Object.entries(envBackup)) {
if (val === undefined) {
delete process.env[key];
} else {
process.env[key] = val;
}
}
});
describe('defaults', () => {
it('returns schema defaults when nothing is set', () => {
const config = new Configuration();
expect(config.chromeExecutablePath).toBeUndefined();
expect(config.headless).toBe(true);
expect(config.xvfb).toBe(false);
expect(config.disableBrowserSandbox).toBe(false);
expect(config.purgeOnStart).toBe(true);
expect(config.persistStorage).toBe(true);
expect(config.maxUsedCpuRatio).toBe(0.95);
expect(config.availableMemoryRatio).toBe(0.25);
expect(config.persistStateIntervalMillis).toBe(60_000);
expect(config.systemInfoIntervalMillis).toBe(1_000);
});
// `containerized` and `logLevel` are intentionally optional — `undefined` is a meaningful signal:
// - `containerized`: consumers fall back to runtime detection via `isContainerized()` using
// `config.containerized ?? (await isContainerized())`; defaulting to `false` would disable
// auto-detection.
// - `logLevel`: the log system already has its own default (INFO). Defaulting here would
// cause every `new Configuration()` to override any previously-configured log level.
it('returns undefined for optional fields with no default', () => {
const config = new Configuration();
expect(config.memoryMbytes).toBeUndefined();
expect(config.chromeExecutablePath).toBeUndefined();
expect(config.defaultBrowserPath).toBeUndefined();
expect(config.containerized).toBeUndefined();
expect(config.logLevel).toBeUndefined();
});
});
describe('priority: constructor > env > crawlee.json > defaults', () => {
it('constructor options override env vars', () => {
setEnv('CRAWLEE_HEADLESS', 'true');
const config = new Configuration({ headless: false });
expect(config.headless).toBe(false);
});
it('env vars override defaults', () => {
setEnv('CRAWLEE_HEADLESS', 'false');
const config = new Configuration();
expect(config.headless).toBe(false);
});
it('constructor options override defaults', () => {
const config = new Configuration({ persistStateIntervalMillis: 30_000 });
expect(config.persistStateIntervalMillis).toBe(30_000);
});
it('constructor options override env vars for string fields', () => {
setEnv('CRAWLEE_CHROME_EXECUTABLE_PATH', 'from-env');
const config = new Configuration({ chromeExecutablePath: 'from-constructor' });
expect(config.chromeExecutablePath).toBe('from-constructor');
});
it('constructor options override env vars for number fields', () => {
setEnv('CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS', '99999');
const config = new Configuration({ persistStateIntervalMillis: 10_000 });
expect(config.persistStateIntervalMillis).toBe(10_000);
});
describe('with crawlee.json', () => {
const crawleeJsonPath = join(process.cwd(), 'crawlee.json');
afterEach(() => {
if (existsSync(crawleeJsonPath)) {
unlinkSync(crawleeJsonPath);
}
});
it('crawlee.json overrides defaults', () => {
writeFileSync(crawleeJsonPath, JSON.stringify({ persistStateIntervalMillis: 5_000 }));
const config = new Configuration();
expect(config.persistStateIntervalMillis).toBe(5_000);
});
it('env vars override crawlee.json', () => {
writeFileSync(crawleeJsonPath, JSON.stringify({ persistStateIntervalMillis: 5_000 }));
setEnv('CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS', '7000');
const config = new Configuration();
expect(config.persistStateIntervalMillis).toBe(7_000);
});
it('constructor options override crawlee.json and env vars', () => {
writeFileSync(crawleeJsonPath, JSON.stringify({ persistStateIntervalMillis: 5_000 }));
setEnv('CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS', '7000');
const config = new Configuration({ persistStateIntervalMillis: 10_000 });
expect(config.persistStateIntervalMillis).toBe(10_000);
});
});
});
describe('env var coercion', () => {
it('coerces boolean env vars', () => {
setEnv('CRAWLEE_HEADLESS', 'false');
expect(new Configuration().headless).toBe(false);
setEnv('CRAWLEE_HEADLESS', '0');
expect(new Configuration().headless).toBe(false);
setEnv('CRAWLEE_HEADLESS', 'true');
expect(new Configuration().headless).toBe(true);
setEnv('CRAWLEE_HEADLESS', '1');
expect(new Configuration().headless).toBe(true);
});
it('treats empty-string env var as unset across all field types', () => {
// Empty boolean env var falls through to default (true), not coerced to false
setEnv('CRAWLEE_HEADLESS', '');
expect(new Configuration().headless).toBe(true);
// Empty number env var falls through to default (60_000), not coerced to 0
setEnv('CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS', '');
expect(new Configuration().persistStateIntervalMillis).toBe(60_000);
// Empty string env var falls through to default ('./storage'), not coerced to ''
setEnv('CRAWLEE_STORAGE_DIR', '');
expect(new Configuration().storageDir).toBe('./storage');
// Optional fields with no default stay undefined
setEnv('CRAWLEE_MEMORY_MBYTES', '');
expect(new Configuration().memoryMbytes).toBeUndefined();
});
it('coerces number env vars', () => {
setEnv('CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS', '30000');
expect(new Configuration().persistStateIntervalMillis).toBe(30_000);
setEnv('CRAWLEE_MEMORY_MBYTES', '512');
expect(new Configuration().memoryMbytes).toBe(512);
});
it('coerces log level from string name', () => {
setEnv('CRAWLEE_LOG_LEVEL', 'DEBUG');
expect(new Configuration().logLevel).toBe(LogLevel.DEBUG);
});
it('coerces log level from numeric string', () => {
setEnv('CRAWLEE_LOG_LEVEL', '5');
expect(new Configuration().logLevel).toBe(LogLevel.DEBUG);
});
it('coerces log level case-insensitively', () => {
setEnv('CRAWLEE_LOG_LEVEL', 'info');
expect(new Configuration().logLevel).toBe(LogLevel.INFO);
});
});
describe('direct property access', () => {
it('exposes resolved values as instance properties', () => {
const config = new Configuration({
headless: false,
chromeExecutablePath: '/usr/bin/chromium',
persistStateIntervalMillis: 5_000,
});
expect(config.headless).toBe(false);
expect(config.chromeExecutablePath).toBe('/usr/bin/chromium');
expect(config.persistStateIntervalMillis).toBe(5_000);
});
});
describe('immutability', () => {
it('throws TypeError when assigning to a config property', () => {
const config = new Configuration();
expect(() => {
(config as any).headless = false;
}).toThrow(TypeError);
expect(() => {
(config as any).headless = false;
}).toThrow('Configuration is immutable');
});
});
describe('schema validation of constructor options', () => {
it('validates and coerces constructor options through the schema', () => {
// Pass a string where a number is expected — schema should coerce it
const config = new Configuration({ memoryMbytes: '512' as any });
expect(config.memoryMbytes).toBe(512);
expect(typeof config.memoryMbytes).toBe('number');
});
it('validates boolean constructor options through the schema', () => {
// Pass a string where a boolean is expected — schema should coerce it
const config = new Configuration({ headless: '0' as any });
expect(config.headless).toBe(false);
});
it('rejects invalid constructor options via schema', () => {
// Pass a completely invalid value — schema should throw
expect(() => {
const config = new Configuration({ memoryMbytes: 'not-a-number' as any });
// Access the property to trigger resolution
void config.memoryMbytes;
}).toThrow();
});
});
describe('crawlee.json file loading', () => {
const crawleeJsonPath = join(process.cwd(), 'crawlee.json');
let fileCreated = false;
afterEach(() => {
if (fileCreated) {
try {
unlinkSync(crawleeJsonPath);
} catch {
/* ignore */
}
fileCreated = false;
}
});
it('loads values from crawlee.json', () => {
writeFileSync(crawleeJsonPath, JSON.stringify({ chromeExecutablePath: 'from-file' }));
fileCreated = true;
const config = new Configuration();
expect(config.chromeExecutablePath).toBe('from-file');
});
it('constructor options override crawlee.json', () => {
writeFileSync(crawleeJsonPath, JSON.stringify({ chromeExecutablePath: 'from-file' }));
fileCreated = true;
const config = new Configuration({ chromeExecutablePath: 'from-constructor' });
expect(config.chromeExecutablePath).toBe('from-constructor');
});
it('env vars override crawlee.json', () => {
writeFileSync(crawleeJsonPath, JSON.stringify({ headless: false }));
fileCreated = true;
setEnv('CRAWLEE_HEADLESS', 'true');
const config = new Configuration();
expect(config.headless).toBe(true);
});
it('validates and coerces crawlee.json values through the schema', () => {
// JSON numbers are already numbers, but string values should be coerced
writeFileSync(crawleeJsonPath, JSON.stringify({ memoryMbytes: '256' }));
fileCreated = true;
const config = new Configuration();
expect(config.memoryMbytes).toBe(256);
expect(typeof config.memoryMbytes).toBe('number');
});
it('loads crawlee.json saved with a UTF-8 byte order mark', () => {
// e.g. written by PowerShell 5.1 `Set-Content -Encoding UTF8` or "UTF-8 with BOM" in an editor
writeFileSync(crawleeJsonPath, `\uFEFF${JSON.stringify({ chromeExecutablePath: 'from-file' })}`);
fileCreated = true;
const config = new Configuration();
expect(config.chromeExecutablePath).toBe('from-file');
});
it('handles missing crawlee.json gracefully', () => {
// No file created — should fall through to defaults
const config = new Configuration();
expect(config.storageDir).toBe('./storage');
});
it('handles malformed crawlee.json gracefully', () => {
writeFileSync(crawleeJsonPath, 'not valid json{{{');
fileCreated = true;
const config = new Configuration();
expect(config.storageDir).toBe('./storage');
});
});
describe('subclass field registration', () => {
it('subclass can define additional fields via static fields override', () => {
const extendedFields = {
...crawleeConfigFields,
customFlag: field(coerceBoolean.default(false), 'MY_CUSTOM_FLAG'),
};
class ExtendedConfig extends Configuration {
protected static override fields = extendedFields;
}
const config = new ExtendedConfig();
expect((config as any).customFlag).toBe(false);
setEnv('MY_CUSTOM_FLAG', 'true');
const config2 = new ExtendedConfig();
expect((config2 as any).customFlag).toBe(true);
});
});
it('does not change EventEmitter.defaultMaxListeners on import', async () => {
const original = EventEmitter.defaultMaxListeners;
EventEmitter.defaultMaxListeners = 7;
// The vitest setup already imported the module, so force it to be evaluated again
vitest.resetModules();
await import('../../packages/core/src/configuration.js');
const afterImport = EventEmitter.defaultMaxListeners;
EventEmitter.defaultMaxListeners = original;
expect(afterImport).toBe(7);
});
});