Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat: add support for parsing comma-separated list environment variables (#2765)

František Nesveda4e50c474f60d
diff --git a/packages/core/src/configuration.ts b/packages/core/src/configuration.ts
index 396c82cde..d5ed9b4f5 100644
--- a/packages/core/src/configuration.ts
+++ b/packages/core/src/configuration.ts
@@ -253,6 +253,8 @@ export class Configuration {
protected static INTEGER_VARS = ['memoryMbytes', 'persistStateIntervalMillis', 'systemInfoIntervalMillis'];
+ protected static COMMA_SEPARATED_LIST_VARS: string[] = [];
+
protected static DEFAULTS: Dictionary = {
defaultKeyValueStoreId: 'default',
defaultDatasetId: 'default',
@@ -344,6 +346,13 @@ export class Configuration {
return !['0', 'false', ''].includes(String(value).toLowerCase());
}
+ if (Configuration.COMMA_SEPARATED_LIST_VARS.includes(key)) {
+ if (!value) return [];
+ return String(value)
+ .split(',')
+ .map((v) => v.trim());
+ }
+
return value;
}