Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: casting of int/bool environment variables (e.g. `APIFY_LOCAL_STORAGE_ENABLE_WAL_MODE`) (#1146)

Martin Adámek090bbbee7986
diff --git a/CHANGELOG.md b/CHANGELOG.md
index d7d05bbca..c16f1201c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,7 @@
+2.0.7 / 2021/08/31
+====================
+- Fix casting of int/bool environment variables (e.g. `APIFY_LOCAL_STORAGE_ENABLE_WAL_MODE`), closes #956
+
2.0.6 / 2021/08/27
====================
- Fix deprecation messages logged from `ProxyConfiguration` and `CheerioCrawler`.
diff --git a/src/configuration.js b/src/configuration.js
index c5f51cb49..e41b2f1b7 100644
--- a/src/configuration.js
+++ b/src/configuration.js
@@ -169,11 +169,11 @@ export class Configuration {
* @private
*/
_castEnvValue(key, value) {
- if (key in Configuration.INTEGER_VARS) {
+ if (Configuration.INTEGER_VARS.includes(key)) {
return +value;
}
- if (key in Configuration.BOOLEAN_VARS) {
+ if (Configuration.BOOLEAN_VARS.includes(key)) {
// 0, false and empty string are considered falsy values
return !['0', 'false', ''].includes(value.toLowerCase());
}
diff --git a/test/apify.test.js b/test/apify.test.js
index b35c5d8f2..cc6143b6a 100644
--- a/test/apify.test.js
+++ b/test/apify.test.js
@@ -12,7 +12,7 @@ import { ProxyConfiguration } from '../build/proxy_configuration';
import { SessionPool } from '../build/session_pool/session_pool';
// NOTE: test use of require() here because this is how its done in acts
-const { Apify, RequestList, utils: { log, sleep } } = require('../build/index');
+const { Apify, Configuration, RequestList, utils: { log, sleep } } = require('../build/index');
/**
* Helper function that enables testing of Apify.main()
@@ -666,5 +666,12 @@ describe('new Apify({ ... })', () => {
expect(openStorageSpy).toBeCalledWith(queueId, options);
expect(openStorageSpy).toBeCalledTimes(1);
});
+
+ test('openRequestQueue works with APIFY_LOCAL_STORAGE_ENABLE_WAL_MODE=false', async () => {
+ process.env.APIFY_LOCAL_STORAGE_ENABLE_WAL_MODE = 'false';
+ const config = new Configuration();
+ const enableWalMode = config.get('localStorageEnableWalMode');
+ expect(enableWalMode).toBe(false);
+ });
});
});