Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Improving getEnv()
Marek Trunkat620b6100630d
diff --git a/package.json b/package.jsonindex d118369b1..c72ab1b82 100644--- a/package.json+++ b/package.json@@ -74,6 +74,7 @@"basic-auth-parser": "^0.0.2","chai": "^4.2.0","chai-as-promised": "^7.1.1",+ "chai-subset": "^1.6.0","eslint": "^5.9.0","eslint-config-airbnb": "^17.1.0","eslint-config-airbnb-base": "^13.1.0",diff --git a/src/actor.js b/src/actor.jsindex bb9ecde53..48a794447 100644--- a/src/actor.js+++ b/src/actor.js@@ -4,10 +4,10 @@ import Promise from 'bluebird';import log from 'apify-shared/log';import { checkParamOrThrow } from 'apify-client/build/utils';import { APIFY_PROXY_VALUE_REGEX } from 'apify-shared/regexs';-import { ENV_VARS, LOCAL_ENV_VARS, ACT_JOB_TERMINAL_STATUSES, ACT_JOB_STATUSES } from 'apify-shared/consts';+import { ENV_VARS, INTEGER_ENV_VARS, LOCAL_ENV_VARS, ACT_JOB_TERMINAL_STATUSES, ACT_JOB_STATUSES } from 'apify-shared/consts';import { EXIT_CODES } from './constants';import { initializeEvents, stopEvents } from './events';-import { apifyClient, addCharsetToContentType } from './utils';+import { apifyClient, addCharsetToContentType, snakeCaseToCamelCase } from './utils';import { maybeStringify } from './key_value_store';import { ApifyCallError } from './errors';@@ -113,18 +113,22 @@ const waitForRunToFinish = async ({ actId, runId, token, waitSecs, taskId }) =>export const getEnv = () => {// NOTE: Don't throw if env vars are invalid to simplify local development and debugging of actorsconst env = process.env || {};- return {- actId: env[ENV_VARS.ACT_ID] || null,- actRunId: env[ENV_VARS.ACT_RUN_ID] || null,- userId: env[ENV_VARS.USER_ID] || null,- token: env[ENV_VARS.TOKEN] || null,- startedAt: tryParseDate(env[ENV_VARS.STARTED_AT]) || null,- timeoutAt: tryParseDate(env[ENV_VARS.TIMEOUT_AT]) || null,- defaultKeyValueStoreId: env[ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID] || null,- defaultDatasetId: env[ENV_VARS.DEFAULT_DATASET_ID] || null,- // internalPort: parseInt(env[ENV_VARS.INTERNAL_PORT], 10) || null,- memoryMbytes: parseInt(env[ENV_VARS.MEMORY_MBYTES], 10) || null,- };+ const envVars = {};++ _.mapObject(ENV_VARS, (fullName, shortName) => {+ const camelCaseName = snakeCaseToCamelCase(shortName);+ let value = env[fullName];++ // Parse dates and integers.+ if (value && fullName.endsWith('_AT')) value = tryParseDate(value);+ else if (_.contains(INTEGER_ENV_VARS, fullName)) value = parseInt(value, 10);++ envVars[camelCaseName] = value || value === 0+ ? value+ : null;+ });++ return envVars;};/**diff --git a/src/utils.js b/src/utils.jsindex f2b8aef58..00bab64cd 100644--- a/src/utils.js+++ b/src/utils.js@@ -502,7 +502,6 @@ export const ensureTokenOrLocalStorageEnvExists = (storageName) => {const SKIP_TAGS_REGEX = /^(script|style|canvas|svg|noscript)$/i;const BLOCK_TAGS_REGEX = /^(p|h1|h2|h3|h4|h5|h6|ol|ul|li|pre|address|blockquote|dl|div|fieldset|form|table|tr|select|option)$/i;-/*** The function converts a HTML document to a plain text.*@@ -606,6 +605,25 @@ const createRequestDebugInfo = (request, response = {}, additionalFields = {}) =);};+/**+ * Converts SNAKE_CASE to camelCase.+ *+ * @param {String} snakeCaseStr+ * @return {String}+ * @ignore+ */+export const snakeCaseToCamelCase = (snakeCaseStr) => {+ return snakeCaseStr+ .toLowerCase()+ .split('_')+ .map((part, index) => {+ return index > 0+ ? part.charAt(0).toUpperCase() + part.slice(1)+ : part;+ })+ .join('');+};+/*** A namespace that contains various utilities.*diff --git a/test/_helper.js b/test/_helper.jsindex 818a57378..b63f96e0f 100644--- a/test/_helper.js+++ b/test/_helper.js@@ -1,8 +1,13 @@import fs from 'fs-extra';import path from 'path';-import { expect } from 'chai';+import chai, { expect } from 'chai';+import chaiSubset from 'chai-subset';+import chaiAsPromised from 'chai-as-promised';import { ENV_VARS } from 'apify-shared/consts';+chai.use(chaiSubset);+chai.use(chaiAsPromised);+export const LOCAL_STORAGE_DIR = path.join(__dirname, '..', 'tmp', 'local-emulation-dir');// Log unhandled rejections.diff --git a/test/actor.js b/test/actor.jsindex 11cc96cb6..8b75256a5 100644--- a/test/actor.js+++ b/test/actor.js@@ -1,7 +1,6 @@import path from 'path';import _ from 'underscore';import chai, { expect } from 'chai';-import chaiAsPromised from 'chai-as-promised';import sinon from 'sinon';import { delayPromise } from 'apify-shared/utilities';import { ENV_VARS, ACT_JOB_STATUSES, LOCAL_ENV_VARS } from 'apify-shared/consts';@@ -146,7 +145,7 @@ describe('Apify.getEnv()', () => {setEnv(expectedEnv);const env = Apify.getEnv();- expect(env).to.eql(expectedEnv);+ expect(env).to.containSubset(expectedEnv);});it('works with with non-null values', () => {@@ -165,11 +164,10 @@ describe('Apify.getEnv()', () => {setEnv(expectedEnv);const env = Apify.getEnv();- expect(env).to.eql(expectedEnv);+ expect(env).to.containSubset(expectedEnv);});});-describe('Apify.main()', () => {it('throws on invalid args', () => {expect(() => {diff --git a/test/utils.js b/test/utils.jsindex 3d8c3e8b0..74170d5c4 100644--- a/test/utils.js+++ b/test/utils.js@@ -1,6 +1,7 @@import chai, { expect } from 'chai';import chaiAsPromised from 'chai-as-promised';import sinon from 'sinon';+import _ from 'underscore';import fs from 'fs';import path from 'path';import os from 'os';@@ -752,25 +753,25 @@ describe('utils.htmlToText()', () => {/* eslint-disable */checkHtmlToText(html1,- `Let's start with a simple text.-The ships hung in the sky, much the way that bricks don't.+ `Let's start with a simple text.+The ships hung in the sky, much the way that bricks don't.These aren't the Droids you're looking forI'm sorry, Dave. I'm afraid I can't do that.I'm sorry, Dave. I'm afraid I can't do that.A1 A2 A3\tB1 B2 B3 B 4\t-This is some text with inline elements and HTML entities (>bla<)+This is some text with inline elements and HTML entities (>bla<)Testafewlinebreaks-Spaces in an inline text should be completely ignored.+Spaces in an inline text should be completely ignored.But,a pre-formattedblock should be keptpre-formatted.-The Greatest Science Fiction Quotes Of All Time+The Greatest Science Fiction Quotes Of All TimeDon't know, I don't know such stuff. I just do eyes, ju-, ju-, just eyes... just genetic design, just eyes. You Nexus, huh? I design your eyes.`,true,);@@ -854,3 +855,18 @@ describe('utils.createRequestDebugInfo()', () => {});});});++describe('utils.snakeCaseToCamelCase()', () => {+ it('should camel case all sneaky cases of snake case', () => {+ const tests = {+ aaa_bbb_: 'aaaBbb',+ '': '',+ AaA_bBb_cCc: 'aaaBbbCcc',+ a_1_b_1a: 'a1B1a',+ };++ _.mapObject(tests, (camelCase, snakeCase) => {+ expect(utils.snakeCaseToCamelCase(snakeCase)).to.be.eql(camelCase);+ });+ });+});