Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Added support for APIFY_API_BASE_URL to enable testing
Jan Curnb5d7db119542
diff --git a/src/actor.js b/src/actor.jsindex 4d763f1d1..983105955 100644--- a/src/actor.js+++ b/src/actor.js@@ -1,14 +1,25 @@import fs from 'fs';+import url from 'url';import ApifyClient from 'apify-client';import { APIFY_ENV_VARS, EXIT_CODES, KV_STORE_KEYS } from './constants';import { getPromisesDependency, newPromise, nodeifyPromise } from './utils';/* global process */+// TODO: protocol/host/port/basePath should be replaced with baseUrl+const clientOpts = {};+if (process.env[APIFY_ENV_VARS.API_BASE_URL]) {+ const parsed = url.parse(process.env[APIFY_ENV_VARS.API_BASE_URL]);+ clientOpts.protocol = parsed.protocol;+ clientOpts.host = parsed.hostname;+ clientOpts.port = parsed.port;+ clientOpts.basePath = parsed.pathname;+}+/**- * Exported to enable unit testing mock-up.+ * Exported to enable mocking up in unit tests.*/-export const apifyClient = new ApifyClient();+export const apifyClient = new ApifyClient(clientOpts);/**diff --git a/src/constants.js b/src/constants.jsindex 4326f71d3..01ba007c3 100644--- a/src/constants.js+++ b/src/constants.js@@ -21,6 +21,7 @@ export const APIFY_ENV_VARS = {TIMEOUT_AT: 'APIFY_TIMEOUT_AT',DEFAULT_KEY_VALUE_STORE_ID: 'APIFY_DEFAULT_KEY_VALUE_STORE_ID',WATCH_FILE: 'APIFY_WATCH_FILE',+ API_BASE_URL: 'APIFY_API_BASE_URL',};