Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Feature/get proxy url opts (#118)
Marek Trunkát1bc28aa06ad9
diff --git a/src/actor.js b/src/actor.jsindex 04388b799..9a5723913 100644--- a/src/actor.js+++ b/src/actor.js@@ -415,6 +415,11 @@ export const getApifyProxyUrl = (opts = {}) => {password = process.env[ENV_VARS.PROXY_PASSWORD],hostname = process.env[ENV_VARS.PROXY_HOSTNAME] || DEFAULT_PROXY_HOSTNAME,port = parseInt(process.env[ENV_VARS.PROXY_PORT], 10) || DEFAULT_PROXY_PORT,++ // This is used only internaly. Some other function calling this function use different naming for groups and session+ // parameters so we need to override this in error messages.+ groupsParamName = 'opts.groups',+ sessionParamName = 'opts.session',} = opts;const getMissingParamErrorMgs = (param, env) => `Apify Proxy ${param} must be provided as parameter or "${env}" environment variable!`;@@ -422,8 +427,8 @@ export const getApifyProxyUrl = (opts = {}) => {throw new Error(`The "${param}" option can only contain the following characters: 0-9, a-z, A-Z, ".", "_" and "~"`);};- checkParamOrThrow(groups, 'opts.groups', 'Maybe [String]');- checkParamOrThrow(session, 'opts.session', 'Maybe Number | String');+ checkParamOrThrow(groups, groupsParamName, 'Maybe [String]');+ checkParamOrThrow(session, sessionParamName, 'Maybe Number | String');checkParamOrThrow(password, 'opts.password', 'String', getMissingParamErrorMgs('password', ENV_VARS.PROXY_PASSWORD));checkParamOrThrow(hostname, 'opts.hostname', 'String', getMissingParamErrorMgs('hostname', ENV_VARS.PROXY_HOSTNAME));checkParamOrThrow(port, 'opts.port', 'Number', getMissingParamErrorMgs('port', ENV_VARS.PROXY_PORT));diff --git a/src/puppeteer.js b/src/puppeteer.jsindex 053c234f3..6cafe3cb7 100644--- a/src/puppeteer.js+++ b/src/puppeteer.js@@ -173,6 +173,8 @@ export const launchPuppeteer = (opts = {}) => {optsCopy.proxyUrl = getApifyProxyUrl({groups: optsCopy.apifyProxyGroups,session: optsCopy.apifyProxySession,+ groupsParamName: 'opts.apifyProxyGroups',+ sessionParamName: 'opts.apifyProxySession',});}diff --git a/test/puppeteer.js b/test/puppeteer.jsindex 7be25fb8e..24fe4c65f 100644--- a/test/puppeteer.js+++ b/test/puppeteer.js@@ -200,6 +200,8 @@ describe('Apify.launchPuppeteer()', () => {.withArgs({session: 'xxx',groups: ['yyy'],+ groupsParamName: 'opts.apifyProxyGroups',+ sessionParamName: 'opts.apifyProxySession',}).returns(null); // Return null so that it doesn't start proxy-chain