Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Feature/get proxy url opts (#118)

Marek Trunkát1bc28aa06ad9
diff --git a/src/actor.js b/src/actor.js
index 04388b799..9a5723913 100644
--- a/src/actor.js
+++ b/src/actor.js
@@ -415,6 +415,11 @@ export const getApifyProxyUrl = (opts = {}) => {
password = process.env[ENV_VARS.PROXY_PASSWORD],
hostname = process.env[ENV_VARS.PROXY_HOSTNAME] || DEFAULT_PROXY_HOSTNAME,
port = parseInt(process.env[ENV_VARS.PROXY_PORT], 10) || DEFAULT_PROXY_PORT,
+
+ // This is used only internaly. Some other function calling this function use different naming for groups and session
+ // parameters so we need to override this in error messages.
+ groupsParamName = 'opts.groups',
+ sessionParamName = 'opts.session',
} = opts;
const getMissingParamErrorMgs = (param, env) => `Apify Proxy ${param} must be provided as parameter or "${env}" environment variable!`;
@@ -422,8 +427,8 @@ export const getApifyProxyUrl = (opts = {}) => {
throw new Error(`The "${param}" option can only contain the following characters: 0-9, a-z, A-Z, ".", "_" and "~"`);
};
- checkParamOrThrow(groups, 'opts.groups', 'Maybe [String]');
- checkParamOrThrow(session, 'opts.session', 'Maybe Number | String');
+ checkParamOrThrow(groups, groupsParamName, 'Maybe [String]');
+ checkParamOrThrow(session, sessionParamName, 'Maybe Number | String');
checkParamOrThrow(password, 'opts.password', 'String', getMissingParamErrorMgs('password', ENV_VARS.PROXY_PASSWORD));
checkParamOrThrow(hostname, 'opts.hostname', 'String', getMissingParamErrorMgs('hostname', ENV_VARS.PROXY_HOSTNAME));
checkParamOrThrow(port, 'opts.port', 'Number', getMissingParamErrorMgs('port', ENV_VARS.PROXY_PORT));
diff --git a/src/puppeteer.js b/src/puppeteer.js
index 053c234f3..6cafe3cb7 100644
--- a/src/puppeteer.js
+++ b/src/puppeteer.js
@@ -173,6 +173,8 @@ export const launchPuppeteer = (opts = {}) => {
optsCopy.proxyUrl = getApifyProxyUrl({
groups: optsCopy.apifyProxyGroups,
session: optsCopy.apifyProxySession,
+ groupsParamName: 'opts.apifyProxyGroups',
+ sessionParamName: 'opts.apifyProxySession',
});
}
diff --git a/test/puppeteer.js b/test/puppeteer.js
index 7be25fb8e..24fe4c65f 100644
--- a/test/puppeteer.js
+++ b/test/puppeteer.js
@@ -200,6 +200,8 @@ describe('Apify.launchPuppeteer()', () => {
.withArgs({
session: 'xxx',
groups: ['yyy'],
+ groupsParamName: 'opts.apifyProxyGroups',
+ sessionParamName: 'opts.apifyProxySession',
})
.returns(null); // Return null so that it doesn't start proxy-chain