Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Renaming params of getApifyProxyUrl() (#78)

Marek Trunkátc9862f183612
diff --git a/src/actor.js b/src/actor.js
index 12a07e0ac..95788cea5 100644
--- a/src/actor.js
+++ b/src/actor.js
@@ -1,6 +1,7 @@
import _ from 'underscore';
import Promise from 'bluebird';
import fs from 'fs';
+import log from 'apify-shared/log';
import { checkParamOrThrow } from 'apify-client/build/utils';
import { PROXY_GROUP_NAME_REGEX, PROXY_SESSION_ID_REGEX } from 'apify-shared/regexs';
import { ENV_VARS, EXIT_CODES, ACT_TASK_TERMINAL_STATUSES, ACT_TASK_STATUSES, DEFAULT_PROXY_HOSTNAME, DEFAULT_PROXY_PORT } from './constants';
@@ -375,8 +376,8 @@ export const call = (actId, input, opts = {}) => {
*
* @param {Object} opts
* @param {String} opts.password User proxy password. By default, it is taken from the `APIFY_PROXY_PASSWORD` environment variable.
- * @param {String} [opts.groups] Proxy groups to be used.
- * @param {String} [opts.session] Session ID that identifies requests that should use the same proxy connection.
+ * @param {String} [opts.apifyProxyGroups] Proxy groups to be used.
+ * @param {String} [opts.apifyProxySession] Session ID that identifies requests that should use the same proxy connection.
* @returns {String} Returns proxy URL.
*
* @memberof module:Apify
@@ -384,9 +385,20 @@ export const call = (actId, input, opts = {}) => {
* @name getApifyProxyUrl
*/
export const getApifyProxyUrl = (opts = {}) => {
+ // For backwards compatibility.
+ // TODO: remove this when we release v1.0.0
+ if (!opts.apifyProxyGroups && opts.groups) {
+ log.warning('Parameter `groups` of Apify.getApifyProxyUrl() is deprecated!!! Use `apifyProxyGroups` instead!');
+ opts.apifyProxyGroups = opts.groups;
+ }
+ if (!opts.apifyProxySession && opts.session) {
+ log.warning('Parameter `session` of Apify.getApifyProxyUrl() is deprecated!!! Use `apifyProxySession` instead!');
+ opts.apifyProxySession = opts.session;
+ }
+
const {
- groups,
- session,
+ apifyProxyGroups,
+ apifyProxySession,
password = process.env[ENV_VARS.PROXY_PASSWORD],
hostname = process.env[ENV_VARS.PROXY_HOSTNAME] || DEFAULT_PROXY_HOSTNAME,
port = parseInt(process.env[ENV_VARS.PROXY_PORT], 10) || DEFAULT_PROXY_PORT,
@@ -395,24 +407,24 @@ export const getApifyProxyUrl = (opts = {}) => {
const getMissingParamErrorMgs = (param, env) => `Apify Proxy ${param} must be provided as parameter or "${env}" environment variable!`;
const throwNonAlphanumericError = (param) => { throw new Error(`Only alphanumeric characters and underscore are allowed in ${param}`); };
- checkParamOrThrow(groups, 'opts.groups', 'Maybe [String]');
- checkParamOrThrow(session, 'opts.session', 'Maybe Number | String');
+ checkParamOrThrow(apifyProxyGroups, 'opts.apifyProxyGroups', 'Maybe [String]');
+ checkParamOrThrow(apifyProxySession, 'opts.apifyProxySession', 'Maybe Number | String');
checkParamOrThrow(password, 'opts.password', 'String', getMissingParamErrorMgs('password', ENV_VARS.PROXY_PASSWORD));
checkParamOrThrow(hostname, 'opts.hostname', 'String', getMissingParamErrorMgs('hostname', ENV_VARS.PROXY_HOSTNAME));
checkParamOrThrow(port, 'opts.port', 'Number', getMissingParamErrorMgs('port', ENV_VARS.PROXY_PORT));
let username;
- if (groups || session) {
+ if (apifyProxyGroups || apifyProxySession) {
const parts = [];
- if (groups && groups.length) {
- if (!groups.every(group => PROXY_GROUP_NAME_REGEX.test(group))) throwNonAlphanumericError('proxy group names');
- parts.push(`GROUPS-${groups.join('+')}`);
+ if (apifyProxyGroups && apifyProxyGroups.length) {
+ if (!apifyProxyGroups.every(group => PROXY_GROUP_NAME_REGEX.test(group))) throwNonAlphanumericError('proxy group name');
+ parts.push(`GROUPS-${apifyProxyGroups.join('+')}`);
}
- if (session) {
- if (!PROXY_SESSION_ID_REGEX.test(session)) throwNonAlphanumericError('proxy session ID');
- parts.push(`SESSION-${session}`);
+ if (apifyProxySession) {
+ if (!PROXY_SESSION_ID_REGEX.test(apifyProxySession)) throwNonAlphanumericError('proxy session ID');
+ parts.push(`SESSION-${apifyProxySession}`);
}
username = parts.join(',');
diff --git a/src/puppeteer.js b/src/puppeteer.js
index a5a45d15c..0e0ff826a 100644
--- a/src/puppeteer.js
+++ b/src/puppeteer.js
@@ -1,6 +1,5 @@
import { checkParamOrThrow } from 'apify-client/build/utils';
import { anonymizeProxy, closeAnonymizedProxy } from 'proxy-chain';
-import { cryptoRandomObjectId } from 'apify-shared/utilities';
import { ENV_VARS, DEFAULT_USER_AGENT } from './constants';
import { newPromise, getTypicalChromeExecutablePath } from './utils';
import { getApifyProxyUrl } from './actor';
@@ -149,10 +148,9 @@ export const launchPuppeteer = (opts = {}) => {
opts.executablePath = process.env[ENV_VARS.CHROME_EXECUTABLE_PATH] || getTypicalChromeExecutablePath();
}
if (opts.useApifyProxy) {
- opts.proxyUrl = getApifyProxyUrl({
- groups: opts.apifyProxyGroups,
- session: opts.apifyProxySession || cryptoRandomObjectId(),
- });
+ const { apifyProxyGroups, apifyProxySession } = opts;
+
+ opts.proxyUrl = getApifyProxyUrl({ apifyProxyGroups, apifyProxySession });
}
// When User-Agent is not set and we're using Chromium or headless mode,
diff --git a/test/actor.js b/test/actor.js
index b533e80aa..7ea1f35a6 100644
--- a/test/actor.js
+++ b/test/actor.js
@@ -703,6 +703,42 @@ describe('Apify.getApifyProxyUrl()', () => {
process.env[ENV_VARS.PROXY_HOSTNAME] = 'my.host.com';
process.env[ENV_VARS.PROXY_PORT] = 123;
+ expect(Apify.getApifyProxyUrl({
+ apifyProxySession: 'XYZ',
+ apifyProxyGroups: ['g1', 'g2', 'g3'],
+ })).to.be.eql('http://GROUPS-g1+g2+g3,SESSION-XYZ:[email protected]:123');
+
+ expect(Apify.getApifyProxyUrl({
+ apifyProxyGroups: ['g1', 'g2', 'g3'],
+ })).to.be.eql('http://GROUPS-g1+g2+g3:[email protected]:123');
+
+ expect(Apify.getApifyProxyUrl({
+ apifyProxySession: 'XYZ',
+ })).to.be.eql('http://SESSION-XYZ:[email protected]:123');
+
+ expect(Apify.getApifyProxyUrl()).to.be.eql('http://auto:[email protected]:123');
+
+ delete process.env[ENV_VARS.PROXY_PASSWORD];
+ delete process.env[ENV_VARS.PROXY_HOSTNAME];
+ delete process.env[ENV_VARS.PROXY_PORT];
+
+ expect(Apify.getApifyProxyUrl({ password: 'xyz' })).to.be.eql(`http://auto:xyz@${DEFAULT_PROXY_HOSTNAME}:${DEFAULT_PROXY_PORT}`);
+
+ expect(() => Apify.getApifyProxyUrl()).to.throw();
+
+ expect(Apify.getApifyProxyUrl({
+ password: 'xyz',
+ hostname: 'your.host.com',
+ port: 345,
+ })).to.be.eql('http://auto:[email protected]:345');
+ });
+
+ // Test old params - session, groups
+ it('should be backwards compatible', () => {
+ process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';
+ process.env[ENV_VARS.PROXY_HOSTNAME] = 'my.host.com';
+ process.env[ENV_VARS.PROXY_PORT] = 123;
+
expect(Apify.getApifyProxyUrl({
session: 'XYZ',
groups: ['g1', 'g2', 'g3'],
diff --git a/test/puppeteer.js b/test/puppeteer.js
index 7be25fb8e..249fcba1b 100644
--- a/test/puppeteer.js
+++ b/test/puppeteer.js
@@ -198,8 +198,8 @@ describe('Apify.launchPuppeteer()', () => {
mock.expects('getApifyProxyUrl')
.once()
.withArgs({
- session: 'xxx',
- groups: ['yyy'],
+ apifyProxySession: 'xxx',
+ apifyProxyGroups: ['yyy'],
})
.returns(null); // Return null so that it doesn't start proxy-chain