Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Proxy configuration (#681)
Ondra Urban1f7bf2392e77
diff --git a/docs/guides/session_management.md b/docs/guides/session_management.mdindex 199cc67b3..d654b8913 100644--- a/docs/guides/session_management.md+++ b/docs/guides/session_management.md@@ -16,12 +16,13 @@ Now let's take a look at how to use a Session pool.**Example usage in [`PuppeteerCrawler`](/docs/api/puppeteer-crawler)**```javascript++const proxyConfiguration = await Apify.createProxyConfiguration();+const crawler = new Apify.PuppeteerCrawler({requestQueue,- launchPuppeteerOptions: {- // To use the proxy IP session rotation logic, you must turn the proxy usage on.- useApifyProxy: true,- },+ // To use the proxy IP session rotation logic, you must turn the proxy usage on.+ proxyConfiguration,// Activates the Session pool.useSessionPool: true,// Overrides default Session pool configuration@@ -31,7 +32,7 @@ const crawler = new Apify.PuppeteerCrawler({// Set to true if you want the crawler to save cookies per session,// and set the cookies to page before navigation automatically.persistCookiesPerSession: true,- handlePageFunction: async ({request, page, session}) => {+ handlePageFunction: async ({ request, page, session }) => {const title = await page.title();if (title === "Blocked") {@@ -49,10 +50,12 @@ const crawler = new Apify.PuppeteerCrawler({**Example usage in [`CheerioCrawler`](/docs/api/cheerio-crawler)**```javascript+ const proxyConfiguration = await Apify.createProxyConfiguration();+const crawler = new Apify.CheerioCrawler({requestQueue,// To use the proxy IP session rotation logic, you must turn the proxy usage on.- useApifyProxy: true,+ proxyConfiguration,// Activates the Session pool.useSessionPool: true,// Overrides default Session pool configuration.@@ -80,15 +83,19 @@ const crawler = new Apify.PuppeteerCrawler({**Example usage in [`BasicCrawler`](/docs/api/basic-crawler)**```javascript+ const proxyConfiguration = await Apify.createProxyConfiguration();+const crawler = new Apify.BasicCrawler({requestQueue,+ // Allows access to proxyInfo object in handleRequestFunction+ proxyConfiguration,useSessionPool: true,sessionPoolOptions: {maxPoolSize: 100},- handleRequestFunction: async ({request, session}) => {+ handleRequestFunction: async ({request, session, proxyInfo }) => {// To use the proxy IP session rotation logic, you must turn the proxy usage on.- const proxyUrl = Apify.getApifyProxyUrl({session});+ const proxyUrl = proxyInfo.url;const requestOptions = {url: request.url,proxyUrl,diff --git a/src/actor.js b/src/actor.jsindex 7495af974..79e9f7be5 100644--- a/src/actor.js+++ b/src/actor.js@@ -1,10 +1,9 @@import * as path from 'path';import * as _ from 'underscore';import { checkParamOrThrow } from 'apify-client/build/utils';-import { APIFY_PROXY_VALUE_REGEX } from 'apify-shared/regexs';-import { ENV_VARS, INTEGER_ENV_VARS, LOCAL_ENV_VARS, ACT_JOB_STATUSES } from 'apify-shared/consts';+import { ENV_VARS, INTEGER_ENV_VARS, ACT_JOB_STATUSES } from 'apify-shared/consts';import log from './utils_log';-import { EXIT_CODES, COUNTRY_CODE_REGEX } from './constants';+import { EXIT_CODES } from './constants';import { initializeEvents, stopEvents } from './events';import {apifyClient,@@ -601,113 +600,6 @@ export const metamorph = async (targetActorId, input, options = {}) => {await sleep(optionsCopy.customAfterSleepMillis || METAMORPH_AFTER_SLEEP_MILLIS);};-/**- * Constructs an Apify Proxy URL using the specified settings.- * The proxy URL can be used from Apify actors, web browsers or any other HTTP- * proxy-enabled applications.- *- * For more information, see the [Apify Proxy](https://my.apify.com/proxy) page in the app- * or the [documentation](https://docs.apify.com/proxy).- *- * @param {Object} [options]- * Object with the props below:- * @param {string} [options.password]- * User's password for the proxy. By default, it is taken from the `APIFY_PROXY_PASSWORD`- * environment variable, which is automatically set by the system when running the actors- * on the Apify cloud, or when using the [Apify CLI](https://github.com/apifytech/apify-cli)- * package and the user previously logged in (called `apify login`).- * @param {string[]} [options.groups]- * Array of Apify Proxy groups to be used. If not provided, the proxy will select- * the groups automatically.- * @param {string} [options.session]- * Apify Proxy session identifier to be used by the Chrome browser. All HTTP requests- * going through the proxy with the same session identifier will use the same target- * proxy server (i.e. the same IP address), unless using Residential proxies. The identifier- * can only contain the following characters: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.- * @param {string} [options.country]- * If set and relevant proxies are available in your Apify account, all proxied requests will- * use IP addresses that are geolocated to the specified country. For example `GB` for IPs- * from Great Britain. Note that online services often have their own rules for handling- * geolocation and thus the country selection is a best attempt at geolocation, rather than- * a guaranteed hit. This parameter is optional, by default, each proxied request is assigned- * an IP address from a random country. The country code needs to be a two letter ISO country code. See the- * [full list of available country codes](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2#Officially_assigned_code_elements).- * This parameter is optional, by default, the proxy uses all available proxy servers from all countries.- *- * @returns {string} Returns the proxy URL, e.g. `http://auto:[email protected]:8000`.- *- * @memberof module:Apify- * @function- * @name getApifyProxyUrl- */-export const getApifyProxyUrl = (options = {}) => {- // For backwards compatibility.- // TODO: remove this when we release v1.0.0- if (!options.groups && options.apifyProxyGroups) {- log.warning('Parameter `apifyProxyGroups` of Apify.getApifyProxyUrl() is deprecated!!! Use `groups` instead!');- options.groups = options.apifyProxyGroups;- }- if (!options.session && options.apifyProxySession) {- log.warning('Parameter `apifyProxySession` of Apify.getApifyProxyUrl() is deprecated!!! Use `session` instead!');- options.session = options.apifyProxySession;- }-- const {- groups,- session,- country,- password = process.env[ENV_VARS.PROXY_PASSWORD],- hostname = process.env[ENV_VARS.PROXY_HOSTNAME] || LOCAL_ENV_VARS[ENV_VARS.PROXY_HOSTNAME],- port = parseInt(process.env[ENV_VARS.PROXY_PORT] || LOCAL_ENV_VARS[ENV_VARS.PROXY_PORT], 10),-- // This is used only internaly. Some other function calling this function use different naming for groups and session- // parameters so we need to override this in error messages.- groupsParamName = 'opts.groups',- sessionParamName = 'opts.session',- countryParamName = 'opts.country',- } = options;-- const getMissingParamErrorMgs = (param, env) => `Apify Proxy ${param} must be provided as parameter or "${env}" environment variable!`;- const throwInvalidProxyValueError = (param) => {- throw new Error(`The "${param}" option can only contain the following characters: 0-9, a-z, A-Z, ".", "_" and "~"`);- };- const throwInvalidCountryCode = (code) => {- throw new Error(`The "${code}" option must be a valid two letter country code according to ISO 3166-1 alpha-2`);- };-- checkParamOrThrow(groups, groupsParamName, 'Maybe [String]');- checkParamOrThrow(session, sessionParamName, 'Maybe Number | String');- checkParamOrThrow(country, countryParamName, 'Maybe String');- checkParamOrThrow(password, 'opts.password', 'String', getMissingParamErrorMgs('password', ENV_VARS.PROXY_PASSWORD));- checkParamOrThrow(hostname, 'opts.hostname', 'String', getMissingParamErrorMgs('hostname', ENV_VARS.PROXY_HOSTNAME));- checkParamOrThrow(port, 'opts.port', 'Number', getMissingParamErrorMgs('port', ENV_VARS.PROXY_PORT));-- let username;-- if (groups || session || country) {- const parts = [];-- if (groups && groups.length) {- if (!groups.every(group => APIFY_PROXY_VALUE_REGEX.test(group))) throwInvalidProxyValueError('groups');- parts.push(`groups-${groups.join('+')}`);- }- if (session) {- if (!APIFY_PROXY_VALUE_REGEX.test(session)) throwInvalidProxyValueError('session');- parts.push(`session-${session}`);- }- if (country) {- if (!COUNTRY_CODE_REGEX.test(country)) throwInvalidCountryCode(country);- parts.push(`country-${country}`);- }-- username = parts.join(',');- } else {- username = 'auto';- }-- return `http://${username}:${password}@${hostname}:${port}`;-};-/**** Creates an ad-hoc webhook for the current actor run, which lets you receive a notification when the actor run finished or failed.diff --git a/src/crawlers/basic_crawler.js b/src/crawlers/basic_crawler.jsindex 706451c2c..a70c48d43 100644--- a/src/crawlers/basic_crawler.js+++ b/src/crawlers/basic_crawler.js@@ -18,6 +18,7 @@ import Request from '../request';import { QueueOperationInfo } from '../request_queue';import { Session } from '../session_pool/session';import { SessionPoolOptions } from '../session_pool/session_pool';+import { ProxyConfiguration, ProxyInfo } from '../proxy_configuration';/* eslint-enable no-unused-vars,import/named,import/no-duplicates,import/order *//**@@ -42,7 +43,9 @@ const SAFE_MIGRATION_WAIT_MILLIS = 20000;* ```* {* request: Request,- * autoscaledPool: AutoscaledPool+ * autoscaledPool: AutoscaledPool,+ * session: Session,+ * proxyInfo: ProxyInfo* }* ```* where the {@link Request} instance represents the URL to crawl.@@ -103,6 +106,13 @@ const SAFE_MIGRATION_WAIT_MILLIS = 20000;* If set to true. Basic crawler will initialize the {@link SessionPool} with the corresponding `sessionPoolOptions`.* The session instance will be than available in the `handleRequestFunction`.* @property {SessionPoolOptions} [sessionPoolOptions] The configuration options for {SessionPool} to use.+ * @property {ProxyConfiguration} [proxyConfiguration]+ * If set, `BasicCrawler` will be configured to use+ * [Apify Proxy](https://my.apify.com/proxy) for all connections.+ * For more information, see the [documentation](https://docs.apify.com/proxy)+ *+ * The proxyConfiguration only provides access to the proxyInfo object but the `BasicCrawler` itself+ * is not responsible for using that proxy information to make the proxied HTTP requests.*//**@@ -186,6 +196,7 @@ class BasicCrawler {},maxRequestRetries = 3,maxRequestsPerCrawl,+ proxyConfiguration,autoscaledPoolOptions = {},sessionPoolOptions = {},useSessionPool = false,@@ -199,6 +210,7 @@ class BasicCrawler {} = options;checkParamPrototypeOrThrow(requestList, 'options.requestList', RequestList, 'Apify.RequestList', true);+ checkParamPrototypeOrThrow(proxyConfiguration, 'options.proxyConfiguration', ProxyConfiguration, 'Apify.proxyConfiguration', true);checkParamPrototypeOrThrow(requestQueue, 'options.requestQueue', [RequestQueue, RequestQueueLocal], 'Apify.RequestQueue', true);checkParamOrThrow(handleRequestFunction, 'options.handleRequestFunction', 'Function');checkParamOrThrow(handleRequestTimeoutSecs, 'options.handleRequestTimeoutSecs', 'Number');@@ -216,6 +228,7 @@ class BasicCrawler {this.log = log;this.requestList = requestList;this.requestQueue = requestQueue;+ this.proxyConfiguration = proxyConfiguration;this.handleRequestFunction = handleRequestFunction;this.handleRequestTimeoutMillis = handleRequestTimeoutSecs * 1000;this.handleFailedRequestFunction = handleFailedRequestFunction;@@ -380,6 +393,7 @@ class BasicCrawler {let request;let session;+ let proxyInfo;if (this.useSessionPool) {[request, session] = await Promise.all([this._fetchNextRequest(), this.sessionPool.getSession()]);@@ -387,6 +401,10 @@ class BasicCrawler {request = await this._fetchNextRequest();}+ if (this.proxyConfiguration) {+ proxyInfo = this.proxyConfiguration.getInfo(session ? session.id : undefined);+ }+if (!request) return;// Reset loadedUrl so an old one is not carried over to retries.@@ -396,7 +414,7 @@ class BasicCrawler {this.stats.startJob(statisticsId);try {await addTimeoutToPromise(- this.handleRequestFunction({ request, autoscaledPool: this.autoscaledPool, session }),+ this.handleRequestFunction({ request, autoscaledPool: this.autoscaledPool, session, proxyInfo }),this.handleRequestTimeoutMillis,`handleRequestFunction timed out after ${this.handleRequestTimeoutMillis / 1000} seconds.`,);@@ -519,6 +537,7 @@ export default BasicCrawler;* to pause the crawler by calling {@link AutoscaledPool#pause}* or to abort it by calling {@link AutoscaledPool#abort}.* @property {Session} [session]+ * @property {ProxyInfo} [proxyInfo]*//**diff --git a/src/crawlers/cheerio_crawler.js b/src/crawlers/cheerio_crawler.jsindex 4031f93f0..bce24a3e1 100644--- a/src/crawlers/cheerio_crawler.js+++ b/src/crawlers/cheerio_crawler.js@@ -7,7 +7,6 @@ import * as htmlparser from 'htmlparser2';import * as iconv from 'iconv-lite';import * as _ from 'underscore';import * as util from 'util';-import { getApifyProxyUrl } from '../actor';import { BASIC_CRAWLER_TIMEOUT_MULTIPLIER } from '../constants';import { TimeoutError } from '../errors';import { addTimeoutToPromise, parseContentTypeFromResponse } from '../utils';@@ -23,6 +22,7 @@ import AutoscaledPool, { AutoscaledPoolOptions } from '../autoscaling/autoscaledimport { HandleFailedRequest } from './basic_crawler';import Request from '../request';import { RequestList } from '../request_list';+import { ProxyConfiguration } from '../proxy_configuration';import { RequestQueue } from '../request_queue';import { Session } from '../session_pool/session';import { SessionPoolOptions } from '../session_pool/session_pool';@@ -76,6 +76,9 @@ const DEFAULT_AUTOSCALED_POOL_OPTIONS = {** // Session object, useful to work around anti-scraping protections* session: Session+ *+ * // ProxyInfo object with information about currently used proxy+ * proxyInfo: ProxyInfo* }* ```*@@ -132,20 +135,10 @@ const DEFAULT_AUTOSCALED_POOL_OPTIONS = {* Timeout in which the HTTP request to the resource needs to finish, given in seconds.* @property {boolean} [ignoreSslErrors=true]* If set to true, SSL certificate errors will be ignored.- * @property {boolean} [useApifyProxy=false]- * If set to `true`, `CheerioCrawler` will be configured to use+ * @property {ProxyConfiguration} [proxyConfiguration]+ * If set, `CheerioCrawler` will be configured to use* [Apify Proxy](https://my.apify.com/proxy) for all connections.* For more information, see the [documentation](https://docs.apify.com/proxy)- * @property {string[]} [apifyProxyGroups]- * An array of proxy groups to be used- * by the [Apify Proxy](https://docs.apify.com/proxy).- * Only applied if the `useApifyProxy` option is `true`.- * @property {string} [apifyProxySession]- * Apify Proxy session identifier to be used with requests made by `CheerioCrawler`.- * All HTTP requests going through the proxy with the same session identifier- * will use the same target proxy server (i.e. the same IP address).- * The identifier can only contain the following characters: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.- * Only applied if the `useApifyProxy` option is `true`.* @property {string[]} [proxyUrls]* An array of custom proxy URLs to be used by the `CheerioCrawler` instance.* The provided custom proxies' order will be randomized and the resulting list rotated.@@ -314,13 +307,11 @@ class CheerioCrawler {requestTimeoutSecs = 30,handlePageTimeoutSecs = 60,ignoreSslErrors = true,- useApifyProxy = false,- apifyProxyGroups,- apifyProxySession,proxyUrls,additionalMimeTypes = [],suggestResponseEncoding,forceResponseEncoding,+ proxyConfiguration,// Autoscaled pool shorthandsminConcurrency,@@ -344,9 +335,6 @@ class CheerioCrawler {checkParamOrThrow(requestTimeoutSecs, 'options.requestTimeoutSecs', 'Number');checkParamOrThrow(handlePageTimeoutSecs, 'options.handlePageTimeoutSecs', 'Number');checkParamOrThrow(ignoreSslErrors, 'options.ignoreSslErrors', 'Maybe Boolean');- checkParamOrThrow(useApifyProxy, 'options.useApifyProxy', 'Maybe Boolean');- checkParamOrThrow(apifyProxyGroups, 'options.apifyProxyGroups', 'Maybe [String]');- checkParamOrThrow(apifyProxySession, 'options.apifyProxySession', 'Maybe String');checkParamOrThrow(proxyUrls, 'options.proxyUrls', 'Maybe [String]');checkParamOrThrow(prepareRequestFunction, 'options.prepareRequestFunction', 'Maybe Function');checkParamOrThrow(additionalMimeTypes, 'options.additionalMimeTypes', '[String]');@@ -360,15 +348,11 @@ class CheerioCrawler {// Enforce valid proxy configurationif (proxyUrls && !proxyUrls.length) throw new Error('Parameter "options.proxyUrls" of type Array must not be empty');- if (useApifyProxy && proxyUrls) throw new Error('Cannot combine "options.useApifyProxy" with "options.proxyUrls"!');+ if (proxyConfiguration && proxyUrls) throw new Error('Cannot combine "options.proxyConfiguration" with "options.proxyUrls"!');if (persistCookiesPerSession && !useSessionPool) {throw new Error('Cannot use "options.persistCookiesPerSession" without "options.useSessionPool"');}- if (apifyProxySession && useSessionPool) {- throw new Error('Cannot use "options.apifyProxySession" with "options.useSessionPool"');- }-this.supportedMimeTypes = new Set(DEFAULT_MIME_TYPES);if (additionalMimeTypes.length) this._extendSupportedMimeTypes(additionalMimeTypes);@@ -387,14 +371,12 @@ class CheerioCrawler {this.handlePageTimeoutMillis = handlePageTimeoutSecs * 1000;this.requestTimeoutMillis = requestTimeoutSecs * 1000;this.ignoreSslErrors = ignoreSslErrors;- this.useApifyProxy = useApifyProxy;- this.apifyProxyGroups = apifyProxyGroups;- this.apifyProxySession = apifyProxySession;this.proxyUrls = _.shuffle(proxyUrls);this.suggestResponseEncoding = suggestResponseEncoding;this.forceResponseEncoding = forceResponseEncoding;this.lastUsedProxyUrlIndex = 0;this.prepareRequestFunction = prepareRequestFunction;+ this.proxyConfiguration = proxyConfiguration;this.persistCookiesPerSession = persistCookiesPerSession;this.useSessionPool = useSessionPool;this.sessionPoolOptions = sessionPoolOptions;@@ -409,6 +391,7 @@ class CheerioCrawler {handleRequestFunction: (...args) => this._handleRequestFunction(...args),handleRequestTimeoutSecs: handlePageTimeoutSecs * BASIC_CRAWLER_TIMEOUT_MULTIPLIER,handleFailedRequestFunction,+ proxyConfiguration,// Autoscaled pool options.minConcurrency,@@ -484,9 +467,10 @@ class CheerioCrawler {* @param {Request} options.request* @param {AutoscaledPool} options.autoscaledPool* @param {Session} options.session+ * @param {ProxyInfo} options.proxyInfo* @ignore*/- async _handleRequestFunction({ request, autoscaledPool, session }) {+ async _handleRequestFunction({ request, autoscaledPool, session, proxyInfo }) {if (this.prepareRequestFunction) await this.prepareRequestFunction({ request, session });const { dom, isXml, body, contentType, response } = await addTimeoutToPromise(this._requestFunction({ request, session }),@@ -533,6 +517,7 @@ class CheerioCrawler {response,autoscaledPool,session,+ proxyInfo,};return addTimeoutToPromise(this.handlePageFunction(context),@@ -649,13 +634,8 @@ class CheerioCrawler {* @ignore*/_getProxyUrl(session = {}) {- if (this.useApifyProxy) {- return getApifyProxyUrl({- groups: this.apifyProxyGroups,- session: session.id || this.apifyProxySession,- groupsParamName: 'options.apifyProxyGroups',- sessionParamName: 'options.apifyProxySession',- });+ if (this.proxyConfiguration) {+ return this.proxyConfiguration.getUrl(session.id);}if (this.proxyUrls) {return this.proxyUrls[this.lastUsedProxyUrlIndex++ % this.proxyUrls.length];@@ -795,6 +775,9 @@ export default CheerioCrawler;* to pause the crawler by calling {@link AutoscaledPool#pause}* or to abort it by calling {@link AutoscaledPool#abort}.* @property {Session} [session]+ * @property {ProxyInfo} [proxyInfo]+ * An object with information about currently used proxy by the crawler+ * and configured by the {@link ProxyConfiguration} class.*//**diff --git a/src/crawlers/puppeteer_crawler.js b/src/crawlers/puppeteer_crawler.jsindex 4e567b403..14ff1b73f 100644--- a/src/crawlers/puppeteer_crawler.js+++ b/src/crawlers/puppeteer_crawler.js@@ -36,6 +36,7 @@ import { SessionPoolOptions } from '../session_pool/session_pool';* puppeteerPool: PuppeteerPool,* autoscaledPool: AutoscaledPool,* session: Session,+ * proxyInfo: ProxyInfo,* }* ```*@@ -76,8 +77,6 @@ import { SessionPoolOptions } from '../session_pool/session_pool';* Note that a single page object is only used to process a single request and it is closed afterwards.** By default, the function invokes {@link puppeteer#gotoExtended} with a timeout of 60 seconds.- * For details, see source code on- * [GitHub](https://github.com/apifytech/apify-js/blob/master/src/crawlers/puppeteer_crawler.js#L292).* @property {number} [gotoTimeoutSecs=60]* Timeout in which page navigation needs to finish, in seconds. When `gotoFunction()` is used and thus the default* function is overridden, this timeout will not be used and needs to be configured in the new `gotoFunction()`.@@ -93,10 +92,6 @@ import { SessionPoolOptions } from '../session_pool/session_pool';* ```* Where the {@link Request} instance corresponds to the failed request, and the `Error` instance* represents the last error thrown during processing of the request.- *- * See- * [source code](https://github.com/apifytech/apify-js/blob/master/src/crawlers/puppeteer_crawler.js#L301)- * for the default implementation of this function.* @property {number} [maxRequestRetries=3]* Indicates how many times the request is retried if either `handlePageFunction()` or `gotoFunction()` fails.* @property {number} [maxRequestsPerCrawl]@@ -136,6 +131,10 @@ import { SessionPoolOptions } from '../session_pool/session_pool';* Custom options passed to the underlying {@link SessionPool} constructor.* @property {boolean} [persistCookiesPerSession=false]* Automatically saves cookies to Session. Works only if Session Pool is used.+ * @property {ProxyConfiguration} [proxyConfiguration]+ * If set, `PuppeteerCrawler` will be configured to use+ * [Apify Proxy](https://my.apify.com/proxy) for all connections.+ * For more information, see the [documentation](https://docs.apify.com/proxy)*//**@@ -239,6 +238,7 @@ class PuppeteerCrawler {sessionPoolOptions = {},persistCookiesPerSession = false,useSessionPool = false,+ proxyConfiguration,} = options;checkParamOrThrow(handlePageFunction, 'options.handlePageFunction', 'Function');@@ -277,6 +277,7 @@ class PuppeteerCrawler {log: this.log,};this.persistCookiesPerSession = persistCookiesPerSession;+ this.proxyConfiguration = proxyConfiguration;/** @ignore */this.basicCrawler = new BasicCrawler({@@ -311,6 +312,11 @@ class PuppeteerCrawler {this.sessionPool = await openSessionPool(this.sessionPoolOptions);this.puppeteerPoolOptions.sessionPool = this.sessionPool;}++ if (this.proxyConfiguration) {+ this.puppeteerPoolOptions.proxyConfiguration = this.proxyConfiguration;+ }+this.puppeteerPoolOptions.log = this.log;this.puppeteerPool = new PuppeteerPool(this.puppeteerPoolOptions);try {@@ -336,20 +342,27 @@ class PuppeteerCrawler {*/async _handleRequestFunction({ request, autoscaledPool }) {let session;+ let proxyInfo;const page = await this.puppeteerPool.newPage();if (this.sessionPool) {const browser = page.browser();session = browser[BROWSER_SESSION_KEY_NAME];+// setting cookies to pageif (this.persistCookiesPerSession) {await page.setCookie(...session.getPuppeteerCookies(request.url));}}+ if (this.proxyConfiguration) {+ proxyInfo = this.proxyConfiguration.getInfo(session ? session.id : undefined);+ }++try {- const response = await this.gotoFunction({ page, request, autoscaledPool, puppeteerPool: this.puppeteerPool, session });+ const response = await this.gotoFunction({ page, request, autoscaledPool, puppeteerPool: this.puppeteerPool, session, proxyInfo });await this.puppeteerPool.serveLiveViewSnapshot(page);request.loadedUrl = page.url();@@ -360,7 +373,7 @@ class PuppeteerCrawler {}await addTimeoutToPromise(- this.handlePageFunction({ page, request, autoscaledPool, puppeteerPool: this.puppeteerPool, response, session }),+ this.handlePageFunction({ page, request, autoscaledPool, puppeteerPool: this.puppeteerPool, response, session, proxyInfo }),this.handlePageTimeoutMillis,`handlePageFunction timed out after ${this.handlePageTimeoutMillis / 1000} seconds.`,);@@ -415,6 +428,7 @@ export default PuppeteerCrawler;* to pause the crawler by calling {@link AutoscaledPool#pause}* or to abort it by calling {@link AutoscaledPool#abort}.* @property {Session} [session]+ * @property {ProxyInfo} [proxyInfo]*//**@@ -432,6 +446,7 @@ export default PuppeteerCrawler;* @property {AutoscaledPool} autoscaledPool An instance of the `AutoscaledPool`.* @property {PuppeteerPool} puppeteerPool An instance of the {@link PuppeteerPool} used by this `PuppeteerCrawler`.* @property {Session} [session] `Session` object for this request.+ * @property {ProxyInfo} [proxyInfo]*//**diff --git a/src/main.js b/src/main.jsindex f8577f214..175322a8b 100644--- a/src/main.js+++ b/src/main.js@@ -1,6 +1,6 @@import { EventEmitter } from 'events';import log from './utils_log';-import { main, getEnv, call, callTask, getApifyProxyUrl, metamorph, addWebhook } from './actor';+import { main, getEnv, call, callTask, metamorph, addWebhook } from './actor';import AutoscaledPool from './autoscaling/autoscaled_pool';import BasicCrawler from './crawlers/basic_crawler';import CheerioCrawler from './crawlers/cheerio_crawler';@@ -12,6 +12,7 @@ import PuppeteerCrawler from './crawlers/puppeteer_crawler';import PuppeteerPool from './puppeteer_pool';import Request from './request';import { RequestList, openRequestList } from './request_list';+import { createProxyConfiguration } from './proxy_configuration';import { openRequestQueue } from './request_queue';import { apifyClient, getMemoryInfo, isAtHome, publicUtils } from './utils';import { puppeteerUtils } from './puppeteer_utils';@@ -49,7 +50,6 @@ export {callTask,metamorph,getMemoryInfo,- getApifyProxyUrl,isAtHome,apifyClient as client,addWebhook,@@ -85,6 +85,8 @@ export {openSessionPool,+ createProxyConfiguration,+LiveViewServer,Session,diff --git a/src/proxy_configuration.js b/src/proxy_configuration.jsnew file mode 100644index 000000000..9c41180b3--- /dev/null+++ b/src/proxy_configuration.js@@ -0,0 +1,366 @@+import { checkParamOrThrow } from 'apify-client/build/utils';+import { ENV_VARS, LOCAL_ENV_VARS } from 'apify-shared/consts';+import { APIFY_PROXY_VALUE_REGEX } from 'apify-shared/regexs';+import { COUNTRY_CODE_REGEX } from './constants';+import { apifyClient } from './utils';+import { requestAsBrowser } from './utils_request';+import log from './utils_log';++// CONSTANTS+const PROTOCOL = 'http';+const APIFY_PROXY_STATUS_URL = 'http://proxy.apify.com/?format=json';++/**+ * @typedef ProxyConfigurationOptions+ * @property {string} [password] - User's password for the proxy. By default, it is taken from the `APIFY_PROXY_PASSWORD`+ * environment variable, which is automatically set by the system when running the actors.+ * @property {string[]} [groups] - An array of proxy groups to be used+ * by the [Apify Proxy](https://docs.apify.com/proxy). If not provided, the proxy will select+ * the groups automatically.+ * @property {string} [countryCode] - If set and relevant proxies are available in your Apify account, all proxied requests will+ * use IP addresses that are geolocated to the specified country. For example `GB` for IPs+ * from Great Britain. Note that online services often have their own rules for handling+ * geolocation and thus the country selection is a best attempt at geolocation, rather than+ * a guaranteed hit. This parameter is optional, by default, each proxied request is assigned+ * an IP address from a random country. The country code needs to be a two letter ISO country code. See the+ * [full list of available country codes](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2#Officially_assigned_code_elements).+ * This parameter is optional, by default, the proxy uses all available proxy servers from all countries.+ * on the Apify cloud, or when using the [Apify CLI](https://github.com/apifytech/apify-cli).+ * @property {string[]} [apifyProxyGroups] - Same option as `groups` which can be used to+ * configurate the proxy by UI input schema. You should use the `groups` option in your crawler code.+ * @property {string} [apifyProxyCountry] - Same option as `countryCode` which can be used to+ * configurate the proxy by UI input schema. You should use the `countryCode` option in your crawler code.+ */++/**+ * The main purpose of the ProxyInfo object is to provide information+ * about the proxy used by the crawler for the current request.+ * Outside of crawlers, you can get this object by calling {@link ProxyConfiguration.getInfo}.+ *+ * **Example usage:**+ *+ * ```javascript+ *+ * const proxyConfiguration = await Apify.createProxyConfiguration({+ * groups: ['GROUP1', 'GROUP2'] // List of Apify Proxy groups+ * countryCode: 'US',+ * });+ *+ * // Getting proxyInfo object by calling class method directly+ * const proxyInfo = proxyConfiguration.getInfo();+ *+ * // In crawler+ * const crawler = new Apify.CheerioCrawler({+ * // ...+ * proxyConfiguration,+ * handlePageFunction: ({ proxyInfo }) => {+ * // Getting used Proxy URL+ * const proxyUrl = proxyInfo.url;+ *+ * // Getting ID of used Session+ * const sessionIdentifier = proxyInfo.sessionId;+ * }+ * })+ *+ * ```+ * @typedef ProxyInfo+ * @property {string} [sessionId] - The identifier of used {@link Session}, if used.+ * @property {string} url - The proxy URL.+ * @property {string[]} groups - An array of proxy groups to be used+ * by the [Apify Proxy](https://docs.apify.com/proxy). If not provided, the proxy will select+ * the groups automatically.+ * @property {string} [countryCode] - If set and relevant proxies are available in your Apify account, all proxied requests will+ * use IP addresses that are geolocated to the specified country. For example `GB` for IPs+ * from Great Britain. Note that online services often have their own rules for handling+ * geolocation and thus the country selection is a best attempt at geolocation, rather than+ * a guaranteed hit. This parameter is optional, by default, each proxied request is assigned+ * an IP address from a random country. The country code needs to be a two letter ISO country code. See the+ * [full list of available country codes](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2#Officially_assigned_code_elements).+ * This parameter is optional, by default, the proxy uses all available proxy servers from all countries.+ * @property {string} password - User's password for the proxy. By default, it is taken from the `APIFY_PROXY_PASSWORD`+ * environment variable, which is automatically set by the system when running the actors+ * on the Apify cloud, or when using the [Apify CLI](https://github.com/apifytech/apify-cli).+ * @property {string} hostname - Hostname of your proxy.+ * @property {string} port - Proxy port.+ */++/**+ * Configures connection to a proxy server with the provided options. Proxy servers are used to prevent target websites from blocking+ * your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures+ * them to use the selected proxies for all connections. You can get information about the currently used proxy by inspecting+ * the {@link ProxyInfo} property in your crawler's page function. There, you can inspect the proxy's URL and other attributes.+ *+ * The proxy servers are managed by [Apify Proxy](https://docs.apify.com/proxy). To be able to use Apify Proxy,+ * you need an Apify account and access to the selected proxies. If you provide no configuration option,+ * the proxies will be managed automatically using a smart algorithm.+ *+ * **Example usage:**+ *+ * ```javascript+ *+ * const proxyConfiguration = await Apify.createProxyConfiguration({+ * groups: ['GROUP1', 'GROUP2'] // List of Apify Proxy groups+ * countryCode: 'US',+ * });+ *+ * const crawler = new Apify.CheerioCrawler({+ * // ...+ * proxyConfiguration,+ * handlePageFunction: ({ proxyInfo }) => {+ * const usedProxyUrl = proxyInfo.url; // Getting the Proxy URL+ * }+ * })+ *+ * ```+ * @hideconstructor+ */+export class ProxyConfiguration {+ /**+ * Configuration of proxy.+ *+ * @param {ProxyConfigurationOptions} [options] All `ProxyConfiguration` options.+ */+ constructor(options = {}) {+ const {+ groups = [],+ apifyProxyGroups = [],+ countryCode,+ apifyProxyCountry,+ password = process.env[ENV_VARS.PROXY_PASSWORD],+ } = options;++ const groupsToUse = groups.length ? groups : apifyProxyGroups;+ const countryCodeToUse = countryCode || apifyProxyCountry;+ const hostname = process.env[ENV_VARS.PROXY_HOSTNAME] || LOCAL_ENV_VARS[ENV_VARS.PROXY_HOSTNAME];+ const port = Number(process.env[ENV_VARS.PROXY_PORT] || LOCAL_ENV_VARS[ENV_VARS.PROXY_PORT]);++ // Validation+ checkParamOrThrow(groupsToUse, 'opts.groups', '[String]');+ checkParamOrThrow(countryCodeToUse, 'opts.countryCode', 'Maybe String');+ checkParamOrThrow(password, 'opts.password', 'Maybe String');+ this._validateArgumentStructure(groupsToUse, countryCodeToUse);++ this.groups = groupsToUse;+ this.countryCode = countryCodeToUse;+ this.password = password;+ this.hostname = hostname;+ this.port = port;+ }++ /**+ * Loads proxy password if token is provided and checks access to Apify Proxy and provided proxy groups.+ * Also checks if country has access to Apify Proxy groups if the country code is provided.+ *+ * You should use the {@link Apify.createProxyConfiguration} function to create a pre-initialized+ * `ProxyConfiguration` instance instead of calling this manually.+ *+ * @returns {Promise<void>}+ */+ async initialize() {+ await this._setPasswordIfToken();++ await this._checkAccess();++ // TODO: Validate proxyUrl each of custom proxies+ }+++ /**+ * Gets information about proxy and its configuration.+ * @param {string} sessionId+ * Apify Proxy [Session](https://docs.apify.com/proxy/datacenter-proxy#session-persistence) identifier+ * to be used with requests.+ * All HTTP requests going through the proxy with the same session identifier+ * will use the same target proxy server (i.e. the same IP address).+ * The identifier can only contain the following characters: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.+ * @return {ProxyInfo} represents information about used proxy configuration.+ */+ getInfo(sessionId) {+ if (sessionId) this._validateSessionArgumentStructure(sessionId);+ const { groups, countryCode, password, port, hostname } = this;+ const url = this.getUrl(sessionId);++ return {+ sessionId,+ url,+ groups,+ countryCode,+ password,+ hostname,+ port,+ };+ }++ /**+ * Returns the proxy URL.+ * @param {string} sessionId+ * Apify Proxy [Session](https://docs.apify.com/proxy/datacenter-proxy#session-persistence) identifier+ * to be used with requests.+ * All HTTP requests going through the proxy with the same session identifier+ * will use the same target proxy server (i.e. the same IP address).+ * The identifier can only contain the following characters: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.+ * @return {string} represents the proxy URL.+ */+ getUrl(sessionId) {+ if (sessionId) this._validateSessionArgumentStructure(sessionId);+ const username = this._getUsername(sessionId);+ const { password, hostname, port } = this;++ return `${PROTOCOL}://${username}:${password}@${hostname}:${port}`;+ }++ /**+ * Returns proxy username.+ * @return {string} the proxy username+ * @param {string} sessionId+ * @ignore+ */+ _getUsername(sessionId) {+ let username;+ const { groups, countryCode } = this;+ const parts = [];++ if (groups && groups.length) {+ parts.push(`groups-${groups.join('+')}`);+ }+ if (sessionId) {+ parts.push(`session-${sessionId}`);+ }+ if (countryCode) {+ parts.push(`country-${countryCode}`);+ }++ username = parts.join(',');++ if (parts.length === 0) username = 'auto';++ return username;+ }++ /**+ * Checks if Apify Token is provided in env+ * and gets the password via API and sets it to env+ * @returns {Promise<void>}+ * @ignore+ */+ async _setPasswordIfToken() {+ const token = process.env[ENV_VARS.TOKEN] || LOCAL_ENV_VARS[ENV_VARS.TOKEN];+ if (token) {+ const { proxy: { password } } = await apifyClient.users.getUser({ token, userId: 'me' });+ if (this.password) {+ if (this.password !== password) {+ log.warning('The Apify Proxy password you provided belongs to'+ + ' a different user than the Apify token you are using. Are you sure this is correct?');+ }+ } else {+ this.password = password;+ }+ }+ if (!this.password) {+ throw new Error(`Apify Proxy password must be provided using options.password or the "${ENV_VARS.PROXY_PASSWORD}" environment variable!`);+ }+ }++ /**+ * Checks the status of Apify Proxy and throws an error if the status is not "connected".+ * @returns {Promise<void>}+ * @ignore+ */+ async _checkAccess() {+ const url = APIFY_PROXY_STATUS_URL;+ const proxyUrl = this.getUrl();+ const { countryCode } = this;+ const { body: { connected, connectionError } } = await requestAsBrowser({ url, proxyUrl, countryCode, json: true });+ if (!connected) this._throwApifyProxyConnectionError(connectionError);+ }++ /**+ * Validates session argument structure+ * @param {string} sessionId+ * @ignore+ */+ _validateSessionArgumentStructure(sessionId) {+ if (!APIFY_PROXY_VALUE_REGEX.test(sessionId)) this._throwInvalidProxyValueError(sessionId);+ }++ /**+ * Validates groups and countryCode options correct structure+ * @param {string[]} groups+ * @param {string} countryCode+ * @ignore+ */+ _validateArgumentStructure(groups, countryCode) {+ for (const group of groups) {+ if (!APIFY_PROXY_VALUE_REGEX.test(group)) this._throwInvalidProxyValueError(group);+ }+ if (countryCode) {+ if (!COUNTRY_CODE_REGEX.test(countryCode)) this._throwInvalidCountryCode(countryCode);+ }+ }++ /**+ * Throws invalid proxy value error+ * @param {string} param+ * @ignore+ */+ _throwInvalidProxyValueError(param) {+ throw new Error(`The provided proxy group name "${param}" can only contain the following characters: 0-9, a-z, A-Z, ".", "_" and "~"`);+ }++ /**+ * Throws invalid country code error+ * @param {string} code+ * @ignore+ */+ _throwInvalidCountryCode(code) {+ throw new Error(`The provided country code "${code}" is not valid. Please use a two letter country code according to ISO 3166-1 alpha-2`);+ }++ /**+ * Throws Apify Proxy is not connected+ * @ignore+ */+ _throwApifyProxyConnectionError(errorMessage) {+ throw new Error(errorMessage);+ }+}++/**+ * Creates a proxy configuration and returns a promise resolving to an instance+ * of the {@link ProxyConfiguration} class that is already initialized.+ *+ * Configures connection to a proxy server with the provided options. Proxy servers are used to prevent target websites from blocking+ * your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures+ * them to use the selected proxies for all connections.+ *+ * For more details and code examples, see the {@link ProxyConfiguration} class.+ *+ * ```javascript+ *+ * // Returns initialized proxy configuration class+ * const proxyConfiguration = await Apify.createProxyConfiguration({+ * groups: ['GROUP1', 'GROUP2'] // List of Apify proxy groups+ * countryCode: 'US'+ * });+ *+ * const crawler = new Apify.CheerioCrawler({+ * // ...+ * proxyConfiguration,+ * handlePageFunction: ({ proxyInfo }) => {+ * const usedProxyUrl = proxyInfo.url; // Getting the Proxy URL+ * }+ * })+ *+ * ```+* @param {ProxyConfigurationOptions} proxyConfigurationOptions+* @returns {Promise<ProxyConfiguration>}+* @memberof module:Apify+* @name createProxyConfiguration+* @function+ */+export const createProxyConfiguration = async (proxyConfigurationOptions) => {+ const proxyConfiguration = new ProxyConfiguration(proxyConfigurationOptions);+ await proxyConfiguration.initialize();++ return proxyConfiguration;+};diff --git a/src/puppeteer.js b/src/puppeteer.jsindex 012b5f54f..541b0306b 100644--- a/src/puppeteer.js+++ b/src/puppeteer.js@@ -6,12 +6,10 @@ import { Browser } from 'puppeteer'; // eslint-disable-line no-unused-varsimport { DEFAULT_USER_AGENT } from './constants';import log from './utils_log';import { getTypicalChromeExecutablePath, isAtHome } from './utils';-import { getApifyProxyUrl } from './actor';import applyStealthToBrowser, { StealthOptions } from './stealth/stealth'; // eslint-disable-line no-unused-vars,import/namedconst LAUNCH_PUPPETEER_LOG_OMIT_OPTS = [- 'proxyUrl', 'userAgent', 'useApifyProxy', 'apifyProxyGroups',- 'apifyProxySession', 'puppeteerModule', 'stealthOptions',+ 'proxyUrl', 'userAgent', 'puppeteerModule', 'stealthOptions',];const LAUNCH_PUPPETEER_DEFAULT_VIEWPORT = {@@ -127,20 +125,6 @@ const getPuppeteerOrThrow = (puppeteerModule = 'puppeteer') => {* is taken from the `APIFY_CHROME_EXECUTABLE_PATH` environment variable if provided,* or defaults to the typical Google Chrome executable location specific for the operating system.* By default, this option is `false`.- * @property {boolean} [useApifyProxy=false]- * If set to `true`, Puppeteer will be configured to use- * [Apify Proxy](https://my.apify.com/proxy) for all connections.- * For more information, see the [documentation](https://docs.apify.com/proxy)- * @property {string[]} [apifyProxyGroups]- * An array of proxy groups to be used- * by the [Apify Proxy](https://docs.apify.com/proxy).- * Only applied if the `useApifyProxy` option is `true`.- * @property {string} [apifyProxySession]- * Apify Proxy session identifier to be used by all the Chrome browsers.- * All HTTP requests going through the proxy with the same session identifier- * will use the same target proxy server (i.e. the same IP address).- * The identifier can only contain the following characters: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.- * Only applied if the `useApifyProxy` option is `true`.* @property {(string|Object)} [puppeteerModule]* Either a require path (`string`) to a package to be used instead of default `puppeteer`,* or an already required module (`Object`). This enables usage of various Puppeteer@@ -209,11 +193,9 @@ export const launchPuppeteer = async (options = {}) => {checkParamOrThrow(options, 'options', 'Object');checkParamOrThrow(options.args, 'options.args', 'Maybe [String]');checkParamOrThrow(options.proxyUrl, 'options.proxyUrl', 'Maybe String');- checkParamOrThrow(options.useApifyProxy, 'options.useApifyProxy', 'Maybe Boolean');checkParamOrThrow(options.puppeteerModule, 'options.puppeteerModule', 'Maybe String|Object');checkParamOrThrow(options.stealth, 'options.stealth', 'Maybe Boolean');checkParamOrThrow(options.stealthOptions, 'options.stealthOptions', 'Maybe Object');- if (options.useApifyProxy && options.proxyUrl) throw new Error('Cannot combine "options.useApifyProxy" with "options.proxyUrl"!');if (options.liveView || options.liveViewOptions) {log.deprecated('Live view is no longer available in Apify.launchPuppeteer() and launchPuppeteerOptions. '+ 'Use options.useLiveView in PuppeteerPool for an updated version. '@@ -232,14 +214,7 @@ export const launchPuppeteer = async (options = {}) => {if (optsCopy.useChrome && (optsCopy.executablePath === undefined || optsCopy.executablePath === null)) {optsCopy.executablePath = process.env[ENV_VARS.CHROME_EXECUTABLE_PATH] || getTypicalChromeExecutablePath();}- if (optsCopy.useApifyProxy) {- optsCopy.proxyUrl = getApifyProxyUrl({- groups: optsCopy.apifyProxyGroups,- session: optsCopy.apifyProxySession,- groupsParamName: 'options.apifyProxyGroups',- sessionParamName: 'options.apifyProxySession',- });- }+if (optsCopy.defaultViewport === undefined) {optsCopy.defaultViewport = LAUNCH_PUPPETEER_DEFAULT_VIEWPORT;}diff --git a/src/puppeteer_pool.js b/src/puppeteer_pool.jsindex dbd35af94..210321b1c 100644--- a/src/puppeteer_pool.js+++ b/src/puppeteer_pool.js@@ -90,19 +90,24 @@ class PuppeteerInstance {* Note that the new browser starts with empty cookies, local storage etc. so this setting doesn't affect anonymity of your crawler.** Beware that the disk cache directories can consume a lot of disk space.- * To limit the space consumed, you can pass the `--disk-cache-size=X` argument to `launchPuppeteerargs`,+ * To limit the space consumed, you can pass the `--disk-cache-size=X` argument to `launchPuppeteer` `args`,* where `X` is the approximate maximum number of bytes for disk cache.** Do not use the `recycleDiskCache` setting together with `--disk-cache-dir`- * argument in `launchPuppeteerargs`, the behavior is undefined.+ * argument in `launchPuppeteer` `args`, the behavior is undefined.* @property {boolean} [useIncognitoPages]* With this option selected, all pages will be opened in a new incognito browser context, which means* that they will not share cookies or cache and their resources will not be throttled by one another.+ * @property {SessionPool} [sessionPool]+ * A pool of Session instances.* @property {string[]} [proxyUrls]* An array of custom proxy URLs to be used by the `PuppeteerPool` instance.* The provided custom proxies' order will be randomized and the resulting list rotated.* Custom proxies are not compatible with Apify Proxy and an attempt to use both* configuration options will cause an error to be thrown on startup.+ * @property {ProxyConfiguration} [proxyConfiguration]+ * If set, `PuppeteerPool` will be configured to use+ * [Apify Proxy](https://my.apify.com/proxy) for all connections.*//**@@ -123,18 +128,16 @@ class PuppeteerInstance {** ```javascript* const puppeteerPool = new PuppeteerPool({- * launchPuppeteerFunction: () => {- * // Use a new proxy with a new IP address for each new Chrome instance- * return Apify.launchPuppeteer({- * useApifyProxy: true,- * apifyProxySession: Math.random(),- * });+ * launchPuppeteerOptions: {+ * // Use Chrome instead of Chromium+ * useChrome: true,+ * // Hide from bot detection+ * stealth: true,* },* });** const page1 = await puppeteerPool.newPage();* const page2 = await puppeteerPool.newPage();- * const page3 = await puppeteerPool.newPage();** // ... do something with the pages ...*@@ -164,6 +167,7 @@ class PuppeteerPool {recycleDiskCache = false,useIncognitoPages = false,proxyUrls,+ proxyConfiguration,useLiveView = false,sessionPool,log = defaultLog,@@ -197,9 +201,12 @@ class PuppeteerPool {checkParamOrThrow(proxyUrls, 'options.proxyUrls', 'Maybe Array');// Enforce non-empty proxyUrls arrayif (proxyUrls && !proxyUrls.length) throw new Error('Parameter "options.proxyUrls" of type Array must not be empty');+ if (proxyConfiguration && proxyUrls) throw new Error('Cannot combine "options.proxyConfiguration" with "options.proxyUrls"!');+checkParamOrThrow(useLiveView, 'options.useLiveView', 'Maybe Boolean');checkParamOrThrow(sessionPool, 'options.sessionPool', 'Maybe Object');+// Config.this.sessionPool = sessionPool;this.reusePages = reusePages;@@ -215,6 +222,7 @@ class PuppeteerPool {this.recycledDiskCacheDirs = recycleDiskCache ? new LinkedList() : null;this.useIncognitoPages = useIncognitoPages;this.proxyUrls = proxyUrls ? _.shuffle(proxyUrls) : null;+ this.proxyConfiguration = proxyConfiguration;this.liveViewServer = useLiveView ? new LiveViewServer() : null;this.launchPuppeteerFunction = async () => {// Do not modify passed launchPuppeteerOptions!@@ -241,7 +249,10 @@ class PuppeteerPool {let session;if (sessionPool) {session = await sessionPool.getSession();- opts.apifyProxySession = session.id;+ }++ if (proxyConfiguration) {+ opts.proxyUrl = proxyConfiguration.getUrl(session ? session.id : undefined);}const browser = await launchPuppeteerFunction(opts);diff --git a/src/session_pool/session.js b/src/session_pool/session.jsindex eeb9c6921..4bb136159 100644--- a/src/session_pool/session.js+++ b/src/session_pool/session.js@@ -350,6 +350,7 @@ export class Session {/*** Calculate cookie expiration date* @return {Date} - calculated date by session max age seconds.+ * @private*/_getDefaultCookieExpirationDate(maxAgeSecs) {return new Date(Date.now() + (maxAgeSecs * 1000));diff --git a/test/actor.test.js b/test/actor.test.jsindex 25a93b9ee..3c79b64f5 100644--- a/test/actor.test.js+++ b/test/actor.test.js@@ -1,7 +1,7 @@import path from 'path';import _ from 'underscore';import sinon from 'sinon';-import { ENV_VARS, ACT_JOB_STATUSES, LOCAL_ENV_VARS } from 'apify-shared/consts';+import { ENV_VARS, ACT_JOB_STATUSES } from 'apify-shared/consts';import { ApifyCallError } from '../build/errors';import { sleep } from '../build/utils';@@ -1042,136 +1042,6 @@ describe('Apify.metamorph()', () => {});});-describe('Apify.getApifyProxyUrl()', () => {- test('should work', () => {- process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';- process.env[ENV_VARS.PROXY_HOSTNAME] = 'my.host.com';- process.env[ENV_VARS.PROXY_PORT] = 123;-- expect(Apify.getApifyProxyUrl({- session: 'XYZ',- groups: ['g1', 'g2', 'g3'],- country: 'US',- })).toBe('http://groups-g1+g2+g3,session-XYZ,country-US:[email protected]:123');-- expect(Apify.getApifyProxyUrl({- session: 'XYZ',- groups: ['g1', 'g2', 'g3'],- })).toBe('http://groups-g1+g2+g3,session-XYZ:[email protected]:123');-- expect(Apify.getApifyProxyUrl({- groups: ['g1', 'g2', 'g3'],- })).toBe('http://groups-g1+g2+g3:[email protected]:123');-- expect(Apify.getApifyProxyUrl({- session: 'XYZ',- })).toBe('http://session-XYZ:[email protected]:123');-- expect(Apify.getApifyProxyUrl({ country: 'US' })).toBe('http://country-US:[email protected]:123');-- expect(Apify.getApifyProxyUrl()).toBe('http://auto:[email protected]:123');--- delete process.env[ENV_VARS.PROXY_PASSWORD];- delete process.env[ENV_VARS.PROXY_HOSTNAME];- delete process.env[ENV_VARS.PROXY_PORT];-- expect(Apify.getApifyProxyUrl({ password: 'xyz' })).toEqual(- `http://auto:xyz@${LOCAL_ENV_VARS[ENV_VARS.PROXY_HOSTNAME]}:${LOCAL_ENV_VARS[ENV_VARS.PROXY_PORT]}`,- );-- expect(() => Apify.getApifyProxyUrl()).toThrowError();-- expect(Apify.getApifyProxyUrl({- password: 'xyz',- hostname: 'your.host.com',- port: 345,- })).toBe('http://auto:[email protected]:345');- });-- // Test old params - session, groups- test('should be backwards compatible', () => {- const ll = log.getLevel();- log.setLevel(log.LEVELS.ERROR);- process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';- process.env[ENV_VARS.PROXY_HOSTNAME] = 'my.host.com';- process.env[ENV_VARS.PROXY_PORT] = 123;-- expect(Apify.getApifyProxyUrl({- apifyProxySession: 'XYZ',- apifyProxyGroups: ['g1', 'g2', 'g3'],- })).toBe('http://groups-g1+g2+g3,session-XYZ:[email protected]:123');-- expect(Apify.getApifyProxyUrl({- apifyProxyGroups: ['g1', 'g2', 'g3'],- })).toBe('http://groups-g1+g2+g3:[email protected]:123');-- expect(Apify.getApifyProxyUrl({- apifyProxySession: 'XYZ',- })).toBe('http://session-XYZ:[email protected]:123');-- expect(Apify.getApifyProxyUrl()).toBe('http://auto:[email protected]:123');-- delete process.env[ENV_VARS.PROXY_PASSWORD];- delete process.env[ENV_VARS.PROXY_HOSTNAME];- delete process.env[ENV_VARS.PROXY_PORT];-- expect(Apify.getApifyProxyUrl({ password: 'xyz' })).toEqual(- `http://auto:xyz@${LOCAL_ENV_VARS[ENV_VARS.PROXY_HOSTNAME]}:${LOCAL_ENV_VARS[ENV_VARS.PROXY_PORT]}`,- );-- expect(() => Apify.getApifyProxyUrl()).toThrowError();-- expect(Apify.getApifyProxyUrl({- password: 'xyz',- hostname: 'your.host.com',- port: 345,- })).toBe('http://auto:[email protected]:345');- log.setLevel(ll);- });-- test('should throw on invalid proxy args', () => {- process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';- process.env[ENV_VARS.PROXY_HOSTNAME] = 'my.host.com';- process.env[ENV_VARS.PROXY_PORT] = 123;-- expect(() => Apify.getApifyProxyUrl({ session: 'a-b' })).toThrowError();- expect(() => Apify.getApifyProxyUrl({ session: 'a$b' })).toThrowError();- expect(() => Apify.getApifyProxyUrl({ session: {} })).toThrowError();- expect(() => Apify.getApifyProxyUrl({ session: new Date() })).toThrowError();- expect(() => Apify.getApifyProxyUrl({ apifyProxySession: new Date() })).toThrowError();-- expect(() => Apify.getApifyProxyUrl({ session: 'a_b' })).not.toThrowError();- expect(() => Apify.getApifyProxyUrl({ session: '0.34252352' })).not.toThrowError();- expect(() => Apify.getApifyProxyUrl({ session: 'aaa~BBB' })).not.toThrowError();- expect(() => Apify.getApifyProxyUrl({ session: 'a_1_b' })).not.toThrowError();- expect(() => Apify.getApifyProxyUrl({ session: 'a_2' })).not.toThrowError();- expect(() => Apify.getApifyProxyUrl({ session: 'a' })).not.toThrowError();- expect(() => Apify.getApifyProxyUrl({ session: '1' })).not.toThrowError();-- expect(() => Apify.getApifyProxyUrl({ groups: [new Date()] })).toThrowError();- expect(() => Apify.getApifyProxyUrl({ groups: [{}, 'fff', 'ccc'] })).toThrowError();- expect(() => Apify.getApifyProxyUrl({ groups: ['ffff', 'ff-hf', 'ccc'] })).toThrowError();- expect(() => Apify.getApifyProxyUrl({ groups: ['ffff', 'fff', 'cc$c'] })).toThrowError();- expect(() => Apify.getApifyProxyUrl({ apifyProxyGroups: [new Date()] })).toThrowError();--- expect(() => Apify.getApifyProxyUrl({ country: new Date() })).toThrow();- expect(() => Apify.getApifyProxyUrl({ country: 'aa' })).toThrow();- expect(() => Apify.getApifyProxyUrl({ country: 'aB' })).toThrow();- expect(() => Apify.getApifyProxyUrl({ country: 'Ba' })).toThrow();- expect(() => Apify.getApifyProxyUrl({ country: '11' })).toThrow();- expect(() => Apify.getApifyProxyUrl({ country: 'DDDD' })).toThrow();- expect(() => Apify.getApifyProxyUrl({ country: 'dddd' })).toThrow();- expect(() => Apify.getApifyProxyUrl({ country: 1111 })).toThrow();-- delete process.env[ENV_VARS.PROXY_PASSWORD];- delete process.env[ENV_VARS.PROXY_HOSTNAME];- delete process.env[ENV_VARS.PROXY_PORT];- });-});--describe('Apify.addWebhook()', () => {test('works', async () => {const runId = 'my-run-id';diff --git a/test/crawlers/cheerio_crawler.test.js b/test/crawlers/cheerio_crawler.test.jsindex 21f7ba949..df8624f5d 100644--- a/test/crawlers/cheerio_crawler.test.js+++ b/test/crawlers/cheerio_crawler.test.js@@ -15,6 +15,7 @@ import LocalStorageDirEmulator from '../local_storage_dir_emulator';import * as utilsRequest from '../../build/utils_request';import CrawlerExtension from '../../build/crawlers/crawler_extension';+// Add common props to mocked request responses.const responseMock = {url: 'loadedUrl',@@ -599,20 +600,26 @@ describe('CheerioCrawler', () => {expect(proxies[3]).toEqual(shuffled[0]);});- test('should work with useApifyProxy', async () => {+ test('should work with proxyConfiguration', async () => {process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';- const proxies = [];- const useApifyProxy = true;+ const status = { connected: true };+ const fakeCall = async () => {+ return { body: status };+ };- const proxy = Apify.getApifyProxyUrl();+ const stub = sinon.stub(utilsRequest, 'requestAsBrowser').callsFake(fakeCall);+ const proxyConfiguration = await Apify.createProxyConfiguration({+ groups: ['GROUP1', 'GROUP2'],+ });const crawler = new Apify.CheerioCrawler({requestList,handlePageFunction: async () => {},- useApifyProxy,+ proxyConfiguration,});+ const proxies = [];crawler._requestFunction = async ({ request }) => {const opts = crawler._getRequestOptions(request);proxies.push(opts.proxyUrl);@@ -620,52 +627,56 @@ describe('CheerioCrawler', () => {return { dom: {}, response: responseMock };};+ const proxyUrl = crawler.proxyConfiguration.getUrl();await crawler.run();- delete process.env[ENV_VARS.PROXY_PASSWORD];- // expect(proxies).to.have.lengthOf(1);- expect(proxies[0]).toEqual(proxy);- expect(proxies[1]).toEqual(proxy);- expect(proxies[2]).toEqual(proxy);- expect(proxies[3]).toEqual(proxy);+ expect(proxies[0]).toEqual(proxyUrl);+ expect(proxies[1]).toEqual(proxyUrl);+ expect(proxies[2]).toEqual(proxyUrl);+ expect(proxies[3]).toEqual(proxyUrl);++ delete process.env[ENV_VARS.PROXY_PASSWORD];+ stub.restore();});- test('should work with useApifyProxy and other opts', async () => {+ test('handlePageFunction should expose the proxyInfo object with sessions correctly', async () => {process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';- const proxies = [];- const useApifyProxy = true;- const apifyProxyGroups = ['GROUP1', 'GROUP2'];- const apifyProxySession = 'session';+ const status = { connected: true };+ const fakeCall = async () => {+ return { body: status };+ };- const proxy = Apify.getApifyProxyUrl({- groups: apifyProxyGroups,- session: apifyProxySession,- });+ const stub = sinon.stub(utilsRequest, 'requestAsBrowser').callsFake(fakeCall);++ const proxyConfiguration = await Apify.createProxyConfiguration();+ const proxies = [];+ const sessions = [];+ const handlePageFunction = async ({ session, proxyInfo }) => {+ proxies.push(proxyInfo);+ sessions.push(session);+ };const crawler = new Apify.CheerioCrawler({requestList,- handlePageFunction: async () => {- },- useApifyProxy,- apifyProxyGroups,- apifyProxySession,+ handlePageFunction,+ proxyConfiguration,+ useSessionPool: true,});crawler._requestFunction = async ({ request }) => {- const opts = crawler._getRequestOptions(request);- proxies.push(opts.proxyUrl);// it needs to return something valid- return { dom: {}, response: responseMock };+ return { dom: {}, response: { url: request.url } };};await crawler.run();- delete process.env[ENV_VARS.PROXY_PASSWORD];- // expect(proxies).to.have.lengthOf(1);- expect(proxies[0]).toEqual(proxy);- expect(proxies[1]).toEqual(proxy);- expect(proxies[2]).toEqual(proxy);- expect(proxies[3]).toEqual(proxy);+ expect(proxies[0]).toEqual(proxyConfiguration.getInfo(sessions[0].id));+ expect(proxies[1]).toEqual(proxyConfiguration.getInfo(sessions[1].id));+ expect(proxies[2]).toEqual(proxyConfiguration.getInfo(sessions[2].id));+ expect(proxies[3]).toEqual(proxyConfiguration.getInfo(sessions[3].id));++ delete process.env[ENV_VARS.PROXY_PASSWORD];+ stub.restore();});describe('throws', () => {@@ -677,19 +688,28 @@ describe('CheerioCrawler', () => {log.setLevel(log.LEVELS.ERROR);});- test('when proxyUrls is used together with useApifyProxy', async () => {+ test('when proxyUrls is used together with proxyConfiguration', async () => {+ process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';+ const status = { connected: true };+ const fakeCall = async () => {+ return { body: status };+ };++ const stub = sinon.stub(utilsRequest, 'requestAsBrowser').callsFake(fakeCall);+ const proxyConfiguration = await Apify.createProxyConfiguration();try {new Apify.CheerioCrawler({requestList,handlePageFunction: async () => {},proxyUrls: ['http://proxy.com:1111', 'http://proxy.com:2222', 'http://proxy.com:3333'],- useApifyProxy: true,+ proxyConfiguration,});throw new Error('Invalid error.');} catch (err) {- expect(err.message).toMatch('useApifyProxy');+ expect(err.message).toMatch('proxyConfiguration');}+ stub.restore();});test('when proxyUrls array is empty', async () => {@@ -860,32 +880,38 @@ describe('CheerioCrawler', () => {process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';const requestListNew = new Apify.RequestList({ sources: sourcesNew });- let sessionUsed;- let requestUsed;+ let usedSession;+ const usedRequests = [];+ const status = { connected: true };+const fakeCall = async (opt) => {- requestUsed = opt;- throw new Error('throwing here to abort rest of execution, its not needed');+ usedRequests.push(opt);+ return { body: status };};+const stub = sinon.stub(utilsRequest, 'requestAsBrowser').callsFake(fakeCall);++ const proxyConfiguration = await Apify.createProxyConfiguration();const cheerioCrawler = new Apify.CheerioCrawler({requestList: requestListNew,maxRequestRetries: 0,handlePageFunction: async () => {},handleFailedRequestFunction: async () => {},useSessionPool: true,- useApifyProxy: true,+ proxyConfiguration,});const oldHandleRequestF = cheerioCrawler._handleRequestFunction;cheerioCrawler._handleRequestFunction = async (opts) => {- sessionUsed = opts.session;+ usedSession = opts.session;return oldHandleRequestF.call(cheerioCrawler, opts);};await requestListNew.initialize();await cheerioCrawler.run();- expect(requestUsed.proxyUrl.includes(sessionUsed.id)).toBeTruthy();+ const cheerioCrawlerRequest = usedRequests[1];+ expect(cheerioCrawlerRequest.proxyUrl.includes(usedSession.id)).toBeTruthy();stub.restore();});});@@ -944,7 +970,6 @@ describe('CheerioCrawler', () => {const extension = new DummyExtension({useSessionPool: true,prepareRequestFunction,- apifyProxyGroups: ['SHADER'],handlePageFunction: undefined,});const cheerioCrawler = new Apify.CheerioCrawler({@@ -961,7 +986,6 @@ describe('CheerioCrawler', () => {expect(cheerioCrawler.useSessionPool).toEqual(true);expect(cheerioCrawler.prepareRequestFunction).toEqual(prepareRequestFunction);expect(cheerioCrawler.handlePageFunction).toBeUndefined();- expect(cheerioCrawler.apifyProxyGroups[0]).toEqual('SHADER');});});});diff --git a/test/crawlers/puppeteer_crawler.test.js b/test/crawlers/puppeteer_crawler.test.jsindex 9d884fbdb..068f5c2bb 100644--- a/test/crawlers/puppeteer_crawler.test.js+++ b/test/crawlers/puppeteer_crawler.test.js@@ -1,7 +1,9 @@import { ENV_VARS } from 'apify-shared/consts';+import sinon from 'sinon';import log from '../../build/utils_log';import * as Apify from '../../build';import LocalStorageDirEmulator from '../local_storage_dir_emulator';+import * as utilsRequest from '../../build/utils_request';describe('PuppeteerCrawler', () => {let prevEnvHeadless;@@ -161,4 +163,94 @@ describe('PuppeteerCrawler', () => {expect(loadedCookies).toHaveLength(4);loadedCookies.forEach(cookie => expect(cookie).toEqual('TEST=12321312312'));});++ describe('proxy', () => {+ let requestList;+ beforeEach(async () => {+ requestList = new Apify.RequestList({+ sources: [+ { url: 'http://example.com/?q=1' },+ { url: 'http://example.com/?q=2' },+ { url: 'http://example.com/?q=3' },+ { url: 'http://example.com/?q=4' },+ ],+ });+ await requestList.initialize();+ });++ afterEach(() => {+ requestList = null;+ });++ test('browser should launch with correct proxyUrl', async () => {+ process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';+ const status = { connected: true };+ const fakeCall = async () => {+ return { body: status };+ };++ const stub = sinon.stub(utilsRequest, 'requestAsBrowser').callsFake(fakeCall);+ const proxyConfiguration = await Apify.createProxyConfiguration();+ const generatedProxyUrl = proxyConfiguration.getUrl();+ let browserProxy;+ const launchPuppeteerFunction = async (opts) => {+ browserProxy = opts.proxyUrl;+ const browser = await Apify.launchPuppeteer(opts);++ return browser;+ };++ const puppeteerCrawler = new Apify.PuppeteerCrawler({+ requestList,+ handlePageFunction: async () => {},+ launchPuppeteerFunction,+ proxyConfiguration,+ });++ await puppeteerCrawler.run();+ delete process.env[ENV_VARS.PROXY_PASSWORD];++ expect(browserProxy).toEqual(generatedProxyUrl);++ stub.restore();+ });++ test('handlePageFunction should expose the proxyInfo object with sessions correctly', async () => {+ process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';+ const status = { connected: true };+ const fakeCall = async () => {+ return { body: status };+ };++ const stub = sinon.stub(utilsRequest, 'requestAsBrowser').callsFake(fakeCall);++ const proxyConfiguration = await Apify.createProxyConfiguration();+ const proxies = [];+ const sessions = [];+ const handlePageFunction = async ({ session, proxyInfo }) => {+ proxies.push(proxyInfo);+ sessions.push(session);+ };++ const puppeteerCrawler = new Apify.PuppeteerCrawler({+ requestList,+ handlePageFunction,+ proxyConfiguration,+ useSessionPool: true,+ sessionPoolOptions: {+ maxPoolSize: 1,+ },+ });++ await puppeteerCrawler.run();++ expect(proxies[0]).toEqual(proxyConfiguration.getInfo(sessions[0].id));+ expect(proxies[1]).toEqual(proxyConfiguration.getInfo(sessions[1].id));+ expect(proxies[2]).toEqual(proxyConfiguration.getInfo(sessions[2].id));+ expect(proxies[3]).toEqual(proxyConfiguration.getInfo(sessions[3].id));++ delete process.env[ENV_VARS.PROXY_PASSWORD];+ stub.restore();+ });+ });});diff --git a/test/proxy_configuration.test.js b/test/proxy_configuration.test.jsnew file mode 100644index 000000000..d28260f40--- /dev/null+++ b/test/proxy_configuration.test.js@@ -0,0 +1,276 @@+import sinon from 'sinon';+import { ENV_VARS, LOCAL_ENV_VARS } from 'apify-shared/consts';+import Apify from '../build/index';+import * as requestUtils from '../build/utils_request';+import * as utils from '../build/utils';+import { ProxyConfiguration } from '../build/proxy_configuration';+import log from '../build/utils_log';++const { apifyClient } = utils;++const groups = ['GROUP1', 'GROUP2'];+const hostname = LOCAL_ENV_VARS[ENV_VARS.PROXY_HOSTNAME];+const port = Number(LOCAL_ENV_VARS[ENV_VARS.PROXY_PORT]);+const password = 'test12345';+const countryCode = 'CZ';+const sessionId = 538909250932;+const basicOpts = {+ groups,+ countryCode,+ password,+};+const basicOptsProxyUrl = 'http://groups-GROUP1+GROUP2,session-538909250932,country-CZ:[email protected]:8000';+const proxyUrlNoSession = 'http://groups-GROUP1+GROUP2,country-CZ:[email protected]:8000';++describe('ProxyConfiguration', () => {+ test('should accept all options', () => {+ const proxyConfiguration = new ProxyConfiguration(basicOpts);++ expect(proxyConfiguration).toBeInstanceOf(ProxyConfiguration);+ expect(proxyConfiguration.groups).toBe(groups);+ expect(proxyConfiguration.countryCode).toBe(countryCode);+ expect(proxyConfiguration.password).toBe(password);+ expect(proxyConfiguration.hostname).toBe(hostname);+ expect(proxyConfiguration.port).toBe(port);+ });++ test('getUrl() should return proxy URL', () => {+ const proxyConfiguration = new ProxyConfiguration(basicOpts);++ expect(proxyConfiguration.getUrl(sessionId)).toBe(basicOptsProxyUrl);+ });++ test('getInfo() should return ProxyInfo object', () => {+ const proxyConfiguration = new ProxyConfiguration(basicOpts);+ const url = basicOptsProxyUrl;++ const proxyInfo = {+ sessionId,+ url,+ groups,+ countryCode,+ password,+ hostname,+ port,+ };+ expect(proxyConfiguration.getInfo(sessionId)).toStrictEqual(proxyInfo);+ });++ test('actor UI input schema should work', () => {+ const apifyProxyGroups = ['GROUP1', 'GROUP2'];+ const apifyProxyCountry = 'CZ';++ const input = {+ useApifyProxy: true,+ apifyProxyGroups,+ apifyProxyCountry,+ };++ const proxyConfiguration = new ProxyConfiguration(input);++ expect(proxyConfiguration.groups).toStrictEqual(apifyProxyGroups);+ expect(proxyConfiguration.countryCode).toStrictEqual(apifyProxyCountry);+ });++ test('should throw on invalid arguments structure', () => {+ // Group value+ const invalidGroups = ['GROUP1*'];+ let opts = Object.assign({}, basicOpts);+ opts.groups = invalidGroups;+ try {+ // eslint-disable-next-line no-unused-vars+ const proxyConfiguration = new ProxyConfiguration(opts);+ throw new Error('wrong error');+ } catch (err) {+ expect(err.message).toMatch('The provided proxy group name "GROUP1*"');+ }++ // Country code+ const invalidCountryCode = 'CZE';+ opts = Object.assign({}, basicOpts);+ opts.countryCode = invalidCountryCode;+ try {+ // eslint-disable-next-line no-unused-vars+ const proxyConfiguration = new ProxyConfiguration(opts);+ throw new Error('wrong error');+ } catch (err) {+ expect(err.message).toMatch('The provided country code "CZE"');+ }+ });++ test('should throw on invalid groups and countryCode args', async () => {+ expect(() => new ProxyConfiguration({ groups: [new Date()] })).toThrowError();+ expect(() => new ProxyConfiguration({ groups: [{}, 'fff', 'ccc'] })).toThrowError();+ expect(() => new ProxyConfiguration({ groups: ['ffff', 'ff-hf', 'ccc'] })).toThrowError();+ expect(() => new ProxyConfiguration({ groups: ['ffff', 'fff', 'cc$c'] })).toThrowError();+ expect(() => new ProxyConfiguration({ apifyProxyGroups: [new Date()] })).toThrowError();+++ expect(() => new ProxyConfiguration({ countryCode: new Date() })).toThrow();+ expect(() => new ProxyConfiguration({ countryCode: 'aa' })).toThrow();+ expect(() => new ProxyConfiguration({ countryCode: 'aB' })).toThrow();+ expect(() => new ProxyConfiguration({ countryCode: 'Ba' })).toThrow();+ expect(() => new ProxyConfiguration({ countryCode: '11' })).toThrow();+ expect(() => new ProxyConfiguration({ countryCode: 'DDDD' })).toThrow();+ expect(() => new ProxyConfiguration({ countryCode: 'dddd' })).toThrow();+ expect(() => new ProxyConfiguration({ countryCode: 1111 })).toThrow();+ });++ test('getUrl() should throw invalid session argument', () => {+ const proxyConfiguration = new ProxyConfiguration();++ expect(() => proxyConfiguration.getUrl('a-b')).toThrowError();+ expect(() => proxyConfiguration.getUrl('a$b')).toThrowError();+ expect(() => proxyConfiguration.getUrl({})).toThrowError();+ expect(() => proxyConfiguration.getUrl(new Date())).toThrowError();++ expect(() => proxyConfiguration.getUrl('a_b')).not.toThrowError();+ expect(() => proxyConfiguration.getUrl('0.34252352')).not.toThrowError();+ expect(() => proxyConfiguration.getUrl('aaa~BBB')).not.toThrowError();+ expect(() => proxyConfiguration.getUrl('a_1_b')).not.toThrowError();+ expect(() => proxyConfiguration.getUrl('a_2')).not.toThrowError();+ expect(() => proxyConfiguration.getUrl('a')).not.toThrowError();+ expect(() => proxyConfiguration.getUrl('1')).not.toThrowError();+ });+});++describe('Apify.createProxyConfiguration()', () => {+ test('should work with all options', async () => {+ const mock = sinon.mock(requestUtils);+ const status = { connected: true };+ const proxyUrl = proxyUrlNoSession;+ const url = 'http://proxy.apify.com/?format=json';++ mock.expects('requestAsBrowser')+ .once()+ .withArgs({ url, proxyUrl, countryCode, json: true })+ .resolves({ body: status });++ const proxyConfiguration = await Apify.createProxyConfiguration(basicOpts);++ expect(proxyConfiguration).toBeInstanceOf(ProxyConfiguration);+ expect(proxyConfiguration.groups).toBe(groups);+ expect(proxyConfiguration.countryCode).toBe(countryCode);+ expect(proxyConfiguration.password).toBe(password);+ expect(proxyConfiguration.hostname).toBe(hostname);+ expect(proxyConfiguration.port).toBe(port);++ mock.verify();+ });++ test('should work without password (with token)', async () => {+ const opts = basicOpts;+ opts.password = null;++ const requestUtilsMock = sinon.mock(requestUtils);+ const status = { connected: true };+ const proxyUrl = proxyUrlNoSession;+ const url = 'http://proxy.apify.com/?format=json';++ requestUtilsMock.expects('requestAsBrowser')+ .once()+ .withArgs({ url, proxyUrl, countryCode, json: true })+ .resolves({ body: status });++ const clientUsersMock = sinon.mock(apifyClient.users);+ const token = '123456789';+ process.env.APIFY_TOKEN = token;+ const data = { proxy: { password } };++ clientUsersMock.expects('getUser')+ .once()+ .withArgs({ token, userId: 'me' })+ .returns(Promise.resolve(data));+++ const proxyConfiguration = await Apify.createProxyConfiguration(opts);++ expect(proxyConfiguration).toBeInstanceOf(ProxyConfiguration);+ expect(proxyConfiguration.groups).toBe(groups);+ expect(proxyConfiguration.countryCode).toBe(countryCode);+ expect(proxyConfiguration.password).toBe(password);+ expect(proxyConfiguration.hostname).toBe(hostname);+ expect(proxyConfiguration.port).toBe(port);++ requestUtilsMock.verify();+ clientUsersMock.verify();+ });++ test('should show warning log', async () => {+ const logMock = sinon.mock(log);+ process.env.APIFY_TOKEN = '123456789';++ const status = { connected: true };+ const fakeUserObjectProxyData = { password: '987654321' };++ const fakeRequestAsBrowser = async () => {+ return { body: status };+ };+ const stub1 = sinon.stub(requestUtils, 'requestAsBrowser').callsFake(fakeRequestAsBrowser);++ const fakeGetUser = async () => {+ return { proxy: fakeUserObjectProxyData };+ };+ const stub2 = sinon.stub(apifyClient.users, 'getUser').callsFake(fakeGetUser);++ // eslint-disable-next-line no-unused-vars+ const proxyConfiguration = await Apify.createProxyConfiguration(basicOpts);++ logMock.expects('warning').once();++ stub1.restore();+ stub2.restore();+ logMock.restore();+ });++ test('should throw missing password', async () => {+ delete process.env[ENV_VARS.PROXY_PASSWORD];+ delete process.env[ENV_VARS.TOKEN];++ const status = { connected: true };++ const fakeCall = async () => {+ return { body: status };+ };++ const stub = sinon.stub(requestUtils, 'requestAsBrowser').callsFake(fakeCall);++ try {+ // eslint-disable-next-line no-unused-vars+ const proxyConfiguration = await Apify.createProxyConfiguration();+ throw new Error('wrong error');+ } catch (err) {+ expect(err.message).toMatch('Apify Proxy password must be provided');+ }+ stub.restore();+ });++ test('should throw when group is not available', async () => {+ delete process.env[ENV_VARS.PROXY_PASSWORD];+ process.env.APIFY_TOKEN = '123456789';+ const connectionError = 'Invalid username: proxy group "GROUP2" not found or not accessible.';+ const status = { connected: false, connectionError };+ const fakeUserObjectProxyData = { password };++ const fakeRequestAsBrowser = async () => {+ return { body: status };+ };+ const stub1 = sinon.stub(requestUtils, 'requestAsBrowser').callsFake(fakeRequestAsBrowser);+++ const fakeGetUser = async () => {+ return { proxy: fakeUserObjectProxyData };+ };+ const stub2 = sinon.stub(apifyClient.users, 'getUser').callsFake(fakeGetUser);++ try {+ // eslint-disable-next-line no-unused-vars+ const proxyConfiguration = await Apify.createProxyConfiguration({ groups });+ throw new Error('wrong error');+ } catch (err) {+ expect(err.message).toMatch(connectionError);+ }+ stub1.restore();+ stub2.restore();+ });+});diff --git a/test/puppeteer.test.js b/test/puppeteer.test.jsindex c87a0fc52..99fc48560 100644--- a/test/puppeteer.test.js+++ b/test/puppeteer.test.js@@ -7,7 +7,6 @@ import _ from 'underscore';import sinon from 'sinon';import { ENV_VARS } from 'apify-shared/consts';import Apify from '../build/index';-import * as actor from '../build/actor';import * as utils from '../build/utils';let prevEnvHeadless;@@ -75,7 +74,6 @@ describe('Apify.launchPuppeteer()', () => {expect(Apify.launchPuppeteer({ proxyUrl: 'socks4://user:[email protected]:1234' })).rejects.toThrow(Error);expect(Apify.launchPuppeteer({ proxyUrl: 'socks5://user:[email protected]:1234' })).rejects.toThrow(Error);expect(Apify.launchPuppeteer({ proxyUrl: ' something really bad' })).rejects.toThrow(Error);- expect(Apify.launchPuppeteer({ proxyUrl: 'xxx', useApifyProxy: true })).rejects.toThrow(Error);expect(Apify.launchPuppeteer({ args: 'wrong args' })).rejects.toThrow(Error);expect(Apify.launchPuppeteer({ args: [12, 34] })).rejects.toThrow(Error);@@ -204,51 +202,6 @@ describe('Apify.launchPuppeteer()', () => {}});- test('should allow to use Apify proxy', async () => {- process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';- process.env[ENV_VARS.PROXY_HOSTNAME] = 'my.host.com';- process.env[ENV_VARS.PROXY_PORT] = 123;-- const mock = sinon.mock(actor);- mock.expects('getApifyProxyUrl')- .once()- .withArgs({- session: 'xxx',- groups: ['yyy'],- groupsParamName: 'options.apifyProxyGroups',- sessionParamName: 'options.apifyProxySession',- })- .returns(null); // Return null so that it doesn't start proxy-chain-- try {- await Apify- .launchPuppeteer({- useApifyProxy: true,- apifyProxySession: 'xxx',- apifyProxyGroups: ['yyy'],- headless: true,- })- .then(browser => browser.close());- } finally {- mock.verify();- mock.restore();- delete process.env[ENV_VARS.PROXY_PASSWORD];- delete process.env[ENV_VARS.PROXY_HOSTNAME];- delete process.env[ENV_VARS.PROXY_PORT];- }- });-- test('should throw when useApifyProxy=true and proxy password is not set', () => {- const opts = {- useApifyProxy: true,- apifyProxySession: 'xxx',- apifyProxyGroups: ['yyy'],- headless: true,- };-- expect(Apify.launchPuppeteer(opts)).rejects.toThrow(Error);- });-test('puppeteerModule option works with type string', async () => {let browser;try {diff --git a/test/puppeteer_pool.test.js b/test/puppeteer_pool.test.jsindex ea2d551c0..c4b73a8a7 100644--- a/test/puppeteer_pool.test.js+++ b/test/puppeteer_pool.test.js@@ -1,6 +1,7 @@import fs from 'fs';import _ from 'underscore';import { ENV_VARS } from 'apify-shared/consts';+import sinon from 'sinon';import log from '../build/utils_log';import * as Apify from '../build/index';import { launchPuppeteer } from '../build/puppeteer';@@ -8,6 +9,7 @@ import { SessionPool } from '../build/session_pool/session_pool';import { BROWSER_SESSION_KEY_NAME } from '../build/puppeteer_pool';import { sleep } from '../build/utils';import LocalStorageDirEmulator from './local_storage_dir_emulator';+import * as utilsRequest from '../build/utils_request';const shortSleep = (millis = 25) => new Promise(resolve => setTimeout(resolve, millis));@@ -517,6 +519,48 @@ describe('PuppeteerPool', () => {});});+ describe('the proxyConfiguration', () => {+ test('correctly creates the proxyUrl', async () => {+ process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';+ const status = { connected: true };+ const fakeCall = async () => {+ return { body: status };+ };++ const stub = sinon.stub(utilsRequest, 'requestAsBrowser').callsFake(fakeCall);+ const proxyConfiguration = await Apify.createProxyConfiguration({+ groups: ['G1', 'G2'],+ });++ const optionsLog = [];+ const pool = new Apify.PuppeteerPool({+ maxOpenPagesPerInstance: 1,+ proxyConfiguration,+ launchPuppeteerFunction: async (launchOpts) => {+ optionsLog.push(launchOpts);+ return launchPuppeteer(launchOpts);+ },+ });++ const proxyUrl = pool.proxyConfiguration.getUrl();+ // Open 4 browsers to do rotation cycle+ await pool.newPage();+ await pool.newPage();+ await pool.newPage();+ await pool.newPage();+ await pool.destroy();++ expect(optionsLog).toHaveLength(4);+ expect(optionsLog[0].proxyUrl).toEqual(proxyUrl);+ expect(optionsLog[1].proxyUrl).toEqual(proxyUrl);+ expect(optionsLog[2].proxyUrl).toEqual(proxyUrl);+ expect(optionsLog[3].proxyUrl).toEqual(proxyUrl);++ delete process.env[ENV_VARS.PROXY_PASSWORD];+ stub.restore();+ });+ });+describe('the proxyUrls parameter', () => {test('supports rotation of custom proxies', async () => {const optionsLog = [];@@ -553,22 +597,31 @@ describe('PuppeteerPool', () => {if (pool) await pool.destroy();});- test('when used with useApifyProxy', async () => {- pool = new Apify.PuppeteerPool({- maxOpenPagesPerInstance: 1,- proxyUrls: ['http://proxy.com:1111', 'http://proxy.com:2222', 'http://proxy.com:3333'],- launchPuppeteerOptions: {- useApifyProxy: true,- apifyProxyGroups: ['G1', 'G2'],- },+ test('when used with proxyConfiguration', async () => {+ process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';+ const status = { connected: true };+ const fakeCall = async () => {+ return { body: status };+ };++ const stub = sinon.stub(utilsRequest, 'requestAsBrowser').callsFake(fakeCall);+ const proxyConfiguration = await Apify.createProxyConfiguration({+ groups: ['G1', 'G2'],});try {- await pool.newPage();+ pool = new Apify.PuppeteerPool({+ maxOpenPagesPerInstance: 1,+ proxyUrls: ['http://proxy.com:1111', 'http://proxy.com:2222', 'http://proxy.com:3333'],+ proxyConfiguration,+ });throw new Error('Invalid error.');} catch (err) {- expect(err.stack).toMatch('useApifyProxy');+ expect(err.stack).toMatch('options.proxyConfiguration');}++ delete process.env[ENV_VARS.PROXY_PASSWORD];+ stub.restore();});test('when empty', async () => {diff --git a/test/session_pool/session.test.js b/test/session_pool/session.test.jsindex 744437e8f..954b123b0 100644--- a/test/session_pool/session.test.js+++ b/test/session_pool/session.test.js@@ -1,11 +1,10 @@import { Session } from '../../build/session_pool/session';import { SessionPool } from '../../build/session_pool/session_pool';+import { ProxyConfiguration } from '../../build/proxy_configuration';import EVENTS from '../../build/session_pool/events';import { STATUS_CODES_BLOCKED } from '../../build/constants';import Apify from '../../build';-import { getApifyProxyUrl } from '../../build/actor';-describe('Session - testing session behaviour ', () => {let sessionPool;@@ -121,10 +120,11 @@ describe('Session - testing session behaviour ', () => {});test('should be valid proxy session', () => {+ const proxyConfiguration = new ProxyConfiguration({ password: '12312' });session = new Session({ sessionPool });let error;try {- getApifyProxyUrl({ session: session.id, password: '12312' });+ proxyConfiguration.getUrl(session.id);} catch (e) {error = e;}diff --git a/website/sidebars.json b/website/sidebars.jsonindex 3911ef08d..447fd62a9 100644--- a/website/sidebars.json+++ b/website/sidebars.json@@ -38,6 +38,7 @@"api/puppeteer-pool","api/session","api/session-pool",+ "api/proxy-configuration","api/snapshotter","api/system-status"]@@ -73,6 +74,8 @@"typedefs/cheerio-crawler-options","typedefs/puppeteer-crawler-options","typedefs/puppeteer-pool-options",+ "typedefs/proxy-configuration-options",+ "typedefs/proxy-info","typedefs/logger-options","typedefs/launch-puppeteer-options","typedefs/request-options",