Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Feature/proxy configuration custom urls (#706)

Jan Bárta02b8e9e7296a
diff --git a/src/crawlers/basic_crawler.js b/src/crawlers/basic_crawler.js
index a70c48d43..6bf8d4fcc 100644
--- a/src/crawlers/basic_crawler.js
+++ b/src/crawlers/basic_crawler.js
@@ -18,7 +18,6 @@ import Request from '../request';
import { QueueOperationInfo } from '../request_queue';
import { Session } from '../session_pool/session';
import { SessionPoolOptions } from '../session_pool/session_pool';
-import { ProxyConfiguration, ProxyInfo } from '../proxy_configuration';
/* eslint-enable no-unused-vars,import/named,import/no-duplicates,import/order */
/**
@@ -45,7 +44,6 @@ const SAFE_MIGRATION_WAIT_MILLIS = 20000;
* request: Request,
* autoscaledPool: AutoscaledPool,
* session: Session,
- * proxyInfo: ProxyInfo
* }
* ```
* where the {@link Request} instance represents the URL to crawl.
@@ -106,13 +104,6 @@ const SAFE_MIGRATION_WAIT_MILLIS = 20000;
* If set to true. Basic crawler will initialize the {@link SessionPool} with the corresponding `sessionPoolOptions`.
* The session instance will be than available in the `handleRequestFunction`.
* @property {SessionPoolOptions} [sessionPoolOptions] The configuration options for {SessionPool} to use.
- * @property {ProxyConfiguration} [proxyConfiguration]
- * If set, `BasicCrawler` will be configured to use
- * [Apify Proxy](https://my.apify.com/proxy) for all connections.
- * For more information, see the [documentation](https://docs.apify.com/proxy)
- *
- * The proxyConfiguration only provides access to the proxyInfo object but the `BasicCrawler` itself
- * is not responsible for using that proxy information to make the proxied HTTP requests.
*/
/**
@@ -196,7 +187,6 @@ class BasicCrawler {
},
maxRequestRetries = 3,
maxRequestsPerCrawl,
- proxyConfiguration,
autoscaledPoolOptions = {},
sessionPoolOptions = {},
useSessionPool = false,
@@ -210,7 +200,6 @@ class BasicCrawler {
} = options;
checkParamPrototypeOrThrow(requestList, 'options.requestList', RequestList, 'Apify.RequestList', true);
- checkParamPrototypeOrThrow(proxyConfiguration, 'options.proxyConfiguration', ProxyConfiguration, 'Apify.proxyConfiguration', true);
checkParamPrototypeOrThrow(requestQueue, 'options.requestQueue', [RequestQueue, RequestQueueLocal], 'Apify.RequestQueue', true);
checkParamOrThrow(handleRequestFunction, 'options.handleRequestFunction', 'Function');
checkParamOrThrow(handleRequestTimeoutSecs, 'options.handleRequestTimeoutSecs', 'Number');
@@ -228,7 +217,6 @@ class BasicCrawler {
this.log = log;
this.requestList = requestList;
this.requestQueue = requestQueue;
- this.proxyConfiguration = proxyConfiguration;
this.handleRequestFunction = handleRequestFunction;
this.handleRequestTimeoutMillis = handleRequestTimeoutSecs * 1000;
this.handleFailedRequestFunction = handleFailedRequestFunction;
@@ -393,7 +381,6 @@ class BasicCrawler {
let request;
let session;
- let proxyInfo;
if (this.useSessionPool) {
[request, session] = await Promise.all([this._fetchNextRequest(), this.sessionPool.getSession()]);
@@ -401,10 +388,6 @@ class BasicCrawler {
request = await this._fetchNextRequest();
}
- if (this.proxyConfiguration) {
- proxyInfo = this.proxyConfiguration.getInfo(session ? session.id : undefined);
- }
-
if (!request) return;
// Reset loadedUrl so an old one is not carried over to retries.
@@ -414,7 +397,7 @@ class BasicCrawler {
this.stats.startJob(statisticsId);
try {
await addTimeoutToPromise(
- this.handleRequestFunction({ request, autoscaledPool: this.autoscaledPool, session, proxyInfo }),
+ this.handleRequestFunction({ request, autoscaledPool: this.autoscaledPool, session }),
this.handleRequestTimeoutMillis,
`handleRequestFunction timed out after ${this.handleRequestTimeoutMillis / 1000} seconds.`,
);
@@ -537,7 +520,6 @@ export default BasicCrawler;
* to pause the crawler by calling {@link AutoscaledPool#pause}
* or to abort it by calling {@link AutoscaledPool#abort}.
* @property {Session} [session]
- * @property {ProxyInfo} [proxyInfo]
*/
/**
diff --git a/src/crawlers/cheerio_crawler.js b/src/crawlers/cheerio_crawler.js
index bce24a3e1..17e9ea50e 100644
--- a/src/crawlers/cheerio_crawler.js
+++ b/src/crawlers/cheerio_crawler.js
@@ -7,6 +7,7 @@ import * as htmlparser from 'htmlparser2';
import * as iconv from 'iconv-lite';
import * as _ from 'underscore';
import * as util from 'util';
+import { checkParamPrototypeOrThrow } from 'apify-shared/utilities';
import { BASIC_CRAWLER_TIMEOUT_MULTIPLIER } from '../constants';
import { TimeoutError } from '../errors';
import { addTimeoutToPromise, parseContentTypeFromResponse } from '../utils';
@@ -136,14 +137,9 @@ const DEFAULT_AUTOSCALED_POOL_OPTIONS = {
* @property {boolean} [ignoreSslErrors=true]
* If set to true, SSL certificate errors will be ignored.
* @property {ProxyConfiguration} [proxyConfiguration]
- * If set, `CheerioCrawler` will be configured to use
- * [Apify Proxy](https://my.apify.com/proxy) for all connections.
- * For more information, see the [documentation](https://docs.apify.com/proxy)
- * @property {string[]} [proxyUrls]
- * An array of custom proxy URLs to be used by the `CheerioCrawler` instance.
- * The provided custom proxies' order will be randomized and the resulting list rotated.
- * Custom proxies are not compatible with Apify Proxy and an attempt to use both
- * configuration options will cause an error to be thrown on startup.
+ * If set, `CheerioCrawler` will be configured for all connections to use
+ * [Apify Proxy](https://my.apify.com/proxy) or your own Proxy URLs provided and rotated according to the configuration.
+ * For more information, see the [documentation](https://docs.apify.com/proxy).
* @property {HandleFailedRequest} [handleFailedRequestFunction]
* A function to handle requests that failed more than `option.maxRequestRetries` times.
*
@@ -307,7 +303,6 @@ class CheerioCrawler {
requestTimeoutSecs = 30,
handlePageTimeoutSecs = 60,
ignoreSslErrors = true,
- proxyUrls,
additionalMimeTypes = [],
suggestResponseEncoding,
forceResponseEncoding,
@@ -335,7 +330,6 @@ class CheerioCrawler {
checkParamOrThrow(requestTimeoutSecs, 'options.requestTimeoutSecs', 'Number');
checkParamOrThrow(handlePageTimeoutSecs, 'options.handlePageTimeoutSecs', 'Number');
checkParamOrThrow(ignoreSslErrors, 'options.ignoreSslErrors', 'Maybe Boolean');
- checkParamOrThrow(proxyUrls, 'options.proxyUrls', 'Maybe [String]');
checkParamOrThrow(prepareRequestFunction, 'options.prepareRequestFunction', 'Maybe Function');
checkParamOrThrow(additionalMimeTypes, 'options.additionalMimeTypes', '[String]');
checkParamOrThrow(suggestResponseEncoding, 'options.suggestResponseEncoding', 'Maybe String');
@@ -343,12 +337,10 @@ class CheerioCrawler {
checkParamOrThrow(useSessionPool, 'options.useSessionPool', 'Boolean');
checkParamOrThrow(sessionPoolOptions, 'options.sessionPoolOptions', 'Object');
checkParamOrThrow(persistCookiesPerSession, 'options.persistCookiesPerSession', 'Boolean');
+ checkParamPrototypeOrThrow(proxyConfiguration, 'options.proxyConfiguration', ProxyConfiguration, 'ProxyConfiguration', true);
this.log = defaultLog.child({ prefix: 'CheerioCrawler' });
- // Enforce valid proxy configuration
- if (proxyUrls && !proxyUrls.length) throw new Error('Parameter "options.proxyUrls" of type Array must not be empty');
- if (proxyConfiguration && proxyUrls) throw new Error('Cannot combine "options.proxyConfiguration" with "options.proxyUrls"!');
if (persistCookiesPerSession && !useSessionPool) {
throw new Error('Cannot use "options.persistCookiesPerSession" without "options.useSessionPool"');
}
@@ -371,10 +363,8 @@ class CheerioCrawler {
this.handlePageTimeoutMillis = handlePageTimeoutSecs * 1000;
this.requestTimeoutMillis = requestTimeoutSecs * 1000;
this.ignoreSslErrors = ignoreSslErrors;
- this.proxyUrls = _.shuffle(proxyUrls);
this.suggestResponseEncoding = suggestResponseEncoding;
this.forceResponseEncoding = forceResponseEncoding;
- this.lastUsedProxyUrlIndex = 0;
this.prepareRequestFunction = prepareRequestFunction;
this.proxyConfiguration = proxyConfiguration;
this.persistCookiesPerSession = persistCookiesPerSession;
@@ -470,10 +460,18 @@ class CheerioCrawler {
* @param {ProxyInfo} options.proxyInfo
* @ignore
*/
- async _handleRequestFunction({ request, autoscaledPool, session, proxyInfo }) {
+ async _handleRequestFunction({ request, autoscaledPool, session }) {
if (this.prepareRequestFunction) await this.prepareRequestFunction({ request, session });
+
+ let proxyInfo;
+ let proxyUrl;
+ if (this.proxyConfiguration) {
+ proxyInfo = this.proxyConfiguration.newProxyInfo(session ? session.id : undefined);
+ proxyUrl = proxyInfo.url;
+ }
+
const { dom, isXml, body, contentType, response } = await addTimeoutToPromise(
- this._requestFunction({ request, session }),
+ this._requestFunction({ request, session, proxyUrl }),
this.requestTimeoutMillis,
`request timed out after ${this.requestTimeoutMillis / 1000} seconds.`,
);
@@ -534,9 +532,10 @@ class CheerioCrawler {
* @param {Object} options
* @param {Request} options.request
* @param {Session} options.session
+ * @param {string} options.proxyUrl
* @ignore
*/
- async _requestFunction({ request, session }) {
+ async _requestFunction({ request, session, proxyUrl }) {
// Using the streaming API of Request to be able to
// handle the response based on headers receieved.
@@ -545,7 +544,7 @@ class CheerioCrawler {
headers.Cookie = session.getCookieString(request.url);
}
- const opts = this._getRequestOptions(request, session);
+ const opts = this._getRequestOptions(request, session, proxyUrl);
let responseStream;
try {
@@ -590,15 +589,16 @@ class CheerioCrawler {
* Combines the provided `requestOptions` with mandatory (non-overridable) values.
* @param {Request} request
* @param {Session} [session]
+ * @param {string} [proxyUrl]
* @ignore
*/
- _getRequestOptions(request, session) {
+ _getRequestOptions(request, session, proxyUrl) {
const mandatoryRequestOptions = {
url: request.url,
method: request.method,
headers: Object.assign({}, request.headers),
ignoreSslErrors: this.ignoreSslErrors,
- proxyUrl: this._getProxyUrl(session),
+ proxyUrl,
stream: true,
useCaseSensitiveHeaders: true,
abortFunction: (res) => {
@@ -626,23 +626,6 @@ class CheerioCrawler {
return Object.assign({}, this.requestOptions, mandatoryRequestOptions);
}
- /**
- * Enables the use of a proxy by returning a proxy URL
- * based on configured options or null if no proxy is used.
- * @param {Session} [session]
- * @returns {(string|null)}
- * @ignore
- */
- _getProxyUrl(session = {}) {
- if (this.proxyConfiguration) {
- return this.proxyConfiguration.getUrl(session.id);
- }
- if (this.proxyUrls) {
- return this.proxyUrls[this.lastUsedProxyUrlIndex++ % this.proxyUrls.length];
- }
- return null;
- }
-
_encodeResponse(request, response, encoding) {
if (this.forceResponseEncoding) {
encoding = this.forceResponseEncoding;
diff --git a/src/crawlers/puppeteer_crawler.js b/src/crawlers/puppeteer_crawler.js
index 14ff1b73f..716ec8cbb 100644
--- a/src/crawlers/puppeteer_crawler.js
+++ b/src/crawlers/puppeteer_crawler.js
@@ -1,6 +1,6 @@
import { checkParamOrThrow } from 'apify-client/build/utils';
import * as _ from 'underscore';
-import PuppeteerPool, { BROWSER_SESSION_KEY_NAME } from '../puppeteer_pool'; // eslint-disable-line import/no-duplicates
+import PuppeteerPool from '../puppeteer_pool'; // eslint-disable-line import/no-duplicates
import { BASIC_CRAWLER_TIMEOUT_MULTIPLIER } from '../constants';
import { gotoExtended } from '../puppeteer_utils';
import { openSessionPool } from '../session_pool/session_pool'; // eslint-disable-line import/no-duplicates
@@ -132,9 +132,9 @@ import { SessionPoolOptions } from '../session_pool/session_pool';
* @property {boolean} [persistCookiesPerSession=false]
* Automatically saves cookies to Session. Works only if Session Pool is used.
* @property {ProxyConfiguration} [proxyConfiguration]
- * If set, `PuppeteerCrawler` will be configured to use
- * [Apify Proxy](https://my.apify.com/proxy) for all connections.
- * For more information, see the [documentation](https://docs.apify.com/proxy)
+ * If set, `PuppeteerCrawler` will be configured for all connections to use
+ * [Apify Proxy](https://my.apify.com/proxy) or your own Proxy URLs provided and rotated according to the configuration.
+ * For more information, see the [documentation](https://docs.apify.com/proxy).
*/
/**
@@ -251,6 +251,11 @@ class PuppeteerCrawler {
checkParamOrThrow(sessionPoolOptions, 'options.sessionPoolOptions', 'Object');
checkParamOrThrow(persistCookiesPerSession, 'options.persistCookiesPerSession', 'Boolean');
+ if (proxyConfiguration && (launchPuppeteerOptions && launchPuppeteerOptions.proxyUrl)) {
+ throw new Error('It is not possible to combine "options.proxyConfiguration" together with '
+ + 'custom "proxyUrl" option from "options.launchPuppeteerOptions".');
+ }
+
this.log = defaultLog.child({ prefix: 'PuppeteerCrawler' });
if (options.gotoTimeoutSecs && options.gotoFunction) {
@@ -345,10 +350,11 @@ class PuppeteerCrawler {
let proxyInfo;
const page = await this.puppeteerPool.newPage();
+ // eslint-disable-next-line no-underscore-dangle
+ const browserInstance = this.puppeteerPool._getBrowserInstance(page);
if (this.sessionPool) {
- const browser = page.browser();
- session = browser[BROWSER_SESSION_KEY_NAME];
-
+ // eslint-disable-next-line prefer-destructuring
+ session = browserInstance.session;
// setting cookies to page
if (this.persistCookiesPerSession) {
@@ -357,7 +363,8 @@ class PuppeteerCrawler {
}
if (this.proxyConfiguration) {
- proxyInfo = this.proxyConfiguration.getInfo(session ? session.id : undefined);
+ // eslint-disable-next-line prefer-destructuring
+ proxyInfo = browserInstance.proxyInfo;
}
diff --git a/src/proxy_configuration.js b/src/proxy_configuration.js
index 9c41180b3..68342b1e2 100644
--- a/src/proxy_configuration.js
+++ b/src/proxy_configuration.js
@@ -1,6 +1,7 @@
import { checkParamOrThrow } from 'apify-client/build/utils';
import { ENV_VARS, LOCAL_ENV_VARS } from 'apify-shared/consts';
import { APIFY_PROXY_VALUE_REGEX } from 'apify-shared/regexs';
+import { URL } from 'url';
import { COUNTRY_CODE_REGEX } from './constants';
import { apifyClient } from './utils';
import { requestAsBrowser } from './utils_request';
@@ -12,12 +13,14 @@ const APIFY_PROXY_STATUS_URL = 'http://proxy.apify.com/?format=json';
/**
* @typedef ProxyConfigurationOptions
- * @property {string} [password] - User's password for the proxy. By default, it is taken from the `APIFY_PROXY_PASSWORD`
+ * @property {string} [password]
+ * User's password for the proxy. By default, it is taken from the `APIFY_PROXY_PASSWORD`
* environment variable, which is automatically set by the system when running the actors.
- * @property {string[]} [groups] - An array of proxy groups to be used
- * by the [Apify Proxy](https://docs.apify.com/proxy). If not provided, the proxy will select
- * the groups automatically.
- * @property {string} [countryCode] - If set and relevant proxies are available in your Apify account, all proxied requests will
+ * @property {string[]} [groups]
+ * An array of proxy groups to be used by the [Apify Proxy](https://docs.apify.com/proxy).
+ * If not provided, the proxy will select the groups automatically.
+ * @property {string} [countryCode]
+ * If set and relevant proxies are available in your Apify account, all proxied requests will
* use IP addresses that are geolocated to the specified country. For example `GB` for IPs
* from Great Britain. Note that online services often have their own rules for handling
* geolocation and thus the country selection is a best attempt at geolocation, rather than
@@ -26,16 +29,26 @@ const APIFY_PROXY_STATUS_URL = 'http://proxy.apify.com/?format=json';
* [full list of available country codes](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2#Officially_assigned_code_elements).
* This parameter is optional, by default, the proxy uses all available proxy servers from all countries.
* on the Apify cloud, or when using the [Apify CLI](https://github.com/apifytech/apify-cli).
- * @property {string[]} [apifyProxyGroups] - Same option as `groups` which can be used to
- * configurate the proxy by UI input schema. You should use the `groups` option in your crawler code.
- * @property {string} [apifyProxyCountry] - Same option as `countryCode` which can be used to
- * configurate the proxy by UI input schema. You should use the `countryCode` option in your crawler code.
+ * @property {string[]} [apifyProxyGroups]
+ * Same option as `groups` which can be used to
+ * configurate the proxy by UI input schema. You should use the `groups` option in your crawler code.
+ * @property {string} [apifyProxyCountry]
+ * Same option as `countryCode` which can be used to
+ * configurate the proxy by UI input schema. You should use the `countryCode` option in your crawler code.
+ * @property {string[]} [proxyUrls]
+ * An array of custom proxy URLs to be rotated.
+ * Custom proxies are not compatible with Apify Proxy and an attempt to use both
+ * configuration options will cause an error to be thrown on initialize.
+ * @property {function} [newUrlFunction]
+ * Custom function that allows you to generate the new proxy URL dynamically. It gets the `sessionId` as a parameter
+ * and should always return stringified proxy URL.
+ * This function is used to generate the URL when {@link ProxyConfiguration.newUrl} or {@link ProxyConfiguration.newProxyInfo} is called.
*/
/**
* The main purpose of the ProxyInfo object is to provide information
- * about the proxy used by the crawler for the current request.
- * Outside of crawlers, you can get this object by calling {@link ProxyConfiguration.getInfo}.
+ * about the current proxy connection used by the crawler for the request.
+ * Outside of crawlers, you can get this object by calling {@link ProxyConfiguration.newProxyInfo}.
*
* **Example usage:**
*
@@ -47,14 +60,14 @@ const APIFY_PROXY_STATUS_URL = 'http://proxy.apify.com/?format=json';
* });
*
* // Getting proxyInfo object by calling class method directly
- * const proxyInfo = proxyConfiguration.getInfo();
+ * const proxyInfo = proxyConfiguration.newProxyInfo();
*
* // In crawler
* const crawler = new Apify.CheerioCrawler({
* // ...
* proxyConfiguration,
* handlePageFunction: ({ proxyInfo }) => {
- * // Getting used Proxy URL
+ * // Getting used proxy URL
* const proxyUrl = proxyInfo.url;
*
* // Getting ID of used Session
@@ -64,12 +77,15 @@ const APIFY_PROXY_STATUS_URL = 'http://proxy.apify.com/?format=json';
*
* ```
* @typedef ProxyInfo
- * @property {string} [sessionId] - The identifier of used {@link Session}, if used.
- * @property {string} url - The proxy URL.
- * @property {string[]} groups - An array of proxy groups to be used
- * by the [Apify Proxy](https://docs.apify.com/proxy). If not provided, the proxy will select
- * the groups automatically.
- * @property {string} [countryCode] - If set and relevant proxies are available in your Apify account, all proxied requests will
+ * @property {string} [sessionId]
+ * The identifier of used {@link Session}, if used.
+ * @property {string} url
+ * The URL of the proxy.
+ * @property {string[]} groups
+ * An array of proxy groups to be used by the [Apify Proxy](https://docs.apify.com/proxy).
+ * If not provided, the proxy will select the groups automatically.
+ * @property {string} [countryCode]
+ * If set and relevant proxies are available in your Apify account, all proxied requests will
* use IP addresses that are geolocated to the specified country. For example `GB` for IPs
* from Great Britain. Note that online services often have their own rules for handling
* geolocation and thus the country selection is a best attempt at geolocation, rather than
@@ -77,11 +93,14 @@ const APIFY_PROXY_STATUS_URL = 'http://proxy.apify.com/?format=json';
* an IP address from a random country. The country code needs to be a two letter ISO country code. See the
* [full list of available country codes](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2#Officially_assigned_code_elements).
* This parameter is optional, by default, the proxy uses all available proxy servers from all countries.
- * @property {string} password - User's password for the proxy. By default, it is taken from the `APIFY_PROXY_PASSWORD`
+ * @property {string} password
+ * User's password for the proxy. By default, it is taken from the `APIFY_PROXY_PASSWORD`
* environment variable, which is automatically set by the system when running the actors
* on the Apify cloud, or when using the [Apify CLI](https://github.com/apifytech/apify-cli).
- * @property {string} hostname - Hostname of your proxy.
- * @property {string} port - Proxy port.
+ * @property {string} hostname
+ * Hostname of your proxy.
+ * @property {string} port
+ * Proxy port.
*/
/**
@@ -94,6 +113,9 @@ const APIFY_PROXY_STATUS_URL = 'http://proxy.apify.com/?format=json';
* you need an Apify account and access to the selected proxies. If you provide no configuration option,
* the proxies will be managed automatically using a smart algorithm.
*
+ * If you want to use your own proxies, use the {@link ProxyConfigurationOptions.proxyUrls} option. Your list of proxy URLs will
+ * be rotated by the configuration if this option is provided.
+ *
* **Example usage:**
*
* ```javascript
@@ -107,7 +129,7 @@ const APIFY_PROXY_STATUS_URL = 'http://proxy.apify.com/?format=json';
* // ...
* proxyConfiguration,
* handlePageFunction: ({ proxyInfo }) => {
- * const usedProxyUrl = proxyInfo.url; // Getting the Proxy URL
+ * const usedProxyUrl = proxyInfo.url; // Getting the proxy URL
* }
* })
*
@@ -126,7 +148,9 @@ export class ProxyConfiguration {
apifyProxyGroups = [],
countryCode,
apifyProxyCountry,
+ proxyUrls,
password = process.env[ENV_VARS.PROXY_PASSWORD],
+ newUrlFunction,
} = options;
const groupsToUse = groups.length ? groups : apifyProxyGroups;
@@ -138,17 +162,31 @@ export class ProxyConfiguration {
checkParamOrThrow(groupsToUse, 'opts.groups', '[String]');
checkParamOrThrow(countryCodeToUse, 'opts.countryCode', 'Maybe String');
checkParamOrThrow(password, 'opts.password', 'Maybe String');
- this._validateArgumentStructure(groupsToUse, countryCodeToUse);
+ checkParamOrThrow(proxyUrls, 'options.proxyUrls', 'Maybe [String]');
+ checkParamOrThrow(newUrlFunction, 'options.newUrlFunction', 'Maybe Function');
+ if (((proxyUrls || newUrlFunction) && ((groupsToUse.length) || countryCodeToUse))) {
+ this._throwCannotCombineCustomWithApify();
+ }
+ if (proxyUrls && newUrlFunction) this._throwCannotCombineCustomMethods();
+ if (proxyUrls) this._validateProxyUrls(proxyUrls);
+ if (countryCodeToUse) this._validateCountryCode(countryCodeToUse);
+ this._validateGroupsStructure(groupsToUse);
this.groups = groupsToUse;
this.countryCode = countryCodeToUse;
this.password = password;
this.hostname = hostname;
this.port = port;
+ this.nextCustomUrlIndex = 0;
+ this.proxyUrls = proxyUrls;
+ this.usedProxyUrls = new Map();
+ this.newUrlFunction = newUrlFunction;
+ this.usesApifyProxy = !this.proxyUrls && !this.newUrlFunction;
}
/**
- * Loads proxy password if token is provided and checks access to Apify Proxy and provided proxy groups.
+ * Loads proxy password if token is provided and checks access to Apify Proxy and provided proxy groups
+ * if Apify Proxy configuration is used.
* Also checks if country has access to Apify Proxy groups if the country code is provided.
*
* You should use the {@link Apify.createProxyConfiguration} function to create a pre-initialized
@@ -157,28 +195,34 @@ export class ProxyConfiguration {
* @returns {Promise<void>}
*/
async initialize() {
- await this._setPasswordIfToken();
-
- await this._checkAccess();
+ if (this.usesApifyProxy) {
+ await this._setPasswordIfToken();
- // TODO: Validate proxyUrl each of custom proxies
+ await this._checkAccess();
+ }
}
/**
- * Gets information about proxy and its configuration.
- * @param {string} sessionId
- * Apify Proxy [Session](https://docs.apify.com/proxy/datacenter-proxy#session-persistence) identifier
- * to be used with requests.
- * All HTTP requests going through the proxy with the same session identifier
+ * This function creates a new {@link ProxyInfo} info object.
+ * It is used by CheerioCrawler and PuppeteerCrawler to generate proxy URLs and also to allow the user to inspect
+ * the currently used proxy via the handlePageFunction parameter: proxyInfo.
+ * Use it if you want to work with a rich representation of a proxy URL.
+ * If you need the URL string only, use {@link ProxyConfiguration.newUrl}.
+ * @param {string} [sessionId]
+ * Represents the identifier of user {@link Session} that can be managed by the {@link SessionPool} or
+ * you can use the Apify Proxy [Session](https://docs.apify.com/proxy/datacenter-proxy#session-persistence) identifier.
+ *
+ * All the HTTP requests going through the proxy with the same session identifier
* will use the same target proxy server (i.e. the same IP address).
* The identifier can only contain the following characters: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.
- * @return {ProxyInfo} represents information about used proxy configuration.
+ * @return {ProxyInfo} represents information about used proxy and its configuration.
*/
- getInfo(sessionId) {
+ newProxyInfo(sessionId) {
if (sessionId) this._validateSessionArgumentStructure(sessionId);
- const { groups, countryCode, password, port, hostname } = this;
- const url = this.getUrl(sessionId);
+ const url = this.newUrl(sessionId);
+
+ const { groups, countryCode, password, port, hostname } = this.usesApifyProxy ? this : new URL(url);
return {
sessionId,
@@ -192,17 +236,24 @@ export class ProxyConfiguration {
}
/**
- * Returns the proxy URL.
- * @param {string} sessionId
- * Apify Proxy [Session](https://docs.apify.com/proxy/datacenter-proxy#session-persistence) identifier
- * to be used with requests.
- * All HTTP requests going through the proxy with the same session identifier
+ * Returns a new stringified URL of the proxy server.
+ * @param {string} [sessionId]
+ * Represents the identifier of user {@link Session} that can be managed by the {@link SessionPool} or
+ * you can use the Apify Proxy [Session](https://docs.apify.com/proxy/datacenter-proxy#session-persistence) identifier.
+ *
+ * All the HTTP requests going through the proxy with the same session identifier
* will use the same target proxy server (i.e. the same IP address).
* The identifier can only contain the following characters: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.
* @return {string} represents the proxy URL.
*/
- getUrl(sessionId) {
+ newUrl(sessionId) {
if (sessionId) this._validateSessionArgumentStructure(sessionId);
+ if (this.newUrlFunction) {
+ return this._callNewUrlFunction(sessionId);
+ }
+ if (this.proxyUrls) {
+ return this._handleCustomUrl(sessionId);
+ }
const username = this._getUsername(sessionId);
const { password, hostname, port } = this;
@@ -211,8 +262,8 @@ export class ProxyConfiguration {
/**
* Returns proxy username.
+ * @param {string} [sessionId]
* @return {string} the proxy username
- * @param {string} sessionId
* @ignore
*/
_getUsername(sessionId) {
@@ -268,12 +319,49 @@ export class ProxyConfiguration {
*/
async _checkAccess() {
const url = APIFY_PROXY_STATUS_URL;
- const proxyUrl = this.getUrl();
+ const proxyUrl = this.newUrl();
const { countryCode } = this;
const { body: { connected, connectionError } } = await requestAsBrowser({ url, proxyUrl, countryCode, json: true });
if (!connected) this._throwApifyProxyConnectionError(connectionError);
}
+ /**
+ * Handles custom url rotation with session
+ * @param {string} [sessionId]
+ * @returns {string} url
+ * @ignore
+ */
+ _handleCustomUrl(sessionId) {
+ let customUrlToUse;
+ if (sessionId) {
+ if (this.usedProxyUrls.has(sessionId)) {
+ customUrlToUse = this.usedProxyUrls.get(sessionId);
+ } else {
+ customUrlToUse = this.proxyUrls[this.nextCustomUrlIndex++ % this.proxyUrls.length];
+ this.usedProxyUrls.set(sessionId, customUrlToUse);
+ }
+ } else {
+ customUrlToUse = this.proxyUrls[this.nextCustomUrlIndex++ % this.proxyUrls.length];
+ }
+ return customUrlToUse;
+ }
+
+ /**
+ * Calls the custom newUrlFunction and checks format of its return value
+ * @param {string} [sessionId]
+ * @ignore
+ */
+ _callNewUrlFunction(sessionId) {
+ const urlToReturn = this.newUrlFunction(sessionId);
+ try {
+ // eslint-disable-next-line no-new
+ new URL(urlToReturn);
+ } catch (err) {
+ this._throwNewUrlFunctionInvalidReturn(urlToReturn);
+ }
+ return urlToReturn;
+ }
+
/**
* Validates session argument structure
* @param {string} sessionId
@@ -284,18 +372,49 @@ export class ProxyConfiguration {
}
/**
- * Validates groups and countryCode options correct structure
+ * Validates groups option structure
* @param {string[]} groups
- * @param {string} countryCode
* @ignore
*/
- _validateArgumentStructure(groups, countryCode) {
+ _validateGroupsStructure(groups) {
for (const group of groups) {
if (!APIFY_PROXY_VALUE_REGEX.test(group)) this._throwInvalidProxyValueError(group);
}
- if (countryCode) {
- if (!COUNTRY_CODE_REGEX.test(countryCode)) this._throwInvalidCountryCode(countryCode);
- }
+ }
+
+ /**
+ * Validates countryCode option
+ * @param {string} countryCode
+ * @ignore
+ */
+ _validateCountryCode(countryCode) {
+ if (!COUNTRY_CODE_REGEX.test(countryCode)) this._throwInvalidCountryCode(countryCode);
+ }
+
+ /**
+ * Validates the structure of all URLs from proxyUrls option
+ * @param {string[]} proxyUrls
+ * @ignore
+ */
+ _validateProxyUrls(proxyUrls) {
+ if (!proxyUrls.length) this._throwProxyUrlsIsEmpty();
+ proxyUrls.forEach((customUrl) => {
+ try {
+ // eslint-disable-next-line no-new
+ new URL(customUrl);
+ } catch (err) {
+ this._throwInvalidCustomUrl(customUrl);
+ }
+ });
+ }
+
+ /**
+ * Throws invalid custom newUrlFunction return
+ * @param {string} url
+ * @ignore
+ */
+ _throwNewUrlFunctionInvalidReturn(url) {
+ throw new Error(`The return value "${url}" of provided "options.newUrlFunction" is not a valid URL.`);
}
/**
@@ -323,6 +442,41 @@ export class ProxyConfiguration {
_throwApifyProxyConnectionError(errorMessage) {
throw new Error(errorMessage);
}
+
+ /**
+ * Throws custom URLs is provided but empty
+ * @ignore
+ */
+ _throwProxyUrlsIsEmpty() {
+ throw new Error('Parameter "options.proxyUrls" of type Array must not be empty!');
+ }
+
+ /**
+ * Throws cannot combine custom proxies with Apify Proxy
+ * @ignore
+ */
+ _throwCannotCombineCustomWithApify() {
+ throw new Error('Cannot combine custom proxies with Apify Proxy!'
+ + 'It is not allowed to set "options.proxyUrls" or "options.newUrlFunction" combined with '
+ + '"options.groups" or "options.apifyProxyGroups" and "options.countryCode" or "options.apifyProxyCountry".');
+ }
+
+ /**
+ * Throws cannot combine custom 2 custom methods
+ * @ignore
+ */
+ _throwCannotCombineCustomMethods() {
+ throw new Error('Cannot combine custom proxies "options.proxyUrls" with custom generating function "options.newUrlFunction".');
+ }
+
+ /**
+ * Throws invalid custom proxy URL
+ * @param {string} url
+ * @ignore
+ */
+ _throwInvalidCustomUrl(url) {
+ throw new Error(`The provided proxy URL "${url}" is not a valid URL.`);
+ }
}
/**
@@ -347,12 +501,12 @@ export class ProxyConfiguration {
* // ...
* proxyConfiguration,
* handlePageFunction: ({ proxyInfo }) => {
- * const usedProxyUrl = proxyInfo.url; // Getting the Proxy URL
+ * const usedProxyUrl = proxyInfo.url; // Getting the proxy URL
* }
* })
*
* ```
-* @param {ProxyConfigurationOptions} proxyConfigurationOptions
+* @param {ProxyConfigurationOptions} [proxyConfigurationOptions]
* @returns {Promise<ProxyConfiguration>}
* @memberof module:Apify
* @name createProxyConfiguration
diff --git a/src/puppeteer_pool.js b/src/puppeteer_pool.js
index 210321b1c..6e90ec2cc 100644
--- a/src/puppeteer_pool.js
+++ b/src/puppeteer_pool.js
@@ -10,16 +10,13 @@ import defaultLog from './utils_log';
import { addTimeoutToPromise } from './utils';
import LiveViewServer from './live_view/live_view_server';
import EVENTS from './session_pool/events';
-
// TYPE IMPORTS
/* eslint-disable no-unused-vars,import/named,import/order */
-import { Page, Browser } from 'puppeteer';
+import { Browser, Page } from 'puppeteer';
import { launchPuppeteer, LaunchPuppeteerOptions } from './puppeteer';
import { SessionPool } from './session_pool/session_pool';
/* eslint-enable no-unused-vars */
-export const BROWSER_SESSION_KEY_NAME = 'APIFY_SESSION';
-
const PROCESS_KILL_TIMEOUT_MILLIS = 5000;
const PAGE_CLOSE_KILL_TIMEOUT_MILLIS = 1000;
@@ -32,16 +29,42 @@ const DISK_CACHE_DIR = path.join(os.tmpdir(), 'puppeteer_disk_cache-');
*
* @ignore
*/
-class PuppeteerInstance {
- constructor(id, browserPromise) {
+class PuppeteerInstance { // TODO: this is in progress and it will be refactored later
+ constructor(options) {
+ const {
+ id,
+ sessionPool,
+ proxyConfiguration,
+ launchPuppeteerFunction,
+ } = options;
+
this.id = id;
+ this.proxyInfo = null;
+ this.session = null;
this.activePages = 0;
this.totalPages = 0;
- this.browserPromise = browserPromise;
+ this.browserPromise = null;
this.lastPageOpenedAt = Date.now();
this.killed = false;
this.childProcess = null;
this.recycleDiskCacheDir = null;
+ this.sessionPool = sessionPool;
+ this.proxyConfiguration = proxyConfiguration;
+ this.launchPuppeteerFunction = launchPuppeteerFunction;
+ }
+
+ async launch() {
+ this.browserPromise = new Promise(async (resolve) => {
+ if (this.sessionPool) {
+ this.session = await this.sessionPool.getSession();
+ }
+
+ if (this.proxyConfiguration) {
+ this.proxyInfo = this.proxyConfiguration.newProxyInfo(this.session ? this.session.id : undefined);
+ }
+ const proxyUrl = this.proxyInfo ? this.proxyInfo.url : null;
+ resolve(this.launchPuppeteerFunction({ proxyUrl }));
+ });
}
}
@@ -100,14 +123,10 @@ class PuppeteerInstance {
* that they will not share cookies or cache and their resources will not be throttled by one another.
* @property {SessionPool} [sessionPool]
* A pool of Session instances.
- * @property {string[]} [proxyUrls]
- * An array of custom proxy URLs to be used by the `PuppeteerPool` instance.
- * The provided custom proxies' order will be randomized and the resulting list rotated.
- * Custom proxies are not compatible with Apify Proxy and an attempt to use both
- * configuration options will cause an error to be thrown on startup.
* @property {ProxyConfiguration} [proxyConfiguration]
- * If set, `PuppeteerPool` will be configured to use
- * [Apify Proxy](https://my.apify.com/proxy) for all connections.
+ * If set, `PuppeteerPool` will be configured for all connections to use
+ * [Apify Proxy](https://my.apify.com/proxy) or your own Proxy URLs provided and rotated according to the configuration.
+ * For more information, see the [documentation](https://docs.apify.com/proxy).
*/
/**
@@ -166,7 +185,6 @@ class PuppeteerPool {
launchPuppeteerOptions,
recycleDiskCache = false,
useIncognitoPages = false,
- proxyUrls,
proxyConfiguration,
useLiveView = false,
sessionPool,
@@ -198,14 +216,15 @@ class PuppeteerPool {
checkParamOrThrow(launchPuppeteerOptions, 'options.launchPuppeteerOptions', 'Maybe Object');
checkParamOrThrow(recycleDiskCache, 'options.recycleDiskCache', 'Maybe Boolean');
checkParamOrThrow(useIncognitoPages, 'options.useIncognitoPages', 'Maybe Boolean');
- checkParamOrThrow(proxyUrls, 'options.proxyUrls', 'Maybe Array');
- // Enforce non-empty proxyUrls array
- if (proxyUrls && !proxyUrls.length) throw new Error('Parameter "options.proxyUrls" of type Array must not be empty');
- if (proxyConfiguration && proxyUrls) throw new Error('Cannot combine "options.proxyConfiguration" with "options.proxyUrls"!');
checkParamOrThrow(useLiveView, 'options.useLiveView', 'Maybe Boolean');
checkParamOrThrow(sessionPool, 'options.sessionPool', 'Maybe Object');
+ checkParamOrThrow(proxyConfiguration, 'options.sessionPool', 'Maybe Object');
+ if (proxyConfiguration && (launchPuppeteerOptions && launchPuppeteerOptions.proxyUrl)) {
+ throw new Error('It is not possible to combine "options.proxyConfiguration" together with '
+ + 'custom "proxyUrl" option from "options.launchPuppeteerOptions".');
+ }
// Config.
this.sessionPool = sessionPool;
@@ -221,10 +240,10 @@ class PuppeteerPool {
*/
this.recycledDiskCacheDirs = recycleDiskCache ? new LinkedList() : null;
this.useIncognitoPages = useIncognitoPages;
- this.proxyUrls = proxyUrls ? _.shuffle(proxyUrls) : null;
this.proxyConfiguration = proxyConfiguration;
this.liveViewServer = useLiveView ? new LiveViewServer() : null;
- this.launchPuppeteerFunction = async () => {
+ this.launchPuppeteerOptions = launchPuppeteerOptions;
+ this.launchPuppeteerFunction = async ({ proxyUrl }) => {
// Do not modify passed launchPuppeteerOptions!
const opts = _.clone(launchPuppeteerOptions) || {};
opts.args = _.clone(opts.args || []);
@@ -238,21 +257,11 @@ class PuppeteerPool {
opts.args.push(`--disk-cache-dir=${diskCacheDir}`);
}
- // Rotate custom proxyUrls.
- if (this.proxyUrls) {
- opts.proxyUrl = this.proxyUrls[this.lastUsedProxyUrlIndex++ % this.proxyUrls.length];
- }
-
// Set timeout for browser launch.
if (opts.timeout == null) opts.timeout = this.puppeteerOperationTimeoutMillis;
- let session;
- if (sessionPool) {
- session = await sessionPool.getSession();
- }
-
if (proxyConfiguration) {
- opts.proxyUrl = proxyConfiguration.getUrl(session ? session.id : undefined);
+ opts.proxyUrl = proxyUrl;
}
const browser = await launchPuppeteerFunction(opts);
@@ -262,8 +271,6 @@ class PuppeteerPool {
}
browser.recycleDiskCacheDir = diskCacheDir;
- if (session) browser[BROWSER_SESSION_KEY_NAME] = session;
-
return browser;
};
@@ -283,6 +290,7 @@ class PuppeteerPool {
// are no references to the stored pages.
this.closedPages = new WeakSet();
this.pagesToInstancesMap = new WeakMap();
+ this.browsersToInstancesMap = new WeakMap();
this.liveViewSnapshotsInProgress = new WeakMap();
@@ -305,33 +313,34 @@ class PuppeteerPool {
const id = this.browserCounter++;
this.log.debug('Launching new browser', { id });
- const browserPromise = this.launchPuppeteerFunction();
-
- const instance = new PuppeteerInstance(id, browserPromise);
+ const instance = new PuppeteerInstance({
+ id,
+ sessionPool: this.sessionPool,
+ proxyConfiguration: this.proxyConfiguration,
+ launchPuppeteerOptions: this.launchPuppeteerOptions,
+ launchPuppeteerFunction: this.launchPuppeteerFunction,
+ });
this.activeInstances[id] = instance;
// Handle the async stuff elsewhere.
- this._initBrowser(browserPromise, instance);
+ this._initBrowser(instance);
return instance;
}
/**
* Takes care of async processes in PuppeteerInstance construction with a Browser.
- * @param {Promise<Browser>} browserPromise
* @param {PuppeteerInstance} instance
* @returns {Promise<void>}
* @ignore
*/
- async _initBrowser(browserPromise, instance) {
+ async _initBrowser(instance) {
const { id } = instance;
let browser;
try {
- browser = await browserPromise;
-
- if (this.sessionPool) {
- instance.session = browser[BROWSER_SESSION_KEY_NAME];
- }
+ await instance.launch();
+ browser = await instance.browserPromise;
+ this.browsersToInstancesMap.set(browser, instance);
} catch (err) {
this.log.exception(err, 'Browser launch failed', { id });
delete this.activeInstances[id];
@@ -773,6 +782,19 @@ class PuppeteerPool {
}, PAGE_CLOSE_KILL_TIMEOUT_MILLIS);
}
}
+
+
+ /**
+ * Gets Puppeteer Instance by the page.
+ *
+ * @param {Page} page
+ * @return {PuppeteerInstance}
+ * @ignore
+ */
+ _getBrowserInstance(page) {
+ const browser = page.browser();
+ return this.browsersToInstancesMap.get(browser);
+ }
}
export default PuppeteerPool;
diff --git a/test/crawlers/cheerio_crawler.test.js b/test/crawlers/cheerio_crawler.test.js
index df8624f5d..a786e698d 100644
--- a/test/crawlers/cheerio_crawler.test.js
+++ b/test/crawlers/cheerio_crawler.test.js
@@ -574,33 +574,7 @@ describe('CheerioCrawler', () => {
requestList = null;
});
- test('should work with proxyUrls array', async () => {
- const proxies = [];
- const crawler = new Apify.CheerioCrawler({
- requestList,
- handlePageFunction: async () => {
- },
- proxyUrls: ['http://proxy.com:1111', 'http://proxy.com:2222', 'http://proxy.com:3333'],
- });
-
- crawler._requestFunction = async ({ request }) => {
- const opts = crawler._getRequestOptions(request);
- proxies.push(opts.proxyUrl);
- // it needs to return something valid
- return { dom: {}, response: responseMock };
- };
-
- const shuffled = crawler.proxyUrls;
- await crawler.run();
-
- expect(proxies).toHaveLength(4);
- expect(proxies[0]).toEqual(shuffled[0]);
- expect(proxies[1]).toEqual(shuffled[1]);
- expect(proxies[2]).toEqual(shuffled[2]);
- expect(proxies[3]).toEqual(shuffled[0]);
- });
-
- test('should work with proxyConfiguration', async () => {
+ test('should work with Apify Proxy configuration', async () => {
process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';
const status = { connected: true };
const fakeCall = async () => {
@@ -620,14 +594,13 @@ describe('CheerioCrawler', () => {
});
const proxies = [];
- crawler._requestFunction = async ({ request }) => {
- const opts = crawler._getRequestOptions(request);
- proxies.push(opts.proxyUrl);
+ crawler._requestFunction = async ({ proxyUrl }) => {
+ proxies.push(proxyUrl);
// it needs to return something valid
return { dom: {}, response: responseMock };
};
- const proxyUrl = crawler.proxyConfiguration.getUrl();
+ const proxyUrl = crawler.proxyConfiguration.newUrl();
await crawler.run();
expect(proxies[0]).toEqual(proxyUrl);
@@ -670,61 +643,82 @@ describe('CheerioCrawler', () => {
await crawler.run();
- expect(proxies[0]).toEqual(proxyConfiguration.getInfo(sessions[0].id));
- expect(proxies[1]).toEqual(proxyConfiguration.getInfo(sessions[1].id));
- expect(proxies[2]).toEqual(proxyConfiguration.getInfo(sessions[2].id));
- expect(proxies[3]).toEqual(proxyConfiguration.getInfo(sessions[3].id));
+ expect(proxies[0]).toEqual(proxyConfiguration.newProxyInfo(sessions[0].id));
+ expect(proxies[1]).toEqual(proxyConfiguration.newProxyInfo(sessions[1].id));
+ expect(proxies[2]).toEqual(proxyConfiguration.newProxyInfo(sessions[2].id));
+ expect(proxies[3]).toEqual(proxyConfiguration.newProxyInfo(sessions[3].id));
delete process.env[ENV_VARS.PROXY_PASSWORD];
stub.restore();
});
- describe('throws', () => {
- /* eslint-disable no-new */
- beforeEach(() => {
- log.setLevel(log.LEVELS.OFF);
+ test('should work with proxyUrls configuration', async () => {
+ process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';
+ const proxies = [];
+
+ const proxyConfiguration = await Apify.createProxyConfiguration({
+ proxyUrls: ['http://proxy.com:1111', 'http://proxy.com:2222', 'http://proxy.com:3333'],
});
- afterEach(async () => {
- log.setLevel(log.LEVELS.ERROR);
+
+ const crawler = new Apify.CheerioCrawler({
+ requestList,
+ handlePageFunction: async () => {
+ },
+ proxyConfiguration,
+ maxConcurrency: 1,
});
- test('when proxyUrls is used together with proxyConfiguration', async () => {
- process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';
- const status = { connected: true };
- const fakeCall = async () => {
- return { body: status };
- };
+ crawler._requestFunction = async ({ request, session, proxyUrl }) => {
+ const opts = crawler._getRequestOptions(request, session, proxyUrl);
+ proxies.push(opts.proxyUrl);
+ // it needs to return something valid
+ return { dom: {}, response: responseMock };
+ };
- const stub = sinon.stub(utilsRequest, 'requestAsBrowser').callsFake(fakeCall);
- const proxyConfiguration = await Apify.createProxyConfiguration();
- try {
- new Apify.CheerioCrawler({
- requestList,
- handlePageFunction: async () => {
- },
- proxyUrls: ['http://proxy.com:1111', 'http://proxy.com:2222', 'http://proxy.com:3333'],
- proxyConfiguration,
- });
- throw new Error('Invalid error.');
- } catch (err) {
- expect(err.message).toMatch('proxyConfiguration');
- }
- stub.restore();
+ await crawler.run();
+
+ const { proxyUrls } = proxyConfiguration;
+ expect(proxies[0]).toEqual(proxyUrls[0]);
+ expect(proxies[1]).toEqual(proxyUrls[1]);
+ expect(proxies[2]).toEqual(proxyUrls[2]);
+ expect(proxies[3]).toEqual(proxyUrls[0]);
+
+ delete process.env[ENV_VARS.PROXY_PASSWORD];
+ });
+
+ test('should correctly pass the url to proxyInfo with proxyUrls configuration and sessionPool', async () => {
+ process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';
+ const proxies = [];
+ const proxyChecks = [];
+
+ const proxyConfiguration = await Apify.createProxyConfiguration({
+ proxyUrls: ['http://proxy.com:1111', 'http://proxy.com:2222', 'http://proxy.com:3333'],
});
- test('when proxyUrls array is empty', async () => {
- try {
- new Apify.CheerioCrawler({
- requestList,
- handlePageFunction: async () => {
- },
- proxyUrls: [],
- });
- throw new Error('Invalid error.');
- } catch (err) {
- expect(err.message).toMatch('must not be empty');
- }
+ const crawler = new Apify.CheerioCrawler({
+ requestList,
+ handlePageFunction: async ({ proxyInfo }) => {
+ proxyChecks.push(proxyInfo);
+ },
+ proxyConfiguration,
+ useSessionPool: true,
});
+
+ crawler._requestFunction = async ({ request, session, proxyUrl }) => {
+ const opts = crawler._getRequestOptions(request, session, proxyUrl);
+ proxies.push(opts.proxyUrl);
+ // it needs to return something valid
+ return { dom: {}, response: responseMock };
+ };
+
+ await crawler.run();
+
+ expect(proxies[0]).toEqual(proxyChecks[0].url);
+ expect(proxies[1]).toEqual(proxyChecks[1].url);
+ expect(proxies[2]).toEqual(proxyChecks[2].url);
+ expect(proxies[3]).toEqual(proxyChecks[3].url);
+
+ delete process.env[ENV_VARS.PROXY_PASSWORD];
});
});
@@ -815,6 +809,7 @@ describe('CheerioCrawler', () => {
test('should throw when "options.useSessionPool" false and "options.persistCookiesPerSession" is true', async () => {
try {
+ // eslint-disable-next-line no-new
new Apify.CheerioCrawler({
requestList: await getRequestListForMock(port, {
diff --git a/test/crawlers/puppeteer_crawler.test.js b/test/crawlers/puppeteer_crawler.test.js
index 068f5c2bb..402653409 100644
--- a/test/crawlers/puppeteer_crawler.test.js
+++ b/test/crawlers/puppeteer_crawler.test.js
@@ -191,7 +191,7 @@ describe('PuppeteerCrawler', () => {
const stub = sinon.stub(utilsRequest, 'requestAsBrowser').callsFake(fakeCall);
const proxyConfiguration = await Apify.createProxyConfiguration();
- const generatedProxyUrl = proxyConfiguration.getUrl();
+ const generatedProxyUrl = proxyConfiguration.newUrl();
let browserProxy;
const launchPuppeteerFunction = async (opts) => {
browserProxy = opts.proxyUrl;
@@ -244,13 +244,76 @@ describe('PuppeteerCrawler', () => {
await puppeteerCrawler.run();
- expect(proxies[0]).toEqual(proxyConfiguration.getInfo(sessions[0].id));
- expect(proxies[1]).toEqual(proxyConfiguration.getInfo(sessions[1].id));
- expect(proxies[2]).toEqual(proxyConfiguration.getInfo(sessions[2].id));
- expect(proxies[3]).toEqual(proxyConfiguration.getInfo(sessions[3].id));
+ expect(proxies[0].sessionId).toEqual(sessions[0].id);
+ expect(proxies[1].sessionId).toEqual(sessions[1].id);
+ expect(proxies[2].sessionId).toEqual(sessions[2].id);
+ expect(proxies[3].sessionId).toEqual(sessions[3].id);
delete process.env[ENV_VARS.PROXY_PASSWORD];
stub.restore();
});
+
+ test('browser should launch with rotated custom proxy', async () => {
+ process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';
+
+ const proxyConfiguration = await Apify.createProxyConfiguration({
+ proxyUrls: ['http://proxy.com:1111', 'http://proxy.com:2222', 'http://proxy.com:3333'],
+ });
+
+ const browserProxies = [];
+ const launchPuppeteerFunction = async (opts) => {
+ browserProxies.push(opts.proxyUrl);
+ const browser = await Apify.launchPuppeteer(opts);
+
+ return browser;
+ };
+
+ const puppeteerCrawler = new Apify.PuppeteerCrawler({
+ requestList,
+ handlePageFunction: async () => {},
+ gotoFunction: async () => {},
+ launchPuppeteerFunction,
+ puppeteerPoolOptions: {
+ retireInstanceAfterRequestCount: 1,
+ },
+ proxyConfiguration,
+ });
+
+ await puppeteerCrawler.run();
+
+ const proxiesToUse = proxyConfiguration.proxyUrls;
+ expect(browserProxies[0]).toEqual(proxiesToUse[0]);
+ expect(browserProxies[1]).toEqual(proxiesToUse[1]);
+ expect(browserProxies[2]).toEqual(proxiesToUse[2]);
+ expect(browserProxies[3]).toEqual(proxiesToUse[0]);
+
+ delete process.env[ENV_VARS.PROXY_PASSWORD];
+ });
+
+ test('should throw on proxyConfiguration together with proxyUrl from launchPuppeteerOptions', async () => {
+ process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';
+
+ const proxyConfiguration = await Apify.createProxyConfiguration({
+ proxyUrls: ['http://proxy.com:1111', 'http://proxy.com:2222', 'http://proxy.com:3333'],
+ });
+
+ try {
+ // eslint-disable-next-line no-unused-vars
+ const puppeteerCrawler = new Apify.PuppeteerCrawler({
+ requestList,
+ handlePageFunction: async () => {},
+ gotoFunction: async () => {},
+ proxyConfiguration,
+ launchPuppeteerOptions: {
+ proxyUrl: 'http://proxy.com:1111',
+ },
+ });
+ throw new Error('wrong error');
+ } catch (err) {
+ expect(err.message).toMatch('It is not possible to combine "options.proxyConfiguration"');
+ }
+
+ delete process.env[ENV_VARS.PROXY_PASSWORD];
+ });
});
});
diff --git a/test/proxy_configuration.test.js b/test/proxy_configuration.test.js
index d28260f40..a6ee0206c 100644
--- a/test/proxy_configuration.test.js
+++ b/test/proxy_configuration.test.js
@@ -34,13 +34,13 @@ describe('ProxyConfiguration', () => {
expect(proxyConfiguration.port).toBe(port);
});
- test('getUrl() should return proxy URL', () => {
+ test('newUrl() should return proxy URL', () => {
const proxyConfiguration = new ProxyConfiguration(basicOpts);
- expect(proxyConfiguration.getUrl(sessionId)).toBe(basicOptsProxyUrl);
+ expect(proxyConfiguration.newUrl(sessionId)).toBe(basicOptsProxyUrl);
});
- test('getInfo() should return ProxyInfo object', () => {
+ test('newProxyInfo() should return ProxyInfo object', () => {
const proxyConfiguration = new ProxyConfiguration(basicOpts);
const url = basicOptsProxyUrl;
@@ -53,7 +53,7 @@ describe('ProxyConfiguration', () => {
hostname,
port,
};
- expect(proxyConfiguration.getInfo(sessionId)).toStrictEqual(proxyInfo);
+ expect(proxyConfiguration.newProxyInfo(sessionId)).toStrictEqual(proxyInfo);
});
test('actor UI input schema should work', () => {
@@ -116,21 +116,182 @@ describe('ProxyConfiguration', () => {
expect(() => new ProxyConfiguration({ countryCode: 1111 })).toThrow();
});
- test('getUrl() should throw invalid session argument', () => {
+ test('newUrl() should throw invalid session argument', () => {
const proxyConfiguration = new ProxyConfiguration();
- expect(() => proxyConfiguration.getUrl('a-b')).toThrowError();
- expect(() => proxyConfiguration.getUrl('a$b')).toThrowError();
- expect(() => proxyConfiguration.getUrl({})).toThrowError();
- expect(() => proxyConfiguration.getUrl(new Date())).toThrowError();
-
- expect(() => proxyConfiguration.getUrl('a_b')).not.toThrowError();
- expect(() => proxyConfiguration.getUrl('0.34252352')).not.toThrowError();
- expect(() => proxyConfiguration.getUrl('aaa~BBB')).not.toThrowError();
- expect(() => proxyConfiguration.getUrl('a_1_b')).not.toThrowError();
- expect(() => proxyConfiguration.getUrl('a_2')).not.toThrowError();
- expect(() => proxyConfiguration.getUrl('a')).not.toThrowError();
- expect(() => proxyConfiguration.getUrl('1')).not.toThrowError();
+ expect(() => proxyConfiguration.newUrl('a-b')).toThrowError();
+ expect(() => proxyConfiguration.newUrl('a$b')).toThrowError();
+ expect(() => proxyConfiguration.newUrl({})).toThrowError();
+ expect(() => proxyConfiguration.newUrl(new Date())).toThrowError();
+
+ expect(() => proxyConfiguration.newUrl('a_b')).not.toThrowError();
+ expect(() => proxyConfiguration.newUrl('0.34252352')).not.toThrowError();
+ expect(() => proxyConfiguration.newUrl('aaa~BBB')).not.toThrowError();
+ expect(() => proxyConfiguration.newUrl('a_1_b')).not.toThrowError();
+ expect(() => proxyConfiguration.newUrl('a_2')).not.toThrowError();
+ expect(() => proxyConfiguration.newUrl('a')).not.toThrowError();
+ expect(() => proxyConfiguration.newUrl('1')).not.toThrowError();
+ expect(() => proxyConfiguration.newUrl(123456)).not.toThrowError();
+ });
+
+ test('should throw invalid newUrlFunction return value', async () => {
+ const newUrlFunction = () => {
+ return 'http://proxy.com:1111*invalid_url';
+ };
+ const proxyConfiguration = new ProxyConfiguration({
+ newUrlFunction,
+ });
+ try {
+ // eslint-disable-next-line no-unused-vars
+ proxyConfiguration.newUrl();
+ throw new Error('wrong error');
+ } catch (err) {
+ expect(err.message).toMatch('The return value "http://proxy.com:1111*invalid_url"');
+ }
+ });
+
+ test('newUrlFunction should correctly generate URLs', async () => {
+ const customUrls = ['http://proxy.com:1111', 'http://proxy.com:2222', 'http://proxy.com:3333',
+ 'http://proxy.com:4444', 'http://proxy.com:5555', 'http://proxy.com:6666'];
+ const newUrlFunction = () => {
+ return customUrls.pop();
+ };
+ const proxyConfiguration = new ProxyConfiguration({
+ newUrlFunction,
+ });
+
+ // through newUrl()
+ expect(proxyConfiguration.newUrl()).toEqual('http://proxy.com:6666');
+ expect(proxyConfiguration.newUrl()).toEqual('http://proxy.com:5555');
+ expect(proxyConfiguration.newUrl()).toEqual('http://proxy.com:4444');
+
+ // through newProxyInfo()
+ expect(proxyConfiguration.newProxyInfo().url).toEqual('http://proxy.com:3333');
+ expect(proxyConfiguration.newProxyInfo().url).toEqual('http://proxy.com:2222');
+ expect(proxyConfiguration.newProxyInfo().url).toEqual('http://proxy.com:1111');
+ });
+
+ describe('With proxyUrls options', () => {
+ test('should rotate custom URLs correctly', async () => {
+ const proxyConfiguration = new ProxyConfiguration({
+ proxyUrls: ['http://proxy.com:1111', 'http://proxy.com:2222', 'http://proxy.com:3333'],
+ });
+
+ const { proxyUrls } = proxyConfiguration;
+ expect(proxyConfiguration.newUrl()).toEqual(proxyUrls[0]);
+ expect(proxyConfiguration.newUrl()).toEqual(proxyUrls[1]);
+ expect(proxyConfiguration.newUrl()).toEqual(proxyUrls[2]);
+ expect(proxyConfiguration.newUrl()).toEqual(proxyUrls[0]);
+ expect(proxyConfiguration.newUrl()).toEqual(proxyUrls[1]);
+ expect(proxyConfiguration.newUrl()).toEqual(proxyUrls[2]);
+ });
+
+ test('newProxyInfo() should return correctly rotated URL', async () => {
+ const proxyConfiguration = new ProxyConfiguration({
+ proxyUrls: ['http://proxy.com:1111', 'http://proxy.com:2222', 'http://proxy.com:3333'],
+ });
+
+ const { proxyUrls } = proxyConfiguration;
+ expect(proxyConfiguration.newProxyInfo().url).toEqual(proxyUrls[0]);
+ expect(proxyConfiguration.newProxyInfo().url).toEqual(proxyUrls[1]);
+ expect(proxyConfiguration.newProxyInfo().url).toEqual(proxyUrls[2]);
+ expect(proxyConfiguration.newProxyInfo().url).toEqual(proxyUrls[0]);
+ expect(proxyConfiguration.newProxyInfo().url).toEqual(proxyUrls[1]);
+ expect(proxyConfiguration.newProxyInfo().url).toEqual(proxyUrls[2]);
+ });
+
+ test('should rotate custom URLs with sessions correctly', async () => {
+ const sessions = ['sesssion_01', 'sesssion_02', 'sesssion_03', 'sesssion_04', 'sesssion_05', 'sesssion_06'];
+ const proxyConfiguration = new ProxyConfiguration({
+ proxyUrls: ['http://proxy.com:1111', 'http://proxy.com:2222', 'http://proxy.com:3333'],
+ });
+
+ const { proxyUrls } = proxyConfiguration;
+ // should use same proxy URL
+ expect(proxyConfiguration.newUrl(sessions[0])).toEqual(proxyUrls[0]);
+ expect(proxyConfiguration.newUrl(sessions[0])).toEqual(proxyUrls[0]);
+ expect(proxyConfiguration.newUrl(sessions[0])).toEqual(proxyUrls[0]);
+
+ // should rotate different proxies
+ expect(proxyConfiguration.newUrl(sessions[1])).toEqual(proxyUrls[1]);
+ expect(proxyConfiguration.newUrl(sessions[2])).toEqual(proxyUrls[2]);
+ expect(proxyConfiguration.newUrl(sessions[3])).toEqual(proxyUrls[0]);
+ expect(proxyConfiguration.newUrl(sessions[4])).toEqual(proxyUrls[1]);
+ expect(proxyConfiguration.newUrl(sessions[5])).toEqual(proxyUrls[2]);
+
+ // should remember already used session
+ expect(proxyConfiguration.newUrl(sessions[1])).toEqual(proxyUrls[1]);
+ expect(proxyConfiguration.newUrl(sessions[3])).toEqual(proxyUrls[0]);
+ });
+
+ test('should throw cannot combine custom proxies with Apify Proxy', async () => {
+ const proxyUrls = ['http://proxy.com:1111', 'http://proxy.com:2222', 'http://proxy.com:3333'];
+ const newUrlFunction = () => {
+ return proxyUrls[Math.floor(Math.random() * proxyUrls.length)];
+ };
+ try {
+ // eslint-disable-next-line no-unused-vars
+ const proxyConfiguration = new ProxyConfiguration({
+ groups: ['GROUP1'],
+ proxyUrls,
+ });
+ throw new Error('wrong error');
+ } catch (err) {
+ expect(err.message).toMatch('Cannot combine custom proxies with Apify Proxy!');
+ }
+
+ try {
+ // eslint-disable-next-line no-unused-vars
+ const proxyConfiguration = new ProxyConfiguration({
+ groups: ['GROUP1'],
+ newUrlFunction,
+ });
+ throw new Error('wrong error');
+ } catch (err) {
+ expect(err.message).toMatch('Cannot combine custom proxies with Apify Proxy!');
+ }
+ });
+
+ test('should throw cannot combine custom methods', async () => {
+ const proxyUrls = ['http://proxy.com:1111', 'http://proxy.com:2222', 'http://proxy.com:3333'];
+ const newUrlFunction = () => {
+ return proxyUrls[Math.floor(Math.random() * proxyUrls.length)];
+ };
+ try {
+ // eslint-disable-next-line no-unused-vars
+ const proxyConfiguration = new ProxyConfiguration({
+ proxyUrls,
+ newUrlFunction,
+ });
+ throw new Error('wrong error');
+ } catch (err) {
+ expect(err.message).toMatch('Cannot combine custom proxies "options.proxyUrls"');
+ }
+ });
+
+ test('should throw proxyUrls array is empty', async () => {
+ try {
+ // eslint-disable-next-line no-unused-vars
+ const proxyConfiguration = new ProxyConfiguration({
+ proxyUrls: [],
+ });
+ throw new Error('wrong error');
+ } catch (err) {
+ expect(err.message).toMatch('must not be empty');
+ }
+ });
+
+ test('should throw invalid custom URL form', async () => {
+ try {
+ // eslint-disable-next-line no-unused-vars
+ const proxyConfiguration = new ProxyConfiguration({
+ proxyUrls: ['http://proxy.com:1111*invalid_url'],
+ });
+ throw new Error('wrong error');
+ } catch (err) {
+ expect(err.message).toEqual('The provided proxy URL "http://proxy.com:1111*invalid_url" is not a valid URL.');
+ }
+ });
});
});
diff --git a/test/puppeteer_pool.test.js b/test/puppeteer_pool.test.js
index c4b73a8a7..4d0677b9f 100644
--- a/test/puppeteer_pool.test.js
+++ b/test/puppeteer_pool.test.js
@@ -6,7 +6,6 @@ import log from '../build/utils_log';
import * as Apify from '../build/index';
import { launchPuppeteer } from '../build/puppeteer';
import { SessionPool } from '../build/session_pool/session_pool';
-import { BROWSER_SESSION_KEY_NAME } from '../build/puppeteer_pool';
import { sleep } from '../build/utils';
import LocalStorageDirEmulator from './local_storage_dir_emulator';
import * as utilsRequest from '../build/utils_request';
@@ -533,6 +532,7 @@ describe('PuppeteerPool', () => {
});
const optionsLog = [];
+ const proxyUrls = [];
const pool = new Apify.PuppeteerPool({
maxOpenPagesPerInstance: 1,
proxyConfiguration,
@@ -542,37 +542,46 @@ describe('PuppeteerPool', () => {
},
});
- const proxyUrl = pool.proxyConfiguration.getUrl();
// Open 4 browsers to do rotation cycle
- await pool.newPage();
- await pool.newPage();
- await pool.newPage();
- await pool.newPage();
+ const page1 = await pool.newPage();
+ const page2 = await pool.newPage();
+ const page3 = await pool.newPage();
+ const page4 = await pool.newPage();
+ /* eslint-disable no-underscore-dangle */
+ proxyUrls.push(pool._getBrowserInstance(page1).proxyInfo.url);
+ proxyUrls.push(pool._getBrowserInstance(page2).proxyInfo.url);
+ proxyUrls.push(pool._getBrowserInstance(page3).proxyInfo.url);
+ proxyUrls.push(pool._getBrowserInstance(page4).proxyInfo.url);
+
await pool.destroy();
+
expect(optionsLog).toHaveLength(4);
- expect(optionsLog[0].proxyUrl).toEqual(proxyUrl);
- expect(optionsLog[1].proxyUrl).toEqual(proxyUrl);
- expect(optionsLog[2].proxyUrl).toEqual(proxyUrl);
- expect(optionsLog[3].proxyUrl).toEqual(proxyUrl);
+ expect(optionsLog[0].proxyUrl).toEqual(proxyUrls[0]);
+ expect(optionsLog[1].proxyUrl).toEqual(proxyUrls[1]);
+ expect(optionsLog[2].proxyUrl).toEqual(proxyUrls[2]);
+ expect(optionsLog[3].proxyUrl).toEqual(proxyUrls[3]);
delete process.env[ENV_VARS.PROXY_PASSWORD];
stub.restore();
});
- });
- describe('the proxyUrls parameter', () => {
test('supports rotation of custom proxies', async () => {
+ process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';
const optionsLog = [];
+
+ const proxyConfiguration = await Apify.createProxyConfiguration({
+ proxyUrls: ['http://proxy.com:1111', 'http://proxy.com:2222', 'http://proxy.com:3333'],
+ });
const pool = new Apify.PuppeteerPool({
maxOpenPagesPerInstance: 1,
- proxyUrls: ['http://proxy.com:1111', 'http://proxy.com:2222', 'http://proxy.com:3333'],
launchPuppeteerFunction: async (launchOpts) => {
optionsLog.push(launchOpts);
return launchPuppeteer(launchOpts);
},
+ proxyConfiguration,
});
- const proxies = [...pool.proxyUrls];
+ const proxies = proxyConfiguration.proxyUrls;
// Open 4 browsers to do full rotation cycle
await pool.newPage();
await pool.newPage();
@@ -585,56 +594,31 @@ describe('PuppeteerPool', () => {
expect(optionsLog[1].proxyUrl).toEqual(proxies[1]);
expect(optionsLog[2].proxyUrl).toEqual(proxies[2]);
expect(optionsLog[3].proxyUrl).toEqual(proxies[0]);
+
+ delete process.env[ENV_VARS.PROXY_PASSWORD];
});
- describe('throws', () => {
- let pool;
- beforeEach(() => {
- log.setLevel(log.LEVELS.OFF);
- });
- afterEach(async () => {
- log.setLevel(log.LEVELS.ERROR);
- if (pool) await pool.destroy();
- });
+ test('should throw on proxyConfiguration together with proxyUrl from launchPuppeteerOptions', async () => {
+ process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';
- test('when used with proxyConfiguration', async () => {
- process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';
- const status = { connected: true };
- const fakeCall = async () => {
- return { body: status };
- };
+ const proxyConfiguration = await Apify.createProxyConfiguration({
+ proxyUrls: ['http://proxy.com:1111', 'http://proxy.com:2222', 'http://proxy.com:3333'],
+ });
- const stub = sinon.stub(utilsRequest, 'requestAsBrowser').callsFake(fakeCall);
- const proxyConfiguration = await Apify.createProxyConfiguration({
- groups: ['G1', 'G2'],
+ try {
+ // eslint-disable-next-line no-unused-vars
+ const pool = new Apify.PuppeteerPool({
+ proxyConfiguration,
+ launchPuppeteerOptions: {
+ proxyUrl: 'http://proxy.com:1111',
+ },
});
+ throw new Error('wrong error');
+ } catch (err) {
+ expect(err.message).toMatch('It is not possible to combine "options.proxyConfiguration"');
+ }
- try {
- pool = new Apify.PuppeteerPool({
- maxOpenPagesPerInstance: 1,
- proxyUrls: ['http://proxy.com:1111', 'http://proxy.com:2222', 'http://proxy.com:3333'],
- proxyConfiguration,
- });
- throw new Error('Invalid error.');
- } catch (err) {
- expect(err.stack).toMatch('options.proxyConfiguration');
- }
-
- delete process.env[ENV_VARS.PROXY_PASSWORD];
- stub.restore();
- });
-
- test('when empty', async () => {
- try {
- pool = new Apify.PuppeteerPool({
- maxOpenPagesPerInstance: 1,
- proxyUrls: [],
- });
- throw new Error('Invalid error.');
- } catch (err) {
- expect(err.message).toMatch('must not be empty');
- }
- });
+ delete process.env[ENV_VARS.PROXY_PASSWORD];
});
});
@@ -728,10 +712,12 @@ describe('PuppeteerPool', () => {
});
expect(pool.sessionPool.constructor.name).toEqual('SessionPool');
const page = await pool.newPage();
- const browser = page.browser();
- expect(browser[BROWSER_SESSION_KEY_NAME].id).toEqual(sessionPool.sessions[0].id);
+ // eslint-disable-next-line no-underscore-dangle
+ const browserSession = pool._getBrowserInstance(page).session;
+
+ expect(browserSession.id).toEqual(sessionPool.sessions[0].id);
expect(
- Object.values(pool.activeInstances).filter(instance => instance.session.id === browser[BROWSER_SESSION_KEY_NAME].id),
+ Object.values(pool.activeInstances).filter(instance => instance.session.id === browserSession.id),
).toHaveLength(1);
sessionPool.sessions[0].retire();
diff --git a/test/session_pool/session.test.js b/test/session_pool/session.test.js
index 954b123b0..099ec4d5b 100644
--- a/test/session_pool/session.test.js
+++ b/test/session_pool/session.test.js
@@ -124,7 +124,7 @@ describe('Session - testing session behaviour ', () => {
session = new Session({ sessionPool });
let error;
try {
- proxyConfiguration.getUrl(session.id);
+ proxyConfiguration.newUrl(session.id);
} catch (e) {
error = e;
}