Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Moving some constants to apify-shared and changing proxy configuration (#71)

Marek Trunkát052017904ad7
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ad1d17171..e4027396b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,17 +1,23 @@
xxxxxxxxxxxxxxxxxxx
===================
+- WARNING: `disableProxy` configuration of `PuppeteerCrawler`, `BasicCrawler` and `PuppeteerPool` removed. By default no proxy is used.
+ You must either use new configuration `launchPuppeteerOptions.useApifyProxy = true` to use Apify Proxy or provide own proxy via
+ `launchPuppeteerOptions.proxyUrl`.
+- WARNING: `groups` parameter of `PuppeteerCrawler`, `BasicCrawler` and `PuppeteerPool` removed. Use `launchPuppeteerOptions.apifyProxyGroups` instead.
+- WARNING: `session` and `groups` parameters of `Apify.launchPuppeteer()` are now validated to contain only alphanumberic
+ characters and underscores.
- `Apify.call()` now throws an `ApifyCallError` error if run doesn't succeed
- Renamed options `abortInstanceAfterRequestCount` of `PuppeteerPool` and `PuppeteerCrawler` to retireInstanceAfterRequestCcount
- Logs are now in plain text instead of JSON for better readability.
0.5.22 / 2018-04-12
===================
+- WARNING: `AutoscaledPool` was completely redesigned. Check documentation for reference. It still supports previous
+ configuration parameters for backwards compatibility but in the future compatibility will break.
- `handleFailedRequestFunction` in both `BasicCrawler` and `PuppeteerCrawler` has now also error object
available in `ops.error`.
- Request Queue storage type implemented. See documentation for more information.
- `BasicCrawler` and `PuppeteerCrawler` now supports both `RequestList` and `RequestQueue`.
-- `AutoscaledPool` was completely redesigned. Check documentation for reference. It still supports previous
- configuration parameters for backwards compatibility but in the future compatibility will break.
- `launchPuppeteer()` changes `User-Agent` only when in headless mode or if not using full Google Chrome,
to reduce chance of detection of the crawler.
- Apify package now supports Node 7 and newer.
diff --git a/package.json b/package.json
index d7d6b0fd0..9475bda31 100644
--- a/package.json
+++ b/package.json
@@ -45,7 +45,7 @@
},
"dependencies": {
"apify-client": "^0.2.7",
- "apify-shared": "^0.0.28",
+ "apify-shared": "^0.0.29",
"bluebird": "^3.5.0",
"content-type": "^1.0.3",
"fs-extra": "^5.0.0",
diff --git a/src/actor.js b/src/actor.js
index 19de1abdc..12a07e0ac 100644
--- a/src/actor.js
+++ b/src/actor.js
@@ -2,6 +2,7 @@ import _ from 'underscore';
import Promise from 'bluebird';
import fs from 'fs';
import { checkParamOrThrow } from 'apify-client/build/utils';
+import { PROXY_GROUP_NAME_REGEX, PROXY_SESSION_ID_REGEX } from 'apify-shared/regexs';
import { ENV_VARS, EXIT_CODES, ACT_TASK_TERMINAL_STATUSES, ACT_TASK_STATUSES, DEFAULT_PROXY_HOSTNAME, DEFAULT_PROXY_PORT } from './constants';
import { initializeEvents, stopEvents } from './events';
import { newPromise, apifyClient, addCharsetToContentType } from './utils';
@@ -391,21 +392,28 @@ export const getApifyProxyUrl = (opts = {}) => {
port = parseInt(process.env[ENV_VARS.PROXY_PORT], 10) || DEFAULT_PROXY_PORT,
} = opts;
- // TODO: Check that session and groups are alphanumeric!
+ const getMissingParamErrorMgs = (param, env) => `Apify Proxy ${param} must be provided as parameter or "${env}" environment variable!`;
+ const throwNonAlphanumericError = (param) => { throw new Error(`Only alphanumeric characters and underscore are allowed in ${param}`); };
- checkParamOrThrow(groups, 'opts.groups', 'Maybe Array');
+ checkParamOrThrow(groups, 'opts.groups', 'Maybe [String]');
checkParamOrThrow(session, 'opts.session', 'Maybe Number | String');
- checkParamOrThrow(password, 'opts.password', 'String');
- checkParamOrThrow(hostname, 'opts.hostname', 'String');
- checkParamOrThrow(port, 'opts.port', 'Number');
+ checkParamOrThrow(password, 'opts.password', 'String', getMissingParamErrorMgs('password', ENV_VARS.PROXY_PASSWORD));
+ checkParamOrThrow(hostname, 'opts.hostname', 'String', getMissingParamErrorMgs('hostname', ENV_VARS.PROXY_HOSTNAME));
+ checkParamOrThrow(port, 'opts.port', 'Number', getMissingParamErrorMgs('port', ENV_VARS.PROXY_PORT));
let username;
if (groups || session) {
const parts = [];
- if (groups && groups.length) parts.push(`GROUPS-${groups.join('+')}`);
- if (session) parts.push(`SESSION-${session}`);
+ if (groups && groups.length) {
+ if (!groups.every(group => PROXY_GROUP_NAME_REGEX.test(group))) throwNonAlphanumericError('proxy group names');
+ parts.push(`GROUPS-${groups.join('+')}`);
+ }
+ if (session) {
+ if (!PROXY_SESSION_ID_REGEX.test(session)) throwNonAlphanumericError('proxy session ID');
+ parts.push(`SESSION-${session}`);
+ }
username = parts.join(',');
} else {
diff --git a/src/basic_crawler.js b/src/basic_crawler.js
index 411acf6f7..0e57bd96c 100644
--- a/src/basic_crawler.js
+++ b/src/basic_crawler.js
@@ -1,10 +1,11 @@
import { checkParamOrThrow } from 'apify-client/build/utils';
import _ from 'underscore';
import log from 'apify-shared/log';
-import { isPromise, checkParamPrototypeOrThrow } from './utils';
+import { checkParamPrototypeOrThrow } from 'apify-shared/utilities';
import AutoscaledPool from './autoscaled_pool';
import RequestList from './request_list';
import { RequestQueue, RequestQueueLocal } from './request_queue';
+import { isPromise } from './utils';
const DEFAULT_OPTIONS = {
maxRequestRetries: 3,
diff --git a/src/dataset.js b/src/dataset.js
index 5900d9e68..4e70c0fbe 100644
--- a/src/dataset.js
+++ b/src/dataset.js
@@ -19,10 +19,10 @@ const { datasets } = apifyClient;
const datasetsCache = new LruCache({ maxLength: MAX_OPENED_STORES }); // Open Datasets are stored here.
/**
- * Dataset class provides easy interface to Apify Dataset storage type. Dataset should be opened using
- * `Apify.openDataset()` function.
+ * The `Dataset` class provides a simple interface to the [Apify Dataset](https://www.apify.com/docs/storage#dataset) storage.
+ * You should not instantiate this class directly, use the [Apify.openDataset()](#module-Apify-openDataset) function.
*
- * Basic usage of Dataset:
+ * Example usage:
*
* ```javascript
* const dataset = await Apify.openDataset('my-dataset-id');
@@ -55,7 +55,7 @@ export class Dataset {
}
/**
- * This is a local representation of a dataset.
+ * This is a local emulation of a dataset.
*
* @ignore
*/
@@ -120,20 +120,36 @@ const getOrCreateDataset = (datasetIdOrName) => {
});
};
+
/**
- * Opens dataset and returns its object.
+ * Opens a dataset and returns a promise resolving to an instance of the [Dataset](#Dataset) object.
+ *
+ * Dataset is an append-only storage that is useful for storing sequential or tabular results.
+ * For more information, see [Dataset documentation](https://www.apify.com/docs/storage#dataset).
+ *
+ * Example usage:
*
* ```javascript
- * const dataset = await Apify.openDataset('my-dataset-id');
+ * const store = await Apify.openDataset(); // Opens the default dataset of the run.
+ * const storeWithName = await Apify.openDataset('some-name'); // Opens dataset with name 'some-name'.
+ *
+ * // Write a single row to dataset
* await dataset.pushData({ foo: 'bar' });
+ *
+ * // Write multiple rows
+ * await dataset.pushData([
+ * { foo: 'bar2', col2: 'val2' },
+ * { col3: 123 },
+ * ]);
* ```
*
- * If the `APIFY_LOCAL_EMULATION_DIR` environment variable is defined, the value this function
- * returns an instance `DatasetLocal` which is an local emulation of dataset.
- * This is useful for local development and debugging of your acts.
+ * If the `APIFY_LOCAL_EMULATION_DIR` environment variable is set, the result of this function
+ * is an instance of the `DatasetLocal` class which stores the data in a local directory
+ * rather than Apify cloud. This is useful for local development and debugging of your acts.
*
- * @param {string} datasetIdOrName ID or name of the dataset to be opened.
- * @returns {Promise<Dataset>} Returns a promise that resolves to a Dataset object.
+ * @param {string} datasetIdOrName ID or name of the dataset to be opened. If no value is provided
+ * then the function opens the default dataset associated with the act run.
+ * @returns {Promise<Dataset>} Returns a promise that resolves to a `Dataset` object.
*
* @memberof module:Apify
* @name openDataset
diff --git a/src/key_value_store.js b/src/key_value_store.js
index 87c639311..1de60ee19 100644
--- a/src/key_value_store.js
+++ b/src/key_value_store.js
@@ -86,19 +86,27 @@ export const maybeStringify = (value, options) => {
return value;
};
+
/**
- * KeyValueStore class provides easy interface to Apify key-value store storage type. Key-value store should be opened using
- * `Apify.openKeyValueStore()` function.
+ * The `KeyValueStore` class provides a simple interface to the [Apify Key-value stores](https://www.apify.com/docs/storage#kv-store).
+ * You should not instantiate this class directly, use the
+ * [Apify.openKeyValueStore()](#module-Apify-openKeyValueStore) function.
*
- * Basic usage of key-value store:
+ * Example usage:
*
* ```javascript
- * const store = await Apify.openKeyValueStore('my-store-id');
+ * // Opens default key-value store of the run.
+ * const store = await Apify.openKeyValueStore();
+ *
+ * // Opens key-value store called 'some-name', belonging to the current Apify user account.
+ * const storeWithName = await Apify.openKeyValueStore('some-name');
+ *
+ * // Write and read data record
* await store.setValue('some-key', { foo: 'bar' });
* const value = store.getValue('some-key');
* ```
*
- * @param {String} storeId - ID of the store.
+ * @param {String} storeId - ID of the key-value store.
*/
export class KeyValueStore {
constructor(storeId) {
@@ -107,9 +115,12 @@ export class KeyValueStore {
this.storeId = storeId;
}
+ // TODO: Move here the Apify.getValue()/setValue() documentation, and link it from there.
+ // This place should be the main source of information.
+
/**
- * Stores object or an array of objects in the dataset.
- * The function has no result, but throws on invalid args or other errors.
+ * Gets a record from the current key-value store using its key.
+ * For more details, see [Apify.getValue](#module-Apify-getValue).
*
* @param {String} key Record key.
* @return {Promise}
@@ -123,8 +134,8 @@ export class KeyValueStore {
}
/**
- * Stores object or an array of objects in the dataset.
- * The function has no result, but throws on invalid args or other errors.
+ * Stores a record to the key-value stores.
+ * The function has no result, but throws on invalid arguments or other errors.
*
* @param {String} key Record key.
* @param {Object|String|Buffer} value Record value. If content type is not provided then the value is stringified to JSON.
@@ -251,19 +262,27 @@ const getOrCreateKeyValueStore = (storeIdOrName) => {
});
};
+
/**
- * Opens key-value store and returns its object.</p>
+ * Opens a key-value store and returns a promise resolving to an instance
+ * of the [KeyValueStore](#KeyValueStore) class.
+ *
+ * Key-value store is a simple storage for records, where each record has a unique key.
+ * For more information, see [Key-value store documentation](https://www.apify.com/docs/storage#dataset).
+ *
+ * Example usage:
*
* ```javascript
* const store = await Apify.openKeyValueStore('my-store-id');
* await store.setValue('some-key', { foo: 'bar' });
* ```
*
- * If the `APIFY_LOCAL_EMULATION_DIR` environment variable is defined, the value this function
- * returns an instance `KeyValueStoreLocal` which is an local emulation of key-value store.
- * This is useful for local development and debugging of your acts.
+ * If the `APIFY_LOCAL_EMULATION_DIR` environment variable is set, the result of this function
+ * is an instance of the `KeyValueStoreLocal` class which stores the records in a local directory
+ * rather than Apify cloud. This is useful for local development and debugging of your acts.
*
- * @param {string} storeIdOrName ID or name of the key-value store to be opened.
+ * @param {string} storeIdOrName ID or name of the key-value store to be opened. If no value is
+ * provided then the function opens the default key-value store associated with the act run.
* @returns {Promise<KeyValueStore>} Returns a promise that resolves to a KeyValueStore object.
*
* @memberof module:Apify
diff --git a/src/puppeteer.js b/src/puppeteer.js
index 72dee2133..a5a45d15c 100644
--- a/src/puppeteer.js
+++ b/src/puppeteer.js
@@ -1,28 +1,116 @@
import { checkParamOrThrow } from 'apify-client/build/utils';
import { anonymizeProxy, closeAnonymizedProxy } from 'proxy-chain';
+import { cryptoRandomObjectId } from 'apify-shared/utilities';
import { ENV_VARS, DEFAULT_USER_AGENT } from './constants';
import { newPromise, getTypicalChromeExecutablePath } from './utils';
+import { getApifyProxyUrl } from './actor';
/* global process, require */
/**
- * Launches headless Chrome using Puppeteer pre-configured to work with the Apify Actor platform.
- * The function has the same argument and return value as `puppeteer.launch()`.
+ * @typedef {Object} LaunchPuppeteerOptions
+ * @property {String} [opts.proxyUrl] URL to a HTTP proxy server. It must define the port number,
+ * and it might also contain proxy username and password.
+ * For example: `http://bob:[email protected]:1234`.
+ * @property {String} [opts.userAgent] HTTP `User-Agent` header used by the browser.
+ * If not provided, the function sets `User-Agent` to a reasonable default
+ * to reduce the chance of detection of the crawler.
+ * @property {String} [opts.useChrome=false] If `true` and `opts.executablePath` is not set,
+ * Puppeteer will launch full Google Chrome browser available on the machine
+ * rather than the bundled Chromium. The path to Chrome executable
+ * is taken from the `APIFY_CHROME_EXECUTABLE_PATH` environment variable if provided,
+ * or defaults to the typical Google Chrome executable location specific for the operating system.
+ * By default, this option is `false`.
+ * @property {String} [opts.useApifyProxy=false] If set to `true` then Puppeteer will be configured to use
+ * <a href="https://www.apify.com/docs/proxy" target="_blank">Apify Proxy</a>.
+ * @property {String} [opts.apifyProxyGroups] An array of proxy groups to be used
+ * when using the <a href="https://www.apify.com/docs/proxy" target="_blank">Apify Proxy</a>.
+ * @property {String} [opts.apifyProxySession] <a href="https://www.apify.com/docs/proxy" target="_blank">Apify Proxy</a> session ID that
+ * identifies requests that should use the same proxy connection.
+ */
+
+/**
+ * Launches Puppeteer with proxy used via `proxy-chain` package.
+ *
+ * @ignore
+ */
+const launchPuppeteerWithProxy = (puppeteer, opts) => {
+ let anonymizedProxyUrl;
+
+ // Parse and validate proxy URL and anonymize it
+ // NOTE: anonymizeProxy() throws on invalid proxy URL, so it must not be inside a Promise!
+ return anonymizeProxy(opts.proxyUrl)
+ .then((result) => {
+ anonymizedProxyUrl = result;
+ opts.args.push(`--proxy-server=${anonymizedProxyUrl}`);
+ })
+ .then(() => puppeteer.launch(opts))
+ // Close anonymization proxy server when Puppeteer finishes
+ .then((browser) => {
+ const cleanUp = () => {
+ // Don't wait for finish, only log errors
+ closeAnonymizedProxy(anonymizedProxyUrl, true)
+ .catch((err) => {
+ console.log(`WARNING: closeAnonymizedProxy() failed with: ${err.stack || err}`);
+ });
+ };
+
+ browser.on('disconnected', cleanUp);
+
+ const prevClose = browser.close.bind(browser);
+ browser.close = () => {
+ cleanUp();
+ return prevClose();
+ };
+
+ return browser;
+ });
+};
+
+/**
+ * Requires `puppeteer` package or throws meaningful error if not installed.
+ *
+ * @ignore
+ */
+const getPuppeteerOrThrow = () => {
+ try {
+ // This is an optional dependency because it is quite large, only require it when used (ie. image with Chrome)
+ return require('puppeteer'); // eslint-disable-line global-require
+ } catch (err) {
+ if (err.code === 'MODULE_NOT_FOUND') err.message = 'Cannot find module \'puppeteer\'. Did you choose the correct base Docker image?';
+
+ throw err;
+ }
+};
+
+/**
+ * Launches headless Chrome using Puppeteer pre-configured to work within the Apify platform.
+ * The function has the same argument and the return value as `puppeteer.launch()`.
* See <a href="https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#puppeteerlaunchoptions" target="_blank">
* Puppeteer documentation</a> for more details.
*
* The `launchPuppeteer()` function alters the following Puppeteer options:
*
* <ul>
- * <li>Passes the setting from the `APIFY_HEADLESS` environment variable to the `headless` option,
+ * <li>
+ * Passes the setting from the `APIFY_HEADLESS` environment variable to the `headless` option,
* unless it was already defined by the caller or `APIFY_XVFB` environment variable is set to `1`.
- * Note that Apify Actor cloud platform automatically sets `APIFY_HEADLESS=1` to all running acts.</li>
- * <li>Takes the `proxyUrl` option, checks it and adds it to `args` as `--proxy-server=XXX`.
+ * Note that Apify Actor cloud platform automatically sets `APIFY_HEADLESS=1` to all running acts.
+ * </li>
+ * <li>
+ * Takes the `proxyUrl` option, checks it and adds it to `args` as `--proxy-server=XXX`.
* If the proxy uses authentication, the function sets up an anonymous proxy HTTP
* to make the proxy work with headless Chrome. For more information, read the
* <a href="https://blog.apify.com/249a21a79212" target="_blank">blog post about proxy-chain library</a>.
* </li>
- * <li>Adds `--no-sandbox` to `args` to enable running headless Chrome in a Docker container on the Apify Actor platform.</li>
+ * <li>
+ * If `opts.useApifyProxy` is `true` then the function generates a URL of
+ * <a href="https://www.apify.com/docs/proxy" target="_blank">Apify Proxy</a>
+ * based on `opts.apifyProxyGroups` and `opts.apifyProxySession` and passes it as `opts.proxyUrl`.
+ * </li>
+ * <li>
+ * The function adds `--no-sandbox` to `args` to enable running headless Chrome in a Docker container on the Apify platform.
+ * </li>
* </ul>
*
* To use this function, you need to have the <a href="https://www.npmjs.com/package/puppeteer" target="_blank">puppeteer</a>
@@ -33,19 +121,8 @@ import { newPromise, getTypicalChromeExecutablePath } from './utils';
*
* For an example of usage, see the <a href="https://www.apify.com/apify/example-puppeteer">apify/example-puppeteer</a> act.
*
- * @param {Object} [opts] Optional settings passed to `puppeteer.launch()`. Additionally the object can contain the following fields:
- * @param {String} [opts.proxyUrl] URL to a HTTP proxy server.
- * Port number must be specified. Proxy username and password might also be provided.
- * For example, `http://bob:[email protected]:1234`.
- * @param {String} [opts.userAgent] HTTP `User-Agent` header used by the browser.
- * If not provided, the function sets `User-Agent` to a reasonable default
- * to reduce the chance of detection of the crawler.
- * @param {String} [opts.useChrome=false] If `true` and `opts.executablePath` is not set,
- * Puppeteer will launch full Google Chrome browser available on the machine
- * rather than the bundled Chromium. The path to Chrome executable
- * is taken from the `APIFY_CHROME_EXECUTABLE_PATH` environment variable if provided,
- * or defaults to the typical Google Chrome executable location specific for the operating system.
- * By default, this option is `false`.
+ * @param {LaunchPuppeteerOptions} [opts] Optional settings passed to `puppeteer.launch()`. Additionally the object can
+ * contain the following fields:
* @returns {Promise} Promise object that resolves to Puppeteer's `Browser` instance.
*
* @memberof module:Apify
@@ -53,22 +130,15 @@ import { newPromise, getTypicalChromeExecutablePath } from './utils';
* @instance
* @function
*/
-export const launchPuppeteer = (opts) => {
- if (opts === undefined || opts === null) opts = {};
-
+export const launchPuppeteer = (opts = {}) => {
checkParamOrThrow(opts, 'opts', 'Object');
checkParamOrThrow(opts.args, 'opts.args', 'Maybe [String]');
checkParamOrThrow(opts.proxyUrl, 'opts.proxyUrl', 'Maybe String');
+ checkParamOrThrow(opts.useApifyProxy, 'opts.useApifyProxy', 'Maybe Boolean');
- let puppeteer;
- try {
- // This is an optional dependency because it is quite large, only require it when used (ie. image with Chrome)
- puppeteer = require('puppeteer'); // eslint-disable-line global-require
- } catch (err) {
- if (err.code === 'MODULE_NOT_FOUND') err.message = 'Cannot find module \'puppeteer\'. Did you choose the correct base Docker image?';
+ if (opts.useApifyProxy && opts.proxyUrl) throw new Error('Cannot combine "opts.useApifyProxy" with "opts.proxyUrl"!');
- throw err;
- }
+ const puppeteer = getPuppeteerOrThrow();
opts.args = opts.args || [];
opts.args.push('--no-sandbox');
@@ -78,6 +148,12 @@ export const launchPuppeteer = (opts) => {
if (opts.useChrome && (opts.executablePath === undefined || opts.executablePath === null)) {
opts.executablePath = process.env[ENV_VARS.CHROME_EXECUTABLE_PATH] || getTypicalChromeExecutablePath();
}
+ if (opts.useApifyProxy) {
+ opts.proxyUrl = getApifyProxyUrl({
+ groups: opts.apifyProxyGroups,
+ session: opts.apifyProxySession || cryptoRandomObjectId(),
+ });
+ }
// When User-Agent is not set and we're using Chromium or headless mode,
// it is better to use DEFAULT_USER_AGENT to reduce chance of detection
@@ -89,45 +165,10 @@ export const launchPuppeteer = (opts) => {
opts.args.push(`--user-agent=${userAgent}`);
}
- let anonymizedProxyUrl;
- let promise;
-
- // Parse and validate proxy URL and anonymize it
- if (opts.proxyUrl) {
- // NOTE: anonymizeProxy() throws on invalid proxy URL, so it must not be inside a Promise!
- promise = anonymizeProxy(opts.proxyUrl)
- .then((result) => {
- anonymizedProxyUrl = result;
- opts.args.push(`--proxy-server=${anonymizedProxyUrl}`);
- })
- .then(() => puppeteer.launch(opts));
- } else {
- promise = puppeteer.launch(opts);
- }
-
- // Close anonymization proxy server when Puppeteer finishes
- if (opts.proxyUrl) {
- promise = promise.then((browser) => {
- const cleanUp = () => {
- // Don't wait for finish, only log errors
- closeAnonymizedProxy(anonymizedProxyUrl, true)
- .catch((err) => {
- console.log(`WARNING: closeAnonymizedProxy() failed with: ${err.stack || err}`);
- });
- };
-
- browser.on('disconnected', cleanUp);
-
- const prevClose = browser.close.bind(browser);
- browser.close = () => {
- cleanUp();
- return prevClose();
- };
-
- return browser;
- });
- }
+ const browserPromise = opts.proxyUrl
+ ? launchPuppeteerWithProxy(puppeteer, opts)
+ : puppeteer.launch(opts);
- // Ensure that the returned promise is of type set in setPromiseDependency()
- return newPromise().then(() => promise);
+ // Ensure that the returned promise is of type Bluebird.
+ return newPromise().then(() => browserPromise);
};
diff --git a/src/puppeteer_crawler.js b/src/puppeteer_crawler.js
index 4003a977e..2d7da70d4 100644
--- a/src/puppeteer_crawler.js
+++ b/src/puppeteer_crawler.js
@@ -79,8 +79,6 @@ const DEFAULT_OPTIONS = {
* After the limit is reached the browser will be retired and new requests will
* be handled by a new browser instance.
* See `retireInstanceAfterRequestCount` parameter of `PuppeteerPool`.
- * @param {Function} [options.launchPuppeteerFunction] Overrides how new Puppeteer instance gets launched. See `launchPuppeteerFunction` parameter of
- * `PuppeteerPool`.
* @param {Number} [options.instanceKillerIntervalMillis=60000] How often the launched Puppeteer instances are checked whether they can be
* closed. See `instanceKillerIntervalMillis` parameter of `PuppeteerPool`.
* @param {Number} [options.killInstanceAfterMillis=300000] If Puppeteer instance reaches the `options.retireInstanceAfterRequestCount` limit then
@@ -90,8 +88,11 @@ const DEFAULT_OPTIONS = {
* `killInstanceAfterMillis` parameter of `PuppeteerPool`.
* @param {Object} [options.puppeteerConfig={ dumpio: process.env.NODE_ENV !== 'production', slowMo: 0, args: []}] Default options for each
* new Puppeteer instance. See `puppeteerConfig` parameter of `PuppeteerPool`.
- * @param {Boolean} [options.disableProxy=false] Disables proxying through Apify proxy. See `disableProxy` parameter of `PuppeteerPool`.
- * @param {Array} [options.groups] Apify proxy groups to be used. See `Apify.getApifyProxyUrl()` for more details.
+ * @param {Function} [options.launchPuppeteerFunction=launchPuppeteerOptions&nbsp;=>&nbsp;Apify.launchPuppeteer(launchPuppeteerOptions)]
+ * Overrides the default function to launch a new Puppeteer instance.
+ * See `launchPuppeteerFunction` parameter of `PuppeteerPool`.
+ * @param {LaunchPuppeteerOptions} [options.launchPuppeteerOptions] Options used by `Apify.launchPuppeteer()` to start new Puppeteer instances.
+ * See `launchPuppeteerOptions` parameter of `PuppeteerPool`.
*/
export default class PuppeteerCrawler {
constructor(opts) {
@@ -122,15 +123,10 @@ export default class PuppeteerCrawler {
// Puppeteer Pool options
maxOpenPagesPerInstance,
retireInstanceAfterRequestCount,
- launchPuppeteerFunction,
instanceKillerIntervalMillis,
killInstanceAfterMillis,
-
- // TODO: 'groups' is not a great name, but we need to
- // review proxy settings in general
- groups,
- puppeteerConfig,
- disableProxy,
+ launchPuppeteerFunction,
+ launchPuppeteerOptions,
} = _.defaults(opts, DEFAULT_OPTIONS);
checkParamOrThrow(handlePageFunction, 'opts.handlePageFunction', 'Function');
@@ -145,12 +141,10 @@ export default class PuppeteerCrawler {
this.puppeteerPool = new PuppeteerPool({
maxOpenPagesPerInstance,
retireInstanceAfterRequestCount,
- launchPuppeteerFunction,
instanceKillerIntervalMillis,
killInstanceAfterMillis,
- groups,
- puppeteerConfig,
- disableProxy,
+ launchPuppeteerFunction,
+ launchPuppeteerOptions,
});
this.basicCrawler = new BasicCrawler({
diff --git a/src/puppeteer_pool.js b/src/puppeteer_pool.js
index de6f7ccc2..0c186865e 100644
--- a/src/puppeteer_pool.js
+++ b/src/puppeteer_pool.js
@@ -1,17 +1,8 @@
import _ from 'underscore';
import log from 'apify-shared/log';
import Promise from 'bluebird';
-import { cryptoRandomObjectId } from 'apify-shared/utilities';
import { checkParamOrThrow } from 'apify-client/build/utils';
import { launchPuppeteer } from './puppeteer';
-import { getApifyProxyUrl } from './actor';
-import { isProduction } from './utils';
-
-const DEFAULT_PUPPETEER_CONFIG = {
- dumpio: !isProduction(),
- slowMo: 0,
- args: [],
-};
const PROCESS_KILL_TIMEOUT_MILLIS = 5000;
@@ -26,30 +17,7 @@ const DEFAULT_OPTIONS = {
killInstanceAfterMillis: 5 * 60 * 1000,
// TODO: use settingsRotator()
- launchPuppeteerFunction: ({ groups, puppeteerConfig, disableProxy = false }) => {
- checkParamOrThrow(groups, 'opts.groups', 'Maybe Array');
- checkParamOrThrow(puppeteerConfig, 'opts.puppeteerConfig', 'Maybe Object');
- checkParamOrThrow(disableProxy, 'opts.disableProxy', 'Maybe Boolean');
-
- const config = Object.assign({}, DEFAULT_PUPPETEER_CONFIG, puppeteerConfig);
-
- // TODO: this should be in Apify.launchPuppeteer(). But is this needed at all?
- // It might be confusing because the feature has the same name
- // as Chrome command line flag, so people will assume it's doing just that.
- // For simplicity I'd just remove it...
- if (config.disableWebSecurity) {
- config.ignoreHTTPSErrors = true;
- config.args.push('--disable-web-security');
- }
-
- // TODO: Maybe we should move this whole logic directly to Apify.launchPuppeteer().
- // E.g. if process.env.APIFY_PROXY_HOST is defined, then puppeteer should use it with "auto".
- if (!disableProxy) {
- config.proxyUrl = getApifyProxyUrl({ groups, session: cryptoRandomObjectId() });
- }
-
- return launchPuppeteer(config);
- },
+ launchPuppeteerFunction: launchPuppeteerOptions => launchPuppeteer(launchPuppeteerOptions),
};
/**
@@ -93,17 +61,15 @@ class PuppeteerInstance {
* @param {Number} [options.retireInstanceAfterRequestCount=150] Maximum number of requests that can be processed by a single browser instance.
* After the limit is reached the browser will be retired and new requests will
* be handled by a new browser instance.
- * @param {Function} [options.launchPuppeteerFunction] Overrides the default function to launch a new Puppeteer instance.
* @param {Number} [options.instanceKillerIntervalMillis=60000] How often opened Puppeteer instances are checked wheter they can be
* closed.
* @param {Number} [options.killInstanceAfterMillis=300000] If Puppeteer instance reaches the `options.retireInstanceAfterRequestCount` limit then
* it is considered retired and no more tabs will be opened. After the last tab is closed the
* whole browser is closed too. This parameter defines a time limit for inactivity after
* which the browser is closed even if there are pending open tabs.
- * @param {Object} [options.puppeteerConfig={ dumpio: process.env.NODE_ENV !== 'production', slowMo: 0, args: []}] Configuration of Puppeteer
- * instances.
- * @param {Boolean} [options.disableProxy=false] Disables proxying through Apify proxy.
- * @param {Array} [options.groups] Apify proxy groups to be used. See `Apify.getApifyProxyUrl()` for more details.
+ * @param {Function} [options.launchPuppeteerFunction=launchPuppeteerOptions&nbsp;=>&nbsp;Apify.launchPuppeteer(launchPuppeteerOptions)]
+ * Overrides the default function to launch a new Puppeteer instance.
+ * @param {LaunchPuppeteerOptions} [options.launchPuppeteerOptions] Options used by `Apify.launchPuppeteer()` to start new Puppeteer instances.
*/
export default class PuppeteerPool {
constructor(opts = {}) {
@@ -121,6 +87,7 @@ export default class PuppeteerPool {
launchPuppeteerFunction,
instanceKillerIntervalMillis,
killInstanceAfterMillis,
+ launchPuppeteerOptions,
} = _.defaults(opts, DEFAULT_OPTIONS);
checkParamOrThrow(maxOpenPagesPerInstance, 'opts.maxOpenPagesPerInstance', 'Number');
@@ -128,12 +95,13 @@ export default class PuppeteerPool {
checkParamOrThrow(launchPuppeteerFunction, 'opts.launchPuppeteerFunction', 'Function');
checkParamOrThrow(instanceKillerIntervalMillis, 'opts.instanceKillerIntervalMillis', 'Number');
checkParamOrThrow(killInstanceAfterMillis, 'opts.killInstanceAfterMillis', 'Number');
+ checkParamOrThrow(launchPuppeteerOptions, 'opts.launchPuppeteerOptions', 'Maybe Object');
// Config.
this.maxOpenPagesPerInstance = maxOpenPagesPerInstance;
this.retireInstanceAfterRequestCount = retireInstanceAfterRequestCount;
this.killInstanceAfterMillis = killInstanceAfterMillis;
- this.launchPuppeteerFunction = () => launchPuppeteerFunction(opts);
+ this.launchPuppeteerFunction = () => launchPuppeteerFunction(launchPuppeteerOptions);
// State.
this.browserCounter = 0;
diff --git a/src/request_queue.js b/src/request_queue.js
index 43539cc11..b8a60fed8 100644
--- a/src/request_queue.js
+++ b/src/request_queue.js
@@ -4,13 +4,13 @@ import path from 'path';
import { checkParamOrThrow } from 'apify-client/build/utils';
import LruCache from 'apify-shared/lru_cache';
import ListDictionary from 'apify-shared/list_dictionary';
-import { delayPromise } from 'apify-shared/utilities';
+import { delayPromise, checkParamPrototypeOrThrow } from 'apify-shared/utilities';
import Promise from 'bluebird';
import crypto from 'crypto';
import _ from 'underscore';
import Request from './request';
-import { ENV_VARS } from '../build/constants';
-import { ensureDirExists, checkParamPrototypeOrThrow, apifyClient } from './utils';
+import { ENV_VARS, REQUEST_QUEUE_HEAD_MAX_LIMIT } from '../build/constants';
+import { ensureDirExists, apifyClient } from './utils';
export const LOCAL_EMULATION_SUBDIR = 'request-queues';
const MAX_OPENED_QUEUES = 1000;
@@ -25,9 +25,6 @@ export const API_PROCESSED_REQUESTS_DELAY_MILLIS = 10 * 1000;
// How many times we try to get queue head with queueModifiedAt older than API_PROCESSED_REQUESTS_DELAY_MILLIS.
export const MAX_QUERIES_FOR_CONSISTENCY = 6;
-// TODO: this should be in shared.
-export const QUERY_HEAD_MAX_LIMIT = 1000;
-
const writeFilePromised = Promise.promisify(fs.writeFile);
const readdirPromised = Promise.promisify(fs.readdir);
const readFilePromised = Promise.promisify(fs.readFile);
@@ -94,19 +91,25 @@ const validateReclaimRequestParams = (request, opts) => {
*/
/**
- * Provides a simple interface to the Apify Request Queue API,
- * which is used to manage a dynamic queue of web pages to crawl.
- * The instance of this object can be obtained using the
- * `Apify.openRequestQueue()` function.
+ * Provides a simple interface to the [Apify Request Queue](https://www.apify.com/docs/storage#queue)
+ * storage, which is used to manage a dynamic queue of web pages to crawl.
+ *
+ * You should not instantiate this class directly, but use the
+ * [Apify.openRequestQueue()](#module-Apify-openRequestQueue) function.
*
* Example usage:
*
* ```javascript
- * const queue = await Apify.openRequestQueue('my-queue-id');
+ * // Opens default request queue of the run.
+ * const queue = await Apify.openRequestQueue();
*
- * await queue.addRequest(new Apify.Request({ url: 'http://example.com/aaa'});
- * await queue.addRequest(new Apify.Request({ url: 'http://example.com/bbb'});
- * await queue.addRequest(new Apify.Request({ url: 'http://example.com/foo/bar'}, { forefront: true });
+ * // Opens request queue called 'some-name'.
+ * const queueWithName = await Apify.openRequestQueue('some-name');
+ *
+ * // Enqueue few requests
+ * await queue.addRequest(new Apify.Request({ url: 'http://example.com/aaa'}));
+ * await queue.addRequest(new Apify.Request({ url: 'http://example.com/bbb'}));
+ * await queue.addRequest(new Apify.Request({ url: 'http://example.com/foo/bar'}, { forefront: true }));
*
* // Get requests from queue
* const request1 = queue.fetchNextRequest();
@@ -351,7 +354,7 @@ export class RequestQueue {
// low and contained only requests in progress.
//
// If limit was not reached in the call then there are no more requests to be returned.
- const shouldRepeatWithHigherLimit = !this.queueHeadDict.length() && limitReached && prevLimit < QUERY_HEAD_MAX_LIMIT;
+ const shouldRepeatWithHigherLimit = !this.queueHeadDict.length() && limitReached && prevLimit < REQUEST_QUEUE_HEAD_MAX_LIMIT;
// If checkModifiedAt=true then we must ensure that queueModifiedAt is older than
// queryStartedAt for at least API_PROCESSED_REQUESTS_DELAY_MILLIS.
@@ -371,7 +374,7 @@ export class RequestQueue {
}
const nextLimit = shouldRepeatWithHigherLimit
- ? prevLimit + QUERY_HEAD_MAX_LIMIT
+ ? prevLimit + QUERY_HEAD_BUFFER
: prevLimit;
const delayMillis = shouldRepeatForConsistency
@@ -430,6 +433,7 @@ export class RequestQueueLocal {
this.localHandledEmulationPath = path.join(this.localEmulationPath, 'handled');
this.localPendingEmulationPath = path.join(this.localEmulationPath, 'pending');
+ this.queueOrderNoCounter = 0; // Counter used in _getQueueOrderNo to ensure there won't be a collision.
this.pendingCount = 0;
this.inProgressCount = 0;
this.requestIdToQueueOrderNo = {};
@@ -480,7 +484,7 @@ export class RequestQueueLocal {
const sgn = (forefront ? 1 : 2) * (10 ** 15);
const base = (10 ** (13)); // Date.now() returns int with 13 numbers.
// We always add pending count for a case that two pages are insterted at the same millisecond.
- const now = Date.now() + this.pendingCount;
+ const now = Date.now() + this.queueOrderNoCounter++;
const queueOrderNo = forefront
? sgn + (base - now)
: sgn + (base + now);
@@ -661,11 +665,15 @@ const getOrCreateQueue = (queueIdOrName) => {
};
/**
- * Opens a request queue and returns its `RequestQueue` object.
+ * Opens a request queue and returns a promise resolving to an instance
+ * of the [RequestQueue](#RequestQueue) class.
+ *
+ * Example usage:
*
* ```javascript
* const queue = await Apify.openRequestQueue('my-queue-id');
*
+ * // Enqueue some requests
* await queue.addRequest(new Apify.Request({ url: 'http://example.com/aaa'});
* await queue.addRequest(new Apify.Request({ url: 'http://example.com/bbb'});
* await queue.addRequest(new Apify.Request({ url: 'http://example.com/foo/bar'}, { forefront: true });
@@ -687,7 +695,8 @@ const getOrCreateQueue = (queueIdOrName) => {
* This is useful for local development and debugging of the acts.
*
* @param {string} queueIdOrName ID or name of the request queue to be opened.
- * @returns {Promise<RequestQueue>} Returns a promise that resolves to a `RequestQueue` object.
+ * @returns {Promise<RequestQueue>} Returns a promise that resolves to a `RequestQueue` object. If no value is provided
+ * then the function opens the default request queue associated with the act run.
*
* @memberof module:Apify
* @name openRequestQueue
diff --git a/src/utils.js b/src/utils.js
index 355ef9c4d..7d31cb353 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -2,7 +2,6 @@ import Promise from 'bluebird';
import contentTypeParser from 'content-type';
import os from 'os';
import fs from 'fs';
-import _ from 'underscore';
import fsExtra from 'fs-extra';
import ApifyClient from 'apify-client';
import { ENV_VARS } from './constants';
@@ -175,22 +174,6 @@ export const isPromise = (maybePromise) => {
return maybePromise && typeof maybePromise.then === 'function' && typeof maybePromise.catch === 'function';
};
-/**
- * Helper function for validation if parameter is an instance of given prototype or multiple prototypes.
- * TODO: Move this to shared package along with checkParamOrThrow
- *
- * @ignore
- */
-export const checkParamPrototypeOrThrow = (paramVal, paramName, prototypes, prototypeName, isOptional = false) => {
- if (isOptional && (paramVal === undefined || paramVal === null)) return;
-
- const hasCorrectPrototype = prototypes instanceof Array
- ? _.some(prototypes, prototype => paramVal instanceof prototype)
- : paramVal instanceof prototypes;
-
- if (!hasCorrectPrototype) throw new Error(`Parameter "${paramName}" must be an instance of ${prototypeName}`);
-};
-
/**
* Returns true if node is in production environment and false otherwise.
*
diff --git a/test/actor.js b/test/actor.js
index c8bede8ad..b533e80aa 100644
--- a/test/actor.js
+++ b/test/actor.js
@@ -732,4 +732,25 @@ describe('Apify.getApifyProxyUrl()', () => {
port: 345,
})).to.be.eql('http://auto:[email protected]:345');
});
+
+ it('should throw on invalid proxy args', () => {
+ process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';
+ process.env[ENV_VARS.PROXY_HOSTNAME] = 'my.host.com';
+ process.env[ENV_VARS.PROXY_PORT] = 123;
+
+ expect(() => Apify.getApifyProxyUrl({ session: 'a.b' })).to.throw();
+ expect(() => Apify.getApifyProxyUrl({ session: 'a-b' })).to.throw();
+ expect(() => Apify.getApifyProxyUrl({ session: 'ab_' })).to.throw();
+ expect(() => Apify.getApifyProxyUrl({ session: 'a$b' })).to.throw();
+
+ expect(() => Apify.getApifyProxyUrl({ session: 'a_b' })).to.not.throw();
+ expect(() => Apify.getApifyProxyUrl({ session: 'a_1_b' })).to.not.throw();
+ expect(() => Apify.getApifyProxyUrl({ session: 'a_2' })).to.not.throw();
+ expect(() => Apify.getApifyProxyUrl({ session: 'a' })).to.not.throw();
+ expect(() => Apify.getApifyProxyUrl({ session: '1' })).to.not.throw();
+
+ delete process.env[ENV_VARS.PROXY_PASSWORD];
+ delete process.env[ENV_VARS.PROXY_HOSTNAME];
+ delete process.env[ENV_VARS.PROXY_PORT];
+ });
});
diff --git a/test/puppeteer.js b/test/puppeteer.js
index 19cb3436d..7be25fb8e 100644
--- a/test/puppeteer.js
+++ b/test/puppeteer.js
@@ -7,12 +7,11 @@ import Promise from 'bluebird';
import _ from 'underscore';
import sinon from 'sinon';
import Apify from '../build/index';
+import * as actor from '../build/actor';
import { ENV_VARS } from '../build/constants';
import * as utils from '../build/utils';
-
let prevEnvHeadless;
-
let proxyServer;
let proxyPort; // eslint-disable-line no-unused-vars
const proxyAuth = { scheme: 'Basic', username: 'username', password: 'password' };
@@ -76,6 +75,7 @@ describe('Apify.launchPuppeteer()', () => {
expect(() => Apify.launchPuppeteer({ proxyUrl: 'invalid://somehost:1234' })).to.throw(Error);
expect(() => Apify.launchPuppeteer({ proxyUrl: 'https://somehost:1234' })).to.throw(Error);
expect(() => Apify.launchPuppeteer({ proxyUrl: ' something really bad' })).to.throw(Error);
+ expect(() => Apify.launchPuppeteer({ proxyUrl: 'xxx', useApifyProxy: true })).to.throw(Error);
expect(() => Apify.launchPuppeteer({ args: 'wrong args' })).to.throw(Error);
expect(() => Apify.launchPuppeteer({ args: [12, 34] })).to.throw(Error);
@@ -188,4 +188,46 @@ describe('Apify.launchPuppeteer()', () => {
mock.restore();
});
});
+
+ it('should allow to use Apify proxy', () => {
+ process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';
+ process.env[ENV_VARS.PROXY_HOSTNAME] = 'my.host.com';
+ process.env[ENV_VARS.PROXY_PORT] = 123;
+
+ const mock = sinon.mock(actor);
+ mock.expects('getApifyProxyUrl')
+ .once()
+ .withArgs({
+ session: 'xxx',
+ groups: ['yyy'],
+ })
+ .returns(null); // Return null so that it doesn't start proxy-chain
+
+ return Apify
+ .launchPuppeteer({
+ useApifyProxy: true,
+ apifyProxySession: 'xxx',
+ apifyProxyGroups: ['yyy'],
+ headless: true,
+ })
+ .then(browser => browser.close())
+ .finally(() => {
+ mock.verify();
+ mock.restore();
+ delete process.env[ENV_VARS.PROXY_PASSWORD];
+ delete process.env[ENV_VARS.PROXY_HOSTNAME];
+ delete process.env[ENV_VARS.PROXY_PORT];
+ });
+ });
+
+ it('should throw when useApifyProxy=true and proxy password is not set', () => {
+ const opts = {
+ useApifyProxy: true,
+ apifyProxySession: 'xxx',
+ apifyProxyGroups: ['yyy'],
+ headless: true,
+ };
+
+ expect(() => Apify.launchPuppeteer(opts)).to.throw(Error);
+ });
});
diff --git a/test/puppeteer_crawler.js b/test/puppeteer_crawler.js
index 81eeddecb..3f53986ca 100644
--- a/test/puppeteer_crawler.js
+++ b/test/puppeteer_crawler.js
@@ -45,7 +45,6 @@ describe('PuppeteerCrawler', () => {
maxConcurrency: 1,
handlePageFunction,
handleFailedRequestFunction: ({ request }) => failed.push(request),
- disableProxy: true,
});
await requestList.initialize();
@@ -79,7 +78,6 @@ describe('PuppeteerCrawler', () => {
failed.push(request);
errors.push(error);
},
- disableProxy: true,
pageOpsTimeoutMillis: 900,
});
diff --git a/test/request_queue.js b/test/request_queue.js
index 8b25f6cda..c1fc09cf5 100644
--- a/test/request_queue.js
+++ b/test/request_queue.js
@@ -24,7 +24,7 @@ describe('RequestQueue', () => {
describe('local', async () => {
it('should work', async () => {
- const queue = new RequestQueueLocal('my-queue', LOCAL_EMULATION_DIR);
+ const queue = new RequestQueueLocal('my-queue-0', LOCAL_EMULATION_DIR);
await queue.addRequest(new Apify.Request({ url: 'http://example.com/first' }));
await queue.addRequest(new Apify.Request({ url: 'http://example.com/middle' }));
@@ -67,7 +67,7 @@ describe('RequestQueue', () => {
});
it('supports forefront param in reclaimRequest()', async () => {
- const queue = new RequestQueueLocal('my-queue', LOCAL_EMULATION_DIR);
+ const queue = new RequestQueueLocal('my-queue-1', LOCAL_EMULATION_DIR);
await queue.addRequest(new Apify.Request({ url: 'http://example.com/first' }));
await queue.addRequest(new Apify.Request({ url: 'http://example.com/middle' }));
diff --git a/test/utils.js b/test/utils.js
index 74dfe5bce..5e368d03d 100644
--- a/test/utils.js
+++ b/test/utils.js
@@ -182,22 +182,6 @@ describe('utils.isPromise()', () => {
});
});
-describe('utils.checkParamPrototypeOrThrow()', () => {
- it('works', () => {
- // One prototype
- expect(() => utils.checkParamPrototypeOrThrow(new Date(), 'param', Date, 'Date')).to.not.throw();
- expect(() => utils.checkParamPrototypeOrThrow(null, 'param', Function, 'Date', true)).to.not.throw();
- expect(() => utils.checkParamPrototypeOrThrow(undefined, 'param', Function, 'Date', true)).to.not.throw();
- expect(() => utils.checkParamPrototypeOrThrow(new Date(), 'param', Function, 'Date')).to.throw();
-
- // Multiple prototypes
- expect(() => utils.checkParamPrototypeOrThrow(new Date(), 'param', [Date, Function], 'Date')).to.not.throw();
- expect(() => utils.checkParamPrototypeOrThrow(new Date(), 'param', [Function, Date], 'Date')).to.not.throw();
- expect(() => utils.checkParamPrototypeOrThrow(new Date(), 'param', [Function, String], 'Date')).to.throw();
- expect(() => utils.checkParamPrototypeOrThrow(new Date(), 'param', [], 'Date')).to.throw();
- });
-});
-
describe('utils.newPromise()', () => {
it('works', () => {
if (!utils.isPromise(utils.newPromise())) throw new Error('utils.newPromise() must return a promise!');