Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Development of better proxy chains

Jan Curnb5a64c513448
diff --git a/.gitignore b/.gitignore
index 5fa5425a1..c54aa7c5c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -10,4 +10,5 @@ logs
pids
.idea
yarn.lock
+package-lock.json
docs
\ No newline at end of file
diff --git a/package.json b/package.json
index fc44e5941..87b647780 100644
--- a/package.json
+++ b/package.json
@@ -44,12 +44,13 @@
"apify-client": ">=0.1.13",
"bluebird": "^3.5.0",
"content-type": "^1.0.3",
- "portastic": "^1.0.1",
+ "proxy-chain": "^0.1.2",
"rimraf": "^2.6.1",
"tmp": "^0.0.33",
"underscore": "^1.8.3"
},
"devDependencies": {
+ "apify-jsdoc-template": "github:Apifier/apify-jsdoc-template",
"babel-cli": "^6.6.5",
"babel-core": "^6.1.21",
"babel-eslint": "^7.2.2",
@@ -65,14 +66,14 @@
"eslint-plugin-promise": "^3.4.2",
"eslint-plugin-react": "^7.0.1",
"isparta": "^4.0.0",
+ "jsdoc": "^3.4.3",
+ "jsdoc-export-default-interop": "^0.3.1",
"mocha": "^3.2.0",
+ "portastic": "^1.0.1",
"proxy": "^0.2.4",
- "sinon": "^2.2.0",
- "sinon-stub-promise": "^4.0.0",
"request": "^2.81.0",
- "jsdoc": "^3.4.3",
- "jsdoc-export-default-interop": "^0.3.1",
- "apify-jsdoc-template": "github:Apifier/apify-jsdoc-template"
+ "sinon": "^2.2.0",
+ "sinon-stub-promise": "^4.0.0"
},
"optionalDependencies": {
"puppeteer": "^0.12.0",
diff --git a/src/browser.js b/src/browser.js
index bb17e31a1..c3888fef6 100644
--- a/src/browser.js
+++ b/src/browser.js
@@ -1,7 +1,6 @@
// import { ChromeLauncher } from 'lighthouse/lighthouse-cli/chrome-launcher';
import { ENV_VARS } from './constants';
import { newPromise, nodeifyPromise, parseUrl } from './utils';
-import { ProxyChain } from './proxy_chain';
/* global process, require */
diff --git a/src/proxy_chain.js b/src/proxy_chain.js
deleted file mode 100644
index f99d30239..000000000
--- a/src/proxy_chain.js
+++ /dev/null
@@ -1,284 +0,0 @@
-import childProcess from 'child_process';
-import fs from 'fs';
-import path from 'path';
-import tmp from 'tmp';
-import portastic from 'portastic';
-import Promise from 'bluebird';
-import rimraf from 'rimraf';
-import { parseUrl, redactParsedUrl } from './utils';
-
-/* globals process */
-
-// Constants, exported to simplify unit-testing.
-export const PROXY_CHAIN = {
- PORT_FROM: 55000,
- PORT_TO: 65000,
- HOST: '127.0.0.1',
- PROTOCOL: 'http:',
- SQUID_CMD: 'squid',
- SQUID_CHECK_ARGS: ['-v'],
- SQUID_BATCH_TIMEOUT: 10000,
- TMP_DIR_TEMPLATE: '/tmp/apify-squid-XXXXXX',
- CONF_FILE_NAME: 'squid.conf',
- PID_FILE_NAME: 'squid.pid',
-};
-
-const tmpDirPromised = Promise.promisify(tmp.dir);
-const fsWriteFilePromised = Promise.promisify(fs.writeFile);
-// const execFilePromised = Promise.promisify(childProcess.execFile, { multiArgs: true });
-
-/*
- * Run Squid process to get its version and wait for the finish
- * @returns Promise
- const printSquidVersion = () => {
- // TODO: check that the version is at least 3.3 and throw error otherwise!
- console.log(`${PROXY_CHAIN.LOG_PREFIX}Checking Squid installation with '${PROXY_CHAIN.SQUID_CMD}
- ${PROXY_CHAIN.SQUID_CHECK_ARGS.join(' ')}'`); // eslint-disable-line max-len
- const options = {
- timeout: PROXY_CHAIN.SQUID_BATCH_TIMEOUT,
- };
- return execFilePromised(PROXY_CHAIN.SQUID_CMD, PROXY_CHAIN.SQUID_CHECK_ARGS, options)
- .then((array) => {
- const stdout = array[0];
- console.log(`${PROXY_CHAIN.LOG_PREFIX}${(stdout || '').split('\n')[0]}`);
- this.isInitialized = true;
- })
- .catch((err) => {
- // Give a user-friendly message for this common error
- if (err.code === 'ENOENT') {
- err = new Error(`'${PROXY_CHAIN.SQUID_CMD}' command not found in the PATH`);
- }
- throw err;
- });
- };
-*/
-
-
-/**
- * The class is used to manage a local Squid proxy instance
- * that forwards HTTP requests to a user-provided parent proxy.
- * The point of this is that there's no easy way to programmatically pass proxy
- * authentication credentials to web browsers such as Chrome, so we need to setup
- * a local open proxy that can forward to parent proxy with authentication.
- */
-export class ProxyChain {
- /**
- * Creates a new instance of ProxyChain class.
- * @param parsedProxyUrl Proxy URL parsed by calling parseUrl()
- */
- constructor(parsedProxyUrl) {
- if (!parsedProxyUrl.hostname || !parsedProxyUrl.port) throw new Error('Proxy URL must contain both hostname and port');
- if (parsedProxyUrl.scheme !== 'http') throw new Error('Only "http" proxy protocol is currently supported');
-
- this.parsedProxyUrl = parsedProxyUrl;
-
- // PID of the Squid proxy process
- this.squidPid = null;
-
- // TCP port where Squid is listening
- this.squidPort = null;
-
- // Path to temporary directory with config files
- this.tmpDir = null;
- }
-
- _getChildProxyUrl() {
- return `${PROXY_CHAIN.PROTOCOL}//${PROXY_CHAIN.HOST}:${this.squidPort}`;
- }
-
- _getParentProxyUrl() {
- return redactParsedUrl(this.parsedProxyUrl);
- }
-
- /**
- * Creates the temporary directory, writes a config to it and starts Squid process.
- * @returns Promise
- * @private
- */
- start() {
- // Create temporary directory, if not created yet
- return tmpDirPromised({ template: PROXY_CHAIN.TMP_DIR_TEMPLATE })
- .then((tmpDir) => {
- this.tmpDir = tmpDir;
-
- return ProxyChain._findFreePort();
- })
- .then((port) => {
- this.squidPort = port;
-
- // Write config file
- return this._writeSquidConf();
- })
- .then((squidConfPath) => {
- // Start Squid process
- const cmd = PROXY_CHAIN.SQUID_CMD;
- const args = ['-f', squidConfPath, '-N'];
- const options = { cwd: this.tmpDir };
- console.log(`Starting proxy chain: ${cmd} ${args.join(' ')}`);
- const proc = childProcess.spawn(cmd, args, options);
-
- this.squidPid = proc.pid;
-
- // Wait for Squid process to be running or fail
- return new Promise((resolve, reject) => {
- let isFinished = false;
- const intervalId = setInterval(() => {
- if (this._isSquidRunning()) {
- clearInterval(intervalId);
- isFinished = true;
- resolve();
- }
- // console.log("NOT RUNNING");
- }, 50);
-
- proc.on('exit', (code, signal) => {
- if (isFinished) {
- console.log(`Squid process ${this.squidPid} exited (code: ${code}, signal: ${signal}, temp dir: ${this.tmpDir})`);
- return;
- }
-
- clearInterval(intervalId);
- isFinished = true;
- const msg = `Squid process ${this.squidPid} exited unexpectedly (code: ${code}, signal: ${signal}, temp dir: ${this.tmpDir})`;
- reject(new Error(msg));
- });
-
- proc.on('error', (err) => {
- if (isFinished) {
- console.log(`Squid process ${this.squidPid} failed: ${err}`);
- return;
- }
-
- // Give a user-friendly message for this common error
- if (err.code === 'ENOENT') {
- err = new Error(`'${PROXY_CHAIN.SQUID_CMD}' command not found in the PATH`);
- }
-
- clearInterval(intervalId);
- isFinished = true;
- reject(err);
- });
-
- // Print stdout/stderr to simplify debugging
- const printLog = (data) => {
- console.log(`Squid process ${this.squidPid} says: ${data}`);
- };
- proc.stdout.on('data', printLog);
- proc.stderr.on('data', printLog);
- });
- })
- .then(() => {
- const childProxyUrl = this._getChildProxyUrl();
- console.log(`Started proxy chain ${childProxyUrl} => ${this._getParentProxyUrl()} (Squid pid: ${this.squidPid}, temp dir: ${this.tmpDir})`); // eslint-disable-line max-len
-
- return parseUrl(childProxyUrl);
- });
- }
-
- static _findFreePort() {
- // Let 'min' be a random value in the first half of the PORT_FROM-PORT_TO range,
- // to reduce a chance of collision if other ProxyChain is started at the same time.
- const half = Math.floor((PROXY_CHAIN.PORT_TO - PROXY_CHAIN.PORT_FROM) / 2);
-
- const opts = {
- min: PROXY_CHAIN.PORT_FROM + Math.floor(Math.random() * half),
- max: PROXY_CHAIN.PORT_TO,
- retrieve: 1,
- };
-
- return portastic.find(opts)
- .then((ports) => {
- if (ports.length < 1) throw new Error(`There are no more free ports in range from ${PROXY_CHAIN.PORT_FROM} to ${PROXY_CHAIN.PORT_TO}`); // eslint-disable-line max-len
- return ports[0];
- });
- }
-
- /**
- * Determines whether the Squid process is still running.
- * @returns Boolean
- * @private
- */
- _isSquidRunning() {
- // Inspired by https://github.com/nisaacson/is-running/blob/master/index.js
- if (!this.squidPid) return false;
- try {
- return !!process.kill(this.squidPid, 0);
- } catch (e) {
- return e.code === 'EPERM';
- }
- }
-
- /**
- * Generates Squid proxy configuration file and writes it to the temporary directory.
- * @returns Promise Promise resolving to the path to the configuration file.
- * @private
- */
- _writeSquidConf() {
- if (!this.tmpDir) throw new Error('Temporary directory not created yet');
-
- const proxy = this.parsedProxyUrl;
-
- // NOTE: need to set pid_filename to isolate our squid instance from others possibly running in the system
- const conf = `
-visible_hostname apify-actor
-# debug_options 44,9 28,9
-http_access allow all
-never_direct allow all
-# this probably requires --enable-http-violations compile option
-via off
-forwarded_for transparent
-access_log none
-#access_log daemon:${path.join(this.tmpDir, 'access.log')} squid
-# An attempt to reduce memory consumption (proxy shouldn't store anything)
-cache deny all
-memory_pools off
-cache_mem 16 MB
-cache_store_log none
-cache_log /dev/null
-#cache_log ${path.join(this.tmpDir, 'cache.log')}
-logfile_rotate 0
-pid_filename ${path.join(this.tmpDir, PROXY_CHAIN.PID_FILE_NAME)}
-
-http_port ${this.squidPort}
-cache_peer ${proxy.hostname} parent ${proxy.port} 0 no-query login=${proxy.auth} connect-fail-limit=99999999 proxy-only name=my_peer
-cache_peer_access my_peer allow all
-`;
- const filePath = path.join(this.tmpDir, PROXY_CHAIN.CONF_FILE_NAME);
-
- // console.log('CONFIG');
- // console.log(conf);
-
- return fsWriteFilePromised(filePath, conf)
- .then(() => {
- return filePath;
- });
- }
-
- /**
- * Removes all proxy chains and terminates the squid process.
- * @returns {*}
- */
- shutdown() {
- const isRunning = this._isSquidRunning();
-
- console.log(`Shutting down proxy chain ${this._getChildProxyUrl()} => ${this._getParentProxyUrl()} (Squid pid: ${this.squidPid}, running: ${isRunning})`); // eslint-disable-line max-len
-
- if (isRunning) {
- try {
- process.kill(this.squidPid, 'SIGKILL');
- } catch (err) {
- console.log(`WARNING: Failed to kill Squid process ${this.squidPid}: ${err}`);
- }
- }
-
- if (this.tmpDir) {
- const tmpDir = this.tmpDir;
- rimraf(tmpDir, { glob: false }, (err) => {
- if (err) {
- console.log(`WARNING: Failed to delete temporary directory at ${tmpDir}: ${err}`);
- }
- });
- this.tmpDir = null;
- }
- }
-}
diff --git a/src/puppeteer.js b/src/puppeteer.js
index 0b71f701b..9d7a4861c 100644
--- a/src/puppeteer.js
+++ b/src/puppeteer.js
@@ -1,5 +1,6 @@
import { checkParamOrThrow } from 'apify-client/build/utils';
-import { parseUrl } from './utils';
+import { anonymizeProxy, closeAnonymizedProxy } from 'proxy-chain';
+import { newPromise } from './utils';
import { ENV_VARS } from './constants';
/* global process, require */
@@ -15,50 +16,71 @@ import { ENV_VARS } from './constants';
* <ul>
* <li>Passes the value of the `APIFY_HEADLESS` environment variable to the `headless` option,
* unless it was already defined by the caller.</li>
- * <li>Takes the `proxyUrl` option and adds it to `env` under the `HTTPS_PROXY` or `HTTP_PROXY` key.</li>
+ * <li>Takes the `proxyUrl` option, checks it and adds it to the `--proxy-server=XXX` command-line option `args`.
+ * If the proxy uses authentication, the function sets up an anonymous proxy HTTP that will work with headless Chrome.
+ * </li>
* <li>Adds `--no-sandbox` to `args` to enable running headless Chrome in a Docker container on the Actor platform.</li>
* </ul>
* </p>
- * @param options Optional settings passed to `puppeteer.launch()`.
+ * @param [opts] Optional settings passed to `puppeteer.launch()`.
* Additionally, the options can contain the `proxyUrl` property to specify a proxy server
- * (e.g. `http://example.com:1234`).
- * @returns {Promise} Promise object returned by `puppeteer.launch()`
+ * (e.g. `http://username:[email protected]:1234`).
+ * @returns {Promise} Promise object that resolves to Puppeteer's `Browser` instance.
*/
-export const launchPuppeteer = (opts = {}) => {
- let puppeteer;
+export const launchPuppeteer = (opts) => {
+ if (opts === undefined || opts === null) opts = {};
+
+ checkParamOrThrow(opts, 'opts', 'Object');
+ checkParamOrThrow(opts.args, 'opts.args', 'Maybe [String]');
+ let puppeteer;
try {
// This is an optional dependency because it is quite large, only require it when used (ie. image with Chrome)
puppeteer = require('puppeteer'); // eslint-disable-line global-require
} catch (err) {
- if (err.code === 'MODULE_NOT_FOUND') err.message = 'Cannot find module \'puppeteer\'. Did you choose the wrong docker image?';
+ if (err.code === 'MODULE_NOT_FOUND') err.message = 'Cannot find module \'puppeteer\'. Did you choose the correct base Docker image?';
throw err;
}
- checkParamOrThrow(opts, 'opts', 'Object');
- checkParamOrThrow(opts.env, 'opts.env', 'Maybe Object');
- checkParamOrThrow(opts.args, 'opts.args', 'Maybe [String]');
-
- const { proxyUrl } = opts;
+ opts.args = opts.args || [];
+ opts.args.push('--no-sandbox');
+ if (opts.headless === undefined || opts.headless === null) {
+ opts.headless = !!process.env[ENV_VARS.HEADLESS];
+ }
- if (proxyUrl) {
- checkParamOrThrow(proxyUrl, 'opts.proxyUrl', 'String');
+ let anonymizedProxyUrl;
+ let promise;
- const { host, port, protocol, password } = parseUrl(proxyUrl);
- if (!host || !port) throw new Error('Invalid "proxyUrl" option: the URL must contain hostname and port number.');
- if (protocol !== 'http:' && protocol !== 'https:') throw new Error('Invalid "proxyUrl" option: protocol must be http or https.');
- if (password) throw new Error('Invalid "proxyUrl" option: password is not currently supported.');
+ // Parse proxy URL and if it contains a password then setup ProxyChain server
+ if (opts.proxyUrl) {
+ checkParamOrThrow(opts.proxyUrl, 'opts.proxyUrl', 'String');
- const proxyVarName = proxyUrl.startsWith('https') ? 'HTTPS_PROXY' : 'HTTP_PROXY';
- opts.env = opts.env || {};
- opts.env[proxyVarName] = opts.proxyUrl;
+ // NOTE: anonymizeProxy() throws on invalid proxyUrl, so it must not be in a Promise!
+ promise = anonymizeProxy(opts.proxyUrl)
+ .then((result) => {
+ anonymizedProxyUrl = result;
+ opts.args.push(`--proxy-server=${anonymizedProxyUrl}`);
+ })
+ .then(() => puppeteer.launch(opts));
+ } else {
+ promise = puppeteer.launch(opts);
}
- opts.args = opts.args || [];
- opts.args.push('--no-sandbox');
+ // --user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36
- if (opts.headless === undefined || opts.headless === null) opts.headless = !!process.env[ENV_VARS.HEADLESS];
+ // Close ProxyChain server when Puppeteer finishes
+ if (anonymizedProxyUrl) {
+ promise = promise.then((browser) => {
+ browser.on('disconnected', () => {
+ console.log('Puppeteer disconnected');
+ // Don't wait for finish
+ closeAnonymizedProxy(anonymizedProxyUrl, true);
+ });
+ return browser;
+ });
+ }
- return puppeteer.launch(opts);
+ // Ensure that the returned promise is of type set in setPromiseDependency()
+ return newPromise().then(() => promise);
};
diff --git a/src/utils.js b/src/utils.js
index 0bf770585..61359a790 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -1,9 +1,10 @@
import urlModule from 'url';
+import Promise from 'bluebird';
import contentTypeParser from 'content-type';
import ApifyClient from 'apify-client';
import { ENV_VARS } from './constants';
-let PromisesDependency = typeof Promise === 'function' ? Promise : null;
+let PromisesDependency = Promise;
/* global process */
@@ -22,6 +23,7 @@ let PromisesDependency = typeof Promise === 'function' ? Promise : null;
* &nbsp;
* Apify.setPromisesDependency(Promise);
* </code></pre>
+ * By default, the package uses the `bluebird` promises.
* @param [Constructor] dep Reference to a Promise constructor
*/
export const setPromisesDependency = (dep) => {
@@ -33,6 +35,7 @@ export const setPromisesDependency = (dep) => {
* @memberof module:Apify
* @function
* @description Gets the promise dependency set by <a href="#module-Apify-setPromisesDependency"><code>Apify.setPromisesDependency</code></a>.
+ * By default, the package uses the `bluebird` promises.
* @returns {Constructor} Reference to a Promise constructor
*/
export const getPromisesDependency = () => {
diff --git a/test/proxy_chain.js b/test/proxy_chain.js
deleted file mode 100644
index 0daf1ed2a..000000000
--- a/test/proxy_chain.js
+++ /dev/null
@@ -1,268 +0,0 @@
-import fs from 'fs';
-import _ from 'underscore';
-import { expect, assert } from 'chai';
-import proxy from 'proxy';
-import http from 'http';
-import portastic from 'portastic';
-import basicAuthParser from 'basic-auth-parser';
-import Promise from 'bluebird';
-import request from 'request';
-
-import { parseUrl } from '../build/utils';
-import { ProxyChain, PROXY_CHAIN } from '../build/proxy_chain';
-
-/* globals process */
-
-const ORIG_PROXY_CHAIN = _.clone(PROXY_CHAIN);
-
-let proxyServer;
-let proxyPort; // eslint-disable-line no-unused-vars
-const proxyAuth = { scheme: 'Basic', username: 'username', password: 'password' };
-let wasProxyCalled = false; // eslint-disable-line no-unused-vars
-
-// Setup local proxy server for the tests
-before(() => {
- // Find free port for the proxy
- return portastic.find({ min: 50000, max: 50100 }).then((ports) => {
- return new Promise((resolve, reject) => {
- const httpServer = http.createServer();
-
- // Setup proxy authorization
- httpServer.authenticate = function (req, fn) {
- // parse the "Proxy-Authorization" header
- const auth = req.headers['proxy-authorization'];
- if (!auth) {
- // optimization: don't invoke the child process if no
- // "Proxy-Authorization" header was given
- // console.log('not Proxy-Authorization');
- return fn(null, false);
- }
- const parsed = basicAuthParser(auth);
- const isEqual = _.isEqual(parsed, proxyAuth);
- // console.log('Parsed "Proxy-Authorization": parsed: %j expected: %j : %s', parsed, proxyAuth, isEqual);
- if (isEqual) wasProxyCalled = true;
- fn(null, isEqual);
- };
-
- httpServer.on('error', reject);
-
- proxyServer = proxy(httpServer);
- proxyServer.listen(ports[0], () => {
- proxyPort = proxyServer.address().port;
- resolve();
- });
- });
- });
-});
-
-after(function () {
- this.timeout(60 * 1000);
- if (proxyServer) return Promise.promisify(proxyServer.close).bind(proxyServer)();
-});
-
-
-const requestPromised = (opts) => {
- // console.log('requestPromised');
- // console.dir(opts);
- return new Promise((resolve, reject) => {
- request(opts, (error, response, body) => {
- if (error) return reject(error);
- if (response.statusCode !== 200) {
- console.log('ERROR VOLE');
- // console.dir(response);
- console.dir(body);
-
- return reject(new Error(`Received invalid response code: ${response.statusCode}`));
- }
- if (opts.expectBodyContainsText) expect(body).to.contain(opts.expectBodyContainsText);
- resolve();
- });
- });
-};
-
-
-describe('ProxyChain', function () {
- // Need larger timeout for Travis CI
- this.timeout(1000 * 1000);
-
- it('throws nice error when "squid" command not found', () => {
- PROXY_CHAIN.SQUID_CMD = 'command-that-does-not-exist';
- const pc = new ProxyChain(parseUrl('http://whatever.com:1234'));
- return pc.start()
- .then(() => {
- assert.fail();
- })
- .catch((err) => {
- expect(err.message).to.contain('command not found in the PATH');
- })
- .finally(() => {
- Object.assign(PROXY_CHAIN, ORIG_PROXY_CHAIN);
- });
- });
-
- it('throws an error when "squid" command exits with non-zero code', () => {
- PROXY_CHAIN.SQUID_CMD = 'false';
- const pc = new ProxyChain(parseUrl('http://whatever.com:1234'));
- return pc.start()
- .then(() => {
- assert.fail();
- })
- .catch((err) => {
- expect(err.message).to.contain('exited unexpectedly');
- })
- .finally(() => {
- Object.assign(PROXY_CHAIN, ORIG_PROXY_CHAIN);
- });
- });
-
- it('throws nice error when no more free ports available', () => {
- // Testing proxy is already listening on proxyPort
- PROXY_CHAIN.PORT_FROM = proxyPort;
- PROXY_CHAIN.PORT_TO = proxyPort;
- const pc = new ProxyChain(parseUrl('http://whatever.com:1234'));
- return pc.start()
- .then(() => {
- assert.fail();
- })
- .catch((err) => {
- expect(err.message).to.contain('There are no more free ports');
- })
- .finally(() => {
- Object.assign(PROXY_CHAIN, ORIG_PROXY_CHAIN);
- });
- });
-
- it('throws for invalid args', () => {
- assert.throws(() => { new ProxyChain(null); }, Error); // eslint-disable-line no-new
- });
-
- it('throws for unsupported proxy protocols', () => {
- assert.throws(() => { new ProxyChain(parseUrl('socks://whatever.com')); }, Error); // eslint-disable-line no-new
- assert.throws(() => { new ProxyChain(parseUrl('https://whatever.com')); }, Error); // eslint-disable-line no-new
- assert.throws(() => { new ProxyChain(parseUrl('socks5://whatever.com')); }, Error); // eslint-disable-line no-new
- });
-
- it('throws for invalid URLs', () => {
- assert.throws(() => { new ProxyChain(parseUrl('://whatever.com')); }, Error); // eslint-disable-line no-new
- assert.throws(() => { new ProxyChain(parseUrl('https://whatever.com')); }, Error); // eslint-disable-line no-new
- assert.throws(() => { new ProxyChain(parseUrl('socks5://whatever.com')); }, Error); // eslint-disable-line no-new
- assert.throws(() => { new ProxyChain(parseUrl('http://no-port-provided')); }, Error); // eslint-disable-line no-new
- });
-
- it('works well', () => {
- // TODO: test maybe 5 proxies at the same time
- const proxyChain1 = new ProxyChain(parseUrl(`http://${proxyAuth.username}:${proxyAuth.password}@127.0.0.1:${proxyPort}`));
- const proxyChain2 = new ProxyChain(parseUrl(`http://${proxyAuth.username}:${proxyAuth.password}@127.0.0.1:${proxyPort}`));
- let parsedChildProxyUrl1;
- let parsedChildProxyUrl2;
- return Promise.resolve()
- .then(() => {
- return Promise.all([proxyChain1.start(), proxyChain2.start()]);
- })
- .then((results) => {
- parsedChildProxyUrl1 = results[0];
- parsedChildProxyUrl2 = results[1];
- expect(parsedChildProxyUrl1.port).to.not.equal(proxyPort);
- expect(parsedChildProxyUrl2.port).to.not.equal(proxyPort);
- expect(parsedChildProxyUrl1.port).to.not.equal(parsedChildProxyUrl2.port);
-
- // Test call through proxy 1
- wasProxyCalled = false;
- return requestPromised({
- uri: 'https://www.example.com', // Test HTTPS
- proxy: `${parsedChildProxyUrl1.protocol}//${parsedChildProxyUrl1.host}`,
- expectBodyContainsText: 'Example Domain',
- });
- })
- .then(() => {
- expect(wasProxyCalled).to.equal(true);
- })
- .then(() => {
- // Test call through proxy 2
- wasProxyCalled = false;
- return requestPromised({
- uri: 'http://www.example.com', // Test HTTP
- proxy: `${parsedChildProxyUrl2.protocol}//${parsedChildProxyUrl2.host}`,
- expectBodyContainsText: 'Example Domain',
- });
- })
- .then(() => {
- expect(wasProxyCalled).to.equal(true);
- })
- .then(() => {
- // Test again call through proxy 1
- wasProxyCalled = false;
- return requestPromised({
- uri: 'http://www.example.com',
- proxy: `${parsedChildProxyUrl1.protocol}//${parsedChildProxyUrl1.host}`,
- expectBodyContainsText: 'Example Domain',
- });
- })
- .then(() => {
- expect(wasProxyCalled).to.equal(true);
- })
- .then(() => {
- proxyChain1.shutdown();
- proxyChain2.shutdown();
-
- // Test these can be called multiple times
- proxyChain1.shutdown();
- proxyChain2.shutdown();
- });
- });
-
- it('fails with invalid proxy credentials', () => {
- const proxyChain = new ProxyChain(parseUrl(`http://username:[email protected]:${proxyPort}`));
-
- return Promise.resolve()
- .then(() => {
- return proxyChain.start();
- })
- .then((parsedChildProxyUrl) => {
- expect(parsedChildProxyUrl.port).to.not.equal(proxyPort);
- wasProxyCalled = false;
- return requestPromised({
- uri: 'https://www.example.com',
- proxy: `${parsedChildProxyUrl.protocol}//${parsedChildProxyUrl.host}`,
- expectBodyContainsText: 'Example Domain',
- });
- })
- .then(() => {
- assert.fail();
- })
- .catch((err) => {
- expect(err.message).to.contains('tunneling socket could not be established');
- expect(wasProxyCalled).to.equal(false);
- })
- .then(() => {
- proxyChain.shutdown();
- });
- });
-
- it('cleans up properly after shutdown', () => {
- const proxyChain = new ProxyChain(parseUrl(`http://any:[email protected]:${proxyPort}`));
- let tmpDir;
-
- return Promise.resolve()
- .then(() => {
- return proxyChain.start();
- })
- .then(() => {
- tmpDir = proxyChain.tmpDir;
- proxyChain.shutdown();
- })
- .then(() => {
- return new Promise((resolve) => {
- setTimeout(resolve, 100);
- });
- })
- .then(() => {
- expect(fs.existsSync(tmpDir)).to.equal(false);
- expect(proxyChain._isSquidRunning()).to.equal(false);
- });
- });
-
- after(() => {
- Object.assign(PROXY_CHAIN, ORIG_PROXY_CHAIN);
- });
-});
diff --git a/test/puppeteer.js b/test/puppeteer.js
index 14cc9d8da..3cf1d440a 100644
--- a/test/puppeteer.js
+++ b/test/puppeteer.js
@@ -1,16 +1,76 @@
import { expect } from 'chai';
+import proxy from 'proxy';
+import http from 'http';
+import portastic from 'portastic';
+import basicAuthParser from 'basic-auth-parser';
+import Promise from 'bluebird';
+import _ from 'underscore';
import Apify from '../build/index';
import { ENV_VARS } from '../build/constants';
-process.env[ENV_VARS.HEADLESS] = true;
+let prevEnvHeadless;
+
+let proxyServer;
+let proxyPort; // eslint-disable-line no-unused-vars
+const proxyAuth = { scheme: 'Basic', username: 'username', password: 'password' };
+let wasProxyCalled = false; // eslint-disable-line no-unused-vars
+
+// Setup local proxy server for the tests
+before(() => {
+ prevEnvHeadless = process.env[ENV_VARS.HEADLESS];
+ process.env[ENV_VARS.HEADLESS] = true;
+
+ // Find free port for the proxy
+ return portastic.find({ min: 50000, max: 50100 }).then((ports) => {
+ return new Promise((resolve, reject) => {
+ const httpServer = http.createServer();
+
+ // Setup proxy authorization
+ httpServer.authenticate = function (req, fn) {
+ // parse the "Proxy-Authorization" header
+ const auth = req.headers['proxy-authorization'];
+ if (!auth) {
+ // optimization: don't invoke the child process if no
+ // "Proxy-Authorization" header was given
+ // console.log('not Proxy-Authorization');
+ return fn(null, false);
+ }
+ const parsed = basicAuthParser(auth);
+ const isEqual = _.isEqual(parsed, proxyAuth);
+ // console.log('Parsed "Proxy-Authorization": parsed: %j expected: %j : %s', parsed, proxyAuth, isEqual);
+ if (isEqual) wasProxyCalled = true;
+ fn(null, isEqual);
+ };
+
+ httpServer.on('error', reject);
+
+ proxyServer = proxy(httpServer);
+ proxyServer.listen(ports[0], () => {
+ proxyPort = proxyServer.address().port;
+ resolve();
+ });
+ });
+ });
+});
+
+after(function () {
+ process.env[ENV_VARS.HEADLESS] = prevEnvHeadless;
+
+ this.timeout(5 * 1000);
+ if (proxyServer) return Promise.promisify(proxyServer.close).bind(proxyServer)();
+});
+
describe('Apify.launchPuppeteer()', () => {
it('throws on invalid args', () => {
expect(() => Apify.launchPuppeteer({ proxyUrl: 'invalidurl' })).to.throw(Error);
expect(() => Apify.launchPuppeteer({ proxyUrl: 'http://host-without-port' })).to.throw(Error);
expect(() => Apify.launchPuppeteer({ proxyUrl: 'invalid://somehost:1234' })).to.throw(Error);
- expect(() => Apify.launchPuppeteer({ proxyUrl: 'http://username:[email protected]:30' })).to.throw(Error);
- expect(() => Apify.launchPuppeteer({ proxyUrl: 'http://host-with-port.com:30' })).to.not.throw();
+ expect(() => Apify.launchPuppeteer({ proxyUrl: 'https://somehost:1234' })).to.throw(Error);
+ expect(() => Apify.launchPuppeteer({ proxyUrl: ' something really bad' })).to.throw(Error);
+
+ expect(() => Apify.launchPuppeteer({ args: 'wrong args' })).to.throw(Error);
+ expect(() => Apify.launchPuppeteer({ args: [12, 34] })).to.throw(Error);
});
it('opens https://www.example.com', () => {
@@ -27,10 +87,41 @@ describe('Apify.launchPuppeteer()', () => {
.then((openedPage) => {
page = openedPage;
- return page.goto('https://example.com');
+ return page.goto('https://www.example.com');
})
.then(() => page.content())
.then(html => expect(html).to.include('<h1>Example Domain</h1>'))
.then(() => browser.close());
});
+
+ it('opens https://www.example.com via proxy with authentication', () => {
+ let browser;
+ let page;
+
+ // Test headless parameter
+ process.env[ENV_VARS.HEADLESS] = false;
+
+ return Apify
+ .launchPuppeteer({
+ headless: true,
+ proxyUrl: `http://username:[email protected]:${proxyPort}`,
+ })
+ .then((createdBrowser) => {
+ browser = createdBrowser;
+
+ return browser.newPage();
+ })
+ .then((openedPage) => {
+ page = openedPage;
+
+ return page.goto('https://example.com');
+ })
+ .then(() => {
+ expect(wasProxyCalled).to.eql(true);
+
+ return page.content();
+ })
+ .then(html => expect(html).to.include('<h1>Example Domain</h1>'))
+ .then(() => browser.close());
+ });
});