Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Replaced Squid-based proxy chain with proxy-chain package, launchPuppeteer() uses better default User-Agent, better unit tests, removed parseUrl/redactUrl/redactParsedUrl functions

Jan Curn3a4a4c13d8ab
diff --git a/src/puppeteer.js b/src/puppeteer.js
index 2eabce385..ecea42860 100644
--- a/src/puppeteer.js
+++ b/src/puppeteer.js
@@ -6,6 +6,9 @@ import { ENV_VARS } from './constants';
/* global process, require */
+const DEFAULT_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36'; // eslint-disable-line max-len
+
+
/**
* @memberof module:Apify
* @function
@@ -19,7 +22,8 @@ import { ENV_VARS } from './constants';
* unless it was already defined by the caller. Note that this environment variable is automatically set to `1`
* in acts running on the Apify Actor cloud platform.</li>
* <li>Takes the `proxyUrl` option, checks it and adds it to `args` as `--proxy-server=XXX`.
- * If the proxy uses authentication, the function sets up an anonymous proxy HTTP that will work with headless Chrome.
+ * If the proxy uses authentication, the function sets up an anonymous proxy HTTP
+ * to make the proxy work with headless Chrome.
* </li>
* <li>Adds `--no-sandbox` to `args` to enable running headless Chrome in a Docker container on the Actor platform.</li>
* </ul>
@@ -36,6 +40,7 @@ import { ENV_VARS } from './constants';
* @param {String} [opts.proxyUrl] - URL to a proxy server. Currently only `http://` scheme is supported.
* Port number must be specified. Proxy authentication is also supported. For example, `http://bob:[email protected]:1234`.
* @param {String} [opts.userAgent] - Default User-Agent for the browser.
+ * If not provided, the function sets it to a reasonable default.
* @returns {Promise} Promise object that resolves to Puppeteer's `Browser` instance.
*/
export const launchPuppeteer = (opts) => {
@@ -43,6 +48,7 @@ export const launchPuppeteer = (opts) => {
checkParamOrThrow(opts, 'opts', 'Object');
checkParamOrThrow(opts.args, 'opts.args', 'Maybe [String]');
+ checkParamOrThrow(opts.proxyUrl, 'opts.proxyUrl', 'Maybe String');
let puppeteer;
try {
@@ -56,19 +62,17 @@ export const launchPuppeteer = (opts) => {
opts.args = opts.args || [];
opts.args.push('--no-sandbox');
+ opts.args.push(`--user-agent=${opts.userAgent || DEFAULT_USER_AGENT}`);
if (opts.headless === undefined || opts.headless === null) {
opts.headless = process.env[ENV_VARS.HEADLESS] === '1';
}
- if (opts.userAgent) opts.args.push(`--user-agent=${opts.userAgent}`);
let anonymizedProxyUrl;
let promise;
- // Parse proxy URL and if it contains a password then setup ProxyChain server
+ // Parse and validate proxy URL and anonymize it
if (opts.proxyUrl) {
- checkParamOrThrow(opts.proxyUrl, 'opts.proxyUrl', 'String');
-
- // NOTE: anonymizeProxy() throws on invalid proxyUrl, so it must not be in a Promise!
+ // NOTE: anonymizeProxy() throws on invalid proxy URL, so it must not be inside a Promise!
promise = anonymizeProxy(opts.proxyUrl)
.then((result) => {
anonymizedProxyUrl = result;
@@ -79,16 +83,25 @@ export const launchPuppeteer = (opts) => {
promise = puppeteer.launch(opts);
}
- // --user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36
-
- // Close ProxyChain server when Puppeteer finishes
- if (anonymizedProxyUrl) {
+ // Close anonymization proxy server when Puppeteer finishes
+ if (opts.proxyUrl) {
promise = promise.then((browser) => {
- browser.on('disconnected', () => {
- console.log('Puppeteer disconnected');
- // Don't wait for finish
- closeAnonymizedProxy(anonymizedProxyUrl, true);
- });
+ const cleanUp = () => {
+ // Don't wait for finish, only log errors
+ closeAnonymizedProxy(anonymizedProxyUrl, true)
+ .catch((err) => {
+ console.log(`WARNING: closeAnonymizedProxy() failed with: ${err.stack || err}`);
+ });
+ };
+
+ browser.on('disconnected', cleanUp);
+
+ const prevClose = browser.close.bind(browser);
+ browser.close = () => {
+ cleanUp();
+ return prevClose();
+ };
+
return browser;
});
}
diff --git a/src/utils.js b/src/utils.js
index 61359a790..fb89f0b17 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -1,16 +1,17 @@
-import urlModule from 'url';
import Promise from 'bluebird';
import contentTypeParser from 'content-type';
import ApifyClient from 'apify-client';
import { ENV_VARS } from './constants';
-let PromisesDependency = Promise;
+// For backward compatibility, re-export functions that were moved to proxy-chain package
+// TODO: eventually get rid of this
+export { parseUrl, redactUrl, redactParsedUrl } from 'proxy-chain';
/* global process */
-// TODO: add methods to override console.log() and console.error(), add unit tests for that!
+let PromisesDependency = Promise;
-// TODO: use bluebird promise by default instead of native promise, rename newPromise() to newUserPromise(), get rid of nodeifyPromise()
+// TODO: add methods to override console.log() and console.error(), add unit tests for that!
/**
* @memberof module:Apify
@@ -101,64 +102,6 @@ export const newClient = () => {
};
-/**
- * Sames are Node's url.parse() just adds the 'username', 'password' and 'scheme' fields.
- * @param url
- * @ignore
- */
-export const parseUrl = (url) => {
- const parsed = urlModule.parse(url);
-
- parsed.username = null;
- parsed.password = null;
- parsed.scheme = null;
-
- if (parsed.auth) {
- const matches = /^([^:]+)(:?)(.*)$/.exec(parsed.auth);
- if (matches && matches.length === 4) {
- parsed.username = matches[1];
- if (matches[2] === ':') parsed.password = matches[3];
- }
- }
-
- if (parsed.protocol) {
- const matches = /^([a-z0-9]+):$/i.exec(parsed.protocol);
- if (matches && matches.length === 2) {
- parsed.scheme = matches[1];
- }
- }
-
- return parsed;
-};
-
-
-/**
- * Redacts password from a URL, so that it can be shown in logs, results etc.
- * For example, converts URL such as
- * 'https://username:[email protected]/path#hash'
- * to 'https://username:<redacted>@www.example.com/path#hash'
- * @param url URL, it must contain at least protocol and hostname
- * @param passwordReplacement The string that replaces password, by default it is '<redacted>'
- * @returns {string}
- * @ignore
- */
-export const redactUrl = (url, passwordReplacement) => {
- return redactParsedUrl(parseUrl(url), passwordReplacement);
-};
-
-export const redactParsedUrl = (parsedUrl, passwordReplacement = '<redacted>') => {
- const p = parsedUrl;
- let auth = null;
- if (p.username) {
- if (p.password) {
- auth = `${p.username}:${passwordReplacement}`;
- } else {
- auth = `${p.username}`;
- }
- }
- return `${p.protocol}//${auth || ''}${auth ? '@' : ''}${p.host}${p.path || ''}${p.hash || ''}`;
-};
-
/**
* Adds charset=utf-8 to given content type if this parameter is missing.
*
diff --git a/src/webdriver.js b/src/webdriver.js
index 8d2f89fc5..a1294386f 100644
--- a/src/webdriver.js
+++ b/src/webdriver.js
@@ -1,6 +1,7 @@
// import { ChromeLauncher } from 'lighthouse/lighthouse-cli/chrome-launcher';
+import { anonymizeProxy, closeAnonymizedProxy } from 'proxy-chain';
import { ENV_VARS } from './constants';
-import { newPromise, nodeifyPromise, parseUrl } from './utils';
+import { newPromise, nodeifyPromise } from './utils';
/* global process, require */
@@ -41,17 +42,12 @@ export class Browser {
constructor(options) {
this.options = Object.assign(getDefaultBrowseOptions(), options);
- if (this.options.proxyUrl) {
- const parsed = parseUrl(this.options.proxyUrl);
- if (!parsed.host || !parsed.port) throw new Error('Invalid "proxyUrl" option: the URL must contain hostname and port number.');
- if (parsed.scheme !== 'http') throw new Error('Invalid "proxyUrl" option: only HTTP proxy type is currently supported.');
- this.parsedProxyUrl = parsed;
- }
-
// This is an optional dependency because it is quite large, only require it when used
const { Capabilities, Builder } = require('selenium-webdriver'); // eslint-disable-line global-require
const chrome = require('selenium-webdriver/chrome'); // eslint-disable-line global-require
+ this.anonymizedProxyUrl = null;
+
// logging.installConsoleHandler();
// logging.getLogger('webdriver.http').setLevel(logging.Level.ALL);
@@ -80,9 +76,6 @@ export class Browser {
// Instance of Selenium's WebDriver
this.webDriver = null;
-
- // Instance of ProxyChain or null if not used
- this.proxyChain = null;
}
/**
@@ -90,9 +83,37 @@ export class Browser {
* @returns Promise
*/
_initialize() {
+ let promise = null;
+
+ // Applies options.proxyUrl setting to the WebDriver's Capabilities and Chrome Options.
+ // For proxy servers with authentication, this class starts a local proxy server
+ // NOTE: to view effective proxy settings in Chrome, open chrome://net-internals/#proxy
+ if (this.options.proxyUrl) {
+ // NOTE: call anonymizeProxy() outside of promise, so that errors in proxyUrl are thrown!
+ promise = anonymizeProxy(this.options.proxyUrl)
+ .then((result) => {
+ this.anonymizedProxyUrl = result;
+
+ if (/^chrome$/i.test(this.options.browserName)) {
+ // In Chrome, Capabilities.setProxy() has no effect,
+ // so we setup the proxy manually
+ this.chromeOptions.addArguments(`--proxy-server=${this.anonymizedProxyUrl}`);
+ } else {
+ const proxyConfig = {
+ proxyType: 'MANUAL',
+ httpProxy: this.anonymizedProxyUrl,
+ sslProxy: this.anonymizedProxyUrl,
+ ftpProxy: this.anonymizedProxyUrl,
+ };
+ this.capabilities.setProxy(proxyConfig);
+ }
+ });
+ }
+
+ // Ensure that the returned promise is of type set in setPromiseDependency()
return newPromise()
.then(() => {
- return this._setupProxy();
+ return promise;
})
.then(() => {
this.webDriver = this.builder
@@ -105,70 +126,18 @@ export class Browser {
});
}
- /**
- * Applies options.proxyUrl setting to the WebDriver's Capabilities and Chrome Options.
- * For proxy servers with authentication, this class starts a local
- * Privoxy process to proxy-chain to the target proxy server and enable browser
- * no to use authentication, because it's typically not supported.
- * @param capabilities
- * @param chromeOpts
- */
- _setupProxy() {
- if (!this.parsedProxyUrl) return null;
-
- // NOTE: to view effective proxy settings in Chrome, open chrome://net-internals/#proxy
- // https://sites.google.com/a/chromium.org/chromedriver/capabilities
- // https://github.com/haad/proxychains/blob/0f61bd071389398a4c8378847a45973577593e6f/src/proxychains.conf
- // https://www.rootusers.com/configure-squid-proxy-to-forward-to-a-parent-proxy/
- // https://gist.github.com/leefsmp/3e4385e08ea27e30ba96
- // https://github.com/tinyproxy/tinyproxy
-
- return newPromise().then(() => {
- // If target proxy has no authentication, pass it directly to the browser.
- if (!this.parsedProxyUrl.auth) {
- return this.parsedProxyUrl;
- }
-
- // Otherwise we need to setup an open child proxy
- // that will forward to the original proxy with authentication
- this.proxyChain = new ProxyChain(this.parsedProxyUrl);
- return this.proxyChain.start();
- })
- .then((effectiveParsedProxyUrl) => {
- if (/^chrome$/i.test(this.options.browserName)) {
- // In Chrome, Capabilities.setProxy() has no effect,
- // so we setup the proxy manually
- this.chromeOptions.addArguments(`--proxy-server=${effectiveParsedProxyUrl.host}`);
- } else {
- const proxyConfig = {
- proxyType: 'MANUAL',
- httpProxy: effectiveParsedProxyUrl.host,
- sslProxy: effectiveParsedProxyUrl.host,
- ftpProxy: effectiveParsedProxyUrl.host,
- // socksProxy: this.parsedProxyUrl.host,
- //socksUsername: parsed.username,
- //socksPassword: parsed.password,
- // noProxy: '', // Do not skip proxy for any address
- };
- this.capabilities.setProxy(proxyConfig);
-
- // console.dir(this.capabilities);
- }
- });
- }
-
close() {
- if (this.proxyChain) {
- this.proxyChain.shutdown();
- this.proxyChain = null;
- }
-
return newPromise()
.then(() => {
if (this.webDriver) {
return this.webDriver.quit();
}
})
+ .then(() => {
+ if (this.anonymizedProxyUrl) {
+ return closeAnonymizedProxy(this.anonymizedProxyUrl, true);
+ }
+ })
.then(() => {
this.webDriver = null;
});
@@ -356,12 +325,12 @@ export const launchWebDriver = (opts, callback) => {
.then(() => {
// TODO: for some reason this doesn't work, the proxy chain will never shut down!!
// we'll need to find a way to fix this!
- browser.webDriver.onQuit_ = () => {
- if (browser.proxyChain) {
- browser.proxyChain.shutdown();
- browser.proxyChain = null;
- }
- };
+ // browser.webDriver.onQuit_ = () => {
+ // if (browser.proxyChain) {
+ // browser.proxyChain.shutdown();
+ // browser.proxyChain = null;
+ // }
+ // };
return browser.webDriver;
});
diff --git a/test/puppeteer.js b/test/puppeteer.js
index 71985ff14..787406545 100644
--- a/test/puppeteer.js
+++ b/test/puppeteer.js
@@ -19,7 +19,7 @@ let wasProxyCalled = false; // eslint-disable-line no-unused-vars
// Setup local proxy server for the tests
before(() => {
prevEnvHeadless = process.env[ENV_VARS.HEADLESS];
- process.env[ENV_VARS.HEADLESS] = true;
+ process.env[ENV_VARS.HEADLESS] = '1';
// Find free port for the proxy
return portastic.find({ min: 50000, max: 50100 }).then((ports) => {
@@ -38,7 +38,7 @@ before(() => {
}
const parsed = basicAuthParser(auth);
const isEqual = _.isEqual(parsed, proxyAuth);
- // console.log('Parsed "Proxy-Authorization": parsed: %j expected: %j : %s', parsed, proxyAuth, isEqual);
+ console.log('Parsed "Proxy-Authorization": parsed: %j expected: %j : %s', parsed, proxyAuth, isEqual);
if (isEqual) wasProxyCalled = true;
fn(null, isEqual);
};
@@ -64,6 +64,11 @@ after(function () {
describe('Apify.launchPuppeteer()', () => {
it('throws on invalid args', () => {
+ expect(() => Apify.launchPuppeteer('some non-object')).to.throw(Error);
+ expect(() => Apify.launchPuppeteer(1234)).to.throw(Error);
+
+ expect(() => Apify.launchPuppeteer({ proxyUrl: 234 })).to.throw(Error);
+ expect(() => Apify.launchPuppeteer({ proxyUrl: {} })).to.throw(Error);
expect(() => Apify.launchPuppeteer({ proxyUrl: 'invalidurl' })).to.throw(Error);
expect(() => Apify.launchPuppeteer({ proxyUrl: 'http://host-without-port' })).to.throw(Error);
expect(() => Apify.launchPuppeteer({ proxyUrl: 'invalid://somehost:1234' })).to.throw(Error);
@@ -102,27 +107,26 @@ describe('Apify.launchPuppeteer()', () => {
// Test headless parameter
process.env[ENV_VARS.HEADLESS] = false;
- return Apify
- .launchPuppeteer({
- headless: true,
- proxyUrl: `http://username:[email protected]:${proxyPort}`,
- })
- .then((createdBrowser) => {
- browser = createdBrowser;
-
- return browser.newPage();
- })
- .then((openedPage) => {
- page = openedPage;
-
- return page.goto('https://example.com');
- })
- .then(() => {
- expect(wasProxyCalled).to.eql(true);
-
- return page.content();
- })
- .then(html => expect(html).to.include('<h1>Example Domain</h1>'))
- .then(() => browser.close());
+ return Apify.launchPuppeteer({
+ headless: true,
+ proxyUrl: `http://username:[email protected]:${proxyPort}`,
+ })
+ .then((createdBrowser) => {
+ browser = createdBrowser;
+
+ return browser.newPage();
+ })
+ .then((openedPage) => {
+ page = openedPage;
+
+ return page.goto('https://example.com');
+ })
+ .then(() => {
+ expect(wasProxyCalled).to.eql(true);
+
+ return page.content();
+ })
+ .then(html => expect(html).to.include('<h1>Example Domain</h1>'))
+ .then(() => browser.close());
});
});
diff --git a/test/utils.js b/test/utils.js
index 07941fd89..cc0a3b939 100644
--- a/test/utils.js
+++ b/test/utils.js
@@ -1,5 +1,3 @@
-import _ from 'underscore';
-import urlModule from 'url';
import BluebirdPromise from 'bluebird';
import { expect } from 'chai';
import * as utils from '../build/utils';
@@ -35,7 +33,6 @@ describe('Apify.xxxPromisesDependency()', () => {
});
});
-
describe('utils.newClient()', () => {
it('reads environment variables correctly', () => {
process.env.APIFY_API_BASE_URL = 'http://www.example.com:1234/path/';
@@ -65,76 +62,6 @@ describe('utils.newClient()', () => {
});
});
-
-const testUrl = (url, extras) => {
- const parsed1 = utils.parseUrl(url);
- const parsed2 = urlModule.parse(url);
- expect(parsed1).to.eql(_.extend(parsed2, extras));
-};
-
-describe('utils.parseUrl()', () => {
- it('works', () => {
- testUrl('https://username:[email protected]:12345/some/path', {
- scheme: 'https',
- username: 'username',
- password: 'password',
- });
-
- testUrl('http://us-er+na12345me:@www.example.com:12345/some/path', {
- scheme: 'http',
- username: 'us-er+na12345me',
- password: '',
- });
-
- testUrl('socks5://[email protected]:12345/some/path', {
- scheme: 'socks5',
- username: 'username',
- password: null,
- });
-
- testUrl('FTP://@www.example.com:12345/some/path', {
- scheme: 'ftp',
- username: null,
- password: null,
- });
-
- testUrl('HTTP://www.example.com:12345/some/path', {
- scheme: 'http',
- username: null,
- password: null,
- });
-
- testUrl('www.example.com:12345/some/path', {
- scheme: null,
- username: null,
- password: null,
- });
- });
-});
-
-
-describe('utils.redactUrl()', () => {
- it('works', () => {
- expect(utils.redactUrl('https://username:[email protected]:1234/path#hash'))
- .to.eql('https://username:<redacted>@www.example.com:1234/path#hash');
-
- expect(utils.redactUrl('https://[email protected]:1234/path#hash'))
- .to.eql('https://[email protected]:1234/path#hash');
-
- expect(utils.redactUrl('https://username:[email protected]:1234/path#hash', '<xxx>'))
- .to.eql('https://username:<xxx>@www.example.com:1234/path#hash');
-
- expect(utils.redactUrl('ftp://@www.example.com/path/path2'))
- .to.eql('ftp://www.example.com/path/path2');
-
- expect(utils.redactUrl('ftp://www.example.com'))
- .to.eql('ftp://www.example.com/');
-
- expect(utils.redactUrl('ftp://example.com/'))
- .to.eql('ftp://example.com/');
- });
-});
-
describe('utils.addCharsetToContentType()', () => {
it('works', () => {
expect(utils.addCharsetToContentType('application/json; charset=something')).to.eql('application/json; charset=something');