Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Merge branch 'master' into feature/proxy-chain
Jan Curn5abb77dc2961
diff --cc src/puppeteer.jsindex 9d7a4861c,30c886ddf..2eabce385--- a/src/puppeteer.js+++ b/src/puppeteer.js@@@ -11,29 -10,34 +11,40 @@@ import { ENV_VARS } from './constants'* @function* @description <p>Launches headless Chrome using Puppeteer pre-configured to work with the Apify Actor platform.* The function has the same argument and return value as `puppeteer.launch()`.- * See {@link https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#puppeteerlaunchoptions|Puppeteer documentation} for more details.</p>+ * See {@link https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#puppeteerlaunchoptions|Puppeteer documentation} for more details.+ * </p>* <p>The `launchPuppeteer()` function alters the following Puppeteer options:* <ul>- * <li>Passes the value of the `APIFY_HEADLESS` environment variable to the `headless` option,- * unless it was already defined by the caller.</li>- * <li>Takes the `proxyUrl` option, checks it and adds it to the `--proxy-server=XXX` command-line option `args`.+ * <li>Passes the setting from the `APIFY_HEADLESS` environment variable to the `headless` option,+ * unless it was already defined by the caller. Note that this environment variable is automatically set to `1`+ * in acts running on the Apify Actor cloud platform.</li>- * <li>Takes the `proxyUrl` option and adds it to `env` under the `HTTPS_PROXY` or `HTTP_PROXY` key.</li>++ * <li>Takes the `proxyUrl` option, checks it and adds it to `args` as `--proxy-server=XXX`.+ * If the proxy uses authentication, the function sets up an anonymous proxy HTTP that will work with headless Chrome.+ * </li>* <li>Adds `--no-sandbox` to `args` to enable running headless Chrome in a Docker container on the Actor platform.</li>* </ul>* </p>- * @param [opts] Optional settings passed to `puppeteer.launch()`.- * Additionally, the options can contain the `proxyUrl` property to specify a proxy server- * (e.g. `http://username:[email protected]:1234`).+ * <p>+ * To use this function, you need to have the+ * {@link https://www.npmjs.com/package/puppeteer|puppeteer}+ * NPM package installed in your project.+ * For example, you can use the `apify/actor-node-puppeteer` base Docker image for your act - see+ * {@link https://www.apify.com/docs/actor#base-images|documentation}+ * for more details.+ * </p>+ * @param {Object} [opts] Optional settings passed to `puppeteer.launch()`. Additionally the object can contain the following fields:+ * @param {String} [opts.proxyUrl] - URL to a proxy server. Currently only `http://` scheme is supported.- * Port number must be specified. Proxy authentication is not supported yet. For example, `http://example.com:1234`.++ * Port number must be specified. Proxy authentication is also supported. For example, `http://bob:[email protected]:1234`.+ * @param {String} [opts.userAgent] - Default User-Agent for the browser.- * @returns {Promise} Promise object returned by `puppeteer.launch()`+ * @returns {Promise} Promise object that resolves to Puppeteer's `Browser` instance.*/-export const launchPuppeteer = (opts = {}) => {- let puppeteer;+export const launchPuppeteer = (opts) => {+ if (opts === undefined || opts === null) opts = {};++ checkParamOrThrow(opts, 'opts', 'Object');+ checkParamOrThrow(opts.args, 'opts.args', 'Maybe [String]');+ let puppeteer;try {// This is an optional dependency because it is quite large, only require it when used (ie. image with Chrome)puppeteer = require('puppeteer'); // eslint-disable-line global-require@@@ -43,44 -47,30 +54,45 @@@throw err;}- checkParamOrThrow(opts, 'opts', 'Object');- checkParamOrThrow(opts.env, 'opts.env', 'Maybe Object');- checkParamOrThrow(opts.args, 'opts.args', 'Maybe [String]');-- const { proxyUrl } = opts;+ opts.args = opts.args || [];+ opts.args.push('--no-sandbox');+ if (opts.headless === undefined || opts.headless === null) {- opts.headless = !!process.env[ENV_VARS.HEADLESS];++ opts.headless = process.env[ENV_VARS.HEADLESS] === '1';+ }++ if (opts.userAgent) opts.args.push(`--user-agent=${opts.userAgent}`);- if (proxyUrl) {- checkParamOrThrow(proxyUrl, 'opts.proxyUrl', 'String');+ let anonymizedProxyUrl;+ let promise;- const { host, port, protocol, password } = parseUrl(proxyUrl);- if (!host || !port) throw new Error('Invalid "proxyUrl" option: the URL must contain hostname and port number.');- if (protocol !== 'http:' && protocol !== 'https:') throw new Error('Invalid "proxyUrl" option: protocol must be http or https.');- if (password) throw new Error('Invalid "proxyUrl" option: password is not currently supported.');+ // Parse proxy URL and if it contains a password then setup ProxyChain server+ if (opts.proxyUrl) {+ checkParamOrThrow(opts.proxyUrl, 'opts.proxyUrl', 'String');- const proxyVarName = proxyUrl.startsWith('https') ? 'HTTPS_PROXY' : 'HTTP_PROXY';- opts.env = opts.env || {};- opts.env[proxyVarName] = opts.proxyUrl;+ // NOTE: anonymizeProxy() throws on invalid proxyUrl, so it must not be in a Promise!+ promise = anonymizeProxy(opts.proxyUrl)+ .then((result) => {+ anonymizedProxyUrl = result;+ opts.args.push(`--proxy-server=${anonymizedProxyUrl}`);+ })+ .then(() => puppeteer.launch(opts));+ } else {+ promise = puppeteer.launch(opts);}- opts.args = opts.args || [];- opts.args.push('--no-sandbox');+ // --user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36- if (opts.userAgent) opts.args.push(`--user-agent=${opts.userAgent}`);- if (opts.headless === undefined || opts.headless === null) opts.headless = process.env[ENV_VARS.HEADLESS] === '1';+ // Close ProxyChain server when Puppeteer finishes+ if (anonymizedProxyUrl) {+ promise = promise.then((browser) => {+ browser.on('disconnected', () => {+ console.log('Puppeteer disconnected');+ // Don't wait for finish+ closeAnonymizedProxy(anonymizedProxyUrl, true);+ });+ return browser;+ });+ }- return puppeteer.launch(opts);+ // Ensure that the returned promise is of type set in setPromiseDependency()+ return newPromise().then(() => promise);};diff --cc test/puppeteer.jsindex 3cf1d440a,0790eaa4e..71985ff14--- a/test/puppeteer.js+++ b/test/puppeteer.js@@@ -8,58 -2,7 +8,59 @@@ import _ from 'underscore'import Apify from '../build/index';import { ENV_VARS } from '../build/constants';-process.env[ENV_VARS.HEADLESS] = '1';+++let prevEnvHeadless;++let proxyServer;+let proxyPort; // eslint-disable-line no-unused-vars+const proxyAuth = { scheme: 'Basic', username: 'username', password: 'password' };+let wasProxyCalled = false; // eslint-disable-line no-unused-vars++// Setup local proxy server for the tests+before(() => {+ prevEnvHeadless = process.env[ENV_VARS.HEADLESS];+ process.env[ENV_VARS.HEADLESS] = true;++ // Find free port for the proxy+ return portastic.find({ min: 50000, max: 50100 }).then((ports) => {+ return new Promise((resolve, reject) => {+ const httpServer = http.createServer();++ // Setup proxy authorization+ httpServer.authenticate = function (req, fn) {+ // parse the "Proxy-Authorization" header+ const auth = req.headers['proxy-authorization'];+ if (!auth) {+ // optimization: don't invoke the child process if no+ // "Proxy-Authorization" header was given+ // console.log('not Proxy-Authorization');+ return fn(null, false);+ }+ const parsed = basicAuthParser(auth);+ const isEqual = _.isEqual(parsed, proxyAuth);+ // console.log('Parsed "Proxy-Authorization": parsed: %j expected: %j : %s', parsed, proxyAuth, isEqual);+ if (isEqual) wasProxyCalled = true;+ fn(null, isEqual);+ };++ httpServer.on('error', reject);++ proxyServer = proxy(httpServer);+ proxyServer.listen(ports[0], () => {+ proxyPort = proxyServer.address().port;+ resolve();+ });+ });+ });+});++after(function () {+ process.env[ENV_VARS.HEADLESS] = prevEnvHeadless;++ this.timeout(5 * 1000);+ if (proxyServer) return Promise.promisify(proxyServer.close).bind(proxyServer)();+});+describe('Apify.launchPuppeteer()', () => {it('throws on invalid args', () => {