Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Development of support for proxies
Jan Curn5f346feb615f
diff --git a/README.md b/README.mdindex d3c4433cc..1eb60a76e 100644--- a/README.md+++ b/README.md@@ -181,7 +181,7 @@ automation of a web browser.The simplest way to launch a new web browser process is to call:```javascript-const browser = await Apifier.browse('https://www.example.com/', { browser: 'chrome' });+const browser = await Apifier.browse('https://www.example.com/', { browserName: 'chrome' });```The first parameter is the URL of the initial page, by default it is `about:blank`.@@ -190,20 +190,28 @@ It has the following properties:```javascript{- // Indicates whether the browser should be opened in headless mode (i.e. without window).+ // The type of the web browser to use.+ // See https://github.com/SeleniumHQ/selenium/wiki/DesiredCapabilities for possible options.+ // By default it is 'chrome'.+ browserName: String,++ // Indicates whether the browser should be opened in headless mode (i.e. without windows).// By default, this value is generated based on the APIFY_HEADLESS environment variable.headless: Boolean,- // Type of the web browser, currently only 'chrome' is supported.- browser: 'chrome',+ // URL of the proxy server, e.g. 'socks://username:[email protected]:55555'.+ // By default it is null.+ proxyUrl: String,- // URL of the proxy server, e.g. 'socks://username:[email protected]:55555'- proxyUrl: null,+ // Overrides the User-Agent HTTP header of the web browser.+ // By default it is null, which means the browser uses its default User-Agent.+ userAgent: String,}```The `Apifier.browse()` function returns a promise resolving to a new instance of the `Browser` class,-which represents a web browser process. Currently it only contains the `webDriver` property,+which represents a web browser instance (possibly with multiple windows or tabs).+Currently it only contains the `webDriver` property,which is an instance of Selenium's[WebDriver](http://seleniumhq.github.io/selenium/docs/api/javascript/module/selenium-webdriver/index_exports_WebDriver.html)class:diff --git a/package.json b/package.jsonindex 5925ac86d..7ddc1a3b0 100644--- a/package.json+++ b/package.json@@ -48,6 +48,7 @@"babel-eslint": "^7.2.2","babel-preset-es2015": "^6.1.18","babel-preset-stage-0": "^6.1.18",+ "basic-auth-parser": "^0.0.2","bluebird": "^3.5.0","chai": "^3.4.1","eslint": "^3.19.0",@@ -60,6 +61,7 @@"isparta": "^4.0.0","mocha": "^3.2.0","portastic": "^1.0.1",+ "proxy": "^0.2.4","sinon": "^2.2.0","sinon-stub-promise": "^4.0.0","tmp": "^0.0.31",diff --git a/src/browser.js b/src/browser.jsindex 0604c793d..f468c6747 100644--- a/src/browser.js+++ b/src/browser.js@@ -1,7 +1,6 @@-import urlModule from 'url';// import { ChromeLauncher } from 'lighthouse/lighthouse-cli/chrome-launcher';import { APIFY_ENV_VARS } from './constants';-import { newPromise, nodeifyPromise } from './utils';+import { newPromise, nodeifyPromise, parseUrl } from './utils';/* global process, require */@@ -20,8 +19,9 @@ import { newPromise, nodeifyPromise } from './utils';export const getDefaultBrowseOptions = () => {return {headless: !!process.env[APIFY_ENV_VARS.HEADLESS],- browser: 'chrome',+ browserName: 'chrome',proxyUrl: null,+ userAgent: null,};};@@ -70,44 +70,53 @@ export const browse = (url, options = null, callback = null) => {// (inspired by Lighthouse, see lighthouse/lighthouse-cli/chrome-launcher)const chromeOpts = new chrome.Options();+ // Define capabilities of the web browser,+ // see https://github.com/SeleniumHQ/selenium/wiki/DesiredCapabilities for reference.+ const capabilities = {+ browserName: options.browserName,+ };+// Disable built-in Google Translate servicechromeOpts.addArguments('--disable-translate');+// Disable fetching safebrowsing lists, likely redundant due to disable-background-networkingchromeOpts.addArguments('--safebrowsing-disable-auto-update');// Run in headless mode if requested- if (options.headless) chromeOpts.addArguments('--headless', '--disable-gpu', '--no-sandbox');+ if (options.headless) {+ chromeOpts.addArguments('--headless', '--disable-gpu', '--no-sandbox');+ }++ // TODO: add unit test!+ if (options.userAgent) {+ chromeOpts.addArguments(`--user-agent=${options.userAgent}`);+ }if (options.proxyUrl) {- // TODO- urlModule.parse(options.proxyUrl);-- /* --proxy-server- proxy = {'address': '123.123.123.123:2345',- 'usernmae': 'johnsmith123',- 'password': 'iliketurtles'}-- or use https://github.com/SeleniumHQ/docker-selenium/wiki/Corporate-Proxies-- capabilities = dict(DesiredCapabilities.CHROME)- capabilities['proxy'] = {'proxyType': 'MANUAL',- 'httpProxy': proxy['address'],- 'ftpProxy': proxy['address'],- 'sslProxy': proxy['address'],- 'noProxy': '',- 'class': "org.openqa.selenium.Proxy",- 'autodetect': False}-- capabilities['proxy']['socksUsername'] = proxy['username']- capabilities['proxy']['socksPassword'] = proxy['password']-- driver = webdriver.Chrome(executable_path=[path to your chromedriver], desired_capabilities=capabilities)- */+ const parsed = parseUrl(options.proxyUrl);++ if (!parsed.host || !parsed.port) throw new Error('Invalid "proxyUrl" option: the URL must contain hostname and port number.');+ if (parsed.scheme !== 'http') throw new Error('Invalid "proxyUrl" option: only HTTP proxy type is currently supported.');++ // NOTE: to view effective proxy settings in Chrome, open chrome://net-internals/#proxy++ capabilities.proxy = {+ proxyType: 'manual',+ httpProxy: parsed.host,+ ftpProxy: parsed.host,+ sslProxy: parsed.host,+ socksProxy: parsed.host,+ socksUsername: parsed.username,+ socksPassword: parsed.password,+ noProxy: '', // Do not skip proxy for any address+ };++ console.dir(capabilities);}const webDriver = new Builder()- .forBrowser(options.browser).setChromeOptions(chromeOpts)+ .withCapabilities(capabilities).build();const browser = new Browser(webDriver);diff --git a/src/utils.js b/src/utils.jsindex 91e98e8a5..8d84ba180 100644--- a/src/utils.js+++ b/src/utils.js@@ -1,3 +1,4 @@+import urlModule from 'url';import ApifyClient from 'apify-client';import { APIFY_ENV_VARS } from './constants';@@ -74,3 +75,33 @@ export const newClient = () => {return new ApifyClient(opts);};+++/**+ * Sames are Node's url.parse() just adds the 'username', 'password' and 'scheme' fields.+ * @param url+ */+export const parseUrl = (url) => {+ const parsed = urlModule.parse(url);++ parsed.username = null;+ parsed.password = null;+ parsed.scheme = null;++ if (parsed.auth) {+ const matches = /^([^:]+)(:?)(.*)$/.exec(parsed.auth);+ if (matches && matches.length === 4) {+ parsed.username = matches[1];+ if (matches[2] === ':') parsed.password = matches[3];+ }+ }++ if (parsed.protocol) {+ const matches = /^([a-z0-9]+):$/i.exec(parsed.protocol);+ if (matches && matches.length === 2) {+ parsed.scheme = matches[1];+ }+ }++ return parsed;+};diff --git a/test/browser.js b/test/browser.jsindex 0ab17693e..071afcfb5 100644--- a/test/browser.js+++ b/test/browser.js@@ -1,27 +1,78 @@+import _ from 'underscore';import { expect, assert } from 'chai';-// import CDP from 'chrome-remote-interface';+import proxy from 'proxy';+import http from 'http';+import portastic from 'portastic';+import basicAuthParser from 'basic-auth-parser';+import Promise from 'bluebird';+import { getDefaultBrowseOptions } from '../build/browser';import Apifier from '../build/index';/* globals process */+let proxyServer;+let proxyPort;+const proxyAuth = { scheme: 'Basic', username: 'username', password: 'password' };+let wasProxyCalled = false;++// Setup local proxy server for the tests+before(() => {+ // Find free port for the proxy+ return portastic.find({ min: 50000, max: 50100 }).then((ports) => {+ return new Promise((resolve, reject) => {+ const httpServer = http.createServer();++ // Setup proxy authorization+ httpServer.authenticate = function (req, fn) {+ // parse the "Proxy-Authorization" header+ const auth = req.headers['proxy-authorization'];+ if (!auth) {+ // optimization: don't invoke the child process if no+ // "Proxy-Authorization" header was given+ // console.log('not Proxy-Authorization');+ return fn(null, false);+ }+ const parsed = basicAuthParser(auth);+ const isEqual = _.isEqual(parsed, proxyAuth);+ console.log('parsed "Proxy-Authorization": %j - %s', parsed, isEqual);+ if (isEqual) wasProxyCalled = true;+ fn(null, isEqual);+ };++ httpServer.on('error', reject);++ proxyServer = proxy(httpServer);+ proxyServer.listen(ports[0], () => {+ proxyPort = proxyServer.address().port;+ resolve();+ });+ });+ });+});++after(() => {+ if (proxyServer) return Promise.promisify(proxyServer.close).bind(proxyServer)();+});describe('getDefaultBrowseOptions()', () => {it('it works', () => {process.env.APIFY_HEADLESS = '1';const opts1 = getDefaultBrowseOptions();expect(opts1).to.eql({- browser: 'chrome',+ browserName: 'chrome',headless: true,proxyUrl: null,+ userAgent: null,});delete process.env.APIFY_HEADLESS;const opts2 = getDefaultBrowseOptions();expect(opts2).to.eql({- browser: 'chrome',+ browserName: 'chrome',headless: false,proxyUrl: null,+ userAgent: null,});});});@@ -31,6 +82,18 @@ describe('Apifier.browse()', function () {// Need a large timeout to run unit tests on Travis CIthis.timeout(300 * 1000);+ it('throws with invalid params', () => {+ assert.throws(() => {+ Apifier.browse('http://www.blabla.bla', { proxyUrl: 'invalidurl' });+ }, Error);+ assert.throws(() => {+ Apifier.browse('http://www.blabla.bla', { proxyUrl: 'http://host-without-port' });+ }, Error);+ assert.throws(() => {+ Apifier.browse('http://www.blabla.bla', { proxyUrl: 'invalid://somehost:1234' });+ }, Error);+ });+it('opens about:blank with no args', () => {process.env.APIFY_HEADLESS = '1';let browser;@@ -85,6 +148,36 @@ describe('Apifier.browse()', function () {}});});++ /* TODO !!!!+ it('works with proxy server', () => {+ let browser;+ wasProxyCalled = false;+ const opts = {+ headless: false,+ browserName: 'firefox',+ proxyUrl: `http://${proxyAuth.username}:${proxyAuth.password}@127.0.0.1:${proxyPort}`,+ };+ return Apifier.browse('https://www.example.com', opts)+ .then((res) => {+ browser = res;+ })+ .then(() => {+ return new Promise((resolve) => {+ setTimeout(resolve, 60 * 1000);+ });+ })+ .then(() => {+ expect(browser.constructor.name).to.eql('Browser');+ return browser.webDriver.getCurrentUrl();+ })+ .then((url) => {+ expect(wasProxyCalled).to.be.true();+ expect(url).to.eql('https://www.example.com/');+ return browser.close();+ });+ });+ */});diff --git a/test/utils.js b/test/utils.jsindex 31a705a7a..0268e01e5 100644--- a/test/utils.js+++ b/test/utils.js@@ -1,3 +1,5 @@+import _ from 'underscore';+import urlModule from 'url';import BluebirdPromise from 'bluebird';import { expect } from 'chai';import * as utils from '../build/utils';@@ -34,7 +36,7 @@ describe('Apifier.xxxPromisesDependency()', () => {});-describe('utils.newClient', () => {+describe('utils.newClient()', () => {it('reads environment variables correctly', () => {process.env.APIFY_API_BASE_URL = 'http://www.example.com:1234/path/';process.env.APIFY_USER_ID = 'userId';@@ -49,3 +51,50 @@ describe('utils.newClient', () => {expect(opts.baseUrl).to.eql('http://www.example.com:1234/path/');});});+++const testUrl = (url, extras) => {+ const parsed1 = utils.parseUrl(url);+ const parsed2 = urlModule.parse(url);+ expect(parsed1).to.eql(_.extend(parsed2, extras));+};++describe('utils.parseUrl()', () => {+ it('works', () => {+ testUrl('https://username:[email protected]:12345/some/path', {+ scheme: 'https',+ username: 'username',+ password: 'password',+ });++ testUrl('http://us-er+na12345me:@www.example.com:12345/some/path', {+ scheme: 'http',+ username: 'us-er+na12345me',+ password: '',+ });++ testUrl('socks5://[email protected]:12345/some/path', {+ scheme: 'socks5',+ username: 'username',+ password: null,+ });++ testUrl('FTP://@www.example.com:12345/some/path', {+ scheme: 'ftp',+ username: null,+ password: null,+ });++ testUrl('HTTP://www.example.com:12345/some/path', {+ scheme: 'http',+ username: null,+ password: null,+ });++ testUrl('www.example.com:12345/some/path', {+ scheme: null,+ username: null,+ password: null,+ });+ });+});