Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Developmed of Squid proxy chain
Jan Curnc15e1bfd6bd1
diff --git a/.eslintrc.json b/.eslintrc.jsonindex fe91d93e8..5ee33f47e 100644--- a/.eslintrc.json+++ b/.eslintrc.json@@ -21,6 +21,7 @@"func-names": 0,"import/prefer-default-export": 0,"import/no-absolute-path": 0,- "import/no-extraneous-dependencies": ["error", { "devDependencies": ["**/test/*.js"] }]+ "import/no-extraneous-dependencies": ["error", { "devDependencies": ["**/test/*.js"] }],+ "no-underscore-dangle": 0}}diff --git a/package.json b/package.jsonindex 980ce439e..8c66adbd2 100644--- a/package.json+++ b/package.json@@ -41,7 +41,9 @@},"dependencies": {"apify-client": ">=0.1.13",- "underscore": "^1.8.3"+ "underscore": "^1.8.3",+ "portastic": "^1.0.1",+ "tmp": "^0.0.31"},"devDependencies": {"babel-cli": "^6.6.5",@@ -61,7 +63,6 @@"eslint-plugin-react": "^7.0.1","isparta": "^4.0.0","mocha": "^3.2.0",- "portastic": "^1.0.1","proxy": "^0.2.4","sinon": "^2.2.0","sinon-stub-promise": "^4.0.0",diff --git a/src/browser.js b/src/browser.jsindex 9f3acae47..6613b59c3 100644--- a/src/browser.js+++ b/src/browser.js@@ -1,6 +1,7 @@// import { ChromeLauncher } from 'lighthouse/lighthouse-cli/chrome-launcher';import { APIFY_ENV_VARS } from './constants';import { newPromise, nodeifyPromise, parseUrl } from './utils';+import { ProxyChainManager } from './proxy_chain_manager';/* global process, require */@@ -12,6 +13,7 @@ import { newPromise, nodeifyPromise, parseUrl } from './utils';// logging.installConsoleHandler();// logging.getLogger('webdriver.http').setLevel(logging.Level.ALL);+/*** Gets the default options for the browse() function, generated from current process environment* variables. This is function to enable unit testing.@@ -25,49 +27,7 @@ export const getDefaultBrowseOptions = () => {};};--// Base Squid proxy configuraton - enable all connection, disable all log files-const SQUID_CONF_BASE = `-http_access allow all-never_direct allow all-access_log none-cache_store_log none-cache_log /dev/null-logfile_rotate 0-`;--const getSquidConfForProxy = (parsedProxyUrl, squidPort) => {- const peerName = `peer${squidPort}`;- const aclName = `acl${squidPort}`;- const str = `http_port ${squidPort}\n`- + `cache_peer ${parsedProxyUrl.host} parent ${parsedProxyUrl.port} 0 no-query login=${parsedProxyUrl.auth} connect-fail-limit=99999999 proxy-only name=${peerName}\n` // eslint-disable-line max-len- + `acl ${aclName} myport ${squidPort}\n`- + `cache_peer_access ${peerName} allow ${aclName}\n`;- return str;-};-----class SquidProxyManager {- constructor() {- // A dictionary of all settings- this.squidPortToParsedProxyUrl = {};- }-- /**- *- * @param parsedProxyUrl- * @return Promise Promise resolving to a handle for the proxy.- */- addProxy(parsedProxyUrl) {- }-- removeProxy(handle) {- }-}--const squidProxyManager = new SquidProxyManager();+const proxyChainManager = new ProxyChainManager();/**@@ -114,7 +74,7 @@ export class Browser {// Instance of Selenium's WebDriverthis.webDriver = null;- this.proxyHandle = null;+ this.parsedChildProxyUrl = null;}/**@@ -157,16 +117,30 @@ export class Browser {// https://github.com/tinyproxy/tinyproxyreturn newPromise().then(() => {+ // If target proxy has no authentication, pass it directly to the browser.+ if (!this.parsedProxyUrl.auth) {+ return this.parsedProxyUrl;+ }++ // Otherwise we need to setup an open child proxy+ // that will forward to the original proxy with authentication+ return proxyChainManager.addProxyChain(this.parsedProxyUrl)+ .then((parsedChildProxyUrl) => {+ this.parsedChildProxyUrl = parsedChildProxyUrl;+ return parsedChildProxyUrl;+ });+ })+ .then((effectiveParsedProxyUrl) => {if (/^chrome$/i.test(this.options.browserName)) {// In Chrome Capabilities.setProxy() has no effect,// so we setup the proxy manually- this.chromeOptions.addArguments(`--proxy-server=${this.parsedProxyUrl.host}`);+ this.chromeOptions.addArguments(`--proxy-server=${effectiveParsedProxyUrl.host}`);} else {const proxyConfig = {proxyType: 'MANUAL',- httpProxy: this.parsedProxyUrl.host,- sslProxy: this.parsedProxyUrl.host,- ftpProxy: this.parsedProxyUrl.host,+ httpProxy: effectiveParsedProxyUrl.host,+ sslProxy: effectiveParsedProxyUrl.host,+ ftpProxy: effectiveParsedProxyUrl.host,// socksProxy: this.parsedProxyUrl.host,//socksUsername: parsed.username,//socksPassword: parsed.password,@@ -180,17 +154,20 @@ export class Browser {}close() {- if (this.proxy) {- this.proxy.close();- this.proxy = null;- }-return newPromise()+ .then(() => {+ if (this.parsedChildProxyUrl) {+ return proxyChainManager.removeProxyChain(this.parsedChildProxyUrl);+ }+ }).then(() => {if (this.webDriver) {- this.webDriver.quit();- this.webDriver = null;+ return this.webDriver.quit();}+ })+ .then(() => {+ this.parsedChildProxyUrl = null;+ this.webDriver = null;});}}diff --git a/src/proxy_chain_manager.js b/src/proxy_chain_manager.jsnew file mode 100644index 000000000..b4a2bd31f--- /dev/null+++ b/src/proxy_chain_manager.js@@ -0,0 +1,130 @@+import _ from 'underscore';+import portastic from 'portastic';++/* globals process */++const PROXY_CHAIN_PORT_FROM = 50000;+const PROXY_CHAIN_PORT_TO = 60000;+const PROXY_CHAIN_HOST = '127.0.0.1';+const PROXY_CHAIN_PROTOCOL = 'http:';++export class ProxyChainManager {+ constructor() {+ // Dictionary of all proxy chains, key is Squid proxy port, value is parsed+ // URL of the original parent proxy.+ this.portToParsedProxyUrl = {};++ // PID of the Squid proxy process+ this.squidPid = null;+ }++ /**+ * Sets up an open child proxy which forwards to a specified parent proxy with authentication.+ * @param parsedChildProxyUrl Parsed URL of the parent proxy.+ * @return Promise Promise resolving to the parsed URL of the child proxy.+ */+ addProxyChain(parsedProxyUrl) {+ let parsedChildProxyUrl;+ return ProxyChainManager._findFreePort()+ .then((port) => {+ this.portToParsedProxyUrl[port] = _.clone(parsedProxyUrl);+ parsedChildProxyUrl = parseUrl(`${PROXY_CHAIN_PROTOCOL}//${PROXY_CHAIN_HOST}:${port}`);+ return this._manageSquidProcess();+ })+ .then(() => {+ return parsedChildProxyUrl;+ });+ }++ /**+ * Removes a child proxy.+ * @param childParsedUrl Result of the previous call from the addProxyChain() function.+ * @return Promise Returns a promise that resolves when the proxy chain is reconfigured.+ */+ removeProxyChain(parsedChildProxyUrl) {+ if (parsedChildProxyUrl.protocol === PROXY_CHAIN_PROTOCOL+ && parsedChildProxyUrl.host === PROXY_CHAIN_HOST+ && this.portToParsedProxyUrl[parsedChildProxyUrl.port]) {+ delete this.portToParsedProxyUrl[parsedChildProxyUrl.port];+ return this._manageSquidProcess();+ }+ }++ static _findFreePort() {+ return portastic.find({+ min: PROXY_CHAIN_PORT_FROM,+ max: PROXY_CHAIN_PORT_TO,+ retrieve: 1,+ })+ .then((ports) => {+ if (ports.length < 1) throw new Error(`There are no more free ports in range from ${PROXY_CHAIN_PORT_FROM} to ${PROXY_CHAIN_PORT_TO}`);+ return ports[0];+ });+ }++ /**+ * Manages Squid proxy process, which means the function either starts it, updates its configuration or kills it,+ * depending on the state of the `portToParsedProxyUrl` field.+ * @private+ */+ _manageSquidProcess() {+ // First, check whether the Squid process is still running+ // (inspired by https://github.com/nisaacson/is-running/blob/master/index.js)+ if (this.squidPid) {+ let isRunning;+ try {+ isRunning = !!process.kill(this.squidPid, 0);+ } catch (e) {+ isRunning = e.code === 'EPERM';+ }+ if (!isRunning) this.squidPid = null;+ }++ // If Squid shouldn't be running but sill is, kill it+ if (_.isEmpty(this.portToParsedProxyUrl)) {+ if (this.squidPid) {+ process.kill(this.squidPid);+ this.squidPid = null;+ }+ return;+ }++ const squidConf = this._generateSquidConf();++ if (!this.squidPid) {+ // Squid is not running => start it+ // TODO+ } else {+ // Squid is already running => reconfigure it+ // TODO+ }+ }++ static _generateSquidConfForPort(parsedProxyUrl, squidPort) {+ const peerName = `peer${squidPort}`;+ const aclName = `acl${squidPort}`;+ const str = `http_port ${squidPort}\n`+ + `cache_peer ${parsedProxyUrl.host} parent ${parsedProxyUrl.port} 0 no-query login=${parsedProxyUrl.auth} connect-fail-limit=99999999 proxy-only name=${peerName}\n` // eslint-disable-line max-len+ + `acl ${aclName} myport ${squidPort}\n`+ + `cache_peer_access ${peerName} allow ${aclName}\n`;+ return str;+ }++ _generateSquidConf() {+ const chainConfs = _.mapObject(this.portToParsedProxyUrl, (parsedProxyUrl, port) => {+ return this._generateSquidConfForPort(parsedProxyUrl, port);+ }).join('\n');++ const conf = `+http_access allow all+never_direct allow all+access_log none+cache_store_log none+cache_log /dev/null+logfile_rotate 0++${chainConfs}+`;+ return conf;+ }+}