Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Developmed of Squid proxy chain and unit tests
Jan Curn3feed0413428
diff --git a/package.json b/package.jsonindex f7347b46d..a0c0eb7b2 100644--- a/package.json+++ b/package.json@@ -66,8 +66,7 @@"proxy": "^0.2.4","sinon": "^2.2.0","sinon-stub-promise": "^4.0.0",- "tmp": "^0.0.31",- "underscore": "^1.8.3"+ "request": "^2.81.0"},"optionalDependencies": {"selenium-webdriver": "^3.4.0"diff --git a/src/proxy_chain_manager.js b/src/proxy_chain_manager.jsindex 242ba80d0..1d9de9227 100644--- a/src/proxy_chain_manager.js+++ b/src/proxy_chain_manager.js@@ -9,24 +9,29 @@ import { parseUrl } from './utils';/* globals process */-// Exported to allow unit-testing+// Constants, exported to simplify unit-testing.export const PROXY_CHAIN = {PORT_FROM: 55000,PORT_TO: 60000,HOST: '127.0.0.1',PROTOCOL: 'http:',SQUID_CMD: 'squid',+ TMP_DIR_TEMPLATE: '/tmp/apify-squid-XXXXXX',+ CONF_FILE_NAME: 'squid.conf',+ PID_FILE_NAME: 'squid.pid',+ ERROR_PREFIX: 'ProxyChainManager: ',+ LOG_PREFIX: 'ProxyChainManager: ',};const tmpDirPromised = Promise.promisify(tmp.dir);const fsWriteFilePromised = Promise.promisify(fs.writeFile);/**- * The class is used to manage a local Squid proxy process instance+ * The class is used to manage a local Squid proxy instance* that forwards HTTP requests to user-provided parent proxies.* The point of this is that there's no easy way to programmatically pass proxy* authentication credentials to web browsers such as Chrome, so we need to setup- * local open proxy that can forward to parent proxy with authentication.+ * a local open proxy that can forward to parent proxy with authentication.*/export class ProxyChainManager {constructor() {@@ -80,7 +85,7 @@ export class ProxyChainManager {retrieve: 1,}).then((ports) => {- if (ports.length < 1) throw new Error(`ProxyChainManager: There are no more free ports in range from ${PROXY_CHAIN.PORT_FROM} to ${PROXY_CHAIN.PORT_TO}`); // eslint-disable-line max-len+ if (ports.length < 1) throw new Error(`${PROXY_CHAIN.ERROR_PREFIX}There are no more free ports in range from ${PROXY_CHAIN.PORT_FROM} to ${PROXY_CHAIN.PORT_TO}`); // eslint-disable-line max-lenreturn ports[0];});}@@ -130,7 +135,7 @@ export class ProxyChainManager {.then(() => {// Create temporary directory, if not created yetif (!this.tmpDir) {- return tmpDirPromised({ template: '/tmp/squid-XXXXXX' })+ return tmpDirPromised({ template: PROXY_CHAIN.TMP_DIR_TEMPLATE }).then((tmpDir) => {this.tmpDir = tmpDir;});@@ -142,9 +147,13 @@ export class ProxyChainManager {}).then((squidConfPath) => {// Start Squid process- const args = [`-f ${squidConfPath}`, '-N'];+ const cmd = PROXY_CHAIN.SQUID_CMD;+ const args = ['-f', squidConfPath, '-N'];const options = { cwd: this.tmpDir };- const process = childProcess.spawn(PROXY_CHAIN.SQUID_CMD, args, options);+ console.log(`${PROXY_CHAIN.LOG_PREFIX}Starting local Squid proxy using: ${cmd} ${args.join(' ')}`);+ const proc = childProcess.spawn(PROXY_CHAIN.SQUID_CMD, args, options);++ this.squidPid = proc.pid;// Wait for Squid process to be running or failreturn new Promise((resolve, reject) => {@@ -155,21 +164,40 @@ export class ProxyChainManager {isFinished = true;resolve();}+ // console.log("NOT RUNNING");}, 50);- process.on('error', (err) => {+ proc.on('exit', (code, signal) => {+ if (isFinished) {+ console.log(`${PROXY_CHAIN.LOG_PREFIX}Squid process exited (code: ${code}, signal: ${signal})`);+ return;+ }++ clearInterval(intervalId);+ isFinished = true;+ reject(new Error(`${PROXY_CHAIN.ERROR_PREFIX}Squid process exited unexpectedly (code: ${code}, signal: ${signal})`));+ });++ proc.on('error', (err) => {if (isFinished) {- console.log(`ProxyChainManager: Squid process failed: ${err}`);+ console.log(`${PROXY_CHAIN.LOG_PREFIX}Squid process failed: ${err}`);return;}// Give a user-friendly message for this common error- if (err.code === 'ENOENT') err = new Error('ProxyChainManager: "squid" command not found in the PATH');+ if (err.code === 'ENOENT') err = new Error(`${PROXY_CHAIN.ERROR_PREFIX}"squid" command not found in the PATH`);clearInterval(intervalId);isFinished = true;reject(err);});++ // Print stdout/stderr to simplify debugging+ const printLog = (data) => {+ console.log(`${PROXY_CHAIN.LOG_PREFIX}: squid: ${data}`);+ };+ proc.stdout.on('data', printLog);+ proc.stderr.on('data', printLog);});});}@@ -184,24 +212,24 @@ export class ProxyChainManager {// Run Squid process to reconfigure the other running instance and wait for the finishconst args = [`-f ${squidConfPath}`, '-k reconfigure'];const options = { cwd: this.tmpDir };- const process = childProcess.spawn(PROXY_CHAIN.SQUID_CMD, args, options);+ const proc = childProcess.spawn(PROXY_CHAIN.SQUID_CMD, args, options);// Wait for Squid reconfiguration process to exit or failreturn new Promise((resolve, reject) => {let isFinished = false;- process.on('exit', (code, signal) => {+ proc.on('exit', (code, signal) => {if (!isFinished) {isFinished = true;if (code !== 0) {- return reject(new Error(`ProxyChainManager: Squid reconfiguration process failed (exit code: ${code}, signal: ${signal})`)); // eslint-disable-line max-len+ return reject(new Error(`${PROXY_CHAIN.ERROR_PREFIX}Squid reconfiguration process failed (exit code: ${code}, signal: ${signal})`)); // eslint-disable-line max-len}resolve();}});- process.on('error', (err) => {+ proc.on('error', (err) => {if (!isFinished) {isFinished = true;- return reject(new Error(`ProxyChainManager: Squid reconfiguration process failed: ${err}`));+ return reject(new Error(`${PROXY_CHAIN.ERROR_PREFIX}Squid reconfiguration process failed: ${err}`));}});});@@ -212,7 +240,7 @@ export class ProxyChainManager {const peerName = `peer${squidPort}`;const aclName = `acl${squidPort}`;const str = `http_port ${squidPort}\n`- + `cache_peer ${parsedProxyUrl.host} parent ${parsedProxyUrl.port} 0 no-query login=${parsedProxyUrl.auth} connect-fail-limit=99999999 proxy-only name=${peerName}\n` // eslint-disable-line max-len+ + `cache_peer ${parsedProxyUrl.hostname} parent ${parsedProxyUrl.port} 0 no-query login=${parsedProxyUrl.auth} connect-fail-limit=99999999 proxy-only name=${peerName}\n` // eslint-disable-line max-len+ `acl ${aclName} myport ${squidPort}\n`+ `cache_peer_access ${peerName} allow ${aclName}\n`;return str;@@ -238,13 +266,12 @@ access_log nonecache_store_log nonecache_log /dev/nulllogfile_rotate 0-pid_filename ${path.join(this.tmpDir, 'squid.pid')}+pid_filename ${path.join(this.tmpDir, PROXY_CHAIN.PID_FILE_NAME)}${chainConfs}`;- const filePath = path.join(this.tmpDir, 'squid.conf');+ const filePath = path.join(this.tmpDir, PROXY_CHAIN.CONF_FILE_NAME);- console.log(`Squid config:\n${conf}`);return fsWriteFilePromised(filePath, conf).then(() => {return filePath;diff --git a/test/browser.js b/test/browser.jsindex 3edccd2a2..6a22c22ad 100644--- a/test/browser.js+++ b/test/browser.js@@ -82,7 +82,7 @@ describe('getDefaultBrowseOptions()', () => {describe('Apifier.browse()', function () {// Need a large timeout to run unit tests on Travis CIthis.timeout(300 * 1000);-/*+it('throws with invalid params', () => {assert.throws(() => {Apifier.browse('http://www.blabla.bla', { proxyUrl: 'invalidurl' });@@ -114,23 +114,25 @@ describe('Apifier.browse()', function () {delete process.env.APIFY_HEADLESS;let browser;return Apifier.browse('https://www.example.com', { headless: true })- .then((res) => {- browser = res;- expect(browser.constructor.name).to.eql('Browser');- return browser.webDriver.getCurrentUrl();- })- .then((url) => {- expect(url).to.eql('https://www.example.com/');- return browser.close();- });+ .then((res) => {+ browser = res;+ expect(browser.constructor.name).to.eql('Browser');+ return browser.webDriver.getCurrentUrl();+ })+ .then((url) => {+ expect(url).to.eql('https://www.example.com/');+ return browser.close();+ });});it('works with empty options and callback', () => {+ let browser;return new Promise((resolve, reject) => {try {process.env.APIFY_HEADLESS = '1';- const retVal = Apifier.browse('about:blank', {}, (err, browser) => {+ const retVal = Apifier.browse('about:blank', {}, (err, result) => {if (err) return reject(err);+ browser = result;try {expect(browser.constructor.name).to.eql('Browser');browser.webDriver.getCurrentUrl()@@ -147,9 +149,10 @@ describe('Apifier.browse()', function () {} catch (e) {reject(e);}+ }).then(() => {+ return browser.close();});});-*/it('works with proxy server', () => {let browser;@@ -160,23 +163,21 @@ describe('Apifier.browse()', function () {proxyUrl: `http://${proxyAuth.username}:${proxyAuth.password}@127.0.0.1:${proxyPort}`,};return Apifier.browse('https://www.example.com', opts)- .then((res) => {- browser = res;+ .then((result) => {+ browser = result;}).then(() => {- return browser.webDriver.sleep(300 * 1000);+ // return browser.webDriver.sleep(300 * 1000);}).then(() => {return browser.webDriver.getAllWindowHandles();})- .then((handles) => {- console.dir(handles);-+ .then(() => {expect(browser.constructor.name).to.eql('Browser');return browser.webDriver.getCurrentUrl();}).then((url) => {- expect(wasProxyCalled).to.be.true();+ expect(wasProxyCalled).to.equal(true);expect(url).to.eql('https://www.example.com/');return browser.close();});diff --git a/test/proxy_chain_manager.js b/test/proxy_chain_manager.jsindex ffd770b02..28286f60c 100644--- a/test/proxy_chain_manager.js+++ b/test/proxy_chain_manager.js@@ -5,9 +5,8 @@ import http from 'http';import portastic from 'portastic';import basicAuthParser from 'basic-auth-parser';import Promise from 'bluebird';+import request from 'request';-import { getDefaultBrowseOptions } from '../build/browser';-import Apifier from '../build/index';import { parseUrl } from '../build/utils';import { ProxyChainManager, PROXY_CHAIN } from '../build/proxy_chain_manager';@@ -63,12 +62,16 @@ after(function () {describe('ProxyChainManager.addProxyChain()', function () {// Need larger timeout for Travis CI- this.timeout(10 * 1000);+ this.timeout(100 * 1000);++ let mng;+ before(() => {+ mng = new ProxyChainManager();+ });it('throws nice error when "squid" command not found', () => {- const mng = new ProxyChainManager();PROXY_CHAIN.SQUID_CMD = 'command-that-does-not-exist';- return mng.addProxyChain(parseUrl(`http://localhost:${proxyPort}`))+ return mng.addProxyChain(parseUrl(`http://${proxyAuth.username}:${proxyAuth.password}@127.0.0.1:${proxyPort}`)).then(() => {assert.fail();})@@ -81,11 +84,10 @@ describe('ProxyChainManager.addProxyChain()', function () {});it('throws nice error when no more free ports available', () => {- const mng = new ProxyChainManager();// we're listening on proxyPortPROXY_CHAIN.PORT_FROM = proxyPort;PROXY_CHAIN.PORT_TO = proxyPort;- return mng.addProxyChain(parseUrl(`http://localhost:${proxyPort}`))+ return mng.addProxyChain(parseUrl(`http://${proxyAuth.username}:${proxyAuth.password}@127.0.0.1:${proxyPort}`)).then(() => {assert.fail();})@@ -96,4 +98,46 @@ describe('ProxyChainManager.addProxyChain()', function () {Object.assign(PROXY_CHAIN, ORIG_PROXY_CHAIN);});});++ it('creates a working proxy chain', () => {+ return mng.addProxyChain(parseUrl(`http://${proxyAuth.username}:${proxyAuth.password}@127.0.0.1:${proxyPort}`))+ .then((parsedChildProxyUrl) => {+ expect(parsedChildProxyUrl.port).to.not.equal(proxyPort);+ // console.log(`http://${proxyAuth.username}:${proxyAuth.password}@127.0.0.1:${proxyPort}`);+ // console.dir(parsedChildProxyUrl);++ return new Promise((resolve, reject) => {+ const opts = {+ uri: 'https://www.example.com',+ proxy: `${parsedChildProxyUrl.protocol}//${parsedChildProxyUrl.host}:${parsedChildProxyUrl.port}`,+ };+ request(opts, (error, response, body) => {+ if (error) return reject(error);+ if (response.statusCode !== 200) return reject(new Error(`Received invalid response code: ${response.statusCode}`));+ expect(body).to.contain('Example Domain');+ resolve();+ });+ });+ })+ .then(() => {+ expect(mng._isSquidRunning()).to.equal(true);+ expect(wasProxyCalled).to.equal(true);+ });+ });++ it('handles adding one more proxy', () => {+ // TODO+ });++ it('handles removal of first proxy', () => {++ });++ it('handles removal of second proxy', () => {++ });++ after(() => {+ Object.assign(PROXY_CHAIN, ORIG_PROXY_CHAIN);+ });});