Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Development of ProxyChainManager and unit tests

Jan Curn361d31ce1fc9
diff --git a/src/proxy_chain_manager.js b/src/proxy_chain_manager.js
index 40d5cd5f8..6e4e62add 100644
--- a/src/proxy_chain_manager.js
+++ b/src/proxy_chain_manager.js
@@ -40,7 +40,8 @@ export class ProxyChainManager {
// Indicates whether _initialize() succeeded
this.isInitialized = false;
- // Dictionary of all proxy chains, key is Squid proxy port, value is parsed
+ // Dictionary of all proxy chains, key is Squid proxy port, value is an object such as:
+ // { parsed
// URL of the original parent proxy.
this.portToParsedProxyUrl = {};
@@ -73,6 +74,7 @@ export class ProxyChainManager {
return execFilePromised(PROXY_CHAIN.SQUID_CMD, PROXY_CHAIN.SQUID_CHECK_ARGS, options)
.then((array) => {
const stdout = array[0];
+ // TODO: check that the version is at least 3.3 and throw error otherwise!
console.log(`${PROXY_CHAIN.LOG_PREFIX}${(stdout || '').split('\n')[0]}`);
this.isInitialized = true;
})
@@ -99,7 +101,7 @@ export class ProxyChainManager {
return ProxyChainManager._findFreePort()
.then((port) => {
this.portToParsedProxyUrl[port] = _.clone(parsedProxyUrl);
- const parentProxyUrl = `${parsedProxyUrl.protocol}//${parsedProxyUrl.username}:${parsedProxyUrl.password ? '<redacted>' : ''}@${parsedProxyUrl.host}`; // eslint-disable-line max-len
+ const parentProxyUrl = `${parsedProxyUrl.protocol}//${parsedProxyUrl.username || ''}:${parsedProxyUrl.password ? '<redacted>' : ''}@${parsedProxyUrl.host}`; // eslint-disable-line max-len
const childProxyUrl = `${PROXY_CHAIN.PROTOCOL}//${PROXY_CHAIN.HOST}:${port}`;
parsedChildProxyUrl = parseUrl(childProxyUrl);
@@ -124,8 +126,10 @@ export class ProxyChainManager {
console.log(`${PROXY_CHAIN.LOG_PREFIX}Removing proxy chain ${childProxyUrl}`);
delete this.portToParsedProxyUrl[parsedChildProxyUrl.port];
- return this._manageSquidProcess();
+ return this._manageSquidProcess()
+ .then(() => true);
}
+ return Promise.resolve(false);
}
static _findFreePort() {
@@ -173,8 +177,8 @@ export class ProxyChainManager {
// If Squid is no longer needed but it is still running, kill it to save system resources
if (_.isEmpty(this.portToParsedProxyUrl)) {
if (this.squidPid) {
- process.kill(this.squidPid);
- this.squidPid = null;
+ console.log(`${PROXY_CHAIN.LOG_PREFIX}Killing Squid process ${this.squidPid}`);
+ process.kill(this.squidPid, 'SIGKILL');
}
return;
}
@@ -216,6 +220,7 @@ export class ProxyChainManager {
clearInterval(intervalId);
isFinished = true;
+ this.squidPid = null;
reject(new Error(`Squid process ${this.squidPid} exited unexpectedly (code: ${code}, signal: ${signal})`)); // eslint-disable-line max-len
});
@@ -227,6 +232,7 @@ export class ProxyChainManager {
clearInterval(intervalId);
isFinished = true;
+ this.squidPid = null;
reject(err);
});
@@ -247,12 +253,20 @@ export class ProxyChainManager {
return this._writeSquidConf();
})
.then((squidConfPath) => {
- const args = [`-f ${squidConfPath}`, '-k reconfigure'];
+ const args = ['-f', squidConfPath, '-k', 'reconfigure'];
const options = {
cwd: this.tmpDir,
timeout: PROXY_CHAIN.SQUID_BATCH_TIMEOUT,
};
return execFilePromised(PROXY_CHAIN.SQUID_CMD, args, options)
+ .then((arr) => {
+ console.dir(arr);
+
+ // TODO
+ return new Promise((resolve) => {
+ setTimeout(resolve, 1000);
+ });
+ })
.catch((err) => {
// Use better error message
throw new Error(`Squid reconfiguration failed (${err})`);
@@ -265,8 +279,10 @@ export class ProxyChainManager {
const aclName = `acl${squidPort}`;
const str = `http_port ${squidPort}\n`
+ `cache_peer ${parsedProxyUrl.hostname} parent ${parsedProxyUrl.port} 0 no-query login=${parsedProxyUrl.auth} connect-fail-limit=99999999 proxy-only name=${peerName}\n` // eslint-disable-line max-len
- + `acl ${aclName} myport ${squidPort}\n`
- + `cache_peer_access ${peerName} allow ${aclName}\n`;
+ // + `acl ${aclName} myportname ${squidPort}\n`
+ + `acl ${aclName} myportname ${squidPort}\n`
+ + `cache_peer_access ${peerName} allow ${aclName}\n`
+ + `cache_peer_access ${peerName} deny all\n`;
return str;
}
@@ -280,15 +296,20 @@ export class ProxyChainManager {
const chainConfs = _.values(_.mapObject(this.portToParsedProxyUrl, (parsedProxyUrl, port) => {
return ProxyChainManager._generateSquidConfForPort(parsedProxyUrl, port);
- }));
+ })).join('\n');
// NOTE: set pid_filename to isolate our squid instance from others possibly running in the system
const conf = `
+# debug_options 44,9 28,9
http_access allow all
never_direct allow all
-access_log none
+via off # this probably requires --enable-http-violations compile option
+forwarded_for transparent
+#access_log none
+access_log daemon:${path.join(this.tmpDir, 'access.log')} squid
cache_store_log none
-cache_log /dev/null
+#cache_log /dev/null
+cache_log ${path.join(this.tmpDir, 'cache.log')}
logfile_rotate 0
pid_filename ${path.join(this.tmpDir, PROXY_CHAIN.PID_FILE_NAME)}
@@ -296,9 +317,22 @@ ${chainConfs}
`;
const filePath = path.join(this.tmpDir, PROXY_CHAIN.CONF_FILE_NAME);
+ console.log('CONFIG');
+ console.log(conf);
+
return fsWriteFilePromised(filePath, conf)
.then(() => {
return filePath;
});
}
+
+ /**
+ * Removes all proxy chains and terminates the squid process.
+ * @return {*}
+ */
+ shutdown() {
+ this.portToParsedProxyUrl = {};
+
+ return this._manageSquidProcess();
+ }
}
diff --git a/test/proxy_chain_manager.js b/test/proxy_chain_manager.js
index a7560a56d..092ad9cee 100644
--- a/test/proxy_chain_manager.js
+++ b/test/proxy_chain_manager.js
@@ -16,7 +16,7 @@ const ORIG_PROXY_CHAIN = _.clone(PROXY_CHAIN);
let proxyServer;
let proxyPort; // eslint-disable-line no-unused-vars
-const proxyAuth = { scheme: 'Basic', username: 'username', password: 'password' };
+let proxyAuth = { scheme: 'Basic', username: 'username', password: 'password' };
let wasProxyCalled = false; // eslint-disable-line no-unused-vars
// Setup local proxy server for the tests
@@ -38,7 +38,7 @@ before(() => {
}
const parsed = basicAuthParser(auth);
const isEqual = _.isEqual(parsed, proxyAuth);
- console.log('parsed "Proxy-Authorization": %j - %s', parsed, isEqual);
+ console.log('Parsed "Proxy-Authorization": parsed: %j expected: %j : %s', parsed, proxyAuth, isEqual);
if (isEqual) wasProxyCalled = true;
fn(null, isEqual);
};
@@ -60,9 +60,29 @@ after(function () {
});
+const requestPromised = (opts) => {
+ console.log('requestPromised');
+ console.dir(opts);
+ return new Promise((resolve, reject) => {
+ request(opts, (error, response, body) => {
+ if (error) return reject(error);
+ if (response.statusCode !== 200) {
+ console.log('ERROR VOLE');
+ // console.dir(response);
+ console.dir(body);
+
+ return reject(new Error(`Received invalid response code: ${response.statusCode}`));
+ }
+ if (opts.expectBodyContainsText) expect(body).to.contain(opts.expectBodyContainsText);
+ resolve();
+ });
+ });
+};
+
+
describe('ProxyChainManager.addProxyChain()', function () {
// Need larger timeout for Travis CI
- this.timeout(100 * 1000);
+ this.timeout(1000 * 1000);
let mng;
before(() => {
@@ -127,45 +147,131 @@ describe('ProxyChainManager.addProxyChain()', function () {
assert.throws(() => { mng.addProxyChain(parseUrl('http://no-port-provided')); }, Error);
});
- it('creates a working proxy chain', () => {
- return mng.addProxyChain(parseUrl(`http://${proxyAuth.username}:${proxyAuth.password}@127.0.0.1:${proxyPort}`))
+ it('correctly handles add and removal', () => {
+ let parsedChildProxyUrl1;
+ let parsedChildProxyUrl2;
+ let proxy1;
+ let proxy2;
+ const proxyAuth1 = { scheme: 'Basic', username: 'user1', password: 'password1' };
+ const proxyAuth2 = { scheme: 'Basic', username: 'user2', password: 'password2' };
+ return Promise.resolve()
+ .then(() => {
+ // Add first proxy
+ console.log('*** Add first proxy');
+ proxyAuth = proxyAuth1;
+ return mng.addProxyChain(parseUrl(`http://${proxyAuth.username}:${proxyAuth.password}@127.0.0.1:${proxyPort}`));
+ })
.then((parsedChildProxyUrl) => {
+ // Test first proxy works
+ console.log('*** Test first proxy works');
expect(parsedChildProxyUrl.port).to.not.equal(proxyPort);
- // console.log(`http://${proxyAuth.username}:${proxyAuth.password}@127.0.0.1:${proxyPort}`);
- // console.dir(parsedChildProxyUrl);
-
- return new Promise((resolve, reject) => {
- const opts = {
- uri: 'https://www.example.com',
- proxy: `${parsedChildProxyUrl.protocol}//${parsedChildProxyUrl.host}:${parsedChildProxyUrl.port}`,
- };
- request(opts, (error, response, body) => {
- if (error) return reject(error);
- if (response.statusCode !== 200) return reject(new Error(`Received invalid response code: ${response.statusCode}`));
- expect(body).to.contain('Example Domain');
- resolve();
+ wasProxyCalled = false;
+ parsedChildProxyUrl1 = parsedChildProxyUrl;
+ proxy1 = `${parsedChildProxyUrl.protocol}//${parsedChildProxyUrl.host}`;
+ return requestPromised({
+ uri: 'https://www.example.com', // Test HTTPS
+ proxy: proxy1,
+ expectBodyContainsText: 'Example Domain',
+ })
+ .then(() => {
+ expect(wasProxyCalled).to.equal(true);
+ expect(mng._isSquidRunning()).to.equal(true);
+ });
+ })
+ .then(() => {
+ // Add second proxy
+ // NOTE: the host must be different than the first proxy (e.g. 127.0.0.1 vs localhost) !!!
+ proxyAuth = proxyAuth2;
+ return mng.addProxyChain(parseUrl(`http://${proxyAuth.username}:${proxyAuth.password}@localhost:${proxyPort}`));
+ })
+ .then((parsedChildProxyUrl) => {
+ // Test second proxy works
+ console.log('*** Test second proxy works');
+ expect(parsedChildProxyUrl.port).to.not.equal(proxyPort);
+ wasProxyCalled = false;
+ parsedChildProxyUrl2 = parsedChildProxyUrl;
+ proxy2 = `${parsedChildProxyUrl.protocol}//${parsedChildProxyUrl.host}}`;
+ return requestPromised({
+ uri: 'http://www.example.com', // Test HTTP
+ proxy: proxy2,
+ expectBodyContainsText: 'Example Domain',
+ })
+ .then(() => {
+ expect(wasProxyCalled).to.equal(true);
+ });
+ })
+ .then(() => {
+ // Test first proxy still works
+ console.log('*** Test first proxy still works');
+ return new Promise(() => {}); // TODO
+ proxyAuth = proxyAuth1;
+ wasProxyCalled = false;
+ return requestPromised({
+ uri: 'http://www.example.com',
+ proxy: proxy1,
+ expectBodyContainsText: 'Example Domain',
+ })
+ .then(() => {
+ expect(wasProxyCalled).to.equal(true);
+ })
+ .catch((err) => {
+ console.dir(err);
+ throw err;
+ });
+ })
+ .then(() => {
+ // Remove first proxy
+ console.log('*** Remove first proxy');
+ return mng.removeProxyChain(parsedChildProxyUrl1)
+ .then((wasRemoved) => {
+ expect(wasRemoved).to.equal(true);
});
+ })
+ .then(() => {
+ // Remove first proxy again (should have no effect)
+ console.log('*** Remove first proxy again');
+ return mng.removeProxyChain(parsedChildProxyUrl1)
+ .then((wasRemoved) => {
+ expect(wasRemoved).to.equal(false);
+ });
+ })
+ .then(() => {
+ // Check that second proxy still works
+ console.log('*** Check that second proxy still works');
+ proxyAuth = proxyAuth2;
+ wasProxyCalled = false;
+ return requestPromised({
+ uri: 'http://www.example.com',
+ proxy: proxy2,
+ expectBodyContainsText: 'Example Domain',
});
})
.then(() => {
- expect(mng._isSquidRunning()).to.equal(true);
+ // Check that first proxy no longer works
+ console.log('*** Check that first proxy no longer works');
+ proxyAuth = proxyAuth1;
expect(wasProxyCalled).to.equal(true);
+ return requestPromised({
+ uri: 'http://www.example.com',
+ proxy: proxy2,
+ expectBodyContainsText: 'Example Domain',
+ })
+ .then(() => {
+ assert.fail();
+ })
+ .catch((err) => {
+ expect(err.message).to.contain('There are no more free ports');
+ });
+ })
+ .then(() => {
+ // Remove second proxy
+ return mng.removeProxyChain(parsedChildProxyUrl2);
});
});
- it('handles adding one more proxy', () => {
- // TODO
- });
-
- it('handles removal of first proxy', () => {
-
- });
-
- it('handles removal of second proxy', () => {
-
- });
-
after(() => {
Object.assign(PROXY_CHAIN, ORIG_PROXY_CHAIN);
+
+ //return mng.shutdown();
});
});