Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Developmed of Squid proxy chain
Jan Curnaf7b1cc1efbb
diff --git a/src/browser.js b/src/browser.jsindex 76dc9414b..9f3acae47 100644--- a/src/browser.js+++ b/src/browser.js@@ -26,6 +26,50 @@ export const getDefaultBrowseOptions = () => {};+// Base Squid proxy configuraton - enable all connection, disable all log files+const SQUID_CONF_BASE = `+http_access allow all+never_direct allow all+access_log none+cache_store_log none+cache_log /dev/null+logfile_rotate 0+`;++const getSquidConfForProxy = (parsedProxyUrl, squidPort) => {+ const peerName = `peer${squidPort}`;+ const aclName = `acl${squidPort}`;+ const str = `http_port ${squidPort}\n`+ + `cache_peer ${parsedProxyUrl.host} parent ${parsedProxyUrl.port} 0 no-query login=${parsedProxyUrl.auth} connect-fail-limit=99999999 proxy-only name=${peerName}\n` // eslint-disable-line max-len+ + `acl ${aclName} myport ${squidPort}\n`+ + `cache_peer_access ${peerName} allow ${aclName}\n`;+ return str;+};+++++class SquidProxyManager {+ constructor() {+ // A dictionary of all settings+ this.squidPortToParsedProxyUrl = {};+ }++ /**+ *+ * @param parsedProxyUrl+ * @return Promise Promise resolving to a handle for the proxy.+ */+ addProxy(parsedProxyUrl) {+ }++ removeProxy(handle) {+ }+}++const squidProxyManager = new SquidProxyManager();++/*** Represents a single web browser process.* Currently it is just a thin wrapper of Selenium's WebDriver instance.@@ -69,6 +113,8 @@ export class Browser {// Instance of Selenium's WebDriverthis.webDriver = null;++ this.proxyHandle = null;}/**@@ -175,27 +221,6 @@ export const browse = (url, options = null, callback = null) => {return nodeifyPromise(promise, callback);};-/*-// Base Squid proxy configuraton - enable all connection, disable all log files-const SQUID_CONF_BASE = `-http_access allow all-never_direct allow all-access_log none-cache_store_log none-cache_log /dev/null-logfile_rotate 0-`;-const getSquidConfForProxy = (parsedProxyUrl, squidPort) => {- const peerName = `peer${squidPort}`;- const aclName = `acl${squidPort}`;- const str = `http_port ${squidPort}\n`- + `cache_peer ${parsedProxyUrl.host} parent ${parsedProxyUrl.port} 0 no-query- login=${parsedProxyUrl.auth} connect-fail-limit=99999999 proxy-only name=${peerName}\n` // eslint-disable-line max-len- + `acl ${aclName} myport ${squidPort}\n`- + `cache_peer_access ${peerName} allow ${aclName}\n`;- return str;-};-*/// /**// * Launches a debugging instance of Chrome on port 9222, without Selenium.diff --git a/test/browser.js b/test/browser.jsindex 956b44283..3edccd2a2 100644--- a/test/browser.js+++ b/test/browser.js@@ -82,7 +82,7 @@ describe('getDefaultBrowseOptions()', () => {describe('Apifier.browse()', function () {// Need a large timeout to run unit tests on Travis CIthis.timeout(300 * 1000);-+/*it('throws with invalid params', () => {assert.throws(() => {Apifier.browse('http://www.blabla.bla', { proxyUrl: 'invalidurl' });@@ -149,8 +149,8 @@ describe('Apifier.browse()', function () {}});});+*/- /*it('works with proxy server', () => {let browser;wasProxyCalled = false;@@ -180,7 +180,7 @@ describe('Apifier.browse()', function () {expect(url).to.eql('https://www.example.com/');return browser.close();});- }); */+ });});