Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Fixed .travis.yml, development of proxyUrl setting
Jan Curnd1484ee8c4e7
diff --git a/.travis.yml b/.travis.ymlindex d469ff3c5..4dd70a111 100644--- a/.travis.yml+++ b/.travis.yml@@ -4,10 +4,10 @@ before_install:- curl -sS https://dl-ssl.google.com/linux/linux_signing_key.pub | sudo apt-key add -- sudo apt-get -qq update- sudo apt-get install -y google-chrome-beta- - curl -sS http://chromedriver.storage.googleapis.com/2.29/chromedriver_linux64.zip > /tmp.zip- - unzip /tmp.zip- - rm /tmp.zip- - mv /chromedriver /bin+ - sudo curl -sS http://chromedriver.storage.googleapis.com/2.29/chromedriver_linux64.zip > /tmp.zip+ - sudo unzip /tmp.zip+ - sudo rm /tmp.zip+ - sudo mv /chromedriver /binlanguage: node_jsnode_js:- "7"diff --git a/src/browser.js b/src/browser.jsindex 6e07838f0..b76ddfcbe 100644--- a/src/browser.js+++ b/src/browser.js@@ -1,3 +1,4 @@+import urlModule from 'url';import { ChromeLauncher } from 'lighthouse/lighthouse-cli/chrome-launcher';import { APIFY_ENV_VARS } from './constants';import { newPromise, nodeifyPromise } from './utils';@@ -19,7 +20,8 @@ import { newPromise, nodeifyPromise } from './utils';export const getDefaultBrowseOptions = () => {return {headless: !!process.env[APIFY_ENV_VARS.HEADLESS],- browser: 'chrome', // Current default+ browser: 'chrome',+ proxyUrl: null,};};@@ -67,13 +69,41 @@ export const browse = (url, options = null, callback = null) => {// to enable browser automation, here we add a few other ones// (inspired by Lighthouse, see lighthouse/lighthouse-cli/chrome-launcher)const chromeOpts = new chrome.Options();+// Disable built-in Google Translate servicechromeOpts.addArguments('--disable-translate');// Disable fetching safebrowsing lists, likely redundant due to disable-background-networkingchromeOpts.addArguments('--safebrowsing-disable-auto-update');+// Run in headless mode if requestedif (options.headless) chromeOpts.addArguments('--headless', '--disable-gpu', '--no-sandbox');+ if (options.proxyUrl) {+ // TODO+ urlModule.parse(options.proxyUrl);++ /* --proxy-server+ proxy = {'address': '123.123.123.123:2345',+ 'usernmae': 'johnsmith123',+ 'password': 'iliketurtles'}+++ capabilities = dict(DesiredCapabilities.CHROME)+ capabilities['proxy'] = {'proxyType': 'MANUAL',+ 'httpProxy': proxy['address'],+ 'ftpProxy': proxy['address'],+ 'sslProxy': proxy['address'],+ 'noProxy': '',+ 'class': "org.openqa.selenium.Proxy",+ 'autodetect': False}++ capabilities['proxy']['socksUsername'] = proxy['username']+ capabilities['proxy']['socksPassword'] = proxy['password']++ driver = webdriver.Chrome(executable_path=[path to your chromedriver], desired_capabilities=capabilities)+ */+ }+const webDriver = new Builder().forBrowser(options.browser).setChromeOptions(chromeOpts)