Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Fixed .travis.yml, development of proxyUrl setting

Jan Curnd1484ee8c4e7
diff --git a/.travis.yml b/.travis.yml
index d469ff3c5..4dd70a111 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -4,10 +4,10 @@ before_install:
- curl -sS https://dl-ssl.google.com/linux/linux_signing_key.pub | sudo apt-key add -
- sudo apt-get -qq update
- sudo apt-get install -y google-chrome-beta
- - curl -sS http://chromedriver.storage.googleapis.com/2.29/chromedriver_linux64.zip > /tmp.zip
- - unzip /tmp.zip
- - rm /tmp.zip
- - mv /chromedriver /bin
+ - sudo curl -sS http://chromedriver.storage.googleapis.com/2.29/chromedriver_linux64.zip > /tmp.zip
+ - sudo unzip /tmp.zip
+ - sudo rm /tmp.zip
+ - sudo mv /chromedriver /bin
language: node_js
node_js:
- "7"
diff --git a/src/browser.js b/src/browser.js
index 6e07838f0..b76ddfcbe 100644
--- a/src/browser.js
+++ b/src/browser.js
@@ -1,3 +1,4 @@
+import urlModule from 'url';
import { ChromeLauncher } from 'lighthouse/lighthouse-cli/chrome-launcher';
import { APIFY_ENV_VARS } from './constants';
import { newPromise, nodeifyPromise } from './utils';
@@ -19,7 +20,8 @@ import { newPromise, nodeifyPromise } from './utils';
export const getDefaultBrowseOptions = () => {
return {
headless: !!process.env[APIFY_ENV_VARS.HEADLESS],
- browser: 'chrome', // Current default
+ browser: 'chrome',
+ proxyUrl: null,
};
};
@@ -67,13 +69,41 @@ export const browse = (url, options = null, callback = null) => {
// to enable browser automation, here we add a few other ones
// (inspired by Lighthouse, see lighthouse/lighthouse-cli/chrome-launcher)
const chromeOpts = new chrome.Options();
+
// Disable built-in Google Translate service
chromeOpts.addArguments('--disable-translate');
// Disable fetching safebrowsing lists, likely redundant due to disable-background-networking
chromeOpts.addArguments('--safebrowsing-disable-auto-update');
+
// Run in headless mode if requested
if (options.headless) chromeOpts.addArguments('--headless', '--disable-gpu', '--no-sandbox');
+ if (options.proxyUrl) {
+ // TODO
+ urlModule.parse(options.proxyUrl);
+
+ /* --proxy-server
+ proxy = {'address': '123.123.123.123:2345',
+ 'usernmae': 'johnsmith123',
+ 'password': 'iliketurtles'}
+
+
+ capabilities = dict(DesiredCapabilities.CHROME)
+ capabilities['proxy'] = {'proxyType': 'MANUAL',
+ 'httpProxy': proxy['address'],
+ 'ftpProxy': proxy['address'],
+ 'sslProxy': proxy['address'],
+ 'noProxy': '',
+ 'class': "org.openqa.selenium.Proxy",
+ 'autodetect': False}
+
+ capabilities['proxy']['socksUsername'] = proxy['username']
+ capabilities['proxy']['socksPassword'] = proxy['password']
+
+ driver = webdriver.Chrome(executable_path=[path to your chromedriver], desired_capabilities=capabilities)
+ */
+ }
+
const webDriver = new Builder()
.forBrowser(options.browser)
.setChromeOptions(chromeOpts)