Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Feature/new docker image (#37)
Jan Čurnfbc7062ad3cb
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex fa7985ed6..df14c8c68 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -1,3 +1,11 @@+XXX / 2018-03-06+==================+- If the `APIFY_XVFB` env var is '1', then avoid headless mode+- Updated DEFAULT_USER_AGENT to Linux Chrome+- Consolidated startup options for Chrome - use --disable-dev-shm-usage, skip --no-sandbox,+ use --disable-gpu only on Windows+- Updated docs and package description+0.5.8 / 2018-03-06==================- Puppeteer updated to 1.1.1diff --git a/package.json b/package.jsonindex 5f34237cd..61d9c568c 100644--- a/package.json+++ b/package.json@@ -1,7 +1,7 @@{"name": "apify","version": "0.5.9",- "description": "Apify JavaScript SDK",+ "description": "Apify SDK for JavaScript / Node.js","main": "build/index.js","keywords": ["apify",diff --git a/src/constants.js b/src/constants.jsindex e42550264..68be65dbc 100644--- a/src/constants.js+++ b/src/constants.js@@ -1,6 +1,6 @@import * as consts from 'apify-shared/consts';-consts.DEFAULT_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'; // eslint-disable-line max-len+consts.DEFAULT_USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36'; // eslint-disable-line max-lenconsts.DEFAULT_PROXY_HOSTNAME = 'proxy.apify.com';consts.DEFAULT_PROXY_PORT = 8000;@@ -34,6 +34,7 @@ consts.ENV_VARS = {WATCH_FILE: 'APIFY_WATCH_FILE',API_BASE_URL: 'APIFY_API_BASE_URL',HEADLESS: 'APIFY_HEADLESS',+ XVFB: 'APIFY_XVFB',INTERNAL_PORT: 'APIFY_INTERNAL_PORT',MEMORY_MBYTES: 'APIFY_MEMORY_MBYTES',LOG_LEVEL: 'APIFY_LOG_LEVEL',diff --git a/src/puppeteer.js b/src/puppeteer.jsindex 55a0e3625..9c70bb266 100644--- a/src/puppeteer.js+++ b/src/puppeteer.js@@ -15,14 +15,14 @@ import { newPromise } from './utils';** <ul>* <li>Passes the setting from the `APIFY_HEADLESS` environment variable to the `headless` option,- * unless it was already defined by the caller. Note that this environment variable is automatically set to `1`- * in acts running on the Apify Actor cloud platform.</li>+ * unless it was already defined by the caller or `APIFY_XVFB` environment variable is set to `1`.+ * Note that Apify Actor cloud platform automatically sets `APIFY_HEADLESS=1` to all running acts.</li>* <li>Takes the `proxyUrl` option, checks it and adds it to `args` as `--proxy-server=XXX`.* If the proxy uses authentication, the function sets up an anonymous proxy HTTP* to make the proxy work with headless Chrome. For more information, read the* <a href="https://blog.apify.com/249a21a79212" target="_blank">blog post about proxy-chain library</a>.* </li>- * <li>Adds `--no-sandbox` to `args` to enable running headless Chrome in a Docker container on the Apify Actor platform.</li>+ * <li>Adds `--disable-dev-shm-usage` to `args` to avoid using shared memory space in favour of `/tmp`.</li>* </ul>** To use this function, you need to have the <a href="https://www.npmjs.com/package/puppeteer" target="_blank">puppeteer</a>@@ -31,6 +31,8 @@ import { newPromise } from './utils';* <a href="https://www.apify.com/docs/actor#base-images" target="_blank">documentation</a>* for more details.*+ * For an example of usage, see the <a href="https://www.apify.com/apify/example-puppeteer">apify/example-puppeteer</a> act.+ ** @param {Object} [opts] Optional settings passed to `puppeteer.launch()`. Additionally the object can contain the following fields:* @param {String} [opts.proxyUrl] URL to a HTTP proxy server.* Port number must be specified. Proxy username and password might also be provided.@@ -62,10 +64,10 @@ export const launchPuppeteer = (opts) => {}opts.args = opts.args || [];- opts.args.push('--no-sandbox');+ opts.args.push('--disable-dev-shm-usage');opts.args.push(`--user-agent=${opts.userAgent || DEFAULT_USER_AGENT}`);if (opts.headless === undefined || opts.headless === null) {- opts.headless = process.env[ENV_VARS.HEADLESS] === '1';+ opts.headless = process.env[ENV_VARS.HEADLESS] === '1' && process.env[ENV_VARS.XVFB] !== '1';}let anonymizedProxyUrl;diff --git a/src/webdriver.js b/src/webdriver.jsindex 0cd9200ae..0fb5651b4 100644--- a/src/webdriver.js+++ b/src/webdriver.js@@ -1,6 +1,6 @@// import { ChromeLauncher } from 'lighthouse/lighthouse-cli/chrome-launcher';import { anonymizeProxy, closeAnonymizedProxy } from 'proxy-chain';-import { ENV_VARS } from './constants';+import { DEFAULT_USER_AGENT, ENV_VARS } from './constants';import { newPromise } from './utils';/* global process, require */@@ -23,13 +23,15 @@ import { newPromise } from './utils';*/export const getDefaultBrowseOptions = () => {return {- headless: !!process.env[ENV_VARS.HEADLESS],+ headless: process.env[ENV_VARS.HEADLESS] === '1' && process.env[ENV_VARS.XVFB] !== '1',browserName: 'chrome',proxyUrl: null,userAgent: null,};};+const isWin = process.platform === 'win32';+/*** Represents a single web browser process.* Currently it is just a thin wrapper of Selenium's WebDriver instance.@@ -61,11 +63,14 @@ export class Browser {this.chromeOptions.addArguments('--disable-translate');this.chromeOptions.addArguments('--safebrowsing-disable-auto-update');if (this.options.headless) {- this.chromeOptions.addArguments('--headless', '--disable-gpu', '--no-sandbox');- }- if (this.options.userAgent) {- this.chromeOptions.addArguments(`--user-agent=${this.options.userAgent}`);+ this.chromeOptions.addArguments('--headless', '--disable-dev-shm-usage');+ if (isWin) {+ // Temporarily needed on Windows+ this.chromeOptions.addArguments('--disable-gpu');+ }}+ this.chromeOptions.addArguments(`--user-agent=${this.options.userAgent || DEFAULT_USER_AGENT}`);+if (this.options.extraChromeArguments) {this.chromeOptions.addArguments(this.options.extraChromeArguments);}@@ -280,20 +285,22 @@ export const browse = (url, options) => {* WebDriver</a>* class.*- * If the `APIFY_HEADLESS` environment variable is set to `1`, the function- * runs the web browser in headless mode. Note that this environment variable is automatically set to `1` when- * in acts running on the Apify Actor cloud platform.- ** To use this function, you need to have Google Chrome and* <a href="https://sites.google.com/a/chromium.org/chromedriver/" target="_blank">ChromeDriver</a> installed in your environment.* For example, you can use the `apify/actor-node-chrome` base Docker image for your act - see* <a href="https://www.apify.com/docs/actor#base-images" target="_blank">documentation</a>* for more details.*+ * For an example of usage, see the <a href="https://www.apify.com/apify/example-selenium">apify/example-selenium</a> act.+ ** @param {Object} [opts] Optional settings passed to `puppeteer.launch()`. Additionally the object can contain the following fields:* @param {String} [opts.proxyUrl] - URL to a proxy server. Currently only `http://` scheme is supported.* Port number must be specified. For example, `http://example.com:1234`.- * @param {String} [opts.userAgent] - Default User-Agent for the browser.+ * @param {String} [opts.headless] - Indicates that the browser will be started in headless mode.+ * If the option is not defined, and the `APIFY_HEADLESS` environment variable has value `1`+ * and `APIFY_XVFB` is NOT `1`, the value defaults to `true`, otherwise it will be `false`.+ * @param {String} [opts.userAgent] - User-Agent for the browser.+ * If not provided, the function sets it to a reasonable default.* @returns {Promise}** @memberof module:Apify