Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Added useChrome option to launchPuppeteer() + unit test, renamed unit… (#45)
Jan Čurnb71ab739a2ff
diff --git a/.travis.yml b/.travis.ymlindex 61b0d4f03..88a2a21ff 100644--- a/.travis.yml+++ b/.travis.yml@@ -9,8 +9,6 @@ before_install:- sudo unzip /tmp/chromedriver.zip -d /tmp- sudo rm /tmp/chromedriver.zip- sudo mv /tmp/chromedriver /usr/bin/- - # Travis' VM has preinstalled google-chrome stable, which has no headless mode!- - sudo mv -f /usr/bin/google-chrome-beta /usr/bin/google-chrome | tee- # Print versions to help debugging- google-chrome --version- chromedriver --versiondiff --git a/CHANGELOG.md b/CHANGELOG.mdindex 0198824f0..52f184e4f 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -1,3 +1,7 @@+XXX / 2018-03-20+===================+- Added `useChrome` flag to `launchPuppeteer()` function+0.5.14 / 2018-03-20===================- Removed again the --disable-dev-shm-usage flag when launching headless Chrome,diff --git a/examples/example_autoscaled_pool.js b/examples/example_autoscaled_pool.jsindex c4d4b9168..4b9461acc 100644--- a/examples/example_autoscaled_pool.js+++ b/examples/example_autoscaled_pool.js@@ -1,4 +1,4 @@-const Apify = require('../build');+const Apify = require('apify');Apify.main(async () => {let counter = 0;diff --git a/examples/example_basic_crawler.js b/examples/example_basic_crawler.jsindex 88d9e8839..d0eb88113 100644--- a/examples/example_basic_crawler.js+++ b/examples/example_basic_crawler.js@@ -1,4 +1,4 @@-const Apify = require('../build');+const Apify = require('apify');const rp = require('request-promise');Apify.main(async () => {diff --git a/examples/example_puppeteer_crawler.js b/examples/example_puppeteer_crawler.jsindex 11f52c041..011efcf40 100644--- a/examples/example_puppeteer_crawler.js+++ b/examples/example_puppeteer_crawler.js@@ -1,4 +1,4 @@-const Apify = require('../build');+const Apify = require('apify');Apify.main(async () => {// Create a request list.diff --git a/package.json b/package.jsonindex 7909025f8..d7b35e9f5 100644--- a/package.json+++ b/package.json@@ -44,14 +44,14 @@},"dependencies": {"apify-client": "^0.2.3",- "apify-shared": "0.0.16",+ "apify-shared": "0.0.22","bluebird": "^3.5.0","content-type": "^1.0.3","fs-extra": "^5.0.0","proxy-chain": "^0.1.2","request-promise": "^4.2.2","underscore": "^1.8.3",- "ws": "^4.1.0"+ "ws": "^5.1.0"},"devDependencies": {"apify-jsdoc-template": "github:apifytech/apify-jsdoc-template",@@ -82,7 +82,7 @@"sinon": "^4.1.2","sinon-stub-promise": "^4.0.0","tmp": "^0.0.33",- "why-is-node-running": "^1.2.3"+ "why-is-node-running": "^2.0.2"},"optionalDependencies": {"puppeteer": "^1.2.0",diff --git a/src/constants.js b/src/constants.jsindex 68be65dbc..ebb894beb 100644--- a/src/constants.js+++ b/src/constants.js@@ -39,6 +39,7 @@ consts.ENV_VARS = {MEMORY_MBYTES: 'APIFY_MEMORY_MBYTES',LOG_LEVEL: 'APIFY_LOG_LEVEL',ACTOR_EVENTS_WS_URL: 'APIFY_ACTOR_EVENTS_WS_URL',+ CHROME_EXECUTABLE_PATH: 'APIFY_CHROME_EXECUTABLE_PATH',};consts.KEY_VALUE_STORE_KEYS = {diff --git a/src/puppeteer.js b/src/puppeteer.jsindex 5c96ef562..06175d642 100644--- a/src/puppeteer.js+++ b/src/puppeteer.js@@ -1,7 +1,7 @@import { checkParamOrThrow } from 'apify-client/build/utils';import { anonymizeProxy, closeAnonymizedProxy } from 'proxy-chain';import { ENV_VARS, DEFAULT_USER_AGENT } from './constants';-import { newPromise } from './utils';+import { newPromise, getTypicalChromeExecutablePath } from './utils';/* global process, require */@@ -39,6 +39,11 @@ import { newPromise } from './utils';* For example, `http://bob:[email protected]:1234`.* @param {String} [opts.userAgent] Default User-Agent for the browser.* If not provided, the function sets it to a reasonable default.+ * @param {String} [opts.useChrome=false] If true-ish value and `opts.executablePath` is not set,+ * Puppeteer will launch full Chrome available on the machine rather than the bundled Chromium.+ * The path to Chrome executable is taken from the `APIFY_CHROME_EXECUTABLE_PATH` environment variable if provided,+ * or defaults to the typical Google Chrome executable location specific for the operating system.+ * By default, this option is `false`.* @returns {Promise} Promise object that resolves to Puppeteer's `Browser` instance.** @memberof module:Apify@@ -69,6 +74,9 @@ export const launchPuppeteer = (opts) => {if (opts.headless === undefined || opts.headless === null) {opts.headless = process.env[ENV_VARS.HEADLESS] === '1' && process.env[ENV_VARS.XVFB] !== '1';}+ if (opts.useChrome && (opts.executablePath === undefined || opts.executablePath === null)) {+ opts.executablePath = process.env[ENV_VARS.CHROME_EXECUTABLE_PATH] || getTypicalChromeExecutablePath();+ }let anonymizedProxyUrl;let promise;diff --git a/src/utils.js b/src/utils.jsindex 4a4347536..cb5ae5251 100644--- a/src/utils.js+++ b/src/utils.js@@ -201,3 +201,15 @@ export const isProduction = () => process.env.NODE_ENV !== 'production';* @ignore*/export const ensureDirExists = path => ensureDirPromised(path);++/**+ * Gets a typical path to Chrome executable, depending on the current operating system.+ * @returns {string}+ */+export const getTypicalChromeExecutablePath = () => {+ switch (os.platform()) {+ case 'darwin': return '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome';+ case 'win32': return 'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe';+ default: return 'google-chrome';+ }+};diff --git a/test/autoscaled_pool.js b/test/autoscaled_pool.jsindex e1993be34..b97cb5502 100644--- a/test/autoscaled_pool.js+++ b/test/autoscaled_pool.js@@ -12,7 +12,7 @@ import * as utils from '../build/utils';chai.use(chaiAsPromised);const toBytes = x => x * 1024 * 1024;-describe('autoscaled_pool', () => {+describe('AutoscaledPool', () => {it('should work with concurrency 1', async () => {const range = _.range(0, 10);const result = [];diff --git a/test/basic_crawler.js b/test/basic_crawler.jsindex fd81d5558..12511ec0f 100644--- a/test/basic_crawler.js+++ b/test/basic_crawler.js@@ -7,7 +7,7 @@ import * as Apify from '../build/index';chai.use(chaiAsPromised);-describe('basic_crawler', () => {+describe('BasicCrawler', () => {it('runs in parallel thru all the requests', async () => {const startedAt = Date.now();const sources = _.range(0, 500).map(index => ({ url: `https://example.com/${index}` }));diff --git a/test/key_value_store.js b/test/key_value_store.jsindex 0de475935..0769e3fff 100644--- a/test/key_value_store.js+++ b/test/key_value_store.js@@ -10,7 +10,7 @@ import { LOCAL_EMULATION_DIR, emptyLocalEmulationSubdir, expectNotLocalEmulationchai.use(chaiAsPromised);-describe('key_value_store', () => {+describe('KeyValueStore', () => {before(() => apifyClient.setOptions({ token: 'xxx' }));after(() => apifyClient.setOptions({ token: undefined }));beforeEach(() => emptyLocalEmulationSubdir(LOCAL_EMULATION_SUBDIR));diff --git a/test/puppeteer.js b/test/puppeteer.jsindex e71f44ff9..19cb3436d 100644--- a/test/puppeteer.js+++ b/test/puppeteer.js@@ -5,8 +5,10 @@ import portastic from 'portastic';import basicAuthParser from 'basic-auth-parser';import Promise from 'bluebird';import _ from 'underscore';+import sinon from 'sinon';import Apify from '../build/index';import { ENV_VARS } from '../build/constants';+import * as utils from '../build/utils';let prevEnvHeadless;@@ -130,7 +132,7 @@ describe('Apify.launchPuppeteer()', () => {.then(() => browser.close());});- it('userAgent option works', () => {+ it('supports userAgent option', () => {let browser;let page;const opts = {@@ -157,4 +159,33 @@ describe('Apify.launchPuppeteer()', () => {return browser.close();});});++ it('supports useChrome option', () => {+ const mock = sinon.mock(utils);+ mock.expects('getTypicalChromeExecutablePath').once();++ let browser;+ const opts = {+ useChrome: true,+ headless: true,+ };++ return Apify.launchPuppeteer(opts)+ .then((result) => {+ browser = result;+ })+ .then(() => {+ return browser.newPage();+ })+ .then((page) => {+ return page.content();+ })+ .then(() => {+ return browser.close();+ })+ .finally(() => {+ mock.verify();+ mock.restore();+ });+ });});diff --git a/test/puppeteer_crawler.js b/test/puppeteer_crawler.jsindex e80d47253..a90994408 100644--- a/test/puppeteer_crawler.js+++ b/test/puppeteer_crawler.js@@ -7,7 +7,7 @@ import * as Apify from '../build/index';chai.use(chaiAsPromised);-describe('puppeteer_crawler', () => {+describe('PuppeteerCrawler', () => {let prevEnvHeadless;before(() => {diff --git a/test/puppeteer_pool.js b/test/puppeteer_pool.jsindex b1f4b43c1..226847387 100644--- a/test/puppeteer_pool.js+++ b/test/puppeteer_pool.js@@ -9,7 +9,7 @@ chai.use(chaiAsPromised);const shortSleep = (millis = 25) => new Promise(resolve => setTimeout(resolve, millis));-describe('puppeteer_pool', () => {+describe('PuppeteerPool', () => {let prevEnvHeadless;before(() => {diff --git a/test/request_queue.js b/test/request_queue.jsindex 607da3e7a..219fc3d5b 100644--- a/test/request_queue.js+++ b/test/request_queue.js@@ -8,7 +8,7 @@ import { emptyLocalEmulationSubdir, LOCAL_EMULATION_DIR } from './_helper';chai.use(chaiAsPromised);-describe('request_queue', () => {+describe('RequestQueue', () => {before(() => apifyClient.setOptions({ token: 'xxx' }));after(() => apifyClient.setOptions({ token: undefined }));beforeEach(() => emptyLocalEmulationSubdir(LOCAL_EMULATION_SUBDIR));