Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Added useChrome option to launchPuppeteer() + unit test, renamed unit… (#45)

Jan Čurnb71ab739a2ff
diff --git a/.travis.yml b/.travis.yml
index 61b0d4f03..88a2a21ff 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,8 +9,6 @@ before_install:
- sudo unzip /tmp/chromedriver.zip -d /tmp
- sudo rm /tmp/chromedriver.zip
- sudo mv /tmp/chromedriver /usr/bin/
- - # Travis' VM has preinstalled google-chrome stable, which has no headless mode!
- - sudo mv -f /usr/bin/google-chrome-beta /usr/bin/google-chrome | tee
- # Print versions to help debugging
- google-chrome --version
- chromedriver --version
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0198824f0..52f184e4f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,7 @@
+XXX / 2018-03-20
+===================
+- Added `useChrome` flag to `launchPuppeteer()` function
+
0.5.14 / 2018-03-20
===================
- Removed again the --disable-dev-shm-usage flag when launching headless Chrome,
diff --git a/examples/example_autoscaled_pool.js b/examples/example_autoscaled_pool.js
index c4d4b9168..4b9461acc 100644
--- a/examples/example_autoscaled_pool.js
+++ b/examples/example_autoscaled_pool.js
@@ -1,4 +1,4 @@
-const Apify = require('../build');
+const Apify = require('apify');
Apify.main(async () => {
let counter = 0;
diff --git a/examples/example_basic_crawler.js b/examples/example_basic_crawler.js
index 88d9e8839..d0eb88113 100644
--- a/examples/example_basic_crawler.js
+++ b/examples/example_basic_crawler.js
@@ -1,4 +1,4 @@
-const Apify = require('../build');
+const Apify = require('apify');
const rp = require('request-promise');
Apify.main(async () => {
diff --git a/examples/example_puppeteer_crawler.js b/examples/example_puppeteer_crawler.js
index 11f52c041..011efcf40 100644
--- a/examples/example_puppeteer_crawler.js
+++ b/examples/example_puppeteer_crawler.js
@@ -1,4 +1,4 @@
-const Apify = require('../build');
+const Apify = require('apify');
Apify.main(async () => {
// Create a request list.
diff --git a/package.json b/package.json
index 7909025f8..d7b35e9f5 100644
--- a/package.json
+++ b/package.json
@@ -44,14 +44,14 @@
},
"dependencies": {
"apify-client": "^0.2.3",
- "apify-shared": "0.0.16",
+ "apify-shared": "0.0.22",
"bluebird": "^3.5.0",
"content-type": "^1.0.3",
"fs-extra": "^5.0.0",
"proxy-chain": "^0.1.2",
"request-promise": "^4.2.2",
"underscore": "^1.8.3",
- "ws": "^4.1.0"
+ "ws": "^5.1.0"
},
"devDependencies": {
"apify-jsdoc-template": "github:apifytech/apify-jsdoc-template",
@@ -82,7 +82,7 @@
"sinon": "^4.1.2",
"sinon-stub-promise": "^4.0.0",
"tmp": "^0.0.33",
- "why-is-node-running": "^1.2.3"
+ "why-is-node-running": "^2.0.2"
},
"optionalDependencies": {
"puppeteer": "^1.2.0",
diff --git a/src/constants.js b/src/constants.js
index 68be65dbc..ebb894beb 100644
--- a/src/constants.js
+++ b/src/constants.js
@@ -39,6 +39,7 @@ consts.ENV_VARS = {
MEMORY_MBYTES: 'APIFY_MEMORY_MBYTES',
LOG_LEVEL: 'APIFY_LOG_LEVEL',
ACTOR_EVENTS_WS_URL: 'APIFY_ACTOR_EVENTS_WS_URL',
+ CHROME_EXECUTABLE_PATH: 'APIFY_CHROME_EXECUTABLE_PATH',
};
consts.KEY_VALUE_STORE_KEYS = {
diff --git a/src/puppeteer.js b/src/puppeteer.js
index 5c96ef562..06175d642 100644
--- a/src/puppeteer.js
+++ b/src/puppeteer.js
@@ -1,7 +1,7 @@
import { checkParamOrThrow } from 'apify-client/build/utils';
import { anonymizeProxy, closeAnonymizedProxy } from 'proxy-chain';
import { ENV_VARS, DEFAULT_USER_AGENT } from './constants';
-import { newPromise } from './utils';
+import { newPromise, getTypicalChromeExecutablePath } from './utils';
/* global process, require */
@@ -39,6 +39,11 @@ import { newPromise } from './utils';
* For example, `http://bob:[email protected]:1234`.
* @param {String} [opts.userAgent] Default User-Agent for the browser.
* If not provided, the function sets it to a reasonable default.
+ * @param {String} [opts.useChrome=false] If true-ish value and `opts.executablePath` is not set,
+ * Puppeteer will launch full Chrome available on the machine rather than the bundled Chromium.
+ * The path to Chrome executable is taken from the `APIFY_CHROME_EXECUTABLE_PATH` environment variable if provided,
+ * or defaults to the typical Google Chrome executable location specific for the operating system.
+ * By default, this option is `false`.
* @returns {Promise} Promise object that resolves to Puppeteer's `Browser` instance.
*
* @memberof module:Apify
@@ -69,6 +74,9 @@ export const launchPuppeteer = (opts) => {
if (opts.headless === undefined || opts.headless === null) {
opts.headless = process.env[ENV_VARS.HEADLESS] === '1' && process.env[ENV_VARS.XVFB] !== '1';
}
+ if (opts.useChrome && (opts.executablePath === undefined || opts.executablePath === null)) {
+ opts.executablePath = process.env[ENV_VARS.CHROME_EXECUTABLE_PATH] || getTypicalChromeExecutablePath();
+ }
let anonymizedProxyUrl;
let promise;
diff --git a/src/utils.js b/src/utils.js
index 4a4347536..cb5ae5251 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -201,3 +201,15 @@ export const isProduction = () => process.env.NODE_ENV !== 'production';
* @ignore
*/
export const ensureDirExists = path => ensureDirPromised(path);
+
+/**
+ * Gets a typical path to Chrome executable, depending on the current operating system.
+ * @returns {string}
+ */
+export const getTypicalChromeExecutablePath = () => {
+ switch (os.platform()) {
+ case 'darwin': return '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome';
+ case 'win32': return 'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe';
+ default: return 'google-chrome';
+ }
+};
diff --git a/test/autoscaled_pool.js b/test/autoscaled_pool.js
index e1993be34..b97cb5502 100644
--- a/test/autoscaled_pool.js
+++ b/test/autoscaled_pool.js
@@ -12,7 +12,7 @@ import * as utils from '../build/utils';
chai.use(chaiAsPromised);
const toBytes = x => x * 1024 * 1024;
-describe('autoscaled_pool', () => {
+describe('AutoscaledPool', () => {
it('should work with concurrency 1', async () => {
const range = _.range(0, 10);
const result = [];
diff --git a/test/basic_crawler.js b/test/basic_crawler.js
index fd81d5558..12511ec0f 100644
--- a/test/basic_crawler.js
+++ b/test/basic_crawler.js
@@ -7,7 +7,7 @@ import * as Apify from '../build/index';
chai.use(chaiAsPromised);
-describe('basic_crawler', () => {
+describe('BasicCrawler', () => {
it('runs in parallel thru all the requests', async () => {
const startedAt = Date.now();
const sources = _.range(0, 500).map(index => ({ url: `https://example.com/${index}` }));
diff --git a/test/key_value_store.js b/test/key_value_store.js
index 0de475935..0769e3fff 100644
--- a/test/key_value_store.js
+++ b/test/key_value_store.js
@@ -10,7 +10,7 @@ import { LOCAL_EMULATION_DIR, emptyLocalEmulationSubdir, expectNotLocalEmulation
chai.use(chaiAsPromised);
-describe('key_value_store', () => {
+describe('KeyValueStore', () => {
before(() => apifyClient.setOptions({ token: 'xxx' }));
after(() => apifyClient.setOptions({ token: undefined }));
beforeEach(() => emptyLocalEmulationSubdir(LOCAL_EMULATION_SUBDIR));
diff --git a/test/puppeteer.js b/test/puppeteer.js
index e71f44ff9..19cb3436d 100644
--- a/test/puppeteer.js
+++ b/test/puppeteer.js
@@ -5,8 +5,10 @@ import portastic from 'portastic';
import basicAuthParser from 'basic-auth-parser';
import Promise from 'bluebird';
import _ from 'underscore';
+import sinon from 'sinon';
import Apify from '../build/index';
import { ENV_VARS } from '../build/constants';
+import * as utils from '../build/utils';
let prevEnvHeadless;
@@ -130,7 +132,7 @@ describe('Apify.launchPuppeteer()', () => {
.then(() => browser.close());
});
- it('userAgent option works', () => {
+ it('supports userAgent option', () => {
let browser;
let page;
const opts = {
@@ -157,4 +159,33 @@ describe('Apify.launchPuppeteer()', () => {
return browser.close();
});
});
+
+ it('supports useChrome option', () => {
+ const mock = sinon.mock(utils);
+ mock.expects('getTypicalChromeExecutablePath').once();
+
+ let browser;
+ const opts = {
+ useChrome: true,
+ headless: true,
+ };
+
+ return Apify.launchPuppeteer(opts)
+ .then((result) => {
+ browser = result;
+ })
+ .then(() => {
+ return browser.newPage();
+ })
+ .then((page) => {
+ return page.content();
+ })
+ .then(() => {
+ return browser.close();
+ })
+ .finally(() => {
+ mock.verify();
+ mock.restore();
+ });
+ });
});
diff --git a/test/puppeteer_crawler.js b/test/puppeteer_crawler.js
index e80d47253..a90994408 100644
--- a/test/puppeteer_crawler.js
+++ b/test/puppeteer_crawler.js
@@ -7,7 +7,7 @@ import * as Apify from '../build/index';
chai.use(chaiAsPromised);
-describe('puppeteer_crawler', () => {
+describe('PuppeteerCrawler', () => {
let prevEnvHeadless;
before(() => {
diff --git a/test/puppeteer_pool.js b/test/puppeteer_pool.js
index b1f4b43c1..226847387 100644
--- a/test/puppeteer_pool.js
+++ b/test/puppeteer_pool.js
@@ -9,7 +9,7 @@ chai.use(chaiAsPromised);
const shortSleep = (millis = 25) => new Promise(resolve => setTimeout(resolve, millis));
-describe('puppeteer_pool', () => {
+describe('PuppeteerPool', () => {
let prevEnvHeadless;
before(() => {
diff --git a/test/request_queue.js b/test/request_queue.js
index 607da3e7a..219fc3d5b 100644
--- a/test/request_queue.js
+++ b/test/request_queue.js
@@ -8,7 +8,7 @@ import { emptyLocalEmulationSubdir, LOCAL_EMULATION_DIR } from './_helper';
chai.use(chaiAsPromised);
-describe('request_queue', () => {
+describe('RequestQueue', () => {
before(() => apifyClient.setOptions({ token: 'xxx' }));
after(() => apifyClient.setOptions({ token: undefined }));
beforeEach(() => emptyLocalEmulationSubdir(LOCAL_EMULATION_SUBDIR));