Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Updating launchPuppeter()

Marek Trunkat99752beda0d3
diff --git a/package.json b/package.json
index 7f7285286..06e0b1de7 100644
--- a/package.json
+++ b/package.json
@@ -45,7 +45,6 @@
"bluebird": "^3.5.0",
"content-type": "^1.0.3",
"portastic": "^1.0.1",
- "puppeteer": "^0.12.0",
"rimraf": "^2.6.1",
"tmp": "^0.0.33",
"underscore": "^1.8.3"
@@ -76,6 +75,7 @@
"apify-jsdoc-template": "github:Apifier/apify-jsdoc-template"
},
"optionalDependencies": {
+ "puppeteer": "^0.12.0",
"selenium-webdriver": "^3.4.0"
}
}
diff --git a/src/puppeteer.js b/src/puppeteer.js
index 4190d6479..9e95ffec1 100644
--- a/src/puppeteer.js
+++ b/src/puppeteer.js
@@ -1,8 +1,14 @@
-import puppeteer from 'puppeteer';
import { checkParamOrThrow } from 'apify-client/build/utils';
import { parseUrl } from './utils';
import { ENV_VARS } from './constants';
+/* global process, require */
+
+/**
+ * Gets the default options for the browse() function, generated from current process environment
+ * variables. This is function to enable unit testing.
+ * @ignore
+ */
export const getDefaultPuppeteerOptions = () => ({
args: ['--no-sandbox'],
headless: !!process.env[ENV_VARS.HEADLESS],
@@ -18,6 +24,17 @@ export const getDefaultPuppeteerOptions = () => ({
* @return Returns a promise.
*/
export const launchPuppeteer = (opts = {}) => {
+ let puppeteer;
+
+ try {
+ // This is an optional dependency because it is quite large, only require it when used (ie. image with Chrome)
+ puppeteer = require('puppeteer'); // eslint-disable-line global-require
+ } catch (err) {
+ if (err.code === 'MODULE_NOT_FOUND') err.message = 'Cannot find module \'puppeteer\'. Did you choose the wrong docker image?';
+
+ throw err;
+ }
+
checkParamOrThrow(opts, 'opts', 'Object');
const { proxyUrl } = opts;
@@ -25,9 +42,10 @@ export const launchPuppeteer = (opts = {}) => {
if (proxyUrl) {
checkParamOrThrow(proxyUrl, 'opts.proxyUrl', 'String');
- const { host, port, protocol } = parseUrl(proxyUrl);
+ const { host, port, protocol, password } = parseUrl(proxyUrl);
if (!host || !port) throw new Error('Invalid "proxyUrl" option: the URL must contain hostname and port number.');
if (protocol !== 'http:' && protocol !== 'https:') throw new Error('Invalid "proxyUrl" option: protocol must be http or https.');
+ if (password) throw new Error('Invalid "proxyUrl" option: password is not currently supported.');
const proxyVarName = proxyUrl.startsWith('https') ? 'HTTPS_PROXY' : 'HTTP_PROXY';
opts.env = opts.env || {};
diff --git a/test/puppeteer.js b/test/puppeteer.js
index a0178046f..14cc9d8da 100644
--- a/test/puppeteer.js
+++ b/test/puppeteer.js
@@ -9,6 +9,7 @@ describe('Apify.launchPuppeteer()', () => {
expect(() => Apify.launchPuppeteer({ proxyUrl: 'invalidurl' })).to.throw(Error);
expect(() => Apify.launchPuppeteer({ proxyUrl: 'http://host-without-port' })).to.throw(Error);
expect(() => Apify.launchPuppeteer({ proxyUrl: 'invalid://somehost:1234' })).to.throw(Error);
+ expect(() => Apify.launchPuppeteer({ proxyUrl: 'http://username:[email protected]:30' })).to.throw(Error);
expect(() => Apify.launchPuppeteer({ proxyUrl: 'http://host-with-port.com:30' })).to.not.throw();
});