Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Adding launchPuppeteer()

Marek Trunkatb2bba5c7b4fd
diff --git a/package.json b/package.json
index 04028b31b..7f7285286 100644
--- a/package.json
+++ b/package.json
@@ -45,6 +45,7 @@
"bluebird": "^3.5.0",
"content-type": "^1.0.3",
"portastic": "^1.0.1",
+ "puppeteer": "^0.12.0",
"rimraf": "^2.6.1",
"tmp": "^0.0.33",
"underscore": "^1.8.3"
diff --git a/src/index.js b/src/index.js
index 18869fbee..3c1434926 100644
--- a/src/index.js
+++ b/src/index.js
@@ -2,6 +2,7 @@ import EventEmitter from 'events';
import { setPromisesDependency, getPromisesDependency } from './utils';
import { main, readyFreddy, getEnv, getValue, setValue, apifyClient, call } from './actor';
+import { launchPuppeteer } from './puppeteer';
import { browse } from './browser';
/* globals module */
@@ -17,6 +18,8 @@ const Apify = {
setPromisesDependency,
getPromisesDependency,
browse,
+ launchWebdriver: browse,
+ launchPuppeteer,
client: apifyClient,
events: new EventEmitter(),
};
diff --git a/src/puppeteer.js b/src/puppeteer.js
new file mode 100644
index 000000000..d099b8fa0
--- /dev/null
+++ b/src/puppeteer.js
@@ -0,0 +1,38 @@
+import puppeteer from 'puppeteer';
+import { checkParamOrThrow } from 'apify-client/build/utils';
+import { parseUrl } from './utils';
+import { ENV_VARS } from './constants';
+
+export const PUPPETEER_DEFAULT_OPTS = {
+ args: ['--no-sandbox'],
+ headless: !!process.env[ENV_VARS.HEADLESS],
+ proxyUrl: null,
+};
+
+/**
+ * @memberof module:Apify
+ * @function
+ * @description Launches Puppeteer preconfigured to work with Apify platform.
+ * The result of the function is a result of puppeteer.launch().
+ * @param options Optional settings, their defaults are provided in the PUPPETEER_DEFAULT_OPTS constant.
+ * @return Returns a promise.
+ */
+export const launchPuppeteer = (opts = {}) => {
+ checkParamOrThrow(opts, 'opts', 'Object');
+
+ const { proxyUrl } = opts;
+
+ if (proxyUrl) {
+ checkParamOrThrow(proxyUrl, 'opts.proxyUrl', 'String');
+
+ const { host, port, protocol } = parseUrl(proxyUrl);
+ if (!host || !port) throw new Error('Invalid "proxyUrl" option: the URL must contain hostname and port number.');
+ if (protocol !== 'http:' && protocol !== 'https:') throw new Error('Invalid "proxyUrl" option: protocol must be http or https.');
+
+ const proxyVarName = proxyUrl.startsWith('https') ? 'HTTPS_PROXY' : 'HTTP_PROXY';
+ opts.env = opts.env || {};
+ opts.env[proxyVarName] = opts.proxyUrl;
+ }
+
+ return puppeteer.launch(Object.assign({}, PUPPETEER_DEFAULT_OPTS, opts));
+};
diff --git a/test/puppeteer.js b/test/puppeteer.js
new file mode 100644
index 000000000..d1f82ec0b
--- /dev/null
+++ b/test/puppeteer.js
@@ -0,0 +1,32 @@
+import { expect } from 'chai';
+import Apify from '../build/index';
+
+describe('Apify.launchPuppeteer()', function () {
+ it('throws on invalid args', () => {
+ expect(() => Apify.launchPuppeteer({ proxyUrl: 'invalidurl' })).to.throw(Error);
+ expect(() => Apify.launchPuppeteer({ proxyUrl: 'http://host-without-port' })).to.throw(Error);
+ expect(() => Apify.launchPuppeteer({ proxyUrl: 'invalid://somehost:1234' })).to.throw(Error);
+ expect(() => Apify.launchPuppeteer({ proxyUrl: 'http://host-with-port.com:30' })).to.not.throw();
+ });
+
+ it('opens https://www.example.com', () => {
+ let browser;
+ let page;
+
+ return Apify
+ .launchPuppeteer()
+ .then((createdBrowser) => {
+ browser = createdBrowser;
+
+ return browser.newPage();
+ })
+ .then((openedPage) => {
+ page = openedPage;
+
+ return page.goto('https://example.com');
+ })
+ .then(() => page.content())
+ .then(html => expect(html).to.include('<h1>Example Domain</h1>'))
+ .then(() => browser.close());
+ });
+});