Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Added Apify.launchWebDriver() function

Jan Curn214255bd194c
diff --git a/src/index.js b/src/index.js
index 86c7c4a36..874a8e85f 100644
--- a/src/index.js
+++ b/src/index.js
@@ -3,7 +3,7 @@ import EventEmitter from 'events';
import { setPromisesDependency, getPromisesDependency } from './utils';
import { main, readyFreddy, getEnv, getValue, setValue, apifyClient, call } from './actor';
import { launchPuppeteer } from './puppeteer';
-import { browse } from './browser';
+import { browse, launchWebDriver } from './webdriver';
/* globals module */
@@ -18,7 +18,7 @@ const Apify = {
setPromisesDependency,
getPromisesDependency,
browse,
- launchWebdriver: browse,
+ launchWebDriver,
launchPuppeteer,
client: apifyClient,
events: new EventEmitter(),
diff --git a/src/puppeteer.js b/src/puppeteer.js
index 0b71f701b..42e9e3f45 100644
--- a/src/puppeteer.js
+++ b/src/puppeteer.js
@@ -13,15 +13,16 @@ import { ENV_VARS } from './constants';
* See {@link https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#puppeteerlaunchoptions|Puppeteer documentation} for more details.</p>
* <p>The `launchPuppeteer()` function alters the following Puppeteer options:
* <ul>
- * <li>Passes the value of the `APIFY_HEADLESS` environment variable to the `headless` option,
+ * <li>Passes the setting from the `APIFY_HEADLESS` environment variable to the `headless` option,
* unless it was already defined by the caller.</li>
* <li>Takes the `proxyUrl` option and adds it to `env` under the `HTTPS_PROXY` or `HTTP_PROXY` key.</li>
* <li>Adds `--no-sandbox` to `args` to enable running headless Chrome in a Docker container on the Actor platform.</li>
* </ul>
* </p>
- * @param options Optional settings passed to `puppeteer.launch()`.
- * Additionally, the options can contain the `proxyUrl` property to specify a proxy server
- * (e.g. `http://example.com:1234`).
+ * @param {Object} [opts] Optional settings passed to `puppeteer.launch()`. Additionally the object can contain the following fields:
+ * @param {String} [opts.proxyUrl] - URL to a proxy server. Currently only `http://` scheme is supported.
+ * Port number must be specified. Proxy authentication is not supported yet. For example, `http://example.com:1234`.
+ * @param {String} [opts.userAgent] - Default User-Agent for the browser.
* @returns {Promise} Promise object returned by `puppeteer.launch()`
*/
export const launchPuppeteer = (opts = {}) => {
@@ -58,7 +59,8 @@ export const launchPuppeteer = (opts = {}) => {
opts.args = opts.args || [];
opts.args.push('--no-sandbox');
- if (opts.headless === undefined || opts.headless === null) opts.headless = !!process.env[ENV_VARS.HEADLESS];
+ if (opts.userAgent) opts.args.push(`--user-agent=${opts.userAgent}`);
+ if (opts.headless === undefined || opts.headless === null) opts.headless = process.env[ENV_VARS.HEADLESS] === '1';
return puppeteer.launch(opts);
};
diff --git a/src/browser.js b/src/webdriver.js
similarity index 88%
rename from src/browser.js
rename to src/webdriver.js
index bb17e31a1..26a945c45 100644
--- a/src/browser.js
+++ b/src/webdriver.js
@@ -289,6 +289,8 @@ await browser.close();
```
*/
+// TODO: browse() is only kept for backwards compatibility, get rid of it after no acts are using it!
+
/**
* @memberof module:Apify
* @function
@@ -318,6 +320,46 @@ export const browse = (url, options, callback) => {
};
+/**
+ * @memberof module:Apify
+ * @function
+ * @description Opens a new instance of Chrome web browser
+ * controlled by Selenium WebDriver created using Builder command
+ * (see http://seleniumhq.github.io/selenium/docs/api/javascript/module/selenium-webdriver/index_exports_Builder.html).
+ * If the `APIFY_HEADLESS` environment variable is set to `1`, the function
+ * runs the browser in headless mode.
+ * The result of the function is the new instance of the Selenium WebDriver class.
+ * @param {Object} [opts] Optional settings passed to `puppeteer.launch()`. Additionally the object can contain the following fields:
+ * @param {String} [opts.proxyUrl] - URL to a proxy server. Currently only `http://` scheme is supported.
+ * Port number must be specified. For example, `http://example.com:1234`.
+ * @param {String} [opts.userAgent] - Default User-Agent for the browser.
+ * @param {Function} callback Optional callback.
+ * @returns {Promise} Returns a promise if no callback was provided, otherwise the return value is not defined.
+ */
+export const launchWebDriver = (opts, callback) => {
+ const args = processBrowseArgs(undefined, opts, callback);
+ const browser = new Browser(args.options);
+
+ // NOTE: eventually get rid of the Brower class
+
+ const prevQuit = browser.webDriver.quit;
+ browser.webDriver.quit = () => {
+ if (browser.proxyChain) {
+ browser.proxyChain.shutdown();
+ browser.proxyChain = null;
+ }
+ return prevQuit();
+ };
+
+ const promise = browser._initialize()
+ .then(() => {
+ return browser.webDriver;
+ });
+
+ return nodeifyPromise(promise, args.callback);
+};
+
+
// /**
// * Launches a debugging instance of Chrome on port 9222, without Selenium.
// * This code is kept here for legacy reasons, it's not used.
diff --git a/test/puppeteer.js b/test/puppeteer.js
index 14cc9d8da..0790eaa4e 100644
--- a/test/puppeteer.js
+++ b/test/puppeteer.js
@@ -2,7 +2,7 @@ import { expect } from 'chai';
import Apify from '../build/index';
import { ENV_VARS } from '../build/constants';
-process.env[ENV_VARS.HEADLESS] = true;
+process.env[ENV_VARS.HEADLESS] = '1';
describe('Apify.launchPuppeteer()', () => {
it('throws on invalid args', () => {