Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Fixes in Apify.launchWebDriver(), added unit tests

Jan Curn9b26ccce412a
diff --git a/src/puppeteer.js b/src/puppeteer.js
index 42e9e3f45..30c886ddf 100644
--- a/src/puppeteer.js
+++ b/src/puppeteer.js
@@ -10,15 +10,25 @@ import { ENV_VARS } from './constants';
* @function
* @description <p>Launches headless Chrome using Puppeteer pre-configured to work with the Apify Actor platform.
* The function has the same argument and return value as `puppeteer.launch()`.
- * See {@link https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#puppeteerlaunchoptions|Puppeteer documentation} for more details.</p>
+ * See {@link https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#puppeteerlaunchoptions|Puppeteer documentation} for more details.
+ * </p>
* <p>The `launchPuppeteer()` function alters the following Puppeteer options:
* <ul>
* <li>Passes the setting from the `APIFY_HEADLESS` environment variable to the `headless` option,
- * unless it was already defined by the caller.</li>
+ * unless it was already defined by the caller. Note that this environment variable is automatically set to `1`
+ * in acts running on the Apify Actor cloud platform.</li>
* <li>Takes the `proxyUrl` option and adds it to `env` under the `HTTPS_PROXY` or `HTTP_PROXY` key.</li>
* <li>Adds `--no-sandbox` to `args` to enable running headless Chrome in a Docker container on the Actor platform.</li>
* </ul>
* </p>
+ * <p>
+ * To use this function, you need to have the
+ * {@link https://www.npmjs.com/package/puppeteer|puppeteer}
+ * NPM package installed in your project.
+ * For example, you can use the `apify/actor-node-puppeteer` base Docker image for your act - see
+ * {@link https://www.apify.com/docs/actor#base-images|documentation}
+ * for more details.
+ * </p>
* @param {Object} [opts] Optional settings passed to `puppeteer.launch()`. Additionally the object can contain the following fields:
* @param {String} [opts.proxyUrl] - URL to a proxy server. Currently only `http://` scheme is supported.
* Port number must be specified. Proxy authentication is not supported yet. For example, `http://example.com:1234`.
diff --git a/src/webdriver.js b/src/webdriver.js
index 26a945c45..fc01f8d47 100644
--- a/src/webdriver.js
+++ b/src/webdriver.js
@@ -323,12 +323,24 @@ export const browse = (url, options, callback) => {
/**
* @memberof module:Apify
* @function
- * @description Opens a new instance of Chrome web browser
- * controlled by Selenium WebDriver created using Builder command
- * (see http://seleniumhq.github.io/selenium/docs/api/javascript/module/selenium-webdriver/index_exports_Builder.html).
+ * @description <p>Opens a new instance of Chrome web browser
+ * controlled by {@link http://www.seleniumhq.org/projects/webdriver/|Selenium WebDriver}.
+ * The result of the function is the new instance of the
+ * {@link http://seleniumhq.github.io/selenium/docs/api/javascript/module/selenium-webdriver/index_exports_WebDriver.html|WebDriver}
+ * class.
+ * </p>
+ * <p>
* If the `APIFY_HEADLESS` environment variable is set to `1`, the function
- * runs the browser in headless mode.
- * The result of the function is the new instance of the Selenium WebDriver class.
+ * runs the web browser in headless mode. Note that this environment variable is automatically set to `1` when
+ * in acts running on the Apify Actor cloud platform.
+ * </p>
+ * <p>
+ * To use this function, you need to have Google Chrome and
+ * {@link https://sites.google.com/a/chromium.org/chromedriver/|ChromeDriver} installed in your environment.
+ * For example, you can use the `apify/actor-node-chrome` base Docker image for your act - see
+ * {@link https://www.apify.com/docs/actor#base-images|documentation}
+ * for more details.
+ * </p>
* @param {Object} [opts] Optional settings passed to `puppeteer.launch()`. Additionally the object can contain the following fields:
* @param {String} [opts.proxyUrl] - URL to a proxy server. Currently only `http://` scheme is supported.
* Port number must be specified. For example, `http://example.com:1234`.
@@ -340,19 +352,18 @@ export const launchWebDriver = (opts, callback) => {
const args = processBrowseArgs(undefined, opts, callback);
const browser = new Browser(args.options);
- // NOTE: eventually get rid of the Brower class
-
- const prevQuit = browser.webDriver.quit;
- browser.webDriver.quit = () => {
- if (browser.proxyChain) {
- browser.proxyChain.shutdown();
- browser.proxyChain = null;
- }
- return prevQuit();
- };
-
+ // NOTE: eventually get rid of the Browser class
const promise = browser._initialize()
.then(() => {
+ // TODO: for some reason this doesn't work, the proxy chain will never shut down!!
+ // we'll need to find a way to fix this!
+ browser.webDriver.onQuit_ = () => {
+ if (browser.proxyChain) {
+ browser.proxyChain.shutdown();
+ browser.proxyChain = null;
+ }
+ };
+
return browser.webDriver;
});
diff --git a/test/browser.js b/test/webdriver.js
similarity index 79%
rename from test/browser.js
rename to test/webdriver.js
index e10ad1418..7f52fbede 100644
--- a/test/browser.js
+++ b/test/webdriver.js
@@ -167,6 +167,130 @@ describe('processBrowseArgs()', () => {
});
+describe('Apify.launchWebDriver()', function () {
+ // Need a large timeout to run unit tests on Travis CI
+ this.timeout(300 * 1000);
+
+ it('throws on invalid args', () => {
+ assert.throws(() => {
+ Apify.launchWebDriver({ proxyUrl: 'invalidurl' });
+ }, Error);
+ assert.throws(() => {
+ Apify.launchWebDriver({ proxyUrl: 'http://host-without-port' });
+ }, Error);
+ assert.throws(() => {
+ Apify.launchWebDriver({ proxyUrl: 'invalid://somehost:1234' });
+ }, Error);
+ });
+
+ it('starts with no args', () => {
+ process.env.APIFY_HEADLESS = '1';
+ let webDriver;
+ return Apify.launchWebDriver()
+ .then((res) => {
+ webDriver = res;
+ expect(webDriver.constructor.name).to.eql('Driver');
+ return webDriver.quit();
+ });
+ });
+
+ it('opens https://www.example.com in headless mode', () => {
+ delete process.env.APIFY_HEADLESS;
+ let webDriver;
+ return Apify.launchWebDriver({ headless: true })
+ .then((res) => {
+ webDriver = res;
+ return webDriver.get('https://www.example.com');
+ })
+ .then(() => {
+ expect(webDriver.constructor.name).to.eql('Driver');
+ return webDriver.getCurrentUrl();
+ })
+ .then((url) => {
+ expect(url).to.eql('https://www.example.com/');
+ return webDriver.quit();
+ });
+ });
+
+ it('works with empty options and callback', () => {
+ let webDriver;
+ return new Promise((resolve, reject) => {
+ try {
+ process.env.APIFY_HEADLESS = '1';
+ const retVal = Apify.launchWebDriver({}, (err, result) => {
+ if (err) return reject(err);
+ webDriver = result;
+ try {
+ expect(webDriver.constructor.name).to.eql('Driver');
+ webDriver.getCurrentUrl()
+ .then(resolve)
+ .catch(reject);
+ } catch (e) {
+ reject(e);
+ }
+ });
+ assert(!retVal, 'Apify.browse() with callback should return false-ish value');
+ } catch (e) {
+ reject(e);
+ }
+ }).then(() => {
+ return webDriver.quit();
+ });
+ });
+
+ it('works with proxy server', () => {
+ let webDriver;
+ wasProxyCalled = false;
+ const opts = {
+ headless: true,
+ proxyUrl: `http://${proxyAuth.username}:${proxyAuth.password}@127.0.0.1:${proxyPort}`,
+ };
+ return Apify.launchWebDriver(opts)
+ .then((result) => {
+ webDriver = result;
+ return webDriver.get('https://www.example.com');
+ })
+ .then(() => {
+ // return webDriver.sleep(300 * 1000);
+ })
+ .then(() => {
+ return webDriver.getAllWindowHandles();
+ })
+ .then(() => {
+ expect(webDriver.constructor.name).to.eql('Driver');
+ return webDriver.getCurrentUrl();
+ })
+ .then((url) => {
+ expect(wasProxyCalled).to.equal(true);
+ expect(url).to.eql('https://www.example.com/');
+ return webDriver.quit();
+ });
+ });
+
+ it('userAgent option works', () => {
+ let webDriver;
+ const opts = {
+ headless: true,
+ userAgent: 'MyUserAgent/1234',
+ };
+ return Apify.launchWebDriver(opts)
+ .then((result) => {
+ webDriver = result;
+ // TODO: this is not reliable, we should use our own testing page
+ return webDriver.get('http://www.whoishostingthis.com/tools/user-agent');
+ })
+ .then(() => {
+ expect(webDriver.constructor.name).to.eql('Driver');
+ return webDriver.getPageSource();
+ })
+ .then((source) => {
+ expect(source).to.contain(opts.userAgent);
+ return webDriver.quit();
+ });
+ });
+});
+
+
describe('Apify.browse()', function () {
// Need a large timeout to run unit tests on Travis CI
this.timeout(300 * 1000);