Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Disabled selenium logging, added unit tests for userAgent setting,

Jan Curn39be7717bddf
diff --git a/src/browser.js b/src/browser.js
index 676d2bfc8..2f74b10ee 100644
--- a/src/browser.js
+++ b/src/browser.js
@@ -48,12 +48,11 @@ export class Browser {
}
// This is an optional dependency because it is quite large, only require it when used
- const { Capabilities, Builder, logging } = require('selenium-webdriver'); // eslint-disable-line global-require
+ const { Capabilities, Builder } = require('selenium-webdriver'); // eslint-disable-line global-require
const chrome = require('selenium-webdriver/chrome'); // eslint-disable-line global-require
- logging.installConsoleHandler();
- logging.getLogger('webdriver.http').setLevel(logging.Level.ALL);
-
+ // logging.installConsoleHandler();
+ // logging.getLogger('webdriver.http').setLevel(logging.Level.ALL);
// See https://github.com/SeleniumHQ/selenium/wiki/DesiredCapabilities for reference.
this.capabilities = new Capabilities();
@@ -69,7 +68,6 @@ export class Browser {
if (this.options.headless) {
this.chromeOptions.addArguments('--headless', '--disable-gpu', '--no-sandbox');
}
- // TODO: add unit test!
if (this.options.userAgent) {
this.chromeOptions.addArguments(`--user-agent=${this.options.userAgent}`);
}
@@ -88,7 +86,6 @@ export class Browser {
* @return Promise
*/
_initialize() {
- // logging.installConsoleHandler();
return newPromise()
.then(() => {
return this._setupProxy();
diff --git a/src/proxy_chain.js b/src/proxy_chain.js
index 5c415b039..4067ce457 100644
--- a/src/proxy_chain.js
+++ b/src/proxy_chain.js
@@ -1,7 +1,6 @@
import childProcess from 'child_process';
import fs from 'fs';
import path from 'path';
-import _ from 'underscore';
import tmp from 'tmp';
import portastic from 'portastic';
import Promise from 'bluebird';
diff --git a/test/browser.js b/test/browser.js
index 169856672..aac391cc5 100644
--- a/test/browser.js
+++ b/test/browser.js
@@ -170,7 +170,7 @@ describe('Apifier.browse()', function () {
// Need a large timeout to run unit tests on Travis CI
this.timeout(300 * 1000);
- it('throws with invalid params', () => {
+ it('throws on invalid args', () => {
assert.throws(() => {
Apifier.browse('http://www.blabla.bla', { proxyUrl: 'invalidurl' });
}, Error);
@@ -246,7 +246,7 @@ describe('Apifier.browse()', function () {
wasProxyCalled = false;
const opts = {
url: 'https://www.example.com',
- headless: false,
+ headless: true,
browserName: 'chrome',
proxyUrl: `http://${proxyAuth.username}:${proxyAuth.password}@127.0.0.1:${proxyPort}`,
};
@@ -270,6 +270,28 @@ describe('Apifier.browse()', function () {
return browser.close();
});
});
+
+ it('userAgent option works', () => {
+ let browser;
+ const opts = {
+ url: 'http://www.whatsmyua.info/',
+ headless: true,
+ browserName: 'chrome',
+ userAgent: 'MyUserAgent/1234',
+ };
+ return Apifier.browse(opts)
+ .then((result) => {
+ browser = result;
+ })
+ .then(() => {
+ expect(browser.constructor.name).to.eql('Browser');
+ return browser.webDriver.getPageSource();
+ })
+ .then((source) => {
+ expect(source).to.contain(`rawUa: ${opts.userAgent}`);
+ return browser.close();
+ });
+ });
});