Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Added stealth class and integrate it to the puppeteer. Added tests for the hiding tricks.

Petr Patekbf1b4ed5b2ed
diff --git a/package.json b/package.json
index f3c0da3b1..1720e5bcd 100644
--- a/package.json
+++ b/package.json
@@ -82,6 +82,8 @@
"eslint-plugin-jsx-a11y": "^6.1.2",
"eslint-plugin-promise": "^4.0.1",
"eslint-plugin-react": "^7.11.1",
+ "fpcollect": "^1.0.4",
+ "fpscanner": "^0.1.5",
"jsdoc": "^3.5.5",
"jsdoc-export-default-interop": "^0.3.1",
"mocha": "^5.2.0",
diff --git a/src/puppeteer.js b/src/puppeteer.js
index e352c47dd..a37702a4f 100644
--- a/src/puppeteer.js
+++ b/src/puppeteer.js
@@ -7,6 +7,7 @@ import { DEFAULT_USER_AGENT } from './constants';
import { newPromise, getTypicalChromeExecutablePath, isAtHome } from './utils';
import { getApifyProxyUrl } from './actor';
import { registerBrowserForLiveView } from './puppeteer_live_view_server';
+import Stealth from './stealth/stealth';
/* global process, require */
@@ -139,7 +140,6 @@ const getPuppeteerOrThrow = () => {
throw err;
}
};
-
/**
* Launches headless Chrome using Puppeteer pre-configured to work within the Apify platform.
* The function has the same argument and the return value as `puppeteer.launch()`.
@@ -210,6 +210,18 @@ export const launchPuppeteer = (options = {}) => {
if (options.useApifyProxy && options.proxyUrl) throw new Error('Cannot combine "options.useApifyProxy" with "options.proxyUrl"!');
const puppeteer = getPuppeteerOrThrow();
+
+ // Add stealth
+ if (options.stealth) {
+ const prevLaunch = puppeteer.launch;
+ const newLaunch = async function (...args) {
+ const browser = await prevLaunch.bind(puppeteer)(...args);
+ const stealth = new Stealth(options.stealth);
+ return stealth.getStealthBrowser(browser);
+ };
+
+ puppeteer.launch = newLaunch.bind(puppeteer);
+ }
const optsCopy = Object.assign({}, options);
optsCopy.args = optsCopy.args || [];
diff --git a/src/puppeteer_pool.js b/src/puppeteer_pool.js
index 3f95b5842..50957ad45 100644
--- a/src/puppeteer_pool.js
+++ b/src/puppeteer_pool.js
@@ -7,7 +7,6 @@ import log from 'apify-shared/log';
import LinkedList from 'apify-shared/linked_list';
import rimraf from 'rimraf';
import { checkParamOrThrow } from 'apify-client/build/utils';
-import Stealth from './stealth';
import { launchPuppeteer } from './puppeteer';
import { addTimeoutToPromise } from './utils';
@@ -217,12 +216,8 @@ class PuppeteerPool {
opts.proxyUrl = this.proxyUrls[this.lastUsedProxyUrlIndex++ % this.proxyUrls.length];
}
- let browser = await launchPuppeteerFunction(opts);
+ const browser = await launchPuppeteerFunction(opts);
// Maybe apply hiding tricks
- if (opts.stealth) {
- const stealth = new Stealth(opts.stealth);
- browser = stealth.getStealthBrowser(browser);
- }
if (!browser || typeof browser.newPage !== 'function') {
// eslint-disable-next-line max-len
throw new Error("The custom 'launchPuppeteerFunction' passed to PuppeteerPool must return a promise resolving to Puppeteer's Browser instance.");
diff --git a/src/stealth.js b/src/stealth.js
deleted file mode 100644
index bc7723b22..000000000
--- a/src/stealth.js
+++ /dev/null
@@ -1,324 +0,0 @@
-class Stealth {
- constructor(options) {
- this.options = options;
- }
-
- getStealthBrowser(browser) {
- if (this.options) return browser;
- const modifiedBrowser = browser;
-
- const prevNewPage = browser.newPage;
-
- Object.defineProperty(modifiedBrowser, 'newPage', async (...args) => {
- const page = await prevNewPage(...args);
- await this._applyStealthTricks(page, this.options);
- return Promise.resolve(page);
- });
-
-
- return modifiedBrowser;
- }
-
- _applyStealthTricks(page) {
- const strigifyedFunctions = Object.keys(this.options)
- .filter(key => this.options[key])
- .map(key => ({ func: this.options[key].toString, key }));
- // TODO: ADD TO WINDOW object of browser
- return page.evaluateOnNewDocument((functionsToBeCalled) => {
- for (const { func } of functionsToBeCalled) {
- func.eval()();
- }
- }, strigifyedFunctions);
- }
-
- // TODO: REFACTOR A LOT, but first lets testIt
- addPlugins(window) {
- function mockPluginsAndMimeTypes() {
- /* global MimeType MimeTypeArray PluginArray */
-
- // Disguise custom functions as being native
- const makeFnsNative = (fns = []) => {
- const oldCall = Function.prototype.call;
- function call() {
- return oldCall.apply(this, arguments);
- }
- // eslint-disable-next-line
- Function.prototype.call = call
-
- const nativeToStringFunctionString = Error.toString().replace(
- /Error/g,
- 'toString',
- );
- const oldToString = Function.prototype.toString;
-
- function functionToString() {
- for (const fn of fns) {
- if (this === fn.ref) {
- return `function ${fn.name}() { [native code] }`;
- }
- }
-
- if (this === functionToString) {
- return nativeToStringFunctionString;
- }
- return oldCall.call(oldToString, this);
- }
- // eslint-disable-next-line
- Function.prototype.toString = functionToString
- };
-
- const mockedFns = [];
-
- const fakeData = {
- mimeTypes: [
- {
- type: 'application/pdf',
- suffixes: 'pdf',
- description: '',
- __pluginName: 'Chrome PDF Viewer',
- },
- {
- type: 'application/x-google-chrome-pdf',
- suffixes: 'pdf',
- description: 'Portable Document Format',
- __pluginName: 'Chrome PDF Plugin',
- },
- {
- type: 'application/x-nacl',
- suffixes: '',
- description: 'Native Client Executable',
- enabledPlugin: window.Plugin,
- __pluginName: 'Native Client',
- },
- {
- type: 'application/x-pnacl',
- suffixes: '',
- description: 'Portable Native Client Executable',
- __pluginName: 'Native Client',
- },
- ],
- plugins: [
- {
- name: 'Chrome PDF Plugin',
- filename: 'internal-pdf-viewer',
- description: 'Portable Document Format',
- },
- {
- name: 'Chrome PDF Viewer',
- filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai',
- description: '',
- },
- {
- name: 'Native Client',
- filename: 'internal-nacl-plugin',
- description: '',
- },
- ],
- fns: {
- namedItem: (instanceName) => {
- // Returns the Plugin/MimeType with the specified name.
- const fn = function (name) {
- if (!arguments.length) {
- throw new TypeError(
- `Failed to execute 'namedItem' on '${instanceName}': 1 argument required, but only 0 present.`,
- );
- }
- return this[name] || null;
- };
- mockedFns.push({ ref: fn, name: 'namedItem' });
- return fn;
- },
- item: (instanceName) => {
- // Returns the Plugin/MimeType at the specified index into the array.
- const fn = function (index) {
- if (!arguments.length) {
- throw new TypeError(
- `Failed to execute 'namedItem' on '${instanceName}': 1 argument required, but only 0 present.`,
- );
- }
- return this[index] || null;
- };
- mockedFns.push({ ref: fn, name: 'item' });
- return fn;
- },
- refresh: (instanceName) => {
- // Refreshes all plugins on the current page, optionally reloading documents.
- const fn = function () {
- return undefined;
- };
- mockedFns.push({ ref: fn, name: 'refresh' });
- return fn;
- },
- },
- };
- // Poor mans _.pluck
- const getSubset = (keys, obj) => keys.reduce((a, c) => ({ ...a, [c]: obj[c] }), {});
-
- function generateMimeTypeArray() {
- const arr = fakeData.mimeTypes
- .map(obj => getSubset(['type', 'suffixes', 'description'], obj))
- .map(obj => Object.setPrototypeOf(obj, MimeType.prototype));
- arr.forEach((obj) => {
- arr[obj.type] = obj;
- });
-
- // Mock functions
- arr.namedItem = fakeData.fns.namedItem('MimeTypeArray');
- arr.item = fakeData.fns.item('MimeTypeArray');
-
- return Object.setPrototypeOf(arr, MimeTypeArray.prototype);
- }
-
- const mimeTypeArray = generateMimeTypeArray();
- Object.defineProperty(window.navigator, 'mimeTypes', {
- get: () => mimeTypeArray,
- });
-
- function generatePluginArray() {
- const arr = fakeData.plugins
- .map(obj => getSubset(['name', 'filename', 'description'], obj))
- .map((obj) => {
- const mimes = fakeData.mimeTypes.filter(
- m => m.__pluginName === obj.name,
- );
- // Add mimetypes
- mimes.forEach((mime, index) => {
- window.navigator.mimeTypes[mime.type].enabledPlugin = obj;
- obj[mime.type] = window.navigator.mimeTypes[mime.type];
- obj[index] = window.navigator.mimeTypes[mime.type];
- });
- obj.length = mimes.length;
- return obj;
- })
- .map((obj) => {
- // Mock functions
- obj.namedItem = fakeData.fns.namedItem('Plugin');
- obj.item = fakeData.fns.item('Plugin');
- return obj;
- })
- .map(obj => Object.setPrototypeOf(obj, window.Plugin.prototype));
- arr.forEach((obj) => {
- arr[obj.name] = obj;
- });
-
- // Mock functions
- arr.namedItem = fakeData.fns.namedItem('PluginArray');
- arr.item = fakeData.fns.item('PluginArray');
- arr.refresh = fakeData.fns.refresh('PluginArray');
-
- return Object.setPrototypeOf(arr, PluginArray.prototype);
- }
-
- const pluginArray = generatePluginArray();
- Object.defineProperty(window.navigator, 'plugins', {
- get: () => pluginArray,
- });
-
- // Make mockedFns toString() representation resemble a native function
- makeFnsNative(mockedFns);
- }
- try {
- const isPluginArray = window.navigator.plugins instanceof PluginArray;
- const hasPlugins = isPluginArray && window.navigator.plugins.length > 0;
- if (isPluginArray && hasPlugins) {
- return; // nothing to do here
- }
- mockPluginsAndMimeTypes();
- } catch (err) {}
- }
-
- hideWebDriver(window) {
- const newProto = window.navigator.__proto__;
- delete newProto.webdriver;
- // eslint-disable-next-line
- navigator.__proto__ = newProto
- }
-
- // TODO: Proper runtime mocking
- mockRuntime(window) {
- window.navigator.chrome = {
- runtime: {},
- // etc.
- };
- }
-
- hackPermissions(window) {
- const originalQuery = window.navigator.permissions.query;
- // eslint-disable-next-line
- window.navigator.permissions.__proto__.query = parameters => parameters.name === 'notifications'
- ? Promise.resolve({ state: Notification.permission }) //eslint-disable-line
- : originalQuery(parameters);
-
- // Inspired by: https://github.com/ikarienator/phantomjs_hide_and_seek/blob/master/5.spoofFunctionBind.js
- const oldCall = Function.prototype.call;
- function call() {
- return oldCall.apply(this, arguments); //eslint-disable-line
- }
- // eslint-disable-next-line
- Function.prototype.call = call;
-
- const nativeToStringFunctionString = Error.toString().replace(
- /Error/g,
- 'toString',
- );
- const oldToString = Function.prototype.toString;
-
- function functionToString() {
- if (this === window.navigator.permissions.query) {
- return 'function query() { [native code] }';
- }
- if (this === functionToString) {
- return nativeToStringFunctionString;
- }
- return oldCall.call(oldToString, this);
- }
- // eslint-disable-next-line
- Function.prototype.toString = functionToString
- }
-
- addLanguage(window) {
- Object.defineProperty(window.navigator, 'languages', {
- get: () => ['en-US', 'en'],
- });
- }
-
- emulateWebGL() {
- try {
- /* global WebGLRenderingContext */
- const getParameter = WebGLRenderingContext.getParameter;
- WebGLRenderingContext.prototype.getParameter = function (parameter) {
- // UNMASKED_VENDOR_WEBGL
- if (parameter === 37445) {
- return 'Intel Inc.';
- }
- // UNMASKED_RENDERER_WEBGL
- if (parameter === 37446) {
- return 'Intel(R) Iris(TM) Plus Graphics 640';
- }
-
- return getParameter(parameter);
- };
- } catch (err) {
- console.log('COULD NOT ADD WBEGL');
- }
- }
-
- emulateWindowFrame(window) {
- try {
- if (window.outerWidth && window.outerHeight) {
- return; // nothing to do here
- }
- const windowFrame = 85; // probably OS and WM dependent
- window.outerWidth = window.innerWidth;
- window.outerHeight = window.innerHeight + windowFrame;
- } catch (err) {
- }
- }
-
- emulateConsoleDebug(window) {
- window.console.debug = () => {
- return null;
- };
- }
-}
-export default Stealth;
diff --git a/src/stealth/hiding_tricks.js b/src/stealth/hiding_tricks.js
new file mode 100644
index 000000000..cf782cf85
--- /dev/null
+++ b/src/stealth/hiding_tricks.js
@@ -0,0 +1,356 @@
+/* eslint no-undef: 0 */
+
+/* istanbul ignore next */
+const mockRuntime = () => {
+ window.navigator.chrome = {
+ runtime: {},
+ // etc.
+ };
+};
+
+/* istanbul ignore next */
+const hideWebDriver = () => {
+ const newProto = window.navigator.__proto__; // eslint-disable-line
+ delete newProto.webdriver;
+ // eslint-disable-next-line
+ navigator.__proto__ = newProto
+};
+
+/* istanbul ignore next */
+const hackPermissions = () => {
+ const originalQuery = window.navigator.permissions.query;
+ // eslint-disable-next-line
+ window.navigator.permissions.__proto__.query = parameters => parameters.name === 'notifications'
+ ? Promise.resolve({ state: Notification.permission }) //eslint-disable-line
+ : originalQuery(parameters);
+
+ // Inspired by: https://github.com/ikarienator/phantomjs_hide_and_seek/blob/master/5.spoofFunctionBind.js
+ const oldCall = Function.prototype.call;
+ function call() {
+ return oldCall.apply(this, arguments); //eslint-disable-line
+ }
+ // eslint-disable-next-line
+ Function.prototype.call = call;
+
+ const nativeToStringFunctionString = Error.toString().replace(
+ /Error/g,
+ 'toString',
+ );
+ const oldToString = Function.prototype.toString;
+
+ function functionToString() {
+ if (this === window.navigator.permissions.query) {
+ return 'function query() { [native code] }';
+ }
+ if (this === functionToString) {
+ return nativeToStringFunctionString;
+ }
+ return oldCall.call(oldToString, this);
+ }
+ // eslint-disable-next-line
+ Function.prototype.toString = functionToString
+};
+
+/* istanbul ignore next */
+const addLanguage = () => {
+ Object.defineProperty(window.navigator, 'languages', {
+ get: () => ['en-US', 'en'],
+ configurable: true,
+ });
+};
+
+/* istanbul ignore next */
+const emulateWebGL = () => {
+ try {
+ /* global WebGLRenderingContext */
+ const getParameterOld = WebGLRenderingContext.getParameter;
+ WebGLRenderingContext.prototype.getParameter = function (parameter) {
+ // UNMASKED_VENDOR_WEBGL
+ if (parameter === 37445) {
+ return 'Intel Inc.';
+ }
+ // UNMASKED_RENDERER_WEBGL
+ if (parameter === 37446) {
+ return 'Intel(R) Iris(TM) Plus Graphics 640';
+ }
+
+ return getParameterOld(parameter);
+ };
+ } catch (err) {
+ console.error('hiding_tricks: Could not emulate WebGL');
+ }
+};
+
+/* istanbul ignore next */
+const emulateWindowFrame = () => {
+ try {
+ if (window.outerWidth && window.outerHeight) {
+ return; // nothing to do here
+ }
+ const windowFrame = 85; // probably OS and WM dependent
+ window.outerWidth = window.innerWidth;
+ window.outerHeight = window.innerHeight + windowFrame;
+ } catch (err) {
+ console.error('hiding_tricks: Could not emulate window frame');
+ }
+};
+
+/* istanbul ignore next */
+const addPlugins = () => {
+ function mockPluginsAndMimeTypes() {
+ /* global MimeType MimeTypeArray PluginArray */
+
+ // Disguise custom functions as being native
+ const makeFnsNative = (fns = []) => {
+ const oldCall = Function.prototype.call;
+ function call() {
+ return oldCall.apply(this, arguments); // eslint-disable-line
+ }
+ // eslint-disable-next-line
+ Function.prototype.call = call;
+
+ const nativeToStringFunctionString = Error.toString().replace(
+ /Error/g,
+ 'toString',
+ );
+ const oldToString = Function.prototype.toString;
+
+ function functionToString() {
+ for (const fn of fns) {
+ if (this === fn.ref) {
+ return `function ${fn.name}() { [native code] }`;
+ }
+ }
+
+ if (this === functionToString) {
+ return nativeToStringFunctionString;
+ }
+ return oldCall.call(oldToString, this);
+ }
+ // eslint-disable-next-line
+ Function.prototype.toString = functionToString
+ };
+
+ const mockedFns = [];
+
+ const fakeData = {
+ mimeTypes: [
+ {
+ type: 'application/pdf',
+ suffixes: 'pdf',
+ description: '',
+ __pluginName: 'Chrome PDF Viewer',
+ },
+ {
+ type: 'application/x-google-chrome-pdf',
+ suffixes: 'pdf',
+ description: 'Portable Document Format',
+ __pluginName: 'Chrome PDF Plugin',
+ },
+ {
+ type: 'application/x-nacl',
+ suffixes: '',
+ description: 'Native Client Executable',
+ enabledPlugin: window.Plugin,
+ __pluginName: 'Native Client',
+ },
+ {
+ type: 'application/x-pnacl',
+ suffixes: '',
+ description: 'Portable Native Client Executable',
+ __pluginName: 'Native Client',
+ },
+ ],
+ plugins: [
+ {
+ name: 'Chrome PDF Plugin',
+ filename: 'internal-pdf-viewer',
+ description: 'Portable Document Format',
+ },
+ {
+ name: 'Chrome PDF Viewer',
+ filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai',
+ description: '',
+ },
+ {
+ name: 'Native Client',
+ filename: 'internal-nacl-plugin',
+ description: '',
+ },
+ ],
+ fns: {
+ namedItem: (instanceName) => {
+ // Returns the Plugin/MimeType with the specified name.
+ const fn = function (name) {
+ if (!arguments.length) {
+ throw new TypeError(
+ `Failed to execute 'namedItem' on '${instanceName}': 1 argument required, but only 0 present.`,
+ );
+ }
+ return this[name] || null;
+ };
+ mockedFns.push({ ref: fn, name: 'namedItem' });
+ return fn;
+ },
+ item: (instanceName) => {
+ // Returns the Plugin/MimeType at the specified index into the array.
+ const fn = function (index) {
+ if (!arguments.length) {
+ throw new TypeError(
+ `Failed to execute 'namedItem' on '${instanceName}': 1 argument required, but only 0 present.`,
+ );
+ }
+ return this[index] || null;
+ };
+ mockedFns.push({ ref: fn, name: 'item' });
+ return fn;
+ },
+ refresh: () => {
+ // Refreshes all plugins on the current page, optionally reloading documents.
+ const fn = function () {
+ return undefined;
+ };
+ mockedFns.push({ ref: fn, name: 'refresh' });
+ return fn;
+ },
+ },
+ };
+ // Poor mans _.pluck
+ const getSubset = (keys, obj) => keys.reduce((a, c) => ({ ...a, [c]: obj[c] }), {});
+
+ function generateMimeTypeArray() {
+ const arr = fakeData.mimeTypes
+ .map(obj => getSubset(['type', 'suffixes', 'description'], obj))
+ .map(obj => Object.setPrototypeOf(obj, MimeType.prototype));
+ arr.forEach((obj) => {
+ arr[obj.type] = obj;
+ });
+
+ // Mock functions
+ arr.namedItem = fakeData.fns.namedItem('MimeTypeArray');
+ arr.item = fakeData.fns.item('MimeTypeArray');
+
+ return Object.setPrototypeOf(arr, MimeTypeArray.prototype);
+ }
+
+ const mimeTypeArray = generateMimeTypeArray();
+ Object.defineProperty(window.navigator, 'mimeTypes', {
+ get: () => mimeTypeArray,
+ });
+
+ function generatePluginArray() {
+ const arr = fakeData.plugins
+ .map(obj => getSubset(['name', 'filename', 'description'], obj))
+ .map((obj) => {
+ const mimes = fakeData.mimeTypes.filter(
+ m => m.__pluginName === obj.name, // eslint-disable-line
+ );
+ // Add mimetypes
+ mimes.forEach((mime, index) => {
+ window.navigator.mimeTypes[mime.type].enabledPlugin = obj;
+ obj[mime.type] = window.navigator.mimeTypes[mime.type];
+ obj[index] = window.navigator.mimeTypes[mime.type];
+ });
+ obj.length = mimes.length;
+ return obj;
+ })
+ .map((obj) => {
+ // Mock functions
+ obj.namedItem = fakeData.fns.namedItem('Plugin');
+ obj.item = fakeData.fns.item('Plugin');
+ return obj;
+ })
+ .map(obj => Object.setPrototypeOf(obj, window.Plugin.prototype));
+ arr.forEach((obj) => {
+ arr[obj.name] = obj;
+ });
+
+ // Mock functions
+ arr.namedItem = fakeData.fns.namedItem('PluginArray');
+ arr.item = fakeData.fns.item('PluginArray');
+ arr.refresh = fakeData.fns.refresh('PluginArray');
+
+ return Object.setPrototypeOf(arr, PluginArray.prototype);
+ }
+
+ const pluginArray = generatePluginArray();
+ Object.defineProperty(window.navigator, 'plugins', {
+ get: () => pluginArray,
+ });
+
+ // Make mockedFns toString() representation resemble a native function
+ makeFnsNative(mockedFns);
+ }
+ const isPluginArray = window.navigator.plugins instanceof PluginArray;
+ const hasPlugins = isPluginArray && window.navigator.plugins.length > 0;
+ if (isPluginArray && hasPlugins) {
+ return; // nothing to do here
+ }
+ mockPluginsAndMimeTypes();
+};
+
+/* istanbul ignore next */
+const emulateConsoleDebug = () => {
+ window.console.debug = () => {
+ return null;
+ };
+};
+
+/* istanbul ignore next */
+// Should be mocked more properly - this one will bypass only some stupid tests
+const mockChrome = () => {
+ Object.defineProperty(window, 'chrome', {
+ value: {},
+ });
+};
+
+/* istanbul ignore next */
+// not sure if this hack does not broke iframe on websites... Should figure out how to test properly
+const mocksChromeInIframe = () => {
+ const oldCreate = document.createElement.bind(document);
+ const newCreate = (...args) => {
+ if (args[0] === 'iframe') {
+ const iframe = oldCreate(...args);
+ if (!iframe.contentWindow) {
+ Object.defineProperty(iframe, 'contentWindow', {
+ configurable: true,
+ value: { chrome: {} },
+ });
+ }
+ if (!iframe.contentWindow.chrome) {
+ Object.defineProperty(iframe.contentWindow, 'chrome', {
+ value: {},
+ configurable: true,
+ }); // TODO: ALSO MOCK BETTER
+ }
+ return iframe;
+ }
+ return oldCreate(...args);
+ };
+
+ newCreate.toString = () => 'function createElement() { [native code] }';
+
+ document.createElement = newCreate;
+};
+
+/* istanbul ignore next */
+const mockDeviceMemory = () => {
+ // TODO: Count from env according to - https://developer.mozilla.org/en-US/docs/Web/API/Navigator/deviceMemory
+ Object.defineProperty(navigator, 'deviceMemory', {
+ value: 8,
+ });
+};
+
+export default {
+ addPlugins,
+ emulateWindowFrame,
+ emulateWebGL,
+ emulateConsoleDebug,
+ addLanguage,
+ hideWebDriver,
+ hackPermissions,
+ mockRuntime,
+ mockChrome,
+ mocksChromeInIframe,
+ mockDeviceMemory,
+};
diff --git a/src/stealth/stealth.js b/src/stealth/stealth.js
new file mode 100644
index 000000000..2a549007d
--- /dev/null
+++ b/src/stealth/stealth.js
@@ -0,0 +1,60 @@
+import hidingTricks from './hiding_tricks';
+
+/**
+ * Class that handles hiding headless chrome.
+ * The main purpose of this class is to override newPage function and attached selected tricks.
+ */
+class Stealth {
+ /**
+ * constructor
+ * @param {Object} options
+ * @param {boolean} options.
+ */
+ constructor(options) {
+ this.options = Object.assign({}, options);
+ }
+
+ /**
+ * Modifies the newPage function in order to get.
+ * @param {Browser} browser - puppeteer browser instance
+ * @returns {Promise<Browser>} - Instance of Browser from puppeteer package
+ */
+ getStealthBrowser(browser) {
+ const modifiedBrowser = browser;
+
+ const prevNewPage = browser.newPage;
+ modifiedBrowser.newPage = async (...args) => {
+ const page = await prevNewPage.bind(browser)(...args);
+ await this._applyStealthTricks(page);
+ return Promise.resolve(page);
+ };
+
+
+ return Promise.resolve(modifiedBrowser);
+ }
+
+ /**
+ * Applies stealth tricks to the puppeteer page
+ * @param {Page} page
+ * @returns {Promise}
+ * @private
+ * @ignore
+ */
+ _applyStealthTricks(page) {
+ const functions = Object.keys(this.options)
+ .filter((key) => {
+ return this.options[key];
+ })
+ .map(key => hidingTricks[key].toString());
+
+ /* istanbul ignore next */
+ const addFunctions = (functionsArr) => {
+ for (const func of functionsArr) {
+ eval(func)(); // eslint-disable-line
+ }
+ };
+
+ return page.evaluateOnNewDocument(addFunctions, functions);
+ }
+}
+export default Stealth;
diff --git a/test/stealth/stealth.js b/test/stealth/stealth.js
new file mode 100644
index 000000000..f86a515de
--- /dev/null
+++ b/test/stealth/stealth.js
@@ -0,0 +1,246 @@
+import { expect } from 'chai';
+import scanner from 'fpscanner';
+
+import Apify from '../../build';
+
+const fingerPrintPath = require.resolve('fpcollect/dist/fpCollect.min.js');
+
+const getFingerPrint = async (page) => {
+ await Apify.utils.puppeteer.injectFile(page, fingerPrintPath);
+
+ return page.evaluate(() => fpCollect.generateFingerprint()); // eslint-disable-line
+};
+
+describe('Stealth - testing headless chrome hiding tricks', () => {
+ it('it adds plugins, mimeTypes and passes', async () => {
+ const browser = await Apify.launchPuppeteer({
+ stealth: {
+ addPlugins: true,
+ },
+ headless: true,
+ useChrome: true,
+ });
+
+ const page = await browser.newPage();
+ await page.goto('file://test_website.html');
+
+ const { plugins, mimeTypes } = await getFingerPrint(page);
+
+ expect(plugins.length).to.be.eql(3);
+ expect(mimeTypes.length).to.be.eql(4);
+
+ return browser.close();
+ });
+
+ it('it mocks chrome runtime', async () => {
+ const browser = await Apify.launchPuppeteer({
+ stealth: {
+ mockRuntime: true,
+ },
+ headless: true,
+ useChrome: true,
+ });
+
+ const page = await browser.newPage();
+ await page.goto('file://test_website.html');
+ const chrome = await page.evaluate(() => window.navigator.chrome); // eslint-disable-line
+
+ expect(chrome).to.be.an('object');
+ expect(chrome.runtime).to.be.empty; // eslint-disable-line
+
+ return browser.close();
+ });
+
+ it('it hides webDriver', async () => {
+ const browser = await Apify.launchPuppeteer({
+ stealth: {
+ hideWebDriver: true,
+ },
+ headless: true,
+ useChrome: true,
+ });
+
+ const page = await browser.newPage();
+ await page.goto('file://test_website.html');
+ const { webDriver } = await getFingerPrint(page);
+
+ expect(webDriver).to.be.eql(false);
+
+ return browser.close();
+ });
+
+ it('it hacks permissions', async () => {
+ const browser = await Apify.launchPuppeteer({
+ stealth: {
+ hackPermissions: true,
+ },
+ headless: true,
+ useChrome: true,
+ });
+
+ const page = await browser.newPage();
+ await page.goto('file://test_website.html');
+ const { permissions } = await getFingerPrint(page);
+
+ expect(permissions.state).to.be.eql('denied');
+
+ return browser.close();
+ });
+
+ it('it adds language to navigator', async () => {
+ const browser = await Apify.launchPuppeteer({
+ stealth: {
+ addLanguage: true,
+ },
+ headless: true,
+ useChrome: true,
+ });
+
+ const page = await browser.newPage();
+ await page.goto('file://test_website.html');
+ const { languages } = await getFingerPrint(page);
+
+ expect(languages).to.be.an('array');
+ expect(languages[0]).to.be.eql('en-US');
+
+ return browser.close();
+ });
+
+ it('it emulates WebGL', async () => {
+ const browser = await Apify.launchPuppeteer({
+ stealth: {
+ emulateWebGL: true,
+ },
+ headless: true,
+ useChrome: true,
+ });
+
+ const page = await browser.newPage();
+ await page.goto('file://test_website.html');
+ const { videoCard } = await getFingerPrint(page);
+
+ expect(videoCard[0]).to.be.eql('Intel Inc.');
+ expect(videoCard[1]).to.be.eql('Intel(R) Iris(TM) Plus Graphics 640');
+
+ return browser.close();
+ });
+
+ it('it emulates windowFrame', async () => {
+ const browser = await Apify.launchPuppeteer({
+ stealth: {
+ emulateWindowFrame: true,
+ },
+ headless: true,
+ useChrome: true,
+ });
+
+ const page = await browser.newPage();
+ await page.goto('file://test_website.html');
+ const { screen } = await getFingerPrint(page);
+
+ expect(screen.wOuterHeight).to.be.eql(screen.wInnerHeight + 85);
+
+ return browser.close();
+ });
+
+ it('it emulates console.debug', async () => {
+ const browser = await Apify.launchPuppeteer({
+ stealth: {
+ emulateConsoleDebug: true,
+ },
+ headless: true,
+ useChrome: true,
+ });
+
+ const page = await browser.newPage();
+ await page.goto('file://test_website.html');
+ const returnValue = await page.evaluate(() => console.debug('TEST'));
+
+ expect(returnValue).to.be.eql(null);
+
+ return browser.close();
+ });
+ it('it should mock window.chrome to plain object', async () => {
+ const browser = await Apify.launchPuppeteer({
+ stealth: {
+ mockChrome: true,
+ },
+ headless: true,
+ useChrome: true,
+ });
+
+ const page = await browser.newPage();
+ await page.goto('file://test_website.html');
+ const { hasChrome } = await getFingerPrint(page);
+
+ expect(hasChrome).to.be.eql(true);
+
+ return browser.close();
+ });
+
+ it('it should mock chrome when iframe is created', async () => {
+ const browser = await Apify.launchPuppeteer({
+ stealth: {
+ mocksChromeInIframe: true,
+ },
+ headless: true,
+ useChrome: true,
+ });
+
+ const page = await browser.newPage();
+ await page.goto('file://test_website.html');
+ const { iframeChrome } = await getFingerPrint(page);
+
+ expect(iframeChrome).to.be.eql('object');
+
+ return browser.close();
+ });
+
+ it('it should mock device memory', async () => {
+ const browser = await Apify.launchPuppeteer({
+ stealth: {
+ mockDeviceMemory: true,
+ },
+ headless: true,
+ useChrome: true,
+ });
+
+ const page = await browser.newPage();
+ await page.goto('file://test_website.html');
+ const { deviceMemory } = await getFingerPrint(page);
+
+ expect(deviceMemory).not.to.be.eql(0);
+
+ return browser.close();
+ });
+
+ it('it should bypass all of the known tests for browser fingerprinting', async () => {
+ const browser = await Apify.launchPuppeteer({
+ stealth: {
+ addPlugins: true,
+ emulateWindowFrame: true,
+ hideWebDriver: true,
+ mockRuntime: true,
+ emulateWebGL: true,
+ hackPermissions: true,
+ addLanguage: true,
+ emulateConsoleDebug: true,
+ mockChrome: true,
+ mocksChromeInIframe: true,
+ mockDeviceMemory: true,
+ },
+ headless: true,
+ useChrome: true,
+ });
+
+ const page = await browser.newPage();
+ await page.goto('file://test_website.html');
+ const fingerPrint = await getFingerPrint(page);
+ const testedFingerprint = scanner.analyseFingerprint(fingerPrint);
+ const failedChecks = Object.values(testedFingerprint).filter(val => val.consistent < 3);
+
+ expect(failedChecks.length).to.eql(0);
+
+ return browser.close();
+ });
+});
diff --git a/test/stealth/test_website.html b/test/stealth/test_website.html
new file mode 100644
index 000000000..23f895e68
--- /dev/null
+++ b/test/stealth/test_website.html
@@ -0,0 +1,15 @@
+<!doctype html>
+<html lang="">
+
+<head>
+ <meta charset="utf-8">
+ <title>Test website</title>
+ <meta name="description" content="">
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+</head>
+
+<body>
+<p>Hello world! This is HTML5 Boilerplate.</p>
+</body>
+
+</html>