Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

improved comments and puppeteer initialization.

Petr Patek76da5ff65c3e
diff --git a/package.json b/package.json
index 27dae7931..1ab034a61 100644
--- a/package.json
+++ b/package.json
@@ -85,6 +85,8 @@
"eslint-plugin-jsx-a11y": "^6.1.2",
"eslint-plugin-promise": "^4.0.1",
"eslint-plugin-react": "^7.11.1",
+ "fpcollect": "^1.0.4",
+ "fpscanner": "^0.1.5",
"jsdoc": "^3.5.5",
"jsdoc-export-default-interop": "^0.3.1",
"mocha": "^5.2.0",
diff --git a/src/puppeteer.js b/src/puppeteer.js
index 589b421a3..c5d38303c 100644
--- a/src/puppeteer.js
+++ b/src/puppeteer.js
@@ -6,7 +6,7 @@ import { ENV_VARS } from 'apify-shared/consts';
import { DEFAULT_USER_AGENT } from './constants';
import { getTypicalChromeExecutablePath, isAtHome } from './utils';
import { getApifyProxyUrl } from './actor';
-import Stealth from './stealth/stealth';
+import applyStealthFromBrowser from './stealth/stealth';
/* global process, require */
@@ -190,6 +190,7 @@ export const launchPuppeteer = async (options = {}) => {
checkParamOrThrow(options.proxyUrl, 'options.proxyUrl', 'Maybe String');
checkParamOrThrow(options.useApifyProxy, 'options.useApifyProxy', 'Maybe Boolean');
checkParamOrThrow(options.puppeteerModule, 'options.puppeteerModule', 'Maybe String');
+ checkParamOrThrow(options.stealth, 'options.stealth', 'Maybe Object');
if (options.useApifyProxy && options.proxyUrl) throw new Error('Cannot combine "options.useApifyProxy" with "options.proxyUrl"!');
if (options.liveView || options.liveViewOptions) {
log.deprecated('Live view is no longer available in Apify.launchPuppeteer() and launchPuppeteerOptions. '
@@ -198,17 +199,7 @@ export const launchPuppeteer = async (options = {}) => {
}
const puppeteer = getPuppeteerOrThrow(options.puppeteerModule);
- // Add stealth
- if (options.stealth) {
- const prevLaunch = puppeteer.launch;
- const newLaunch = async function (...args) {
- const browser = await prevLaunch.bind(puppeteer)(...args);
- const stealth = new Stealth(options.stealth);
- return stealth.getStealthBrowser(browser);
- };
-
- puppeteer.launch = newLaunch.bind(puppeteer);
- }
+
const optsCopy = Object.assign({}, options);
optsCopy.args = optsCopy.args || [];
@@ -243,7 +234,17 @@ export const launchPuppeteer = async (options = {}) => {
optsCopy.args.push(`--user-agent=${userAgent}`);
}
- if (optsCopy.proxyUrl) return launchPuppeteerWithProxy(puppeteer, optsCopy);
+ let browser;
+ if (optsCopy.proxyUrl) {
+ browser = await launchPuppeteerWithProxy(puppeteer, optsCopy);
+ } else {
+ browser = await puppeteer.launch(optsCopy);
+ }
+
+ // Add stealth
+ if (optsCopy.stealth) {
+ browser = applyStealthFromBrowser(browser, optsCopy.stealth);
+ }
log.info('Launching Puppeteer', _.omit(optsCopy, LAUNCH_PUPPETEER_LOG_OMIT_OPTS));
- return puppeteer.launch(optsCopy);
+ return browser;
};
diff --git a/src/stealth/hiding_tricks.js b/src/stealth/hiding_tricks.js
index 07692fb97..2d2b1c9f7 100644
--- a/src/stealth/hiding_tricks.js
+++ b/src/stealth/hiding_tricks.js
@@ -1,7 +1,7 @@
/* eslint no-undef: 0 */
+/* istanbul ignore file */
-/* istanbul ignore next */
const hideWebDriver = () => {
const newProto = window.navigator.__proto__; // eslint-disable-line
delete newProto.webdriver;
@@ -9,7 +9,6 @@ const hideWebDriver = () => {
navigator.__proto__ = newProto
};
-/* istanbul ignore next */
const hackPermissions = () => {
const originalQuery = window.navigator.permissions.query;
// eslint-disable-next-line
@@ -44,14 +43,12 @@ const hackPermissions = () => {
Function.prototype.toString = functionToString
};
-/* istanbul ignore next */
const addLanguage = () => {
Object.defineProperty(window.navigator, 'languages', {
- get: () => ['cs', 'en-US', 'en'],
+ get: () => ['en-US', 'en'],
});
};
-/* istanbul ignore next */
const emulateWebGL = () => {
try {
/* global WebGLRenderingContext */
@@ -73,7 +70,6 @@ const emulateWebGL = () => {
}
};
-/* istanbul ignore next */
const emulateWindowFrame = () => {
try {
if (window.outerWidth && window.outerHeight) {
@@ -87,7 +83,6 @@ const emulateWindowFrame = () => {
}
};
-/* istanbul ignore next */
const addPlugins = () => {
function mockPluginsAndMimeTypes() {
/* global MimeType MimeTypeArray PluginArray */
@@ -281,14 +276,12 @@ const addPlugins = () => {
mockPluginsAndMimeTypes();
};
-/* istanbul ignore next */
const emulateConsoleDebug = () => {
window.console.debug = () => {
return null;
};
};
-/* istanbul ignore next */
// Should be mocked more properly - this one will bypass only some stupid tests
const mockChrome = () => {
Object.defineProperty(window, 'chrome', {
@@ -298,7 +291,6 @@ const mockChrome = () => {
});
};
-/* istanbul ignore next */
// not sure if this hack does not broke iframe on websites... Should figure out how to test properly
// Should cover that it is custom function same as the permission query
const mocksChromeInIframe = () => {
@@ -316,7 +308,7 @@ const mocksChromeInIframe = () => {
Object.defineProperty(iframe.contentWindow, 'chrome', {
value: {},
configurable: true,
- }); // TODO: ALSO MOCK BETTER
+ });
}
return iframe;
}
@@ -326,9 +318,33 @@ const mocksChromeInIframe = () => {
newCreate.toString = () => 'function createElement() { [native code] }';
document.createElement = newCreate;
+
+ const oldCall = Function.prototype.call;
+ function call() {
+ return oldCall.apply(this, arguments); //eslint-disable-line
+ }
+ // eslint-disable-next-line
+ Function.prototype.call = call;
+
+ const nativeToStringFunctionString = Error.toString().replace(
+ /Error/g,
+ 'toString',
+ );
+ const oldToString = Function.prototype.toString;
+
+ function functionToString() {
+ if (this === window.document.createElement) {
+ return 'function createElement() { [native code] }';
+ }
+ if (this === functionToString) {
+ return nativeToStringFunctionString;
+ }
+ return oldCall.call(oldToString, this);
+ }
+ // eslint-disable-next-line
+ Function.prototype.toString = functionToString
};
-/* istanbul ignore next */
const mockDeviceMemory = () => {
// TODO: Count from env according to - https://developer.mozilla.org/en-US/docs/Web/API/Navigator/deviceMemory
Object.defineProperty(navigator, 'deviceMemory', {
diff --git a/src/stealth/stealth.js b/src/stealth/stealth.js
index 2a549007d..5f91900ae 100644
--- a/src/stealth/stealth.js
+++ b/src/stealth/stealth.js
@@ -1,60 +1,44 @@
import hidingTricks from './hiding_tricks';
/**
- * Class that handles hiding headless chrome.
- * The main purpose of this class is to override newPage function and attached selected tricks.
- */
-class Stealth {
- /**
- * constructor
- * @param {Object} options
- * @param {boolean} options.
- */
- constructor(options) {
- this.options = Object.assign({}, options);
- }
-
- /**
- * Modifies the newPage function in order to get.
+ * The main purpose of this function is to override newPage function and attached selected tricks.
* @param {Browser} browser - puppeteer browser instance
* @returns {Promise<Browser>} - Instance of Browser from puppeteer package
*/
- getStealthBrowser(browser) {
- const modifiedBrowser = browser;
+export default function applyStealthToBrowser(browser, options) {
+ const modifiedBrowser = browser;
- const prevNewPage = browser.newPage;
- modifiedBrowser.newPage = async (...args) => {
- const page = await prevNewPage.bind(browser)(...args);
- await this._applyStealthTricks(page);
- return Promise.resolve(page);
- };
+ const prevNewPage = browser.newPage;
+ modifiedBrowser.newPage = async (...args) => {
+ const page = await prevNewPage.bind(browser)(...args);
+ await applyStealthTricks(page, options);
+ return Promise.resolve(page);
+ };
- return Promise.resolve(modifiedBrowser);
- }
+ return Promise.resolve(modifiedBrowser);
+}
- /**
+/**
* Applies stealth tricks to the puppeteer page
* @param {Page} page
* @returns {Promise}
* @private
* @ignore
*/
- _applyStealthTricks(page) {
- const functions = Object.keys(this.options)
- .filter((key) => {
- return this.options[key];
- })
- .map(key => hidingTricks[key].toString());
-
- /* istanbul ignore next */
- const addFunctions = (functionsArr) => {
- for (const func of functionsArr) {
+function applyStealthTricks(page, options) {
+ const functions = Object.keys(options)
+ .filter((key) => {
+ return options[key];
+ })
+ .map(key => hidingTricks[key].toString());
+
+ /* istanbul ignore next */
+ const addFunctions = (functionsArr) => {
+ for (const func of functionsArr) {
eval(func)(); // eslint-disable-line
- }
- };
+ }
+ };
- return page.evaluateOnNewDocument(addFunctions, functions);
- }
+ return page.evaluateOnNewDocument(addFunctions, functions);
}
-export default Stealth;
diff --git a/test/stealth/stealth.js b/test/stealth/stealth.js
index f1242d32e..b98ecf308 100644
--- a/test/stealth/stealth.js
+++ b/test/stealth/stealth.js
@@ -82,8 +82,7 @@ describe('Stealth - testing headless chrome hiding tricks', () => {
const { languages } = await getFingerPrint(page);
expect(languages).to.be.an('array');
- expect(languages[0]).to.be.eql('cs');
- expect(languages[1]).to.be.eql('en-US');
+ expect(languages[0]).to.be.eql('en-US');
return browser.close();
});