Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Hotfix stealth (#824)
petrpateka97dd8c7ddd7
diff --git a/src/stealth/hiding_tricks.js b/src/stealth/hiding_tricks.jsindex f5ff5e748..16c1c2dda 100644--- a/src/stealth/hiding_tricks.js+++ b/src/stealth/hiding_tricks.js@@ -48,7 +48,7 @@ const hackPermissions = () => {};const addLanguage = () => {- Object.defineProperty(window.navigator, 'languages', {+ Object.defineProperty(Object.getPrototypeOf(navigator), 'languages', {get: () => ['en-US', 'en'],});};@@ -74,16 +74,12 @@ const emulateWebGL = () => {};const emulateWindowFrame = () => {- try {- if (window.outerWidth && window.outerHeight) {- return; // nothing to do here- }- const windowFrame = 85; // probably OS and WM dependent- window.outerWidth = window.innerWidth;- window.outerHeight = window.innerHeight + windowFrame;- } catch (err) {- console.error('hiding_tricks: Could not emulate window frame');+ if (window.outerWidth && window.outerHeight) {+ return; // nothing to do here}+ const windowFrame = 85; // probably OS and WM dependent+ window.outerWidth = window.innerWidth;+ window.outerHeight = window.innerHeight + windowFrame;};const addPlugins = () => {@@ -285,11 +281,13 @@ const emulateConsoleDebug = () => {// Should be mocked more properly - this one will bypass only some stupid testsconst mockChrome = () => {- Object.defineProperty(window, 'chrome', {- value: {- runtime: {},- },- });+ if (!window.chrome) {+ Object.defineProperty(window, 'chrome', {+ value: {+ runtime: {},+ },+ });+ }};// not sure if this hack does not broke iframe on websites... Should figure out how to test properlydiff --git a/src/stealth/stealth.js b/src/stealth/stealth.jsindex c3519ca26..d7dfee1e4 100644--- a/src/stealth/stealth.js+++ b/src/stealth/stealth.js@@ -1,7 +1,12 @@import * as _ from 'underscore';import { Page, Browser } from 'puppeteer'; // eslint-disable-line no-unused-vars+import { cryptoRandomObjectId } from 'apify-shared/utilities';+import globalLog from '../utils_log';+import hidingTricks from './hiding_tricks';+const log = globalLog.child({ prefix: 'Stealth' });+/*** Configuration of stealth tricks for a proper hiding effect all of them should be set to true.* These tricks are applied only when the `stealth` option is set to `true`.@@ -31,6 +36,9 @@ const DEFAULT_STEALTH_OPTIONS = {mockDeviceMemory: true,};+const STEALTH_ERROR_MESSAGE_PREFIX = 'StealthError';+const MAX_IFRAMES = 10;+const alreadyWrapped = Symbol('alreadyWrapped');/*** The main purpose of this function is to override newPage function and attach selected tricks.* @param {Browser} browser - puppeteer browser instance@@ -46,25 +54,68 @@ export default function applyStealthToBrowser(browser, options) {const prevNewPage = contextPrototype.newPage;- contextPrototype.newPage = async function (...args) {- const page = await prevNewPage.bind(this)(...args);- await applyStealthTricks(page, opts);- return page;- };+ if (!contextPrototype.newPage[alreadyWrapped]) {+ contextPrototype.newPage = async function (...args) {+ const page = await prevNewPage.bind(this)(...args);++ const evaluationDebugMessage = generateEvaluationDebugMessage();+ addStealthDebugToPage(page, evaluationDebugMessage);+ await applyStealthTricks(page, evaluationDebugMessage, opts);++ return page;+ };++ contextPrototype.newPage[alreadyWrapped] = true;+ }return Promise.resolve(modifiedBrowser);}+function generateEvaluationDebugMessage() {+ const minLength = 6;+ const maxLength = 10;+ const randomLength = Math.random() * (maxLength - minLength) + minLength;++ return cryptoRandomObjectId(randomLength);+}+/**+ * Logs the stealth errors in browser to the node stdout.+ * @param page {Page} - puppeteer page instance+ * @param evaluationDebugMessage {string} - debug message+ */+function addStealthDebugToPage(page, evaluationDebugMessage) {+ let warningLogged = false;+ let counter = 1;+ page.on('console', (msg) => {+ const text = msg.text();+ if (text.includes(STEALTH_ERROR_MESSAGE_PREFIX)) {+ log.error(text);+ } else if (text.includes(evaluationDebugMessage)) {+ if (counter > MAX_IFRAMES && !warningLogged) {+ log.warning(+ `Evaluating hiding tricks in too many iframes (limit: ${MAX_IFRAMES}).`+ + 'You might experience some performance issues. Try setting \'stealth\' false',+ );++ warningLogged = true;+ }+ counter++;+ log.debug('Tricks evaluated', { counter });+ }+ });+}+/*** Applies stealth tricks to the puppeteer page* @param {Page} page+ * @param evaluationDebugMessage {string} - debug message* @param {StealthOptions} options* @returns {Promise<void>}* @private* @ignore*/-function applyStealthTricks(page, options) {+function applyStealthTricks(page, evaluationDebugMessage, options) {const functions = Object.keys(options).filter((key) => {return options[key];@@ -72,11 +123,17 @@ function applyStealthTricks(page, options) {.map(key => hidingTricks[key].toString());/* istanbul ignore next */- const addFunctions = (functionsArr) => {+ const addFunctions = (functionsArr, errorMessagePrefix, debugMessage) => {+ console.log(debugMessage);+ // add functionsfor (const func of functionsArr) {- eval(func)(); // eslint-disable-line+ try {+ eval(func)(); // eslint-disable-line+ } catch (e) {+ console.error(`${errorMessagePrefix}: Failed to apply stealth trick reason: ${e.message}`);+ }}};- return page.evaluateOnNewDocument(addFunctions, functions);+ return page.evaluateOnNewDocument(addFunctions, functions, STEALTH_ERROR_MESSAGE_PREFIX, evaluationDebugMessage);}diff --git a/test/stealth/stealth.test.js b/test/stealth/stealth.test.jsindex 8365359f6..4a645b421 100644--- a/test/stealth/stealth.test.js+++ b/test/stealth/stealth.test.js@@ -17,6 +17,8 @@ const getFingerPrint = async (page) => {// we can speed up the test to make the requests to the local static htmldescribe('Stealth - testing headless chrome hiding tricks', () => {let localStorageEmulator;+ let browser;+ let page;beforeAll(async () => {localStorageEmulator = new LocalStorageDirEmulator();@@ -24,6 +26,16 @@ describe('Stealth - testing headless chrome hiding tricks', () => {beforeEach(async () => {await localStorageEmulator.init();+ browser = await Apify.launchPuppeteer({+ stealth: true,+ headless: true,+ useChrome: true,+ });+ page = await browser.newPage();+ });++ afterEach(async () => {+ await browser.close();});afterAll(async () => {@@ -31,161 +43,67 @@ describe('Stealth - testing headless chrome hiding tricks', () => {});test('it adds plugins, mimeTypes and passes', async () => {- const browser = await Apify.launchPuppeteer({- stealth: true,- stealthOptions: {- emulateWindowFrame: false,- emulateWebGL: false,- emulateConsoleDebug: false,- addLanguage: false,- hideWebDriver: false,- hackPermissions: false,- mockChrome: false,- mockChromeInIframe: false,- mockDeviceMemory: false,- },- headless: true,- useChrome: true,- });-- const page = await browser.newPage();await page.goto(testUrl);- const { webDriver } = await getFingerPrint(page);-- // check if disabling works- expect(webDriver).toBe(true);const { plugins, mimeTypes } = await getFingerPrint(page);expect(plugins.length).toBe(3);expect(mimeTypes.length).toBe(4);-- return browser.close();});test('it hides webDriver', async () => {- const browser = await Apify.launchPuppeteer({- stealth: true,- headless: true,- useChrome: true,- });-- const page = await browser.newPage();await page.goto(testUrl);const { webDriver } = await getFingerPrint(page);expect(webDriver).toBe(false);-- return browser.close();});test('it hacks permissions', async () => {- const browser = await Apify.launchPuppeteer({- stealth: true,- headless: true,- useChrome: true,- });-- const page = await browser.newPage();await page.goto(testUrl);const { permissions } = await getFingerPrint(page);expect(permissions.state).toBe('denied');-- return browser.close();});test('it adds language to navigator', async () => {- const browser = await Apify.launchPuppeteer({- stealth: true,- headless: true,- useChrome: true,- });-- const page = await browser.newPage();await page.goto(testUrl);const { languages } = await getFingerPrint(page);expect(Array.isArray(languages)).toBe(true);expect(languages[0]).toBe('en-US');-- return browser.close();});test('it emulates WebGL', async () => {- const browser = await Apify.launchPuppeteer({- stealth: true,- headless: true,- useChrome: true,- });-- const page = await browser.newPage();await page.goto(testUrl);const { videoCard } = await getFingerPrint(page);expect(videoCard[0]).toBe('Intel Inc.');expect(videoCard[1]).toBe('Intel(R) Iris(TM) Plus Graphics 640');-- return browser.close();});test('it emulates windowFrame', async () => {- const browser = await Apify.launchPuppeteer({- stealth: true,- headless: true,- useChrome: true,- });-- const page = await browser.newPage();await page.goto(testUrl);const { screen } = await getFingerPrint(page);expect(screen.wOuterHeight > 0).toBe(true);expect(screen.wOuterWidth > 0).toBe(true);-- return browser.close();});test('it emulates console.debug', async () => {- const browser = await Apify.launchPuppeteer({- stealth: true,- headless: true,- useChrome: true,- });-- const page = await browser.newPage();await page.goto(testUrl);const returnValue = await page.evaluate(() => console.debug('TEST'));expect(returnValue).toBe(null);-- return browser.close();});test('it should mock window.chrome to plain object', async () => {- const browser = await Apify.launchPuppeteer({- stealth: true,- headless: true,- useChrome: true,- });-- const page = await browser.newPage();await page.goto(testUrl);const { hasChrome } = await getFingerPrint(page);const chrome = await page.evaluate(() => window.chrome); //eslint-disable-lineexpect(chrome).toBeInstanceOf(Object);expect(chrome.runtime).toEqual({}); // eslint-disable-lineexpect(hasChrome).toBe(true);-- return browser.close();});test('it should mock chrome when iframe is created', async () => {- const browser = await Apify.launchPuppeteer({- stealth: true,- headless: true,- useChrome: true,- });-- const page = await browser.newPage();await page.goto(testUrl);const { iframeChrome } = await getFingerPrint(page);@@ -195,13 +113,6 @@ describe('Stealth - testing headless chrome hiding tricks', () => {});test('it should not break iframe ', async () => {- const browser = await Apify.launchPuppeteer({- stealth: true,- headless: true,- useChrome: true,- });-- const page = await browser.newPage();const testFuncReturnValue = 'TESTSTRING';await page.goto(testUrl);await page.evaluate((returnValue) => {@@ -215,65 +126,97 @@ describe('Stealth - testing headless chrome hiding tricks', () => {() => document.querySelector('iframe').contentWindow.mySuperFunction(), //eslint-disable-line);expect(realReturn).toEqual(testFuncReturnValue);-- return browser.close();});test('it should mock device memory', async () => {- const browser = await Apify.launchPuppeteer({- stealth: true,- headless: true,- useChrome: true,- });-- const page = await browser.newPage();await page.goto(testUrl);const { deviceMemory } = await getFingerPrint(page);expect(deviceMemory).not.toBe(0);-- return browser.close();});test('it should bypass all of the known tests for browser fingerprinting',async () => {- const browser = await Apify.launchPuppeteer({- stealth: true,- headless: true,- useChrome: true,- });-- const page = await browser.newPage();await page.goto(testUrl);const fingerPrint = await getFingerPrint(page);const testedFingerprint = scanner.analyseFingerprint(fingerPrint);const failedChecks = Object.values(testedFingerprint).filter(val => val.consistent < 3);expect(failedChecks.length).toBe(0);-- return browser.close();},);- test('should work in crawler', async () => {- const requestList = await Apify.openRequestList('test', [testUrl]);- const values = [];- const puppeteerCrawler = new Apify.PuppeteerCrawler({- requestList,- launchPuppeteerOptions: {- stealth: true,- useChrome: true,- headless: true,- },- handlePageFunction: async ({ page }) => {- const fingerprint = await getFingerPrint(page);+ test('logs the evaluation warning in "page" when limit is exceeded', async () => {+ const numberOfIframes = 14;+ let message = '';+ const oldConsoleWarn = console.warn;+ console.warn = (msg) => {+ message = msg;+ return oldConsoleWarn.bind(console);+ };++ await page.goto(testUrl);++ await page.evaluate((iframesCount) => {+ for (let i = 0; i < iframesCount; i++) {+ const iframe = document.createElement('iframe');+ document.body.appendChild(iframe);+ }+ }, numberOfIframes);+++ expect(message.includes('Evaluating hiding tricks in too many iframes')).toBeTruthy();+ });++ test('does not log the message when the iframes are under the limit', async () => {+ const numberOfIframes = 9;+ let message;+ const oldConsoleWarn = console.warn;+ console.warn = (msg) => {+ message = msg;+ return oldConsoleWarn.bind(console);+ };++ await page.goto(testUrl);++ await page.evaluate((iframesCount) => {+ for (let i = 0; i < iframesCount; i++) {+ const iframe = document.createElement('iframe');+ document.body.appendChild(iframe);+ }+ }, numberOfIframes);++ expect(message).toBeUndefined();+ });+++ describe('puppeteer crawler test', async () => {+ let requestList;+ let puppeteerCrawler;++ beforeEach(async () => {+ requestList = await Apify.openRequestList('test', [testUrl]);+ puppeteerCrawler = new Apify.PuppeteerCrawler({+ requestList,+ launchPuppeteerOptions: {+ stealth: true,+ useChrome: true,+ headless: true,+ },+ handlePageFunction: () => {},+ });+ });+ test('should work in crawler', async () => {+ puppeteerCrawler.handlePageFunction = async ({ page: activePage }) => {+ const fingerprint = await getFingerPrint(activePage);values.push(fingerprint);- },+ };+ const values = [];+ await puppeteerCrawler.run();+ const fingerprint = values[0];+ expect(fingerprint.webDriver).toBe(false); // eslint-disable-line+ expect(fingerprint.webDriverValue).toBeUndefined(); // eslint-disable-line});- await puppeteerCrawler.run();- const fingerprint = values[0];- expect(fingerprint.webDriver).toBe(false); // eslint-disable-line- expect(fingerprint.webDriverValue).toBeUndefined(); // eslint-disable-line});});