Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Fix stealth in PuppeteerCrawler and the tests (#921)
Ondra Urban4f98af20c765
diff --git a/package.json b/package.jsonindex b59a7885d..97d0e50c7 100644--- a/package.json+++ b/package.json@@ -53,14 +53,14 @@"dependencies": {"@apify/http-request": "^2.1.1","@apify/ps-tree": "^1.1.3",- "@apify/storage-local": "^1.0.3",+ "@apify/storage-local": "^1.0.4","@types/cheerio": "^0.22.22","@types/domhandler": "^2.4.1","@types/node": "^14","@types/puppeteer": "^5.4.0","@types/socket.io": "^2.1.11","@types/tough-cookie": "^4.0.0",- "apify-client": "^1.0.2",+ "apify-client": "^1.0.3","apify-shared": "^0.5.0","browser-pool": "^1.0.2","cheerio": "1.0.0-rc.3",diff --git a/src/browser_launchers/puppeteer_launcher.js b/src/browser_launchers/puppeteer_launcher.jsindex 7de509f54..119814a52 100644--- a/src/browser_launchers/puppeteer_launcher.js+++ b/src/browser_launchers/puppeteer_launcher.js@@ -140,13 +140,13 @@ export class PuppeteerLauncher extends BrowserLauncher {}if (this.stealthOptions && this.stealthOptions.hideWebDriver) {- launchOptions.args.forEach((e) => {- if (e.startsWith('--disable-blink-features=')) {- e += ',AutomationControlled';- return // eslint-disable-line- }- });- launchOptions.args.push('--disable-blink-features=AutomationControlled');+ const idx = launchOptions.args.findIndex((arg) => arg.startsWith('--disable-blink-features='));+ if (idx !== -1) {+ const arg = launchOptions.args[idx];+ launchOptions.args[idx] = `${arg},AutomationControlled`;+ } else {+ launchOptions.args.push('--disable-blink-features=AutomationControlled');+ }}return launchOptions;diff --git a/src/crawlers/puppeteer_crawler.js b/src/crawlers/puppeteer_crawler.jsindex c6e9c1360..8e8871881 100644--- a/src/crawlers/puppeteer_crawler.js+++ b/src/crawlers/puppeteer_crawler.js@@ -275,8 +275,8 @@ class PuppeteerCrawler extends BrowserCrawler {this.browserPool.postLaunchHooks.push(async (pageId, browserController) => {// @TODO: We can do this better now. It is not necessary to override the page.// we can modify the page in the postPageCreateHook- const { hideWebdriver, ...newStealthoptions } = puppeteerLauncher.stealthOptions;- await applyStealthToBrowser(browserController.browser, newStealthoptions);+ const { hideWebDriver, ...newStealthOptions } = puppeteerLauncher.stealthOptions;+ await applyStealthToBrowser(browserController.browser, newStealthOptions);});}diff --git a/test/stealth/stealth.test.js b/test/stealth/stealth.test.jsindex 06777eb15..e5ea35b37 100644--- a/test/stealth/stealth.test.js+++ b/test/stealth/stealth.test.js@@ -18,8 +18,6 @@ const getFingerPrint = async (page) => {// we can speed up the test to make the requests to the local static htmldescribe('Stealth - testing headless chrome hiding tricks', () => {let localStorageEmulator;- let browser;- let page;beforeAll(async () => {localStorageEmulator = new LocalStorageDirEmulator();@@ -28,191 +26,196 @@ describe('Stealth - testing headless chrome hiding tricks', () => {beforeEach(async () => {const storageDir = await localStorageEmulator.init();utils.apifyStorageLocal = utils.newStorageLocal({ storageDir });- browser = await Apify.launchPuppeteer({- stealth: true,- useChrome: true,- launchOptions: {- headless: true,- },- });- page = await browser.newPage();- });-- afterEach(async () => {- await browser.close();});afterAll(async () => {await localStorageEmulator.destroy();});- test('it adds plugins, mimeTypes and passes', async () => {- await page.goto(testUrl);- const { plugins, mimeTypes } = await getFingerPrint(page);-- expect(plugins.length).toBe(3);- expect(mimeTypes.length).toBe(4);+ test('it works in PuppeteerCrawler', async () => {+ expect.assertions(1);+ const requestList = await Apify.openRequestList(null, [testUrl]);+ const crawler = new Apify.PuppeteerCrawler({+ requestList,+ launchContext: {+ stealth: true,+ useChrome: true,+ launchOptions: {+ headless: true,+ },+ },+ async handlePageFunction({ page }) {+ const { webDriver } = await getFingerPrint(page);+ expect(webDriver).toBe(false);+ },+ });+ await crawler.run();});- test('it hides webDriver', async () => {- await page.goto(testUrl);- const { webDriver } = await getFingerPrint(page);+ describe('work in launchPuppeteer', () => {+ let browser;+ let page;- expect(webDriver).toBe(false);- });+ beforeEach(async () => {+ browser = await Apify.launchPuppeteer({+ stealth: true,+ useChrome: true,+ launchOptions: {+ headless: true,+ },+ });+ page = await browser.newPage();+ });- test('it hacks permissions', async () => {- await page.goto(testUrl);- const { permissions } = await getFingerPrint(page);+ afterEach(async () => {+ await browser.close();+ });- expect(permissions.state).toBe('denied');- });+ test('it adds plugins, mimeTypes and passes', async () => {+ await page.goto(testUrl);+ const { plugins, mimeTypes } = await getFingerPrint(page);- test('it adds language to navigator', async () => {- await page.goto(testUrl);- const { languages } = await getFingerPrint(page);+ expect(plugins.length).toBe(3);+ expect(mimeTypes.length).toBe(4);+ });- expect(Array.isArray(languages)).toBe(true);- expect(languages[0]).toBe('en-US');- });+ test('it hides webDriver', async () => {+ await page.goto(testUrl);+ const { webDriver } = await getFingerPrint(page);- test('it emulates WebGL', async () => {- await page.goto(testUrl);- const { videoCard } = await getFingerPrint(page);+ expect(webDriver).toBe(false);+ });- expect(videoCard[0]).toBe('Intel Inc.');- expect(videoCard[1]).toBe('Intel(R) Iris(TM) Plus Graphics 640');- });+ test('it hacks permissions', async () => {+ await page.goto(testUrl);+ const { permissions } = await getFingerPrint(page);- test('it emulates windowFrame', async () => {- await page.goto(testUrl);- const { screen } = await getFingerPrint(page);+ expect(permissions.state).toBe('denied');+ });- expect(screen.wOuterHeight > 0).toBe(true);- expect(screen.wOuterWidth > 0).toBe(true);- });+ test('it adds language to navigator', async () => {+ await page.goto(testUrl);+ const { languages } = await getFingerPrint(page);- test('it emulates console.debug', async () => {- await page.goto(testUrl);- const returnValue = await page.evaluate(() => console.debug('TEST'));+ expect(Array.isArray(languages)).toBe(true);+ expect(languages[0]).toBe('en-US');+ });- expect(returnValue).toBe(null);- });- test('it should mock window.chrome to plain object', async () => {- await page.goto(testUrl);- const { hasChrome } = await getFingerPrint(page);- const chrome = await page.evaluate(() => window.chrome); //eslint-disable-line- expect(chrome).toBeInstanceOf(Object);- expect(chrome.runtime).toEqual({}); // eslint-disable-line- expect(hasChrome).toBe(true);- });+ test('it emulates WebGL', async () => {+ await page.goto(testUrl);+ const { videoCard } = await getFingerPrint(page);- test('it should mock chrome when iframe is created', async () => {- await page.goto(testUrl);- const { iframeChrome } = await getFingerPrint(page);+ expect(videoCard[0]).toBe('Intel Inc.');+ expect(videoCard[1]).toBe('Intel(R) Iris(TM) Plus Graphics 640');+ });- expect(iframeChrome).toBe('object');+ test('it emulates windowFrame', async () => {+ await page.goto(testUrl);+ const { screen } = await getFingerPrint(page);- return browser.close();- });+ expect(screen.wOuterHeight > 0).toBe(true);+ expect(screen.wOuterWidth > 0).toBe(true);+ });- test('it should not break iframe ', async () => {- const testFuncReturnValue = 'TESTSTRING';- await page.goto(testUrl);- await page.evaluate((returnValue) => {- const { document } = window; //eslint-disable-line- const body = document.querySelector('body');- const iframe = document.createElement('iframe');- iframe.contentWindow.mySuperFunction = () => returnValue;- body.appendChild(iframe);- }, testFuncReturnValue);- const realReturn = await page.evaluate(- () => document.querySelector('iframe').contentWindow.mySuperFunction(), //eslint-disable-line- );- expect(realReturn).toEqual(testFuncReturnValue);- });+ test('it emulates console.debug', async () => {+ await page.goto(testUrl);+ const returnValue = await page.evaluate(() => console.debug('TEST'));- test('it should mock device memory', async () => {- await page.goto(testUrl);- const { deviceMemory } = await getFingerPrint(page);+ expect(returnValue).toBe(null);+ });+ test('it should mock window.chrome to plain object', async () => {+ await page.goto(testUrl);+ const { hasChrome } = await getFingerPrint(page);+ const chrome = await page.evaluate(() => window.chrome); //eslint-disable-line+ expect(chrome).toBeInstanceOf(Object);+ expect(chrome.runtime).toEqual({}); // eslint-disable-line+ expect(hasChrome).toBe(true);+ });- expect(deviceMemory).not.toBe(0);- });+ test('it should mock chrome when iframe is created', async () => {+ await page.goto(testUrl);+ const { iframeChrome } = await getFingerPrint(page);- test(- 'it should bypass all of the known tests for browser fingerprinting',- async () => {+ expect(iframeChrome).toBe('object');++ return browser.close();+ });++ test('it should not break iframe ', async () => {+ const testFuncReturnValue = 'TESTSTRING';await page.goto(testUrl);- const fingerPrint = await getFingerPrint(page);- const testedFingerprint = scanner.analyseFingerprint(fingerPrint);- const failedChecks = Object.values(testedFingerprint).filter((val) => val.consistent < 3);-- expect(failedChecks.length).toBe(0);- },- );-- test('logs the evaluation warning in "page" when limit is exceeded', async () => {- const numberOfIframes = 14;- let message = '';- const oldConsoleWarn = console.warn;- console.warn = (msg) => {- message = msg;- return oldConsoleWarn.bind(console);- };-- await page.goto(testUrl);-- await page.evaluate((iframesCount) => {- for (let i = 0; i < iframesCount; i++) {+ await page.evaluate((returnValue) => {+ const { document } = window; //eslint-disable-line+ const body = document.querySelector('body');const iframe = document.createElement('iframe');- document.body.appendChild(iframe);- }- }, numberOfIframes);+ iframe.contentWindow.mySuperFunction = () => returnValue;+ body.appendChild(iframe);+ }, testFuncReturnValue);+ const realReturn = await page.evaluate(+ () => document.querySelector('iframe').contentWindow.mySuperFunction(), //eslint-disable-line+ );+ expect(realReturn).toEqual(testFuncReturnValue);+ });- expect(message.includes('Evaluating hiding tricks in too many iframes')).toBeTruthy();- });+ test('it should mock device memory', async () => {+ await page.goto(testUrl);+ const { deviceMemory } = await getFingerPrint(page);- test('does not log the message when the iframes are under the limit', async () => {- const numberOfIframes = 9;- let message;- const oldConsoleWarn = console.warn;- console.warn = (msg) => {- message = msg;- return oldConsoleWarn.bind(console);- };+ expect(deviceMemory).not.toBe(0);+ });- await page.goto(testUrl);+ test(+ 'it should bypass all of the known tests for browser fingerprinting',+ async () => {+ await page.goto(testUrl);+ const fingerPrint = await getFingerPrint(page);+ const testedFingerprint = scanner.analyseFingerprint(fingerPrint);+ const failedChecks = Object.values(testedFingerprint).filter((val) => val.consistent < 3);- await page.evaluate((iframesCount) => {- for (let i = 0; i < iframesCount; i++) {- const iframe = document.createElement('iframe');- document.body.appendChild(iframe);- }- }, numberOfIframes);+ expect(failedChecks.length).toBe(0);+ },+ );- expect(message).toBeUndefined();- });+ test('logs the evaluation warning in "page" when limit is exceeded', async () => {+ const numberOfIframes = 14;+ let message = '';+ const oldConsoleWarn = console.warn;+ console.warn = (msg) => {+ message = msg;+ return oldConsoleWarn.bind(console);+ };- test('should work in puppeteer crawler', async () => {- const requestList = await Apify.openRequestList(null, [testUrl]);- const puppeteerCrawler = new Apify.PuppeteerCrawler({- requestList,- launchContext: {- stealth: true,- useChrome: true,- launchOptions: {- headless: true,- },- },- handlePageFunction: async ({ page: activePage }) => {- const fingerprint = await getFingerPrint(activePage);- values.push(fingerprint);- },+ await page.goto(testUrl);++ await page.evaluate((iframesCount) => {+ for (let i = 0; i < iframesCount; i++) {+ const iframe = document.createElement('iframe');+ document.body.appendChild(iframe);+ }+ }, numberOfIframes);++ expect(message.includes('Evaluating hiding tricks in too many iframes')).toBeTruthy();+ });++ test('does not log the message when the iframes are under the limit', async () => {+ const numberOfIframes = 9;+ let message;+ const oldConsoleWarn = console.warn;+ console.warn = (msg) => {+ message = msg;+ return oldConsoleWarn.bind(console);+ };++ await page.goto(testUrl);++ await page.evaluate((iframesCount) => {+ for (let i = 0; i < iframesCount; i++) {+ const iframe = document.createElement('iframe');+ document.body.appendChild(iframe);+ }+ }, numberOfIframes);++ expect(message).toBeUndefined();});- const values = [];- await puppeteerCrawler.run();- const fingerprint = values[0];- expect(fingerprint.webDriver).toBe(false);- expect(fingerprint.webDriverValue).toBeUndefined();});});