Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Added hiding tricks a improved few of them. Iframe chrome mocking needs to be fix and masked.
Petr Patek5e2a92e9115d
diff --git a/src/stealth/hiding_tricks.js b/src/stealth/hiding_tricks.jsindex cf782cf85..07692fb97 100644--- a/src/stealth/hiding_tricks.js+++ b/src/stealth/hiding_tricks.js@@ -1,12 +1,5 @@/* eslint no-undef: 0 */-/* istanbul ignore next */-const mockRuntime = () => {- window.navigator.chrome = {- runtime: {},- // etc.- };-};/* istanbul ignore next */const hideWebDriver = () => {@@ -54,8 +47,7 @@ const hackPermissions = () => {/* istanbul ignore next */const addLanguage = () => {Object.defineProperty(window.navigator, 'languages', {- get: () => ['en-US', 'en'],- configurable: true,+ get: () => ['cs', 'en-US', 'en'],});};@@ -300,12 +292,15 @@ const emulateConsoleDebug = () => {// Should be mocked more properly - this one will bypass only some stupid testsconst mockChrome = () => {Object.defineProperty(window, 'chrome', {- value: {},+ value: {+ runtime: {},+ },});};/* istanbul ignore next */// not sure if this hack does not broke iframe on websites... Should figure out how to test properly+// Should cover that it is custom function same as the permission queryconst mocksChromeInIframe = () => {const oldCreate = document.createElement.bind(document);const newCreate = (...args) => {@@ -349,7 +344,6 @@ export default {addLanguage,hideWebDriver,hackPermissions,- mockRuntime,mockChrome,mocksChromeInIframe,mockDeviceMemory,diff --git a/test/stealth/stealth.js b/test/stealth/stealth.jsindex f86a515de..f7ff20042 100644--- a/test/stealth/stealth.js+++ b/test/stealth/stealth.js@@ -32,25 +32,6 @@ describe('Stealth - testing headless chrome hiding tricks', () => {return browser.close();});- it('it mocks chrome runtime', async () => {- const browser = await Apify.launchPuppeteer({- stealth: {- mockRuntime: true,- },- headless: true,- useChrome: true,- });-- const page = await browser.newPage();- await page.goto('file://test_website.html');- const chrome = await page.evaluate(() => window.navigator.chrome); // eslint-disable-line-- expect(chrome).to.be.an('object');- expect(chrome.runtime).to.be.empty; // eslint-disable-line-- return browser.close();- });-it('it hides webDriver', async () => {const browser = await Apify.launchPuppeteer({stealth: {@@ -172,7 +153,9 @@ describe('Stealth - testing headless chrome hiding tricks', () => {const page = await browser.newPage();await page.goto('file://test_website.html');const { hasChrome } = await getFingerPrint(page);-+ const chrome = await page.evaluate(() => window.chrome); //eslint-disable-line+ expect(chrome).to.be.an('object');+ expect(chrome.runtime).to.not.be.empty; // eslint-disable-lineexpect(hasChrome).to.be.eql(true);return browser.close();@@ -220,7 +203,6 @@ describe('Stealth - testing headless chrome hiding tricks', () => {addPlugins: true,emulateWindowFrame: true,hideWebDriver: true,- mockRuntime: true,emulateWebGL: true,hackPermissions: true,addLanguage: true,