Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
hotfixed webdriver issue. (#909)
petrpatek8951ea87a0d5
diff --git a/src/browser_launchers/puppeteer_launcher.js b/src/browser_launchers/puppeteer_launcher.jsindex 76450b420..7de509f54 100644--- a/src/browser_launchers/puppeteer_launcher.js+++ b/src/browser_launchers/puppeteer_launcher.js@@ -87,8 +87,8 @@ export class PuppeteerLauncher extends BrowserLauncher {const {launcher = BrowserLauncher.requireLauncherOrThrow('puppeteer', 'apify/actor-node-chrome-*'),userAgent,- stealth,- stealthOptions,+ stealth = false,+ stealthOptions = {},...browserLauncherOptions} = launchContext;@@ -96,10 +96,12 @@ export class PuppeteerLauncher extends BrowserLauncher {...browserLauncherOptions,launcher,});-this.userAgent = userAgent;this.stealth = stealth;- this.stealthOptions = stealthOptions;+ this.stealthOptions = {+ hideWebDriver: true,+ ...stealthOptions,+ };this.Plugin = PuppeteerPlugin;}@@ -108,7 +110,9 @@ export class PuppeteerLauncher extends BrowserLauncher {const browser = await super.launch();if (this.stealth) {- applyStealthToBrowser(browser, this.stealthOptions);+ const { hideWebDriver, ...newStealthOptions } = this.stealthOptions;++ applyStealthToBrowser(browser, newStealthOptions);}return browser;@@ -135,6 +139,16 @@ export class PuppeteerLauncher extends BrowserLauncher {launchOptions.args.push(`--user-agent=${userAgent}`);}+ if (this.stealthOptions && this.stealthOptions.hideWebDriver) {+ launchOptions.args.forEach((e) => {+ if (e.startsWith('--disable-blink-features=')) {+ e += ',AutomationControlled';+ return // eslint-disable-line+ }+ });+ launchOptions.args.push('--disable-blink-features=AutomationControlled');+ }+return launchOptions;}}diff --git a/src/crawlers/puppeteer_crawler.js b/src/crawlers/puppeteer_crawler.jsindex 71b6357da..c6e9c1360 100644--- a/src/crawlers/puppeteer_crawler.js+++ b/src/crawlers/puppeteer_crawler.js@@ -244,8 +244,7 @@ class PuppeteerCrawler extends BrowserCrawler {} = options;const {- stealth,- stealthOptions,+ stealth = false,} = launchContext;if (launchContext.proxyUrl) {@@ -276,7 +275,8 @@ class PuppeteerCrawler extends BrowserCrawler {this.browserPool.postLaunchHooks.push(async (pageId, browserController) => {// @TODO: We can do this better now. It is not necessary to override the page.// we can modify the page in the postPageCreateHook- await applyStealthToBrowser(browserController.browser, stealthOptions);+ const { hideWebdriver, ...newStealthoptions } = puppeteerLauncher.stealthOptions;+ await applyStealthToBrowser(browserController.browser, newStealthoptions);});}diff --git a/src/stealth/hiding_tricks.js b/src/stealth/hiding_tricks.jsindex dbd0712ba..45e4c8578 100644--- a/src/stealth/hiding_tricks.js+++ b/src/stealth/hiding_tricks.js@@ -1,18 +1,5 @@/* istanbul ignore file */-const hideWebDriver = () => {- Object.defineProperty(window, 'navigator', {- value: new Proxy(navigator, {- has: (target, key) => (key === 'webdriver' ? false : key in target),- get: (target, key) => (key === 'webdriver' // eslint-disable-line- ? undefined- : typeof target[key] === 'function'- ? target[key].bind(target)- : target[key]),- }),- });-};-const hackPermissions = () => {const originalQuery = window.navigator.permissions.query;// eslint-disable-next-line@@ -357,7 +344,6 @@ export default {emulateWebGL,emulateConsoleDebug,addLanguage,- hideWebDriver,hackPermissions,mockChrome,mockChromeInIframe,diff --git a/src/stealth/stealth.js b/src/stealth/stealth.jsindex 673bc916a..903bea66e 100644--- a/src/stealth/stealth.js+++ b/src/stealth/stealth.js@@ -29,7 +29,6 @@ const DEFAULT_STEALTH_OPTIONS = {emulateWebGL: true,emulateConsoleDebug: true,addLanguage: true,- hideWebDriver: true,hackPermissions: true,mockChrome: true,mockChromeInIframe: true,