🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
refactor(Playwright Engine): Separate what we can for cleaner code and the async function later
Karim shoair889c111855bf
diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.pyindex 8b2895c..e210a7b 100644--- a/scrapling/engines/pw.py+++ b/scrapling/engines/pw.py@@ -1,12 +1,13 @@import json-from scrapling.core._types import Callable, Dict, List, Optional, Union-from scrapling.core.utils import log+from scrapling.core._types import Callable, Dict, Optional, Union+from scrapling.core.utils import log, lru_cachefrom scrapling.engines.constants import (DEFAULT_STEALTH_FLAGS,NSTBROWSER_DEFAULT_QUERY)from scrapling.engines.toolbelt import (Response, StatusText,check_type_validity, construct_cdp_url,construct_proxy_dict, do_nothing,+ do_nothing_async,generate_convincing_referer,generate_headers, intercept_route,js_bypass_path)@@ -94,10 +95,8 @@ class PlaywrightEngine:# '--disable-extensions',]- def _cdp_url_logic(self, flags: Optional[List] = None) -> str:+ def _cdp_url_logic(self) -> str:"""Constructs new CDP URL if NSTBrowser is enabled otherwise return CDP URL as it is-- :param flags: Chrome flags to be added to NSTBrowser query:return: CDP URL"""cdp_url = self.cdp_url@@ -106,7 +105,8 @@ class PlaywrightEngine:config = self.nstbrowser_configelse:query = NSTBROWSER_DEFAULT_QUERY.copy()- if flags:+ if self.stealth:+ flags = self.__set_flags()query.update({"args": dict(zip(flags, [''] * len(flags))), # browser args should be a dictionary})@@ -122,6 +122,68 @@ class PlaywrightEngine:return cdp_url+ @lru_cache(typed=True)+ def __set_flags(self):+ """Returns the flags that will be used while launching the browser if stealth mode is enabled"""+ flags = DEFAULT_STEALTH_FLAGS+ if self.hide_canvas:+ flags += ('--fingerprinting-canvas-image-data-noise',)+ if self.disable_webgl:+ flags += ('--disable-webgl', '--disable-webgl-image-chromium', '--disable-webgl2',)++ return flags++ def __launch_kwargs(self):+ """Creates the arguments we will use while launching playwright's browser"""+ launch_kwargs = {'headless': self.headless, 'ignore_default_args': self.harmful_default_args, 'channel': 'chrome' if self.real_chrome else 'chromium'}+ if self.stealth:+ launch_kwargs.update({'args': self.__set_flags(), 'chromium_sandbox': True})++ return launch_kwargs++ def __context_kwargs(self):+ """Creates the arguments for the browser context"""+ context_kwargs = {+ "proxy": self.proxy,+ "locale": self.locale,+ "color_scheme": 'dark', # Bypasses the 'prefersLightColor' check in creepjs+ "device_scale_factor": 2,+ "extra_http_headers": self.extra_headers if self.extra_headers else {},+ "user_agent": self.useragent if self.useragent else generate_headers(browser_mode=True).get('User-Agent'),+ }+ if self.stealth:+ context_kwargs.update({+ 'is_mobile': False,+ 'has_touch': False,+ # I'm thinking about disabling it to rest from all Service Workers headache but let's keep it as it is for now+ 'service_workers': 'allow',+ 'ignore_https_errors': True,+ 'screen': {'width': 1920, 'height': 1080},+ 'viewport': {'width': 1920, 'height': 1080},+ 'permissions': ['geolocation', 'notifications']+ })++ return context_kwargs++ @lru_cache()+ def __stealth_scripts(self):+ # Basic bypasses nothing fancy as I'm still working on it+ # But with adding these bypasses to the above config, it bypasses many online tests like+ # https://bot.sannysoft.com/+ # https://kaliiiiiiiiii.github.io/brotector/+ # https://pixelscan.net/+ # https://iphey.com/+ # https://www.browserscan.net/bot-detection <== this one also checks for the CDP runtime fingerprint+ # https://arh.antoinevastel.com/bots/areyouheadless/+ # https://prescience-data.github.io/execution-monitor.html+ return tuple(+ js_bypass_path(script) for script in (+ # Order is important+ 'webdriver_fully.js', 'window_chrome.js', 'navigator_plugins.js', 'pdf_viewer.js',+ 'notification_permission.js', 'screen_props.js', 'playwright_fingerprint.js'+ )+ )+def fetch(self, url: str) -> Response:"""Opens up the browser and do your request based on your chosen options.@@ -135,61 +197,14 @@ class PlaywrightEngine:from rebrowser_playwright.sync_api import sync_playwrightwith sync_playwright() as p:- # Handle the UserAgent early- if self.useragent:- extra_headers = {}- useragent = self.useragent- else:- extra_headers = {}- useragent = generate_headers(browser_mode=True).get('User-Agent')-- # Prepare the flags before diving- flags = DEFAULT_STEALTH_FLAGS- if self.hide_canvas:- flags += ['--fingerprinting-canvas-image-data-noise']- if self.disable_webgl:- flags += ['--disable-webgl', '--disable-webgl-image-chromium', '--disable-webgl2']-# Creating the browserif self.cdp_url:- cdp_url = self._cdp_url_logic(flags if self.stealth else None)+ cdp_url = self._cdp_url_logic()browser = p.chromium.connect_over_cdp(endpoint_url=cdp_url)else:- if self.stealth:- browser = p.chromium.launch(- headless=self.headless, args=flags, ignore_default_args=self.harmful_default_args, chromium_sandbox=True, channel='chrome' if self.real_chrome else 'chromium'- )- else:- browser = p.chromium.launch(headless=self.headless, ignore_default_args=self.harmful_default_args, channel='chrome' if self.real_chrome else 'chromium')-- # Creating the context- if self.stealth:- context = browser.new_context(- locale=self.locale,- is_mobile=False,- has_touch=False,- proxy=self.proxy,- color_scheme='dark', # Bypasses the 'prefersLightColor' check in creepjs- user_agent=useragent,- device_scale_factor=2,- # I'm thinking about disabling it to rest from all Service Workers headache but let's keep it as it is for now- service_workers="allow",- ignore_https_errors=True,- extra_http_headers=extra_headers,- screen={"width": 1920, "height": 1080},- viewport={"width": 1920, "height": 1080},- permissions=["geolocation", 'notifications'],- )- else:- context = browser.new_context(- locale=self.locale,- proxy=self.proxy,- color_scheme='dark',- user_agent=useragent,- device_scale_factor=2,- extra_http_headers=extra_headers- )+ browser = p.chromium.launch(**self.__launch_kwargs())+ context = browser.new_context(**self.__context_kwargs())# Finally we are in businesspage = context.new_page()page.set_default_navigation_timeout(self.timeout)@@ -202,22 +217,8 @@ class PlaywrightEngine:page.route("**/*", intercept_route)if self.stealth:- # Basic bypasses nothing fancy as I'm still working on it- # But with adding these bypasses to the above config, it bypasses many online tests like- # https://bot.sannysoft.com/- # https://kaliiiiiiiiii.github.io/brotector/- # https://pixelscan.net/- # https://iphey.com/- # https://www.browserscan.net/bot-detection <== this one also checks for the CDP runtime fingerprint- # https://arh.antoinevastel.com/bots/areyouheadless/- # https://prescience-data.github.io/execution-monitor.html- page.add_init_script(path=js_bypass_path('webdriver_fully.js'))- page.add_init_script(path=js_bypass_path('window_chrome.js'))- page.add_init_script(path=js_bypass_path('navigator_plugins.js'))- page.add_init_script(path=js_bypass_path('pdf_viewer.js'))- page.add_init_script(path=js_bypass_path('notification_permission.js'))- page.add_init_script(path=js_bypass_path('screen_props.js'))- page.add_init_script(path=js_bypass_path('playwright_fingerprint.js'))+ for script in self.__stealth_scripts():+ page.add_init_script(path=script)res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None)page.wait_for_load_state(state="domcontentloaded")diff --git a/scrapling/engines/toolbelt/__init__.py b/scrapling/engines/toolbelt/__init__.pyindex 595929c..4f31f6a 100644--- a/scrapling/engines/toolbelt/__init__.py+++ b/scrapling/engines/toolbelt/__init__.py@@ -1,5 +1,6 @@from .custom import (BaseFetcher, Response, StatusText, check_if_engine_usable,- check_type_validity, do_nothing, get_variable_name)+ check_type_validity, do_nothing, do_nothing_async,+ get_variable_name)from .fingerprints import (generate_convincing_referer, generate_headers,get_os_name)from .navigation import (construct_cdp_url, construct_proxy_dict,diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.pyindex 6632b6b..0db3088 100644--- a/scrapling/engines/toolbelt/custom.py+++ b/scrapling/engines/toolbelt/custom.py@@ -302,3 +302,8 @@ def check_type_validity(variable: Any, valid_types: Union[List[Type], None], defdef do_nothing(page):# Just works as a filler for `page_action` argument in browser enginesreturn page+++async def do_nothing_async(page):+ # Just works as a filler for `page_action` argument in browser engines+ return page