🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix: improve the handling of useragent in chrome fetcher
Karim shoairf9f34e6ee9c0
diff --git a/scrapling/engines/_browsers/_config_tools.py b/scrapling/engines/_browsers/_config_tools.pyindex eaf28d6..f1f4171 100644--- a/scrapling/engines/_browsers/_config_tools.py+++ b/scrapling/engines/_browsers/_config_tools.py@@ -10,6 +10,7 @@ from scrapling.engines.toolbelt.navigation import js_bypass_pathfrom scrapling.engines.toolbelt.fingerprints import generate_headers__default_useragent__ = generate_headers(browser_mode=True).get("User-Agent")+__default_chrome_useragent__ = generate_headers(browser_mode="chrome").get("User-Agent")@lru_cache(1)@@ -89,8 +90,13 @@ def _launch_kwargs("ignore_default_args": HARMFUL_DEFAULT_ARGS,"channel": "chrome" if real_chrome else "chromium","extra_http_headers": extra_headers or tuple(),- "user_agent": useragent or __default_useragent__,}+ # The default useragent in the headful is always correct now in the current versions of Playwright+ if useragent:+ launch_kwargs["user_agent"] = useragent+ elif not useragent and headless:+ launch_kwargs["user_agent"] = __default_chrome_useragent__ if real_chrome else __default_useragent__+if stealth:stealth_args = base_args + _set_flags(hide_canvas, disable_webgl)launch_kwargs.update(diff --git a/scrapling/engines/toolbelt/fingerprints.py b/scrapling/engines/toolbelt/fingerprints.pyindex 7b551d8..4e9263e 100644--- a/scrapling/engines/toolbelt/fingerprints.py+++ b/scrapling/engines/toolbelt/fingerprints.py@@ -13,6 +13,9 @@ from scrapling.core._types import Dict, Literal, Tuple__OS_NAME__ = platform_system()OSName = Literal["linux", "macos", "windows"]+# Current versions hardcoded for now (Playwright doesn't allow to know the version of a browser without launching it)+chromium_version = 141+chrome_version = 143@lru_cache(10, typed=True)@@ -46,7 +49,7 @@ def get_os_name() -> OSName | Tuple:return SUPPORTED_OPERATING_SYSTEMS-def generate_headers(browser_mode: bool = False) -> Dict:+def generate_headers(browser_mode: bool | str = False) -> Dict:"""Generate real browser-like headers using browserforge's generator:param browser_mode: If enabled, the headers created are used for playwright, so it has to match everything@@ -55,7 +58,8 @@ def generate_headers(browser_mode: bool = False) -> Dict:# In the browser mode, we don't care about anything other than matching the OS and the browser type with the browser we are using,# So we don't raise any inconsistency red flags while websites fingerprinting usos_name = get_os_name()- browsers = [Browser(name="chrome", min_version=141, max_version=141)]+ ver = chrome_version if browser_mode and browser_mode == "chrome" else chromium_version+ browsers = [Browser(name="chrome", min_version=ver, max_version=ver)]if not browser_mode:os_name = ("windows", "macos", "linux")browsers.extend(