Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix: improve the handling of useragent in chrome fetcher

Karim shoairf9f34e6ee9c0
diff --git a/scrapling/engines/_browsers/_config_tools.py b/scrapling/engines/_browsers/_config_tools.py
index eaf28d6..f1f4171 100644
--- a/scrapling/engines/_browsers/_config_tools.py
+++ b/scrapling/engines/_browsers/_config_tools.py
@@ -10,6 +10,7 @@ from scrapling.engines.toolbelt.navigation import js_bypass_path
from scrapling.engines.toolbelt.fingerprints import generate_headers
__default_useragent__ = generate_headers(browser_mode=True).get("User-Agent")
+__default_chrome_useragent__ = generate_headers(browser_mode="chrome").get("User-Agent")
@lru_cache(1)
@@ -89,8 +90,13 @@ def _launch_kwargs(
"ignore_default_args": HARMFUL_DEFAULT_ARGS,
"channel": "chrome" if real_chrome else "chromium",
"extra_http_headers": extra_headers or tuple(),
- "user_agent": useragent or __default_useragent__,
}
+ # The default useragent in the headful is always correct now in the current versions of Playwright
+ if useragent:
+ launch_kwargs["user_agent"] = useragent
+ elif not useragent and headless:
+ launch_kwargs["user_agent"] = __default_chrome_useragent__ if real_chrome else __default_useragent__
+
if stealth:
stealth_args = base_args + _set_flags(hide_canvas, disable_webgl)
launch_kwargs.update(
diff --git a/scrapling/engines/toolbelt/fingerprints.py b/scrapling/engines/toolbelt/fingerprints.py
index 7b551d8..4e9263e 100644
--- a/scrapling/engines/toolbelt/fingerprints.py
+++ b/scrapling/engines/toolbelt/fingerprints.py
@@ -13,6 +13,9 @@ from scrapling.core._types import Dict, Literal, Tuple
__OS_NAME__ = platform_system()
OSName = Literal["linux", "macos", "windows"]
+# Current versions hardcoded for now (Playwright doesn't allow to know the version of a browser without launching it)
+chromium_version = 141
+chrome_version = 143
@lru_cache(10, typed=True)
@@ -46,7 +49,7 @@ def get_os_name() -> OSName | Tuple:
return SUPPORTED_OPERATING_SYSTEMS
-def generate_headers(browser_mode: bool = False) -> Dict:
+def generate_headers(browser_mode: bool | str = False) -> Dict:
"""Generate real browser-like headers using browserforge's generator
:param browser_mode: If enabled, the headers created are used for playwright, so it has to match everything
@@ -55,7 +58,8 @@ def generate_headers(browser_mode: bool = False) -> Dict:
# In the browser mode, we don't care about anything other than matching the OS and the browser type with the browser we are using,
# So we don't raise any inconsistency red flags while websites fingerprinting us
os_name = get_os_name()
- browsers = [Browser(name="chrome", min_version=141, max_version=141)]
+ ver = chrome_version if browser_mode and browser_mode == "chrome" else chromium_version
+ browsers = [Browser(name="chrome", min_version=ver, max_version=ver)]
if not browser_mode:
os_name = ("windows", "macos", "linux")
browsers.extend(