🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(browser version): extract current chrome version from playwright package data
Karim shoair2fb5e429f280
diff --git a/pyproject.toml b/pyproject.tomlindex d1944ff..455ee28 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -73,10 +73,10 @@ dependencies = [fetchers = ["click>=8.3.0","curl_cffi>=0.15.0",- "playwright==1.61.0",- "patchright==1.61.2",+ "playwright>=1.61.0",+ "patchright>=1.61.2","browserforge>=1.2.4",- "apify-fingerprint-datapoints>=0.13.0",+ "apify-fingerprint-datapoints>=0.15.0","msgspec>=0.21.1","anyio>=4.13.0","protego>=0.6.2",diff --git a/scrapling/engines/toolbelt/fingerprints.py b/scrapling/engines/toolbelt/fingerprints.pyindex 5aa074e..bb77ea5 100644--- a/scrapling/engines/toolbelt/fingerprints.py+++ b/scrapling/engines/toolbelt/fingerprints.py@@ -2,58 +2,91 @@Functions related to generating headers and fingerprints generally"""+import re+import sysfrom functools import lru_cache+from importlib.util import find_spec+from pathlib import Pathfrom platform import system as platform_systemfrom browserforge.headers import Browser, HeaderGenerator-from browserforge.headers.generator import SUPPORTED_OPERATING_SYSTEMS+from browserforge.headers.generator import SUPPORTED_DEVICES, SUPPORTED_OPERATING_SYSTEMS+from orjson import loadsfrom scrapling.core._types import Dict, Literal, Tuple__OS_NAME__ = platform_system()-OSName = Literal["linux", "macos", "windows"]-# Current versions hardcoded for now (Playwright doesn't allow to know the version of a browser without launching it)-chromium_version = 149-chrome_version = 149+OSName = Literal["linux", "macos", "windows", "android", "ios"]+DESKTOP_OS = ("linux", "macos", "windows")@lru_cache(1, typed=True)def get_os_name() -> OSName | Tuple:- """Get the current OS name in the same format needed for browserforge, if the OS is Unknown, return None so browserforge uses all.+ """Get the current OS name in the same format needed for browserforge, if the OS is Unknown, return all the OSes browserforge supports.- :return: Current OS name or `None` otherwise+ :return: Current OS name or all supported OSes otherwise"""match __OS_NAME__: # pragma: no covercase "Linux":- return "linux"+ return "android" if hasattr(sys, "getandroidapilevel") else "linux"case "Darwin":return "macos"case "Windows":return "windows"+ case "Android":+ return "android"+ case "iOS" | "iPadOS":+ return "ios"case _:return SUPPORTED_OPERATING_SYSTEMS+@lru_cache(2, typed=True)+def driven_browser_version(package: str = "playwright") -> int | None:+ """Get the Chromium major version the installed automation package drives, read from its bundled `browsers.json`.++ :param package: The automation package to inspect, e.g. `"playwright"` or `"patchright"`+ :return: The Chromium major version, or `None` if it couldn't be determined+ """+ spec = find_spec(package)+ if spec is None or not spec.origin:+ return None+ path = Path(spec.origin).parent / "driver" / "package" / "browsers.json"+ if not path.is_file():+ return None+ try:+ browsers = loads(path.read_bytes()).get("browsers", [])+ version = next((entry["browserVersion"] for entry in browsers if entry.get("name") == "chromium"), None)+ return int(version.partition(".")[0]) if version else None+ except (ValueError, TypeError, KeyError):+ return None++def generate_headers(browser_mode: bool | str = False) -> Dict:"""Generate real browser-like headers using browserforge's generator:param browser_mode: If enabled, the headers created are used for playwright, so it has to match everything:return: A dictionary of the generated headers"""- # In the browser mode, we don't care about anything other than matching the OS and the browser type with the browser we are using,- # So we don't raise any inconsistency red flags while websites fingerprinting usos_name = get_os_name()- ver = chrome_version if browser_mode and browser_mode == "chrome" else chromium_version- browsers = [Browser(name="chrome", min_version=ver, max_version=ver)]+ device: str | Tuple = (+ "desktop" if os_name in DESKTOP_OS else "mobile" if isinstance(os_name, str) else SUPPORTED_DEVICES+ )+ if not browser_mode and os_name in DESKTOP_OS:+ os_name = DESKTOP_OS+ browsers = [Browser(name="chrome", min_version=149)]if not browser_mode:- os_name = ("windows", "macos", "linux")- browsers.extend(- [- Browser(name="firefox", min_version=142),- Browser(name="edge", min_version=140),- ]- )- return HeaderGenerator(browser=browsers, os=os_name, device="desktop").generate()+ browsers.extend([Browser(name="firefox", min_version=147), Browser(name="edge", min_version=145)])+ try:+ headers = HeaderGenerator(browser=browsers, os=os_name, device=device).generate()+ except ValueError:+ headers = HeaderGenerator(+ browser=[Browser(name=b.name) for b in browsers], os=os_name, device=device+ ).generate()+ if browser_mode and (driven := driven_browser_version() or driven_browser_version("patchright")):+ if user_agent := headers.get("User-Agent"):+ headers["User-Agent"] = re.sub(r"Chrome/[\d.]+", f"Chrome/{driven}.0.0.0", user_agent)+ return headers__default_useragent__ = generate_headers(browser_mode=False).get("User-Agent")diff --git a/tests/fetchers/test_utils.py b/tests/fetchers/test_utils.pyindex 4679478..35eeb85 100644--- a/tests/fetchers/test_utils.py+++ b/tests/fetchers/test_utils.py@@ -1,3 +1,5 @@+import re+import pytestfrom scrapling.engines.toolbelt.convertor import ResponseFactory@@ -8,7 +10,14 @@ from scrapling.engines.toolbelt.navigation import (create_async_intercept_handler,_is_domain_blocked,)-from scrapling.engines.toolbelt.fingerprints import get_os_name, generate_headers+from browserforge.headers.generator import SUPPORTED_OPERATING_SYSTEMS++from scrapling.engines.toolbelt import fingerprints+from scrapling.engines.toolbelt.fingerprints import (+ get_os_name,+ generate_headers,+ driven_browser_version,+)@pytest.fixture@@ -216,9 +225,7 @@ class TestFingerprintFunctions:"""Test OS name detection"""result = get_os_name()- # Should return one of the known OS names or None- valid_names = ["linux", "macos", "windows", "ios"]- assert result is None or result in valid_names+ assert result in SUPPORTED_OPERATING_SYSTEMS or result == SUPPORTED_OPERATING_SYSTEMSdef test_generate_headers_basic(self):"""Test basic header generation"""@@ -235,6 +242,65 @@ class TestFingerprintFunctions:assert isinstance(headers, dict)assert "User-Agent" in headers+ def test_driven_browser_version(self):+ """Test that the driven Chromium version is read from the installed automation package"""+ assert isinstance(driven_browser_version("playwright"), int)+ assert driven_browser_version("not_a_real_package") is None++ @pytest.mark.parametrize("browser_mode", [True, "chrome"])+ def test_browser_useragent_follows_the_driven_browser(self, monkeypatch, browser_mode):+ """Test that both browser-facing User-Agents track the real driven version, not the fingerprints data"""+ monkeypatch.setattr(fingerprints, "get_os_name", lambda: "macos")+ # A version far above anything the fingerprints data ships, to prove the data no longer caps it+ monkeypatch.setattr(fingerprints, "driven_browser_version", lambda *args: 999)++ assert "Chrome/999.0.0.0" in generate_headers(browser_mode=browser_mode)["User-Agent"]++ def test_http_mode_useragent_is_not_rewritten(self, monkeypatch):+ """Test that the HTTP-mode User-Agent keeps its fingerprints version since it isn't fed to a browser"""+ monkeypatch.setattr(fingerprints, "get_os_name", lambda: "macos")+ monkeypatch.setattr(fingerprints, "driven_browser_version", lambda *args: 999)++ assert "999" not in generate_headers(browser_mode=False)["User-Agent"]++ def test_browser_mode_useragent_falls_back_without_a_driven_version(self, monkeypatch):+ """Test that the User-Agent still generates on the minimum version when the driven version can't be read"""+ monkeypatch.setattr(fingerprints, "get_os_name", lambda: "macos")+ monkeypatch.setattr(fingerprints, "driven_browser_version", lambda *args: None)++ major = int(re.search(r"Chrome/(\d+)", generate_headers(browser_mode=True)["User-Agent"]).group(1))+ assert major >= fingerprints.MINIMUM_VERSION++ def test_generate_headers_survives_unavailable_minimum(self, monkeypatch):+ """Test that generation falls back instead of crashing when the data has nothing at or above the minimum"""+ monkeypatch.setattr(fingerprints, "get_os_name", lambda: "macos")+ monkeypatch.setattr(fingerprints, "MINIMUM_VERSION", 9999)++ assert "Chrome/" in generate_headers(browser_mode=True)["User-Agent"]++ @pytest.mark.parametrize("os_name", ["linux", "macos", "windows"])+ def test_generate_headers_desktop_os_stays_desktop(self, monkeypatch, os_name):+ """Test that desktop OSes never get a mobile User-Agent"""+ monkeypatch.setattr(fingerprints, "get_os_name", lambda: os_name)+ useragent = generate_headers(browser_mode=True)["User-Agent"]++ assert "Android" not in useragent+ assert "Mobile" not in useragent++ def test_generate_headers_android_is_mobile(self, monkeypatch):+ """Test that Android hosts get a mobile User-Agent instead of a desktop one"""+ monkeypatch.setattr(fingerprints, "get_os_name", lambda: "android")++ for browser_mode in (False, True, "chrome"):+ assert "Android" in generate_headers(browser_mode=browser_mode)["User-Agent"]++ def test_generate_headers_unknown_os(self, monkeypatch):+ """Test that header generation still works when the OS can't be detected"""+ monkeypatch.setattr(fingerprints, "get_os_name", lambda: SUPPORTED_OPERATING_SYSTEMS)+ headers = generate_headers(browser_mode=True)++ assert len(headers["User-Agent"]) > 0+class TestResponse:"""Test Response class functionality"""