Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(browser version): extract current chrome version from playwright package data

Karim shoair2fb5e429f280
diff --git a/pyproject.toml b/pyproject.toml
index d1944ff..455ee28 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -73,10 +73,10 @@ dependencies = [
fetchers = [
"click>=8.3.0",
"curl_cffi>=0.15.0",
- "playwright==1.61.0",
- "patchright==1.61.2",
+ "playwright>=1.61.0",
+ "patchright>=1.61.2",
"browserforge>=1.2.4",
- "apify-fingerprint-datapoints>=0.13.0",
+ "apify-fingerprint-datapoints>=0.15.0",
"msgspec>=0.21.1",
"anyio>=4.13.0",
"protego>=0.6.2",
diff --git a/scrapling/engines/toolbelt/fingerprints.py b/scrapling/engines/toolbelt/fingerprints.py
index 5aa074e..bb77ea5 100644
--- a/scrapling/engines/toolbelt/fingerprints.py
+++ b/scrapling/engines/toolbelt/fingerprints.py
@@ -2,58 +2,91 @@
Functions related to generating headers and fingerprints generally
"""
+import re
+import sys
from functools import lru_cache
+from importlib.util import find_spec
+from pathlib import Path
from platform import system as platform_system
from browserforge.headers import Browser, HeaderGenerator
-from browserforge.headers.generator import SUPPORTED_OPERATING_SYSTEMS
+from browserforge.headers.generator import SUPPORTED_DEVICES, SUPPORTED_OPERATING_SYSTEMS
+from orjson import loads
from scrapling.core._types import Dict, Literal, Tuple
__OS_NAME__ = platform_system()
-OSName = Literal["linux", "macos", "windows"]
-# Current versions hardcoded for now (Playwright doesn't allow to know the version of a browser without launching it)
-chromium_version = 149
-chrome_version = 149
+OSName = Literal["linux", "macos", "windows", "android", "ios"]
+DESKTOP_OS = ("linux", "macos", "windows")
@lru_cache(1, typed=True)
def get_os_name() -> OSName | Tuple:
- """Get the current OS name in the same format needed for browserforge, if the OS is Unknown, return None so browserforge uses all.
+ """Get the current OS name in the same format needed for browserforge, if the OS is Unknown, return all the OSes browserforge supports.
- :return: Current OS name or `None` otherwise
+ :return: Current OS name or all supported OSes otherwise
"""
match __OS_NAME__: # pragma: no cover
case "Linux":
- return "linux"
+ return "android" if hasattr(sys, "getandroidapilevel") else "linux"
case "Darwin":
return "macos"
case "Windows":
return "windows"
+ case "Android":
+ return "android"
+ case "iOS" | "iPadOS":
+ return "ios"
case _:
return SUPPORTED_OPERATING_SYSTEMS
+@lru_cache(2, typed=True)
+def driven_browser_version(package: str = "playwright") -> int | None:
+ """Get the Chromium major version the installed automation package drives, read from its bundled `browsers.json`.
+
+ :param package: The automation package to inspect, e.g. `"playwright"` or `"patchright"`
+ :return: The Chromium major version, or `None` if it couldn't be determined
+ """
+ spec = find_spec(package)
+ if spec is None or not spec.origin:
+ return None
+ path = Path(spec.origin).parent / "driver" / "package" / "browsers.json"
+ if not path.is_file():
+ return None
+ try:
+ browsers = loads(path.read_bytes()).get("browsers", [])
+ version = next((entry["browserVersion"] for entry in browsers if entry.get("name") == "chromium"), None)
+ return int(version.partition(".")[0]) if version else None
+ except (ValueError, TypeError, KeyError):
+ return None
+
+
def generate_headers(browser_mode: bool | str = False) -> Dict:
"""Generate real browser-like headers using browserforge's generator
:param browser_mode: If enabled, the headers created are used for playwright, so it has to match everything
:return: A dictionary of the generated headers
"""
- # In the browser mode, we don't care about anything other than matching the OS and the browser type with the browser we are using,
- # So we don't raise any inconsistency red flags while websites fingerprinting us
os_name = get_os_name()
- ver = chrome_version if browser_mode and browser_mode == "chrome" else chromium_version
- browsers = [Browser(name="chrome", min_version=ver, max_version=ver)]
+ device: str | Tuple = (
+ "desktop" if os_name in DESKTOP_OS else "mobile" if isinstance(os_name, str) else SUPPORTED_DEVICES
+ )
+ if not browser_mode and os_name in DESKTOP_OS:
+ os_name = DESKTOP_OS
+ browsers = [Browser(name="chrome", min_version=149)]
if not browser_mode:
- os_name = ("windows", "macos", "linux")
- browsers.extend(
- [
- Browser(name="firefox", min_version=142),
- Browser(name="edge", min_version=140),
- ]
- )
- return HeaderGenerator(browser=browsers, os=os_name, device="desktop").generate()
+ browsers.extend([Browser(name="firefox", min_version=147), Browser(name="edge", min_version=145)])
+ try:
+ headers = HeaderGenerator(browser=browsers, os=os_name, device=device).generate()
+ except ValueError:
+ headers = HeaderGenerator(
+ browser=[Browser(name=b.name) for b in browsers], os=os_name, device=device
+ ).generate()
+ if browser_mode and (driven := driven_browser_version() or driven_browser_version("patchright")):
+ if user_agent := headers.get("User-Agent"):
+ headers["User-Agent"] = re.sub(r"Chrome/[\d.]+", f"Chrome/{driven}.0.0.0", user_agent)
+ return headers
__default_useragent__ = generate_headers(browser_mode=False).get("User-Agent")
diff --git a/tests/fetchers/test_utils.py b/tests/fetchers/test_utils.py
index 4679478..35eeb85 100644
--- a/tests/fetchers/test_utils.py
+++ b/tests/fetchers/test_utils.py
@@ -1,3 +1,5 @@
+import re
+
import pytest
from scrapling.engines.toolbelt.convertor import ResponseFactory
@@ -8,7 +10,14 @@ from scrapling.engines.toolbelt.navigation import (
create_async_intercept_handler,
_is_domain_blocked,
)
-from scrapling.engines.toolbelt.fingerprints import get_os_name, generate_headers
+from browserforge.headers.generator import SUPPORTED_OPERATING_SYSTEMS
+
+from scrapling.engines.toolbelt import fingerprints
+from scrapling.engines.toolbelt.fingerprints import (
+ get_os_name,
+ generate_headers,
+ driven_browser_version,
+)
@pytest.fixture
@@ -216,9 +225,7 @@ class TestFingerprintFunctions:
"""Test OS name detection"""
result = get_os_name()
- # Should return one of the known OS names or None
- valid_names = ["linux", "macos", "windows", "ios"]
- assert result is None or result in valid_names
+ assert result in SUPPORTED_OPERATING_SYSTEMS or result == SUPPORTED_OPERATING_SYSTEMS
def test_generate_headers_basic(self):
"""Test basic header generation"""
@@ -235,6 +242,65 @@ class TestFingerprintFunctions:
assert isinstance(headers, dict)
assert "User-Agent" in headers
+ def test_driven_browser_version(self):
+ """Test that the driven Chromium version is read from the installed automation package"""
+ assert isinstance(driven_browser_version("playwright"), int)
+ assert driven_browser_version("not_a_real_package") is None
+
+ @pytest.mark.parametrize("browser_mode", [True, "chrome"])
+ def test_browser_useragent_follows_the_driven_browser(self, monkeypatch, browser_mode):
+ """Test that both browser-facing User-Agents track the real driven version, not the fingerprints data"""
+ monkeypatch.setattr(fingerprints, "get_os_name", lambda: "macos")
+ # A version far above anything the fingerprints data ships, to prove the data no longer caps it
+ monkeypatch.setattr(fingerprints, "driven_browser_version", lambda *args: 999)
+
+ assert "Chrome/999.0.0.0" in generate_headers(browser_mode=browser_mode)["User-Agent"]
+
+ def test_http_mode_useragent_is_not_rewritten(self, monkeypatch):
+ """Test that the HTTP-mode User-Agent keeps its fingerprints version since it isn't fed to a browser"""
+ monkeypatch.setattr(fingerprints, "get_os_name", lambda: "macos")
+ monkeypatch.setattr(fingerprints, "driven_browser_version", lambda *args: 999)
+
+ assert "999" not in generate_headers(browser_mode=False)["User-Agent"]
+
+ def test_browser_mode_useragent_falls_back_without_a_driven_version(self, monkeypatch):
+ """Test that the User-Agent still generates on the minimum version when the driven version can't be read"""
+ monkeypatch.setattr(fingerprints, "get_os_name", lambda: "macos")
+ monkeypatch.setattr(fingerprints, "driven_browser_version", lambda *args: None)
+
+ major = int(re.search(r"Chrome/(\d+)", generate_headers(browser_mode=True)["User-Agent"]).group(1))
+ assert major >= fingerprints.MINIMUM_VERSION
+
+ def test_generate_headers_survives_unavailable_minimum(self, monkeypatch):
+ """Test that generation falls back instead of crashing when the data has nothing at or above the minimum"""
+ monkeypatch.setattr(fingerprints, "get_os_name", lambda: "macos")
+ monkeypatch.setattr(fingerprints, "MINIMUM_VERSION", 9999)
+
+ assert "Chrome/" in generate_headers(browser_mode=True)["User-Agent"]
+
+ @pytest.mark.parametrize("os_name", ["linux", "macos", "windows"])
+ def test_generate_headers_desktop_os_stays_desktop(self, monkeypatch, os_name):
+ """Test that desktop OSes never get a mobile User-Agent"""
+ monkeypatch.setattr(fingerprints, "get_os_name", lambda: os_name)
+ useragent = generate_headers(browser_mode=True)["User-Agent"]
+
+ assert "Android" not in useragent
+ assert "Mobile" not in useragent
+
+ def test_generate_headers_android_is_mobile(self, monkeypatch):
+ """Test that Android hosts get a mobile User-Agent instead of a desktop one"""
+ monkeypatch.setattr(fingerprints, "get_os_name", lambda: "android")
+
+ for browser_mode in (False, True, "chrome"):
+ assert "Android" in generate_headers(browser_mode=browser_mode)["User-Agent"]
+
+ def test_generate_headers_unknown_os(self, monkeypatch):
+ """Test that header generation still works when the OS can't be detected"""
+ monkeypatch.setattr(fingerprints, "get_os_name", lambda: SUPPORTED_OPERATING_SYSTEMS)
+ headers = generate_headers(browser_mode=True)
+
+ assert len(headers["User-Agent"]) > 0
+
class TestResponse:
"""Test Response class functionality"""