🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
refactor(fingerprints): Remove unused function and improve another
Karim shoair7efbcd33c37f
diff --git a/scrapling/engines/toolbelt/__init__.py b/scrapling/engines/toolbelt/__init__.pyindex b5a6c95..796b89f 100644--- a/scrapling/engines/toolbelt/__init__.py+++ b/scrapling/engines/toolbelt/__init__.py@@ -6,7 +6,12 @@ from .custom import (check_type_validity,get_variable_name,)-from .fingerprints import generate_convincing_referer, generate_headers, get_os_name+from .fingerprints import (+ generate_convincing_referer,+ generate_headers,+ get_os_name,+ __default_useragent__,+)from .navigation import (async_intercept_route,construct_cdp_url,diff --git a/scrapling/engines/toolbelt/fingerprints.py b/scrapling/engines/toolbelt/fingerprints.pyindex 186e4f3..a623039 100644--- a/scrapling/engines/toolbelt/fingerprints.py+++ b/scrapling/engines/toolbelt/fingerprints.py@@ -4,9 +4,8 @@ Functions related to generating headers and fingerprints generallyfrom platform import system as platform_system-from browserforge.fingerprints import Fingerprint, FingerprintGenerator-from browserforge.headers import Browser, HeaderGeneratorfrom tldextract import extract+from browserforge.headers import Browser, HeaderGeneratorfrom scrapling.core._types import Dict, Unionfrom scrapling.core.utils import lru_cache@@ -43,39 +42,26 @@ def get_os_name() -> Union[str, None]:}.get(__OS_NAME__)-def generate_suitable_fingerprint() -> Fingerprint:- """Generates a browserforge's fingerprint that matches the current OS, desktop device, and Chrome with version 128 at least.-- This function was originally created to test Browserforge's injector.- :return: `Fingerprint` object- """- return FingerprintGenerator(- browser=[Browser(name="chrome", min_version=128)],- os=get_os_name(), # None is ignored- device="desktop",- ).generate()--def generate_headers(browser_mode: bool = False) -> Dict:"""Generate real browser-like headers using browserforge's generator:param browser_mode: If enabled, the headers created are used for playwright, so it has to match everything:return: A dictionary of the generated headers"""- if browser_mode:- # In this mode we don't care about anything other than matching the OS and the browser type with the browser we are using,- # So we don't raise any inconsistency red flags while websites fingerprinting us- os_name = get_os_name()- return HeaderGenerator(- browser=[Browser(name="chrome", min_version=130)],- os=os_name, # None is ignored- device="desktop",- ).generate()- else:- # Here it's used for normal requests that aren't done through browsers- browsers = [- Browser(name="chrome", min_version=130),- Browser(name="firefox", min_version=130),- Browser(name="edge", min_version=130),- ]- return HeaderGenerator(browser=browsers, device="desktop").generate()+ # In the browser mode, we don't care about anything other than matching the OS and the browser type with the browser we are using,+ # So we don't raise any inconsistency red flags while websites fingerprinting us+ os_name = get_os_name()+ browsers = [Browser(name="chrome", min_version=130)]+ if not browser_mode:+ os_name = None+ browsers.extend(+ [+ Browser(name="firefox", min_version=130),+ Browser(name="edge", min_version=130),+ ]+ )++ return HeaderGenerator(browser=browsers, os=os_name, device="desktop").generate()+++__default_useragent__ = generate_headers(browser_mode=False).get("User-Agent")