🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
Toolbelt fingerprint functions - Doc strings and type annotations stuff
Karim shoair544d23b696f4
diff --git a/scrapling/engines/toolbelt/__init__.py b/scrapling/engines/toolbelt/__init__.pyindex 7e8368a..08b559b 100644--- a/scrapling/engines/toolbelt/__init__.py+++ b/scrapling/engines/toolbelt/__init__.py@@ -2,7 +2,6 @@ from .fingerprints import (get_os_name,generate_headers,generate_convincing_referer,- generate_suitable_fingerprint,)from .custom import (Response,diff --git a/scrapling/engines/toolbelt/fingerprints.py b/scrapling/engines/toolbelt/fingerprints.pyindex 3969b0a..76f40da 100644--- a/scrapling/engines/toolbelt/fingerprints.py+++ b/scrapling/engines/toolbelt/fingerprints.py@@ -4,27 +4,32 @@ Functions related to generating headers and fingerprints generallyimport platform+from scrapling.core._types import Union, Dict+from tldextract import extract-from browserforge.fingerprints import FingerprintGeneratorfrom browserforge.headers import HeaderGenerator, Browser+from browserforge.fingerprints import FingerprintGenerator, Fingerprint-def generate_convincing_referer(url):- """- Takes the domain from the URL without the subdomain/suffix and make it look like you were searching google for this website+def generate_convincing_referer(url: str) -> str:+ """Takes the domain from the URL without the subdomain/suffix and make it look like you were searching google for this website>>> generate_convincing_referer('https://www.somewebsite.com/blah')'https://www.google.com/search?q=somewebsite':param url: The URL you are about to fetch.- :return:+ :return: Google's search URL of the domain name"""website_name = extract(url).domainreturn f'https://www.google.com/search?q={website_name}'-def get_os_name():- # Get the OS name in the same format needed for browserforge+def get_os_name() -> Union[str, None]:+ """Get the current OS name in the same format needed for browserforge++ :return: Current OS name or `None` otherwise+ """+ #os_name = platform.system()return {'Linux': 'linux',@@ -35,17 +40,25 @@ def get_os_name():}.get(os_name)-def generate_suitable_fingerprint():- # This would be for Browserforge playwright injector- os_name = get_os_name()+def generate_suitable_fingerprint() -> Fingerprint:+ """Generates a browserforge's fingerprint that matches current OS, desktop device, and Chrome with version 128 at least.++ This function was originally created to test Browserforge's injector.+ :return: `Fingerprint` object+ """return FingerprintGenerator(browser=[Browser(name='chrome', min_version=128)],- os=os_name, # None is ignored+ os=get_os_name(), # None is ignoreddevice='desktop').generate()-def generate_headers(browser_mode=False):+def generate_headers(browser_mode: bool = False) -> Dict:+ """Generate real browser-like headers using browserforge's generator++ :param browser_mode: If enabled, the headers created are used for playwright so it have to match everything+ :return: A dictionary of the generated headers+ """if browser_mode:# In this mode we don't care about anything other than matching the OS and the browser type with the browser we are using# So we don't raise any inconsistency red flags while websites fingerprinting us