🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
Small speed boost for fetching by some caching
Karim shoair5ad21260305c
diff --git a/scrapling/engines/toolbelt/fingerprints.py b/scrapling/engines/toolbelt/fingerprints.pyindex 76f40da..71b8e84 100644--- a/scrapling/engines/toolbelt/fingerprints.py+++ b/scrapling/engines/toolbelt/fingerprints.py@@ -4,6 +4,7 @@ Functions related to generating headers and fingerprints generallyimport platform+from scrapling.core.utils import cachefrom scrapling.core._types import Union, Dictfrom tldextract import extract@@ -11,6 +12,7 @@ from browserforge.headers import HeaderGenerator, Browserfrom browserforge.fingerprints import FingerprintGenerator, Fingerprint+@cache(None, typed=True)def generate_convincing_referer(url: str) -> str:"""Takes the domain from the URL without the subdomain/suffix and make it look like you were searching google for this website@@ -24,6 +26,7 @@ def generate_convincing_referer(url: str) -> str:return f'https://www.google.com/search?q={website_name}'+@cache(None, typed=True)def get_os_name() -> Union[str, None]:"""Get the current OS name in the same format needed for browserforgediff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.pyindex e03cfd7..cf73a39 100644--- a/scrapling/engines/toolbelt/navigation.py+++ b/scrapling/engines/toolbelt/navigation.py@@ -6,6 +6,7 @@ import osimport loggingfrom urllib.parse import urlparse, urlencode+from scrapling.core.utils import cachefrom scrapling.core._types import Union, Dict, Optionalfrom scrapling.engines.constants import DEFAULT_DISABLED_RESOURCES@@ -62,6 +63,7 @@ def construct_cdp_url(cdp_url: str, query_params: Optional[Dict] = None) -> str:raise ValueError(f"Invalid CDP URL: {str(e)}")+@cache(None, typed=True)def js_bypass_path(filename: str) -> str:"""Takes the base filename of JS file inside the `bypasses` folder then return the full path of it