🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix: Limit caching in all the library
Karim shoair7bef44ea4caa
diff --git a/scrapling/core/storage_adaptors.py b/scrapling/core/storage_adaptors.pyindex c614643..d991111 100644--- a/scrapling/core/storage_adaptors.py+++ b/scrapling/core/storage_adaptors.py@@ -19,7 +19,7 @@ class StorageSystemMixin(ABC):"""self.url = url- @lru_cache(None, typed=True)+ @lru_cache(126, typed=True)def _get_base_url(self, default_value: str = 'default') -> str:if not self.url or type(self.url) is not str:return default_value@@ -51,7 +51,7 @@ class StorageSystemMixin(ABC):raise NotImplementedError('Storage system must implement `save` method')@staticmethod- @lru_cache(None, typed=True)+ @lru_cache(256, typed=True)def _get_hash(identifier: str) -> str:"""If you want to hash identifier in your storage system, use this safer"""identifier = identifier.lower().strip()@@ -63,7 +63,7 @@ class StorageSystemMixin(ABC):return f"{hash_value}_{len(identifier)}" # Length to reduce collision chance-@lru_cache(None, typed=True)+@lru_cache(10, typed=True)class SQLiteStorageSystem(StorageSystemMixin):"""The recommended system to use, it's race condition safe and thread safe.Mainly built so the library can run in threaded frameworks like scrapy or threaded toolsdiff --git a/scrapling/core/translator.py b/scrapling/core/translator.pyindex cb7f064..263a24a 100644--- a/scrapling/core/translator.py+++ b/scrapling/core/translator.py@@ -139,6 +139,6 @@ class TranslatorMixin:class HTMLTranslator(TranslatorMixin, OriginalHTMLTranslator):- @lru_cache(maxsize=2048)+ @lru_cache(maxsize=256)def css_to_xpath(self, css: str, prefix: str = "descendant-or-self::") -> str:return super().css_to_xpath(css, prefix)diff --git a/scrapling/core/utils.py b/scrapling/core/utils.pyindex 27cb884..e9de112 100644--- a/scrapling/core/utils.py+++ b/scrapling/core/utils.py@@ -115,7 +115,7 @@ class _StorageTools:# return _impl-@lru_cache(None, typed=True)+@lru_cache(256, typed=True)def clean_spaces(string):string = string.replace('\t', ' ')string = re.sub('[\n|\r]', '', string)diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.pyindex ce60a51..45a0ff3 100644--- a/scrapling/engines/pw.py+++ b/scrapling/engines/pw.py@@ -126,7 +126,7 @@ class PlaywrightEngine:return cdp_url- @lru_cache(typed=True)+ @lru_cache(126, typed=True)def __set_flags(self):"""Returns the flags that will be used while launching the browser if stealth mode is enabled"""flags = DEFAULT_STEALTH_FLAGS@@ -169,7 +169,7 @@ class PlaywrightEngine:return context_kwargs- @lru_cache()+ @lru_cache(10)def __stealth_scripts(self):# Basic bypasses nothing fancy as I'm still working on it# But with adding these bypasses to the above config, it bypasses many online tests likediff --git a/scrapling/engines/static.py b/scrapling/engines/static.pyindex 86d6c68..f19b503 100644--- a/scrapling/engines/static.py+++ b/scrapling/engines/static.py@@ -7,7 +7,7 @@ from scrapling.core.utils import log, lru_cachefrom .toolbelt import Response, generate_convincing_referer, generate_headers-@lru_cache(typed=True)+@lru_cache(5, typed=True) # Singleton easilyclass StaticEngine:def __init__(self, url: str, proxy: Optional[str] = None, stealthy_headers: bool = True, follow_redirects: bool = True,diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.pyindex dd31f4f..f00f649 100644--- a/scrapling/engines/toolbelt/custom.py+++ b/scrapling/engines/toolbelt/custom.py@@ -16,7 +16,7 @@ class ResponseEncoding:__ISO_8859_1_CONTENT_TYPES = {"text/plain", "text/html", "text/css", "text/javascript"}@classmethod- @lru_cache(maxsize=None)+ @lru_cache(maxsize=256)def __parse_content_type(cls, header_value: str) -> Tuple[str, Dict[str, str]]:"""Parse content type and parameters from a content-type header value.@@ -38,7 +38,7 @@ class ResponseEncoding:return content_type, params@classmethod- @lru_cache(maxsize=None)+ @lru_cache(maxsize=256)def get_value(cls, content_type: Optional[str], text: Optional[str] = 'test') -> str:"""Determine the appropriate character encoding from a content-type header.diff --git a/scrapling/engines/toolbelt/fingerprints.py b/scrapling/engines/toolbelt/fingerprints.pyindex a7bf633..cfdcf8a 100644--- a/scrapling/engines/toolbelt/fingerprints.py+++ b/scrapling/engines/toolbelt/fingerprints.py@@ -12,7 +12,7 @@ from scrapling.core._types import Dict, Unionfrom scrapling.core.utils import lru_cache-@lru_cache(None, typed=True)+@lru_cache(128, typed=True)def generate_convincing_referer(url: str) -> str:"""Takes the domain from the URL without the subdomain/suffix and make it look like you were searching google for this website@@ -26,7 +26,7 @@ def generate_convincing_referer(url: str) -> str:return f'https://www.google.com/search?q={website_name}'-@lru_cache(None, typed=True)+@lru_cache(128, typed=True)def get_os_name() -> Union[str, None]:"""Get the current OS name in the same format needed for browserforgediff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.pyindex 14a26d0..4d39fb5 100644--- a/scrapling/engines/toolbelt/navigation.py+++ b/scrapling/engines/toolbelt/navigation.py@@ -110,7 +110,7 @@ def construct_cdp_url(cdp_url: str, query_params: Optional[Dict] = None) -> str:raise ValueError(f"Invalid CDP URL: {str(e)}")-@lru_cache(None, typed=True)+@lru_cache(126, typed=True)def js_bypass_path(filename: str) -> str:"""Takes the base filename of JS file inside the `bypasses` folder then return the full path of it