Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

pref(requests): Optimizing memory by specifiying slots

Karim shoairc487faf4fdd3
diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py
index edeaa2f..2b5ba0a 100644
--- a/scrapling/engines/_browsers/_validators.py
+++ b/scrapling/engines/_browsers/_validators.py
@@ -21,7 +21,7 @@ from scrapling.engines.toolbelt.navigation import construct_proxy_dict
# Custom validators for msgspec
@lru_cache(8)
-def _is_invalid_file_path(value: str) -> bool | str:
+def _is_invalid_file_path(value: str) -> bool | str: # pragma: no cover
"""Fast file path validation"""
path = Path(value)
if not path.exists():
@@ -33,7 +33,7 @@ def _is_invalid_file_path(value: str) -> bool | str:
return False
-def _validate_addon_path(value: str) -> None:
+def _validate_addon_path(value: str) -> None: # pragma: no cover
"""Fast addon path validation"""
path = Path(value)
if not path.exists():
@@ -49,7 +49,7 @@ def _is_invalid_cdp_url(cdp_url: str) -> bool | str:
return "CDP URL must use 'ws://' or 'wss://' scheme"
netloc = urlparse(cdp_url).netloc
- if not netloc:
+ if not netloc: # pragma: no cover
return "Invalid hostname for the CDP URL"
return False
@@ -89,7 +89,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):
selector_config: Optional[Dict] = {}
additional_args: Optional[Dict] = {}
- def __post_init__(self):
+ def __post_init__(self): # pragma: no cover
"""Custom validation after msgspec validation"""
if self.page_action and not callable(self.page_action):
raise TypeError(f"page_action must be callable, got {type(self.page_action).__name__}")
@@ -195,7 +195,7 @@ class _fetch_params:
def validate_fetch(
params: List[Tuple], model: type[PlaywrightConfig] | type[CamoufoxConfig], sentinel=None
-) -> _fetch_params:
+) -> _fetch_params: # pragma: no cover
result = {}
overrides = {}
diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py
index 7251209..796df13 100644
--- a/scrapling/engines/static.py
+++ b/scrapling/engines/static.py
@@ -46,6 +46,24 @@ def _select_random_browser(impersonate: ImpersonateType) -> Optional[BrowserType
class _ConfigurationLogic(ABC):
# Core Logic Handler (Internal Engine)
+ __slots__ = (
+ "_default_impersonate",
+ "_stealth",
+ "_default_proxies",
+ "_default_proxy",
+ "_default_proxy_auth",
+ "_default_timeout",
+ "_default_headers",
+ "_default_retries",
+ "_default_retry_delay",
+ "_default_follow_redirects",
+ "_default_max_redirects",
+ "_default_verify",
+ "_default_cert",
+ "_default_http3",
+ "selector_config",
+ )
+
def __init__(self, **kwargs: Unpack[RequestsSession]):
self._default_impersonate = kwargs.get("impersonate", "chrome")
self._stealth = kwargs.get("stealthy_headers", True)
@@ -157,6 +175,8 @@ class _ConfigurationLogic(ABC):
class _SyncSessionLogic(_ConfigurationLogic):
+ __slots__ = ("_curl_session",)
+
def __init__(self, **kwargs: Unpack[RequestsSession]):
super().__init__(**kwargs)
self._curl_session: Optional[CurlSession] = None
@@ -343,6 +363,8 @@ class _SyncSessionLogic(_ConfigurationLogic):
class _ASyncSessionLogic(_ConfigurationLogic):
+ __slots__ = ("_async_curl_session",)
+
def __init__(self, **kwargs: Unpack[RequestsSession]):
super().__init__(**kwargs)
self._async_curl_session: Optional[AsyncCurlSession] = None
@@ -549,6 +571,25 @@ class FetcherSession:
same manager instance while a session is already active is disallowed.
"""
+ __slots__ = (
+ "_default_impersonate",
+ "_stealth",
+ "_default_proxies",
+ "_default_proxy",
+ "_default_proxy_auth",
+ "_default_timeout",
+ "_default_headers",
+ "_default_retries",
+ "_default_retry_delay",
+ "_default_follow_redirects",
+ "_default_max_redirects",
+ "_default_verify",
+ "_default_cert",
+ "_default_http3",
+ "selector_config",
+ "_client",
+ )
+
def __init__(
self,
impersonate: ImpersonateType = "chrome",
@@ -640,6 +681,8 @@ class FetcherSession:
class FetcherClient(_SyncSessionLogic):
+ __slots__ = ("__enter__", "__exit__")
+
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.__enter__: Any = None
@@ -648,6 +691,8 @@ class FetcherClient(_SyncSessionLogic):
class AsyncFetcherClient(_ASyncSessionLogic):
+ __slots__ = ("__aenter__", "__aexit__")
+
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.__aenter__: Any = None