🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
pref(requests): Optimizing memory by specifiying slots
Karim shoairc487faf4fdd3
diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.pyindex edeaa2f..2b5ba0a 100644--- a/scrapling/engines/_browsers/_validators.py+++ b/scrapling/engines/_browsers/_validators.py@@ -21,7 +21,7 @@ from scrapling.engines.toolbelt.navigation import construct_proxy_dict# Custom validators for msgspec@lru_cache(8)-def _is_invalid_file_path(value: str) -> bool | str:+def _is_invalid_file_path(value: str) -> bool | str: # pragma: no cover"""Fast file path validation"""path = Path(value)if not path.exists():@@ -33,7 +33,7 @@ def _is_invalid_file_path(value: str) -> bool | str:return False-def _validate_addon_path(value: str) -> None:+def _validate_addon_path(value: str) -> None: # pragma: no cover"""Fast addon path validation"""path = Path(value)if not path.exists():@@ -49,7 +49,7 @@ def _is_invalid_cdp_url(cdp_url: str) -> bool | str:return "CDP URL must use 'ws://' or 'wss://' scheme"netloc = urlparse(cdp_url).netloc- if not netloc:+ if not netloc: # pragma: no coverreturn "Invalid hostname for the CDP URL"return False@@ -89,7 +89,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):selector_config: Optional[Dict] = {}additional_args: Optional[Dict] = {}- def __post_init__(self):+ def __post_init__(self): # pragma: no cover"""Custom validation after msgspec validation"""if self.page_action and not callable(self.page_action):raise TypeError(f"page_action must be callable, got {type(self.page_action).__name__}")@@ -195,7 +195,7 @@ class _fetch_params:def validate_fetch(params: List[Tuple], model: type[PlaywrightConfig] | type[CamoufoxConfig], sentinel=None-) -> _fetch_params:+) -> _fetch_params: # pragma: no coverresult = {}overrides = {}diff --git a/scrapling/engines/static.py b/scrapling/engines/static.pyindex 7251209..796df13 100644--- a/scrapling/engines/static.py+++ b/scrapling/engines/static.py@@ -46,6 +46,24 @@ def _select_random_browser(impersonate: ImpersonateType) -> Optional[BrowserTypeclass _ConfigurationLogic(ABC):# Core Logic Handler (Internal Engine)+ __slots__ = (+ "_default_impersonate",+ "_stealth",+ "_default_proxies",+ "_default_proxy",+ "_default_proxy_auth",+ "_default_timeout",+ "_default_headers",+ "_default_retries",+ "_default_retry_delay",+ "_default_follow_redirects",+ "_default_max_redirects",+ "_default_verify",+ "_default_cert",+ "_default_http3",+ "selector_config",+ )+def __init__(self, **kwargs: Unpack[RequestsSession]):self._default_impersonate = kwargs.get("impersonate", "chrome")self._stealth = kwargs.get("stealthy_headers", True)@@ -157,6 +175,8 @@ class _ConfigurationLogic(ABC):class _SyncSessionLogic(_ConfigurationLogic):+ __slots__ = ("_curl_session",)+def __init__(self, **kwargs: Unpack[RequestsSession]):super().__init__(**kwargs)self._curl_session: Optional[CurlSession] = None@@ -343,6 +363,8 @@ class _SyncSessionLogic(_ConfigurationLogic):class _ASyncSessionLogic(_ConfigurationLogic):+ __slots__ = ("_async_curl_session",)+def __init__(self, **kwargs: Unpack[RequestsSession]):super().__init__(**kwargs)self._async_curl_session: Optional[AsyncCurlSession] = None@@ -549,6 +571,25 @@ class FetcherSession:same manager instance while a session is already active is disallowed."""+ __slots__ = (+ "_default_impersonate",+ "_stealth",+ "_default_proxies",+ "_default_proxy",+ "_default_proxy_auth",+ "_default_timeout",+ "_default_headers",+ "_default_retries",+ "_default_retry_delay",+ "_default_follow_redirects",+ "_default_max_redirects",+ "_default_verify",+ "_default_cert",+ "_default_http3",+ "selector_config",+ "_client",+ )+def __init__(self,impersonate: ImpersonateType = "chrome",@@ -640,6 +681,8 @@ class FetcherSession:class FetcherClient(_SyncSessionLogic):+ __slots__ = ("__enter__", "__exit__")+def __init__(self, *args, **kwargs):super().__init__(*args, **kwargs)self.__enter__: Any = None@@ -648,6 +691,8 @@ class FetcherClient(_SyncSessionLogic):class AsyncFetcherClient(_ASyncSessionLogic):+ __slots__ = ("__aenter__", "__aexit__")+def __init__(self, *args, **kwargs):super().__init__(*args, **kwargs)self.__aenter__: Any = None