Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix: prevent FetcherSession state corruption and lazy session close crash

yetvalf4186ab9987f
diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py
index 1f4b09b..b74c730 100644
--- a/scrapling/engines/static.py
+++ b/scrapling/engines/static.py
@@ -716,8 +716,13 @@ class FetcherSession:
config["selector_config"] = self.selector_config
config["proxy_rotator"] = self._proxy_rotator
self._client = _SyncSessionLogic(**config)
+ try:
+ result = self._client.__enter__()
+ except Exception:
+ self._client = None
+ raise
self._is_alive = True
- return self._client.__enter__()
+ return result
raise RuntimeError("This FetcherSession instance already has an active synchronous session.")
def __exit__(self, exc_type, exc_val, exc_tb):
@@ -737,8 +742,13 @@ class FetcherSession:
config["selector_config"] = self.selector_config
config["proxy_rotator"] = self._proxy_rotator
self._client = _ASyncSessionLogic(**config)
+ try:
+ result = await self._client.__aenter__()
+ except Exception:
+ self._client = None
+ raise
self._is_alive = True
- return await self._client.__aenter__()
+ return result
raise RuntimeError("This FetcherSession instance already has an active asynchronous session.")
async def __aexit__(self, exc_type, exc_val, exc_tb):
diff --git a/scrapling/spiders/session.py b/scrapling/spiders/session.py
index 536be6d..5799e8c 100644
--- a/scrapling/spiders/session.py
+++ b/scrapling/spiders/session.py
@@ -93,7 +93,9 @@ class SessionManager:
async def close(self) -> None:
"""Close all registered sessions."""
- for session in self._sessions.values():
+ for sid, session in self._sessions.items():
+ if sid in self._lazy_sessions and not session._is_alive:
+ continue
_ = await session.__aexit__(None, None, None)
self._started = False