🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
feat(spiders): The option to start sessions lazily
Karim shoair927cfd85ffe4
diff --git a/scrapling/spiders/session.py b/scrapling/spiders/session.pyindex 452d12b..f5b5a49 100644--- a/scrapling/spiders/session.py+++ b/scrapling/spiders/session.py@@ -1,7 +1,9 @@+from asyncio import Lock+from scrapling.spiders.request import Requestfrom scrapling.engines.static import _ASyncSessionLogicfrom scrapling.engines.toolbelt.convertor import Response-from scrapling.core._types import cast, SUPPORTED_HTTP_METHODS+from scrapling.core._types import Set, cast, SUPPORTED_HTTP_METHODSfrom scrapling.fetchers import AsyncDynamicSession, AsyncStealthySession, FetcherSessionSession = FetcherSession | AsyncDynamicSession | AsyncStealthySession@@ -14,13 +16,16 @@ class SessionManager:self._sessions: dict[str, Session] = {}self._default_session_id: str | None = Noneself._started: bool = False+ self._lazy_sessions: Set[str] = set()+ self._lazy_lock = Lock()- def add(self, session_id: str, session: Session, *, default: bool = False) -> "SessionManager":+ def add(self, session_id: str, session: Session, *, default: bool = False, lazy: bool = False) -> "SessionManager":"""Register a session instance.:param session_id: Name to reference this session in requests:param session: Your pre-configured session instance:param default: If True, this becomes the default session+ :param lazy: If True, the session will be started only when a request uses its ID."""if session_id in self._sessions:raise ValueError(f"Session '{session_id}' already registered")@@ -30,6 +35,9 @@ class SessionManager:if default or self._default_session_id is None:self._default_session_id = session_id+ if lazy:+ self._lazy_sessions.add(session_id)+return selfdef remove(self, session_id: str) -> None:@@ -48,6 +56,8 @@ class SessionManager:raise KeyError(f"Session '{session_id}' not found")session = self._sessions.pop(session_id)+ if session_id in self._lazy_sessions:+ self._lazy_sessions.remove(session_id)if session and self._default_session_id == session_id:self._default_session_id = next(iter(self._sessions), None)@@ -75,8 +85,8 @@ class SessionManager:if self._started:return- for session in self._sessions.values():- if not session._is_alive:+ for sid, session in self._sessions.items():+ if sid not in self._lazy_sessions and not session._is_alive:await session.__aenter__()self._started = True@@ -93,6 +103,11 @@ class SessionManager:session = self.get(sid)if session:+ if sid in self._lazy_sessions and not session._is_alive:+ async with self._lazy_lock:+ if not session._is_alive:+ await session.__aenter__()+if isinstance(session, FetcherSession):session = session._client