🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
refactor: internal API changes to be easily used as indicators for spiders
Karim shoair6a73f9dcd78a
diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.pyindex bb14b5f..cbbdc24 100644--- a/scrapling/engines/_browsers/_base.py+++ b/scrapling/engines/_browsers/_base.py@@ -37,14 +37,14 @@ class SyncSession:self._max_wait_for_page = 60self.playwright: Playwright | Any = Noneself.context: BrowserContext | Any = None- self._closed = False+ self._is_alive = Falsedef start(self):passdef close(self): # pragma: no cover"""Close all resources"""- if self._closed:+ if not self._is_alive:returnif self.context:@@ -55,7 +55,7 @@ class SyncSession:self.playwright.stop()self.playwright = None # pyright: ignore- self._closed = True+ self._is_alive = Falsedef __enter__(self):self.start()@@ -137,7 +137,7 @@ class AsyncSession:self._max_wait_for_page = 60self.playwright: AsyncPlaywright | Any = Noneself.context: AsyncBrowserContext | Any = None- self._closed = False+ self._is_alive = Falseself._lock = Lock()async def start(self):@@ -145,7 +145,7 @@ class AsyncSession:async def close(self):"""Close all resources"""- if self._closed: # pragma: no cover+ if not self._is_alive: # pragma: no coverreturnif self.context:@@ -156,7 +156,7 @@ class AsyncSession:await self.playwright.stop()self.playwright = None # pyright: ignore- self._closed = True+ self._is_alive = Falseasync def __aenter__(self):await self.start()diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.pyindex ff21438..285bb32 100644--- a/scrapling/engines/_browsers/_controllers.py+++ b/scrapling/engines/_browsers/_controllers.py@@ -31,7 +31,6 @@ class DynamicSession(SyncSession, DynamicSessionMixin):"_max_wait_for_page","playwright","context",- "_closed",)def __init__(self, **kwargs: Unpack[PlaywrightSession]):@@ -82,6 +81,8 @@ class DynamicSession(SyncSession, DynamicSessionMixin):if self._config.cookies: # pragma: no coverself.context.add_cookies(self._config.cookies)++ self._is_alive = Trueelse:raise RuntimeError("Session has been already started")@@ -104,7 +105,7 @@ class DynamicSession(SyncSession, DynamicSessionMixin)::return: A `Response` object."""params = _validate(kwargs, self, PlaywrightConfig)- if self._closed: # pragma: no cover+ if not self._is_alive: # pragma: no coverraise RuntimeError("Context manager has been closed")referer = (@@ -211,6 +212,8 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):if self._config.cookies:await self.context.add_cookies(self._config.cookies) # pyright: ignore++ self._is_alive = Trueelse:raise RuntimeError("Session has been already started")@@ -234,7 +237,7 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):"""params = _validate(kwargs, self, PlaywrightConfig)- if self._closed: # pragma: no cover+ if not self._is_alive: # pragma: no coverraise RuntimeError("Context manager has been closed")referer = (diff --git a/scrapling/engines/_browsers/_stealth.py b/scrapling/engines/_browsers/_stealth.pyindex 912d38e..1cce675 100644--- a/scrapling/engines/_browsers/_stealth.py+++ b/scrapling/engines/_browsers/_stealth.py@@ -39,7 +39,6 @@ class StealthySession(SyncSession, StealthySessionMixin):"_max_wait_for_page","playwright","context",- "_closed",)def __init__(self, **kwargs: Unpack[StealthSession]):@@ -191,7 +190,7 @@ class StealthySession(SyncSession, StealthySessionMixin)::return: A `Response` object."""params = _validate(kwargs, self, StealthConfig)- if self._closed: # pragma: no cover+ if not self._is_alive: # pragma: no coverraise RuntimeError("Context manager has been closed")referer = (@@ -404,7 +403,7 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):"""params = _validate(kwargs, self, StealthConfig)- if self._closed: # pragma: no cover+ if not self._is_alive: # pragma: no coverraise RuntimeError("Context manager has been closed")referer = (diff --git a/scrapling/engines/static.py b/scrapling/engines/static.pyindex c7afa79..2386820 100644--- a/scrapling/engines/static.py+++ b/scrapling/engines/static.py@@ -62,6 +62,7 @@ class _ConfigurationLogic(ABC):"_default_cert","_default_http3","selector_config",+ "_is_alive",)def __init__(self, **kwargs: Unpack[RequestsSession]):@@ -80,6 +81,7 @@ class _ConfigurationLogic(ABC):self._default_cert = kwargs.get("cert") or Noneself._default_http3 = kwargs.get("http3", False)self.selector_config = kwargs.get("selector_config") or {}+ self._is_alive = False@staticmethoddef _get_param(kwargs: Dict, key: str, default: Any) -> Any:@@ -183,10 +185,11 @@ class _SyncSessionLogic(_ConfigurationLogic):def __enter__(self):"""Creates and returns a new synchronous Fetcher Session"""- if self._curl_session:+ if self._is_alive:raise RuntimeError("This FetcherSession instance already has an active synchronous session.")self._curl_session = CurlSession()+ self._is_alive = Truereturn selfdef __exit__(self, exc_type, exc_val, exc_tb):@@ -201,7 +204,9 @@ class _SyncSessionLogic(_ConfigurationLogic):self._curl_session.close()self._curl_session = None- def __make_request(self, method: SUPPORTED_HTTP_METHODS, stealth: Optional[bool] = None, **kwargs) -> Response:+ self._is_alive = False++ def _make_request(self, method: SUPPORTED_HTTP_METHODS, stealth: Optional[bool] = None, **kwargs) -> Response:"""Perform an HTTP request using the configured session."""@@ -267,7 +272,7 @@ class _SyncSessionLogic(_ConfigurationLogic)::return: A `Response` object."""stealthy_headers = kwargs.pop("stealthy_headers", None)- return self.__make_request("GET", stealth=stealthy_headers, url=url, **kwargs)+ return self._make_request("GET", stealth=stealthy_headers, url=url, **kwargs)def post(self, url: str, **kwargs: Unpack[DataRequestParams]) -> Response:"""@@ -299,7 +304,7 @@ class _SyncSessionLogic(_ConfigurationLogic)::return: A `Response` object."""stealthy_headers = kwargs.pop("stealthy_headers", None)- return self.__make_request("POST", stealth=stealthy_headers, url=url, **kwargs)+ return self._make_request("POST", stealth=stealthy_headers, url=url, **kwargs)def put(self, url: str, **kwargs: Unpack[DataRequestParams]) -> Response:"""@@ -331,7 +336,7 @@ class _SyncSessionLogic(_ConfigurationLogic)::return: A `Response` object."""stealthy_headers = kwargs.pop("stealthy_headers", None)- return self.__make_request("PUT", stealth=stealthy_headers, url=url, **kwargs)+ return self._make_request("PUT", stealth=stealthy_headers, url=url, **kwargs)def delete(self, url: str, **kwargs: Unpack[DataRequestParams]) -> Response:"""@@ -365,7 +370,7 @@ class _SyncSessionLogic(_ConfigurationLogic):# Careful of sending a body in a DELETE request, it might cause some websites to reject the request as per https://www.rfc-editor.org/rfc/rfc7231#section-4.3.5,# But some websites accept it, it depends on the implementation used.stealthy_headers = kwargs.pop("stealthy_headers", None)- return self.__make_request("DELETE", stealth=stealthy_headers, url=url, **kwargs)+ return self._make_request("DELETE", stealth=stealthy_headers, url=url, **kwargs)class _ASyncSessionLogic(_ConfigurationLogic):@@ -377,10 +382,11 @@ class _ASyncSessionLogic(_ConfigurationLogic):async def __aenter__(self): # pragma: no cover"""Creates and returns a new asynchronous Session."""- if self._async_curl_session:+ if self._is_alive:raise RuntimeError("This FetcherSession instance already has an active asynchronous session.")self._async_curl_session = AsyncCurlSession()+ self._is_alive = Truereturn selfasync def __aexit__(self, exc_type, exc_val, exc_tb):@@ -395,9 +401,9 @@ class _ASyncSessionLogic(_ConfigurationLogic):await self._async_curl_session.close()self._async_curl_session = None- async def __make_request(- self, method: SUPPORTED_HTTP_METHODS, stealth: Optional[bool] = None, **kwargs- ) -> Response:+ self._is_alive = False++ async def _make_request(self, method: SUPPORTED_HTTP_METHODS, stealth: Optional[bool] = None, **kwargs) -> Response:"""Perform an HTTP request using the configured session."""@@ -465,7 +471,7 @@ class _ASyncSessionLogic(_ConfigurationLogic)::return: A `Response` object."""stealthy_headers = kwargs.pop("stealthy_headers", None)- return self.__make_request("GET", stealth=stealthy_headers, url=url, **kwargs)+ return self._make_request("GET", stealth=stealthy_headers, url=url, **kwargs)def post(self, url: str, **kwargs: Unpack[DataRequestParams]) -> Awaitable[Response]:"""@@ -497,7 +503,7 @@ class _ASyncSessionLogic(_ConfigurationLogic)::return: A `Response` object."""stealthy_headers = kwargs.pop("stealthy_headers", None)- return self.__make_request("POST", stealth=stealthy_headers, url=url, **kwargs)+ return self._make_request("POST", stealth=stealthy_headers, url=url, **kwargs)def put(self, url: str, **kwargs: Unpack[DataRequestParams]) -> Awaitable[Response]:"""@@ -529,7 +535,7 @@ class _ASyncSessionLogic(_ConfigurationLogic)::return: A `Response` object."""stealthy_headers = kwargs.pop("stealthy_headers", None)- return self.__make_request("PUT", stealth=stealthy_headers, url=url, **kwargs)+ return self._make_request("PUT", stealth=stealthy_headers, url=url, **kwargs)def delete(self, url: str, **kwargs: Unpack[DataRequestParams]) -> Awaitable[Response]:"""@@ -563,7 +569,7 @@ class _ASyncSessionLogic(_ConfigurationLogic):# Careful of sending a body in a DELETE request, it might cause some websites to reject the request as per https://www.rfc-editor.org/rfc/rfc7231#section-4.3.5,# But some websites accept it, it depends on the implementation used.stealthy_headers = kwargs.pop("stealthy_headers", None)- return self.__make_request("DELETE", stealth=stealthy_headers, url=url, **kwargs)+ return self._make_request("DELETE", stealth=stealthy_headers, url=url, **kwargs)class FetcherSession: