🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
style: Removing dead code/docstrings and correcting type hints
Karim shoaira9d05cc0ef49
diff --git a/scrapling/core/custom_types.py b/scrapling/core/custom_types.pyindex eb7fa34..e9ece7d 100644--- a/scrapling/core/custom_types.py+++ b/scrapling/core/custom_types.py@@ -145,7 +145,7 @@ class TextHandler(str):clean_match: bool = False,case_sensitive: bool = True,check_match: Literal[False] = False,- ) -> "TextHandlers[TextHandler]": ...+ ) -> "TextHandlers": ...def re(self,@@ -241,7 +241,7 @@ class TextHandlers(List[TextHandler]):replace_entities: bool = True,clean_match: bool = False,case_sensitive: bool = True,- ) -> "TextHandlers[TextHandler]":+ ) -> "TextHandlers":"""Call the ``.re()`` method for each element in this list and returntheir results flattened as TextHandlers.diff --git a/scrapling/engines/_browsers/_page.py b/scrapling/engines/_browsers/_page.pyindex 8c80944..821fbbb 100644--- a/scrapling/engines/_browsers/_page.py+++ b/scrapling/engines/_browsers/_page.py@@ -6,7 +6,7 @@ from playwright.async_api import Page as AsyncPagefrom scrapling.core._types import Optional, List, Literal-PageState = Literal["finished", "ready", "busy", "error"] # States that a page can be in+PageState = Literal["ready", "busy", "error"] # States that a page can be in@dataclass@@ -62,12 +62,6 @@ class PagePool:"""Get the total number of pages"""return len(self.pages)- @property- def finished_count(self) -> int:- """Get the number of finished pages"""- with self._lock:- return sum(1 for p in self.pages if p.state == "finished")-@propertydef busy_count(self) -> int:"""Get the number of busy pages"""diff --git a/scrapling/engines/static.py b/scrapling/engines/static.pyindex 3f6cb79..6e5c0de 100644--- a/scrapling/engines/static.py+++ b/scrapling/engines/static.py@@ -94,8 +94,8 @@ class FetcherSession:self.default_http3 = http3self.selector_config = selector_config or {}- self._curl_session: Optional[CurlSession] = None- self._async_curl_session: Optional[AsyncCurlSession] = None+ self._curl_session: Optional[CurlSession] | bool = None+ self._async_curl_session: Optional[AsyncCurlSession] | bool = Nonedef _merge_request_args(self, **kwargs) -> Dict[str, Any]:"""Merge request-specific arguments with default session arguments."""@@ -239,7 +239,6 @@ class FetcherSession:Perform an HTTP request using the configured session.:param method: HTTP method to be used, supported methods are ["GET", "POST", "PUT", "DELETE"]- :param url: Target URL for the request.:param request_args: Arguments to be passed to the session's `request()` method.:param max_retries: Maximum number of retries for the request.:param retry_delay: Number of seconds to wait between retries.@@ -280,7 +279,6 @@ class FetcherSession:Perform an HTTP request using the configured session.:param method: HTTP method to be used, supported methods are ["GET", "POST", "PUT", "DELETE"]- :param url: Target URL for the request.:param request_args: Arguments to be passed to the session's `request()` method.:param max_retries: Maximum number of retries for the request.:param retry_delay: Number of seconds to wait between retries.diff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.pyindex f2f445c..ea5991b 100644--- a/scrapling/engines/toolbelt/navigation.py+++ b/scrapling/engines/toolbelt/navigation.py@@ -4,7 +4,7 @@ Functions related to files and URLsfrom pathlib import Pathfrom functools import lru_cache-from urllib.parse import urlencode, urlparse+from urllib.parse import urlparsefrom playwright.async_api import Route as async_Routefrom msgspec import Struct, structs, convert, ValidationErrordiff --git a/scrapling/parser.py b/scrapling/parser.pyindex 6260988..f0b9e54 100644--- a/scrapling/parser.py+++ b/scrapling/parser.py@@ -239,7 +239,7 @@ class Selector(SelectorsGeneration):)def __handle_element(- self, element: HtmlElement | _ElementUnicodeResult+ self, element: Optional[HtmlElement | _ElementUnicodeResult]) -> Optional[Union[TextHandler, "Selector"]]:"""Used internally in all functions to convert a single element to type (Selector|TextHandler) when possible"""if element is None:@@ -345,7 +345,7 @@ class Selector(SelectorsGeneration):return TextHandler(content)@property- def body(self):+ def body(self) -> str | bytes:"""Return the raw body of the current `Selector` without any processing. Useful for binary and non-HTML requests."""return self._raw_body