Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

style: Removing dead code/docstrings and correcting type hints

Karim shoaira9d05cc0ef49
diff --git a/scrapling/core/custom_types.py b/scrapling/core/custom_types.py
index eb7fa34..e9ece7d 100644
--- a/scrapling/core/custom_types.py
+++ b/scrapling/core/custom_types.py
@@ -145,7 +145,7 @@ class TextHandler(str):
clean_match: bool = False,
case_sensitive: bool = True,
check_match: Literal[False] = False,
- ) -> "TextHandlers[TextHandler]": ...
+ ) -> "TextHandlers": ...
def re(
self,
@@ -241,7 +241,7 @@ class TextHandlers(List[TextHandler]):
replace_entities: bool = True,
clean_match: bool = False,
case_sensitive: bool = True,
- ) -> "TextHandlers[TextHandler]":
+ ) -> "TextHandlers":
"""Call the ``.re()`` method for each element in this list and return
their results flattened as TextHandlers.
diff --git a/scrapling/engines/_browsers/_page.py b/scrapling/engines/_browsers/_page.py
index 8c80944..821fbbb 100644
--- a/scrapling/engines/_browsers/_page.py
+++ b/scrapling/engines/_browsers/_page.py
@@ -6,7 +6,7 @@ from playwright.async_api import Page as AsyncPage
from scrapling.core._types import Optional, List, Literal
-PageState = Literal["finished", "ready", "busy", "error"] # States that a page can be in
+PageState = Literal["ready", "busy", "error"] # States that a page can be in
@dataclass
@@ -62,12 +62,6 @@ class PagePool:
"""Get the total number of pages"""
return len(self.pages)
- @property
- def finished_count(self) -> int:
- """Get the number of finished pages"""
- with self._lock:
- return sum(1 for p in self.pages if p.state == "finished")
-
@property
def busy_count(self) -> int:
"""Get the number of busy pages"""
diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py
index 3f6cb79..6e5c0de 100644
--- a/scrapling/engines/static.py
+++ b/scrapling/engines/static.py
@@ -94,8 +94,8 @@ class FetcherSession:
self.default_http3 = http3
self.selector_config = selector_config or {}
- self._curl_session: Optional[CurlSession] = None
- self._async_curl_session: Optional[AsyncCurlSession] = None
+ self._curl_session: Optional[CurlSession] | bool = None
+ self._async_curl_session: Optional[AsyncCurlSession] | bool = None
def _merge_request_args(self, **kwargs) -> Dict[str, Any]:
"""Merge request-specific arguments with default session arguments."""
@@ -239,7 +239,6 @@ class FetcherSession:
Perform an HTTP request using the configured session.
:param method: HTTP method to be used, supported methods are ["GET", "POST", "PUT", "DELETE"]
- :param url: Target URL for the request.
:param request_args: Arguments to be passed to the session's `request()` method.
:param max_retries: Maximum number of retries for the request.
:param retry_delay: Number of seconds to wait between retries.
@@ -280,7 +279,6 @@ class FetcherSession:
Perform an HTTP request using the configured session.
:param method: HTTP method to be used, supported methods are ["GET", "POST", "PUT", "DELETE"]
- :param url: Target URL for the request.
:param request_args: Arguments to be passed to the session's `request()` method.
:param max_retries: Maximum number of retries for the request.
:param retry_delay: Number of seconds to wait between retries.
diff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.py
index f2f445c..ea5991b 100644
--- a/scrapling/engines/toolbelt/navigation.py
+++ b/scrapling/engines/toolbelt/navigation.py
@@ -4,7 +4,7 @@ Functions related to files and URLs
from pathlib import Path
from functools import lru_cache
-from urllib.parse import urlencode, urlparse
+from urllib.parse import urlparse
from playwright.async_api import Route as async_Route
from msgspec import Struct, structs, convert, ValidationError
diff --git a/scrapling/parser.py b/scrapling/parser.py
index 6260988..f0b9e54 100644
--- a/scrapling/parser.py
+++ b/scrapling/parser.py
@@ -239,7 +239,7 @@ class Selector(SelectorsGeneration):
)
def __handle_element(
- self, element: HtmlElement | _ElementUnicodeResult
+ self, element: Optional[HtmlElement | _ElementUnicodeResult]
) -> Optional[Union[TextHandler, "Selector"]]:
"""Used internally in all functions to convert a single element to type (Selector|TextHandler) when possible"""
if element is None:
@@ -345,7 +345,7 @@ class Selector(SelectorsGeneration):
return TextHandler(content)
@property
- def body(self):
+ def body(self) -> str | bytes:
"""Return the raw body of the current `Selector` without any processing. Useful for binary and non-HTML requests."""
return self._raw_body