🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
style: General type hints fixes and imports optimizing
Karim shoair9bcb9e9d9308
diff --git a/ruff.toml b/ruff.tomlindex 04dadf0..a579697 100644--- a/ruff.toml+++ b/ruff.toml@@ -15,7 +15,7 @@ target-version = "py39"[lint]select = ["E", "F", "W"]-ignore = ["E501", "F401"]+ignore = ["E501", "F401", "F811"][format]# Like Black, use double quotes for strings.diff --git a/scrapling/core/_types.py b/scrapling/core/_types.pyindex 2ed107f..a41e077 100644--- a/scrapling/core/_types.py+++ b/scrapling/core/_types.py@@ -4,6 +4,7 @@ Type definitions for type checking purposes.from typing import (TYPE_CHECKING,+ overload,Any,Callable,Dict,diff --git a/scrapling/core/custom_types.py b/scrapling/core/custom_types.pyindex 4afc926..52dfe2e 100644--- a/scrapling/core/custom_types.py+++ b/scrapling/core/custom_types.py@@ -7,14 +7,15 @@ from orjson import dumps, loadsfrom scrapling.core._types import (Dict,- Iterable,List,+ Union,+ TypeVar,Literal,- Optional,Pattern,+ Iterable,+ Optional,+ Generator,SupportsIndex,- TypeVar,- Union,)from scrapling.core.utils import _is_iterable, flattenfrom scrapling.core._html_utils import _replace_entities@@ -341,7 +342,9 @@ class AttributesHandler(Mapping[str, _TextHandlerType]):"""Acts like the standard dictionary `.get()` method"""return self._data.get(key, default)- def search_values(self, keyword, partial=False):+ def search_values(+ self, keyword: str, partial: bool = False+ ) -> Generator["AttributesHandler", None, None]:"""Search current attributes by values and return a dictionary of each matching item:param keyword: The keyword to search for in the attribute values:param partial: If True, the function will search if keyword in each value instead of perfect matchdiff --git a/scrapling/core/storage.py b/scrapling/core/storage.pyindex 5821707..03ca612 100644--- a/scrapling/core/storage.py+++ b/scrapling/core/storage.py@@ -9,7 +9,7 @@ from orjson import dumps, loadsfrom tldextract import extract as tldfrom scrapling.core.utils import _StorageTools, log-from scrapling.core._types import Dict, Optional, Union+from scrapling.core._types import Dict, Optional, Union, Anyclass StorageSystemMixin(ABC):@@ -106,7 +106,7 @@ class SQLiteStorageSystem(StorageSystemMixin):""")self.connection.commit()- def save(self, element: HtmlElement, identifier: str):+ def save(self, element: HtmlElement, identifier: str) -> None:"""Saves the elements unique properties to the storage for retrieval and relocation later:param element: The element itself which we want to save to storage.@@ -126,7 +126,7 @@ class SQLiteStorageSystem(StorageSystemMixin):self.cursor.fetchall()self.connection.commit()- def retrieve(self, identifier: str) -> Optional[Dict]:+ def retrieve(self, identifier: str) -> Optional[Dict[str, Any]]:"""Using the identifier, we search the storage and return the unique properties of the element:param identifier: This is the identifier that will be used to retrieve the element from the storage. Seediff --git a/scrapling/core/utils.py b/scrapling/core/utils.pyindex e33c914..0219cb0 100644--- a/scrapling/core/utils.py+++ b/scrapling/core/utils.py@@ -5,7 +5,7 @@ from itertools import chainimport orjsonfrom lxml import html-from scrapling.core._types import Any, Dict, Iterable, Union+from scrapling.core._types import Any, Dict, Iterable, Union, List# Using cache on top of a class is a brilliant way to achieve a Singleton design pattern without much codefrom functools import lru_cache # isort:skip@@ -41,8 +41,8 @@ def setup_logger():log = setup_logger()-def is_jsonable(content: Union[bytes, str]) -> bool:- if type(content) is bytes:+def is_jsonable(content: bytes | str) -> bool:+ if isinstance(content, bytes):content = content.decode()try:@@ -52,14 +52,14 @@ def is_jsonable(content: Union[bytes, str]) -> bool:return False-def flatten(lst: Iterable):+def flatten(lst: Iterable[Any]) -> List[Any]:return list(chain.from_iterable(lst))-def _is_iterable(s: Any):+def _is_iterable(obj: Any) -> bool:# This will be used only in regex functions to make sure it's iterable but not string/bytesreturn isinstance(- s,+ obj,(list,tuple,diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.pyindex 60a8dcf..e6557ff 100644--- a/scrapling/engines/_browsers/_validators.py+++ b/scrapling/engines/_browsers/_validators.py@@ -82,7 +82,7 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False):"""Configuration struct for validation"""max_pages: int = 1- headless: Union[bool] = True # noqa: F821+ headless: bool = True # noqa: F821block_images: bool = Falsedisable_resources: bool = Falseblock_webrtc: bool = Falsediff --git a/scrapling/parser.py b/scrapling/parser.pyindex daddc9d..05d2384 100644--- a/scrapling/parser.py+++ b/scrapling/parser.py@@ -1,7 +1,6 @@-import inspectimport osimport re-import typing+from inspect import signaturefrom difflib import SequenceMatcherfrom urllib.parse import urljoin@@ -18,16 +17,17 @@ from lxml.etree import (from scrapling.core._types import (Any,- Callable,Dict,- Generator,- Iterable,List,- Optional,- Pattern,- SupportsIndex,Tuple,Union,+ Pattern,+ Callable,+ Optional,+ Iterable,+ overload,+ Generator,+ SupportsIndex,)from scrapling.core.custom_types import AttributesHandler, TextHandler, TextHandlersfrom scrapling.core.mixins import SelectorsGeneration@@ -248,7 +248,7 @@ class Selector(SelectorsGeneration):def __handle_elements(self, result: List[Union[HtmlElement, _ElementUnicodeResult]]- ) -> Union["Selectors", "TextHandlers", List]:+ ) -> Union["Selectors", "TextHandlers"]:"""Used internally in all functions to convert results to type (Selectors|TextHandlers) in bulk when possible"""if not len(result@@ -761,7 +761,7 @@ class Selector(SelectorsGeneration):patterns.add(arg)elif callable(arg):- if len(inspect.signature(arg).parameters) > 0:+ if len(signature(arg).parameters) > 0:functions.append(arg)else:raise TypeError(@@ -914,7 +914,7 @@ class Selector(SelectorsGeneration):return round((score / checks) * 100, 2)@staticmethod- def __calculate_dict_diff(dict1: dict, dict2: dict) -> float:+ def __calculate_dict_diff(dict1: Dict, dict2: Dict) -> float:"""Used internally to calculate similarity between two dictionaries as SequenceMatcher doesn't accept dictionaries"""score = (SequenceMatcher(None, tuple(dict1.keys()), tuple(dict2.keys())).ratio()@@ -1210,11 +1210,11 @@ class Selectors(List[Selector]):__slots__ = ()- @typing.overload+ @overloaddef __getitem__(self, pos: SupportsIndex) -> Selector:pass- @typing.overload+ @overloaddef __getitem__(self, pos: slice) -> "Selectors":pass