Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

style: General type hints fixes and imports optimizing

Karim shoair9bcb9e9d9308
diff --git a/ruff.toml b/ruff.toml
index 04dadf0..a579697 100644
--- a/ruff.toml
+++ b/ruff.toml
@@ -15,7 +15,7 @@ target-version = "py39"
[lint]
select = ["E", "F", "W"]
-ignore = ["E501", "F401"]
+ignore = ["E501", "F401", "F811"]
[format]
# Like Black, use double quotes for strings.
diff --git a/scrapling/core/_types.py b/scrapling/core/_types.py
index 2ed107f..a41e077 100644
--- a/scrapling/core/_types.py
+++ b/scrapling/core/_types.py
@@ -4,6 +4,7 @@ Type definitions for type checking purposes.
from typing import (
TYPE_CHECKING,
+ overload,
Any,
Callable,
Dict,
diff --git a/scrapling/core/custom_types.py b/scrapling/core/custom_types.py
index 4afc926..52dfe2e 100644
--- a/scrapling/core/custom_types.py
+++ b/scrapling/core/custom_types.py
@@ -7,14 +7,15 @@ from orjson import dumps, loads
from scrapling.core._types import (
Dict,
- Iterable,
List,
+ Union,
+ TypeVar,
Literal,
- Optional,
Pattern,
+ Iterable,
+ Optional,
+ Generator,
SupportsIndex,
- TypeVar,
- Union,
)
from scrapling.core.utils import _is_iterable, flatten
from scrapling.core._html_utils import _replace_entities
@@ -341,7 +342,9 @@ class AttributesHandler(Mapping[str, _TextHandlerType]):
"""Acts like the standard dictionary `.get()` method"""
return self._data.get(key, default)
- def search_values(self, keyword, partial=False):
+ def search_values(
+ self, keyword: str, partial: bool = False
+ ) -> Generator["AttributesHandler", None, None]:
"""Search current attributes by values and return a dictionary of each matching item
:param keyword: The keyword to search for in the attribute values
:param partial: If True, the function will search if keyword in each value instead of perfect match
diff --git a/scrapling/core/storage.py b/scrapling/core/storage.py
index 5821707..03ca612 100644
--- a/scrapling/core/storage.py
+++ b/scrapling/core/storage.py
@@ -9,7 +9,7 @@ from orjson import dumps, loads
from tldextract import extract as tld
from scrapling.core.utils import _StorageTools, log
-from scrapling.core._types import Dict, Optional, Union
+from scrapling.core._types import Dict, Optional, Union, Any
class StorageSystemMixin(ABC):
@@ -106,7 +106,7 @@ class SQLiteStorageSystem(StorageSystemMixin):
""")
self.connection.commit()
- def save(self, element: HtmlElement, identifier: str):
+ def save(self, element: HtmlElement, identifier: str) -> None:
"""Saves the elements unique properties to the storage for retrieval and relocation later
:param element: The element itself which we want to save to storage.
@@ -126,7 +126,7 @@ class SQLiteStorageSystem(StorageSystemMixin):
self.cursor.fetchall()
self.connection.commit()
- def retrieve(self, identifier: str) -> Optional[Dict]:
+ def retrieve(self, identifier: str) -> Optional[Dict[str, Any]]:
"""Using the identifier, we search the storage and return the unique properties of the element
:param identifier: This is the identifier that will be used to retrieve the element from the storage. See
diff --git a/scrapling/core/utils.py b/scrapling/core/utils.py
index e33c914..0219cb0 100644
--- a/scrapling/core/utils.py
+++ b/scrapling/core/utils.py
@@ -5,7 +5,7 @@ from itertools import chain
import orjson
from lxml import html
-from scrapling.core._types import Any, Dict, Iterable, Union
+from scrapling.core._types import Any, Dict, Iterable, Union, List
# Using cache on top of a class is a brilliant way to achieve a Singleton design pattern without much code
from functools import lru_cache # isort:skip
@@ -41,8 +41,8 @@ def setup_logger():
log = setup_logger()
-def is_jsonable(content: Union[bytes, str]) -> bool:
- if type(content) is bytes:
+def is_jsonable(content: bytes | str) -> bool:
+ if isinstance(content, bytes):
content = content.decode()
try:
@@ -52,14 +52,14 @@ def is_jsonable(content: Union[bytes, str]) -> bool:
return False
-def flatten(lst: Iterable):
+def flatten(lst: Iterable[Any]) -> List[Any]:
return list(chain.from_iterable(lst))
-def _is_iterable(s: Any):
+def _is_iterable(obj: Any) -> bool:
# This will be used only in regex functions to make sure it's iterable but not string/bytes
return isinstance(
- s,
+ obj,
(
list,
tuple,
diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py
index 60a8dcf..e6557ff 100644
--- a/scrapling/engines/_browsers/_validators.py
+++ b/scrapling/engines/_browsers/_validators.py
@@ -82,7 +82,7 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False):
"""Configuration struct for validation"""
max_pages: int = 1
- headless: Union[bool] = True # noqa: F821
+ headless: bool = True # noqa: F821
block_images: bool = False
disable_resources: bool = False
block_webrtc: bool = False
diff --git a/scrapling/parser.py b/scrapling/parser.py
index daddc9d..05d2384 100644
--- a/scrapling/parser.py
+++ b/scrapling/parser.py
@@ -1,7 +1,6 @@
-import inspect
import os
import re
-import typing
+from inspect import signature
from difflib import SequenceMatcher
from urllib.parse import urljoin
@@ -18,16 +17,17 @@ from lxml.etree import (
from scrapling.core._types import (
Any,
- Callable,
Dict,
- Generator,
- Iterable,
List,
- Optional,
- Pattern,
- SupportsIndex,
Tuple,
Union,
+ Pattern,
+ Callable,
+ Optional,
+ Iterable,
+ overload,
+ Generator,
+ SupportsIndex,
)
from scrapling.core.custom_types import AttributesHandler, TextHandler, TextHandlers
from scrapling.core.mixins import SelectorsGeneration
@@ -248,7 +248,7 @@ class Selector(SelectorsGeneration):
def __handle_elements(
self, result: List[Union[HtmlElement, _ElementUnicodeResult]]
- ) -> Union["Selectors", "TextHandlers", List]:
+ ) -> Union["Selectors", "TextHandlers"]:
"""Used internally in all functions to convert results to type (Selectors|TextHandlers) in bulk when possible"""
if not len(
result
@@ -761,7 +761,7 @@ class Selector(SelectorsGeneration):
patterns.add(arg)
elif callable(arg):
- if len(inspect.signature(arg).parameters) > 0:
+ if len(signature(arg).parameters) > 0:
functions.append(arg)
else:
raise TypeError(
@@ -914,7 +914,7 @@ class Selector(SelectorsGeneration):
return round((score / checks) * 100, 2)
@staticmethod
- def __calculate_dict_diff(dict1: dict, dict2: dict) -> float:
+ def __calculate_dict_diff(dict1: Dict, dict2: Dict) -> float:
"""Used internally to calculate similarity between two dictionaries as SequenceMatcher doesn't accept dictionaries"""
score = (
SequenceMatcher(None, tuple(dict1.keys()), tuple(dict2.keys())).ratio()
@@ -1210,11 +1210,11 @@ class Selectors(List[Selector]):
__slots__ = ()
- @typing.overload
+ @overload
def __getitem__(self, pos: SupportsIndex) -> Selector:
pass
- @typing.overload
+ @overload
def __getitem__(self, pos: slice) -> "Selectors":
pass