Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

style(autocompletion): Improving type hints for custom types

Karim shoair5047e886a52d
diff --git a/scrapling/core/_types.py b/scrapling/core/_types.py
index 84e9a51..a175077 100644
--- a/scrapling/core/_types.py
+++ b/scrapling/core/_types.py
@@ -3,7 +3,8 @@ Type definitions for type checking purposes.
"""
from typing import (TYPE_CHECKING, Any, Callable, Dict, Generator, Iterable,
- List, Literal, Optional, Pattern, Tuple, Type, Union)
+ List, Literal, Optional, Pattern, Tuple, Type, TypeVar,
+ Union)
SelectorWaitStates = Literal["attached", "detached", "hidden", "visible"]
diff --git a/scrapling/core/custom_types.py b/scrapling/core/custom_types.py
index 0419406..4a6a4c3 100644
--- a/scrapling/core/custom_types.py
+++ b/scrapling/core/custom_types.py
@@ -5,9 +5,13 @@ from types import MappingProxyType
from orjson import dumps, loads
from w3lib.html import replace_entities as _replace_entities
-from scrapling.core._types import Dict, List, Pattern, SupportsIndex, Union
+from scrapling.core._types import (Dict, Iterable, List, Optional, Pattern,
+ SupportsIndex, TypeVar, Union)
from scrapling.core.utils import _is_iterable, flatten
+# Define type variable for AttributeHandler value type
+VT = TypeVar('VT', bound='TextHandler')
+
class TextHandler(str):
"""Extends standard Python string by adding more functionality"""
@@ -19,71 +23,70 @@ class TextHandler(str):
return super().__new__(cls, '')
# Make methods from original `str` class return `TextHandler` instead of returning `str` again
- # Of course, this stupid workaround is only so we can keep the auto-completion working without issues in your IDE
- # and I made sonnet write it for me :)
- def strip(self, chars=None):
+ # Of course, I made sonnet write it for me :)
+ def strip(self, chars: str = None) -> Union[str, 'TextHandler']:
return TextHandler(super().strip(chars))
- def lstrip(self, chars=None):
+ def lstrip(self, chars: str = None) -> Union[str, 'TextHandler']:
return TextHandler(super().lstrip(chars))
- def rstrip(self, chars=None):
+ def rstrip(self, chars: str = None) -> Union[str, 'TextHandler']:
return TextHandler(super().rstrip(chars))
- def capitalize(self):
+ def capitalize(self) -> Union[str, 'TextHandler']:
return TextHandler(super().capitalize())
- def casefold(self):
+ def casefold(self) -> Union[str, 'TextHandler']:
return TextHandler(super().casefold())
- def center(self, width, fillchar=' '):
+ def center(self, width: SupportsIndex, fillchar: str = ' ') -> Union[str, 'TextHandler']:
return TextHandler(super().center(width, fillchar))
- def expandtabs(self, tabsize=8):
+ def expandtabs(self, tabsize: SupportsIndex = 8) -> Union[str, 'TextHandler']:
return TextHandler(super().expandtabs(tabsize))
- def format(self, *args, **kwargs):
+ def format(self, *args: str, **kwargs: str) -> Union[str, 'TextHandler']:
return TextHandler(super().format(*args, **kwargs))
- def format_map(self, mapping):
+ def format_map(self, mapping) -> Union[str, 'TextHandler']:
return TextHandler(super().format_map(mapping))
- def join(self, iterable):
+ def join(self, iterable: Iterable[str]) -> Union[str, 'TextHandler']:
return TextHandler(super().join(iterable))
- def ljust(self, width, fillchar=' '):
+ def ljust(self, width: SupportsIndex, fillchar: str = ' ') -> Union[str, 'TextHandler']:
return TextHandler(super().ljust(width, fillchar))
- def rjust(self, width, fillchar=' '):
+ def rjust(self, width: SupportsIndex, fillchar: str = ' ') -> Union[str, 'TextHandler']:
return TextHandler(super().rjust(width, fillchar))
- def swapcase(self):
+ def swapcase(self) -> Union[str, 'TextHandler']:
return TextHandler(super().swapcase())
- def title(self):
+ def title(self) -> Union[str, 'TextHandler']:
return TextHandler(super().title())
- def translate(self, table):
+ def translate(self, table) -> Union[str, 'TextHandler']:
return TextHandler(super().translate(table))
- def zfill(self, width):
+ def zfill(self, width: SupportsIndex) -> Union[str, 'TextHandler']:
return TextHandler(super().zfill(width))
- def replace(self, old, new, count=-1):
+ def replace(self, old: str, new: str, count: SupportsIndex = -1) -> Union[str, 'TextHandler']:
return TextHandler(super().replace(old, new, count))
- def upper(self):
+ def upper(self) -> Union[str, 'TextHandler']:
return TextHandler(super().upper())
- def lower(self):
+ def lower(self) -> Union[str, 'TextHandler']:
return TextHandler(super().lower())
##############
- def sort(self, reverse: bool = False) -> str:
+ def sort(self, reverse: bool = False) -> Union[str, 'TextHandler']:
"""Return a sorted version of the string"""
return self.__class__("".join(sorted(self, reverse=reverse)))
- def clean(self) -> str:
+ def clean(self) -> Union[str, 'TextHandler']:
"""Return a new version of the string after removing all white spaces and consecutive spaces"""
data = re.sub(r'[\t|\r|\n]', '', self)
data = re.sub(' +', ' ', data)
@@ -210,7 +213,7 @@ class TextHandlers(List[TextHandler]):
get_all = extract
-class AttributesHandler(Mapping):
+class AttributesHandler(Mapping[str, VT]):
"""A read-only mapping to use instead of the standard dictionary for the speed boost but at the same time I use it to add more functionalities.
If standard dictionary is needed, just convert this class to dictionary with `dict` function
"""
@@ -231,7 +234,7 @@ class AttributesHandler(Mapping):
# Fastest read-only mapping type
self._data = MappingProxyType(mapping)
- def get(self, key, default=None):
+ def get(self, key: str, default: Optional[str] = None) -> Union[VT, None]:
"""Acts like standard dictionary `.get()` method"""
return self._data.get(key, default)
@@ -253,7 +256,7 @@ class AttributesHandler(Mapping):
"""Convert current attributes to JSON string if the attributes are JSON serializable otherwise throws error"""
return dumps(dict(self._data))
- def __getitem__(self, key):
+ def __getitem__(self, key: str) -> VT:
return self._data[key]
def __iter__(self):