Scrapy, a fast high-level web crawling & scraping framework for Python.
Add more typing to scrapy/utils/python.py.
Andrey Rakhmatullind400f1ac0664
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.pyindex ae8feaf7d..0b5dc324f 100644--- a/scrapy/utils/python.py+++ b/scrapy/utils/python.py@@ -1,6 +1,7 @@"""This module contains essential stuff that should've come with Python itself ;)"""+import collections.abcimport gcimport inspectimport re@@ -12,9 +13,17 @@ from typing import (Any,AsyncGenerator,AsyncIterable,+ AsyncIterator,+ Callable,+ Dict,+ Generator,Iterable,+ Iterator,+ List,Mapping,Optional,+ Pattern,+ Tuple,Union,overload,)@@ -78,7 +87,7 @@ def is_listlike(x: Any) -> bool:return hasattr(x, "__iter__") and not isinstance(x, (str, bytes))-def unique(list_, key=lambda x: x):+def unique(list_: Iterable, key: Callable[[Any], Any] = lambda x: x) -> list:"""efficient function to uniquify a list preserving item order"""seen = set()result = []@@ -124,7 +133,9 @@ def to_bytes(return text.encode(encoding, errors)-def re_rsearch(pattern, text, chunk_size=1024):+def re_rsearch(+ pattern: Union[str, Pattern], text: str, chunk_size: int = 1024+) -> Optional[Tuple[int, int]]:"""This function does a reverse search in a text using a regular expressiongiven in the attribute 'pattern'.@@ -138,7 +149,7 @@ def re_rsearch(pattern, text, chunk_size=1024):the start position of the match, and the ending (regarding the entire text)."""- def _chunk_iter():+ def _chunk_iter() -> Generator[Tuple[str, int], Any, None]:offset = len(text)while True:offset -= chunk_size * 1024@@ -158,14 +169,14 @@ def re_rsearch(pattern, text, chunk_size=1024):return None-def memoizemethod_noargs(method):+def memoizemethod_noargs(method: Callable) -> Callable:"""Decorator to cache the result of a method (without arguments) using aweak reference to its object"""- cache = weakref.WeakKeyDictionary()+ cache: weakref.WeakKeyDictionary[Any, Any] = weakref.WeakKeyDictionary()@wraps(method)- def new_method(self, *args, **kwargs):+ def new_method(self: Any, *args: Any, **kwargs: Any) -> Any:if self not in cache:cache[self] = method(self, *args, **kwargs)return cache[self]@@ -187,12 +198,12 @@ def binary_is_text(data: bytes) -> bool:return all(c not in _BINARYCHARS for c in data)-def get_func_args(func, stripself=False):+def get_func_args(func: Callable, stripself: bool = False) -> List[str]:"""Return the argument name list of a callable object"""if not callable(func):raise TypeError(f"func must be callable, got '{type(func).__name__}'")- args = []+ args: List[str] = []try:sig = inspect.signature(func)except ValueError:@@ -217,7 +228,7 @@ def get_func_args(func, stripself=False):return args-def get_spec(func):+def get_spec(func: Callable) -> Tuple[List[str], Dict[str, Any]]:"""Returns (args, kwargs) tuple for a function>>> import re>>> get_spec(re.match)@@ -246,7 +257,7 @@ def get_spec(func):else:raise TypeError(f"{type(func)} is not callable")- defaults = spec.defaults or []+ defaults: Tuple[Any, ...] = spec.defaults or ()firstdefault = len(spec.args) - len(defaults)args = spec.args[:firstdefault]@@ -254,7 +265,9 @@ def get_spec(func):return args, kwargs-def equal_attributes(obj1, obj2, attributes):+def equal_attributes(+ obj1: Any, obj2: Any, attributes: Optional[List[Union[str, Callable]]]+) -> bool:"""Compare two objects attributes"""# not attributes given return False by defaultif not attributes:@@ -282,19 +295,20 @@ def without_none_values(iterable: Iterable) -> Iterable:...-def without_none_values(iterable):+def without_none_values(iterable: Union[Mapping, Iterable]) -> Union[dict, Iterable]:"""Return a copy of ``iterable`` with all ``None`` entries removed.If ``iterable`` is a mapping, return a dictionary where all pairs that havevalue ``None`` have been removed."""- try:+ if isinstance(iterable, collections.abc.Mapping):return {k: v for k, v in iterable.items() if v is not None}- except AttributeError:- return type(iterable)((v for v in iterable if v is not None))+ else:+ # the iterable __init__ must take another iterable+ return type(iterable)(v for v in iterable if v is not None) # type: ignore[call-arg]-def global_object_name(obj):+def global_object_name(obj: Any) -> str:"""Return full name of a global object.@@ -307,14 +321,14 @@ def global_object_name(obj):if hasattr(sys, "pypy_version_info"):- def garbage_collect():+ def garbage_collect() -> None:# Collecting weakreferences can take two collections on PyPy.gc.collect()gc.collect()else:- def garbage_collect():+ def garbage_collect() -> None:gc.collect()@@ -329,10 +343,10 @@ class MutableChain(Iterable):def extend(self, *iterables: Iterable) -> None:self.data = chain(self.data, chain.from_iterable(iterables))- def __iter__(self):+ def __iter__(self) -> Iterator:return self- def __next__(self):+ def __next__(self) -> Any:return next(self.data)@@ -353,8 +367,8 @@ class MutableAsyncChain(AsyncIterable):def extend(self, *iterables: Union[Iterable, AsyncIterable]) -> None:self.data = _async_chain(self.data, _async_chain(*iterables))- def __aiter__(self):+ def __aiter__(self) -> AsyncIterator:return self- async def __anext__(self):+ async def __anext__(self) -> Any:return await self.data.__anext__()