Scrapy, a fast high-level web crawling & scraping framework for Python.
Add typing to scrapy/utils/datatypes.py.
Andrey Rakhmatullin7347d0214578
diff --git a/scrapy/resolver.py b/scrapy/resolver.pyindex 6cbe01cbf..e2e8beff4 100644--- a/scrapy/resolver.py+++ b/scrapy/resolver.py@@ -1,3 +1,5 @@+from typing import Any+from twisted.internet import deferfrom twisted.internet.base import ThreadedResolverfrom twisted.internet.interfaces import (@@ -11,7 +13,7 @@ from zope.interface.declarations import implementer, providerfrom scrapy.utils.datatypes import LocalCache# TODO: cache misses-dnscache = LocalCache(10000)+dnscache: LocalCache[str, Any] = LocalCache(10000)@implementer(IResolverSimple)@@ -36,7 +38,7 @@ class CachingThreadedResolver(ThreadedResolver):def install_on_reactor(self):self.reactor.installResolver(self)- def getHostByName(self, name, timeout=None):+ def getHostByName(self, name: str, timeout=None):if name in dnscache:return defer.succeed(dnscache[name])# in Twisted<=16.6, getHostByName() is always called with@@ -110,7 +112,7 @@ class CachingHostnameResolver:def resolveHostName(self,resolutionReceiver,- hostName,+ hostName: str,portNumber=0,addressTypes=None,transportSemantics="TCP",diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.pyindex fa57a4f26..599b201ea 100644--- a/scrapy/utils/datatypes.py+++ b/scrapy/utils/datatypes.py@@ -8,6 +8,10 @@ This module must not depend on any module outside the Standard Library.import collectionsimport weakreffrom collections.abc import Mapping+from typing import Any, Optional, Sequence, TypeVar++_KT = TypeVar("_KT")+_VT = TypeVar("_VT")class CaselessDict(dict):@@ -64,24 +68,24 @@ class CaselessDict(dict):return dict.pop(self, self.normkey(key), *args)-class LocalCache(collections.OrderedDict):+class LocalCache(collections.OrderedDict[_KT, _VT]):"""Dictionary with a finite number of keys.Older items expires first."""- def __init__(self, limit=None):+ def __init__(self, limit: Optional[int] = None):super().__init__()- self.limit = limit+ self.limit: Optional[int] = limit- def __setitem__(self, key, value):+ def __setitem__(self, key: _KT, value: _VT) -> None:if self.limit:while len(self) >= self.limit:self.popitem(last=False)super().__setitem__(key, value)-class LocalWeakReferencedCache(weakref.WeakKeyDictionary):+class LocalWeakReferencedCache(weakref.WeakKeyDictionary[_KT, _VT]):"""A weakref.WeakKeyDictionary implementation that uses LocalCache as itsunderlying data structure, making it ordered and capable of being size-limited.@@ -93,17 +97,17 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary):it cannot be instantiated with an initial dictionary."""- def __init__(self, limit=None):+ def __init__(self, limit: Optional[int] = None):super().__init__()- self.data = LocalCache(limit=limit)+ self.data: LocalCache = LocalCache(limit=limit)- def __setitem__(self, key, value):+ def __setitem__(self, key: _KT, value: _VT) -> None:try:super().__setitem__(key, value)except TypeError:pass # key is not weak-referenceable, skip caching- def __getitem__(self, key):+ def __getitem__(self, key: _KT) -> Optional[_VT]: # type: ignore[override]try:return super().__getitem__(key)except (TypeError, KeyError):@@ -113,8 +117,8 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary):class SequenceExclude:"""Object to test if an item is NOT within some sequence."""- def __init__(self, seq):- self.seq = seq+ def __init__(self, seq: Sequence):+ self.seq: Sequence = seq- def __contains__(self, item):+ def __contains__(self, item: Any) -> bool:return item not in self.seqdiff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.pyindex d861c9ab6..ea3f934c7 100644--- a/scrapy/utils/misc.py+++ b/scrapy/utils/misc.py@@ -214,16 +214,18 @@ def walk_callable(node):yield node-_generator_callbacks_cache = LocalWeakReferencedCache(limit=128)+_generator_callbacks_cache: LocalWeakReferencedCache[+ Callable, bool+] = LocalWeakReferencedCache(limit=128)-def is_generator_with_return_value(callable):+def is_generator_with_return_value(callable: Callable) -> bool:"""Returns True if a callable is a generator function which includes a'return' statement with a value different than None, False otherwise"""if callable in _generator_callbacks_cache:- return _generator_callbacks_cache[callable]+ return bool(_generator_callbacks_cache[callable])def returns_none(return_node):value = return_node.value@@ -248,10 +250,10 @@ def is_generator_with_return_value(callable):for node in walk_callable(tree):if isinstance(node, ast.Return) and not returns_none(node):_generator_callbacks_cache[callable] = True- return _generator_callbacks_cache[callable]+ return bool(_generator_callbacks_cache[callable])_generator_callbacks_cache[callable] = False- return _generator_callbacks_cache[callable]+ return bool(_generator_callbacks_cache[callable])def warn_on_generator_with_return_value(spider: "Spider", callable: Callable) -> None: