Scrapy, a fast high-level web crawling & scraping framework for Python.
Add typing to scrapy/utils/misc.py.
Andrey Rakhmatullinea299dfd7ce8
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.pyindex ea3f934c7..defc8663d 100644--- a/scrapy/utils/misc.py+++ b/scrapy/utils/misc.py@@ -10,7 +10,21 @@ from contextlib import contextmanagerfrom functools import partialfrom importlib import import_modulefrom pkgutil import iter_modules-from typing import TYPE_CHECKING, Any, Callable, Union+from types import ModuleType+from typing import (+ IO,+ TYPE_CHECKING,+ Any,+ Callable,+ Deque,+ Generator,+ Iterable,+ List,+ Optional,+ Pattern,+ Union,+ cast,+)from w3lib.html import replace_entities@@ -26,7 +40,7 @@ if TYPE_CHECKING:_ITERABLE_SINGLE_VALUES = dict, Item, str, bytes-def arg_to_iter(arg):+def arg_to_iter(arg: Any) -> Iterable[Any]:"""Convert an argument to an iterable. The argument can be a None, singlevalue, or an iterable.@@ -35,7 +49,7 @@ def arg_to_iter(arg):if arg is None:return []if not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, "__iter__"):- return arg+ return cast(Iterable[Any], arg)return [arg]@@ -72,7 +86,7 @@ def load_object(path: Union[str, Callable]) -> Any:return obj-def walk_modules(path):+def walk_modules(path: str) -> List[ModuleType]:"""Loads a module and all its submodules from the given module path andreturns them. If *any* module throws an exception while importing, thatexception is thrown back.@@ -80,7 +94,7 @@ def walk_modules(path):For example: walk_modules('scrapy.utils')"""- mods = []+ mods: List[ModuleType] = []mod = import_module(path)mods.append(mod)if hasattr(mod, "__path__"):@@ -94,7 +108,9 @@ def walk_modules(path):return mods-def extract_regex(regex, text, encoding="utf-8"):+def extract_regex(+ regex: Union[str, Pattern], text: str, encoding: str = "utf-8"+) -> List[str]:"""Extract a list of unicode strings from the given text/encoding using the following policies:* if the regex contains a named group called "extract" that will be returned@@ -111,9 +127,11 @@ def extract_regex(regex, text, encoding="utf-8"):regex = re.compile(regex, re.UNICODE)try:- strings = [regex.search(text).group("extract")] # named group+ # named group+ strings = [regex.search(text).group("extract")] # type: ignore[union-attr]except Exception:- strings = regex.findall(text) # full regex or numbered groups+ # full regex or numbered groups+ strings = regex.findall(text)strings = flatten(strings)if isinstance(text, str):@@ -123,7 +141,7 @@ def extract_regex(regex, text, encoding="utf-8"):]-def md5sum(file):+def md5sum(file: IO) -> str:"""Calculate the md5 checksum of a file-like object without reading itswhole content in memory.@@ -140,7 +158,7 @@ def md5sum(file):return m.hexdigest()-def rel_has_nofollow(rel):+def rel_has_nofollow(rel: Optional[str]) -> bool:"""Return True if link rel attribute has nofollow type"""return rel is not None and "nofollow" in rel.replace(",", " ").split()@@ -181,7 +199,7 @@ def create_instance(objcls, settings, crawler, *args, **kwargs):@contextmanager-def set_environ(**kwargs):+def set_environ(**kwargs: str) -> Generator[None, Any, None]:"""Temporarily set environment variables inside the context manager andfully restore previous environment afterwards"""@@ -198,11 +216,11 @@ def set_environ(**kwargs):os.environ[k] = v-def walk_callable(node):+def walk_callable(node: ast.AST) -> Generator[ast.AST, Any, None]:"""Similar to ``ast.walk``, but walks only function body and skips nestedfunctions defined within the node."""- todo = deque([node])+ todo: Deque[ast.AST] = deque([node])walked_func_def = Falsewhile todo:node = todo.popleft()@@ -227,7 +245,7 @@ def is_generator_with_return_value(callable: Callable) -> bool:if callable in _generator_callbacks_cache:return bool(_generator_callbacks_cache[callable])- def returns_none(return_node):+ def returns_none(return_node: ast.Return) -> bool:value = return_node.valuereturn (value is None or isinstance(value, ast.NameConstant) and value.value is Nonediff --git a/scrapy/utils/python.py b/scrapy/utils/python.pyindex 27816c0df..ae8feaf7d 100644--- a/scrapy/utils/python.py+++ b/scrapy/utils/python.py@@ -22,7 +22,7 @@ from typing import (from scrapy.utils.asyncgen import as_async_generator-def flatten(x):+def flatten(x: Iterable) -> list:"""flatten(sequence) -> listReturns a single, flat list which contains all elements retrieved@@ -42,7 +42,7 @@ def flatten(x):return list(iflatten(x))-def iflatten(x):+def iflatten(x: Iterable) -> Iterable:"""iflatten(sequence) -> iteratorSimilar to ``.flatten()``, but returns iterator instead"""