Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add typing to scrapy/utils/misc.py.

Andrey Rakhmatullinea299dfd7ce8
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index ea3f934c7..defc8663d 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -10,7 +10,21 @@ from contextlib import contextmanager
from functools import partial
from importlib import import_module
from pkgutil import iter_modules
-from typing import TYPE_CHECKING, Any, Callable, Union
+from types import ModuleType
+from typing import (
+ IO,
+ TYPE_CHECKING,
+ Any,
+ Callable,
+ Deque,
+ Generator,
+ Iterable,
+ List,
+ Optional,
+ Pattern,
+ Union,
+ cast,
+)
from w3lib.html import replace_entities
@@ -26,7 +40,7 @@ if TYPE_CHECKING:
_ITERABLE_SINGLE_VALUES = dict, Item, str, bytes
-def arg_to_iter(arg):
+def arg_to_iter(arg: Any) -> Iterable[Any]:
"""Convert an argument to an iterable. The argument can be a None, single
value, or an iterable.
@@ -35,7 +49,7 @@ def arg_to_iter(arg):
if arg is None:
return []
if not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, "__iter__"):
- return arg
+ return cast(Iterable[Any], arg)
return [arg]
@@ -72,7 +86,7 @@ def load_object(path: Union[str, Callable]) -> Any:
return obj
-def walk_modules(path):
+def walk_modules(path: str) -> List[ModuleType]:
"""Loads a module and all its submodules from the given module path and
returns them. If *any* module throws an exception while importing, that
exception is thrown back.
@@ -80,7 +94,7 @@ def walk_modules(path):
For example: walk_modules('scrapy.utils')
"""
- mods = []
+ mods: List[ModuleType] = []
mod = import_module(path)
mods.append(mod)
if hasattr(mod, "__path__"):
@@ -94,7 +108,9 @@ def walk_modules(path):
return mods
-def extract_regex(regex, text, encoding="utf-8"):
+def extract_regex(
+ regex: Union[str, Pattern], text: str, encoding: str = "utf-8"
+) -> List[str]:
"""Extract a list of unicode strings from the given text/encoding using the following policies:
* if the regex contains a named group called "extract" that will be returned
@@ -111,9 +127,11 @@ def extract_regex(regex, text, encoding="utf-8"):
regex = re.compile(regex, re.UNICODE)
try:
- strings = [regex.search(text).group("extract")] # named group
+ # named group
+ strings = [regex.search(text).group("extract")] # type: ignore[union-attr]
except Exception:
- strings = regex.findall(text) # full regex or numbered groups
+ # full regex or numbered groups
+ strings = regex.findall(text)
strings = flatten(strings)
if isinstance(text, str):
@@ -123,7 +141,7 @@ def extract_regex(regex, text, encoding="utf-8"):
]
-def md5sum(file):
+def md5sum(file: IO) -> str:
"""Calculate the md5 checksum of a file-like object without reading its
whole content in memory.
@@ -140,7 +158,7 @@ def md5sum(file):
return m.hexdigest()
-def rel_has_nofollow(rel):
+def rel_has_nofollow(rel: Optional[str]) -> bool:
"""Return True if link rel attribute has nofollow type"""
return rel is not None and "nofollow" in rel.replace(",", " ").split()
@@ -181,7 +199,7 @@ def create_instance(objcls, settings, crawler, *args, **kwargs):
@contextmanager
-def set_environ(**kwargs):
+def set_environ(**kwargs: str) -> Generator[None, Any, None]:
"""Temporarily set environment variables inside the context manager and
fully restore previous environment afterwards
"""
@@ -198,11 +216,11 @@ def set_environ(**kwargs):
os.environ[k] = v
-def walk_callable(node):
+def walk_callable(node: ast.AST) -> Generator[ast.AST, Any, None]:
"""Similar to ``ast.walk``, but walks only function body and skips nested
functions defined within the node.
"""
- todo = deque([node])
+ todo: Deque[ast.AST] = deque([node])
walked_func_def = False
while todo:
node = todo.popleft()
@@ -227,7 +245,7 @@ def is_generator_with_return_value(callable: Callable) -> bool:
if callable in _generator_callbacks_cache:
return bool(_generator_callbacks_cache[callable])
- def returns_none(return_node):
+ def returns_none(return_node: ast.Return) -> bool:
value = return_node.value
return (
value is None or isinstance(value, ast.NameConstant) and value.value is None
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 27816c0df..ae8feaf7d 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -22,7 +22,7 @@ from typing import (
from scrapy.utils.asyncgen import as_async_generator
-def flatten(x):
+def flatten(x: Iterable) -> list:
"""flatten(sequence) -> list
Returns a single, flat list which contains all elements retrieved
@@ -42,7 +42,7 @@ def flatten(x):
return list(iflatten(x))
-def iflatten(x):
+def iflatten(x: Iterable) -> Iterable:
"""iflatten(sequence) -> iterator
Similar to ``.flatten()``, but returns iterator instead"""