Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add parameters to iterable generics, replace generators with iterables.

Andrey Rakhmatullin2e214210f670
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index 2453c0d39..f916a3e75 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -140,13 +140,13 @@ class Command(BaseRunSpiderCommand):
@overload
def iterate_spider_output(
- self, result: Union[AsyncGenerator, CoroutineType]
- ) -> Deferred: ...
+ self, result: Union[AsyncGenerator[_T, None], CoroutineType[Any, Any, _T]]
+ ) -> Deferred[_T]: ...
@overload
- def iterate_spider_output(self, result: _T) -> Iterable: ...
+ def iterate_spider_output(self, result: _T) -> Iterable[Any]: ...
- def iterate_spider_output(self, result: Any) -> Union[Iterable, Deferred]:
+ def iterate_spider_output(self, result: Any) -> Union[Iterable[Any], Deferred]:
if inspect.isasyncgen(result):
d = deferred_from_coro(
collect_asyncgen(aiter_errback(result, self.handle_exception))
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index b342ad7a3..dededf99d 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -372,7 +372,10 @@ class ExecutionEngine:
@inlineCallbacks
def open_spider(
- self, spider: Spider, start_requests: Iterable = (), close_if_idle: bool = True
+ self,
+ spider: Spider,
+ start_requests: Iterable[Request] = (),
+ close_if_idle: bool = True,
) -> Generator[Deferred, Any, None]:
if self.slot is not None:
raise RuntimeError(f"No free spider slot when opening {spider.name!r}")
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 566e6628b..3b7492838 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -16,6 +16,7 @@ from typing import (
Set,
Tuple,
Type,
+ TypeVar,
Union,
cast,
)
@@ -47,6 +48,7 @@ if TYPE_CHECKING:
from scrapy.crawler import Crawler
+_T = TypeVar("_T")
QueueTuple = Tuple[Union[Response, Failure], Request, Deferred]
@@ -256,14 +258,14 @@ class Scraper:
def handle_spider_output(
self,
- result: Union[Iterable, AsyncIterable],
+ result: Union[Iterable[_T], AsyncIterable[_T]],
request: Request,
response: Response,
spider: Spider,
) -> Deferred:
if not result:
return defer_succeed(None)
- it: Union[Iterable, AsyncIterable]
+ it: Union[Iterable[_T], AsyncIterable[_T]]
if isinstance(result, AsyncIterable):
it = aiter_errback(
result, self.handle_spider_error, request, response, spider
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 2cef2e1dd..cb1a93a68 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -9,7 +9,6 @@ from inspect import isasyncgenfunction, iscoroutine
from itertools import islice
from typing import (
Any,
- AsyncGenerator,
AsyncIterable,
Callable,
Generator,
@@ -17,6 +16,7 @@ from typing import (
List,
Optional,
Tuple,
+ TypeVar,
Union,
cast,
)
@@ -42,6 +42,7 @@ from scrapy.utils.python import MutableAsyncChain, MutableChain
logger = logging.getLogger(__name__)
+_T = TypeVar("_T")
ScrapeFunc = Callable[[Union[Response, Failure], Request, Spider], Any]
@@ -98,31 +99,39 @@ class SpiderMiddlewareManager(MiddlewareManager):
self,
response: Response,
spider: Spider,
- iterable: Union[Iterable, AsyncIterable],
+ iterable: Union[Iterable[_T], AsyncIterable[_T]],
exception_processor_index: int,
- recover_to: Union[MutableChain, MutableAsyncChain],
- ) -> Union[Generator, AsyncGenerator]:
- def process_sync(iterable: Iterable) -> Generator:
+ recover_to: Union[MutableChain[_T], MutableAsyncChain[_T]],
+ ) -> Union[Iterable[_T], AsyncIterable[_T]]:
+ def process_sync(iterable: Iterable[_T]) -> Iterable[_T]:
try:
yield from iterable
except Exception as ex:
- exception_result = self._process_spider_exception(
- response, spider, Failure(ex), exception_processor_index
+ exception_result = cast(
+ Union[Failure, MutableChain[_T]],
+ self._process_spider_exception(
+ response, spider, Failure(ex), exception_processor_index
+ ),
)
if isinstance(exception_result, Failure):
raise
+ assert isinstance(recover_to, MutableChain)
recover_to.extend(exception_result)
- async def process_async(iterable: AsyncIterable) -> AsyncGenerator:
+ async def process_async(iterable: AsyncIterable[_T]) -> AsyncIterable[_T]:
try:
async for r in iterable:
yield r
except Exception as ex:
- exception_result = self._process_spider_exception(
- response, spider, Failure(ex), exception_processor_index
+ exception_result = cast(
+ Union[Failure, MutableAsyncChain[_T]],
+ self._process_spider_exception(
+ response, spider, Failure(ex), exception_processor_index
+ ),
)
if isinstance(exception_result, Failure):
raise
+ assert isinstance(recover_to, MutableAsyncChain)
recover_to.extend(exception_result)
if isinstance(iterable, AsyncIterable):
@@ -135,7 +144,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
spider: Spider,
_failure: Failure,
start_index: int = 0,
- ) -> Union[Failure, MutableChain]:
+ ) -> Union[Failure, MutableChain[_T], MutableAsyncChain[_T]]:
exception = _failure.value
# don't handle _InvalidOutput exception
if isinstance(exception, _InvalidOutput):
@@ -151,14 +160,18 @@ class SpiderMiddlewareManager(MiddlewareManager):
if _isiterable(result):
# stop exception handling by handing control over to the
# process_spider_output chain if an iterable has been returned
- dfd: Deferred = self._process_spider_output(
- response, spider, result, method_index + 1
+ dfd: Deferred[Union[MutableChain[_T], MutableAsyncChain[_T]]] = (
+ self._process_spider_output(
+ response, spider, result, method_index + 1
+ )
)
# _process_spider_output() returns a Deferred only because of downgrading so this can be
# simplified when downgrading is removed.
if dfd.called:
# the result is available immediately if _process_spider_output didn't do downgrading
- return cast(MutableChain, dfd.result)
+ return cast(
+ Union[MutableChain[_T], MutableAsyncChain[_T]], dfd.result
+ )
# we forbid waiting here because otherwise we would need to return a deferred from
# _process_spider_exception too, which complicates the architecture
msg = f"Async iterable returned from {method.__qualname__} cannot be downgraded"
@@ -181,12 +194,12 @@ class SpiderMiddlewareManager(MiddlewareManager):
self,
response: Response,
spider: Spider,
- result: Union[Iterable, AsyncIterable],
+ result: Union[Iterable[_T], AsyncIterable[_T]],
start_index: int = 0,
- ) -> Generator[Deferred, Any, Union[MutableChain, MutableAsyncChain]]:
+ ) -> Generator[Deferred[Any], Any, Union[MutableChain[_T], MutableAsyncChain[_T]]]:
# items in this iterable do not need to go through the process_spider_output
# chain, they went through it already from the process_spider_exception method
- recovered: Union[MutableChain, MutableAsyncChain]
+ recovered: Union[MutableChain[_T], MutableAsyncChain[_T]]
last_result_is_async = isinstance(result, AsyncIterable)
if last_result_is_async:
recovered = MutableAsyncChain()
@@ -237,7 +250,9 @@ class SpiderMiddlewareManager(MiddlewareManager):
# might fail directly if the output value is not a generator
result = method(response=response, result=result, spider=spider)
except Exception as ex:
- exception_result = self._process_spider_exception(
+ exception_result: Union[
+ Failure, MutableChain[_T], MutableAsyncChain[_T]
+ ] = self._process_spider_exception(
response, spider, Failure(ex), method_index + 1
)
if isinstance(exception_result, Failure):
@@ -267,9 +282,12 @@ class SpiderMiddlewareManager(MiddlewareManager):
return MutableChain(result, recovered) # type: ignore[arg-type]
async def _process_callback_output(
- self, response: Response, spider: Spider, result: Union[Iterable, AsyncIterable]
- ) -> Union[MutableChain, MutableAsyncChain]:
- recovered: Union[MutableChain, MutableAsyncChain]
+ self,
+ response: Response,
+ spider: Spider,
+ result: Union[Iterable[_T], AsyncIterable[_T]],
+ ) -> Union[MutableChain[_T], MutableAsyncChain[_T]]:
+ recovered: Union[MutableChain[_T], MutableAsyncChain[_T]]
if isinstance(result, AsyncIterable):
recovered = MutableAsyncChain()
else:
@@ -293,14 +311,16 @@ class SpiderMiddlewareManager(MiddlewareManager):
spider: Spider,
) -> Deferred:
async def process_callback_output(
- result: Union[Iterable, AsyncIterable]
- ) -> Union[MutableChain, MutableAsyncChain]:
+ result: Union[Iterable[_T], AsyncIterable[_T]]
+ ) -> Union[MutableChain[_T], MutableAsyncChain[_T]]:
return await self._process_callback_output(response, spider, result)
- def process_spider_exception(_failure: Failure) -> Union[Failure, MutableChain]:
+ def process_spider_exception(
+ _failure: Failure,
+ ) -> Union[Failure, MutableChain[_T], MutableAsyncChain[_T]]:
return self._process_spider_exception(response, spider, _failure)
- dfd = mustbe_deferred(
+ dfd: Deferred = mustbe_deferred(
self._process_spider_input, scrape_func, response, request, spider
)
dfd.addCallback(deferred_f_from_coro_f(process_callback_output))
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index 166c4de97..daf193f59 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -14,7 +14,6 @@ from typing import (
AnyStr,
Callable,
Dict,
- Generator,
Iterable,
List,
Mapping,
@@ -242,7 +241,7 @@ class Response(object_ref):
errback: Optional[Callable] = None,
cb_kwargs: Optional[Dict[str, Any]] = None,
flags: Optional[List[str]] = None,
- ) -> Generator[Request, None, None]:
+ ) -> Iterable[Request]:
"""
.. versionadded:: 2.0
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 44c36b682..df4d90829 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -15,7 +15,6 @@ from typing import (
AnyStr,
Callable,
Dict,
- Generator,
Iterable,
List,
Mapping,
@@ -246,7 +245,7 @@ class TextResponse(Response):
flags: Optional[List[str]] = None,
css: Optional[str] = None,
xpath: Optional[str] = None,
- ) -> Generator[Request, None, None]:
+ ) -> Iterable[Request]:
"""
A generator that produces :class:`~.Request` instances to follow all
links in ``urls``. It accepts the same arguments as the :class:`~.Request`'s
diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index cd6e9d04e..41a842386 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -6,8 +6,7 @@ from typing import (
Any,
Callable,
Dict,
- Generator,
- Iterable,
+ Iterator,
List,
Literal,
Optional,
@@ -22,14 +21,12 @@ from lxml import etree # nosec
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Response, TextResponse
from scrapy.selector import Selector
-from scrapy.utils.python import re_rsearch, to_unicode
+from scrapy.utils.python import re_rsearch
logger = logging.getLogger(__name__)
-def xmliter(
- obj: Union[Response, str, bytes], nodename: str
-) -> Generator[Selector, Any, None]:
+def xmliter(obj: Union[Response, str, bytes], nodename: str) -> Iterator[Selector]:
"""Return a iterator of Selector's over all nodes of a XML document,
given the name of the node to iterate. Useful for parsing XML feeds.
@@ -90,7 +87,7 @@ def xmliter_lxml(
nodename: str,
namespace: Optional[str] = None,
prefix: str = "x",
-) -> Generator[Selector, Any, None]:
+) -> Iterator[Selector]:
reader = _StreamReader(obj)
tag = f"{{{namespace}}}{nodename}" if namespace else nodename
iterable = etree.iterparse(
@@ -168,7 +165,7 @@ def csviter(
headers: Optional[List[str]] = None,
encoding: Optional[str] = None,
quotechar: Optional[str] = None,
-) -> Generator[Dict[str, str], Any, None]:
+) -> Iterator[Dict[str, str]]:
"""Returns an iterator of dictionaries from the given csv object
obj can be:
@@ -184,10 +181,13 @@ def csviter(
quotechar is the character used to enclosure fields on the given obj.
"""
- encoding = obj.encoding if isinstance(obj, TextResponse) else encoding or "utf-8"
-
- def row_to_unicode(row_: Iterable) -> List[str]:
- return [to_unicode(field, encoding) for field in row_]
+ if encoding is not None:
+ warn(
+ "The encoding argument of csviter() is ignored and will be removed"
+ " in a future Scrapy version.",
+ category=ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
lines = StringIO(_body_or_str(obj, unicode=True))
@@ -200,13 +200,11 @@ def csviter(
if not headers:
try:
- row = next(csv_r)
+ headers = next(csv_r)
except StopIteration:
return
- headers = row_to_unicode(row)
for row in csv_r:
- row = row_to_unicode(row)
if len(row) != len(headers):
logger.warning(
"ignoring row %(csvlnum)d (length: %(csvrow)d, "
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 49f36de2d..3d11c1035 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -20,8 +20,8 @@ from typing import (
Any,
Callable,
Deque,
- Generator,
Iterable,
+ Iterator,
List,
Optional,
Type,
@@ -227,7 +227,7 @@ def build_from_settings(
@contextmanager
-def set_environ(**kwargs: str) -> Generator[None, Any, None]:
+def set_environ(**kwargs: str) -> Iterator[None]:
"""Temporarily set environment variables inside the context manager and
fully restore previous environment afterwards
"""
@@ -244,7 +244,7 @@ def set_environ(**kwargs: str) -> Generator[None, Any, None]:
os.environ[k] = v
-def walk_callable(node: ast.AST) -> Generator[ast.AST, Any, None]:
+def walk_callable(node: ast.AST) -> Iterable[ast.AST]:
"""Similar to ``ast.walk``, but walks only function body and skips nested
functions defined within the node.
"""
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 37a84a350..059d8e04d 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -15,12 +15,10 @@ from itertools import chain
from typing import (
TYPE_CHECKING,
Any,
- AsyncGenerator,
AsyncIterable,
AsyncIterator,
Callable,
Dict,
- Generator,
Iterable,
Iterator,
List,
@@ -163,7 +161,7 @@ def re_rsearch(
the start position of the match, and the ending (regarding the entire text).
"""
- def _chunk_iter() -> Generator[Tuple[str, int], Any, None]:
+ def _chunk_iter() -> Iterable[Tuple[str, int]]:
offset = len(text)
while True:
offset -= chunk_size * 1024
@@ -351,43 +349,45 @@ else:
gc.collect()
-class MutableChain(Iterable):
+class MutableChain(Iterable[_T]):
"""
Thin wrapper around itertools.chain, allowing to add iterables "in-place"
"""
- def __init__(self, *args: Iterable):
- self.data = chain.from_iterable(args)
+ def __init__(self, *args: Iterable[_T]):
+ self.data: Iterator[_T] = chain.from_iterable(args)
- def extend(self, *iterables: Iterable) -> None:
+ def extend(self, *iterables: Iterable[_T]) -> None:
self.data = chain(self.data, chain.from_iterable(iterables))
- def __iter__(self) -> Iterator:
+ def __iter__(self) -> Iterator[_T]:
return self
- def __next__(self) -> Any:
+ def __next__(self) -> _T:
return next(self.data)
-async def _async_chain(*iterables: Union[Iterable, AsyncIterable]) -> AsyncGenerator:
+async def _async_chain(
+ *iterables: Union[Iterable[_T], AsyncIterable[_T]]
+) -> AsyncIterator[_T]:
for it in iterables:
async for o in as_async_generator(it):
yield o
-class MutableAsyncChain(AsyncIterable):
+class MutableAsyncChain(AsyncIterable[_T]):
"""
Similar to MutableChain but for async iterables
"""
- def __init__(self, *args: Union[Iterable, AsyncIterable]):
- self.data = _async_chain(*args)
+ def __init__(self, *args: Union[Iterable[_T], AsyncIterable[_T]]):
+ self.data: AsyncIterator[_T] = _async_chain(*args)
- def extend(self, *iterables: Union[Iterable, AsyncIterable]) -> None:
+ def extend(self, *iterables: Union[Iterable[_T], AsyncIterable[_T]]) -> None:
self.data = _async_chain(self.data, _async_chain(*iterables))
- def __aiter__(self) -> AsyncIterator:
+ def __aiter__(self) -> AsyncIterator[_T]:
return self
- async def __anext__(self) -> Any:
+ async def __anext__(self) -> _T:
return await self.data.__anext__()
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index 42a6537a8..45b8008f4 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -12,7 +12,6 @@ from typing import (
TYPE_CHECKING,
Any,
Dict,
- Generator,
Iterable,
List,
Optional,
@@ -40,9 +39,7 @@ if TYPE_CHECKING:
from scrapy.crawler import Crawler
-def _serialize_headers(
- headers: Iterable[bytes], request: Request
-) -> Generator[bytes, Any, None]:
+def _serialize_headers(headers: Iterable[bytes], request: Request) -> Iterable[bytes]:
for header in headers:
if header in request.headers:
yield header
diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py
index cf429043d..7a91afe59 100644
--- a/scrapy/utils/sitemap.py
+++ b/scrapy/utils/sitemap.py
@@ -5,7 +5,7 @@ Note: The main purpose of this module is to provide support for the
SitemapSpider, its API is subject to change without notice.
"""
-from typing import Any, Dict, Generator, Iterator, Optional, Union
+from typing import Any, Dict, Iterable, Iterator, Optional, Union
from urllib.parse import urljoin
import lxml.etree # nosec
@@ -42,7 +42,7 @@ class Sitemap:
def sitemap_urls_from_robots(
robots_text: str, base_url: Optional[str] = None
-) -> Generator[str, Any, None]:
+) -> Iterable[str]:
"""Return an iterator over all sitemap urls contained in the given
robots.txt file
"""
diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py
index cbbb01d85..b05135c04 100644
--- a/scrapy/utils/spider.py
+++ b/scrapy/utils/spider.py
@@ -7,7 +7,6 @@ from typing import (
TYPE_CHECKING,
Any,
AsyncGenerator,
- Generator,
Iterable,
Literal,
Optional,
@@ -34,18 +33,20 @@ _T = TypeVar("_T")
# https://stackoverflow.com/questions/60222982
@overload
-def iterate_spider_output(result: AsyncGenerator) -> AsyncGenerator: ... # type: ignore[overload-overlap]
+def iterate_spider_output(result: AsyncGenerator[_T, None]) -> AsyncGenerator[_T, None]: ... # type: ignore[overload-overlap]
@overload
-def iterate_spider_output(result: CoroutineType) -> Deferred: ...
+def iterate_spider_output(result: CoroutineType[Any, Any, _T]) -> Deferred[_T]: ...
@overload
-def iterate_spider_output(result: _T) -> Iterable: ...
+def iterate_spider_output(result: _T) -> Iterable[Any]: ...
-def iterate_spider_output(result: Any) -> Union[Iterable, AsyncGenerator, Deferred]:
+def iterate_spider_output(
+ result: Any,
+) -> Union[Iterable[Any], AsyncGenerator[_T, None], Deferred[_T]]:
if inspect.isasyncgen(result):
return result
if inspect.iscoroutine(result):
@@ -55,7 +56,7 @@ def iterate_spider_output(result: Any) -> Union[Iterable, AsyncGenerator, Deferr
return arg_to_iter(deferred_from_coro(result))
-def iter_spider_classes(module: ModuleType) -> Generator[Type[Spider], Any, None]:
+def iter_spider_classes(module: ModuleType) -> Iterable[Type[Spider]]:
"""Return an iterator over all spider classes defined in the given module
that can be instantiated (i.e. which have name)
"""
diff --git a/tests/test_commands.py b/tests/test_commands.py
index b9d468c66..857a56b73 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -13,7 +13,7 @@ from shutil import copytree, rmtree
from stat import S_IWRITE as ANYONE_WRITE_PERMISSION
from tempfile import TemporaryFile, mkdtemp
from threading import Timer
-from typing import Dict, Generator, Optional, Union
+from typing import Dict, Iterator, Optional, Union
from unittest import skipIf
from pytest import mark
@@ -674,7 +674,7 @@ class BadSpider(scrapy.Spider):
"""
@contextmanager
- def _create_file(self, content, name=None) -> Generator[str, None, None]:
+ def _create_file(self, content, name=None) -> Iterator[str]:
tmpdir = Path(self.mktemp())
tmpdir.mkdir()
if name: