Scrapy, a fast high-level web crawling & scraping framework for Python.
Add parameters to iterable generics, replace generators with iterables.
Andrey Rakhmatullin2e214210f670
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.pyindex 2453c0d39..f916a3e75 100644--- a/scrapy/commands/parse.py+++ b/scrapy/commands/parse.py@@ -140,13 +140,13 @@ class Command(BaseRunSpiderCommand):@overloaddef iterate_spider_output(- self, result: Union[AsyncGenerator, CoroutineType]- ) -> Deferred: ...+ self, result: Union[AsyncGenerator[_T, None], CoroutineType[Any, Any, _T]]+ ) -> Deferred[_T]: ...@overload- def iterate_spider_output(self, result: _T) -> Iterable: ...+ def iterate_spider_output(self, result: _T) -> Iterable[Any]: ...- def iterate_spider_output(self, result: Any) -> Union[Iterable, Deferred]:+ def iterate_spider_output(self, result: Any) -> Union[Iterable[Any], Deferred]:if inspect.isasyncgen(result):d = deferred_from_coro(collect_asyncgen(aiter_errback(result, self.handle_exception))diff --git a/scrapy/core/engine.py b/scrapy/core/engine.pyindex b342ad7a3..dededf99d 100644--- a/scrapy/core/engine.py+++ b/scrapy/core/engine.py@@ -372,7 +372,10 @@ class ExecutionEngine:@inlineCallbacksdef open_spider(- self, spider: Spider, start_requests: Iterable = (), close_if_idle: bool = True+ self,+ spider: Spider,+ start_requests: Iterable[Request] = (),+ close_if_idle: bool = True,) -> Generator[Deferred, Any, None]:if self.slot is not None:raise RuntimeError(f"No free spider slot when opening {spider.name!r}")diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.pyindex 566e6628b..3b7492838 100644--- a/scrapy/core/scraper.py+++ b/scrapy/core/scraper.py@@ -16,6 +16,7 @@ from typing import (Set,Tuple,Type,+ TypeVar,Union,cast,)@@ -47,6 +48,7 @@ if TYPE_CHECKING:from scrapy.crawler import Crawler+_T = TypeVar("_T")QueueTuple = Tuple[Union[Response, Failure], Request, Deferred]@@ -256,14 +258,14 @@ class Scraper:def handle_spider_output(self,- result: Union[Iterable, AsyncIterable],+ result: Union[Iterable[_T], AsyncIterable[_T]],request: Request,response: Response,spider: Spider,) -> Deferred:if not result:return defer_succeed(None)- it: Union[Iterable, AsyncIterable]+ it: Union[Iterable[_T], AsyncIterable[_T]]if isinstance(result, AsyncIterable):it = aiter_errback(result, self.handle_spider_error, request, response, spiderdiff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.pyindex 2cef2e1dd..cb1a93a68 100644--- a/scrapy/core/spidermw.py+++ b/scrapy/core/spidermw.py@@ -9,7 +9,6 @@ from inspect import isasyncgenfunction, iscoroutinefrom itertools import islicefrom typing import (Any,- AsyncGenerator,AsyncIterable,Callable,Generator,@@ -17,6 +16,7 @@ from typing import (List,Optional,Tuple,+ TypeVar,Union,cast,)@@ -42,6 +42,7 @@ from scrapy.utils.python import MutableAsyncChain, MutableChainlogger = logging.getLogger(__name__)+_T = TypeVar("_T")ScrapeFunc = Callable[[Union[Response, Failure], Request, Spider], Any]@@ -98,31 +99,39 @@ class SpiderMiddlewareManager(MiddlewareManager):self,response: Response,spider: Spider,- iterable: Union[Iterable, AsyncIterable],+ iterable: Union[Iterable[_T], AsyncIterable[_T]],exception_processor_index: int,- recover_to: Union[MutableChain, MutableAsyncChain],- ) -> Union[Generator, AsyncGenerator]:- def process_sync(iterable: Iterable) -> Generator:+ recover_to: Union[MutableChain[_T], MutableAsyncChain[_T]],+ ) -> Union[Iterable[_T], AsyncIterable[_T]]:+ def process_sync(iterable: Iterable[_T]) -> Iterable[_T]:try:yield from iterableexcept Exception as ex:- exception_result = self._process_spider_exception(- response, spider, Failure(ex), exception_processor_index+ exception_result = cast(+ Union[Failure, MutableChain[_T]],+ self._process_spider_exception(+ response, spider, Failure(ex), exception_processor_index+ ),)if isinstance(exception_result, Failure):raise+ assert isinstance(recover_to, MutableChain)recover_to.extend(exception_result)- async def process_async(iterable: AsyncIterable) -> AsyncGenerator:+ async def process_async(iterable: AsyncIterable[_T]) -> AsyncIterable[_T]:try:async for r in iterable:yield rexcept Exception as ex:- exception_result = self._process_spider_exception(- response, spider, Failure(ex), exception_processor_index+ exception_result = cast(+ Union[Failure, MutableAsyncChain[_T]],+ self._process_spider_exception(+ response, spider, Failure(ex), exception_processor_index+ ),)if isinstance(exception_result, Failure):raise+ assert isinstance(recover_to, MutableAsyncChain)recover_to.extend(exception_result)if isinstance(iterable, AsyncIterable):@@ -135,7 +144,7 @@ class SpiderMiddlewareManager(MiddlewareManager):spider: Spider,_failure: Failure,start_index: int = 0,- ) -> Union[Failure, MutableChain]:+ ) -> Union[Failure, MutableChain[_T], MutableAsyncChain[_T]]:exception = _failure.value# don't handle _InvalidOutput exceptionif isinstance(exception, _InvalidOutput):@@ -151,14 +160,18 @@ class SpiderMiddlewareManager(MiddlewareManager):if _isiterable(result):# stop exception handling by handing control over to the# process_spider_output chain if an iterable has been returned- dfd: Deferred = self._process_spider_output(- response, spider, result, method_index + 1+ dfd: Deferred[Union[MutableChain[_T], MutableAsyncChain[_T]]] = (+ self._process_spider_output(+ response, spider, result, method_index + 1+ ))# _process_spider_output() returns a Deferred only because of downgrading so this can be# simplified when downgrading is removed.if dfd.called:# the result is available immediately if _process_spider_output didn't do downgrading- return cast(MutableChain, dfd.result)+ return cast(+ Union[MutableChain[_T], MutableAsyncChain[_T]], dfd.result+ )# we forbid waiting here because otherwise we would need to return a deferred from# _process_spider_exception too, which complicates the architecturemsg = f"Async iterable returned from {method.__qualname__} cannot be downgraded"@@ -181,12 +194,12 @@ class SpiderMiddlewareManager(MiddlewareManager):self,response: Response,spider: Spider,- result: Union[Iterable, AsyncIterable],+ result: Union[Iterable[_T], AsyncIterable[_T]],start_index: int = 0,- ) -> Generator[Deferred, Any, Union[MutableChain, MutableAsyncChain]]:+ ) -> Generator[Deferred[Any], Any, Union[MutableChain[_T], MutableAsyncChain[_T]]]:# items in this iterable do not need to go through the process_spider_output# chain, they went through it already from the process_spider_exception method- recovered: Union[MutableChain, MutableAsyncChain]+ recovered: Union[MutableChain[_T], MutableAsyncChain[_T]]last_result_is_async = isinstance(result, AsyncIterable)if last_result_is_async:recovered = MutableAsyncChain()@@ -237,7 +250,9 @@ class SpiderMiddlewareManager(MiddlewareManager):# might fail directly if the output value is not a generatorresult = method(response=response, result=result, spider=spider)except Exception as ex:- exception_result = self._process_spider_exception(+ exception_result: Union[+ Failure, MutableChain[_T], MutableAsyncChain[_T]+ ] = self._process_spider_exception(response, spider, Failure(ex), method_index + 1)if isinstance(exception_result, Failure):@@ -267,9 +282,12 @@ class SpiderMiddlewareManager(MiddlewareManager):return MutableChain(result, recovered) # type: ignore[arg-type]async def _process_callback_output(- self, response: Response, spider: Spider, result: Union[Iterable, AsyncIterable]- ) -> Union[MutableChain, MutableAsyncChain]:- recovered: Union[MutableChain, MutableAsyncChain]+ self,+ response: Response,+ spider: Spider,+ result: Union[Iterable[_T], AsyncIterable[_T]],+ ) -> Union[MutableChain[_T], MutableAsyncChain[_T]]:+ recovered: Union[MutableChain[_T], MutableAsyncChain[_T]]if isinstance(result, AsyncIterable):recovered = MutableAsyncChain()else:@@ -293,14 +311,16 @@ class SpiderMiddlewareManager(MiddlewareManager):spider: Spider,) -> Deferred:async def process_callback_output(- result: Union[Iterable, AsyncIterable]- ) -> Union[MutableChain, MutableAsyncChain]:+ result: Union[Iterable[_T], AsyncIterable[_T]]+ ) -> Union[MutableChain[_T], MutableAsyncChain[_T]]:return await self._process_callback_output(response, spider, result)- def process_spider_exception(_failure: Failure) -> Union[Failure, MutableChain]:+ def process_spider_exception(+ _failure: Failure,+ ) -> Union[Failure, MutableChain[_T], MutableAsyncChain[_T]]:return self._process_spider_exception(response, spider, _failure)- dfd = mustbe_deferred(+ dfd: Deferred = mustbe_deferred(self._process_spider_input, scrape_func, response, request, spider)dfd.addCallback(deferred_f_from_coro_f(process_callback_output))diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.pyindex 166c4de97..daf193f59 100644--- a/scrapy/http/response/__init__.py+++ b/scrapy/http/response/__init__.py@@ -14,7 +14,6 @@ from typing import (AnyStr,Callable,Dict,- Generator,Iterable,List,Mapping,@@ -242,7 +241,7 @@ class Response(object_ref):errback: Optional[Callable] = None,cb_kwargs: Optional[Dict[str, Any]] = None,flags: Optional[List[str]] = None,- ) -> Generator[Request, None, None]:+ ) -> Iterable[Request]:""".. versionadded:: 2.0diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.pyindex 44c36b682..df4d90829 100644--- a/scrapy/http/response/text.py+++ b/scrapy/http/response/text.py@@ -15,7 +15,6 @@ from typing import (AnyStr,Callable,Dict,- Generator,Iterable,List,Mapping,@@ -246,7 +245,7 @@ class TextResponse(Response):flags: Optional[List[str]] = None,css: Optional[str] = None,xpath: Optional[str] = None,- ) -> Generator[Request, None, None]:+ ) -> Iterable[Request]:"""A generator that produces :class:`~.Request` instances to follow alllinks in ``urls``. It accepts the same arguments as the :class:`~.Request`'sdiff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.pyindex cd6e9d04e..41a842386 100644--- a/scrapy/utils/iterators.py+++ b/scrapy/utils/iterators.py@@ -6,8 +6,7 @@ from typing import (Any,Callable,Dict,- Generator,- Iterable,+ Iterator,List,Literal,Optional,@@ -22,14 +21,12 @@ from lxml import etree # nosecfrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.http import Response, TextResponsefrom scrapy.selector import Selector-from scrapy.utils.python import re_rsearch, to_unicode+from scrapy.utils.python import re_rsearchlogger = logging.getLogger(__name__)-def xmliter(- obj: Union[Response, str, bytes], nodename: str-) -> Generator[Selector, Any, None]:+def xmliter(obj: Union[Response, str, bytes], nodename: str) -> Iterator[Selector]:"""Return a iterator of Selector's over all nodes of a XML document,given the name of the node to iterate. Useful for parsing XML feeds.@@ -90,7 +87,7 @@ def xmliter_lxml(nodename: str,namespace: Optional[str] = None,prefix: str = "x",-) -> Generator[Selector, Any, None]:+) -> Iterator[Selector]:reader = _StreamReader(obj)tag = f"{{{namespace}}}{nodename}" if namespace else nodenameiterable = etree.iterparse(@@ -168,7 +165,7 @@ def csviter(headers: Optional[List[str]] = None,encoding: Optional[str] = None,quotechar: Optional[str] = None,-) -> Generator[Dict[str, str], Any, None]:+) -> Iterator[Dict[str, str]]:"""Returns an iterator of dictionaries from the given csv objectobj can be:@@ -184,10 +181,13 @@ def csviter(quotechar is the character used to enclosure fields on the given obj."""- encoding = obj.encoding if isinstance(obj, TextResponse) else encoding or "utf-8"-- def row_to_unicode(row_: Iterable) -> List[str]:- return [to_unicode(field, encoding) for field in row_]+ if encoding is not None:+ warn(+ "The encoding argument of csviter() is ignored and will be removed"+ " in a future Scrapy version.",+ category=ScrapyDeprecationWarning,+ stacklevel=2,+ )lines = StringIO(_body_or_str(obj, unicode=True))@@ -200,13 +200,11 @@ def csviter(if not headers:try:- row = next(csv_r)+ headers = next(csv_r)except StopIteration:return- headers = row_to_unicode(row)for row in csv_r:- row = row_to_unicode(row)if len(row) != len(headers):logger.warning("ignoring row %(csvlnum)d (length: %(csvrow)d, "diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.pyindex 49f36de2d..3d11c1035 100644--- a/scrapy/utils/misc.py+++ b/scrapy/utils/misc.py@@ -20,8 +20,8 @@ from typing import (Any,Callable,Deque,- Generator,Iterable,+ Iterator,List,Optional,Type,@@ -227,7 +227,7 @@ def build_from_settings(@contextmanager-def set_environ(**kwargs: str) -> Generator[None, Any, None]:+def set_environ(**kwargs: str) -> Iterator[None]:"""Temporarily set environment variables inside the context manager andfully restore previous environment afterwards"""@@ -244,7 +244,7 @@ def set_environ(**kwargs: str) -> Generator[None, Any, None]:os.environ[k] = v-def walk_callable(node: ast.AST) -> Generator[ast.AST, Any, None]:+def walk_callable(node: ast.AST) -> Iterable[ast.AST]:"""Similar to ``ast.walk``, but walks only function body and skips nestedfunctions defined within the node."""diff --git a/scrapy/utils/python.py b/scrapy/utils/python.pyindex 37a84a350..059d8e04d 100644--- a/scrapy/utils/python.py+++ b/scrapy/utils/python.py@@ -15,12 +15,10 @@ from itertools import chainfrom typing import (TYPE_CHECKING,Any,- AsyncGenerator,AsyncIterable,AsyncIterator,Callable,Dict,- Generator,Iterable,Iterator,List,@@ -163,7 +161,7 @@ def re_rsearch(the start position of the match, and the ending (regarding the entire text)."""- def _chunk_iter() -> Generator[Tuple[str, int], Any, None]:+ def _chunk_iter() -> Iterable[Tuple[str, int]]:offset = len(text)while True:offset -= chunk_size * 1024@@ -351,43 +349,45 @@ else:gc.collect()-class MutableChain(Iterable):+class MutableChain(Iterable[_T]):"""Thin wrapper around itertools.chain, allowing to add iterables "in-place""""- def __init__(self, *args: Iterable):- self.data = chain.from_iterable(args)+ def __init__(self, *args: Iterable[_T]):+ self.data: Iterator[_T] = chain.from_iterable(args)- def extend(self, *iterables: Iterable) -> None:+ def extend(self, *iterables: Iterable[_T]) -> None:self.data = chain(self.data, chain.from_iterable(iterables))- def __iter__(self) -> Iterator:+ def __iter__(self) -> Iterator[_T]:return self- def __next__(self) -> Any:+ def __next__(self) -> _T:return next(self.data)-async def _async_chain(*iterables: Union[Iterable, AsyncIterable]) -> AsyncGenerator:+async def _async_chain(+ *iterables: Union[Iterable[_T], AsyncIterable[_T]]+) -> AsyncIterator[_T]:for it in iterables:async for o in as_async_generator(it):yield o-class MutableAsyncChain(AsyncIterable):+class MutableAsyncChain(AsyncIterable[_T]):"""Similar to MutableChain but for async iterables"""- def __init__(self, *args: Union[Iterable, AsyncIterable]):- self.data = _async_chain(*args)+ def __init__(self, *args: Union[Iterable[_T], AsyncIterable[_T]]):+ self.data: AsyncIterator[_T] = _async_chain(*args)- def extend(self, *iterables: Union[Iterable, AsyncIterable]) -> None:+ def extend(self, *iterables: Union[Iterable[_T], AsyncIterable[_T]]) -> None:self.data = _async_chain(self.data, _async_chain(*iterables))- def __aiter__(self) -> AsyncIterator:+ def __aiter__(self) -> AsyncIterator[_T]:return self- async def __anext__(self) -> Any:+ async def __anext__(self) -> _T:return await self.data.__anext__()diff --git a/scrapy/utils/request.py b/scrapy/utils/request.pyindex 42a6537a8..45b8008f4 100644--- a/scrapy/utils/request.py+++ b/scrapy/utils/request.py@@ -12,7 +12,6 @@ from typing import (TYPE_CHECKING,Any,Dict,- Generator,Iterable,List,Optional,@@ -40,9 +39,7 @@ if TYPE_CHECKING:from scrapy.crawler import Crawler-def _serialize_headers(- headers: Iterable[bytes], request: Request-) -> Generator[bytes, Any, None]:+def _serialize_headers(headers: Iterable[bytes], request: Request) -> Iterable[bytes]:for header in headers:if header in request.headers:yield headerdiff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.pyindex cf429043d..7a91afe59 100644--- a/scrapy/utils/sitemap.py+++ b/scrapy/utils/sitemap.py@@ -5,7 +5,7 @@ Note: The main purpose of this module is to provide support for theSitemapSpider, its API is subject to change without notice."""-from typing import Any, Dict, Generator, Iterator, Optional, Union+from typing import Any, Dict, Iterable, Iterator, Optional, Unionfrom urllib.parse import urljoinimport lxml.etree # nosec@@ -42,7 +42,7 @@ class Sitemap:def sitemap_urls_from_robots(robots_text: str, base_url: Optional[str] = None-) -> Generator[str, Any, None]:+) -> Iterable[str]:"""Return an iterator over all sitemap urls contained in the givenrobots.txt file"""diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.pyindex cbbb01d85..b05135c04 100644--- a/scrapy/utils/spider.py+++ b/scrapy/utils/spider.py@@ -7,7 +7,6 @@ from typing import (TYPE_CHECKING,Any,AsyncGenerator,- Generator,Iterable,Literal,Optional,@@ -34,18 +33,20 @@ _T = TypeVar("_T")# https://stackoverflow.com/questions/60222982@overload-def iterate_spider_output(result: AsyncGenerator) -> AsyncGenerator: ... # type: ignore[overload-overlap]+def iterate_spider_output(result: AsyncGenerator[_T, None]) -> AsyncGenerator[_T, None]: ... # type: ignore[overload-overlap]@overload-def iterate_spider_output(result: CoroutineType) -> Deferred: ...+def iterate_spider_output(result: CoroutineType[Any, Any, _T]) -> Deferred[_T]: ...@overload-def iterate_spider_output(result: _T) -> Iterable: ...+def iterate_spider_output(result: _T) -> Iterable[Any]: ...-def iterate_spider_output(result: Any) -> Union[Iterable, AsyncGenerator, Deferred]:+def iterate_spider_output(+ result: Any,+) -> Union[Iterable[Any], AsyncGenerator[_T, None], Deferred[_T]]:if inspect.isasyncgen(result):return resultif inspect.iscoroutine(result):@@ -55,7 +56,7 @@ def iterate_spider_output(result: Any) -> Union[Iterable, AsyncGenerator, Deferrreturn arg_to_iter(deferred_from_coro(result))-def iter_spider_classes(module: ModuleType) -> Generator[Type[Spider], Any, None]:+def iter_spider_classes(module: ModuleType) -> Iterable[Type[Spider]]:"""Return an iterator over all spider classes defined in the given modulethat can be instantiated (i.e. which have name)"""diff --git a/tests/test_commands.py b/tests/test_commands.pyindex b9d468c66..857a56b73 100644--- a/tests/test_commands.py+++ b/tests/test_commands.py@@ -13,7 +13,7 @@ from shutil import copytree, rmtreefrom stat import S_IWRITE as ANYONE_WRITE_PERMISSIONfrom tempfile import TemporaryFile, mkdtempfrom threading import Timer-from typing import Dict, Generator, Optional, Union+from typing import Dict, Iterator, Optional, Unionfrom unittest import skipIffrom pytest import mark@@ -674,7 +674,7 @@ class BadSpider(scrapy.Spider):"""@contextmanager- def _create_file(self, content, name=None) -> Generator[str, None, None]:+ def _create_file(self, content, name=None) -> Iterator[str]:tmpdir = Path(self.mktemp())tmpdir.mkdir()if name: