Scrapy, a fast high-level web crawling & scraping framework for Python.
Add/update typing, cleanup iterator/iterable inconsistencies.
Andrey Rakhmatullin61197d3dba53
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.pyindex 8ce57af2f..0630ce625 100644--- a/scrapy/core/scraper.py+++ b/scrapy/core/scraper.py@@ -1,10 +1,8 @@"""This module implements the Scraper component which parses responses andextracts information from them"""-import collectionsimport loggingfrom collections import deque-from collections.abc import Iterable-from typing import Union+from typing import AsyncGenerator, AsyncIterable, Generator, Iterable, Unionfrom itemadapter import is_itemfrom twisted.internet import defer@@ -188,7 +186,8 @@ class Scraper:def handle_spider_output(self, result: Iterable, request: Request, response: Response, spider: Spider):if not result:return defer_succeed(None)- if isinstance(result, collections.abc.AsyncIterable):+ it: Union[Generator, AsyncGenerator]+ if isinstance(result, AsyncIterable):it = aiter_errback(result, self.handle_spider_error, request, response, spider)dfd = parallel_async(it, self.concurrent_items, self._process_spidermw_output,request, response, spider)diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.pyindex 6fcbc7492..9dd6c462d 100644--- a/scrapy/core/spidermw.py+++ b/scrapy/core/spidermw.py@@ -3,9 +3,8 @@ Spider Middleware managerSee documentation in docs/topics/spider-middleware.rst"""-import collections.abcfrom itertools import islice-from typing import Any, Callable, Generator, Iterable, Union, AsyncIterable+from typing import Any, Callable, Generator, Iterable, Union, AsyncIterable, AsyncGeneratorfrom twisted.internet.defer import Deferredfrom twisted.python.failure import Failure@@ -61,8 +60,9 @@ class SpiderMiddlewareManager(MiddlewareManager):return scrape_func(Failure(), request, spider)return scrape_func(response, request, spider)- def _evaluate_iterable(self, response: Response, spider: Spider, iterable: Iterable,- exception_processor_index: int, recover_to: MutableChain) -> Generator:+ def _evaluate_iterable(self, response: Response, spider: Spider, iterable: Union[Iterable, AsyncIterable],+ exception_processor_index: int, recover_to: Union[MutableChain, MutableAsyncChain]+ ) -> Union[Generator, AsyncGenerator]:@_process_iterable_universalasync def _evaluate_async_iterable(iterable):try:@@ -100,11 +100,13 @@ class SpiderMiddlewareManager(MiddlewareManager):return _failuredef _process_spider_output(self, response: Response, spider: Spider,- result: Iterable, start_index: int = 0) -> MutableChain:+ result: Union[Iterable, AsyncIterable], start_index: int = 0+ ) -> Union[MutableChain, MutableAsyncChain]:# items in this iterable do not need to go through the process_spider_output# chain, they went through it already from the process_spider_exception method- last_result_async = isinstance(result, collections.abc.AsyncIterator)- if last_result_async:+ recovered: Union[MutableChain, MutableAsyncChain]+ last_result_is_async = isinstance(result, AsyncIterable)+ if last_result_is_async:recovered = MutableAsyncChain()else:recovered = MutableChain()@@ -127,30 +129,32 @@ class SpiderMiddlewareManager(MiddlewareManager):msg = (f"Middleware {method.__qualname__} must return an "f"iterable, got {type(result)}")raise _InvalidOutput(msg)- if last_result_async and isinstance(result, collections.abc.Iterator):+ if last_result_is_async and isinstance(result, Iterable):raise TypeError(f"Synchronous {method.__qualname__} called with an async iterable")- last_result_async = isinstance(result, collections.abc.AsyncIterator)+ last_result_is_async = isinstance(result, AsyncIterable)- if last_result_async:+ if last_result_is_async:return MutableAsyncChain(result, recovered)else:- return MutableChain(result, recovered)+ return MutableChain(result, recovered) # type: ignore[arg-type]- def _process_callback_output(self, response: Response, spider: Spider, result: Iterable) -> MutableChain:- if isinstance(result, collections.abc.AsyncIterator):+ def _process_callback_output(self, response: Response, spider: Spider, result: Union[Iterable, AsyncIterable]+ ) -> Union[MutableChain, MutableAsyncChain]:+ recovered: Union[MutableChain, MutableAsyncChain]+ if isinstance(result, AsyncIterable):recovered = MutableAsyncChain()else:recovered = MutableChain()result = self._evaluate_iterable(response, spider, result, 0, recovered)result = self._process_spider_output(response, spider, result)- if isinstance(result, collections.abc.AsyncIterator):+ if isinstance(result, AsyncIterable):return MutableAsyncChain(result, recovered)else:- return MutableChain(result, recovered)+ return MutableChain(result, recovered) # type: ignore[arg-type]def scrape_response(self, scrape_func: ScrapeFunc, response: Response, request: Request,spider: Spider) -> Deferred:- def process_callback_output(result: Iterable) -> MutableChain:+ def process_callback_output(result: Union[Iterable, AsyncIterable]) -> Union[MutableChain, MutableAsyncChain]:return self._process_callback_output(response, spider, result)def process_spider_exception(_failure: Failure) -> Union[Failure, MutableChain]:diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.pyindex 92118ddb9..ae9a79989 100644--- a/scrapy/utils/asyncgen.py+++ b/scrapy/utils/asyncgen.py@@ -1,7 +1,6 @@-import collectionsimport functoolsimport inspect-from collections.abc import AsyncIterable+from typing import AsyncGenerator, AsyncIterable, Callable, Generator, Iterable, Unionasync def collect_asyncgen(result: AsyncIterable):@@ -11,9 +10,9 @@ async def collect_asyncgen(result: AsyncIterable):return results-async def as_async_generator(it):- """ Wraps an iterator (sync or async) into an async generator. """- if isinstance(it, collections.abc.AsyncIterator):+async def as_async_generator(it: Union[Iterable, AsyncIterable]) -> AsyncGenerator:+ """ Wraps an iterable (sync or async) into an async generator. """+ if isinstance(it, AsyncIterable):async for r in it:yield relse:@@ -22,7 +21,7 @@ async def as_async_generator(it):# https://stackoverflow.com/a/66170760/113586-def _process_iterable_universal(process_async):+def _process_iterable_universal(process_async: Callable):""" Takes a function that takes an async iterable, args and kwargs. Returnsa function that takes any iterable, args and kwargs.@@ -33,7 +32,7 @@ def _process_iterable_universal(process_async):# If this stops working, all internal uses can be just replaced with manually-written# process_sync functions.- def process_sync(iterable, *args, **kwargs):+ def process_sync(iterable: Iterable, *args, **kwargs) -> Generator:agen = process_async(as_async_generator(iterable), *args, **kwargs)if not inspect.isasyncgen(agen):raise ValueError(f"process_async returned wrong type {type(agen)}")@@ -52,11 +51,11 @@ def _process_iterable_universal(process_async):f"you can't use {_process_iterable_universal.__name__} with it.")@functools.wraps(process_async)- def process(iterable, *args, **kwargs):- if inspect.isasyncgen(iterable):+ def process(iterable: Union[Iterable, AsyncIterable], *args, **kwargs) -> Union[Generator, AsyncGenerator]:+ if isinstance(iterable, AsyncIterable):# call process_async directlyreturn process_async(iterable, *args, **kwargs)- if hasattr(iterable, '__iter__'):+ if isinstance(iterable, Iterable):# convert process_async to process_syncreturn process_sync(iterable, *args, **kwargs)raise TypeError(f"Wrong iterable type {type(iterable)}")diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.pyindex f81faf6dd..39c8a85e9 100644--- a/scrapy/utils/defer.py+++ b/scrapy/utils/defer.py@@ -3,9 +3,21 @@ Helper functions for dealing with Twisted deferreds"""import asyncioimport inspect-from collections.abc import Coroutine+from asyncio import Futurefrom functools import wraps-from typing import Any, Callable, Generator, Iterable+from typing import (+ Any,+ AsyncGenerator,+ AsyncIterable,+ Callable,+ Coroutine,+ Generator,+ Iterable,+ Iterator,+ List,+ Optional,+ Union+)from twisted.internet import deferfrom twisted.internet.defer import Deferred, DeferredList, ensureDeferred@@ -80,8 +92,8 @@ def parallel(iterable: Iterable, count: int, callable: Callable, *args, **named)return DeferredList([coop.coiterate(work) for _ in range(count)])-class _AsyncCooperatorAdapter:- """ A class that wraps an async iterator into a normal iterator suitable+class _AsyncCooperatorAdapter(Iterator):+ """ A class that wraps an async iterable into a normal iterator suitablefor using in Cooperator.coiterate(). As it's only needed for parallel_async(),it calls the callable directly in the callback, instead of providing a moregeneric interface.@@ -125,30 +137,30 @@ class _AsyncCooperatorAdapter:Cooperator/CooperativeTask and use it instead of this adapter to achieve the samegoal."""- def __init__(self, aiterator, callable, *callable_args, **callable_kwargs):- self.aiterator = aiterator+ def __init__(self, aiterable: AsyncIterable, callable: Callable, *callable_args, **callable_kwargs):+ self.aiterator = aiterable.__aiter__()self.callable = callableself.callable_args = callable_argsself.callable_kwargs = callable_kwargsself.finished = False- self.waiting_deferreds = []- self.anext_deferred = None+ self.waiting_deferreds: List[Deferred] = []+ self.anext_deferred: Optional[Deferred] = None- def _callback(self, result):+ def _callback(self, result: Any) -> None:# This gets called when the result from aiterator.__anext__() is available.# It calls the callable on it and sends the result to the oldest waiting Deferred# (by chaining if the result is a Deferred too or by firing if not).self.anext_deferred = Noneresult = self.callable(result, *self.callable_args, **self.callable_kwargs)d = self.waiting_deferreds.pop(0)- if isinstance(result, defer.Deferred):+ if isinstance(result, Deferred):result.chainDeferred(d)else:d.callback(None)if self.waiting_deferreds:self._call_anext()- def _errback(self, failure):+ def _errback(self, failure: Failure) -> None:# This gets called on any exceptions in aiterator.__anext__().# It handles StopAsyncIteration by stopping the iteration and reraises all others.self.anext_deferred = None@@ -157,29 +169,29 @@ class _AsyncCooperatorAdapter:for d in self.waiting_deferreds:d.callback(None)- def _call_anext(self):+ def _call_anext(self) -> None:# This starts waiting for the next result from aiterator.# If aiterator is exhausted, _errback will be called.self.anext_deferred = deferred_from_coro(self.aiterator.__anext__())self.anext_deferred.addCallbacks(self._callback, self._errback)- def __next__(self):+ def __next__(self) -> Deferred:# This puts a new Deferred into self.waiting_deferreds and returns it.# It also calls __anext__() if needed.if self.finished:raise StopIteration- d = defer.Deferred()+ d = Deferred()self.waiting_deferreds.append(d)if not self.anext_deferred:self._call_anext()return d-def parallel_async(async_iterable, count, callable, *args, **named):- """ Like parallel but for async iterables """+def parallel_async(async_iterable: AsyncIterable, count: int, callable: Callable, *args, **named) -> DeferredList:+ """ Like parallel but for async iterators """coop = Cooperator()work = _AsyncCooperatorAdapter(async_iterable, callable, *args, **named)- dl = defer.DeferredList([coop.coiterate(work) for _ in range(count)])+ dl = DeferredList([coop.coiterate(work) for _ in range(count)])return dl@@ -232,7 +244,7 @@ def iter_errback(iterable: Iterable, errback: Callable, *a, **kw) -> Generator:errback(failure.Failure(), *a, **kw)-async def aiter_errback(aiterable, errback, *a, **kw):+async def aiter_errback(aiterable: AsyncIterable, errback: Callable, *a, **kw) -> AsyncGenerator:"""Wraps an async iterable calling an errback if an error is caught whileiterating it. Similar to scrapy.utils.defer.iter_errback()"""@@ -290,13 +302,13 @@ def maybeDeferred_coro(f: Callable, *args, **kw) -> Deferred:return defer.succeed(result)-def deferred_to_future(d):+def deferred_to_future(d: Deferred) -> Future:""" Wraps a Deferred into a Future. Requires the asyncio reactor."""return d.asFuture(asyncio.get_event_loop())-def maybe_deferred_to_future(d):+def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]:""" Converts a Deferred to something that can be awaited in a callback or other user coroutine.If the asyncio reactor is installed, coroutines are wrapped into Futures, and only Futures can bediff --git a/scrapy/utils/python.py b/scrapy/utils/python.pyindex 8b823d174..d086347bc 100644--- a/scrapy/utils/python.py+++ b/scrapy/utils/python.py@@ -8,9 +8,9 @@ import reimport sysimport warningsimport weakref-from collections.abc import Iterablefrom functools import partial, wrapsfrom itertools import chain+from typing import AsyncIterable, Iterable, Unionfrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.utils.asyncgen import as_async_generator@@ -345,7 +345,7 @@ class MutableChain(Iterable):def __init__(self, *args: Iterable):self.data = chain.from_iterable(args)- def extend(self, *iterables: Iterable):+ def extend(self, *iterables: Iterable) -> None:self.data = chain(self.data, chain.from_iterable(iterables))def __iter__(self):@@ -359,22 +359,22 @@ class MutableChain(Iterable):return self.__next__()-async def _async_chain(*iterables):+async def _async_chain(*iterables: Union[Iterable, AsyncIterable]):for it in iterables:async for o in as_async_generator(it):yield o-class MutableAsyncChain:+class MutableAsyncChain(AsyncIterable):"""Similar to MutableChain but for async iterables"""- def __init__(self, *args):+ def __init__(self, *args: Union[Iterable, AsyncIterable]):self.data = _async_chain(*args)- def extend(self, *aiterables):- self.data = _async_chain(self.data, _async_chain(*aiterables))+ def extend(self, *iterables: Union[Iterable, AsyncIterable]) -> None:+ self.data = _async_chain(self.data, _async_chain(*iterables))def __aiter__(self):return self