Scrapy, a fast high-level web crawling & scraping framework for Python.
Add typing for middleware and coroutine related code.
Andrey Rakhmatullina9e96f990778
diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.pyindex b0e612e43..441fc9fa6 100644--- a/scrapy/core/downloader/middleware.py+++ b/scrapy/core/downloader/middleware.py@@ -3,8 +3,12 @@ Downloader Middleware managerSee documentation in docs/topics/downloader-middleware.rst"""+from typing import Callable, Union+from twisted.internet import defer+from twisted.python.failure import Failure+from scrapy import Spiderfrom scrapy.exceptions import _InvalidOutputfrom scrapy.http import Request, Responsefrom scrapy.middleware import MiddlewareManager@@ -29,9 +33,9 @@ class DownloaderMiddlewareManager(MiddlewareManager):if hasattr(mw, 'process_exception'):self.methods['process_exception'].appendleft(mw.process_exception)- def download(self, download_func, request, spider):+ def download(self, download_func: Callable, request: Request, spider: Spider):@defer.inlineCallbacks- def process_request(request):+ def process_request(request: Request):for method in self.methods['process_request']:response = yield deferred_from_coro(method(request=request, spider=spider))if response is not None and not isinstance(response, (Response, Request)):@@ -45,7 +49,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):return (yield download_func(request=request, spider=spider))@defer.inlineCallbacks- def process_response(response):+ def process_response(response: Union[Response, Request]):if response is None:raise TypeError("Received None in process_response")elif isinstance(response, Request):@@ -64,7 +68,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):return response@defer.inlineCallbacks- def process_exception(failure):+ def process_exception(failure: Failure):exception = failure.valuefor method in self.methods['process_exception']:response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider))diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.pyindex 289292da7..b09adf8e2 100644--- a/scrapy/core/spidermw.py+++ b/scrapy/core/spidermw.py@@ -3,25 +3,32 @@ Spider Middleware managerSee documentation in docs/topics/spider-middleware.rst"""+from collections.abc import Iterable, AsyncIterablefrom itertools import islice+from typing import Callable, Union, Anyfrom twisted.python.failure import Failure+from scrapy import Request, Spiderfrom scrapy.exceptions import _InvalidOutput+from scrapy.http import Responsefrom scrapy.middleware import MiddlewareManagerfrom scrapy.utils.conf import build_component_listfrom scrapy.utils.defer import mustbe_deferredfrom scrapy.utils.python import MutableChain-def _isiterable(possible_iterator):- return hasattr(possible_iterator, '__iter__')+def _isiterable(o):+ return isinstance(o, Iterable)def _fname(f):return f"{f.__self__.__class__.__name__}.{f.__func__.__name__}"+ScrapeFunc = Callable[[Union[Response, Failure], Request, Spider], Any]++class SpiderMiddlewareManager(MiddlewareManager):component_name = 'spider middleware'@@ -41,7 +48,7 @@ class SpiderMiddlewareManager(MiddlewareManager):process_spider_exception = getattr(mw, 'process_spider_exception', None)self.methods['process_spider_exception'].appendleft(process_spider_exception)- def _process_spider_input(self, scrape_func, response, request, spider):+ def _process_spider_input(self, scrape_func: ScrapeFunc, response: Response, request: Request, spider: Spider):for method in self.methods['process_spider_input']:try:result = method(response=response, spider=spider)@@ -55,7 +62,8 @@ class SpiderMiddlewareManager(MiddlewareManager):return scrape_func(Failure(), request, spider)return scrape_func(response, request, spider)- def _evaluate_iterable(self, response, spider, iterable, exception_processor_index, recover_to):+ def _evaluate_iterable(self, response: Response, spider: Spider, iterable: Iterable,+ exception_processor_index: int, recover_to: MutableChain):try:for r in iterable:yield r@@ -66,7 +74,7 @@ class SpiderMiddlewareManager(MiddlewareManager):raiserecover_to.extend(exception_result)- def _process_spider_exception(self, response, spider, _failure, start_index=0):+ def _process_spider_exception(self, response: Response, spider: Spider, _failure: Failure, start_index=0):exception = _failure.value# don't handle _InvalidOutput exceptionif isinstance(exception, _InvalidOutput):@@ -88,7 +96,8 @@ class SpiderMiddlewareManager(MiddlewareManager):raise _InvalidOutput(msg)return _failure- def _process_spider_output(self, response, spider, result, start_index=0):+ def _process_spider_output(self, response: Response, spider: Spider,+ result: Iterable, start_index=0):# items in this iterable do not need to go through the process_spider_output# chain, they went through it already from the process_spider_exception methodrecovered = MutableChain()@@ -114,21 +123,21 @@ class SpiderMiddlewareManager(MiddlewareManager):return MutableChain(result, recovered)- def _process_callback_output(self, response, spider, result):+ def _process_callback_output(self, response: Response, spider: Spider, result: Iterable):recovered = MutableChain()result = self._evaluate_iterable(response, spider, result, 0, recovered)return MutableChain(self._process_spider_output(response, spider, result), recovered)- def scrape_response(self, scrape_func, response, request, spider):- def process_callback_output(result):+ def scrape_response(self, scrape_func: ScrapeFunc, response: Response, request: Request, spider: Spider):+ def process_callback_output(result: Iterable):return self._process_callback_output(response, spider, result)- def process_spider_exception(_failure):+ def process_spider_exception(_failure: Failure):return self._process_spider_exception(response, spider, _failure)dfd = mustbe_deferred(self._process_spider_input, scrape_func, response, request, spider)dfd.addCallbacks(callback=process_callback_output, errback=process_spider_exception)return dfd- def process_start_requests(self, start_requests, spider):+ def process_start_requests(self, start_requests, spider: Spider):return self._process_chain('process_start_requests', start_requests, spider)diff --git a/scrapy/middleware.py b/scrapy/middleware.pyindex 5040378ea..c53cfb814 100644--- a/scrapy/middleware.py+++ b/scrapy/middleware.py@@ -1,6 +1,7 @@-from collections import defaultdict, dequeimport loggingimport pprint+from collections import defaultdict, deque+from typing import Callablefrom scrapy.exceptions import NotConfiguredfrom scrapy.utils.misc import create_instance, load_object@@ -16,7 +17,7 @@ class MiddlewareManager:def __init__(self, *middlewares):self.middlewares = middlewares- self.methods = defaultdict(deque)+ self.methods: dict[str, deque[Callable]] = defaultdict(deque)for mw in middlewares:self._add_middleware(mw)@@ -58,13 +59,13 @@ class MiddlewareManager:if hasattr(mw, 'close_spider'):self.methods['close_spider'].appendleft(mw.close_spider)- def _process_parallel(self, methodname, obj, *args):+ def _process_parallel(self, methodname: str, obj, *args):return process_parallel(self.methods[methodname], obj, *args)- def _process_chain(self, methodname, obj, *args):+ def _process_chain(self, methodname: str, obj, *args):return process_chain(self.methods[methodname], obj, *args)- def _process_chain_both(self, cb_methodname, eb_methodname, obj, *args):+ def _process_chain_both(self, cb_methodname: str, eb_methodname: str, obj, *args):return process_chain_both(self.methods[cb_methodname],self.methods[eb_methodname], obj, *args)diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.pyindex 7f697af5f..c290e376c 100644--- a/scrapy/utils/asyncgen.py+++ b/scrapy/utils/asyncgen.py@@ -1,4 +1,7 @@-async def collect_asyncgen(result):+from collections.abc import AsyncIterable+++async def collect_asyncgen(result: AsyncIterable):results = []async for x in result:results.append(x)diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.pyindex 6db9cc117..c382a00f7 100644--- a/scrapy/utils/defer.py+++ b/scrapy/utils/defer.py@@ -3,16 +3,19 @@ Helper functions for dealing with Twisted deferreds"""import asyncioimport inspect+from collections.abc import Coroutinefrom functools import wraps+from typing import Callable, Iterable, Anyfrom twisted.internet import defer, taskfrom twisted.python import failure+from twisted.python.failure import Failurefrom scrapy.exceptions import IgnoreRequestfrom scrapy.utils.reactor import is_asyncio_reactor_installed-def defer_fail(_failure):+def defer_fail(_failure: Failure):"""Same as twisted.internet.defer.fail but delay calling errback untilnext reactor loop@@ -47,7 +50,7 @@ def defer_result(result):return defer_succeed(result)-def mustbe_deferred(f, *args, **kw):+def mustbe_deferred(f: Callable, *args, **kw):"""Same as twisted.internet.defer.maybeDeferred, but delay callingcallback/errback to next reactor loop"""@@ -64,7 +67,7 @@ def mustbe_deferred(f, *args, **kw):return defer_result(result)-def parallel(iterable, count, callable, *args, **named):+def parallel(iterable: Iterable, count: int, callable: Callable, *args, **named):"""Execute a callable over the objects in the given iterable, in parallel,using no more than ``count`` concurrent calls.@@ -75,7 +78,7 @@ def parallel(iterable, count, callable, *args, **named):return defer.DeferredList([coop.coiterate(work) for _ in range(count)])-def process_chain(callbacks, input, *a, **kw):+def process_chain(callbacks: Iterable[Callable], input, *a, **kw):"""Return a Deferred built by chaining the given callbacks"""d = defer.Deferred()for x in callbacks:@@ -84,7 +87,7 @@ def process_chain(callbacks, input, *a, **kw):return d-def process_chain_both(callbacks, errbacks, input, *a, **kw):+def process_chain_both(callbacks: Iterable[Callable], errbacks: Iterable[Callable], input, *a, **kw):"""Return a Deferred built by chaining the given callbacks and errbacks"""d = defer.Deferred()for cb, eb in zip(callbacks, errbacks):@@ -100,7 +103,7 @@ def process_chain_both(callbacks, errbacks, input, *a, **kw):return d-def process_parallel(callbacks, input, *a, **kw):+def process_parallel(callbacks: Iterable[Callable], input, *a, **kw):"""Return a Deferred with the output of all successful calls to the givencallbacks"""@@ -110,7 +113,7 @@ def process_parallel(callbacks, input, *a, **kw):return d-def iter_errback(iterable, errback, *a, **kw):+def iter_errback(iterable: Iterable, errback: Callable, *a, **kw):"""Wraps an iterable calling an errback if an error is caught whileiterating it."""@@ -124,7 +127,7 @@ def iter_errback(iterable, errback, *a, **kw):errback(failure.Failure(), *a, **kw)-def deferred_from_coro(o):+def deferred_from_coro(o) -> Any:"""Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine"""if isinstance(o, defer.Deferred):return o@@ -139,7 +142,7 @@ def deferred_from_coro(o):return o-def deferred_f_from_coro_f(coro_f):+def deferred_f_from_coro_f(coro_f: Callable[..., Coroutine]):""" Converts a coroutine function into a function that returns a Deferred.The coroutine function will be called at the time when the wrapper is called. Wrapper args will be passed to it.@@ -151,7 +154,7 @@ def deferred_f_from_coro_f(coro_f):return f-def maybeDeferred_coro(f, *args, **kw):+def maybeDeferred_coro(f: Callable, *args, **kw):""" Copy of defer.maybeDeferred that also converts coroutines to Deferreds. """try:result = f(*args, **kw)diff --git a/scrapy/utils/python.py b/scrapy/utils/python.pyindex 5703fd4c3..bcc12f24f 100644--- a/scrapy/utils/python.py+++ b/scrapy/utils/python.py@@ -8,6 +8,7 @@ import reimport sysimport warningsimport weakref+from collections.abc import Iterablefrom functools import partial, wrapsfrom itertools import chain@@ -335,15 +336,15 @@ else:gc.collect()-class MutableChain:+class MutableChain(Iterable):"""Thin wrapper around itertools.chain, allowing to add iterables "in-place""""- def __init__(self, *args):+ def __init__(self, *args: Iterable):self.data = chain.from_iterable(args)- def extend(self, *iterables):+ def extend(self, *iterables: Iterable):self.data = chain(self.data, chain.from_iterable(iterables))def __iter__(self):