Scrapy, a fast high-level web crawling & scraping framework for Python.
Add typing also for return values, other small fixes.
Andrey Rakhmatullin76fa2257ef02
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.pyindex dc0e58095..05df8c988 100644--- a/scrapy/core/spidermw.py+++ b/scrapy/core/spidermw.py@@ -3,10 +3,10 @@ Spider Middleware managerSee documentation in docs/topics/spider-middleware.rst"""-from collections.abc import Iterablefrom itertools import islice-from typing import Callable, Union, Any+from typing import Callable, Union, Any, Generator, Iterable+from twisted.internet.defer import Deferredfrom twisted.python.failure import Failurefrom scrapy import Request, Spider@@ -18,11 +18,11 @@ from scrapy.utils.defer import mustbe_deferredfrom scrapy.utils.python import MutableChain-def _isiterable(o):- return isinstance(o, Iterable)+ScrapeFunc = Callable[[Union[Response, Failure], Request, Spider], Any]-ScrapeFunc = Callable[[Union[Response, Failure], Request, Spider], Any]+def _isiterable(o) -> bool:+ return isinstance(o, Iterable)class SpiderMiddlewareManager(MiddlewareManager):@@ -44,7 +44,8 @@ class SpiderMiddlewareManager(MiddlewareManager):process_spider_exception = getattr(mw, 'process_spider_exception', None)self.methods['process_spider_exception'].appendleft(process_spider_exception)- def _process_spider_input(self, scrape_func: ScrapeFunc, response: Response, request: Request, spider: Spider):+ def _process_spider_input(self, scrape_func: ScrapeFunc, response: Response, request: Request,+ spider: Spider) -> Any:for method in self.methods['process_spider_input']:try:result = method(response=response, spider=spider)@@ -59,7 +60,7 @@ class SpiderMiddlewareManager(MiddlewareManager):return scrape_func(response, request, spider)def _evaluate_iterable(self, response: Response, spider: Spider, iterable: Iterable,- exception_processor_index: int, recover_to: MutableChain):+ exception_processor_index: int, recover_to: MutableChain) -> Generator:try:for r in iterable:yield r@@ -70,7 +71,8 @@ class SpiderMiddlewareManager(MiddlewareManager):raiserecover_to.extend(exception_result)- def _process_spider_exception(self, response: Response, spider: Spider, _failure: Failure, start_index=0):+ def _process_spider_exception(self, response: Response, spider: Spider, _failure: Failure,+ start_index: int = 0) -> Union[Failure, MutableChain]:exception = _failure.value# don't handle _InvalidOutput exceptionif isinstance(exception, _InvalidOutput):@@ -93,7 +95,7 @@ class SpiderMiddlewareManager(MiddlewareManager):return _failuredef _process_spider_output(self, response: Response, spider: Spider,- result: Iterable, start_index=0):+ result: Iterable, start_index: int = 0) -> MutableChain:# items in this iterable do not need to go through the process_spider_output# chain, they went through it already from the process_spider_exception methodrecovered = MutableChain()@@ -119,21 +121,22 @@ class SpiderMiddlewareManager(MiddlewareManager):return MutableChain(result, recovered)- def _process_callback_output(self, response: Response, spider: Spider, result: Iterable):+ def _process_callback_output(self, response: Response, spider: Spider, result: Iterable) -> MutableChain:recovered = MutableChain()result = self._evaluate_iterable(response, spider, result, 0, recovered)return MutableChain(self._process_spider_output(response, spider, result), recovered)- def scrape_response(self, scrape_func: ScrapeFunc, response: Response, request: Request, spider: Spider):- def process_callback_output(result: Iterable):+ def scrape_response(self, scrape_func: ScrapeFunc, response: Response, request: Request,+ spider: Spider) -> Deferred:+ def process_callback_output(result: Iterable) -> MutableChain:return self._process_callback_output(response, spider, result)- def process_spider_exception(_failure: Failure):+ def process_spider_exception(_failure: Failure) -> Union[Failure, MutableChain]:return self._process_spider_exception(response, spider, _failure)dfd = mustbe_deferred(self._process_spider_input, scrape_func, response, request, spider)dfd.addCallbacks(callback=process_callback_output, errback=process_spider_exception)return dfd- def process_start_requests(self, start_requests, spider: Spider):+ def process_start_requests(self, start_requests, spider: Spider) -> Deferred:return self._process_chain('process_start_requests', start_requests, spider)diff --git a/scrapy/middleware.py b/scrapy/middleware.pyindex c53cfb814..3f8c1cbf5 100644--- a/scrapy/middleware.py+++ b/scrapy/middleware.py@@ -1,9 +1,13 @@import loggingimport pprintfrom collections import defaultdict, deque-from typing import Callable+from typing import Callable, Dict, Deque+from twisted.internet import defer++from scrapy import Spiderfrom scrapy.exceptions import NotConfigured+from scrapy.settings import Settingsfrom scrapy.utils.misc import create_instance, load_objectfrom scrapy.utils.defer import process_parallel, process_chain, process_chain_both@@ -17,16 +21,16 @@ class MiddlewareManager:def __init__(self, *middlewares):self.middlewares = middlewares- self.methods: dict[str, deque[Callable]] = defaultdict(deque)+ self.methods: Dict[str, Deque[Callable]] = defaultdict(deque)for mw in middlewares:self._add_middleware(mw)@classmethod- def _get_mwlist_from_settings(cls, settings):+ def _get_mwlist_from_settings(cls, settings: Settings) -> list:raise NotImplementedError@classmethod- def from_settings(cls, settings, crawler=None):+ def from_settings(cls, settings: Settings, crawler=None):mwlist = cls._get_mwlist_from_settings(settings)middlewares = []enabled = []@@ -53,24 +57,24 @@ class MiddlewareManager:def from_crawler(cls, crawler):return cls.from_settings(crawler.settings, crawler)- def _add_middleware(self, mw):+ def _add_middleware(self, mw) -> None:if hasattr(mw, 'open_spider'):self.methods['open_spider'].append(mw.open_spider)if hasattr(mw, 'close_spider'):self.methods['close_spider'].appendleft(mw.close_spider)- def _process_parallel(self, methodname: str, obj, *args):+ def _process_parallel(self, methodname: str, obj, *args) -> defer.Deferred:return process_parallel(self.methods[methodname], obj, *args)- def _process_chain(self, methodname: str, obj, *args):+ def _process_chain(self, methodname: str, obj, *args) -> defer.Deferred:return process_chain(self.methods[methodname], obj, *args)- def _process_chain_both(self, cb_methodname: str, eb_methodname: str, obj, *args):+ def _process_chain_both(self, cb_methodname: str, eb_methodname: str, obj, *args) -> defer.Deferred:return process_chain_both(self.methods[cb_methodname],self.methods[eb_methodname], obj, *args)- def open_spider(self, spider):+ def open_spider(self, spider: Spider) -> defer.Deferred:return self._process_parallel('open_spider', spider)- def close_spider(self, spider):+ def close_spider(self, spider: Spider) -> defer.Deferred:return self._process_parallel('close_spider', spider)diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.pyindex c382a00f7..095eae94c 100644--- a/scrapy/utils/defer.py+++ b/scrapy/utils/defer.py@@ -5,7 +5,7 @@ import asyncioimport inspectfrom collections.abc import Coroutinefrom functools import wraps-from typing import Callable, Iterable, Any+from typing import Callable, Iterable, Any, Generatorfrom twisted.internet import defer, taskfrom twisted.python import failure@@ -15,7 +15,7 @@ from scrapy.exceptions import IgnoreRequestfrom scrapy.utils.reactor import is_asyncio_reactor_installed-def defer_fail(_failure: Failure):+def defer_fail(_failure: Failure) -> defer.Deferred:"""Same as twisted.internet.defer.fail but delay calling errback untilnext reactor loop@@ -28,7 +28,7 @@ def defer_fail(_failure: Failure):return d-def defer_succeed(result):+def defer_succeed(result) -> defer.Deferred:"""Same as twisted.internet.defer.succeed but delay calling callback untilnext reactor loop@@ -41,7 +41,7 @@ def defer_succeed(result):return d-def defer_result(result):+def defer_result(result) -> defer.Deferred:if isinstance(result, defer.Deferred):return resultelif isinstance(result, failure.Failure):@@ -50,7 +50,7 @@ def defer_result(result):return defer_succeed(result)-def mustbe_deferred(f: Callable, *args, **kw):+def mustbe_deferred(f: Callable, *args, **kw) -> defer.Deferred:"""Same as twisted.internet.defer.maybeDeferred, but delay callingcallback/errback to next reactor loop"""@@ -67,7 +67,7 @@ def mustbe_deferred(f: Callable, *args, **kw):return defer_result(result)-def parallel(iterable: Iterable, count: int, callable: Callable, *args, **named):+def parallel(iterable: Iterable, count: int, callable: Callable, *args, **named) -> defer.DeferredList:"""Execute a callable over the objects in the given iterable, in parallel,using no more than ``count`` concurrent calls.@@ -78,7 +78,7 @@ def parallel(iterable: Iterable, count: int, callable: Callable, *args, **named)return defer.DeferredList([coop.coiterate(work) for _ in range(count)])-def process_chain(callbacks: Iterable[Callable], input, *a, **kw):+def process_chain(callbacks: Iterable[Callable], input, *a, **kw) -> defer.Deferred:"""Return a Deferred built by chaining the given callbacks"""d = defer.Deferred()for x in callbacks:@@ -87,7 +87,7 @@ def process_chain(callbacks: Iterable[Callable], input, *a, **kw):return d-def process_chain_both(callbacks: Iterable[Callable], errbacks: Iterable[Callable], input, *a, **kw):+def process_chain_both(callbacks: Iterable[Callable], errbacks: Iterable[Callable], input, *a, **kw) -> defer.Deferred:"""Return a Deferred built by chaining the given callbacks and errbacks"""d = defer.Deferred()for cb, eb in zip(callbacks, errbacks):@@ -103,7 +103,7 @@ def process_chain_both(callbacks: Iterable[Callable], errbacks: Iterable[Callablreturn d-def process_parallel(callbacks: Iterable[Callable], input, *a, **kw):+def process_parallel(callbacks: Iterable[Callable], input, *a, **kw) -> defer.Deferred:"""Return a Deferred with the output of all successful calls to the givencallbacks"""@@ -113,7 +113,7 @@ def process_parallel(callbacks: Iterable[Callable], input, *a, **kw):return d-def iter_errback(iterable: Iterable, errback: Callable, *a, **kw):+def iter_errback(iterable: Iterable, errback: Callable, *a, **kw) -> Generator:"""Wraps an iterable calling an errback if an error is caught whileiterating it."""@@ -142,7 +142,7 @@ def deferred_from_coro(o) -> Any:return o-def deferred_f_from_coro_f(coro_f: Callable[..., Coroutine]):+def deferred_f_from_coro_f(coro_f: Callable[..., Coroutine]) -> Callable:""" Converts a coroutine function into a function that returns a Deferred.The coroutine function will be called at the time when the wrapper is called. Wrapper args will be passed to it.@@ -154,7 +154,7 @@ def deferred_f_from_coro_f(coro_f: Callable[..., Coroutine]):return f-def maybeDeferred_coro(f: Callable, *args, **kw):+def maybeDeferred_coro(f: Callable, *args, **kw) -> defer.Deferred:""" Copy of defer.maybeDeferred that also converts coroutines to Deferreds. """try:result = f(*args, **kw)