Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add typing for middleware and coroutine related code.

Andrey Rakhmatullina9e96f990778
diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py
index b0e612e43..441fc9fa6 100644
--- a/scrapy/core/downloader/middleware.py
+++ b/scrapy/core/downloader/middleware.py
@@ -3,8 +3,12 @@ Downloader Middleware manager
See documentation in docs/topics/downloader-middleware.rst
"""
+from typing import Callable, Union
+
from twisted.internet import defer
+from twisted.python.failure import Failure
+from scrapy import Spider
from scrapy.exceptions import _InvalidOutput
from scrapy.http import Request, Response
from scrapy.middleware import MiddlewareManager
@@ -29,9 +33,9 @@ class DownloaderMiddlewareManager(MiddlewareManager):
if hasattr(mw, 'process_exception'):
self.methods['process_exception'].appendleft(mw.process_exception)
- def download(self, download_func, request, spider):
+ def download(self, download_func: Callable, request: Request, spider: Spider):
@defer.inlineCallbacks
- def process_request(request):
+ def process_request(request: Request):
for method in self.methods['process_request']:
response = yield deferred_from_coro(method(request=request, spider=spider))
if response is not None and not isinstance(response, (Response, Request)):
@@ -45,7 +49,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
return (yield download_func(request=request, spider=spider))
@defer.inlineCallbacks
- def process_response(response):
+ def process_response(response: Union[Response, Request]):
if response is None:
raise TypeError("Received None in process_response")
elif isinstance(response, Request):
@@ -64,7 +68,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
return response
@defer.inlineCallbacks
- def process_exception(failure):
+ def process_exception(failure: Failure):
exception = failure.value
for method in self.methods['process_exception']:
response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider))
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 289292da7..b09adf8e2 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -3,25 +3,32 @@ Spider Middleware manager
See documentation in docs/topics/spider-middleware.rst
"""
+from collections.abc import Iterable, AsyncIterable
from itertools import islice
+from typing import Callable, Union, Any
from twisted.python.failure import Failure
+from scrapy import Request, Spider
from scrapy.exceptions import _InvalidOutput
+from scrapy.http import Response
from scrapy.middleware import MiddlewareManager
from scrapy.utils.conf import build_component_list
from scrapy.utils.defer import mustbe_deferred
from scrapy.utils.python import MutableChain
-def _isiterable(possible_iterator):
- return hasattr(possible_iterator, '__iter__')
+def _isiterable(o):
+ return isinstance(o, Iterable)
def _fname(f):
return f"{f.__self__.__class__.__name__}.{f.__func__.__name__}"
+ScrapeFunc = Callable[[Union[Response, Failure], Request, Spider], Any]
+
+
class SpiderMiddlewareManager(MiddlewareManager):
component_name = 'spider middleware'
@@ -41,7 +48,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
process_spider_exception = getattr(mw, 'process_spider_exception', None)
self.methods['process_spider_exception'].appendleft(process_spider_exception)
- def _process_spider_input(self, scrape_func, response, request, spider):
+ def _process_spider_input(self, scrape_func: ScrapeFunc, response: Response, request: Request, spider: Spider):
for method in self.methods['process_spider_input']:
try:
result = method(response=response, spider=spider)
@@ -55,7 +62,8 @@ class SpiderMiddlewareManager(MiddlewareManager):
return scrape_func(Failure(), request, spider)
return scrape_func(response, request, spider)
- def _evaluate_iterable(self, response, spider, iterable, exception_processor_index, recover_to):
+ def _evaluate_iterable(self, response: Response, spider: Spider, iterable: Iterable,
+ exception_processor_index: int, recover_to: MutableChain):
try:
for r in iterable:
yield r
@@ -66,7 +74,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
raise
recover_to.extend(exception_result)
- def _process_spider_exception(self, response, spider, _failure, start_index=0):
+ def _process_spider_exception(self, response: Response, spider: Spider, _failure: Failure, start_index=0):
exception = _failure.value
# don't handle _InvalidOutput exception
if isinstance(exception, _InvalidOutput):
@@ -88,7 +96,8 @@ class SpiderMiddlewareManager(MiddlewareManager):
raise _InvalidOutput(msg)
return _failure
- def _process_spider_output(self, response, spider, result, start_index=0):
+ def _process_spider_output(self, response: Response, spider: Spider,
+ result: Iterable, start_index=0):
# items in this iterable do not need to go through the process_spider_output
# chain, they went through it already from the process_spider_exception method
recovered = MutableChain()
@@ -114,21 +123,21 @@ class SpiderMiddlewareManager(MiddlewareManager):
return MutableChain(result, recovered)
- def _process_callback_output(self, response, spider, result):
+ def _process_callback_output(self, response: Response, spider: Spider, result: Iterable):
recovered = MutableChain()
result = self._evaluate_iterable(response, spider, result, 0, recovered)
return MutableChain(self._process_spider_output(response, spider, result), recovered)
- def scrape_response(self, scrape_func, response, request, spider):
- def process_callback_output(result):
+ def scrape_response(self, scrape_func: ScrapeFunc, response: Response, request: Request, spider: Spider):
+ def process_callback_output(result: Iterable):
return self._process_callback_output(response, spider, result)
- def process_spider_exception(_failure):
+ def process_spider_exception(_failure: Failure):
return self._process_spider_exception(response, spider, _failure)
dfd = mustbe_deferred(self._process_spider_input, scrape_func, response, request, spider)
dfd.addCallbacks(callback=process_callback_output, errback=process_spider_exception)
return dfd
- def process_start_requests(self, start_requests, spider):
+ def process_start_requests(self, start_requests, spider: Spider):
return self._process_chain('process_start_requests', start_requests, spider)
diff --git a/scrapy/middleware.py b/scrapy/middleware.py
index 5040378ea..c53cfb814 100644
--- a/scrapy/middleware.py
+++ b/scrapy/middleware.py
@@ -1,6 +1,7 @@
-from collections import defaultdict, deque
import logging
import pprint
+from collections import defaultdict, deque
+from typing import Callable
from scrapy.exceptions import NotConfigured
from scrapy.utils.misc import create_instance, load_object
@@ -16,7 +17,7 @@ class MiddlewareManager:
def __init__(self, *middlewares):
self.middlewares = middlewares
- self.methods = defaultdict(deque)
+ self.methods: dict[str, deque[Callable]] = defaultdict(deque)
for mw in middlewares:
self._add_middleware(mw)
@@ -58,13 +59,13 @@ class MiddlewareManager:
if hasattr(mw, 'close_spider'):
self.methods['close_spider'].appendleft(mw.close_spider)
- def _process_parallel(self, methodname, obj, *args):
+ def _process_parallel(self, methodname: str, obj, *args):
return process_parallel(self.methods[methodname], obj, *args)
- def _process_chain(self, methodname, obj, *args):
+ def _process_chain(self, methodname: str, obj, *args):
return process_chain(self.methods[methodname], obj, *args)
- def _process_chain_both(self, cb_methodname, eb_methodname, obj, *args):
+ def _process_chain_both(self, cb_methodname: str, eb_methodname: str, obj, *args):
return process_chain_both(self.methods[cb_methodname],
self.methods[eb_methodname], obj, *args)
diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py
index 7f697af5f..c290e376c 100644
--- a/scrapy/utils/asyncgen.py
+++ b/scrapy/utils/asyncgen.py
@@ -1,4 +1,7 @@
-async def collect_asyncgen(result):
+from collections.abc import AsyncIterable
+
+
+async def collect_asyncgen(result: AsyncIterable):
results = []
async for x in result:
results.append(x)
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 6db9cc117..c382a00f7 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -3,16 +3,19 @@ Helper functions for dealing with Twisted deferreds
"""
import asyncio
import inspect
+from collections.abc import Coroutine
from functools import wraps
+from typing import Callable, Iterable, Any
from twisted.internet import defer, task
from twisted.python import failure
+from twisted.python.failure import Failure
from scrapy.exceptions import IgnoreRequest
from scrapy.utils.reactor import is_asyncio_reactor_installed
-def defer_fail(_failure):
+def defer_fail(_failure: Failure):
"""Same as twisted.internet.defer.fail but delay calling errback until
next reactor loop
@@ -47,7 +50,7 @@ def defer_result(result):
return defer_succeed(result)
-def mustbe_deferred(f, *args, **kw):
+def mustbe_deferred(f: Callable, *args, **kw):
"""Same as twisted.internet.defer.maybeDeferred, but delay calling
callback/errback to next reactor loop
"""
@@ -64,7 +67,7 @@ def mustbe_deferred(f, *args, **kw):
return defer_result(result)
-def parallel(iterable, count, callable, *args, **named):
+def parallel(iterable: Iterable, count: int, callable: Callable, *args, **named):
"""Execute a callable over the objects in the given iterable, in parallel,
using no more than ``count`` concurrent calls.
@@ -75,7 +78,7 @@ def parallel(iterable, count, callable, *args, **named):
return defer.DeferredList([coop.coiterate(work) for _ in range(count)])
-def process_chain(callbacks, input, *a, **kw):
+def process_chain(callbacks: Iterable[Callable], input, *a, **kw):
"""Return a Deferred built by chaining the given callbacks"""
d = defer.Deferred()
for x in callbacks:
@@ -84,7 +87,7 @@ def process_chain(callbacks, input, *a, **kw):
return d
-def process_chain_both(callbacks, errbacks, input, *a, **kw):
+def process_chain_both(callbacks: Iterable[Callable], errbacks: Iterable[Callable], input, *a, **kw):
"""Return a Deferred built by chaining the given callbacks and errbacks"""
d = defer.Deferred()
for cb, eb in zip(callbacks, errbacks):
@@ -100,7 +103,7 @@ def process_chain_both(callbacks, errbacks, input, *a, **kw):
return d
-def process_parallel(callbacks, input, *a, **kw):
+def process_parallel(callbacks: Iterable[Callable], input, *a, **kw):
"""Return a Deferred with the output of all successful calls to the given
callbacks
"""
@@ -110,7 +113,7 @@ def process_parallel(callbacks, input, *a, **kw):
return d
-def iter_errback(iterable, errback, *a, **kw):
+def iter_errback(iterable: Iterable, errback: Callable, *a, **kw):
"""Wraps an iterable calling an errback if an error is caught while
iterating it.
"""
@@ -124,7 +127,7 @@ def iter_errback(iterable, errback, *a, **kw):
errback(failure.Failure(), *a, **kw)
-def deferred_from_coro(o):
+def deferred_from_coro(o) -> Any:
"""Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine"""
if isinstance(o, defer.Deferred):
return o
@@ -139,7 +142,7 @@ def deferred_from_coro(o):
return o
-def deferred_f_from_coro_f(coro_f):
+def deferred_f_from_coro_f(coro_f: Callable[..., Coroutine]):
""" Converts a coroutine function into a function that returns a Deferred.
The coroutine function will be called at the time when the wrapper is called. Wrapper args will be passed to it.
@@ -151,7 +154,7 @@ def deferred_f_from_coro_f(coro_f):
return f
-def maybeDeferred_coro(f, *args, **kw):
+def maybeDeferred_coro(f: Callable, *args, **kw):
""" Copy of defer.maybeDeferred that also converts coroutines to Deferreds. """
try:
result = f(*args, **kw)
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 5703fd4c3..bcc12f24f 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -8,6 +8,7 @@ import re
import sys
import warnings
import weakref
+from collections.abc import Iterable
from functools import partial, wraps
from itertools import chain
@@ -335,15 +336,15 @@ else:
gc.collect()
-class MutableChain:
+class MutableChain(Iterable):
"""
Thin wrapper around itertools.chain, allowing to add iterables "in-place"
"""
- def __init__(self, *args):
+ def __init__(self, *args: Iterable):
self.data = chain.from_iterable(args)
- def extend(self, *iterables):
+ def extend(self, *iterables: Iterable):
self.data = chain(self.data, chain.from_iterable(iterables))
def __iter__(self):