Scrapy, a fast high-level web crawling & scraping framework for Python.
Add `DownloaderMiddlewareManager.download_async()`, deprecate returning Deferreds from downloader mws (#7069)
Andrey Rakhmatullin5105f55a9806
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rstindex 6f36acc68..6d24482ec 100644--- a/docs/topics/downloader-middleware.rst+++ b/docs/topics/downloader-middleware.rst@@ -68,7 +68,8 @@ defines one or more of these methods:.. class:: DownloaderMiddleware- .. note:: Any of the downloader middleware methods may also return a deferred.+ .. note:: Any of the downloader middleware methods may be defined as a+ coroutine function (``async def``)... method:: process_request(request)diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.pyindex 0d23aa85c..109a6eaec 100644--- a/scrapy/core/downloader/__init__.py+++ b/scrapy/core/downloader/__init__.py@@ -23,6 +23,7 @@ from scrapy.utils.decorators import _warn_spider_argfrom scrapy.utils.defer import (_defer_sleep_async,_schedule_coro,+ deferred_from_coro,maybe_deferred_to_future,)from scrapy.utils.deprecate import warn_on_deprecated_spider_attribute@@ -139,7 +140,11 @@ class Downloader:) -> Generator[Deferred[Any], Any, Response | Request]:self.active.add(request)try:- return (yield self.middleware.download(self._enqueue_request, request))+ return (+ yield deferred_from_coro(+ self.middleware.download_async(self._enqueue_request, request)+ )+ )finally:self.active.remove(request)@@ -178,10 +183,7 @@ class Downloader:return key# passed as download_func into self.middleware.download() in self.fetch()- @inlineCallbacks- def _enqueue_request(- self, request: Request- ) -> Generator[Deferred[Any], Any, Response]:+ async def _enqueue_request(self, request: Request) -> Response:key, slot = self._get_slot(request)request.meta[self.DOWNLOAD_SLOT] = keyslot.active.add(request)@@ -194,7 +196,7 @@ class Downloader:slot.queue.append((request, d))self._process_queue(slot)try:- return (yield d) # fired in _wait_for_download()+ return await maybe_deferred_to_future(d) # fired in _wait_for_download()finally:slot.active.remove(request)diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.pyindex a822ba811..431579856 100644--- a/scrapy/core/downloader/middleware.py+++ b/scrapy/core/downloader/middleware.py@@ -7,19 +7,25 @@ See documentation in docs/topics/downloader-middleware.rstfrom __future__ import annotationsimport warnings+from functools import wrapsfrom typing import TYPE_CHECKING, Any, cast-from twisted.internet.defer import Deferred, inlineCallbacks-from scrapy.exceptions import ScrapyDeprecationWarning, _InvalidOutputfrom scrapy.http import Request, Responsefrom scrapy.middleware import MiddlewareManagerfrom scrapy.utils.conf import build_component_list-from scrapy.utils.defer import _defer_sleep, deferred_from_coro-from scrapy.utils.deprecate import argument_is_required+from scrapy.utils.defer import (+ _defer_sleep_async,+ deferred_from_coro,+ ensure_awaitable,+ maybe_deferred_to_future,+)+from scrapy.utils.python import global_object_nameif TYPE_CHECKING:- from collections.abc import Callable, Generator+ from collections.abc import Callable, Coroutine++ from twisted.internet.defer import Deferredfrom scrapy import Spiderfrom scrapy.settings import BaseSettings@@ -43,36 +49,42 @@ class DownloaderMiddlewareManager(MiddlewareManager):self.methods["process_exception"].appendleft(mw.process_exception)self._check_mw_method_spider_arg(mw.process_exception)- @inlineCallbacksdef download(self,- download_func: Callable[[Request], Deferred[Response]],+ download_func: Callable[[Request, Spider], Deferred[Response]],+ request: Request,+ spider: Spider,+ ) -> Deferred[Response | Request]:+ warnings.warn(+ "DownloaderMiddlewareManager.download() is deprecated, use download_async() instead",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )++ @wraps(download_func)+ async def download_func_wrapped(request: Request) -> Response:+ return await maybe_deferred_to_future(download_func(request, spider))++ self._set_compat_spider(spider)+ return deferred_from_coro(self.download_async(download_func_wrapped, request))++ async def download_async(+ self,+ download_func: Callable[[Request], Coroutine[Any, Any, Response]],request: Request,- spider: Spider | None = None,- ) -> Generator[Deferred[Any], Any, Response | Request]:- if argument_is_required(download_func, "spider"):- warnings.warn(- "The spider argument of download_func is deprecated"- " and will not be passed in future Scrapy versions.",- ScrapyDeprecationWarning,- stacklevel=2,- )- need_spider_arg = True- else:- need_spider_arg = False-- @inlineCallbacks- def process_request(- request: Request,- ) -> Generator[Deferred[Any], Any, Response | Request]:+ ) -> Response | Request:+ async def process_request(request: Request) -> Response | Request:for method in self.methods["process_request"]:method = cast("Callable", method)if method in self._mw_methods_requiring_spider:- response = yield deferred_from_coro(- method(request=request, spider=self._spider)+ response = await ensure_awaitable(+ method(request=request, spider=self._spider),+ _warn=global_object_name(method),)else:- response = yield deferred_from_coro(method(request=request))+ response = await ensure_awaitable(+ method(request=request), _warn=global_object_name(method)+ )if response is not None and not isinstance(response, (Response, Request)):@@ -82,14 +94,9 @@ class DownloaderMiddlewareManager(MiddlewareManager):)if response:return response- if need_spider_arg:- return (yield download_func(request, self._spider)) # type: ignore[call-arg]- return (yield download_func(request))-- @inlineCallbacks- def process_response(- response: Response | Request,- ) -> Generator[Deferred[Any], Any, Response | Request]:+ return await download_func(request)++ async def process_response(response: Response | Request) -> Response | Request:if response is None:raise TypeError("Received None in process_response")if isinstance(response, Request):@@ -98,12 +105,14 @@ class DownloaderMiddlewareManager(MiddlewareManager):for method in self.methods["process_response"]:method = cast("Callable", method)if method in self._mw_methods_requiring_spider:- response = yield deferred_from_coro(- method(request=request, response=response, spider=self._spider)+ response = await ensure_awaitable(+ method(request=request, response=response, spider=self._spider),+ _warn=global_object_name(method),)else:- response = yield deferred_from_coro(- method(request=request, response=response)+ response = await ensure_awaitable(+ method(request=request, response=response),+ _warn=global_object_name(method),)if not isinstance(response, (Response, Request)):raise _InvalidOutput(@@ -114,21 +123,20 @@ class DownloaderMiddlewareManager(MiddlewareManager):return responsereturn response- @inlineCallbacks- def process_exception(- exception: Exception,- ) -> Generator[Deferred[Any], Any, Response | Request]:+ async def process_exception(exception: Exception) -> Response | Request:for method in self.methods["process_exception"]:method = cast("Callable", method)if method in self._mw_methods_requiring_spider:- response = yield deferred_from_coro(+ response = await ensure_awaitable(method(request=request, exception=exception, spider=self._spider- )+ ),+ _warn=global_object_name(method),)else:- response = yield deferred_from_coro(- method(request=request, exception=exception)+ response = await ensure_awaitable(+ method(request=request, exception=exception),+ _warn=global_object_name(method),)if response is not None and not isinstance(response, (Response, Request)@@ -141,14 +149,11 @@ class DownloaderMiddlewareManager(MiddlewareManager):return responseraise exception- if spider:- self._warn_spider_arg("download")- self._set_compat_spider(spider)try:- result: Response | Request = yield process_request(request)+ result: Response | Request = await process_request(request)except Exception as ex:- yield _defer_sleep()+ await _defer_sleep_async()# either returns a request or response (which we pass to process_response())# or reraises the exception- result = yield process_exception(ex)- return (yield process_response(result))+ result = await process_exception(ex)+ return await process_response(result)diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.pyindex 6f694753a..61fecfc60 100644--- a/scrapy/core/spidermw.py+++ b/scrapy/core/spidermw.py@@ -436,7 +436,17 @@ class SpiderMiddlewareManager(MiddlewareManager):self, spider: Spider | None = None) -> AsyncIterator[Any] | None:if spider:- self._warn_spider_arg("process_start")+ if self.crawler:+ msg = (+ "Passing a spider argument to SpiderMiddlewareManager.process_start() is deprecated"+ " and the passed value is ignored."+ )+ else:+ msg = (+ "Passing a spider argument to SpiderMiddlewareManager.process_start() is deprecated,"+ " SpiderMiddlewareManager should be instantiated with a Crawler instance instead."+ )+ warn(msg, category=ScrapyDeprecationWarning, stacklevel=2)self._set_compat_spider(spider)self._check_deprecated_start_requests_use()if self._use_start_requests:diff --git a/scrapy/middleware.py b/scrapy/middleware.pyindex 21a2ebfa3..83362784b 100644--- a/scrapy/middleware.py+++ b/scrapy/middleware.py@@ -82,19 +82,6 @@ class MiddlewareManager(ABC):f" {self._compat_spider} and {spider}")- def _warn_spider_arg(self, method_name: str) -> None:- if self.crawler:- msg = (- f"Passing a spider argument to {type(self).__name__}.{method_name}() is deprecated"- " and the passed value is ignored."- )- else:- msg = (- f"Passing a spider argument to {type(self).__name__}.{method_name}() is deprecated,"- f" {type(self).__name__} should be instantiated with a Crawler instance instead."- )- warnings.warn(msg, category=ScrapyDeprecationWarning, stacklevel=3)-@classmethod@abstractmethoddef _get_mwlist_from_settings(cls, settings: Settings) -> list[Any]:diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.pyindex 8ca3bc0bd..c27f8bb34 100644--- a/scrapy/utils/defer.py+++ b/scrapy/utils/defer.py@@ -517,14 +517,14 @@ def _schedule_coro(coro: Coroutine[Any, Any, Any]) -> None:@overload-def ensure_awaitable(o: Awaitable[_T]) -> Awaitable[_T]: ...+def ensure_awaitable(o: Awaitable[_T], _warn: str | None = None) -> Awaitable[_T]: ...@overload-def ensure_awaitable(o: _T) -> Awaitable[_T]: ...+def ensure_awaitable(o: _T, _warn: str | None = None) -> Awaitable[_T]: ...-def ensure_awaitable(o: _T | Awaitable[_T]) -> Awaitable[_T]:+def ensure_awaitable(o: _T | Awaitable[_T], _warn: str | None = None) -> Awaitable[_T]:"""Convert any value to an awaitable object.For a :class:`~twisted.internet.defer.Deferred` object, use@@ -535,6 +535,13 @@ def ensure_awaitable(o: _T | Awaitable[_T]) -> Awaitable[_T]:.. versionadded:: VERSION"""if isinstance(o, Deferred):+ if _warn:+ warnings.warn(+ f"{_warn} returned a Deferred, this is deprecated."+ f" Please refactor this function to return a coroutine.",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )return maybe_deferred_to_future(o)if inspect.isawaitable(o):return odiff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.pyindex 03b941e4f..bc1748dfb 100644--- a/tests/test_downloadermiddleware.py+++ b/tests/test_downloadermiddleware.py@@ -50,10 +50,10 @@ class TestManagerBase:if not response:response = Response(request.url)- def download_func(request: Request) -> Deferred[Response]:- return succeed(response)+ async def download_func(request: Request) -> Response:+ return response- return await maybe_deferred_to_future(mwman.download(download_func, request))+ return await mwman.download_async(download_func, request)class TestDefaults(TestManagerBase):@@ -135,7 +135,7 @@ class TestResponseFromProcessRequest(TestManagerBase):async with self.get_mwman() as mwman:mwman._add_middleware(ResponseMiddleware())- result = await maybe_deferred_to_future(mwman.download(download_func, req))+ result = await mwman.download_async(download_func, req)assert result is respassert not download_func.called@@ -163,7 +163,7 @@ class TestResponseFromProcessException(TestManagerBase):async with self.get_mwman() as mwman:mwman._add_middleware(ResponseMiddleware())- result = await maybe_deferred_to_future(mwman.download(download_func, req))+ result = await mwman.download_async(download_func, req)assert result is respassert calls == ["process_exception",@@ -219,7 +219,7 @@ class TestInvalidOutput(TestManagerBase):class TestMiddlewareUsingDeferreds(TestManagerBase):- """Middlewares using Deferreds should work"""+ """Middlewares using Deferreds (deprecated) should work"""@deferred_f_from_coro_fasync def test_deferred(self):@@ -239,7 +239,11 @@ class TestMiddlewareUsingDeferreds(TestManagerBase):async with self.get_mwman() as mwman:mwman._add_middleware(DeferredMiddleware())- result = await maybe_deferred_to_future(mwman.download(download_func, req))+ with pytest.warns(+ ScrapyDeprecationWarning,+ match="returned a Deferred, this is deprecated",+ ):+ result = await mwman.download_async(download_func, req)assert result is respassert not download_func.called@@ -260,7 +264,7 @@ class TestMiddlewareUsingCoro(TestManagerBase):async with self.get_mwman() as mwman:mwman._add_middleware(CoroMiddleware())- result = await maybe_deferred_to_future(mwman.download(download_func, req))+ result = await mwman.download_async(download_func, req)assert result is respassert not download_func.called@@ -278,14 +282,14 @@ class TestMiddlewareUsingCoro(TestManagerBase):async with self.get_mwman() as mwman:mwman._add_middleware(CoroMiddleware())- result = await maybe_deferred_to_future(mwman.download(download_func, req))+ result = await mwman.download_async(download_func, req)assert result is respassert not download_func.calledclass TestDownloadDeprecated(TestManagerBase):@deferred_f_from_coro_f- async def test_download_func_spider_arg(self):+ async def test_mwman_download(self):req = Request("http://example.com/index.html")resp = Response(req.url, status=200)@@ -295,24 +299,7 @@ class TestDownloadDeprecated(TestManagerBase):async with self.get_mwman() as mwman:with pytest.warns(ScrapyDeprecationWarning,- match="The spider argument of download_func is deprecated",- ):- ret = await maybe_deferred_to_future(mwman.download(download_func, req))- assert isinstance(ret, Response)-- @deferred_f_from_coro_f- async def test_mwman_download_spider_arg(self):- req = Request("http://example.com/index.html")- resp = Response(req.url, status=200)-- def download_func(request: Request) -> Deferred[Response]:- return succeed(resp)-- async with self.get_mwman() as mwman:- with pytest.warns(- ScrapyDeprecationWarning,- match=r"Passing a spider argument to DownloaderMiddlewareManager.download\(\)"- r" is deprecated and the passed value is ignored.",+ match=r"DownloaderMiddlewareManager.download\(\) is deprecated, use download_async\(\) instead",):ret = await maybe_deferred_to_future(mwman.download(download_func, req, mwman.crawler.spider)@@ -353,6 +340,6 @@ class TestDeprecatedSpiderArg(TestManagerBase):),):mwman._add_middleware(DeprecatedSpiderArgMiddleware())- result = await maybe_deferred_to_future(mwman.download(download_func, req))+ result = await mwman.download_async(download_func, req)assert result is respassert not download_func.called