Scrapy, a fast high-level web crawling & scraping framework for Python.
Deprecate maybeDeferred_coro(). (#7212)
Andrey Rakhmatullin1af283387fda
diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.pyindex 21ad6fb1a..14d53f1c5 100644--- a/scrapy/pipelines/__init__.py+++ b/scrapy/pipelines/__init__.py@@ -16,7 +16,7 @@ from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.middleware import MiddlewareManagerfrom scrapy.utils.asyncio import is_asyncio_availablefrom scrapy.utils.conf import build_component_list-from scrapy.utils.defer import deferred_from_coro, ensure_awaitable, maybeDeferred_coro+from scrapy.utils.defer import _maybeDeferred_coro, deferred_from_coro, ensure_awaitablefrom scrapy.utils.python import global_object_nameif TYPE_CHECKING:@@ -70,8 +70,8 @@ class ItemPipelineManager(MiddlewareManager):method: Callable[..., Coroutine[Any, Any, None] | Deferred[None] | None],) -> Deferred[None]:if method in self._mw_methods_requiring_spider:- return maybeDeferred_coro(method, self._spider)- return maybeDeferred_coro(method)+ return _maybeDeferred_coro(method, True, self._spider)+ return _maybeDeferred_coro(method, True)dfds = [get_dfd(m) for m in methods]d: Deferred[list[tuple[bool, None]]] = DeferredList(diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.pyindex af7c50d5c..218dc8d7a 100644--- a/scrapy/utils/defer.py+++ b/scrapy/utils/defer.py@@ -419,6 +419,18 @@ def deferred_f_from_coro_f(def maybeDeferred_coro(f: Callable[_P, Any], *args: _P.args, **kw: _P.kwargs+) -> Deferred[Any]: # pragma: no cover+ """Copy of defer.maybeDeferred that also converts coroutines to Deferreds."""+ warnings.warn(+ "maybeDeferred_coro() is deprecated and will be removed in a future Scrapy version.",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )+ return _maybeDeferred_coro(f, False, *args, **kw)+++def _maybeDeferred_coro(+ f: Callable[_P, Any], warn: bool, *args: _P.args, **kw: _P.kwargs) -> Deferred[Any]:"""Copy of defer.maybeDeferred that also converts coroutines to Deferreds."""try:@@ -426,17 +438,28 @@ def maybeDeferred_coro(except: # noqa: E722 # pylint: disable=bare-exceptreturn fail(failure.Failure(captureVars=Deferred.debug))+ # when the deprecation period has ended we need to make sure the behavior+ # of the public maybeDeferred_coro() function isn't changed, or drop it in+ # the same releaseif isinstance(result, Deferred):- warnings.warn(- f"{global_object_name(f)} returned a Deferred, this is deprecated."- f" Please refactor this function to return a coroutine.",- ScrapyDeprecationWarning,- stacklevel=2,- )+ if warn:+ warnings.warn(+ f"{global_object_name(f)} returned a Deferred, this is deprecated."+ f" Please refactor this function to return a coroutine.",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )return resultif asyncio.isfuture(result) or inspect.isawaitable(result):return deferred_from_coro(result)if isinstance(result, failure.Failure):+ if warn:+ warnings.warn(+ f"{global_object_name(f)} returned a Failure, this is deprecated."+ f" Please refactor this function to return a coroutine.",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )return fail(result)return succeed(result)diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.pyindex a06936544..1f7426e59 100644--- a/scrapy/utils/signal.py+++ b/scrapy/utils/signal.py@@ -22,9 +22,9 @@ from twisted.python.failure import Failurefrom scrapy.exceptions import ScrapyDeprecationWarning, StopDownloadfrom scrapy.utils.asyncio import is_asyncio_availablefrom scrapy.utils.defer import (+ _maybeDeferred_coro,ensure_awaitable,maybe_deferred_to_future,- maybeDeferred_coro,)from scrapy.utils.log import failure_to_exc_infofrom scrapy.utils.python import global_object_name@@ -114,8 +114,14 @@ def _send_catch_log_deferred(spider = named.get("spider")dfds: list[Deferred[tuple[TypingAny, TypingAny]]] = []for receiver in liveReceivers(getAllReceivers(sender, signal)):- d: Deferred[TypingAny] = maybeDeferred_coro(- robustApply, receiver, signal=signal, sender=sender, *arguments, **named+ d: Deferred[TypingAny] = _maybeDeferred_coro(+ robustApply,+ True,+ receiver,+ signal=signal,+ sender=sender,+ *arguments,+ **named,)d.addErrback(logerror, receiver)# TODO https://pylint.readthedocs.io/en/latest/user_guide/messages/warning/cell-var-from-loop.html