Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Deprecate maybeDeferred_coro(). (#7212)

Andrey Rakhmatullin1af283387fda
diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py
index 21ad6fb1a..14d53f1c5 100644
--- a/scrapy/pipelines/__init__.py
+++ b/scrapy/pipelines/__init__.py
@@ -16,7 +16,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.middleware import MiddlewareManager
from scrapy.utils.asyncio import is_asyncio_available
from scrapy.utils.conf import build_component_list
-from scrapy.utils.defer import deferred_from_coro, ensure_awaitable, maybeDeferred_coro
+from scrapy.utils.defer import _maybeDeferred_coro, deferred_from_coro, ensure_awaitable
from scrapy.utils.python import global_object_name
if TYPE_CHECKING:
@@ -70,8 +70,8 @@ class ItemPipelineManager(MiddlewareManager):
method: Callable[..., Coroutine[Any, Any, None] | Deferred[None] | None],
) -> Deferred[None]:
if method in self._mw_methods_requiring_spider:
- return maybeDeferred_coro(method, self._spider)
- return maybeDeferred_coro(method)
+ return _maybeDeferred_coro(method, True, self._spider)
+ return _maybeDeferred_coro(method, True)
dfds = [get_dfd(m) for m in methods]
d: Deferred[list[tuple[bool, None]]] = DeferredList(
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index af7c50d5c..218dc8d7a 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -419,6 +419,18 @@ def deferred_f_from_coro_f(
def maybeDeferred_coro(
f: Callable[_P, Any], *args: _P.args, **kw: _P.kwargs
+) -> Deferred[Any]: # pragma: no cover
+ """Copy of defer.maybeDeferred that also converts coroutines to Deferreds."""
+ warnings.warn(
+ "maybeDeferred_coro() is deprecated and will be removed in a future Scrapy version.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
+ return _maybeDeferred_coro(f, False, *args, **kw)
+
+
+def _maybeDeferred_coro(
+ f: Callable[_P, Any], warn: bool, *args: _P.args, **kw: _P.kwargs
) -> Deferred[Any]:
"""Copy of defer.maybeDeferred that also converts coroutines to Deferreds."""
try:
@@ -426,17 +438,28 @@ def maybeDeferred_coro(
except: # noqa: E722 # pylint: disable=bare-except
return fail(failure.Failure(captureVars=Deferred.debug))
+ # when the deprecation period has ended we need to make sure the behavior
+ # of the public maybeDeferred_coro() function isn't changed, or drop it in
+ # the same release
if isinstance(result, Deferred):
- warnings.warn(
- f"{global_object_name(f)} returned a Deferred, this is deprecated."
- f" Please refactor this function to return a coroutine.",
- ScrapyDeprecationWarning,
- stacklevel=2,
- )
+ if warn:
+ warnings.warn(
+ f"{global_object_name(f)} returned a Deferred, this is deprecated."
+ f" Please refactor this function to return a coroutine.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
return result
if asyncio.isfuture(result) or inspect.isawaitable(result):
return deferred_from_coro(result)
if isinstance(result, failure.Failure):
+ if warn:
+ warnings.warn(
+ f"{global_object_name(f)} returned a Failure, this is deprecated."
+ f" Please refactor this function to return a coroutine.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
return fail(result)
return succeed(result)
diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py
index a06936544..1f7426e59 100644
--- a/scrapy/utils/signal.py
+++ b/scrapy/utils/signal.py
@@ -22,9 +22,9 @@ from twisted.python.failure import Failure
from scrapy.exceptions import ScrapyDeprecationWarning, StopDownload
from scrapy.utils.asyncio import is_asyncio_available
from scrapy.utils.defer import (
+ _maybeDeferred_coro,
ensure_awaitable,
maybe_deferred_to_future,
- maybeDeferred_coro,
)
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.python import global_object_name
@@ -114,8 +114,14 @@ def _send_catch_log_deferred(
spider = named.get("spider")
dfds: list[Deferred[tuple[TypingAny, TypingAny]]] = []
for receiver in liveReceivers(getAllReceivers(sender, signal)):
- d: Deferred[TypingAny] = maybeDeferred_coro(
- robustApply, receiver, signal=signal, sender=sender, *arguments, **named
+ d: Deferred[TypingAny] = _maybeDeferred_coro(
+ robustApply,
+ True,
+ receiver,
+ signal=signal,
+ sender=sender,
+ *arguments,
+ **named,
)
d.addErrback(logerror, receiver)
# TODO https://pylint.readthedocs.io/en/latest/user_guide/messages/warning/cell-var-from-loop.html