Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add async def support to signal handlers that already supported Deferreds.

Andrey Rakhmatullin3faef2d08277
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index bbd5ebe52..a2c24e5fb 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -140,3 +140,20 @@ def deferred_from_coro(o):
# wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor
return defer.Deferred.fromFuture(asyncio.ensure_future(o))
return o
+
+
+def maybeDeferred_coro(f, *args, **kw):
+ """ Copy of defer.maybeDeferred that also converts coroutines to Deferreds. """
+ try:
+ result = f(*args, **kw)
+ except: # noqa: E722
+ return defer.fail(failure.Failure(captureVars=defer.Deferred.debug))
+
+ if isinstance(result, defer.Deferred):
+ return result
+ elif _isfuture(result) or inspect.isawaitable(result):
+ return deferred_from_coro(result)
+ elif isinstance(result, failure.Failure):
+ return defer.fail(result)
+ else:
+ return defer.succeed(result)
diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py
index de00bac49..60c561da6 100644
--- a/scrapy/utils/signal.py
+++ b/scrapy/utils/signal.py
@@ -2,12 +2,14 @@
import logging
-from twisted.internet.defer import maybeDeferred, DeferredList, Deferred
+from twisted.internet.defer import DeferredList, Deferred
from twisted.python.failure import Failure
from pydispatch.dispatcher import Any, Anonymous, liveReceivers, \
getAllReceivers, disconnect
from pydispatch.robustapply import robustApply
+
+from scrapy.utils.defer import maybeDeferred_coro
from scrapy.utils.log import failure_to_exc_info
logger = logging.getLogger(__name__)
@@ -61,7 +63,7 @@ def send_catch_log_deferred(signal=Any, sender=Anonymous, *arguments, **named):
spider = named.get('spider', None)
dfds = []
for receiver in liveReceivers(getAllReceivers(sender, signal)):
- d = maybeDeferred(robustApply, receiver, signal=signal, sender=sender,
+ d = maybeDeferred_coro(robustApply, receiver, signal=signal, sender=sender,
*arguments, **named)
d.addErrback(logerror, receiver)
d.addBoth(lambda result: (receiver, result))
diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py
index 16b7c5c68..e5f6f0ed4 100644
--- a/tests/test_utils_signal.py
+++ b/tests/test_utils_signal.py
@@ -1,3 +1,6 @@
+import asyncio
+
+from pytest import mark
from testfixtures import LogCapture
from twisted.trial import unittest
from twisted.python.failure import Failure
@@ -5,6 +8,7 @@ from twisted.internet import defer, reactor
from pydispatch import dispatcher
from scrapy.utils.signal import send_catch_log, send_catch_log_deferred
+from scrapy.utils.test import get_from_asyncio_queue
class SendCatchLogTest(unittest.TestCase):
@@ -54,7 +58,7 @@ class SendCatchLogDeferredTest(SendCatchLogTest):
return send_catch_log_deferred(signal, *a, **kw)
-class SendCatchLogDeferredTest2(SendCatchLogTest):
+class SendCatchLogDeferredTest2(SendCatchLogDeferredTest):
def ok_handler(self, arg, handlers_called):
handlers_called.add(self.ok_handler)
@@ -63,8 +67,24 @@ class SendCatchLogDeferredTest2(SendCatchLogTest):
reactor.callLater(0, d.callback, "OK")
return d
- def _get_result(self, signal, *a, **kw):
- return send_catch_log_deferred(signal, *a, **kw)
+
+class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest):
+
+ async def ok_handler(self, arg, handlers_called):
+ handlers_called.add(self.ok_handler)
+ assert arg == 'test'
+ await defer.succeed(42)
+ return "OK"
+
+
+class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest):
+
+ async def ok_handler(self, arg, handlers_called):
+ handlers_called.add(self.ok_handler)
+ assert arg == 'test'
+ await asyncio.sleep(0.2)
+ return await get_from_asyncio_queue("OK")
class SendCatchLogTest2(unittest.TestCase):