Scrapy, a fast high-level web crawling & scraping framework for Python.
Add async def support to signal handlers that already supported Deferreds.
Andrey Rakhmatullin3faef2d08277
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.pyindex bbd5ebe52..a2c24e5fb 100644--- a/scrapy/utils/defer.py+++ b/scrapy/utils/defer.py@@ -140,3 +140,20 @@ def deferred_from_coro(o):# wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactorreturn defer.Deferred.fromFuture(asyncio.ensure_future(o))return o+++def maybeDeferred_coro(f, *args, **kw):+ """ Copy of defer.maybeDeferred that also converts coroutines to Deferreds. """+ try:+ result = f(*args, **kw)+ except: # noqa: E722+ return defer.fail(failure.Failure(captureVars=defer.Deferred.debug))++ if isinstance(result, defer.Deferred):+ return result+ elif _isfuture(result) or inspect.isawaitable(result):+ return deferred_from_coro(result)+ elif isinstance(result, failure.Failure):+ return defer.fail(result)+ else:+ return defer.succeed(result)diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.pyindex de00bac49..60c561da6 100644--- a/scrapy/utils/signal.py+++ b/scrapy/utils/signal.py@@ -2,12 +2,14 @@import logging-from twisted.internet.defer import maybeDeferred, DeferredList, Deferred+from twisted.internet.defer import DeferredList, Deferredfrom twisted.python.failure import Failurefrom pydispatch.dispatcher import Any, Anonymous, liveReceivers, \getAllReceivers, disconnectfrom pydispatch.robustapply import robustApply++from scrapy.utils.defer import maybeDeferred_corofrom scrapy.utils.log import failure_to_exc_infologger = logging.getLogger(__name__)@@ -61,7 +63,7 @@ def send_catch_log_deferred(signal=Any, sender=Anonymous, *arguments, **named):spider = named.get('spider', None)dfds = []for receiver in liveReceivers(getAllReceivers(sender, signal)):- d = maybeDeferred(robustApply, receiver, signal=signal, sender=sender,+ d = maybeDeferred_coro(robustApply, receiver, signal=signal, sender=sender,*arguments, **named)d.addErrback(logerror, receiver)d.addBoth(lambda result: (receiver, result))diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.pyindex 16b7c5c68..e5f6f0ed4 100644--- a/tests/test_utils_signal.py+++ b/tests/test_utils_signal.py@@ -1,3 +1,6 @@+import asyncio++from pytest import markfrom testfixtures import LogCapturefrom twisted.trial import unittestfrom twisted.python.failure import Failure@@ -5,6 +8,7 @@ from twisted.internet import defer, reactorfrom pydispatch import dispatcherfrom scrapy.utils.signal import send_catch_log, send_catch_log_deferred+from scrapy.utils.test import get_from_asyncio_queueclass SendCatchLogTest(unittest.TestCase):@@ -54,7 +58,7 @@ class SendCatchLogDeferredTest(SendCatchLogTest):return send_catch_log_deferred(signal, *a, **kw)-class SendCatchLogDeferredTest2(SendCatchLogTest):+class SendCatchLogDeferredTest2(SendCatchLogDeferredTest):def ok_handler(self, arg, handlers_called):handlers_called.add(self.ok_handler)@@ -63,8 +67,24 @@ class SendCatchLogDeferredTest2(SendCatchLogTest):reactor.callLater(0, d.callback, "OK")return d- def _get_result(self, signal, *a, **kw):- return send_catch_log_deferred(signal, *a, **kw)++class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest):++ async def ok_handler(self, arg, handlers_called):+ handlers_called.add(self.ok_handler)+ assert arg == 'test'+ await defer.succeed(42)+ return "OK"++[email protected]_asyncio()+class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest):++ async def ok_handler(self, arg, handlers_called):+ handlers_called.add(self.ok_handler)+ assert arg == 'test'+ await asyncio.sleep(0.2)+ return await get_from_asyncio_queue("OK")class SendCatchLogTest2(unittest.TestCase):