Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add deferred_from_coro.

Andrey Rakhmatullin50aa6ef22cc9
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 20ce59297..bbd5ebe52 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -1,10 +1,14 @@
"""
Helper functions for dealing with Twisted deferreds
"""
+import asyncio
+import inspect
+
from twisted.internet import defer, task
from twisted.python import failure
from scrapy.exceptions import IgnoreRequest
+from scrapy.utils.asyncio import is_asyncio_reactor_installed
def defer_fail(_failure):
@@ -114,3 +118,25 @@ def iter_errback(iterable, errback, *a, **kw):
break
except Exception:
errback(failure.Failure(), *a, **kw)
+
+
+def _isfuture(o):
+ # workaround for Python before 3.5.3 not having asyncio.isfuture
+ if hasattr(asyncio, 'isfuture'):
+ return asyncio.isfuture(o)
+ return isinstance(o, asyncio.Future)
+
+
+def deferred_from_coro(o):
+ """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine"""
+ if isinstance(o, defer.Deferred):
+ return o
+ if _isfuture(o) or inspect.isawaitable(o):
+ if not is_asyncio_reactor_installed():
+ # wrapping the coroutine directly into a Deferred, this doesn't work correctly with coroutines
+ # that use asyncio, e.g. "await asyncio.sleep(1)"
+ return defer.ensureDeferred(o)
+ else:
+ # wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor
+ return defer.Deferred.fromFuture(asyncio.ensure_future(o))
+ return o