Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Get the event loop from event_loop_policy to avoid a deprecation warning (#5689)

Godson3a34fa839938
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 8fcf31cab..38aefd6d0 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -26,7 +26,7 @@ from twisted.python import failure
from twisted.python.failure import Failure
from scrapy.exceptions import IgnoreRequest
-from scrapy.utils.reactor import is_asyncio_reactor_installed
+from scrapy.utils.reactor import is_asyncio_reactor_installed, get_asyncio_event_loop_policy
def defer_fail(_failure: Failure) -> Deferred:
@@ -269,7 +269,8 @@ def deferred_from_coro(o) -> Any:
return ensureDeferred(o)
else:
# wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor
- return Deferred.fromFuture(asyncio.ensure_future(o))
+ event_loop = get_asyncio_event_loop_policy().get_event_loop()
+ return Deferred.fromFuture(asyncio.ensure_future(o, loop=event_loop))
return o
@@ -320,7 +321,8 @@ def deferred_to_future(d: Deferred) -> Future:
d = treq.get('https://example.com/additional')
additional_response = await deferred_to_future(d)
"""
- return d.asFuture(asyncio.get_event_loop())
+ policy = get_asyncio_event_loop_policy()
+ return d.asFuture(policy.get_event_loop())
def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]:
diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
index 652733ce8..ddf354d88 100644
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -51,6 +51,19 @@ class CallLaterOnce:
return self._func(*self._a, **self._kw)
+def get_asyncio_event_loop_policy():
+ policy = asyncio.get_event_loop_policy()
+ if (
+ sys.version_info >= (3, 8)
+ and sys.platform == "win32"
+ and not isinstance(policy, asyncio.WindowsSelectorEventLoopPolicy)
+ ):
+ policy = asyncio.WindowsSelectorEventLoopPolicy()
+ asyncio.set_event_loop_policy(policy)
+
+ return policy
+
+
def install_reactor(reactor_path, event_loop_path=None):
"""Installs the :mod:`~twisted.internet.reactor` with the specified
import path. Also installs the asyncio event loop with the specified import
@@ -58,16 +71,14 @@ def install_reactor(reactor_path, event_loop_path=None):
reactor_class = load_object(reactor_path)
if reactor_class is asyncioreactor.AsyncioSelectorReactor:
with suppress(error.ReactorAlreadyInstalledError):
- if sys.version_info >= (3, 8) and sys.platform == "win32":
- policy = asyncio.get_event_loop_policy()
- if not isinstance(policy, asyncio.WindowsSelectorEventLoopPolicy):
- asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
+ policy = get_asyncio_event_loop_policy()
if event_loop_path is not None:
event_loop_class = load_object(event_loop_path)
event_loop = event_loop_class()
asyncio.set_event_loop(event_loop)
else:
- event_loop = asyncio.get_event_loop()
+ event_loop = policy.get_event_loop()
+
asyncioreactor.install(eventloop=event_loop)
else:
*module, _ = reactor_path.split(".")
diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py
index 295323e4d..741c6a505 100644
--- a/tests/test_utils_asyncio.py
+++ b/tests/test_utils_asyncio.py
@@ -1,3 +1,4 @@
+import warnings
from unittest import TestCase
from pytest import mark
@@ -13,5 +14,6 @@ class AsyncioTest(TestCase):
self.assertEqual(is_asyncio_reactor_installed(), self.reactor_pytest == 'asyncio')
def test_install_asyncio_reactor(self):
- # this should do nothing
- install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
+ with warnings.catch_warnings(record=True) as w:
+ install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
+ self.assertEqual(len(w), 0)