Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Fix CI issues related to asyncio (#5782)

Adrián Chaves724b0332878b
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index ddacfaa49..0da2e3638 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -26,7 +26,7 @@ from twisted.python import failure
from twisted.python.failure import Failure
from scrapy.exceptions import IgnoreRequest
-from scrapy.utils.reactor import is_asyncio_reactor_installed, get_asyncio_event_loop_policy
+from scrapy.utils.reactor import is_asyncio_reactor_installed, _get_asyncio_event_loop
def defer_fail(_failure: Failure) -> Deferred:
@@ -267,7 +267,7 @@ def deferred_from_coro(o) -> Any:
# that use asyncio, e.g. "await asyncio.sleep(1)"
return ensureDeferred(o)
# wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor
- event_loop = get_asyncio_event_loop_policy().get_event_loop()
+ event_loop = _get_asyncio_event_loop()
return Deferred.fromFuture(asyncio.ensure_future(o, loop=event_loop))
return o
@@ -318,8 +318,7 @@ def deferred_to_future(d: Deferred) -> Future:
d = treq.get('https://example.com/additional')
additional_response = await deferred_to_future(d)
"""
- policy = get_asyncio_event_loop_policy()
- return d.asFuture(policy.get_event_loop())
+ return d.asFuture(_get_asyncio_event_loop())
def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]:
diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
index e6b8de292..6a051d8b0 100644
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -1,6 +1,7 @@
import asyncio
import sys
from contextlib import suppress
+from warnings import catch_warnings, filterwarnings
from twisted.internet import asyncioreactor, error
@@ -81,6 +82,10 @@ def install_reactor(reactor_path, event_loop_path=None):
installer()
+def _get_asyncio_event_loop():
+ return set_asyncio_event_loop(None)
+
+
def set_asyncio_event_loop(event_loop_path):
"""Sets and returns the event loop with specified import path."""
policy = get_asyncio_event_loop_policy()
@@ -90,11 +95,26 @@ def set_asyncio_event_loop(event_loop_path):
asyncio.set_event_loop(event_loop)
else:
try:
- event_loop = policy.get_event_loop()
+ with catch_warnings():
+ # In Python 3.10.9, 3.11.1, 3.12 and 3.13, a DeprecationWarning
+ # is emitted about the lack of a current event loop, because in
+ # Python 3.14 and later `get_event_loop` will raise a
+ # RuntimeError in that event. Because our code is already
+ # prepared for that future behavior, we ignore the deprecation
+ # warning.
+ filterwarnings(
+ "ignore",
+ message="There is no current event loop",
+ category=DeprecationWarning,
+ )
+ event_loop = policy.get_event_loop()
except RuntimeError:
- # `get_event_loop` is expected to fail when called from a new thread
- # with no asyncio event loop yet installed. Such is the case when
- # called from `scrapy shell`
+ # `get_event_loop` raises RuntimeError when called with no asyncio
+ # event loop yet installed in the following scenarios:
+ # - From a thread other than the main thread. For example, when
+ # using ``scrapy shell``.
+ # - Previsibly on Python 3.14 and later.
+ # https://github.com/python/cpython/issues/100160#issuecomment-1345581902
event_loop = policy.new_event_loop()
asyncio.set_event_loop(event_loop)
return event_loop
diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py
index 741c6a505..42780ace7 100644
--- a/tests/test_utils_asyncio.py
+++ b/tests/test_utils_asyncio.py
@@ -14,6 +14,9 @@ class AsyncioTest(TestCase):
self.assertEqual(is_asyncio_reactor_installed(), self.reactor_pytest == 'asyncio')
def test_install_asyncio_reactor(self):
+ from twisted.internet import reactor as original_reactor
with warnings.catch_warnings(record=True) as w:
install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
self.assertEqual(len(w), 0)
+ from twisted.internet import reactor
+ assert original_reactor == reactor