Scrapy, a fast high-level web crawling & scraping framework for Python.
Fix CI issues related to asyncio (#5782)
Adrián Chaves724b0332878b
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.pyindex ddacfaa49..0da2e3638 100644--- a/scrapy/utils/defer.py+++ b/scrapy/utils/defer.py@@ -26,7 +26,7 @@ from twisted.python import failurefrom twisted.python.failure import Failurefrom scrapy.exceptions import IgnoreRequest-from scrapy.utils.reactor import is_asyncio_reactor_installed, get_asyncio_event_loop_policy+from scrapy.utils.reactor import is_asyncio_reactor_installed, _get_asyncio_event_loopdef defer_fail(_failure: Failure) -> Deferred:@@ -267,7 +267,7 @@ def deferred_from_coro(o) -> Any:# that use asyncio, e.g. "await asyncio.sleep(1)"return ensureDeferred(o)# wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor- event_loop = get_asyncio_event_loop_policy().get_event_loop()+ event_loop = _get_asyncio_event_loop()return Deferred.fromFuture(asyncio.ensure_future(o, loop=event_loop))return o@@ -318,8 +318,7 @@ def deferred_to_future(d: Deferred) -> Future:d = treq.get('https://example.com/additional')additional_response = await deferred_to_future(d)"""- policy = get_asyncio_event_loop_policy()- return d.asFuture(policy.get_event_loop())+ return d.asFuture(_get_asyncio_event_loop())def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]:diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.pyindex e6b8de292..6a051d8b0 100644--- a/scrapy/utils/reactor.py+++ b/scrapy/utils/reactor.py@@ -1,6 +1,7 @@import asyncioimport sysfrom contextlib import suppress+from warnings import catch_warnings, filterwarningsfrom twisted.internet import asyncioreactor, error@@ -81,6 +82,10 @@ def install_reactor(reactor_path, event_loop_path=None):installer()+def _get_asyncio_event_loop():+ return set_asyncio_event_loop(None)++def set_asyncio_event_loop(event_loop_path):"""Sets and returns the event loop with specified import path."""policy = get_asyncio_event_loop_policy()@@ -90,11 +95,26 @@ def set_asyncio_event_loop(event_loop_path):asyncio.set_event_loop(event_loop)else:try:- event_loop = policy.get_event_loop()+ with catch_warnings():+ # In Python 3.10.9, 3.11.1, 3.12 and 3.13, a DeprecationWarning+ # is emitted about the lack of a current event loop, because in+ # Python 3.14 and later `get_event_loop` will raise a+ # RuntimeError in that event. Because our code is already+ # prepared for that future behavior, we ignore the deprecation+ # warning.+ filterwarnings(+ "ignore",+ message="There is no current event loop",+ category=DeprecationWarning,+ )+ event_loop = policy.get_event_loop()except RuntimeError:- # `get_event_loop` is expected to fail when called from a new thread- # with no asyncio event loop yet installed. Such is the case when- # called from `scrapy shell`+ # `get_event_loop` raises RuntimeError when called with no asyncio+ # event loop yet installed in the following scenarios:+ # - From a thread other than the main thread. For example, when+ # using ``scrapy shell``.+ # - Previsibly on Python 3.14 and later.+ # https://github.com/python/cpython/issues/100160#issuecomment-1345581902event_loop = policy.new_event_loop()asyncio.set_event_loop(event_loop)return event_loopdiff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.pyindex 741c6a505..42780ace7 100644--- a/tests/test_utils_asyncio.py+++ b/tests/test_utils_asyncio.py@@ -14,6 +14,9 @@ class AsyncioTest(TestCase):self.assertEqual(is_asyncio_reactor_installed(), self.reactor_pytest == 'asyncio')def test_install_asyncio_reactor(self):+ from twisted.internet import reactor as original_reactorwith warnings.catch_warnings(record=True) as w:install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")self.assertEqual(len(w), 0)+ from twisted.internet import reactor+ assert original_reactor == reactor