Scrapy, a fast high-level web crawling & scraping framework for Python.
Deprecate `is_asyncio_reactor_installed()` (#8128)
Andrey Rakhmatullina527168ad088
diff --git a/docs/news.rst b/docs/news.rstindex b50878f29..7482f7358 100644--- a/docs/news.rst+++ b/docs/news.rst@@ -2765,7 +2765,7 @@ New features(:gh:`4463`, :gh:`6804`)- Added :func:`scrapy.utils.asyncio.is_asyncio_available` as an alternative- to :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` with a+ to ``scrapy.utils.reactor.is_asyncio_reactor_installed`` with afuture-proof name and semantics.(:gh:`6827`)@@ -3012,7 +3012,7 @@ Scrapy 2.13.2 (2025-06-09)(:gh:`6861`, :gh:`6863`)- Added a note about the behavior change of- :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` to its docs and+ ``scrapy.utils.reactor.is_asyncio_reactor_installed`` to its docs andto the "Backward-incompatible changes" section of :ref:`the Scrapy 2.13.0release notes <release-2.13.0>`.(:gh:`6866`)@@ -3178,7 +3178,7 @@ Backward-incompatible changesalso enforced for start requests.(:gh:`6777`)-- Calling :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` without+- Calling ``scrapy.utils.reactor.is_asyncio_reactor_installed`` withoutan installed reactor now raises an exception instead of installing areactor. This shouldn't affect normal Scrapy use cases, but it may affect3rd-party test suites that use Scrapy internals such as@@ -3488,7 +3488,7 @@ Bug fixes.. _scrapy-poet: https://github.com/scrapinghub/scrapy-poet-- Calling :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` without+- Calling ``scrapy.utils.reactor.is_asyncio_reactor_installed`` withoutan installed reactor now raises an exception instead of installing areactor.(:gh:`6732`, :gh:`6735`)diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rstindex 21f4f1ba2..b3cd35f10 100644--- a/docs/topics/asyncio.rst+++ b/docs/topics/asyncio.rst@@ -137,7 +137,6 @@ example:).. autofunction:: scrapy.utils.asyncio.is_asyncio_available-.. autofunction:: scrapy.utils.reactor.is_asyncio_reactor_installed.. _asyncio-without-reactor:@@ -259,9 +258,6 @@ Here are some examples of APIs and patterns that need a replacement::class:`~scrapy.crawler.CrawlerRunner`. You should use:class:`~scrapy.crawler.AsyncCrawlerProcess` or:class:`~scrapy.crawler.AsyncCrawlerRunner` respectively instead.-* Checking whether ``asyncio`` support is available with- :func:`scrapy.utils.reactor.is_asyncio_reactor_installed`. You should use- :func:`scrapy.utils.asyncio.is_asyncio_available` instead.Scrapy provides unified helpers for some of these examples:diff --git a/scrapy/crawler.py b/scrapy/crawler.pyindex 3da43d88f..dd1593118 100644--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@ -32,8 +32,8 @@ from scrapy.utils.misc import build_from_crawler, load_objectfrom scrapy.utils.ossignal import install_shutdown_handlers, signal_namesfrom scrapy.utils.reactor import (_asyncio_reactor_path,+ _is_asyncio_reactor_installed,install_reactor,- is_asyncio_reactor_installed,is_reactor_installed,set_asyncio_event_loop,verify_installed_asyncio_event_loop,@@ -204,7 +204,7 @@ class Crawler:if reactor_class:# We need to check that the correct reactor is installed.verify_installed_reactor(reactor_class)- if is_asyncio_reactor_installed() and event_loop:+ if _is_asyncio_reactor_installed() and event_loop:verify_installed_asyncio_event_loop(event_loop)if self._init_reactor or reactor_class:@@ -666,7 +666,7 @@ class AsyncCrawlerRunner(CrawlerRunnerBase):raise RuntimeError("We expected a Twisted reactor to be installed but it isn't.")- if not is_asyncio_reactor_installed():+ if not _is_asyncio_reactor_installed():raise RuntimeError(f"When TWISTED_REACTOR_ENABLED is True, {type(self).__name__} "f"requires that the installed Twisted reactor is "diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.pyindex 577e1df19..da88883e1 100644--- a/scrapy/utils/asyncio.py+++ b/scrapy/utils/asyncio.py@@ -13,7 +13,7 @@ from twisted.internet.task import LoopingCall, deferLaterfrom twisted.internet.threads import deferToThreadfrom scrapy.utils.asyncgen import as_async_generator-from scrapy.utils.reactor import is_asyncio_reactor_installed, is_reactor_installed+from scrapy.utils.reactor import _is_asyncio_reactor_installed, is_reactor_installedif TYPE_CHECKING:from twisted.internet.base import DelayedCall@@ -31,6 +31,20 @@ _P = ParamSpec("_P")logger = logging.getLogger(__name__)+def _has_running_loop() -> bool:+ """Check if there is a running asyncio event loop in the current thread.++ Can't easily check for an installed but not running one, and if we+ checked that there could be false positives due to some 3rd-party code+ installing it as a side effect (e.g. by calling get_event_loop()).+ """+ try:+ asyncio.get_running_loop()+ except RuntimeError:+ return False+ return True++def is_asyncio_available() -> bool:"""Check if it's possible to call asyncio code that relies on the asyncio event loop.@@ -45,8 +59,8 @@ def is_asyncio_available() -> bool:Code that doesn't directly require a Twisted reactor should use thisfunction while code that requires- :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` should use- :func:`~scrapy.utils.reactor.is_asyncio_reactor_installed`.+ :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` should+ also use :func:`~scrapy.utils.reactor.is_reactor_installed`.When this returns ``True``, an asyncio loop is installed and used byScrapy. It's possible to call functions that require it, such as@@ -70,15 +84,7 @@ def is_asyncio_available() -> bool:loop, even if no Twisted reactor is installed."""- # Check if there is a running asyncio loop.- # Can't easily check for an installed but not running one, and if we- # checked that there could be false positives due to some 3rd-party code- # installing it as a side effect (e.g. by calling get_event_loop()).- try:- asyncio.get_running_loop()- except RuntimeError:- pass- else:+ if _has_running_loop():return True# Check if there is an installed asyncio reactor (it doesn't need to be@@ -89,7 +95,7 @@ def is_asyncio_available() -> bool:" or running asyncio loop.")- return is_asyncio_reactor_installed()+ return _is_asyncio_reactor_installed()class _QueueEnd:diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.pyindex 4353066ee..8354f9878 100644--- a/scrapy/utils/reactor.py+++ b/scrapy/utils/reactor.py@@ -10,6 +10,7 @@ from warnings import catch_warnings, filterwarningsfrom twisted.internet import asyncioreactor, errorfrom twisted.internet.defer import Deferred+from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.utils.misc import load_objectfrom scrapy.utils.python import global_object_name@@ -224,27 +225,34 @@ def is_reactor_installed() -> bool:return "twisted.internet.reactor" in sys.modules-def is_asyncio_reactor_installed() -> bool:+def _is_asyncio_reactor_installed() -> bool:"""Check whether the installed reactor is :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`.Raise a :exc:`RuntimeError` if no reactor is installed.-- In a future Scrapy version, when Scrapy supports running without a Twisted- reactor, this function won't be useful for checking if it's possible to use- asyncio features, so the code that that doesn't directly require a Twisted- reactor should use :func:`scrapy.utils.asyncio.is_asyncio_available`- instead of this function.-- .. versionchanged:: 2.13- In earlier Scrapy versions this function silently installed the default- reactor if there was no reactor installed. Now it raises an exception to- prevent silent problems in this case."""if not is_reactor_installed():raise RuntimeError(- "is_asyncio_reactor_installed() called without an installed reactor."+ "_is_asyncio_reactor_installed() called without an installed reactor.")from twisted.internet import reactorreturn isinstance(reactor, asyncioreactor.AsyncioSelectorReactor)+++def is_asyncio_reactor_installed() -> bool:+ """Check whether the installed reactor is :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`.++ Raise a :exc:`RuntimeError` if no reactor is installed.+ """+ warnings.warn(+ "is_asyncio_reactor_installed() is deprecated."+ " If you need to check for asyncio support, use is_asyncio_available()."+ " If you need to check whether a reactor is installed or not,"+ " use is_reactor_installed() or is_reactorless()."+ " If you need to check that specifically the asyncio reactor (or the non-asyncio one)"+ " is installed, use is_reactor_installed() and is_asyncio_available().",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )+ return _is_asyncio_reactor_installed()diff --git a/scrapy/utils/reactorless.py b/scrapy/utils/reactorless.pyindex 243d80821..5fc8d2beb 100644--- a/scrapy/utils/reactorless.py+++ b/scrapy/utils/reactorless.py@@ -5,7 +5,7 @@ import sysfrom importlib.abc import MetaPathFinderfrom typing import TYPE_CHECKING-from scrapy.utils.asyncio import is_asyncio_available+from scrapy.utils.asyncio import _has_running_loopfrom scrapy.utils.reactor import is_reactor_installedif TYPE_CHECKING:@@ -22,13 +22,21 @@ def is_reactorless() -> bool:wrong when executed very early, before the reactor and/or the asyncio eventloop are initialized.- .. note:: As this function uses- :func:`scrapy.utils.asyncio.is_asyncio_available()`, it has the same- limitations for detecting a running asyncio event loop as that one.+ .. note:: As this function uses :func:`asyncio.get_running_loop()`, it will+ only detect the event loop if called in the same thread and from the+ code that runs inside that loop (this shouldn't be a problem when+ calling it from code such as spiders and Scrapy components, if Scrapy+ is run using one of the supported ways)... versionadded:: 2.15.0"""- return is_asyncio_available() and not is_reactor_installed()+ if is_reactor_installed():+ return False+ if _has_running_loop():+ return True+ raise RuntimeError(+ "is_reactorless() called without an installed reactor or running asyncio loop."+ )class ReactorImportHook(MetaPathFinder):diff --git a/scrapy/utils/test.py b/scrapy/utils/test.pyindex f9e4d3989..d3b69e3ed 100644--- a/scrapy/utils/test.py+++ b/scrapy/utils/test.py@@ -15,7 +15,7 @@ from twisted.web.client import Agentfrom scrapy.crawler import AsyncCrawlerRunner, CrawlerRunner, CrawlerRunnerBasefrom scrapy.exceptions import ScrapyDeprecationWarning-from scrapy.utils.reactor import is_asyncio_reactor_installed, is_reactor_installed+from scrapy.utils.reactor import _is_asyncio_reactor_installed, is_reactor_installedfrom scrapy.utils.spider import DefaultSpiderif TYPE_CHECKING:@@ -40,7 +40,7 @@ def get_reactor_settings() -> dict[str, Any]:"""settings: dict[str, Any] = {}if is_reactor_installed():- if not is_asyncio_reactor_installed():+ if not _is_asyncio_reactor_installed():settings["TWISTED_REACTOR"] = Noneelse:# We are either running Scrapy tests for the reactorless mode, ordiff --git a/tests/AsyncCrawlerProcess/asyncio_enabled_no_reactor.py b/tests/AsyncCrawlerProcess/asyncio_enabled_no_reactor.pyindex 3c47eb826..0b7ca1ecd 100644--- a/tests/AsyncCrawlerProcess/asyncio_enabled_no_reactor.py+++ b/tests/AsyncCrawlerProcess/asyncio_enabled_no_reactor.py@@ -1,11 +1,11 @@import scrapyfrom scrapy.crawler import AsyncCrawlerProcess-from scrapy.utils.reactor import is_asyncio_reactor_installed+from scrapy.utils.reactor import _is_asyncio_reactor_installedclass ReactorCheckExtension:def __init__(self):- if not is_asyncio_reactor_installed():+ if not _is_asyncio_reactor_installed():raise RuntimeError("ReactorCheckExtension requires the asyncio reactor.")diff --git a/tests/AsyncCrawlerProcess/asyncio_enabled_reactor.py b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor.pyindex aad439759..920792c59 100644--- a/tests/AsyncCrawlerProcess/asyncio_enabled_reactor.py+++ b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor.py@@ -1,37 +1,37 @@import scrapyfrom scrapy.crawler import AsyncCrawlerProcessfrom scrapy.utils.reactor import (+ _is_asyncio_reactor_installed,install_reactor,- is_asyncio_reactor_installed,is_reactor_installed,)if is_reactor_installed():raise RuntimeError(- "Reactor already installed before is_asyncio_reactor_installed()."+ "Reactor already installed before _is_asyncio_reactor_installed().")try:- is_asyncio_reactor_installed()+ _is_asyncio_reactor_installed()except RuntimeError:passelse:- raise RuntimeError("is_asyncio_reactor_installed() did not raise RuntimeError.")+ raise RuntimeError("_is_asyncio_reactor_installed() did not raise RuntimeError.")if is_reactor_installed():raise RuntimeError(- "Reactor already installed after is_asyncio_reactor_installed()."+ "Reactor already installed after _is_asyncio_reactor_installed().")install_reactor()-if not is_asyncio_reactor_installed():+if not _is_asyncio_reactor_installed():raise RuntimeError("Wrong reactor installed after install_reactor().")class ReactorCheckExtension:def __init__(self):- if not is_asyncio_reactor_installed():+ if not _is_asyncio_reactor_installed():raise RuntimeError("ReactorCheckExtension requires the asyncio reactor.")diff --git a/tests/CrawlerProcess/asyncio_enabled_no_reactor.py b/tests/CrawlerProcess/asyncio_enabled_no_reactor.pyindex 6bb6fb3c6..7d55c5cbe 100644--- a/tests/CrawlerProcess/asyncio_enabled_no_reactor.py+++ b/tests/CrawlerProcess/asyncio_enabled_no_reactor.py@@ -1,11 +1,11 @@import scrapyfrom scrapy.crawler import CrawlerProcess-from scrapy.utils.reactor import is_asyncio_reactor_installed+from scrapy.utils.reactor import _is_asyncio_reactor_installedclass ReactorCheckExtension:def __init__(self):- if not is_asyncio_reactor_installed():+ if not _is_asyncio_reactor_installed():raise RuntimeError("ReactorCheckExtension requires the asyncio reactor.")diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.pyindex 7cfc9f1d2..d3c1e86a9 100644--- a/tests/CrawlerProcess/asyncio_enabled_reactor.py+++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py@@ -2,22 +2,22 @@ import scrapyfrom scrapy.crawler import CrawlerProcessfrom scrapy.utils.asyncio import is_asyncio_availablefrom scrapy.utils.reactor import (+ _is_asyncio_reactor_installed,install_reactor,- is_asyncio_reactor_installed,is_reactor_installed,)if is_reactor_installed():raise RuntimeError(- "Reactor already installed before is_asyncio_reactor_installed()."+ "Reactor already installed before _is_asyncio_reactor_installed().")try:- is_asyncio_reactor_installed()+ _is_asyncio_reactor_installed()except RuntimeError:passelse:- raise RuntimeError("is_asyncio_reactor_installed() did not raise RuntimeError.")+ raise RuntimeError("_is_asyncio_reactor_installed() did not raise RuntimeError.")try:is_asyncio_available()@@ -28,18 +28,18 @@ else:if is_reactor_installed():raise RuntimeError(- "Reactor already installed after is_asyncio_reactor_installed()."+ "Reactor already installed after _is_asyncio_reactor_installed().")install_reactor()-if not is_asyncio_reactor_installed():+if not _is_asyncio_reactor_installed():raise RuntimeError("Wrong reactor installed after install_reactor().")class ReactorCheckExtension:def __init__(self):- if not is_asyncio_reactor_installed():+ if not _is_asyncio_reactor_installed():raise RuntimeError("ReactorCheckExtension requires the asyncio reactor.")if not is_asyncio_available():raise RuntimeError("ReactorCheckExtension requires asyncio support.")diff --git a/tests/test_utils_reactor.py b/tests/test_utils_reactor.pyindex 0493bfe6e..578b11d57 100644--- a/tests/test_utils_reactor.py+++ b/tests/test_utils_reactor.py@@ -8,8 +8,10 @@ import pytestfrom twisted.internet.error import CannotListenErrorfrom twisted.internet.protocol import ServerFactory+from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.utils.reactor import (_asyncio_reactor_path,+ _is_asyncio_reactor_installed,install_reactor,is_asyncio_reactor_installed,listen_tcp,@@ -28,8 +30,15 @@ if TYPE_CHECKING:class TestAsyncio:@pytest.mark.requires_reactor # needs a reactordef test_is_asyncio_reactor_installed(self, reactor_pytest: str) -> None:- # the result should depend only on the pytest --reactor argument- assert is_asyncio_reactor_installed() == (reactor_pytest == "asyncio")+ assert _is_asyncio_reactor_installed() == (reactor_pytest == "asyncio")++ @pytest.mark.requires_reactor # needs a reactor+ def test_is_asyncio_reactor_installed_deprecated(self, reactor_pytest: str) -> None:+ with pytest.warns(+ ScrapyDeprecationWarning,+ match=r"is_asyncio_reactor_installed\(\) is deprecated",+ ):+ assert is_asyncio_reactor_installed() == (reactor_pytest == "asyncio")@pytest.mark.requires_reactor # installs a reactor@pytest.mark.only_asynciodiff --git a/tests/test_utils_reactorless.py b/tests/test_utils_reactorless.pyindex 2a575a8b3..0423ce36d 100644--- a/tests/test_utils_reactorless.py+++ b/tests/test_utils_reactorless.py@@ -1,5 +1,6 @@from __future__ import annotations+import asyncioimport sysimport pytest@@ -7,8 +8,14 @@ import pytestfrom scrapy.utils.reactorless import (ReactorImportHook,install_reactor_import_hook,+ is_reactorless,uninstall_reactor_import_hook,)+from tests.utils.decorators import coroutine_test+++def _no_running_loop() -> asyncio.AbstractEventLoop:+ raise RuntimeError("no running event loop")class TestReactorImportHook:@@ -48,3 +55,22 @@ class TestReactorImportHook:):hook.find_spec("twisted.internet.reactor", None)assert hook.find_spec("twisted.internet.defer", None) is None+++class TestIsReactorless:+ @coroutine_test+ async def test_is_asyncio_reactor_installed(self, reactor_pytest: str) -> None:+ assert is_reactorless() == (reactor_pytest == "none")++ @pytest.mark.requires_reactor # needs a reactor+ def test_reactor_installed_no_running_loop(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ monkeypatch.setattr(asyncio, "get_running_loop", _no_running_loop)+ assert not is_reactorless()++ def test_no_reactor_no_running_loop(self, monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.delitem(sys.modules, "twisted.internet.reactor", raising=False)+ monkeypatch.setattr(asyncio, "get_running_loop", _no_running_loop)+ with pytest.raises(RuntimeError, match=r"is_reactorless\(\) called without"):+ is_reactorless()diff --git a/tests/test_zz_resources.py b/tests/test_zz_resources.pyindex aa6e98e7d..bc8fc8363 100644--- a/tests/test_zz_resources.py+++ b/tests/test_zz_resources.py@@ -8,7 +8,7 @@ import loggingimport pytestfrom scrapy.utils.log import LogCounterHandler-from scrapy.utils.reactor import is_asyncio_reactor_installed, is_reactor_installed+from scrapy.utils.reactor import _is_asyncio_reactor_installed, is_reactor_installedfrom tests.utils.decorators import coroutine_test@@ -54,8 +54,8 @@ def test_installed_reactor(reactor_pytest: str) -> None:"""Test that the correct reactor is installed."""match reactor_pytest:case "asyncio":- assert is_asyncio_reactor_installed()+ assert _is_asyncio_reactor_installed()case "default":- assert not is_asyncio_reactor_installed()+ assert not _is_asyncio_reactor_installed()case "none":assert not is_reactor_installed()