Scrapy, a fast high-level web crawling & scraping framework for Python.
Shortcut for install_reactor() with the asyncio reactor/ (#8125)
Andrey Rakhmatullinae97d4f632e9
diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rstindex e3258b037..21f4f1ba2 100644--- a/docs/topics/asyncio.rst+++ b/docs/topics/asyncio.rst@@ -35,7 +35,7 @@ reactor manually. You can do that using.. skip: next.. code-block:: python- install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")+ install_reactor().. _asyncio-preinstalled-reactor:diff --git a/docs/topics/practices.rst b/docs/topics/practices.rstindex 42a791dee..ffbbe2d97 100644--- a/docs/topics/practices.rst+++ b/docs/topics/practices.rst@@ -132,7 +132,7 @@ with simple reactor management code:await runner.crawl(MySpider) # completes when the spider finishes- install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")+ install_reactor()react(deferred_f_from_coro_f(crawl))Same example but using :class:`~scrapy.crawler.CrawlerRunner` and a@@ -422,7 +422,7 @@ Same example using :class:`~scrapy.crawler.AsyncCrawlerRunner`:await runner.join() # completes when both spiders finish- install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")+ install_reactor()react(deferred_f_from_coro_f(crawl))@@ -456,7 +456,7 @@ finishes before starting the next one:await runner.crawl(MySpider2)- install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")+ install_reactor()react(deferred_f_from_coro_f(crawl)).. note:: When running multiple spiders in the same process, :ref:`loggingdiff --git a/scrapy/crawler.py b/scrapy/crawler.pyindex 7d07c8711..07630817e 100644--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@ -959,7 +959,7 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner):if loop_path:verify_installed_asyncio_event_loop(loop_path)else:- install_reactor(_asyncio_reactor_path, loop_path)+ install_reactor(event_loop_path=loop_path)self._initialized_reactor = Trueself._reactorless_main_task: asyncio.Future[None] | None = Nonediff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.pyindex 217249ee5..4353066ee 100644--- a/scrapy/utils/reactor.py+++ b/scrapy/utils/reactor.py@@ -111,10 +111,17 @@ def set_asyncio_event_loop_policy() -> None:asyncio.set_event_loop_policy(policy)-def install_reactor(reactor_path: str, event_loop_path: str | None = None) -> None:+def install_reactor(+ reactor_path: str | None = None, event_loop_path: str | None = None+) -> None:"""Installs the :mod:`~twisted.internet.reactor` with the specifiedimport path. Also installs the asyncio event loop with the specified import- path if the asyncio reactor is enabled"""+ path if the asyncio reactor is enabled.++ If `reactor_path` is `None`, the asyncio reactor will be installed.+ """+ if reactor_path is None:+ reactor_path = _asyncio_reactor_pathreactor_class = load_object(reactor_path)if reactor_class is asyncioreactor.AsyncioSelectorReactor:set_asyncio_event_loop_policy()diff --git a/tests/AsyncCrawlerProcess/asyncio_enabled_reactor.py b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor.pyindex e025e17d1..aad439759 100644--- a/tests/AsyncCrawlerProcess/asyncio_enabled_reactor.py+++ b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor.py@@ -23,7 +23,7 @@ if is_reactor_installed():"Reactor already installed after is_asyncio_reactor_installed().")-install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")+install_reactor()if not is_asyncio_reactor_installed():raise RuntimeError("Wrong reactor installed after install_reactor().")diff --git a/tests/AsyncCrawlerProcess/reactorless_reactor.py b/tests/AsyncCrawlerProcess/reactorless_reactor.pyindex 481112ca7..22015e4b8 100644--- a/tests/AsyncCrawlerProcess/reactorless_reactor.py+++ b/tests/AsyncCrawlerProcess/reactorless_reactor.py@@ -1,6 +1,6 @@from scrapy.crawler import AsyncCrawlerProcessfrom scrapy.utils.reactor import install_reactor-install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")+install_reactor()AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False})diff --git a/tests/AsyncCrawlerRunner/custom_loop_different.py b/tests/AsyncCrawlerRunner/custom_loop_different.pyindex 89cf0e536..ab1c07c16 100644--- a/tests/AsyncCrawlerRunner/custom_loop_different.py+++ b/tests/AsyncCrawlerRunner/custom_loop_different.py@@ -27,5 +27,5 @@ async def main(reactor):await runner.crawl(NoRequestsSpider)-install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")+install_reactor()react(main)diff --git a/tests/AsyncCrawlerRunner/multi_parallel.py b/tests/AsyncCrawlerRunner/multi_parallel.pyindex f1af9f794..009b20ac8 100644--- a/tests/AsyncCrawlerRunner/multi_parallel.py+++ b/tests/AsyncCrawlerRunner/multi_parallel.py@@ -24,5 +24,5 @@ async def main(reactor):await runner.join()-install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")+install_reactor()react(main)diff --git a/tests/AsyncCrawlerRunner/multi_seq.py b/tests/AsyncCrawlerRunner/multi_seq.pyindex 987f7a514..29c4225d0 100644--- a/tests/AsyncCrawlerRunner/multi_seq.py+++ b/tests/AsyncCrawlerRunner/multi_seq.py@@ -23,5 +23,5 @@ async def main(reactor):await runner.crawl(NoRequestsSpider)-install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")+install_reactor()react(main)diff --git a/tests/AsyncCrawlerRunner/reactorless_reactor.py b/tests/AsyncCrawlerRunner/reactorless_reactor.pyindex 4889b2ddc..69856cb8b 100644--- a/tests/AsyncCrawlerRunner/reactorless_reactor.py+++ b/tests/AsyncCrawlerRunner/reactorless_reactor.py@@ -20,5 +20,5 @@ async def main() -> None:await runner.crawl(NoRequestsSpider)-install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")+install_reactor()asyncio.run(main())diff --git a/tests/AsyncCrawlerRunner/simple.py b/tests/AsyncCrawlerRunner/simple.pyindex 29b132d75..a7c1cd137 100644--- a/tests/AsyncCrawlerRunner/simple.py+++ b/tests/AsyncCrawlerRunner/simple.py@@ -24,5 +24,5 @@ async def main(reactor):await runner.crawl(NoRequestsSpider)-install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")+install_reactor()react(main)diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.pyindex 4e8d3db12..7cfc9f1d2 100644--- a/tests/CrawlerProcess/asyncio_enabled_reactor.py+++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py@@ -31,7 +31,7 @@ if is_reactor_installed():"Reactor already installed after is_asyncio_reactor_installed().")-install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")+install_reactor()if not is_asyncio_reactor_installed():raise RuntimeError("Wrong reactor installed after install_reactor().")diff --git a/tests/CrawlerRunner/change_reactor.py b/tests/CrawlerRunner/change_reactor.pyindex c275e0583..5660c57ae 100644--- a/tests/CrawlerRunner/change_reactor.py+++ b/tests/CrawlerRunner/change_reactor.py@@ -20,7 +20,7 @@ configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s", "LOG_LEVEL": "DEBfrom scrapy.utils.reactor import install_reactor # noqa: E402-install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")+install_reactor()runner = CrawlerRunner()diff --git a/tests/CrawlerRunner/custom_loop_different.py b/tests/CrawlerRunner/custom_loop_different.pyindex 86ba1ed47..4cbd381b1 100644--- a/tests/CrawlerRunner/custom_loop_different.py+++ b/tests/CrawlerRunner/custom_loop_different.py@@ -25,5 +25,5 @@ def main(reactor):return runner.crawl(NoRequestsSpider)-install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")+install_reactor()react(main)diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.pyindex bab4f65db..e7ca4b197 100644--- a/tests/CrawlerRunner/ip_address.py+++ b/tests/CrawlerRunner/ip_address.py@@ -8,7 +8,7 @@ from scrapy.utils.reactor import install_reactorfrom tests.mockserver.dns import MockDNSServerfrom tests.mockserver.http import MockServer-install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")+install_reactor()from twisted.names import cache, resolvediff --git a/tests/CrawlerRunner/multi_parallel.py b/tests/CrawlerRunner/multi_parallel.pyindex 51feccd0a..bf07a9ffc 100644--- a/tests/CrawlerRunner/multi_parallel.py+++ b/tests/CrawlerRunner/multi_parallel.py@@ -22,5 +22,5 @@ def main(reactor):return runner.join()-install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")+install_reactor()react(main)diff --git a/tests/CrawlerRunner/multi_seq.py b/tests/CrawlerRunner/multi_seq.pyindex f6549be9b..e384c5249 100644--- a/tests/CrawlerRunner/multi_seq.py+++ b/tests/CrawlerRunner/multi_seq.py@@ -23,5 +23,5 @@ def main(reactor):yield runner.crawl(NoRequestsSpider)-install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")+install_reactor()react(main)diff --git a/tests/CrawlerRunner/simple.py b/tests/CrawlerRunner/simple.pyindex 47c0fe04f..73feec06a 100644--- a/tests/CrawlerRunner/simple.py+++ b/tests/CrawlerRunner/simple.py@@ -22,5 +22,5 @@ def main(reactor):return runner.crawl(NoRequestsSpider)-install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")+install_reactor()react(main)diff --git a/tests/benchmarks/conftest.py b/tests/benchmarks/conftest.pyindex 55356083d..3a5ccd501 100644--- a/tests/benchmarks/conftest.py+++ b/tests/benchmarks/conftest.py@@ -12,7 +12,7 @@ if TYPE_CHECKING:@pytest.fixture(scope="session", autouse=True)def running_reactor() -> Generator[None]:- install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")+ install_reactor()from twisted.internet import reactordiff --git a/tests/test_utils_reactor.py b/tests/test_utils_reactor.pyindex 3be0a03b8..0493bfe6e 100644--- a/tests/test_utils_reactor.py+++ b/tests/test_utils_reactor.py@@ -35,7 +35,7 @@ class TestAsyncio:@pytest.mark.only_asyncio@coroutine_testasync def test_set_asyncio_event_loop(self):- install_reactor(_asyncio_reactor_path)+ install_reactor()assert set_asyncio_event_loop(None) is asyncio.get_running_loop()