Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Shortcut for install_reactor() with the asyncio reactor/ (#8125)

Andrey Rakhmatullinae97d4f632e9
diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst
index e3258b037..21f4f1ba2 100644
--- a/docs/topics/asyncio.rst
+++ b/docs/topics/asyncio.rst
@@ -35,7 +35,7 @@ reactor manually. You can do that using
.. skip: next
.. code-block:: python
- install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
+ install_reactor()
.. _asyncio-preinstalled-reactor:
diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index 42a791dee..ffbbe2d97 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -132,7 +132,7 @@ with simple reactor management code:
await runner.crawl(MySpider) # completes when the spider finishes
- install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
+ install_reactor()
react(deferred_f_from_coro_f(crawl))
Same example but using :class:`~scrapy.crawler.CrawlerRunner` and a
@@ -422,7 +422,7 @@ Same example using :class:`~scrapy.crawler.AsyncCrawlerRunner`:
await runner.join() # completes when both spiders finish
- install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
+ install_reactor()
react(deferred_f_from_coro_f(crawl))
@@ -456,7 +456,7 @@ finishes before starting the next one:
await runner.crawl(MySpider2)
- install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
+ install_reactor()
react(deferred_f_from_coro_f(crawl))
.. note:: When running multiple spiders in the same process, :ref:`logging
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 7d07c8711..07630817e 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -959,7 +959,7 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner):
if loop_path:
verify_installed_asyncio_event_loop(loop_path)
else:
- install_reactor(_asyncio_reactor_path, loop_path)
+ install_reactor(event_loop_path=loop_path)
self._initialized_reactor = True
self._reactorless_main_task: asyncio.Future[None] | None = None
diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
index 217249ee5..4353066ee 100644
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -111,10 +111,17 @@ def set_asyncio_event_loop_policy() -> None:
asyncio.set_event_loop_policy(policy)
-def install_reactor(reactor_path: str, event_loop_path: str | None = None) -> None:
+def install_reactor(
+ reactor_path: str | None = None, event_loop_path: str | None = None
+) -> None:
"""Installs the :mod:`~twisted.internet.reactor` with the specified
import path. Also installs the asyncio event loop with the specified import
- path if the asyncio reactor is enabled"""
+ path if the asyncio reactor is enabled.
+
+ If `reactor_path` is `None`, the asyncio reactor will be installed.
+ """
+ if reactor_path is None:
+ reactor_path = _asyncio_reactor_path
reactor_class = load_object(reactor_path)
if reactor_class is asyncioreactor.AsyncioSelectorReactor:
set_asyncio_event_loop_policy()
diff --git a/tests/AsyncCrawlerProcess/asyncio_enabled_reactor.py b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor.py
index e025e17d1..aad439759 100644
--- a/tests/AsyncCrawlerProcess/asyncio_enabled_reactor.py
+++ b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor.py
@@ -23,7 +23,7 @@ if is_reactor_installed():
"Reactor already installed after is_asyncio_reactor_installed()."
)
-install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
+install_reactor()
if not is_asyncio_reactor_installed():
raise RuntimeError("Wrong reactor installed after install_reactor().")
diff --git a/tests/AsyncCrawlerProcess/reactorless_reactor.py b/tests/AsyncCrawlerProcess/reactorless_reactor.py
index 481112ca7..22015e4b8 100644
--- a/tests/AsyncCrawlerProcess/reactorless_reactor.py
+++ b/tests/AsyncCrawlerProcess/reactorless_reactor.py
@@ -1,6 +1,6 @@
from scrapy.crawler import AsyncCrawlerProcess
from scrapy.utils.reactor import install_reactor
-install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
+install_reactor()
AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False})
diff --git a/tests/AsyncCrawlerRunner/custom_loop_different.py b/tests/AsyncCrawlerRunner/custom_loop_different.py
index 89cf0e536..ab1c07c16 100644
--- a/tests/AsyncCrawlerRunner/custom_loop_different.py
+++ b/tests/AsyncCrawlerRunner/custom_loop_different.py
@@ -27,5 +27,5 @@ async def main(reactor):
await runner.crawl(NoRequestsSpider)
-install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
+install_reactor()
react(main)
diff --git a/tests/AsyncCrawlerRunner/multi_parallel.py b/tests/AsyncCrawlerRunner/multi_parallel.py
index f1af9f794..009b20ac8 100644
--- a/tests/AsyncCrawlerRunner/multi_parallel.py
+++ b/tests/AsyncCrawlerRunner/multi_parallel.py
@@ -24,5 +24,5 @@ async def main(reactor):
await runner.join()
-install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
+install_reactor()
react(main)
diff --git a/tests/AsyncCrawlerRunner/multi_seq.py b/tests/AsyncCrawlerRunner/multi_seq.py
index 987f7a514..29c4225d0 100644
--- a/tests/AsyncCrawlerRunner/multi_seq.py
+++ b/tests/AsyncCrawlerRunner/multi_seq.py
@@ -23,5 +23,5 @@ async def main(reactor):
await runner.crawl(NoRequestsSpider)
-install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
+install_reactor()
react(main)
diff --git a/tests/AsyncCrawlerRunner/reactorless_reactor.py b/tests/AsyncCrawlerRunner/reactorless_reactor.py
index 4889b2ddc..69856cb8b 100644
--- a/tests/AsyncCrawlerRunner/reactorless_reactor.py
+++ b/tests/AsyncCrawlerRunner/reactorless_reactor.py
@@ -20,5 +20,5 @@ async def main() -> None:
await runner.crawl(NoRequestsSpider)
-install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
+install_reactor()
asyncio.run(main())
diff --git a/tests/AsyncCrawlerRunner/simple.py b/tests/AsyncCrawlerRunner/simple.py
index 29b132d75..a7c1cd137 100644
--- a/tests/AsyncCrawlerRunner/simple.py
+++ b/tests/AsyncCrawlerRunner/simple.py
@@ -24,5 +24,5 @@ async def main(reactor):
await runner.crawl(NoRequestsSpider)
-install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
+install_reactor()
react(main)
diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.py
index 4e8d3db12..7cfc9f1d2 100644
--- a/tests/CrawlerProcess/asyncio_enabled_reactor.py
+++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py
@@ -31,7 +31,7 @@ if is_reactor_installed():
"Reactor already installed after is_asyncio_reactor_installed()."
)
-install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
+install_reactor()
if not is_asyncio_reactor_installed():
raise RuntimeError("Wrong reactor installed after install_reactor().")
diff --git a/tests/CrawlerRunner/change_reactor.py b/tests/CrawlerRunner/change_reactor.py
index c275e0583..5660c57ae 100644
--- a/tests/CrawlerRunner/change_reactor.py
+++ b/tests/CrawlerRunner/change_reactor.py
@@ -20,7 +20,7 @@ configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s", "LOG_LEVEL": "DEB
from scrapy.utils.reactor import install_reactor # noqa: E402
-install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
+install_reactor()
runner = CrawlerRunner()
diff --git a/tests/CrawlerRunner/custom_loop_different.py b/tests/CrawlerRunner/custom_loop_different.py
index 86ba1ed47..4cbd381b1 100644
--- a/tests/CrawlerRunner/custom_loop_different.py
+++ b/tests/CrawlerRunner/custom_loop_different.py
@@ -25,5 +25,5 @@ def main(reactor):
return runner.crawl(NoRequestsSpider)
-install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
+install_reactor()
react(main)
diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py
index bab4f65db..e7ca4b197 100644
--- a/tests/CrawlerRunner/ip_address.py
+++ b/tests/CrawlerRunner/ip_address.py
@@ -8,7 +8,7 @@ from scrapy.utils.reactor import install_reactor
from tests.mockserver.dns import MockDNSServer
from tests.mockserver.http import MockServer
-install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
+install_reactor()
from twisted.names import cache, resolve
diff --git a/tests/CrawlerRunner/multi_parallel.py b/tests/CrawlerRunner/multi_parallel.py
index 51feccd0a..bf07a9ffc 100644
--- a/tests/CrawlerRunner/multi_parallel.py
+++ b/tests/CrawlerRunner/multi_parallel.py
@@ -22,5 +22,5 @@ def main(reactor):
return runner.join()
-install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
+install_reactor()
react(main)
diff --git a/tests/CrawlerRunner/multi_seq.py b/tests/CrawlerRunner/multi_seq.py
index f6549be9b..e384c5249 100644
--- a/tests/CrawlerRunner/multi_seq.py
+++ b/tests/CrawlerRunner/multi_seq.py
@@ -23,5 +23,5 @@ def main(reactor):
yield runner.crawl(NoRequestsSpider)
-install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
+install_reactor()
react(main)
diff --git a/tests/CrawlerRunner/simple.py b/tests/CrawlerRunner/simple.py
index 47c0fe04f..73feec06a 100644
--- a/tests/CrawlerRunner/simple.py
+++ b/tests/CrawlerRunner/simple.py
@@ -22,5 +22,5 @@ def main(reactor):
return runner.crawl(NoRequestsSpider)
-install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
+install_reactor()
react(main)
diff --git a/tests/benchmarks/conftest.py b/tests/benchmarks/conftest.py
index 55356083d..3a5ccd501 100644
--- a/tests/benchmarks/conftest.py
+++ b/tests/benchmarks/conftest.py
@@ -12,7 +12,7 @@ if TYPE_CHECKING:
@pytest.fixture(scope="session", autouse=True)
def running_reactor() -> Generator[None]:
- install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
+ install_reactor()
from twisted.internet import reactor
diff --git a/tests/test_utils_reactor.py b/tests/test_utils_reactor.py
index 3be0a03b8..0493bfe6e 100644
--- a/tests/test_utils_reactor.py
+++ b/tests/test_utils_reactor.py
@@ -35,7 +35,7 @@ class TestAsyncio:
@pytest.mark.only_asyncio
@coroutine_test
async def test_set_asyncio_event_loop(self):
- install_reactor(_asyncio_reactor_path)
+ install_reactor()
assert set_asyncio_event_loop(None) is asyncio.get_running_loop()