Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Stop using get_event_loop() in AsyncCrawler*. (#8327)

Andrey Rakhmatullinfbfc89ae7b72
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index f4f09127b..3f5309666 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -16,6 +16,7 @@ from twisted.internet import ssl
from twisted.internet.defer import Deferred, succeed
from twisted.internet.endpoints import TCP4ClientEndpoint
from twisted.internet.protocol import Factory, Protocol, connectionDone
+from twisted.python.failure import Failure
from twisted.web._newclient import (
HEADER,
STATUS,
@@ -68,7 +69,6 @@ from ._base_http import BaseHttpDownloadHandler
if TYPE_CHECKING:
from twisted.internet.base import ReactorBase
from twisted.internet.interfaces import IAddress, IConsumer
- from twisted.python.failure import Failure
from twisted.web._newclient import Request as TxRequest
from scrapy.crawler import Crawler
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index cb42126a8..f57143a0b 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -22,6 +22,7 @@ from scrapy.signalmanager import SignalManager
from scrapy.spiderloader import SpiderLoaderProtocol, get_spider_loader
from scrapy.utils._ossignals import install_shutdown_handlers, signal_names
from scrapy.utils._stopmode import _normalize_stop_mode, _StopMode
+from scrapy.utils.asyncio import _get_running_or_installed_loop, _has_running_loop
from scrapy.utils.defer import _DEFER_DELAY, deferred_from_coro, ensure_awaitable
from scrapy.utils.log import (
_configure_logger_levels,
@@ -665,8 +666,8 @@ class AsyncCrawlerRunner(CrawlerRunnerBase):
When the :setting:`TWISTED_REACTOR_ENABLED` setting is set to ``True``,
this class requires a reactor to be installed and uses it, otherwise it
- requires a reactor to not be installed but requires an asyncio event loop
- to be installed and uses it.
+ requires a reactor to not be installed but requires a running asyncio
+ event loop and uses it.
This class shouldn't be needed (since Scrapy is responsible of using it
accordingly) unless writing scripts that manually handle the crawling
@@ -746,10 +747,21 @@ class AsyncCrawlerRunner(CrawlerRunnerBase):
self.crawlers.discard(crawler)
self.bootstrap_failed |= not getattr(crawler, "spider", None)
+ def _get_event_loop(self) -> asyncio.AbstractEventLoop:
+ if self.settings.getbool("TWISTED_REACTOR_ENABLED") or _has_running_loop():
+ return _get_running_or_installed_loop()
+ warnings.warn(
+ f"Calling {type(self).__name__}.crawl() without a running asyncio"
+ f" event loop when TWISTED_REACTOR_ENABLED is False is deprecated."
+ f" Call it from a coroutine that runs in the event loop instead,"
+ f" e.g. one passed to asyncio.run().",
+ ScrapyDeprecationWarning,
+ stacklevel=4,
+ )
+ return asyncio.get_event_loop()
+
def _crawl(self, crawler: Crawler, *args: Any, **kwargs: Any) -> asyncio.Task[None]:
- # At this point the asyncio loop has been installed either by the user
- # or by AsyncCrawlerProcess (but it isn't running yet, so no asyncio.create_task()).
- loop = asyncio.get_event_loop()
+ loop = self._get_event_loop()
self.crawlers.add(crawler)
task = loop.create_task(self._crawl_and_track(crawler, *args, **kwargs))
@@ -1103,6 +1115,13 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner):
self._initialized_reactor = True
self._reactorless_main_task: asyncio.Future[None] | None = None
+ def _get_event_loop(self) -> asyncio.AbstractEventLoop:
+ # crawl() can be called before start() runs the loop installed by
+ # __init__().
+ if self._reactorless_loop is not None:
+ return self._reactorless_loop
+ return super()._get_event_loop()
+
def _force_stop(self) -> None:
if self.settings.getbool("TWISTED_REACTOR_ENABLED"):
self._stop_reactor()
@@ -1317,7 +1336,8 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner):
from twisted.internet import reactor
if stop_after_crawl:
- loop = asyncio.get_event_loop()
+ # The loop of the installed asyncio reactor, which isn't running yet.
+ loop = _get_running_or_installed_loop()
join_task = loop.create_task(self.join())
join_task.add_done_callback(self._stop_reactor)
diff --git a/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_loop_not_current.py b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_loop_not_current.py
new file mode 100644
index 000000000..411ef77c8
--- /dev/null
+++ b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_loop_not_current.py
@@ -0,0 +1,22 @@
+import asyncio
+
+from twisted.internet import asyncioreactor
+
+import scrapy
+from scrapy.crawler import AsyncCrawlerProcess
+
+# Twisted doesn't make a loop passed to it the current event loop.
+asyncioreactor.install(asyncio.SelectorEventLoop())
+
+
+class NoRequestsSpider(scrapy.Spider):
+ name = "no_request"
+
+ async def start(self):
+ return
+ yield
+
+
+process = AsyncCrawlerProcess()
+process.crawl(NoRequestsSpider)
+process.start()
diff --git a/tests/AsyncCrawlerRunner/reactorless_loop_not_running.py b/tests/AsyncCrawlerRunner/reactorless_loop_not_running.py
new file mode 100644
index 000000000..4cc6b3348
--- /dev/null
+++ b/tests/AsyncCrawlerRunner/reactorless_loop_not_running.py
@@ -0,0 +1,23 @@
+import asyncio
+
+from scrapy import Spider
+from scrapy.crawler import AsyncCrawlerRunner
+from scrapy.utils.log import configure_logging
+
+
+class NoRequestsSpider(Spider):
+ name = "no_request"
+
+ async def start(self):
+ return
+ yield
+
+
+configure_logging()
+loop = asyncio.new_event_loop()
+asyncio.set_event_loop(loop)
+runner = AsyncCrawlerRunner(
+ settings={"TWISTED_REACTOR_ENABLED": False, "MEMUSAGE_ENABLED": False}
+)
+runner.crawl(NoRequestsSpider)
+loop.run_until_complete(runner.join())
diff --git a/tests/test_crawler_subprocess.py b/tests/test_crawler_subprocess.py
index 057c03f35..de3d00813 100644
--- a/tests/test_crawler_subprocess.py
+++ b/tests/test_crawler_subprocess.py
@@ -218,6 +218,11 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
"setting (uvloop.Loop)"
) in log
+ def test_asyncio_enabled_reactor_loop_not_current(self) -> None:
+ log = self.run_script("asyncio_enabled_reactor_loop_not_current.py")
+ assert "Spider closed (finished)" in log
+ assert "ERROR: " not in log
+
def test_default_loop_asyncio_deferred_signal(self) -> None:
log = self.run_script("asyncio_deferred_signal.py")
assert "Spider closed (finished)" in log
@@ -371,11 +376,6 @@ class TestCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
in log
)
- def test_asyncio_enabled_reactor_loop_not_current(self) -> None:
- log = self.run_script("asyncio_enabled_reactor_loop_not_current.py")
- assert "Spider closed (finished)" in log
- assert "ERROR: " not in log
-
class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
@property
@@ -672,6 +672,14 @@ class TestAsyncCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase):
assert log.count("WARNING: AiohttpDownloadHandler is experimental") == 2
assert log.count("WARNING: ") == 2
+ def test_reactorless_loop_not_running(self) -> None:
+ log = self.run_script("reactorless_loop_not_running.py")
+ assert "Spider closed (finished)" in log
+ assert (
+ "Calling AsyncCrawlerRunner.crawl() without a running asyncio event"
+ " loop when TWISTED_REACTOR_ENABLED is False is deprecated"
+ ) in log
+
def test_reactorless_custom_settings(self) -> None:
"""Setting TWISTED_REACTOR_ENABLED=False in spider settings is not
currently supported, AsyncCrawlerRunner will expect a reactor installed