Scrapy, a fast high-level web crawling & scraping framework for Python.
Stop using get_event_loop() in AsyncCrawler*. (#8327)
Andrey Rakhmatullinfbfc89ae7b72
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.pyindex f4f09127b..3f5309666 100644--- a/scrapy/core/downloader/handlers/http11.py+++ b/scrapy/core/downloader/handlers/http11.py@@ -16,6 +16,7 @@ from twisted.internet import sslfrom twisted.internet.defer import Deferred, succeedfrom twisted.internet.endpoints import TCP4ClientEndpointfrom twisted.internet.protocol import Factory, Protocol, connectionDone+from twisted.python.failure import Failurefrom twisted.web._newclient import (HEADER,STATUS,@@ -68,7 +69,6 @@ from ._base_http import BaseHttpDownloadHandlerif TYPE_CHECKING:from twisted.internet.base import ReactorBasefrom twisted.internet.interfaces import IAddress, IConsumer- from twisted.python.failure import Failurefrom twisted.web._newclient import Request as TxRequestfrom scrapy.crawler import Crawlerdiff --git a/scrapy/crawler.py b/scrapy/crawler.pyindex cb42126a8..f57143a0b 100644--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@ -22,6 +22,7 @@ from scrapy.signalmanager import SignalManagerfrom scrapy.spiderloader import SpiderLoaderProtocol, get_spider_loaderfrom scrapy.utils._ossignals import install_shutdown_handlers, signal_namesfrom scrapy.utils._stopmode import _normalize_stop_mode, _StopMode+from scrapy.utils.asyncio import _get_running_or_installed_loop, _has_running_loopfrom scrapy.utils.defer import _DEFER_DELAY, deferred_from_coro, ensure_awaitablefrom scrapy.utils.log import (_configure_logger_levels,@@ -665,8 +666,8 @@ class AsyncCrawlerRunner(CrawlerRunnerBase):When the :setting:`TWISTED_REACTOR_ENABLED` setting is set to ``True``,this class requires a reactor to be installed and uses it, otherwise it- requires a reactor to not be installed but requires an asyncio event loop- to be installed and uses it.+ requires a reactor to not be installed but requires a running asyncio+ event loop and uses it.This class shouldn't be needed (since Scrapy is responsible of using itaccordingly) unless writing scripts that manually handle the crawling@@ -746,10 +747,21 @@ class AsyncCrawlerRunner(CrawlerRunnerBase):self.crawlers.discard(crawler)self.bootstrap_failed |= not getattr(crawler, "spider", None)+ def _get_event_loop(self) -> asyncio.AbstractEventLoop:+ if self.settings.getbool("TWISTED_REACTOR_ENABLED") or _has_running_loop():+ return _get_running_or_installed_loop()+ warnings.warn(+ f"Calling {type(self).__name__}.crawl() without a running asyncio"+ f" event loop when TWISTED_REACTOR_ENABLED is False is deprecated."+ f" Call it from a coroutine that runs in the event loop instead,"+ f" e.g. one passed to asyncio.run().",+ ScrapyDeprecationWarning,+ stacklevel=4,+ )+ return asyncio.get_event_loop()+def _crawl(self, crawler: Crawler, *args: Any, **kwargs: Any) -> asyncio.Task[None]:- # At this point the asyncio loop has been installed either by the user- # or by AsyncCrawlerProcess (but it isn't running yet, so no asyncio.create_task()).- loop = asyncio.get_event_loop()+ loop = self._get_event_loop()self.crawlers.add(crawler)task = loop.create_task(self._crawl_and_track(crawler, *args, **kwargs))@@ -1103,6 +1115,13 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner):self._initialized_reactor = Trueself._reactorless_main_task: asyncio.Future[None] | None = None+ def _get_event_loop(self) -> asyncio.AbstractEventLoop:+ # crawl() can be called before start() runs the loop installed by+ # __init__().+ if self._reactorless_loop is not None:+ return self._reactorless_loop+ return super()._get_event_loop()+def _force_stop(self) -> None:if self.settings.getbool("TWISTED_REACTOR_ENABLED"):self._stop_reactor()@@ -1317,7 +1336,8 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner):from twisted.internet import reactorif stop_after_crawl:- loop = asyncio.get_event_loop()+ # The loop of the installed asyncio reactor, which isn't running yet.+ loop = _get_running_or_installed_loop()join_task = loop.create_task(self.join())join_task.add_done_callback(self._stop_reactor)diff --git a/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_loop_not_current.py b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_loop_not_current.pynew file mode 100644index 000000000..411ef77c8--- /dev/null+++ b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_loop_not_current.py@@ -0,0 +1,22 @@+import asyncio++from twisted.internet import asyncioreactor++import scrapy+from scrapy.crawler import AsyncCrawlerProcess++# Twisted doesn't make a loop passed to it the current event loop.+asyncioreactor.install(asyncio.SelectorEventLoop())+++class NoRequestsSpider(scrapy.Spider):+ name = "no_request"++ async def start(self):+ return+ yield+++process = AsyncCrawlerProcess()+process.crawl(NoRequestsSpider)+process.start()diff --git a/tests/AsyncCrawlerRunner/reactorless_loop_not_running.py b/tests/AsyncCrawlerRunner/reactorless_loop_not_running.pynew file mode 100644index 000000000..4cc6b3348--- /dev/null+++ b/tests/AsyncCrawlerRunner/reactorless_loop_not_running.py@@ -0,0 +1,23 @@+import asyncio++from scrapy import Spider+from scrapy.crawler import AsyncCrawlerRunner+from scrapy.utils.log import configure_logging+++class NoRequestsSpider(Spider):+ name = "no_request"++ async def start(self):+ return+ yield+++configure_logging()+loop = asyncio.new_event_loop()+asyncio.set_event_loop(loop)+runner = AsyncCrawlerRunner(+ settings={"TWISTED_REACTOR_ENABLED": False, "MEMUSAGE_ENABLED": False}+)+runner.crawl(NoRequestsSpider)+loop.run_until_complete(runner.join())diff --git a/tests/test_crawler_subprocess.py b/tests/test_crawler_subprocess.pyindex 057c03f35..de3d00813 100644--- a/tests/test_crawler_subprocess.py+++ b/tests/test_crawler_subprocess.py@@ -218,6 +218,11 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):"setting (uvloop.Loop)") in log+ def test_asyncio_enabled_reactor_loop_not_current(self) -> None:+ log = self.run_script("asyncio_enabled_reactor_loop_not_current.py")+ assert "Spider closed (finished)" in log+ assert "ERROR: " not in log+def test_default_loop_asyncio_deferred_signal(self) -> None:log = self.run_script("asyncio_deferred_signal.py")assert "Spider closed (finished)" in log@@ -371,11 +376,6 @@ class TestCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):in log)- def test_asyncio_enabled_reactor_loop_not_current(self) -> None:- log = self.run_script("asyncio_enabled_reactor_loop_not_current.py")- assert "Spider closed (finished)" in log- assert "ERROR: " not in log-class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):@property@@ -672,6 +672,14 @@ class TestAsyncCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase):assert log.count("WARNING: AiohttpDownloadHandler is experimental") == 2assert log.count("WARNING: ") == 2+ def test_reactorless_loop_not_running(self) -> None:+ log = self.run_script("reactorless_loop_not_running.py")+ assert "Spider closed (finished)" in log+ assert (+ "Calling AsyncCrawlerRunner.crawl() without a running asyncio event"+ " loop when TWISTED_REACTOR_ENABLED is False is deprecated"+ ) in log+def test_reactorless_custom_settings(self) -> None:"""Setting TWISTED_REACTOR_ENABLED=False in spider settings is notcurrently supported, AsyncCrawlerRunner will expect a reactor installed