Scrapy, a fast high-level web crawling & scraping framework for Python.
Verify that the installed asyncio event loop matches ASYNCIO_EVENT_LOOP (#5529)
Adrián Chaves960a7f68f693
diff --git a/scrapy/crawler.py b/scrapy/crawler.pyindex e768bca12..65174d846 100644--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@ -31,7 +31,12 @@ from scrapy.utils.log import ()from scrapy.utils.misc import create_instance, load_objectfrom scrapy.utils.ossignal import install_shutdown_handlers, signal_names-from scrapy.utils.reactor import install_reactor, verify_installed_reactor+from scrapy.utils.reactor import (+ install_reactor,+ is_asyncio_reactor_installed,+ verify_installed_asyncio_event_loop,+ verify_installed_reactor,+)logger = logging.getLogger(__name__)@@ -78,17 +83,20 @@ class Crawler:crawler=self,)- reactor_class = self.settings.get("TWISTED_REACTOR")+ reactor_class = self.settings["TWISTED_REACTOR"]+ event_loop = self.settings["ASYNCIO_EVENT_LOOP"]if init_reactor:# this needs to be done after the spider settings are merged,# but before something imports twisted.internet.reactorif reactor_class:- install_reactor(reactor_class, self.settings["ASYNCIO_EVENT_LOOP"])+ install_reactor(reactor_class, event_loop)else:from twisted.internet import reactor # noqa: F401log_reactor_info()if reactor_class:verify_installed_reactor(reactor_class)+ if is_asyncio_reactor_installed() and event_loop:+ verify_installed_asyncio_event_loop(event_loop)self.extensions = ExtensionManager.from_crawler(self)diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.pyindex bc543b230..652733ce8 100644--- a/scrapy/utils/reactor.py+++ b/scrapy/utils/reactor.py@@ -90,6 +90,24 @@ def verify_installed_reactor(reactor_path):raise Exception(msg)+def verify_installed_asyncio_event_loop(loop_path):+ from twisted.internet import reactor+ loop_class = load_object(loop_path)+ if isinstance(reactor._asyncioEventloop, loop_class):+ return+ installed = (+ f"{reactor._asyncioEventloop.__class__.__module__}"+ f".{reactor._asyncioEventloop.__class__.__qualname__}"+ )+ specified = f"{loop_class.__module__}.{loop_class.__qualname__}"+ raise Exception(+ "Scrapy found an asyncio Twisted reactor already "+ f"installed, and its event loop class ({installed}) does "+ "not match the one specified in the ASYNCIO_EVENT_LOOP "+ f"setting ({specified})"+ )++def is_asyncio_reactor_installed():from twisted.internet import reactorreturn isinstance(reactor, asyncioreactor.AsyncioSelectorReactor)diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.pynew file mode 100644index 000000000..ea8242f67--- /dev/null+++ b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py@@ -0,0 +1,25 @@+import asyncio+import sys++from twisted.internet import asyncioreactor+if sys.version_info >= (3, 8) and sys.platform == "win32":+ asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())+asyncioreactor.install(asyncio.get_event_loop())++import scrapy # noqa: E402+from scrapy.crawler import CrawlerProcess # noqa: E402+++class NoRequestsSpider(scrapy.Spider):+ name = 'no_request'++ def start_requests(self):+ return []+++process = CrawlerProcess(settings={+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",+ "ASYNCIO_EVENT_LOOP": "uvloop.Loop",+})+process.crawl(NoRequestsSpider)+process.start()diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.pynew file mode 100644index 000000000..d24bf3031--- /dev/null+++ b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py@@ -0,0 +1,28 @@+import asyncio+import sys++from uvloop import Loop++from twisted.internet import asyncioreactor+if sys.version_info >= (3, 8) and sys.platform == "win32":+ asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())+asyncio.set_event_loop(Loop())+asyncioreactor.install(asyncio.get_event_loop())++import scrapy # noqa: E402+from scrapy.crawler import CrawlerProcess # noqa: E402+++class NoRequestsSpider(scrapy.Spider):+ name = 'no_request'++ def start_requests(self):+ return []+++process = CrawlerProcess(settings={+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",+ "ASYNCIO_EVENT_LOOP": "uvloop.Loop",+})+process.crawl(NoRequestsSpider)+process.start()diff --git a/tests/test_crawler.py b/tests/test_crawler.pyindex cf15ba9b9..c61d461f7 100644--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@ -454,6 +454,29 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):self.assertIn("Using asyncio event loop: uvloop.Loop", log)self.assertIn("async pipeline opened!", log)+ @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly')+ @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows')+ @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106')+ def test_asyncio_enabled_reactor_same_loop(self):+ log = self.run_script("asyncio_enabled_reactor_same_loop.py")+ self.assertIn("Spider closed (finished)", log)+ self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)+ self.assertIn("Using asyncio event loop: uvloop.Loop", log)++ @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly')+ @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows')+ @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106')+ def test_asyncio_enabled_reactor_different_loop(self):+ log = self.run_script("asyncio_enabled_reactor_different_loop.py")+ self.assertNotIn("Spider closed (finished)", log)+ self.assertIn(+ (+ "does not match the one specified in the ASYNCIO_EVENT_LOOP "+ "setting (uvloop.Loop)"+ ),+ log,+ )+def test_default_loop_asyncio_deferred_signal(self):log = self.run_script("asyncio_deferred_signal.py")self.assertIn("Spider closed (finished)", log)