Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Verify that the installed asyncio event loop matches ASYNCIO_EVENT_LOOP (#5529)

Adrián Chaves960a7f68f693
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index e768bca12..65174d846 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -31,7 +31,12 @@ from scrapy.utils.log import (
)
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
-from scrapy.utils.reactor import install_reactor, verify_installed_reactor
+from scrapy.utils.reactor import (
+ install_reactor,
+ is_asyncio_reactor_installed,
+ verify_installed_asyncio_event_loop,
+ verify_installed_reactor,
+)
logger = logging.getLogger(__name__)
@@ -78,17 +83,20 @@ class Crawler:
crawler=self,
)
- reactor_class = self.settings.get("TWISTED_REACTOR")
+ reactor_class = self.settings["TWISTED_REACTOR"]
+ event_loop = self.settings["ASYNCIO_EVENT_LOOP"]
if init_reactor:
# this needs to be done after the spider settings are merged,
# but before something imports twisted.internet.reactor
if reactor_class:
- install_reactor(reactor_class, self.settings["ASYNCIO_EVENT_LOOP"])
+ install_reactor(reactor_class, event_loop)
else:
from twisted.internet import reactor # noqa: F401
log_reactor_info()
if reactor_class:
verify_installed_reactor(reactor_class)
+ if is_asyncio_reactor_installed() and event_loop:
+ verify_installed_asyncio_event_loop(event_loop)
self.extensions = ExtensionManager.from_crawler(self)
diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
index bc543b230..652733ce8 100644
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -90,6 +90,24 @@ def verify_installed_reactor(reactor_path):
raise Exception(msg)
+def verify_installed_asyncio_event_loop(loop_path):
+ from twisted.internet import reactor
+ loop_class = load_object(loop_path)
+ if isinstance(reactor._asyncioEventloop, loop_class):
+ return
+ installed = (
+ f"{reactor._asyncioEventloop.__class__.__module__}"
+ f".{reactor._asyncioEventloop.__class__.__qualname__}"
+ )
+ specified = f"{loop_class.__module__}.{loop_class.__qualname__}"
+ raise Exception(
+ "Scrapy found an asyncio Twisted reactor already "
+ f"installed, and its event loop class ({installed}) does "
+ "not match the one specified in the ASYNCIO_EVENT_LOOP "
+ f"setting ({specified})"
+ )
+
+
def is_asyncio_reactor_installed():
from twisted.internet import reactor
return isinstance(reactor, asyncioreactor.AsyncioSelectorReactor)
diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py
new file mode 100644
index 000000000..ea8242f67
--- /dev/null
+++ b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py
@@ -0,0 +1,25 @@
+import asyncio
+import sys
+
+from twisted.internet import asyncioreactor
+if sys.version_info >= (3, 8) and sys.platform == "win32":
+ asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
+asyncioreactor.install(asyncio.get_event_loop())
+
+import scrapy # noqa: E402
+from scrapy.crawler import CrawlerProcess # noqa: E402
+
+
+class NoRequestsSpider(scrapy.Spider):
+ name = 'no_request'
+
+ def start_requests(self):
+ return []
+
+
+process = CrawlerProcess(settings={
+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+ "ASYNCIO_EVENT_LOOP": "uvloop.Loop",
+})
+process.crawl(NoRequestsSpider)
+process.start()
diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py
new file mode 100644
index 000000000..d24bf3031
--- /dev/null
+++ b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py
@@ -0,0 +1,28 @@
+import asyncio
+import sys
+
+from uvloop import Loop
+
+from twisted.internet import asyncioreactor
+if sys.version_info >= (3, 8) and sys.platform == "win32":
+ asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
+asyncio.set_event_loop(Loop())
+asyncioreactor.install(asyncio.get_event_loop())
+
+import scrapy # noqa: E402
+from scrapy.crawler import CrawlerProcess # noqa: E402
+
+
+class NoRequestsSpider(scrapy.Spider):
+ name = 'no_request'
+
+ def start_requests(self):
+ return []
+
+
+process = CrawlerProcess(settings={
+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+ "ASYNCIO_EVENT_LOOP": "uvloop.Loop",
+})
+process.crawl(NoRequestsSpider)
+process.start()
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index cf15ba9b9..c61d461f7 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -454,6 +454,29 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
self.assertIn("Using asyncio event loop: uvloop.Loop", log)
self.assertIn("async pipeline opened!", log)
+ @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly')
+ @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows')
+ @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106')
+ def test_asyncio_enabled_reactor_same_loop(self):
+ log = self.run_script("asyncio_enabled_reactor_same_loop.py")
+ self.assertIn("Spider closed (finished)", log)
+ self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
+ self.assertIn("Using asyncio event loop: uvloop.Loop", log)
+
+ @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly')
+ @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows')
+ @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106')
+ def test_asyncio_enabled_reactor_different_loop(self):
+ log = self.run_script("asyncio_enabled_reactor_different_loop.py")
+ self.assertNotIn("Spider closed (finished)", log)
+ self.assertIn(
+ (
+ "does not match the one specified in the ASYNCIO_EVENT_LOOP "
+ "setting (uvloop.Loop)"
+ ),
+ log,
+ )
+
def test_default_loop_asyncio_deferred_signal(self):
log = self.run_script("asyncio_deferred_signal.py")
self.assertIn("Spider closed (finished)", log)