Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add tests for TWISTED_REACTOR in custom_settings.

Andrey Rakhmatullinebcafdf4a9e0
diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings.py b/tests/CrawlerProcess/twisted_reactor_custom_settings.py
new file mode 100644
index 000000000..56304bd23
--- /dev/null
+++ b/tests/CrawlerProcess/twisted_reactor_custom_settings.py
@@ -0,0 +1,14 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+
+class AsyncioReactorSpider(scrapy.Spider):
+ name = 'asyncio_reactor'
+ custom_settings = {
+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+ }
+
+
+process = CrawlerProcess()
+process.crawl(AsyncioReactorSpider)
+process.start()
diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py b/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py
new file mode 100644
index 000000000..9a6c01d72
--- /dev/null
+++ b/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py
@@ -0,0 +1,22 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+
+class PollReactorSpider(scrapy.Spider):
+ name = 'poll_reactor'
+ custom_settings = {
+ "TWISTED_REACTOR": "twisted.internet.pollreactor.PollReactor",
+ }
+
+
+class AsyncioReactorSpider(scrapy.Spider):
+ name = 'asyncio_reactor'
+ custom_settings = {
+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+ }
+
+
+process = CrawlerProcess()
+process.crawl(PollReactorSpider)
+process.crawl(AsyncioReactorSpider)
+process.start()
diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py b/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py
new file mode 100644
index 000000000..1f5a44010
--- /dev/null
+++ b/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py
@@ -0,0 +1,21 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+
+class AsyncioReactorSpider1(scrapy.Spider):
+ name = 'asyncio_reactor1'
+ custom_settings = {
+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+ }
+
+class AsyncioReactorSpider2(scrapy.Spider):
+ name = 'asyncio_reactor2'
+ custom_settings = {
+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+ }
+
+
+process = CrawlerProcess()
+process.crawl(AsyncioReactorSpider1)
+process.crawl(AsyncioReactorSpider2)
+process.start()
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index f445c181e..6d6763aec 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -361,6 +361,30 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
self.assertIn("Spider closed (finished)", log)
self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
+ # https://twistedmatrix.com/trac/ticket/9766
+ @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
+ "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
+ def test_reactor_asyncio_custom_settings(self):
+ log = self.run_script("twisted_reactor_custom_settings.py")
+ self.assertIn("Spider closed (finished)", log)
+ self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
+
+ # https://twistedmatrix.com/trac/ticket/9766
+ @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
+ "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
+ def test_reactor_asyncio_custom_settings_same(self):
+ log = self.run_script("twisted_reactor_custom_settings_same.py")
+ self.assertIn("Spider closed (finished)", log)
+ self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
+
+ # https://twistedmatrix.com/trac/ticket/9766
+ @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
+ "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
+ def test_reactor_asyncio_custom_settings_conflict(self):
+ log = self.run_script("twisted_reactor_custom_settings_conflict.py")
+ self.assertIn("Using reactor: twisted.internet.pollreactor.PollReactor", log)
+ self.assertIn("(twisted.internet.pollreactor.PollReactor) does not match the requested one", log)
+
@mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly')
@mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows')
@mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106')