Scrapy, a fast high-level web crawling & scraping framework for Python.
Add tests for TWISTED_REACTOR in custom_settings.
Andrey Rakhmatullinebcafdf4a9e0
diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings.py b/tests/CrawlerProcess/twisted_reactor_custom_settings.pynew file mode 100644index 000000000..56304bd23--- /dev/null+++ b/tests/CrawlerProcess/twisted_reactor_custom_settings.py@@ -0,0 +1,14 @@+import scrapy+from scrapy.crawler import CrawlerProcess+++class AsyncioReactorSpider(scrapy.Spider):+ name = 'asyncio_reactor'+ custom_settings = {+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",+ }+++process = CrawlerProcess()+process.crawl(AsyncioReactorSpider)+process.start()diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py b/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.pynew file mode 100644index 000000000..9a6c01d72--- /dev/null+++ b/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py@@ -0,0 +1,22 @@+import scrapy+from scrapy.crawler import CrawlerProcess+++class PollReactorSpider(scrapy.Spider):+ name = 'poll_reactor'+ custom_settings = {+ "TWISTED_REACTOR": "twisted.internet.pollreactor.PollReactor",+ }+++class AsyncioReactorSpider(scrapy.Spider):+ name = 'asyncio_reactor'+ custom_settings = {+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",+ }+++process = CrawlerProcess()+process.crawl(PollReactorSpider)+process.crawl(AsyncioReactorSpider)+process.start()diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py b/tests/CrawlerProcess/twisted_reactor_custom_settings_same.pynew file mode 100644index 000000000..1f5a44010--- /dev/null+++ b/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py@@ -0,0 +1,21 @@+import scrapy+from scrapy.crawler import CrawlerProcess+++class AsyncioReactorSpider1(scrapy.Spider):+ name = 'asyncio_reactor1'+ custom_settings = {+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",+ }++class AsyncioReactorSpider2(scrapy.Spider):+ name = 'asyncio_reactor2'+ custom_settings = {+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",+ }+++process = CrawlerProcess()+process.crawl(AsyncioReactorSpider1)+process.crawl(AsyncioReactorSpider2)+process.start()diff --git a/tests/test_crawler.py b/tests/test_crawler.pyindex f445c181e..6d6763aec 100644--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@ -361,6 +361,30 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):self.assertIn("Spider closed (finished)", log)self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)+ # https://twistedmatrix.com/trac/ticket/9766+ @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),+ "the asyncio reactor is broken on Windows when running Python ≥ 3.8")+ def test_reactor_asyncio_custom_settings(self):+ log = self.run_script("twisted_reactor_custom_settings.py")+ self.assertIn("Spider closed (finished)", log)+ self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)++ # https://twistedmatrix.com/trac/ticket/9766+ @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),+ "the asyncio reactor is broken on Windows when running Python ≥ 3.8")+ def test_reactor_asyncio_custom_settings_same(self):+ log = self.run_script("twisted_reactor_custom_settings_same.py")+ self.assertIn("Spider closed (finished)", log)+ self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)++ # https://twistedmatrix.com/trac/ticket/9766+ @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),+ "the asyncio reactor is broken on Windows when running Python ≥ 3.8")+ def test_reactor_asyncio_custom_settings_conflict(self):+ log = self.run_script("twisted_reactor_custom_settings_conflict.py")+ self.assertIn("Using reactor: twisted.internet.pollreactor.PollReactor", log)+ self.assertIn("(twisted.internet.pollreactor.PollReactor) does not match the requested one", log)+@mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly')@mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows')@mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106')