Scrapy, a fast high-level web crawling & scraping framework for Python.
If TWISTED_REACTOR is None, reuse any pre-installed reactor (#5528)
Adrián Chaves4ef71829b22b
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex 4e105642d..f3b28c4c4 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -1638,9 +1638,10 @@ which raises :exc:`Exception`, becomes::The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which-means that Scrapy will install the default reactor defined by Twisted for the-current platform. This is to maintain backward compatibility and avoid possible-problems caused by using a non-default reactor.+means that Scrapy will use the existing reactor if one is already installed, or+install the default reactor defined by Twisted for the current platform. This+is to maintain backward compatibility and avoid possible problems caused by+using a non-default reactor.For additional information, see :doc:`core/howto/choosing-reactor`.diff --git a/scrapy/crawler.py b/scrapy/crawler.pyindex d669d93a8..dcf0c2146 100644--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@ -78,8 +78,7 @@ class Crawler:if reactor_class:install_reactor(reactor_class, self.settings["ASYNCIO_EVENT_LOOP"])else:- from twisted.internet import default- default.install()+ from twisted.internet import reactor # noqa: F401log_reactor_info()if reactor_class:verify_installed_reactor(reactor_class)diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.pyindex 96395543c..bc543b230 100644--- a/scrapy/utils/reactor.py+++ b/scrapy/utils/reactor.py@@ -83,7 +83,7 @@ def verify_installed_reactor(reactor_path):path."""from twisted.internet import reactorreactor_class = load_object(reactor_path)- if not isinstance(reactor, reactor_class):+ if not reactor.__class__ == reactor_class:msg = ("The installed reactor "f"({reactor.__module__}.{reactor.__class__.__name__}) does not "f"match the requested one ({reactor_path})")diff --git a/tests/CrawlerProcess/reactor_default.py b/tests/CrawlerProcess/reactor_default.pynew file mode 100644index 000000000..5a21a3717--- /dev/null+++ b/tests/CrawlerProcess/reactor_default.py@@ -0,0 +1,17 @@+import scrapy+from scrapy.crawler import CrawlerProcess+from twisted.internet import reactor+++class NoRequestsSpider(scrapy.Spider):+ name = 'no_request'++ def start_requests(self):+ return []+++process = CrawlerProcess(settings={})++process.crawl(NoRequestsSpider)+process.start()+diff --git a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.pynew file mode 100644index 000000000..c476722ef--- /dev/null+++ b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py@@ -0,0 +1,20 @@+import scrapy+from scrapy.crawler import CrawlerProcess+from twisted.internet import reactor+++class NoRequestsSpider(scrapy.Spider):+ name = 'no_request'++ def start_requests(self):+ return []+++process = CrawlerProcess(settings={+ "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor",+})++process.crawl(NoRequestsSpider)+process.start()++diff --git a/tests/CrawlerProcess/reactor_select.py b/tests/CrawlerProcess/reactor_select.pynew file mode 100644index 000000000..eac6e2f89--- /dev/null+++ b/tests/CrawlerProcess/reactor_select.py@@ -0,0 +1,19 @@+import scrapy+from scrapy.crawler import CrawlerProcess+from twisted.internet import selectreactor+selectreactor.install()+++class NoRequestsSpider(scrapy.Spider):+ name = 'no_request'++ def start_requests(self):+ return []+++process = CrawlerProcess(settings={})++process.crawl(NoRequestsSpider)+process.start()++diff --git a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.pynew file mode 100644index 000000000..47f480605--- /dev/null+++ b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py@@ -0,0 +1,31 @@+import scrapy+from scrapy.crawler import CrawlerProcess+from twisted.internet.main import installReactor+from twisted.internet.selectreactor import SelectReactor+++class SelectReactorSubclass(SelectReactor):+ pass+++reactor = SelectReactorSubclass()+installReactor(reactor)+++class NoRequestsSpider(scrapy.Spider):+ name = 'no_request'++ def start_requests(self):+ return []+++process = CrawlerProcess(settings={+ "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor",+})++process.crawl(NoRequestsSpider)+process.start()++++diff --git a/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_twisted_reactor_select.pynew file mode 100644index 000000000..e0d2dab26--- /dev/null+++ b/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py@@ -0,0 +1,22 @@+import scrapy+from scrapy.crawler import CrawlerProcess+from twisted.internet import selectreactor+selectreactor.install()+++class NoRequestsSpider(scrapy.Spider):+ name = 'no_request'++ def start_requests(self):+ return []+++process = CrawlerProcess(settings={+ "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor",+})++process.crawl(NoRequestsSpider)+process.start()+++diff --git a/tests/test_crawler.py b/tests/test_crawler.pyindex 957525382..1ff2e8a67 100644--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@ -308,6 +308,57 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)self.assertNotIn("ReactorAlreadyInstalledError", log)+ def test_reactor_default(self):+ log = self.run_script('reactor_default.py')+ self.assertIn('Spider closed (finished)', log)+ self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)+ self.assertNotIn("ReactorAlreadyInstalledError", log)++ def test_reactor_default_twisted_reactor_select(self):+ log = self.run_script('reactor_default_twisted_reactor_select.py')+ if platform.system() == 'Windows':+ # The goal of this test function is to test that, when a reactor is+ # installed (the default one here) and a different reactor is+ # configured (select here), an error raises.+ #+ # In Windows the default reactor is the select reactor, so that+ # error does not raise.+ #+ # If that ever becomes the case on more platforms (i.e. if Linux+ # also starts using the select reactor by default in a future+ # version of Twisted), then we will need to rethink this test.+ self.assertIn('Spider closed (finished)', log)+ else:+ self.assertNotIn('Spider closed (finished)', log)+ self.assertIn(+ (+ "does not match the requested one "+ "(twisted.internet.selectreactor.SelectReactor)"+ ),+ log,+ )++ def test_reactor_select(self):+ log = self.run_script('reactor_select.py')+ self.assertIn('Spider closed (finished)', log)+ self.assertNotIn("ReactorAlreadyInstalledError", log)++ def test_reactor_select_twisted_reactor_select(self):+ log = self.run_script('reactor_select_twisted_reactor_select.py')+ self.assertIn('Spider closed (finished)', log)+ self.assertNotIn("ReactorAlreadyInstalledError", log)++ def test_reactor_select_subclass_twisted_reactor_select(self):+ log = self.run_script('reactor_select_subclass_twisted_reactor_select.py')+ self.assertNotIn('Spider closed (finished)', log)+ self.assertIn(+ (+ "does not match the requested one "+ "(twisted.internet.selectreactor.SelectReactor)"+ ),+ log,+ )+def test_asyncio_enabled_no_reactor(self):log = self.run_script('asyncio_enabled_no_reactor.py')self.assertIn('Spider closed (finished)', log)@@ -340,33 +391,33 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):self.assertNotIn("TimeoutError", log)self.assertNotIn("twisted.internet.error.DNSLookupError", log)- def test_reactor_select(self):+ def test_twisted_reactor_select(self):log = self.run_script("twisted_reactor_select.py")self.assertIn("Spider closed (finished)", log)self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log)@mark.skipif(platform.system() == 'Windows', reason="PollReactor is not supported on Windows")- def test_reactor_poll(self):+ def test_twisted_reactor_poll(self):log = self.run_script("twisted_reactor_poll.py")self.assertIn("Spider closed (finished)", log)self.assertIn("Using reactor: twisted.internet.pollreactor.PollReactor", log)- def test_reactor_asyncio(self):+ def test_twisted_reactor_asyncio(self):log = self.run_script("twisted_reactor_asyncio.py")self.assertIn("Spider closed (finished)", log)self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)- def test_reactor_asyncio_custom_settings(self):+ def test_twisted_reactor_asyncio_custom_settings(self):log = self.run_script("twisted_reactor_custom_settings.py")self.assertIn("Spider closed (finished)", log)self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)- def test_reactor_asyncio_custom_settings_same(self):+ def test_twisted_reactor_asyncio_custom_settings_same(self):log = self.run_script("twisted_reactor_custom_settings_same.py")self.assertIn("Spider closed (finished)", log)self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)- def test_reactor_asyncio_custom_settings_conflict(self):+ def test_twisted_reactor_asyncio_custom_settings_conflict(self):log = self.run_script("twisted_reactor_custom_settings_conflict.py")self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log)self.assertIn("(twisted.internet.selectreactor.SelectReactor) does not match the requested one", log)