Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

If TWISTED_REACTOR is None, reuse any pre-installed reactor (#5528)

Adrián Chaves4ef71829b22b
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 4e105642d..f3b28c4c4 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1638,9 +1638,10 @@ which raises :exc:`Exception`, becomes::
The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which
-means that Scrapy will install the default reactor defined by Twisted for the
-current platform. This is to maintain backward compatibility and avoid possible
-problems caused by using a non-default reactor.
+means that Scrapy will use the existing reactor if one is already installed, or
+install the default reactor defined by Twisted for the current platform. This
+is to maintain backward compatibility and avoid possible problems caused by
+using a non-default reactor.
For additional information, see :doc:`core/howto/choosing-reactor`.
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index d669d93a8..dcf0c2146 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -78,8 +78,7 @@ class Crawler:
if reactor_class:
install_reactor(reactor_class, self.settings["ASYNCIO_EVENT_LOOP"])
else:
- from twisted.internet import default
- default.install()
+ from twisted.internet import reactor # noqa: F401
log_reactor_info()
if reactor_class:
verify_installed_reactor(reactor_class)
diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
index 96395543c..bc543b230 100644
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -83,7 +83,7 @@ def verify_installed_reactor(reactor_path):
path."""
from twisted.internet import reactor
reactor_class = load_object(reactor_path)
- if not isinstance(reactor, reactor_class):
+ if not reactor.__class__ == reactor_class:
msg = ("The installed reactor "
f"({reactor.__module__}.{reactor.__class__.__name__}) does not "
f"match the requested one ({reactor_path})")
diff --git a/tests/CrawlerProcess/reactor_default.py b/tests/CrawlerProcess/reactor_default.py
new file mode 100644
index 000000000..5a21a3717
--- /dev/null
+++ b/tests/CrawlerProcess/reactor_default.py
@@ -0,0 +1,17 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+from twisted.internet import reactor
+
+
+class NoRequestsSpider(scrapy.Spider):
+ name = 'no_request'
+
+ def start_requests(self):
+ return []
+
+
+process = CrawlerProcess(settings={})
+
+process.crawl(NoRequestsSpider)
+process.start()
+
diff --git a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py
new file mode 100644
index 000000000..c476722ef
--- /dev/null
+++ b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py
@@ -0,0 +1,20 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+from twisted.internet import reactor
+
+
+class NoRequestsSpider(scrapy.Spider):
+ name = 'no_request'
+
+ def start_requests(self):
+ return []
+
+
+process = CrawlerProcess(settings={
+ "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor",
+})
+
+process.crawl(NoRequestsSpider)
+process.start()
+
+
diff --git a/tests/CrawlerProcess/reactor_select.py b/tests/CrawlerProcess/reactor_select.py
new file mode 100644
index 000000000..eac6e2f89
--- /dev/null
+++ b/tests/CrawlerProcess/reactor_select.py
@@ -0,0 +1,19 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+from twisted.internet import selectreactor
+selectreactor.install()
+
+
+class NoRequestsSpider(scrapy.Spider):
+ name = 'no_request'
+
+ def start_requests(self):
+ return []
+
+
+process = CrawlerProcess(settings={})
+
+process.crawl(NoRequestsSpider)
+process.start()
+
+
diff --git a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py
new file mode 100644
index 000000000..47f480605
--- /dev/null
+++ b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py
@@ -0,0 +1,31 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+from twisted.internet.main import installReactor
+from twisted.internet.selectreactor import SelectReactor
+
+
+class SelectReactorSubclass(SelectReactor):
+ pass
+
+
+reactor = SelectReactorSubclass()
+installReactor(reactor)
+
+
+class NoRequestsSpider(scrapy.Spider):
+ name = 'no_request'
+
+ def start_requests(self):
+ return []
+
+
+process = CrawlerProcess(settings={
+ "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor",
+})
+
+process.crawl(NoRequestsSpider)
+process.start()
+
+
+
+
diff --git a/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py
new file mode 100644
index 000000000..e0d2dab26
--- /dev/null
+++ b/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py
@@ -0,0 +1,22 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+from twisted.internet import selectreactor
+selectreactor.install()
+
+
+class NoRequestsSpider(scrapy.Spider):
+ name = 'no_request'
+
+ def start_requests(self):
+ return []
+
+
+process = CrawlerProcess(settings={
+ "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor",
+})
+
+process.crawl(NoRequestsSpider)
+process.start()
+
+
+
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 957525382..1ff2e8a67 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -308,6 +308,57 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
self.assertNotIn("ReactorAlreadyInstalledError", log)
+ def test_reactor_default(self):
+ log = self.run_script('reactor_default.py')
+ self.assertIn('Spider closed (finished)', log)
+ self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
+ self.assertNotIn("ReactorAlreadyInstalledError", log)
+
+ def test_reactor_default_twisted_reactor_select(self):
+ log = self.run_script('reactor_default_twisted_reactor_select.py')
+ if platform.system() == 'Windows':
+ # The goal of this test function is to test that, when a reactor is
+ # installed (the default one here) and a different reactor is
+ # configured (select here), an error raises.
+ #
+ # In Windows the default reactor is the select reactor, so that
+ # error does not raise.
+ #
+ # If that ever becomes the case on more platforms (i.e. if Linux
+ # also starts using the select reactor by default in a future
+ # version of Twisted), then we will need to rethink this test.
+ self.assertIn('Spider closed (finished)', log)
+ else:
+ self.assertNotIn('Spider closed (finished)', log)
+ self.assertIn(
+ (
+ "does not match the requested one "
+ "(twisted.internet.selectreactor.SelectReactor)"
+ ),
+ log,
+ )
+
+ def test_reactor_select(self):
+ log = self.run_script('reactor_select.py')
+ self.assertIn('Spider closed (finished)', log)
+ self.assertNotIn("ReactorAlreadyInstalledError", log)
+
+ def test_reactor_select_twisted_reactor_select(self):
+ log = self.run_script('reactor_select_twisted_reactor_select.py')
+ self.assertIn('Spider closed (finished)', log)
+ self.assertNotIn("ReactorAlreadyInstalledError", log)
+
+ def test_reactor_select_subclass_twisted_reactor_select(self):
+ log = self.run_script('reactor_select_subclass_twisted_reactor_select.py')
+ self.assertNotIn('Spider closed (finished)', log)
+ self.assertIn(
+ (
+ "does not match the requested one "
+ "(twisted.internet.selectreactor.SelectReactor)"
+ ),
+ log,
+ )
+
def test_asyncio_enabled_no_reactor(self):
log = self.run_script('asyncio_enabled_no_reactor.py')
self.assertIn('Spider closed (finished)', log)
@@ -340,33 +391,33 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
self.assertNotIn("TimeoutError", log)
self.assertNotIn("twisted.internet.error.DNSLookupError", log)
- def test_reactor_select(self):
+ def test_twisted_reactor_select(self):
log = self.run_script("twisted_reactor_select.py")
self.assertIn("Spider closed (finished)", log)
self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log)
@mark.skipif(platform.system() == 'Windows', reason="PollReactor is not supported on Windows")
- def test_reactor_poll(self):
+ def test_twisted_reactor_poll(self):
log = self.run_script("twisted_reactor_poll.py")
self.assertIn("Spider closed (finished)", log)
self.assertIn("Using reactor: twisted.internet.pollreactor.PollReactor", log)
- def test_reactor_asyncio(self):
+ def test_twisted_reactor_asyncio(self):
log = self.run_script("twisted_reactor_asyncio.py")
self.assertIn("Spider closed (finished)", log)
self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
- def test_reactor_asyncio_custom_settings(self):
+ def test_twisted_reactor_asyncio_custom_settings(self):
log = self.run_script("twisted_reactor_custom_settings.py")
self.assertIn("Spider closed (finished)", log)
self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
- def test_reactor_asyncio_custom_settings_same(self):
+ def test_twisted_reactor_asyncio_custom_settings_same(self):
log = self.run_script("twisted_reactor_custom_settings_same.py")
self.assertIn("Spider closed (finished)", log)
self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
- def test_reactor_asyncio_custom_settings_conflict(self):
+ def test_twisted_reactor_asyncio_custom_settings_conflict(self):
log = self.run_script("twisted_reactor_custom_settings_conflict.py")
self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log)
self.assertIn("(twisted.internet.selectreactor.SelectReactor) does not match the requested one", log)