Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Specify Twisted reactor (TWISTED_REACTOR setting) (#4294)

elacuesta35dafef7f106
diff --git a/docs/faq.rst b/docs/faq.rst
index b65908012..f72e4cf01 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -361,6 +361,17 @@ Note that by doing so, you lose the ability to set a specific timeout for DNS re
(the value of the :setting:`DNS_TIMEOUT` setting is ignored).
+.. _faq-specific-reactor:
+
+How to deal with ``<class 'ValueError'>: filedescriptor out of range in select()`` exceptions?
+----------------------------------------------------------------------------------------------
+
+This issue `has been reported`_ to appear when running broad crawls in macOS, where the default
+Twisted reactor is :class:`twisted.internet.selectreactor.SelectReactor`. Switching to a
+different reactor is possible by using the :setting:`TWISTED_REACTOR` setting.
+
+
+.. _has been reported: https://github.com/scrapy/scrapy/issues/2905
.. _user agents: https://en.wikipedia.org/wiki/User_agent
.. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type)
.. _DFO order: https://en.wikipedia.org/wiki/Depth-first_search
diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst
index 1ab08d949..4922694ee 100644
--- a/docs/topics/broad-crawls.rst
+++ b/docs/topics/broad-crawls.rst
@@ -211,3 +211,10 @@ If your broad crawl shows a high memory usage, in addition to :ref:`crawling in
BFO order <broad-crawls-bfo>` and :ref:`lowering concurrency
<broad-crawls-concurrency>` you should :ref:`debug your memory leaks
<topics-leaks>`.
+
+
+Install a specific Twisted reactor
+==================================
+
+If the crawl is exceeding the system's capabilities, you might want to try
+installing a specific Twisted reactor, via the :setting:`TWISTED_REACTOR` setting.
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 4b770d249..fa63a5807 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -160,27 +160,6 @@ to any particular component. In that case the module of that component will be
shown, typically an extension, middleware or pipeline. It also means that the
component must be enabled in order for the setting to have any effect.
-.. setting:: ASYNCIO_REACTOR
-
-ASYNCIO_REACTOR
----------------
-
-Default: ``False``
-
-Whether to install and require the Twisted reactor that uses the asyncio loop.
-
-When this option is set to ``True``, Scrapy will require
-:class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`. It will
-install this reactor if no reactor is installed yet, such as when using the
-``scrapy`` script or :class:`~scrapy.crawler.CrawlerProcess`. If you are using
-:class:`~scrapy.crawler.CrawlerRunner`, you need to install the correct reactor
-manually. If a different reactor is installed outside Scrapy, it will raise an
-exception.
-
-The default value for this option is currently ``False`` to maintain backward
-compatibility and avoid possible problems caused by using a different Twisted
-reactor.
-
.. setting:: AWS_ACCESS_KEY_ID
AWS_ACCESS_KEY_ID
@@ -1463,6 +1442,30 @@ command.
The project name must not conflict with the name of custom files or directories
in the ``project`` subdirectory.
+.. setting:: TWISTED_REACTOR
+
+TWISTED_REACTOR
+---------------
+
+Default: ``None``
+
+Import path of a given Twisted reactor, for instance:
+:class:`twisted.internet.asyncioreactor.AsyncioSelectorReactor`.
+
+Scrapy will install this reactor if no other is installed yet, such as when
+the ``scrapy`` CLI program is invoked or when using the
+:class:`~scrapy.crawler.CrawlerProcess` class. If you are using the
+:class:`~scrapy.crawler.CrawlerRunner` class, you need to install the correct
+reactor manually. An exception will be raised if the installation fails.
+
+The default value for this option is currently ``None``, which means that Scrapy
+will not attempt to install any specific reactor, and the default one defined by
+Twisted for the current platform will be used. This is to maintain backward
+compatibility and avoid possible problems caused by using a non-default reactor.
+
+For additional information, please see
+:doc:`core/howto/choosing-reactor`.
+
.. setting:: URLLENGTH_LIMIT
diff --git a/pytest.ini b/pytest.ini
index bae68cd3a..552829d4e 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -21,6 +21,7 @@ twisted = 1
markers =
only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
flake8-ignore =
+ W503
# Files that are only meant to provide top-level imports are expected not
# to use any of their imports:
scrapy/core/downloader/handlers/http.py F401
@@ -109,7 +110,7 @@ flake8-ignore =
# scrapy/spidermiddlewares
scrapy/spidermiddlewares/httperror.py E501
scrapy/spidermiddlewares/offsite.py E501
- scrapy/spidermiddlewares/referer.py E501 E129 W503 W504
+ scrapy/spidermiddlewares/referer.py E501 E129 W504
scrapy/spidermiddlewares/urllength.py E501
# scrapy/spiders
scrapy/spiders/__init__.py E501 E402
@@ -129,13 +130,13 @@ flake8-ignore =
scrapy/utils/http.py F403 E226
scrapy/utils/httpobj.py E501
scrapy/utils/iterators.py E501 E701
- scrapy/utils/log.py E128 W503
+ scrapy/utils/log.py E128 E501
scrapy/utils/markup.py F403
scrapy/utils/misc.py E501 E226
scrapy/utils/multipart.py F403
scrapy/utils/project.py E501
scrapy/utils/python.py E501
- scrapy/utils/reactor.py E226
+ scrapy/utils/reactor.py E226 E501
scrapy/utils/reqser.py E501
scrapy/utils/request.py E127 E501
scrapy/utils/response.py E501 E128
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 35c6b7716..49b8e4511 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -13,7 +13,6 @@ from scrapy.extension import ExtensionManager
from scrapy.interfaces import ISpiderLoader
from scrapy.settings import overridden_settings, Settings
from scrapy.signalmanager import SignalManager
-from scrapy.utils.asyncio import install_asyncio_reactor, is_asyncio_reactor_installed
from scrapy.utils.log import (
configure_logging,
get_scrapy_root_handler,
@@ -23,6 +22,7 @@ from scrapy.utils.log import (
)
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
+from scrapy.utils.reactor import install_reactor, verify_installed_reactor
logger = logging.getLogger(__name__)
@@ -138,7 +138,7 @@ class CrawlerRunner:
self._crawlers = set()
self._active = set()
self.bootstrap_failed = False
- self._handle_asyncio_reactor()
+ self._handle_twisted_reactor()
@property
def spiders(self):
@@ -232,10 +232,9 @@ class CrawlerRunner:
while self._active:
yield defer.DeferredList(self._active)
- def _handle_asyncio_reactor(self):
- if self.settings.getbool('ASYNCIO_REACTOR') and not is_asyncio_reactor_installed():
- raise Exception("ASYNCIO_REACTOR is on but the Twisted asyncio "
- "reactor is not installed.")
+ def _handle_twisted_reactor(self):
+ if self.settings.get("TWISTED_REACTOR"):
+ verify_installed_reactor(self.settings["TWISTED_REACTOR"])
class CrawlerProcess(CrawlerRunner):
@@ -324,10 +323,10 @@ class CrawlerProcess(CrawlerRunner):
except RuntimeError: # raised if already stopped or in shutdown stage
pass
- def _handle_asyncio_reactor(self):
- if self.settings.getbool('ASYNCIO_REACTOR'):
- install_asyncio_reactor()
- super()._handle_asyncio_reactor()
+ def _handle_twisted_reactor(self):
+ if self.settings.get("TWISTED_REACTOR"):
+ install_reactor(self.settings["TWISTED_REACTOR"])
+ super()._handle_twisted_reactor()
def _get_spider_loader(settings):
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index c10dc1a1c..c80833710 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -19,8 +19,6 @@ from os.path import join, abspath, dirname
AJAXCRAWL_ENABLED = False
-ASYNCIO_REACTOR = False
-
AUTOTHROTTLE_ENABLED = False
AUTOTHROTTLE_DEBUG = False
AUTOTHROTTLE_MAX_DELAY = 60.0
@@ -291,6 +289,8 @@ TELNETCONSOLE_HOST = '127.0.0.1'
TELNETCONSOLE_USERNAME = 'scrapy'
TELNETCONSOLE_PASSWORD = None
+TWISTED_REACTOR = None
+
SPIDER_CONTRACTS = {}
SPIDER_CONTRACTS_BASE = {
'scrapy.contracts.default.UrlContract': 1,
diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.py
deleted file mode 100644
index 917973de2..000000000
--- a/scrapy/utils/asyncio.py
+++ /dev/null
@@ -1,17 +0,0 @@
-import asyncio
-from contextlib import suppress
-
-from twisted.internet import asyncioreactor
-from twisted.internet.error import ReactorAlreadyInstalledError
-
-
-def install_asyncio_reactor():
- """ Tries to install AsyncioSelectorReactor
- """
- with suppress(ReactorAlreadyInstalledError):
- asyncioreactor.install(asyncio.get_event_loop())
-
-
-def is_asyncio_reactor_installed():
- from twisted.internet import reactor
- return isinstance(reactor, asyncioreactor.AsyncioSelectorReactor)
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 62b43a96c..6a21a4903 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -2,14 +2,14 @@
Helper functions for dealing with Twisted deferreds
"""
import asyncio
-from functools import wraps
import inspect
+from functools import wraps
from twisted.internet import defer, task
from twisted.python import failure
from scrapy.exceptions import IgnoreRequest
-from scrapy.utils.asyncio import is_asyncio_reactor_installed
+from scrapy.utils.reactor import is_asyncio_reactor_installed
def defer_fail(_failure):
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index e4cf0196b..afef2c93f 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -1,17 +1,16 @@
# -*- coding: utf-8 -*-
-import sys
import logging
+import sys
import warnings
from logging.config import dictConfig
-from twisted.python.failure import Failure
from twisted.python import log as twisted_log
+from twisted.python.failure import Failure
import scrapy
-from scrapy.settings import Settings
from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.utils.asyncio import is_asyncio_reactor_installed
+from scrapy.settings import Settings
from scrapy.utils.versions import scrapy_components_versions
@@ -149,8 +148,8 @@ def log_scrapy_info(settings):
{'versions': ", ".join("%s %s" % (name, version)
for name, version in scrapy_components_versions()
if name != "Scrapy")})
- if is_asyncio_reactor_installed():
- logger.debug("Asyncio reactor is installed")
+ from twisted.internet import reactor
+ logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__)
class StreamLogger(object):
diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
index b98fff6ec..80f52a4ef 100644
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -1,4 +1,9 @@
-from twisted.internet import error
+import asyncio
+from contextlib import suppress
+
+from twisted.internet import asyncioreactor, error
+
+from scrapy.utils.misc import load_object
def listen_tcp(portrange, host, factory):
@@ -42,3 +47,31 @@ class CallLaterOnce(object):
def __call__(self):
self._call = None
return self._func(*self._a, **self._kw)
+
+
+def install_reactor(reactor_path):
+ reactor_class = load_object(reactor_path)
+ if reactor_class is asyncioreactor.AsyncioSelectorReactor:
+ with suppress(error.ReactorAlreadyInstalledError):
+ asyncioreactor.install(asyncio.get_event_loop())
+ else:
+ *module, _ = reactor_path.split(".")
+ installer_path = module + ["install"]
+ installer = load_object(".".join(installer_path))
+ with suppress(error.ReactorAlreadyInstalledError):
+ installer()
+
+
+def verify_installed_reactor(reactor_path):
+ from twisted.internet import reactor
+ reactor_class = load_object(reactor_path)
+ if not isinstance(reactor, reactor_class):
+ msg = "The installed reactor ({}.{}) does not match the requested one ({})".format(
+ reactor.__module__, reactor.__class__.__name__, reactor_path
+ )
+ raise Exception(msg)
+
+
+def is_asyncio_reactor_installed():
+ from twisted.internet import reactor
+ return isinstance(reactor, asyncioreactor.AsyncioSelectorReactor)
diff --git a/tests/CrawlerProcess/asyncio_enabled_no_reactor.py b/tests/CrawlerProcess/asyncio_enabled_no_reactor.py
index db1b75931..d1e4a7bb5 100644
--- a/tests/CrawlerProcess/asyncio_enabled_no_reactor.py
+++ b/tests/CrawlerProcess/asyncio_enabled_no_reactor.py
@@ -10,8 +10,7 @@ class NoRequestsSpider(scrapy.Spider):
process = CrawlerProcess(settings={
- 'ASYNCIO_REACTOR': True,
+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
})
-
process.crawl(NoRequestsSpider)
process.start()
diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.py
index cec3c9c25..8568bd8b8 100644
--- a/tests/CrawlerProcess/asyncio_enabled_reactor.py
+++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py
@@ -15,8 +15,7 @@ class NoRequestsSpider(scrapy.Spider):
process = CrawlerProcess(settings={
- 'ASYNCIO_REACTOR': True,
+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
})
-
process.crawl(NoRequestsSpider)
process.start()
diff --git a/tests/CrawlerProcess/twisted_reactor_asyncio.py b/tests/CrawlerProcess/twisted_reactor_asyncio.py
new file mode 100644
index 000000000..c6cbf949b
--- /dev/null
+++ b/tests/CrawlerProcess/twisted_reactor_asyncio.py
@@ -0,0 +1,13 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+
+class AsyncioReactorSpider(scrapy.Spider):
+ name = 'asyncio_reactor'
+
+
+process = CrawlerProcess(settings={
+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+})
+process.crawl(AsyncioReactorSpider)
+process.start()
diff --git a/tests/CrawlerProcess/twisted_reactor_poll.py b/tests/CrawlerProcess/twisted_reactor_poll.py
new file mode 100644
index 000000000..27063260b
--- /dev/null
+++ b/tests/CrawlerProcess/twisted_reactor_poll.py
@@ -0,0 +1,13 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+
+class PollReactorSpider(scrapy.Spider):
+ name = 'poll_reactor'
+
+
+process = CrawlerProcess(settings={
+ "TWISTED_REACTOR": "twisted.internet.pollreactor.PollReactor",
+})
+process.crawl(PollReactorSpider)
+process.start()
diff --git a/tests/CrawlerProcess/twisted_reactor_select.py b/tests/CrawlerProcess/twisted_reactor_select.py
new file mode 100644
index 000000000..9af8ceb4d
--- /dev/null
+++ b/tests/CrawlerProcess/twisted_reactor_select.py
@@ -0,0 +1,13 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+
+class SelectReactorSpider(scrapy.Spider):
+ name = 'epoll_reactor'
+
+
+process = CrawlerProcess(settings={
+ "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor",
+})
+process.crawl(SelectReactorSpider)
+process.start()
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 6024af71c..3612b70c9 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -296,12 +296,14 @@ class BadSpider(scrapy.Spider):
self.assertIn("badspider.py", log)
def test_asyncio_enabled_true(self):
- log = self.get_log(self.debug_log_spider, args=['-s', 'ASYNCIO_REACTOR=True'])
- self.assertIn("DEBUG: Asyncio reactor is installed", log)
+ log = self.get_log(self.debug_log_spider, args=[
+ '-s', 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor'
+ ])
+ self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
def test_asyncio_enabled_false(self):
- log = self.get_log(self.debug_log_spider, args=['-s', 'ASYNCIO_REACTOR=False'])
- self.assertNotIn("DEBUG: Asyncio reactor is installed", log)
+ log = self.get_log(self.debug_log_spider, args=[])
+ self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
class BenchCommandTest(CommandTest):
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 0ce0674de..f8fa26def 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -254,30 +254,38 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
def test_crawler_runner_asyncio_enabled_true(self):
if self.reactor_pytest == 'asyncio':
- runner = CrawlerRunner(settings={'ASYNCIO_REACTOR': True})
+ runner = CrawlerRunner(settings={
+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+ })
else:
- msg = "ASYNCIO_REACTOR is on but the Twisted asyncio reactor is not installed"
+ msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)"
with self.assertRaisesRegex(Exception, msg):
- runner = CrawlerRunner(settings={'ASYNCIO_REACTOR': True})
+ runner = CrawlerRunner(settings={
+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+ })
@defer.inlineCallbacks
def test_crawler_process_asyncio_enabled_true(self):
with LogCapture(level=logging.DEBUG) as log:
if self.reactor_pytest == 'asyncio':
- runner = CrawlerProcess(settings={'ASYNCIO_REACTOR': True})
+ runner = CrawlerProcess(settings={
+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+ })
yield runner.crawl(NoRequestsSpider)
- self.assertIn("Asyncio reactor is installed", str(log))
+ self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", str(log))
else:
- msg = "ASYNCIO_REACTOR is on but the Twisted asyncio reactor is not installed"
+ msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)"
with self.assertRaisesRegex(Exception, msg):
- runner = CrawlerProcess(settings={'ASYNCIO_REACTOR': True})
+ runner = CrawlerProcess(settings={
+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+ })
@defer.inlineCallbacks
def test_crawler_process_asyncio_enabled_false(self):
- runner = CrawlerProcess(settings={'ASYNCIO_REACTOR': False})
+ runner = CrawlerProcess(settings={"TWISTED_REACTOR": None})
with LogCapture(level=logging.DEBUG) as log:
yield runner.crawl(NoRequestsSpider)
- self.assertNotIn("Asyncio reactor is installed", str(log))
+ self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", str(log))
class CrawlerProcessSubprocess(unittest.TestCase):
@@ -294,17 +302,17 @@ class CrawlerProcessSubprocess(unittest.TestCase):
def test_simple(self):
log = self.run_script('simple.py')
self.assertIn('Spider closed (finished)', log)
- self.assertNotIn("DEBUG: Asyncio reactor is installed", log)
+ self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
def test_asyncio_enabled_no_reactor(self):
log = self.run_script('asyncio_enabled_no_reactor.py')
self.assertIn('Spider closed (finished)', log)
- self.assertIn("DEBUG: Asyncio reactor is installed", log)
+ self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
def test_asyncio_enabled_reactor(self):
log = self.run_script('asyncio_enabled_reactor.py')
self.assertIn('Spider closed (finished)', log)
- self.assertIn("DEBUG: Asyncio reactor is installed", log)
+ self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
def test_ipv6_default_name_resolver(self):
log = self.run_script('default_name_resolver.py')
@@ -323,3 +331,18 @@ class CrawlerProcessSubprocess(unittest.TestCase):
"'downloader/exception_type_count/twisted.internet.error.ConnectionRefusedError': 1," in log,
"'downloader/exception_type_count/twisted.internet.error.ConnectError': 1," in log,
]))
+
+ def test_reactor_select(self):
+ log = self.run_script("twisted_reactor_select.py")
+ self.assertIn("Spider closed (finished)", log)
+ self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log)
+
+ def test_reactor_poll(self):
+ log = self.run_script("twisted_reactor_poll.py")
+ self.assertIn("Spider closed (finished)", log)
+ self.assertIn("Using reactor: twisted.internet.pollreactor.PollReactor", log)
+
+ def test_reactor_asyncio(self):
+ log = self.run_script("twisted_reactor_asyncio.py")
+ self.assertIn("Spider closed (finished)", log)
+ self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py
index 44acc24af..295323e4d 100644
--- a/tests/test_utils_asyncio.py
+++ b/tests/test_utils_asyncio.py
@@ -2,7 +2,7 @@ from unittest import TestCase
from pytest import mark
-from scrapy.utils.asyncio import is_asyncio_reactor_installed, install_asyncio_reactor
+from scrapy.utils.reactor import is_asyncio_reactor_installed, install_reactor
@mark.usefixtures('reactor_pytest')
@@ -14,4 +14,4 @@ class AsyncioTest(TestCase):
def test_install_asyncio_reactor(self):
# this should do nothing
- install_asyncio_reactor()
+ install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")