Scrapy, a fast high-level web crawling & scraping framework for Python.
Specify Twisted reactor (TWISTED_REACTOR setting) (#4294)
elacuesta35dafef7f106
diff --git a/docs/faq.rst b/docs/faq.rstindex b65908012..f72e4cf01 100644--- a/docs/faq.rst+++ b/docs/faq.rst@@ -361,6 +361,17 @@ Note that by doing so, you lose the ability to set a specific timeout for DNS re(the value of the :setting:`DNS_TIMEOUT` setting is ignored).+.. _faq-specific-reactor:++How to deal with ``<class 'ValueError'>: filedescriptor out of range in select()`` exceptions?+----------------------------------------------------------------------------------------------++This issue `has been reported`_ to appear when running broad crawls in macOS, where the default+Twisted reactor is :class:`twisted.internet.selectreactor.SelectReactor`. Switching to a+different reactor is possible by using the :setting:`TWISTED_REACTOR` setting.+++.. _has been reported: https://github.com/scrapy/scrapy/issues/2905.. _user agents: https://en.wikipedia.org/wiki/User_agent.. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type).. _DFO order: https://en.wikipedia.org/wiki/Depth-first_searchdiff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rstindex 1ab08d949..4922694ee 100644--- a/docs/topics/broad-crawls.rst+++ b/docs/topics/broad-crawls.rst@@ -211,3 +211,10 @@ If your broad crawl shows a high memory usage, in addition to :ref:`crawling inBFO order <broad-crawls-bfo>` and :ref:`lowering concurrency<broad-crawls-concurrency>` you should :ref:`debug your memory leaks<topics-leaks>`.+++Install a specific Twisted reactor+==================================++If the crawl is exceeding the system's capabilities, you might want to try+installing a specific Twisted reactor, via the :setting:`TWISTED_REACTOR` setting.diff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex 4b770d249..fa63a5807 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -160,27 +160,6 @@ to any particular component. In that case the module of that component will beshown, typically an extension, middleware or pipeline. It also means that thecomponent must be enabled in order for the setting to have any effect.-.. setting:: ASYNCIO_REACTOR--ASYNCIO_REACTOR------------------Default: ``False``--Whether to install and require the Twisted reactor that uses the asyncio loop.--When this option is set to ``True``, Scrapy will require-:class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`. It will-install this reactor if no reactor is installed yet, such as when using the-``scrapy`` script or :class:`~scrapy.crawler.CrawlerProcess`. If you are using-:class:`~scrapy.crawler.CrawlerRunner`, you need to install the correct reactor-manually. If a different reactor is installed outside Scrapy, it will raise an-exception.--The default value for this option is currently ``False`` to maintain backward-compatibility and avoid possible problems caused by using a different Twisted-reactor.-.. setting:: AWS_ACCESS_KEY_IDAWS_ACCESS_KEY_ID@@ -1463,6 +1442,30 @@ command.The project name must not conflict with the name of custom files or directoriesin the ``project`` subdirectory.+.. setting:: TWISTED_REACTOR++TWISTED_REACTOR+---------------++Default: ``None``++Import path of a given Twisted reactor, for instance:+:class:`twisted.internet.asyncioreactor.AsyncioSelectorReactor`.++Scrapy will install this reactor if no other is installed yet, such as when+the ``scrapy`` CLI program is invoked or when using the+:class:`~scrapy.crawler.CrawlerProcess` class. If you are using the+:class:`~scrapy.crawler.CrawlerRunner` class, you need to install the correct+reactor manually. An exception will be raised if the installation fails.++The default value for this option is currently ``None``, which means that Scrapy+will not attempt to install any specific reactor, and the default one defined by+Twisted for the current platform will be used. This is to maintain backward+compatibility and avoid possible problems caused by using a non-default reactor.++For additional information, please see+:doc:`core/howto/choosing-reactor`.+.. setting:: URLLENGTH_LIMITdiff --git a/pytest.ini b/pytest.iniindex bae68cd3a..552829d4e 100644--- a/pytest.ini+++ b/pytest.ini@@ -21,6 +21,7 @@ twisted = 1markers =only_asyncio: marks tests as only enabled when --reactor=asyncio is passedflake8-ignore =+ W503# Files that are only meant to provide top-level imports are expected not# to use any of their imports:scrapy/core/downloader/handlers/http.py F401@@ -109,7 +110,7 @@ flake8-ignore =# scrapy/spidermiddlewaresscrapy/spidermiddlewares/httperror.py E501scrapy/spidermiddlewares/offsite.py E501- scrapy/spidermiddlewares/referer.py E501 E129 W503 W504+ scrapy/spidermiddlewares/referer.py E501 E129 W504scrapy/spidermiddlewares/urllength.py E501# scrapy/spidersscrapy/spiders/__init__.py E501 E402@@ -129,13 +130,13 @@ flake8-ignore =scrapy/utils/http.py F403 E226scrapy/utils/httpobj.py E501scrapy/utils/iterators.py E501 E701- scrapy/utils/log.py E128 W503+ scrapy/utils/log.py E128 E501scrapy/utils/markup.py F403scrapy/utils/misc.py E501 E226scrapy/utils/multipart.py F403scrapy/utils/project.py E501scrapy/utils/python.py E501- scrapy/utils/reactor.py E226+ scrapy/utils/reactor.py E226 E501scrapy/utils/reqser.py E501scrapy/utils/request.py E127 E501scrapy/utils/response.py E501 E128diff --git a/scrapy/crawler.py b/scrapy/crawler.pyindex 35c6b7716..49b8e4511 100644--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@ -13,7 +13,6 @@ from scrapy.extension import ExtensionManagerfrom scrapy.interfaces import ISpiderLoaderfrom scrapy.settings import overridden_settings, Settingsfrom scrapy.signalmanager import SignalManager-from scrapy.utils.asyncio import install_asyncio_reactor, is_asyncio_reactor_installedfrom scrapy.utils.log import (configure_logging,get_scrapy_root_handler,@@ -23,6 +22,7 @@ from scrapy.utils.log import ()from scrapy.utils.misc import create_instance, load_objectfrom scrapy.utils.ossignal import install_shutdown_handlers, signal_names+from scrapy.utils.reactor import install_reactor, verify_installed_reactorlogger = logging.getLogger(__name__)@@ -138,7 +138,7 @@ class CrawlerRunner:self._crawlers = set()self._active = set()self.bootstrap_failed = False- self._handle_asyncio_reactor()+ self._handle_twisted_reactor()@propertydef spiders(self):@@ -232,10 +232,9 @@ class CrawlerRunner:while self._active:yield defer.DeferredList(self._active)- def _handle_asyncio_reactor(self):- if self.settings.getbool('ASYNCIO_REACTOR') and not is_asyncio_reactor_installed():- raise Exception("ASYNCIO_REACTOR is on but the Twisted asyncio "- "reactor is not installed.")+ def _handle_twisted_reactor(self):+ if self.settings.get("TWISTED_REACTOR"):+ verify_installed_reactor(self.settings["TWISTED_REACTOR"])class CrawlerProcess(CrawlerRunner):@@ -324,10 +323,10 @@ class CrawlerProcess(CrawlerRunner):except RuntimeError: # raised if already stopped or in shutdown stagepass- def _handle_asyncio_reactor(self):- if self.settings.getbool('ASYNCIO_REACTOR'):- install_asyncio_reactor()- super()._handle_asyncio_reactor()+ def _handle_twisted_reactor(self):+ if self.settings.get("TWISTED_REACTOR"):+ install_reactor(self.settings["TWISTED_REACTOR"])+ super()._handle_twisted_reactor()def _get_spider_loader(settings):diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex c10dc1a1c..c80833710 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -19,8 +19,6 @@ from os.path import join, abspath, dirnameAJAXCRAWL_ENABLED = False-ASYNCIO_REACTOR = False-AUTOTHROTTLE_ENABLED = FalseAUTOTHROTTLE_DEBUG = FalseAUTOTHROTTLE_MAX_DELAY = 60.0@@ -291,6 +289,8 @@ TELNETCONSOLE_HOST = '127.0.0.1'TELNETCONSOLE_USERNAME = 'scrapy'TELNETCONSOLE_PASSWORD = None+TWISTED_REACTOR = None+SPIDER_CONTRACTS = {}SPIDER_CONTRACTS_BASE = {'scrapy.contracts.default.UrlContract': 1,diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.pydeleted file mode 100644index 917973de2..000000000--- a/scrapy/utils/asyncio.py+++ /dev/null@@ -1,17 +0,0 @@-import asyncio-from contextlib import suppress--from twisted.internet import asyncioreactor-from twisted.internet.error import ReactorAlreadyInstalledError---def install_asyncio_reactor():- """ Tries to install AsyncioSelectorReactor- """- with suppress(ReactorAlreadyInstalledError):- asyncioreactor.install(asyncio.get_event_loop())---def is_asyncio_reactor_installed():- from twisted.internet import reactor- return isinstance(reactor, asyncioreactor.AsyncioSelectorReactor)diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.pyindex 62b43a96c..6a21a4903 100644--- a/scrapy/utils/defer.py+++ b/scrapy/utils/defer.py@@ -2,14 +2,14 @@Helper functions for dealing with Twisted deferreds"""import asyncio-from functools import wrapsimport inspect+from functools import wrapsfrom twisted.internet import defer, taskfrom twisted.python import failurefrom scrapy.exceptions import IgnoreRequest-from scrapy.utils.asyncio import is_asyncio_reactor_installed+from scrapy.utils.reactor import is_asyncio_reactor_installeddef defer_fail(_failure):diff --git a/scrapy/utils/log.py b/scrapy/utils/log.pyindex e4cf0196b..afef2c93f 100644--- a/scrapy/utils/log.py+++ b/scrapy/utils/log.py@@ -1,17 +1,16 @@# -*- coding: utf-8 -*--import sysimport logging+import sysimport warningsfrom logging.config import dictConfig-from twisted.python.failure import Failurefrom twisted.python import log as twisted_log+from twisted.python.failure import Failureimport scrapy-from scrapy.settings import Settingsfrom scrapy.exceptions import ScrapyDeprecationWarning-from scrapy.utils.asyncio import is_asyncio_reactor_installed+from scrapy.settings import Settingsfrom scrapy.utils.versions import scrapy_components_versions@@ -149,8 +148,8 @@ def log_scrapy_info(settings):{'versions': ", ".join("%s %s" % (name, version)for name, version in scrapy_components_versions()if name != "Scrapy")})- if is_asyncio_reactor_installed():- logger.debug("Asyncio reactor is installed")+ from twisted.internet import reactor+ logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__)class StreamLogger(object):diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.pyindex b98fff6ec..80f52a4ef 100644--- a/scrapy/utils/reactor.py+++ b/scrapy/utils/reactor.py@@ -1,4 +1,9 @@-from twisted.internet import error+import asyncio+from contextlib import suppress++from twisted.internet import asyncioreactor, error++from scrapy.utils.misc import load_objectdef listen_tcp(portrange, host, factory):@@ -42,3 +47,31 @@ class CallLaterOnce(object):def __call__(self):self._call = Nonereturn self._func(*self._a, **self._kw)+++def install_reactor(reactor_path):+ reactor_class = load_object(reactor_path)+ if reactor_class is asyncioreactor.AsyncioSelectorReactor:+ with suppress(error.ReactorAlreadyInstalledError):+ asyncioreactor.install(asyncio.get_event_loop())+ else:+ *module, _ = reactor_path.split(".")+ installer_path = module + ["install"]+ installer = load_object(".".join(installer_path))+ with suppress(error.ReactorAlreadyInstalledError):+ installer()+++def verify_installed_reactor(reactor_path):+ from twisted.internet import reactor+ reactor_class = load_object(reactor_path)+ if not isinstance(reactor, reactor_class):+ msg = "The installed reactor ({}.{}) does not match the requested one ({})".format(+ reactor.__module__, reactor.__class__.__name__, reactor_path+ )+ raise Exception(msg)+++def is_asyncio_reactor_installed():+ from twisted.internet import reactor+ return isinstance(reactor, asyncioreactor.AsyncioSelectorReactor)diff --git a/tests/CrawlerProcess/asyncio_enabled_no_reactor.py b/tests/CrawlerProcess/asyncio_enabled_no_reactor.pyindex db1b75931..d1e4a7bb5 100644--- a/tests/CrawlerProcess/asyncio_enabled_no_reactor.py+++ b/tests/CrawlerProcess/asyncio_enabled_no_reactor.py@@ -10,8 +10,7 @@ class NoRequestsSpider(scrapy.Spider):process = CrawlerProcess(settings={- 'ASYNCIO_REACTOR': True,+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",})-process.crawl(NoRequestsSpider)process.start()diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.pyindex cec3c9c25..8568bd8b8 100644--- a/tests/CrawlerProcess/asyncio_enabled_reactor.py+++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py@@ -15,8 +15,7 @@ class NoRequestsSpider(scrapy.Spider):process = CrawlerProcess(settings={- 'ASYNCIO_REACTOR': True,+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",})-process.crawl(NoRequestsSpider)process.start()diff --git a/tests/CrawlerProcess/twisted_reactor_asyncio.py b/tests/CrawlerProcess/twisted_reactor_asyncio.pynew file mode 100644index 000000000..c6cbf949b--- /dev/null+++ b/tests/CrawlerProcess/twisted_reactor_asyncio.py@@ -0,0 +1,13 @@+import scrapy+from scrapy.crawler import CrawlerProcess+++class AsyncioReactorSpider(scrapy.Spider):+ name = 'asyncio_reactor'+++process = CrawlerProcess(settings={+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",+})+process.crawl(AsyncioReactorSpider)+process.start()diff --git a/tests/CrawlerProcess/twisted_reactor_poll.py b/tests/CrawlerProcess/twisted_reactor_poll.pynew file mode 100644index 000000000..27063260b--- /dev/null+++ b/tests/CrawlerProcess/twisted_reactor_poll.py@@ -0,0 +1,13 @@+import scrapy+from scrapy.crawler import CrawlerProcess+++class PollReactorSpider(scrapy.Spider):+ name = 'poll_reactor'+++process = CrawlerProcess(settings={+ "TWISTED_REACTOR": "twisted.internet.pollreactor.PollReactor",+})+process.crawl(PollReactorSpider)+process.start()diff --git a/tests/CrawlerProcess/twisted_reactor_select.py b/tests/CrawlerProcess/twisted_reactor_select.pynew file mode 100644index 000000000..9af8ceb4d--- /dev/null+++ b/tests/CrawlerProcess/twisted_reactor_select.py@@ -0,0 +1,13 @@+import scrapy+from scrapy.crawler import CrawlerProcess+++class SelectReactorSpider(scrapy.Spider):+ name = 'epoll_reactor'+++process = CrawlerProcess(settings={+ "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor",+})+process.crawl(SelectReactorSpider)+process.start()diff --git a/tests/test_commands.py b/tests/test_commands.pyindex 6024af71c..3612b70c9 100644--- a/tests/test_commands.py+++ b/tests/test_commands.py@@ -296,12 +296,14 @@ class BadSpider(scrapy.Spider):self.assertIn("badspider.py", log)def test_asyncio_enabled_true(self):- log = self.get_log(self.debug_log_spider, args=['-s', 'ASYNCIO_REACTOR=True'])- self.assertIn("DEBUG: Asyncio reactor is installed", log)+ log = self.get_log(self.debug_log_spider, args=[+ '-s', 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor'+ ])+ self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)def test_asyncio_enabled_false(self):- log = self.get_log(self.debug_log_spider, args=['-s', 'ASYNCIO_REACTOR=False'])- self.assertNotIn("DEBUG: Asyncio reactor is installed", log)+ log = self.get_log(self.debug_log_spider, args=[])+ self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)class BenchCommandTest(CommandTest):diff --git a/tests/test_crawler.py b/tests/test_crawler.pyindex 0ce0674de..f8fa26def 100644--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@ -254,30 +254,38 @@ class CrawlerRunnerHasSpider(unittest.TestCase):def test_crawler_runner_asyncio_enabled_true(self):if self.reactor_pytest == 'asyncio':- runner = CrawlerRunner(settings={'ASYNCIO_REACTOR': True})+ runner = CrawlerRunner(settings={+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",+ })else:- msg = "ASYNCIO_REACTOR is on but the Twisted asyncio reactor is not installed"+ msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)"with self.assertRaisesRegex(Exception, msg):- runner = CrawlerRunner(settings={'ASYNCIO_REACTOR': True})+ runner = CrawlerRunner(settings={+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",+ })@defer.inlineCallbacksdef test_crawler_process_asyncio_enabled_true(self):with LogCapture(level=logging.DEBUG) as log:if self.reactor_pytest == 'asyncio':- runner = CrawlerProcess(settings={'ASYNCIO_REACTOR': True})+ runner = CrawlerProcess(settings={+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",+ })yield runner.crawl(NoRequestsSpider)- self.assertIn("Asyncio reactor is installed", str(log))+ self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", str(log))else:- msg = "ASYNCIO_REACTOR is on but the Twisted asyncio reactor is not installed"+ msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)"with self.assertRaisesRegex(Exception, msg):- runner = CrawlerProcess(settings={'ASYNCIO_REACTOR': True})+ runner = CrawlerProcess(settings={+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",+ })@defer.inlineCallbacksdef test_crawler_process_asyncio_enabled_false(self):- runner = CrawlerProcess(settings={'ASYNCIO_REACTOR': False})+ runner = CrawlerProcess(settings={"TWISTED_REACTOR": None})with LogCapture(level=logging.DEBUG) as log:yield runner.crawl(NoRequestsSpider)- self.assertNotIn("Asyncio reactor is installed", str(log))+ self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", str(log))class CrawlerProcessSubprocess(unittest.TestCase):@@ -294,17 +302,17 @@ class CrawlerProcessSubprocess(unittest.TestCase):def test_simple(self):log = self.run_script('simple.py')self.assertIn('Spider closed (finished)', log)- self.assertNotIn("DEBUG: Asyncio reactor is installed", log)+ self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)def test_asyncio_enabled_no_reactor(self):log = self.run_script('asyncio_enabled_no_reactor.py')self.assertIn('Spider closed (finished)', log)- self.assertIn("DEBUG: Asyncio reactor is installed", log)+ self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)def test_asyncio_enabled_reactor(self):log = self.run_script('asyncio_enabled_reactor.py')self.assertIn('Spider closed (finished)', log)- self.assertIn("DEBUG: Asyncio reactor is installed", log)+ self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)def test_ipv6_default_name_resolver(self):log = self.run_script('default_name_resolver.py')@@ -323,3 +331,18 @@ class CrawlerProcessSubprocess(unittest.TestCase):"'downloader/exception_type_count/twisted.internet.error.ConnectionRefusedError': 1," in log,"'downloader/exception_type_count/twisted.internet.error.ConnectError': 1," in log,]))++ def test_reactor_select(self):+ log = self.run_script("twisted_reactor_select.py")+ self.assertIn("Spider closed (finished)", log)+ self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log)++ def test_reactor_poll(self):+ log = self.run_script("twisted_reactor_poll.py")+ self.assertIn("Spider closed (finished)", log)+ self.assertIn("Using reactor: twisted.internet.pollreactor.PollReactor", log)++ def test_reactor_asyncio(self):+ log = self.run_script("twisted_reactor_asyncio.py")+ self.assertIn("Spider closed (finished)", log)+ self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.pyindex 44acc24af..295323e4d 100644--- a/tests/test_utils_asyncio.py+++ b/tests/test_utils_asyncio.py@@ -2,7 +2,7 @@ from unittest import TestCasefrom pytest import mark-from scrapy.utils.asyncio import is_asyncio_reactor_installed, install_asyncio_reactor+from scrapy.utils.reactor import is_asyncio_reactor_installed, install_reactor@mark.usefixtures('reactor_pytest')@@ -14,4 +14,4 @@ class AsyncioTest(TestCase):def test_install_asyncio_reactor(self):# this should do nothing- install_asyncio_reactor()+ install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")