Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Rename ASYNCIO_ENABLED to ASYNCIO_REACTOR, change the logic accordingly.

Andrey Rakhmatullindc1ee09481c7
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 62b2870b2..c02f877fc 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -160,26 +160,22 @@ to any particular component. In that case the module of that component will be
shown, typically an extension, middleware or pipeline. It also means that the
component must be enabled in order for the setting to have any effect.
-.. setting:: ASYNCIO_ENABLED
+.. setting:: ASYNCIO_REACTOR
-ASYNCIO_ENABLED
+ASYNCIO_REACTOR
---------------
Default: ``False``
-Whether to support ``async def`` methods and callbacks which use code that
-requires an asyncio loop.
-
-If an ``async def`` coroutine doesn't require the asyncio loop, it will work
-even if this is set to ``False``. Coroutines that require the asyncio loop may
-silently fail to run or raise errors unless this is set to ``True``.
+Whether to install and require the Twisted reactor that uses the asyncio loop.
When this option is set to ``True``, Scrapy will require
:class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`. It will
install this reactor if no reactor is installed yet, such as when using the
``scrapy`` script or :class:`~scrapy.crawler.CrawlerProcess`. If you are using
:class:`~scrapy.crawler.CrawlerRunner`, you need to install the correct reactor
-manually.
+manually. If a different reactor is installed outside Scrapy, it will raise an
+exception.
The default value for this option is currently ``False`` to maintain backward
compatibility and avoid possible problems caused by using a different Twisted
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index a9443f7ac..f87e67d93 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -137,6 +137,7 @@ class CrawlerRunner(object):
self._crawlers = set()
self._active = set()
self.bootstrap_failed = False
+ self._handle_asyncio_reactor()
@property
def spiders(self):
@@ -230,6 +231,11 @@ class CrawlerRunner(object):
while self._active:
yield defer.DeferredList(self._active)
+ def _handle_asyncio_reactor(self):
+ if self.settings.getbool('ASYNCIO_REACTOR') and not is_asyncio_reactor_installed():
+ raise Exception("ASYNCIO_REACTOR is on but the Twisted asyncio "
+ "reactor is not installed.")
+
class CrawlerProcess(CrawlerRunner):
"""
@@ -257,12 +263,6 @@ class CrawlerProcess(CrawlerRunner):
def __init__(self, settings=None, install_root_handler=True):
super(CrawlerProcess, self).__init__(settings)
- if self.settings.getbool('ASYNCIO_ENABLED'):
- install_asyncio_reactor()
- if not is_asyncio_reactor_installed():
- raise Exception("ASYNCIO_ENABLED is on but the Twisted asyncio "
- "reactor is not installed, this is not supported.")
-
install_shutdown_handlers(self._signal_shutdown)
configure_logging(self.settings, install_root_handler)
log_scrapy_info(self.settings)
@@ -333,6 +333,11 @@ class CrawlerProcess(CrawlerRunner):
except RuntimeError: # raised if already stopped or in shutdown stage
pass
+ def _handle_asyncio_reactor(self):
+ if self.settings.getbool('ASYNCIO_REACTOR'):
+ install_asyncio_reactor()
+ super()._handle_asyncio_reactor()
+
def _get_spider_loader(settings):
""" Get SpiderLoader instance from settings """
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index a7792e248..d03fd37b0 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -19,7 +19,7 @@ from os.path import join, abspath, dirname
AJAXCRAWL_ENABLED = False
-ASYNCIO_ENABLED = False
+ASYNCIO_REACTOR = False
AUTOTHROTTLE_ENABLED = False
AUTOTHROTTLE_DEBUG = False
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 6179e1bd1..e4cf0196b 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -11,6 +11,7 @@ from twisted.python import log as twisted_log
import scrapy
from scrapy.settings import Settings
from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.utils.asyncio import is_asyncio_reactor_installed
from scrapy.utils.versions import scrapy_components_versions
@@ -148,8 +149,8 @@ def log_scrapy_info(settings):
{'versions': ", ".join("%s %s" % (name, version)
for name, version in scrapy_components_versions()
if name != "Scrapy")})
- if settings.getbool('ASYNCIO_ENABLED'):
- logger.debug("Asyncio support enabled")
+ if is_asyncio_reactor_installed():
+ logger.debug("Asyncio reactor is installed")
class StreamLogger(object):
diff --git a/tests/CrawlerProcess/asyncio_enabled_no_reactor.py b/tests/CrawlerProcess/asyncio_enabled_no_reactor.py
index dfe028ef4..db1b75931 100644
--- a/tests/CrawlerProcess/asyncio_enabled_no_reactor.py
+++ b/tests/CrawlerProcess/asyncio_enabled_no_reactor.py
@@ -10,7 +10,7 @@ class NoRequestsSpider(scrapy.Spider):
process = CrawlerProcess(settings={
- 'ASYNCIO_ENABLED': True,
+ 'ASYNCIO_REACTOR': True,
})
process.crawl(NoRequestsSpider)
diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.py
index 7a172ea28..cec3c9c25 100644
--- a/tests/CrawlerProcess/asyncio_enabled_reactor.py
+++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py
@@ -15,7 +15,7 @@ class NoRequestsSpider(scrapy.Spider):
process = CrawlerProcess(settings={
- 'ASYNCIO_ENABLED': True,
+ 'ASYNCIO_REACTOR': True,
})
process.crawl(NoRequestsSpider)
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 197d80217..6024af71c 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -296,12 +296,12 @@ class BadSpider(scrapy.Spider):
self.assertIn("badspider.py", log)
def test_asyncio_enabled_true(self):
- log = self.get_log(self.debug_log_spider, args=['-s', 'ASYNCIO_ENABLED=True'])
- self.assertIn("DEBUG: Asyncio support enabled", log)
+ log = self.get_log(self.debug_log_spider, args=['-s', 'ASYNCIO_REACTOR=True'])
+ self.assertIn("DEBUG: Asyncio reactor is installed", log)
def test_asyncio_enabled_false(self):
- log = self.get_log(self.debug_log_spider, args=['-s', 'ASYNCIO_ENABLED=False'])
- self.assertNotIn("DEBUG: Asyncio support enabled", log)
+ log = self.get_log(self.debug_log_spider, args=['-s', 'ASYNCIO_REACTOR=False'])
+ self.assertNotIn("DEBUG: Asyncio reactor is installed", log)
class BenchCommandTest(CommandTest):
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index a2865fcd1..fce60ca37 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -13,7 +13,6 @@ import scrapy
from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess
from scrapy.settings import Settings, default_settings
from scrapy.spiderloader import SpiderLoader
-from scrapy.utils.asyncio import is_asyncio_reactor_installed
from scrapy.utils.log import configure_logging, get_scrapy_root_handler
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.misc import load_object
@@ -209,14 +208,6 @@ class NoRequestsSpider(scrapy.Spider):
return []
-class AsyncioSpider(scrapy.Spider):
- name = 'asyncio'
-
- def start_requests(self):
- self.logger.info('Asyncio support: %s', is_asyncio_reactor_installed())
- return []
-
-
@mark.usefixtures('reactor_pytest')
class CrawlerRunnerHasSpider(unittest.TestCase):
@@ -261,31 +252,32 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
self.assertEqual(runner.bootstrap_failed, True)
+ def test_crawler_runner_asyncio_enabled_true(self):
+ if self.reactor_pytest == 'asyncio':
+ runner = CrawlerRunner(settings={'ASYNCIO_REACTOR': True})
+ else:
+ msg = "ASYNCIO_REACTOR is on but the Twisted asyncio reactor is not installed"
+ with self.assertRaisesRegex(Exception, msg):
+ runner = CrawlerRunner(settings={'ASYNCIO_REACTOR': True})
+
@defer.inlineCallbacks
def test_crawler_process_asyncio_enabled_true(self):
with LogCapture(level=logging.DEBUG) as log:
if self.reactor_pytest == 'asyncio':
- runner = CrawlerProcess(settings={'ASYNCIO_ENABLED': True})
+ runner = CrawlerProcess(settings={'ASYNCIO_REACTOR': True})
yield runner.crawl(NoRequestsSpider)
- self.assertIn("Asyncio support enabled", str(log))
+ self.assertIn("Asyncio reactor is installed", str(log))
else:
- msg = "ASYNCIO_ENABLED is on but the Twisted asyncio reactor is not installed"
+ msg = "ASYNCIO_REACTOR is on but the Twisted asyncio reactor is not installed"
with self.assertRaisesRegex(Exception, msg):
- runner = CrawlerProcess(settings={'ASYNCIO_ENABLED': True})
+ runner = CrawlerProcess(settings={'ASYNCIO_REACTOR': True})
@defer.inlineCallbacks
def test_crawler_process_asyncio_enabled_false(self):
- runner = CrawlerProcess(settings={'ASYNCIO_ENABLED': False})
+ runner = CrawlerProcess(settings={'ASYNCIO_REACTOR': False})
with LogCapture(level=logging.DEBUG) as log:
yield runner.crawl(NoRequestsSpider)
- self.assertNotIn("Asyncio support enabled", str(log))
-
- @defer.inlineCallbacks
- def test_crawler_runner_asyncio_supported(self):
- runner = CrawlerRunner()
- with LogCapture() as log:
- yield runner.crawl(AsyncioSpider)
- log.check_present(('asyncio', 'INFO', 'Asyncio support: %s' % (self.reactor_pytest == 'asyncio')))
+ self.assertNotIn("Asyncio reactor is installed", str(log))
class CrawlerProcessSubprocess(unittest.TestCase):
@@ -302,14 +294,14 @@ class CrawlerProcessSubprocess(unittest.TestCase):
def test_simple(self):
log = self.run_script('simple.py')
self.assertIn('Spider closed (finished)', log)
- self.assertNotIn("DEBUG: Asyncio support enabled", log)
+ self.assertNotIn("DEBUG: Asyncio reactor is installed", log)
def test_asyncio_enabled_no_reactor(self):
log = self.run_script('asyncio_enabled_no_reactor.py')
self.assertIn('Spider closed (finished)', log)
- self.assertIn("DEBUG: Asyncio support enabled", log)
+ self.assertIn("DEBUG: Asyncio reactor is installed", log)
def test_asyncio_enabled_reactor(self):
log = self.run_script('asyncio_enabled_reactor.py')
self.assertIn('Spider closed (finished)', log)
- self.assertIn("DEBUG: Asyncio support enabled", log)
+ self.assertIn("DEBUG: Asyncio reactor is installed", log)