Scrapy, a fast high-level web crawling & scraping framework for Python.
Install the asyncio reactor only in scrapy.cmdline.
Andrey Rakhmatullinb04b541372b2
diff --git a/scrapy/__init__.py b/scrapy/__init__.pyindex 41eaee959..230e5cee3 100644--- a/scrapy/__init__.py+++ b/scrapy/__init__.py@@ -23,28 +23,6 @@ import warningswarnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted')del warnings-# Install twisted asyncio loop-def _install_asyncio_reactor():- global asyncio_supported- try:- import asyncio- from twisted.internet import asyncioreactor- except ImportError:- pass- else:- from twisted.internet.error import ReactorAlreadyInstalledError- try:- asyncioreactor.install(asyncio.get_event_loop())- asyncio_supported = True- except ReactorAlreadyInstalledError:- import twisted.internet.reactor- if isinstance(twisted.internet.reactor,- asyncioreactor.AsyncioSelectorReactor):- asyncio_supported = True-asyncio_supported = False-_install_asyncio_reactor()-del _install_asyncio_reactor-# Apply monkey patches to fix issues in external librariesfrom . import _monkeypatchesdel _monkeypatchesdiff --git a/scrapy/cmdline.py b/scrapy/cmdline.pyindex 418dc1ac9..d66f0cc2d 100644--- a/scrapy/cmdline.py+++ b/scrapy/cmdline.py@@ -7,9 +7,9 @@ import inspectimport pkg_resourcesimport scrapy-from scrapy.crawler import CrawlerProcessfrom scrapy.commands import ScrapyCommandfrom scrapy.exceptions import UsageError+from scrapy.utils.asyncio import install_asyncio_reactor, is_asyncio_supportedfrom scrapy.utils.misc import walk_modulesfrom scrapy.utils.project import inside_project, get_project_settingsfrom scrapy.utils.python import garbage_collect@@ -121,6 +121,10 @@ def execute(argv=None, settings=None):settings['EDITOR'] = editorcheck_deprecated_settings(settings)+ # needs to be before _get_commands_dict() as that imports the command modules+ # which may import twisted.internet.reactor+ install_asyncio_reactor()+inproject = inside_project()cmds = _get_commands_dict(settings, inproject)cmdname = _pop_command_name(argv)@@ -142,6 +146,8 @@ def execute(argv=None, settings=None):opts, args = parser.parse_args(args=argv[1:])_run_print_help(parser, cmd.process_options, args, opts)+ # needs to be after install_asyncio_reactor() as it imports twisted.internet.reactor+ from scrapy.crawler import CrawlerProcesscmd.crawler_process = CrawlerProcess(settings)_run_print_help(parser, _run_command, cmd, args, opts)sys.exit(cmd.exitcode)diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.pynew file mode 100644index 000000000..e9e3bdd88--- /dev/null+++ b/scrapy/utils/asyncio.py@@ -0,0 +1,26 @@+#coding: utf-8+++def install_asyncio_reactor():+ """ Tries to install AsyncioSelectorReactor+ """+ try:+ import asyncio+ from twisted.internet import asyncioreactor+ except ImportError:+ pass+ else:+ from twisted.internet.error import ReactorAlreadyInstalledError+ try:+ asyncioreactor.install(asyncio.get_event_loop())+ except ReactorAlreadyInstalledError:+ pass+++def is_asyncio_supported():+ try:+ import twisted.internet.reactor+ from twisted.internet import asyncioreactor+ return isinstance(twisted.internet.reactor, asyncioreactor.AsyncioSelectorReactor)+ except ImportError:+ return Falsediff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.pyindex 1f6a2584c..955fc820a 100644--- a/scrapy/utils/defer.py+++ b/scrapy/utils/defer.py@@ -8,8 +8,9 @@ import inspectfrom twisted.internet import defer, reactor, taskfrom twisted.python import failure-from scrapy import asyncio_supportedfrom scrapy.exceptions import IgnoreRequest+from scrapy.utils.asyncio import is_asyncio_supported+def defer_fail(_failure):@@ -131,7 +132,7 @@ def deferred_from_coro(o):if isinstance(o, defer.Deferred):return oif asyncio.iscoroutine(o) or isfuture(o) or inspect.isawaitable(o):- if not asyncio_supported:+ if not is_asyncio_supported():raise TypeError('Using coroutines requires installing AsyncioSelectorReactor')return defer.Deferred.fromFuture(asyncio.ensure_future(o))return odiff --git a/tests/mockserver.py b/tests/mockserver.pyindex b6aee009a..d09fbc171 100644--- a/tests/mockserver.py+++ b/tests/mockserver.py@@ -7,8 +7,6 @@ from subprocess import Popen, PIPEfrom OpenSSL import SSLfrom six.moves.urllib.parse import urlencode-import scrapy # needed before importing twisted.internet.reactor-from twisted.web.server import Site, NOT_DONE_YETfrom twisted.web.resource import Resourcefrom twisted.web.static import File