Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Install the asyncio reactor only in scrapy.cmdline.

Andrey Rakhmatullinb04b541372b2
diff --git a/scrapy/__init__.py b/scrapy/__init__.py
index 41eaee959..230e5cee3 100644
--- a/scrapy/__init__.py
+++ b/scrapy/__init__.py
@@ -23,28 +23,6 @@ import warnings
warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted')
del warnings
-# Install twisted asyncio loop
-def _install_asyncio_reactor():
- global asyncio_supported
- try:
- import asyncio
- from twisted.internet import asyncioreactor
- except ImportError:
- pass
- else:
- from twisted.internet.error import ReactorAlreadyInstalledError
- try:
- asyncioreactor.install(asyncio.get_event_loop())
- asyncio_supported = True
- except ReactorAlreadyInstalledError:
- import twisted.internet.reactor
- if isinstance(twisted.internet.reactor,
- asyncioreactor.AsyncioSelectorReactor):
- asyncio_supported = True
-asyncio_supported = False
-_install_asyncio_reactor()
-del _install_asyncio_reactor
-
# Apply monkey patches to fix issues in external libraries
from . import _monkeypatches
del _monkeypatches
diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index 418dc1ac9..d66f0cc2d 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -7,9 +7,9 @@ import inspect
import pkg_resources
import scrapy
-from scrapy.crawler import CrawlerProcess
from scrapy.commands import ScrapyCommand
from scrapy.exceptions import UsageError
+from scrapy.utils.asyncio import install_asyncio_reactor, is_asyncio_supported
from scrapy.utils.misc import walk_modules
from scrapy.utils.project import inside_project, get_project_settings
from scrapy.utils.python import garbage_collect
@@ -121,6 +121,10 @@ def execute(argv=None, settings=None):
settings['EDITOR'] = editor
check_deprecated_settings(settings)
+ # needs to be before _get_commands_dict() as that imports the command modules
+ # which may import twisted.internet.reactor
+ install_asyncio_reactor()
+
inproject = inside_project()
cmds = _get_commands_dict(settings, inproject)
cmdname = _pop_command_name(argv)
@@ -142,6 +146,8 @@ def execute(argv=None, settings=None):
opts, args = parser.parse_args(args=argv[1:])
_run_print_help(parser, cmd.process_options, args, opts)
+ # needs to be after install_asyncio_reactor() as it imports twisted.internet.reactor
+ from scrapy.crawler import CrawlerProcess
cmd.crawler_process = CrawlerProcess(settings)
_run_print_help(parser, _run_command, cmd, args, opts)
sys.exit(cmd.exitcode)
diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.py
new file mode 100644
index 000000000..e9e3bdd88
--- /dev/null
+++ b/scrapy/utils/asyncio.py
@@ -0,0 +1,26 @@
+#coding: utf-8
+
+
+def install_asyncio_reactor():
+ """ Tries to install AsyncioSelectorReactor
+ """
+ try:
+ import asyncio
+ from twisted.internet import asyncioreactor
+ except ImportError:
+ pass
+ else:
+ from twisted.internet.error import ReactorAlreadyInstalledError
+ try:
+ asyncioreactor.install(asyncio.get_event_loop())
+ except ReactorAlreadyInstalledError:
+ pass
+
+
+def is_asyncio_supported():
+ try:
+ import twisted.internet.reactor
+ from twisted.internet import asyncioreactor
+ return isinstance(twisted.internet.reactor, asyncioreactor.AsyncioSelectorReactor)
+ except ImportError:
+ return False
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 1f6a2584c..955fc820a 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -8,8 +8,9 @@ import inspect
from twisted.internet import defer, reactor, task
from twisted.python import failure
-from scrapy import asyncio_supported
from scrapy.exceptions import IgnoreRequest
+from scrapy.utils.asyncio import is_asyncio_supported
+
def defer_fail(_failure):
@@ -131,7 +132,7 @@ def deferred_from_coro(o):
if isinstance(o, defer.Deferred):
return o
if asyncio.iscoroutine(o) or isfuture(o) or inspect.isawaitable(o):
- if not asyncio_supported:
+ if not is_asyncio_supported():
raise TypeError('Using coroutines requires installing AsyncioSelectorReactor')
return defer.Deferred.fromFuture(asyncio.ensure_future(o))
return o
diff --git a/tests/mockserver.py b/tests/mockserver.py
index b6aee009a..d09fbc171 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -7,8 +7,6 @@ from subprocess import Popen, PIPE
from OpenSSL import SSL
from six.moves.urllib.parse import urlencode
-import scrapy # needed before importing twisted.internet.reactor
-
from twisted.web.server import Site, NOT_DONE_YET
from twisted.web.resource import Resource
from twisted.web.static import File