Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Move a bunch of "from twisted.internet import reactor" inside functions.

Andrey Rakhmatullin69cd2e247efe
diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index 3c2efe58f..69e917004 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -6,6 +6,7 @@ import inspect
import pkg_resources
import scrapy
+from scrapy.crawler import CrawlerProcess
from scrapy.commands import ScrapyCommand
from scrapy.exceptions import UsageError
from scrapy.utils.misc import walk_modules
@@ -140,9 +141,6 @@ def execute(argv=None, settings=None):
opts, args = parser.parse_args(args=argv[1:])
_run_print_help(parser, cmd.process_options, args, opts)
- # needs to be after cmd.process_options() as it imports twisted.internet.reactor
- # while commands may want to install the asyncio reactor
- from scrapy.crawler import CrawlerProcess
cmd.crawler_process = CrawlerProcess(settings)
_run_print_help(parser, _run_command, cmd, args, opts)
sys.exit(cmd.exitcode)
diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py
index 7516e2aba..d44a32d5f 100644
--- a/scrapy/commands/shell.py
+++ b/scrapy/commands/shell.py
@@ -7,6 +7,7 @@ from threading import Thread
from scrapy.commands import ScrapyCommand
from scrapy.http import Request
+from scrapy.shell import Shell
from scrapy.utils.spider import spidercls_for_request, DefaultSpider
from scrapy.utils.url import guess_scheme
@@ -69,8 +70,6 @@ class Command(ScrapyCommand):
self._start_crawler_thread()
- # moved from the top-level because it imports twisted.internet.reactor
- from scrapy.shell import Shell
shell = Shell(crawler, update_vars=self.update_vars, code=opts.code)
shell.start(url=url, redirect=not opts.no_redirect)
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 6c7eb737b..450260004 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -3,7 +3,7 @@ import pprint
import signal
import warnings
-from twisted.internet import reactor, defer
+from twisted.internet import defer
from zope.interface.verify import verifyClass, DoesNotImplement
from scrapy import Spider
@@ -261,6 +261,7 @@ class CrawlerProcess(CrawlerRunner):
log_scrapy_info(self.settings)
def _signal_shutdown(self, signum, _):
+ from twisted.internet import reactor
install_shutdown_handlers(self._signal_kill)
signame = signal_names[signum]
logger.info("Received %(signame)s, shutting down gracefully. Send again to force ",
@@ -268,6 +269,7 @@ class CrawlerProcess(CrawlerRunner):
reactor.callFromThread(self._graceful_stop_reactor)
def _signal_kill(self, signum, _):
+ from twisted.internet import reactor
install_shutdown_handlers(signal.SIG_IGN)
signame = signal_names[signum]
logger.info('Received %(signame)s twice, forcing unclean shutdown',
@@ -286,6 +288,7 @@ class CrawlerProcess(CrawlerRunner):
:param boolean stop_after_crawl: stop or not the reactor when all
crawlers have finished
"""
+ from twisted.internet import reactor
if stop_after_crawl:
d = self.join()
# Don't start the reactor if the deferreds are already fired
@@ -300,6 +303,7 @@ class CrawlerProcess(CrawlerRunner):
reactor.run(installSignalHandlers=False) # blocking call
def _get_dns_resolver(self):
+ from twisted.internet import reactor
if self.settings.getbool('DNSCACHE_ENABLED'):
cache_size = self.settings.getint('DNSCACHE_SIZE')
else:
@@ -316,6 +320,7 @@ class CrawlerProcess(CrawlerRunner):
return d
def _stop_reactor(self, _=None):
+ from twisted.internet import reactor
try:
reactor.stop()
except RuntimeError: # raised if already stopped or in shutdown stage
diff --git a/scrapy/shell.py b/scrapy/shell.py
index a649d555f..a23b04df9 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -7,7 +7,7 @@ import os
import signal
import warnings
-from twisted.internet import reactor, threads, defer
+from twisted.internet import threads, defer
from twisted.python import threadable
from w3lib.url import any_to_uri
@@ -98,6 +98,7 @@ class Shell(object):
return spider
def fetch(self, request_or_url, spider=None, redirect=True, **kwargs):
+ from twisted.internet import reactor
if isinstance(request_or_url, Request):
request = request_or_url
else:
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 3b7ef75ab..6a91776c7 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -5,7 +5,7 @@ import asyncio
import asyncio.futures
import inspect
-from twisted.internet import defer, reactor, task
+from twisted.internet import defer, task
from twisted.python import failure
from scrapy.exceptions import IgnoreRequest
@@ -19,6 +19,7 @@ def defer_fail(_failure):
It delays by 100ms so reactor has a chance to go through readers and writers
before attending pending delayed calls, so do not set delay to zero.
"""
+ from twisted.internet import reactor
d = defer.Deferred()
reactor.callLater(0.1, d.errback, _failure)
return d
@@ -31,6 +32,7 @@ def defer_succeed(result):
It delays by 100ms so reactor has a chance to go trough readers and writers
before attending pending delayed calls, so do not set delay to zero.
"""
+ from twisted.internet import reactor
d = defer.Deferred()
reactor.callLater(0.1, d.callback, result)
return d
diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.py
index 7a7aec9be..45c9cef0c 100644
--- a/scrapy/utils/ossignal.py
+++ b/scrapy/utils/ossignal.py
@@ -1,7 +1,5 @@
import signal
-from twisted.internet import reactor
-
signal_names = {}
for signame in dir(signal):
@@ -17,6 +15,7 @@ def install_shutdown_handlers(function, override_sigint=True):
SIGINT handler won't be install if there is already a handler in place
(e.g. Pdb)
"""
+ from twisted.internet import reactor
reactor._handleSignals()
signal.signal(signal.SIGTERM, function)
if signal.getsignal(signal.SIGINT) == signal.default_int_handler or \
diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
index 493d26d4c..b98fff6ec 100644
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -1,8 +1,9 @@
-from twisted.internet import reactor, error
+from twisted.internet import error
def listen_tcp(portrange, host, factory):
"""Like reactor.listenTCP but tries different ports in a range."""
+ from twisted.internet import reactor
assert len(portrange) <= 2, "invalid portrange: %s" % portrange
if not portrange:
return reactor.listenTCP(0, factory, interface=host)
@@ -30,6 +31,7 @@ class CallLaterOnce(object):
self._call = None
def schedule(self, delay=0):
+ from twisted.internet import reactor
if self._call is None:
self._call = reactor.callLater(delay, self)