Scrapy, a fast high-level web crawling & scraping framework for Python.
Move a bunch of "from twisted.internet import reactor" inside functions.
Andrey Rakhmatullin69cd2e247efe
diff --git a/scrapy/cmdline.py b/scrapy/cmdline.pyindex 3c2efe58f..69e917004 100644--- a/scrapy/cmdline.py+++ b/scrapy/cmdline.py@@ -6,6 +6,7 @@ import inspectimport pkg_resourcesimport scrapy+from scrapy.crawler import CrawlerProcessfrom scrapy.commands import ScrapyCommandfrom scrapy.exceptions import UsageErrorfrom scrapy.utils.misc import walk_modules@@ -140,9 +141,6 @@ def execute(argv=None, settings=None):opts, args = parser.parse_args(args=argv[1:])_run_print_help(parser, cmd.process_options, args, opts)- # needs to be after cmd.process_options() as it imports twisted.internet.reactor- # while commands may want to install the asyncio reactor- from scrapy.crawler import CrawlerProcesscmd.crawler_process = CrawlerProcess(settings)_run_print_help(parser, _run_command, cmd, args, opts)sys.exit(cmd.exitcode)diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.pyindex 7516e2aba..d44a32d5f 100644--- a/scrapy/commands/shell.py+++ b/scrapy/commands/shell.py@@ -7,6 +7,7 @@ from threading import Threadfrom scrapy.commands import ScrapyCommandfrom scrapy.http import Request+from scrapy.shell import Shellfrom scrapy.utils.spider import spidercls_for_request, DefaultSpiderfrom scrapy.utils.url import guess_scheme@@ -69,8 +70,6 @@ class Command(ScrapyCommand):self._start_crawler_thread()- # moved from the top-level because it imports twisted.internet.reactor- from scrapy.shell import Shellshell = Shell(crawler, update_vars=self.update_vars, code=opts.code)shell.start(url=url, redirect=not opts.no_redirect)diff --git a/scrapy/crawler.py b/scrapy/crawler.pyindex 6c7eb737b..450260004 100644--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@ -3,7 +3,7 @@ import pprintimport signalimport warnings-from twisted.internet import reactor, defer+from twisted.internet import deferfrom zope.interface.verify import verifyClass, DoesNotImplementfrom scrapy import Spider@@ -261,6 +261,7 @@ class CrawlerProcess(CrawlerRunner):log_scrapy_info(self.settings)def _signal_shutdown(self, signum, _):+ from twisted.internet import reactorinstall_shutdown_handlers(self._signal_kill)signame = signal_names[signum]logger.info("Received %(signame)s, shutting down gracefully. Send again to force ",@@ -268,6 +269,7 @@ class CrawlerProcess(CrawlerRunner):reactor.callFromThread(self._graceful_stop_reactor)def _signal_kill(self, signum, _):+ from twisted.internet import reactorinstall_shutdown_handlers(signal.SIG_IGN)signame = signal_names[signum]logger.info('Received %(signame)s twice, forcing unclean shutdown',@@ -286,6 +288,7 @@ class CrawlerProcess(CrawlerRunner)::param boolean stop_after_crawl: stop or not the reactor when allcrawlers have finished"""+ from twisted.internet import reactorif stop_after_crawl:d = self.join()# Don't start the reactor if the deferreds are already fired@@ -300,6 +303,7 @@ class CrawlerProcess(CrawlerRunner):reactor.run(installSignalHandlers=False) # blocking calldef _get_dns_resolver(self):+ from twisted.internet import reactorif self.settings.getbool('DNSCACHE_ENABLED'):cache_size = self.settings.getint('DNSCACHE_SIZE')else:@@ -316,6 +320,7 @@ class CrawlerProcess(CrawlerRunner):return ddef _stop_reactor(self, _=None):+ from twisted.internet import reactortry:reactor.stop()except RuntimeError: # raised if already stopped or in shutdown stagediff --git a/scrapy/shell.py b/scrapy/shell.pyindex a649d555f..a23b04df9 100644--- a/scrapy/shell.py+++ b/scrapy/shell.py@@ -7,7 +7,7 @@ import osimport signalimport warnings-from twisted.internet import reactor, threads, defer+from twisted.internet import threads, deferfrom twisted.python import threadablefrom w3lib.url import any_to_uri@@ -98,6 +98,7 @@ class Shell(object):return spiderdef fetch(self, request_or_url, spider=None, redirect=True, **kwargs):+ from twisted.internet import reactorif isinstance(request_or_url, Request):request = request_or_urlelse:diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.pyindex 3b7ef75ab..6a91776c7 100644--- a/scrapy/utils/defer.py+++ b/scrapy/utils/defer.py@@ -5,7 +5,7 @@ import asyncioimport asyncio.futuresimport inspect-from twisted.internet import defer, reactor, task+from twisted.internet import defer, taskfrom twisted.python import failurefrom scrapy.exceptions import IgnoreRequest@@ -19,6 +19,7 @@ def defer_fail(_failure):It delays by 100ms so reactor has a chance to go through readers and writersbefore attending pending delayed calls, so do not set delay to zero."""+ from twisted.internet import reactord = defer.Deferred()reactor.callLater(0.1, d.errback, _failure)return d@@ -31,6 +32,7 @@ def defer_succeed(result):It delays by 100ms so reactor has a chance to go trough readers and writersbefore attending pending delayed calls, so do not set delay to zero."""+ from twisted.internet import reactord = defer.Deferred()reactor.callLater(0.1, d.callback, result)return ddiff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.pyindex 7a7aec9be..45c9cef0c 100644--- a/scrapy/utils/ossignal.py+++ b/scrapy/utils/ossignal.py@@ -1,7 +1,5 @@import signal-from twisted.internet import reactor-signal_names = {}for signame in dir(signal):@@ -17,6 +15,7 @@ def install_shutdown_handlers(function, override_sigint=True):SIGINT handler won't be install if there is already a handler in place(e.g. Pdb)"""+ from twisted.internet import reactorreactor._handleSignals()signal.signal(signal.SIGTERM, function)if signal.getsignal(signal.SIGINT) == signal.default_int_handler or \diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.pyindex 493d26d4c..b98fff6ec 100644--- a/scrapy/utils/reactor.py+++ b/scrapy/utils/reactor.py@@ -1,8 +1,9 @@-from twisted.internet import reactor, error+from twisted.internet import errordef listen_tcp(portrange, host, factory):"""Like reactor.listenTCP but tries different ports in a range."""+ from twisted.internet import reactorassert len(portrange) <= 2, "invalid portrange: %s" % portrangeif not portrange:return reactor.listenTCP(0, factory, interface=host)@@ -30,6 +31,7 @@ class CallLaterOnce(object):self._call = Nonedef schedule(self, delay=0):+ from twisted.internet import reactorif self._call is None:self._call = reactor.callLater(delay, self)